Share the readers' place, table-lookup and country helpers, unify the employment vocabulary, and list the enrich switch with the stages
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ca80219bbb
commit
903b69b6d3
34 files changed
+290
-207
No files matched your search
+2
-2
@@ -152,8 +152,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
prompt.add_argument("--remove", action="store_true", help="remove the prompt instead")
|
||||
|
||||
stages = sub.add_parser(
|
||||
"stages", help="show whether extract and summarise run, or switch one")
|
||||
stages.add_argument("name", nargs="?", choices=["extract", "summarise"])
|
||||
"stages", help="show whether extract, summarise and enrich run, or switch one")
|
||||
stages.add_argument("name", nargs="?", choices=["extract", "summarise", "enrich"])
|
||||
stages.add_argument("state", nargs="?", choices=["on", "off"])
|
||||
|
||||
sub.add_parser("seniority", help="per host, how often the stated level equals the model's")
|
||||
|
||||
+10
-4
@@ -13,8 +13,8 @@ finders ask it (`extract.extract_task`, `summarise.summarise_task`).
|
||||
asks no model and is on by default (`enrich.enabled`); its switch is
|
||||
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
|
||||
|
||||
`describe` is what `GET /api/stages` and `app stages` show: for each model
|
||||
stage whether it is on, where that comes from, the model it uses, and who
|
||||
`describe` is what `GET /api/stages` and `app stages` show: for each stage
|
||||
with a switch, the model stages and `enrich`, whether it is on, where that comes from, the model it uses, and who
|
||||
would answer.
|
||||
"""
|
||||
|
||||
@@ -33,6 +33,8 @@ NAMES = ("extract", "summarise")
|
||||
#: Every stage with a switch: the model stages and `enrich`.
|
||||
SWITCHES = (*NAMES, "enrich")
|
||||
|
||||
#: What `describe_one` says of a stage that asks no model.
|
||||
NO_ASK = "reads role pages, asks no model"
|
||||
NO_MODEL = ("no model is reachable: set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN, "
|
||||
"see docs/llm-backends.md")
|
||||
ANSWERS = {
|
||||
@@ -42,7 +44,7 @@ ANSWERS = {
|
||||
|
||||
|
||||
class Stage(BaseModel):
|
||||
"""One model stage as the page shows it. `source` is "stored" when the
|
||||
"""One stage as the page shows it. `source` is "stored" when the
|
||||
switch set it and "config" when it is still config.yaml's default.
|
||||
`backend` is `agent.backend`'s answer, "" for nobody, and `answer` says
|
||||
it in words."""
|
||||
@@ -89,6 +91,10 @@ def answer(config: Config) -> tuple[str, str]:
|
||||
|
||||
|
||||
def describe_one(config: Config, store: Store, name: str) -> Stage:
|
||||
if name not in NAMES:
|
||||
return Stage(name=name, on=stage_on(config, store, name),
|
||||
source="config" if stored(store, name) is None else "stored",
|
||||
model="", backend="", answer=NO_ASK)
|
||||
backend, said = answer(config)
|
||||
return Stage(name=name, on=stage_on(config, store, name),
|
||||
source="config" if stored(store, name) is None else "stored",
|
||||
@@ -97,4 +103,4 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
|
||||
|
||||
|
||||
def describe(config: Config, store: Store) -> list[Stage]:
|
||||
return [describe_one(config, store, name) for name in NAMES]
|
||||
return [describe_one(config, store, name) for name in SWITCHES]
|
||||
+3
-2
@@ -1,5 +1,5 @@
|
||||
/* The Runs section of Sources: the "Run the pull now" button, and the
|
||||
* switches for the two stages that ask a model.
|
||||
* switches for the stages of a pull: the two that ask a model and enrich.
|
||||
*
|
||||
* The switches are `/api/stages`: one row per stage, drawn as the search
|
||||
* rows above are (`.srow`, `.switch`), and a switch is a PUT that stores the
|
||||
@@ -62,6 +62,7 @@
|
||||
const DOES = {
|
||||
extract: "fills in country, seniority, salary and the like from what the posting says",
|
||||
summarise: "writes a short summary and tags for the best roles",
|
||||
enrich: "reads the own page of the best roles whose list gave no text",
|
||||
};
|
||||
|
||||
const el = (tag, className, text) => {
|
||||
@@ -105,7 +106,7 @@
|
||||
if (!stage.on) row.dataset.muted = "true";
|
||||
const name = el("span", "name");
|
||||
name.append(el("span", "", stage.name[0].toUpperCase() + stage.name.slice(1)),
|
||||
el("span", "badge", stage.model));
|
||||
...(stage.model ? [el("span", "badge", stage.model)] : []));
|
||||
row.append(name, stageSwitch(stage, () => renderStages(stages)));
|
||||
const sub = el("span", "sub");
|
||||
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
|
||||
|
||||
+1
-1
@@ -414,7 +414,7 @@ class StageBody(BaseModel):
|
||||
|
||||
@router.get("/stages")
|
||||
def list_stages(store: StoreDep, config: ConfigDep) -> list[jobs.stages.Stage]:
|
||||
"""The two model stages of a pull: whether each is on, whether the
|
||||
"""The stages of a pull with a switch, extract, summarise and enrich: whether each is on, whether the
|
||||
stored switch or config.yaml says so, its model, and who would answer."""
|
||||
return jobs.stages.describe(config, store)
|
||||
|
||||
|
||||
@@ -178,6 +178,29 @@ posting per place, each with its own id, declares `places="copies"`: each
|
||||
copy is a row with its one place, and `dupes.py` folds them into one role
|
||||
that keeps each copy's link.
|
||||
|
||||
**Helpers.** `._shared` holds what more than one host needs; a module keeps
|
||||
only its own word tables and uses these:
|
||||
|
||||
- `place(text, remote=None)` makes a `Place`: remote when the text has the
|
||||
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
|
||||
appends a place unless its text is empty or already in `found`, in the
|
||||
host's order. A host that states remote itself passes it to `place`, or
|
||||
builds the dict and passes that to `add_place`.
|
||||
- `table_word(table, value, log, what)` maps a host's word (stated seniority
|
||||
or employment) through the module's table, without case. The table's keys
|
||||
are lowercase. Empty gives None; a known word mapped to `""` (`experienced`,
|
||||
`permanent`) names no level and gives None; an unknown word gives None and
|
||||
a debug line naming the host, so the table can grow.
|
||||
- `EMPLOYMENTS`, the one employment vocabulary: `full_time`, `part_time`,
|
||||
`contract`, `temporary`, `internship`, `freelance`. Every value of a
|
||||
module's `_EMPLOYMENT` table is one of these or `""`; the test in
|
||||
`tests/test_readers_shared.py` holds every module to it. Seniority values
|
||||
are `app.models.SENIORITIES`.
|
||||
- `country_if_single(codes)` is the code when the role's places share exactly
|
||||
one, else None, for a host that states a country per place.
|
||||
- `remote_word(text)` reads `remote` or `hybrid` from a search provider's
|
||||
headline or town line (Danish, Swedish, English).
|
||||
|
||||
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
|
||||
over as `teaser`, plain text, whether or not the full text comes later. It
|
||||
is shown and embedded until `description` arrives. A reader that sees one
|
||||
|
||||
@@ -8,6 +8,7 @@ import it without either importing the other. The leading underscore keeps
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
@@ -16,6 +17,8 @@ from typing import Any, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
from ._contract import Place
|
||||
|
||||
#: What an undeclared module's parser turns a feed into; a module that
|
||||
#: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`.
|
||||
#: One dict per row, with the keys
|
||||
@@ -192,3 +195,59 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) ->
|
||||
tail = f" per {period}" if period else ""
|
||||
code = str(currency or "").strip().upper()
|
||||
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
||||
|
||||
|
||||
#: The employment words a row's `employment` may hold, for every host. A
|
||||
#: host's table maps its own words onto these; a word that names none of them
|
||||
#: (a permanent contract says nothing about hours) maps to "".
|
||||
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
|
||||
|
||||
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def place(text: str, remote: bool | None = None) -> Place:
|
||||
"""A `Place` for `text`: remote when the text says the word `remote`,
|
||||
otherwise as `remote` says (None when the host says nothing)."""
|
||||
return {"text": text, "remote": True if _REMOTE_WORD.search(text) else remote}
|
||||
|
||||
|
||||
def add_place(found: list[Place], entry: Place) -> None:
|
||||
"""Append `entry` to `found` unless its text is empty or already there;
|
||||
the host's order is kept."""
|
||||
if entry["text"] and entry["text"] not in (p["text"] for p in found):
|
||||
found.append(entry)
|
||||
|
||||
|
||||
def table_word(table: dict[str, str], value: Any, log: logging.Logger, what: str) -> str | None:
|
||||
"""`table`'s word for a host's `value`, looked up without case. None when
|
||||
the host states none, when the table maps the word to "" (a known word
|
||||
that names no level, such as `experienced`), or when the table does not
|
||||
know it; that last case is logged at debug level so the table can grow."""
|
||||
word = str(value or "").strip()
|
||||
if not word:
|
||||
return None
|
||||
found = table.get(word.lower())
|
||||
if found is None:
|
||||
log.debug("%s: unknown %s %r", log.name.rsplit(".", 1)[-1], what, word)
|
||||
return found or None
|
||||
|
||||
|
||||
def country_if_single(codes: Any) -> str | None:
|
||||
"""The one code among `codes` when there is exactly one, else None: a
|
||||
country is stated for a role only when all its places share it."""
|
||||
return next(iter(codes)) if len(codes) == 1 else None
|
||||
|
||||
|
||||
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
||||
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(
|
||||
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
|
||||
|
||||
|
||||
def remote_word(text: str) -> str | None:
|
||||
"""`remote`, `hybrid` or None: the word a search provider's headline or
|
||||
town line says about where the work is done, in Danish, Swedish and
|
||||
English. A board states its own remote flag and needs none of this."""
|
||||
if _REMOTE.search(text):
|
||||
return "remote"
|
||||
return "hybrid" if _HYBRID.search(text) else None
|
||||
@@ -1,19 +0,0 @@
|
||||
"""What the search providers' headlines and town lines say about where the
|
||||
work is done, in Danish, Swedish and English. Shared by the search readers;
|
||||
a board states its own remote flag and needs none of this."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
||||
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(
|
||||
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
|
||||
|
||||
|
||||
def remote_word(text: str) -> str | None:
|
||||
"""`remote`, `hybrid` or None: the word the text says."""
|
||||
if _REMOTE.search(text):
|
||||
return "remote"
|
||||
return "hybrid" if _HYBRID.search(text) else None
|
||||
@@ -24,8 +24,7 @@ from urllib.parse import urlencode
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Key, Place, Reads
|
||||
from ._words import remote_word
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
|
||||
|
||||
NAME = "adzuna"
|
||||
LABEL = "Adzuna"
|
||||
|
||||
@@ -9,14 +9,13 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,7 +41,7 @@ _API = "api.ashbyhq.com"
|
||||
_WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"}
|
||||
|
||||
#: `employmentType` -> the app's employment words.
|
||||
_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "intern",
|
||||
_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship",
|
||||
"contract": "contract", "temporary": "temporary"}
|
||||
|
||||
|
||||
@@ -124,20 +123,12 @@ def _places(j: dict[str, Any], workplace: str | None) -> list[Place]:
|
||||
for s in j.get("secondaryLocations") or []]
|
||||
out: list[Place] = []
|
||||
for text, remote in named:
|
||||
if text and text not in (p["text"] for p in out):
|
||||
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
|
||||
else remote})
|
||||
add_place(out, place(text, remote))
|
||||
return out
|
||||
|
||||
|
||||
def _employment(value: Any) -> str | None:
|
||||
word = str(value or "").strip()
|
||||
if not word:
|
||||
return None
|
||||
found = _EMPLOYMENT.get(word.lower())
|
||||
if found is None:
|
||||
log.debug("ashby: unknown employmentType %r", word)
|
||||
return found
|
||||
return table_word(_EMPLOYMENT, value, log, "employmentType")
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
|
||||
@@ -17,14 +17,13 @@ from __future__ import annotations
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, label_slug_ok, only, parse_dt
|
||||
from ._shared import Get, label_slug_ok, only, parse_dt, place, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -39,7 +38,7 @@ READS = Reads(
|
||||
#: the company's own ("Full-Time", "Part-Time", "Contract", "Intern").
|
||||
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "contractor": "contract",
|
||||
"temporary": "temporary", "intern": "intern", "internship": "intern",
|
||||
"temporary": "temporary", "intern": "internship", "internship": "internship",
|
||||
"freelance": "freelance"}
|
||||
|
||||
NAME = "bamboohr"
|
||||
@@ -97,18 +96,11 @@ def _place(j: dict[str, Any]) -> str:
|
||||
def _places(j: dict[str, Any]) -> list[Place]:
|
||||
"""The one place BambooHR states, if it states any."""
|
||||
text = _place(j)
|
||||
return [{"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}] \
|
||||
if text else []
|
||||
return [place(text)] if text else []
|
||||
|
||||
|
||||
def _employment(j: dict[str, Any]) -> str | None:
|
||||
label = str(j.get("employmentStatusLabel") or "").strip()
|
||||
if not label:
|
||||
return None
|
||||
found = _EMPLOYMENT.get(label.lower())
|
||||
if found is None:
|
||||
log.debug("bamboohr: unknown employmentStatusLabel %r", label)
|
||||
return found
|
||||
return table_word(_EMPLOYMENT, j.get("employmentStatusLabel"), log, "employmentStatusLabel")
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
|
||||
@@ -24,7 +24,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import num, only, parse_dt, pay_line
|
||||
from ._shared import num, only, parse_dt, pay_line, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -65,8 +65,8 @@ _REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
|
||||
#: The name of a `jobEmploymentTypes` entry -> the app's employment words.
|
||||
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "contractor": "contract",
|
||||
"consultant": "contract", "temporary": "temporary", "intern": "intern",
|
||||
"internship": "intern", "freelance": "freelance"}
|
||||
"consultant": "contract", "temporary": "temporary", "intern": "internship",
|
||||
"internship": "internship", "freelance": "freelance"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
@@ -123,12 +123,9 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
||||
def _employment(job: dict[str, Any]) -> str | None:
|
||||
"""The first `jobEmploymentTypes` entry that names a known type."""
|
||||
for entry in job.get("jobEmploymentTypes") or []:
|
||||
name = str(((entry or {}).get("employmentType") or {}).get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
if found := _EMPLOYMENT.get(name.lower()):
|
||||
name = ((entry or {}).get("employmentType") or {}).get("name")
|
||||
if found := table_word(_EMPLOYMENT, name, log, "employment type"):
|
||||
return found
|
||||
log.debug("deel: unknown employment type %r", name)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads, Row
|
||||
from ._countries import name_of
|
||||
from ._shared import Get, Page, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, add_place, is_web_url, only, parse_dt, place
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||
@@ -131,8 +131,7 @@ def _places(locations: Any) -> list[Place]:
|
||||
if name and name not in names:
|
||||
names.append(name)
|
||||
text = ", ".join(names)
|
||||
if text and text not in (p["text"] for p in out):
|
||||
out.append({"text": text, "remote": True if "remote" in text.lower() else None})
|
||||
add_place(out, place(text))
|
||||
return out
|
||||
|
||||
|
||||
|
||||
+7
-11
@@ -27,7 +27,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, Page, only, parse_dt
|
||||
from ._shared import Get, Page, add_place, only, parse_dt, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -69,9 +69,9 @@ DETAIL_QUERY = (
|
||||
_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
|
||||
|
||||
#: `job.employmentType` -> the app's employment words.
|
||||
_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACT": "contract",
|
||||
"CONTRACTOR": "contract", "INTERN": "intern", "INTERNSHIP": "intern",
|
||||
"TEMPORARY": "temporary", "FREELANCE": "freelance"}
|
||||
_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contract": "contract",
|
||||
"contractor": "contract", "intern": "internship", "internship": "internship",
|
||||
"temporary": "temporary", "freelance": "freelance"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
@@ -131,8 +131,7 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
places: list[Place] = []
|
||||
for loc in locations:
|
||||
text = str(loc.get("city") or loc.get("name") or "").strip()
|
||||
if text and text not in (pl["text"] for pl in places):
|
||||
places.append({"text": text, "remote": loc["isRemote"]
|
||||
add_place(places, {"text": text, "remote": loc["isRemote"]
|
||||
if isinstance(loc.get("isRemote"), bool) else None})
|
||||
job = p.get("job") or {}
|
||||
remote = _REMOTE.get(str(job.get("locationType") or "").upper())
|
||||
@@ -150,11 +149,8 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
"remote": remote,
|
||||
"salary_text": "",
|
||||
}
|
||||
if word := str(job.get("employmentType") or "").strip().upper():
|
||||
if employment := _EMPLOYMENT.get(word):
|
||||
row["employment"] = employment
|
||||
else:
|
||||
log.debug("gem: unknown employmentType %r", word)
|
||||
if employment := table_word(_EMPLOYMENT, job.get("employmentType"), log, "employmentType"):
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
|
||||
|
||||
NAME = "greenhouse"
|
||||
LABEL = "Greenhouse"
|
||||
@@ -132,19 +132,16 @@ def _places(j: dict[str, Any]) -> list[Place]:
|
||||
out: list[Place] = []
|
||||
for office in j.get("offices") or []:
|
||||
text = str((office or {}).get("location") or (office or {}).get("name") or "").strip()
|
||||
if text and text not in (p["text"] for p in out):
|
||||
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
|
||||
else None})
|
||||
add_place(out, place(text))
|
||||
if not out or not name or _PLACEHOLDER.fullmatch(name):
|
||||
return out or ([{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I)
|
||||
else None}] if name else [])
|
||||
return out or ([place(name)] if name else [])
|
||||
# A name that says something the offices do not (a "Remote India" beside
|
||||
# an office in Bangalore) is one more place; one they already contain,
|
||||
# or that contains them, is the same place said twice.
|
||||
low = name.lower()
|
||||
if any(p["text"].lower() in low or low in p["text"].lower() for p in out):
|
||||
return out
|
||||
return [{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) else None}, *out]
|
||||
return [place(name), *out]
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
|
||||
@@ -23,7 +23,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -44,7 +44,7 @@ _EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time",
|
||||
"full-time": "full_time", "regular part time": "part_time",
|
||||
"part time": "part_time", "part-time": "part_time", "contract": "contract",
|
||||
"contractor": "contract", "temporary": "temporary", "fixed term": "temporary",
|
||||
"intern": "intern", "internship": "intern", "freelance": "freelance"}
|
||||
"intern": "internship", "internship": "internship", "freelance": "freelance"}
|
||||
|
||||
NAME = "hibob"
|
||||
LABEL = "HiBob"
|
||||
@@ -132,11 +132,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"salary_max": salary_max,
|
||||
"currency": code,
|
||||
}
|
||||
kind = str(ad.get("employmentType") or "").strip()
|
||||
if kind:
|
||||
if employment := _EMPLOYMENT.get(kind.lower()):
|
||||
row["employment"] = employment
|
||||
else:
|
||||
log.debug("hibob: unknown employmentType %r", kind)
|
||||
if employment := table_word(_EMPLOYMENT, ad.get("employmentType"), log, "employmentType"):
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
+7
-13
@@ -38,7 +38,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, Page, only, parse_dt
|
||||
from ._shared import Get, Page, add_place, only, parse_dt, place, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -50,9 +50,9 @@ READS = Reads(
|
||||
)
|
||||
|
||||
#: `employment_type` -> the app's employment words.
|
||||
_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACTOR": "contract",
|
||||
"CONTRACT": "contract", "TEMPORARY": "temporary", "INTERN": "intern",
|
||||
"INTERNSHIP": "intern", "FREELANCE": "freelance"}
|
||||
_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contractor": "contract",
|
||||
"contract": "contract", "temporary": "temporary", "intern": "internship",
|
||||
"internship": "internship", "freelance": "freelance"}
|
||||
|
||||
NAME = "jibe"
|
||||
LABEL = "Jibe"
|
||||
@@ -120,9 +120,7 @@ def _places(j: dict[str, Any]) -> list[Place]:
|
||||
out: list[Place] = []
|
||||
for part in line.split(";"):
|
||||
text = part.strip()
|
||||
if text and text not in (p["text"] for p in out):
|
||||
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
|
||||
else None})
|
||||
add_place(out, place(text))
|
||||
return out
|
||||
|
||||
|
||||
@@ -162,11 +160,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
code = str(j.get("country_code") or "").strip().upper()
|
||||
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code):
|
||||
row["country"] = code
|
||||
kind = str(j.get("employment_type") or "").strip()
|
||||
if kind:
|
||||
if employment := _EMPLOYMENT.get(kind.upper()):
|
||||
row["employment"] = employment
|
||||
else:
|
||||
log.debug("jibe: unknown employment_type %r", kind)
|
||||
if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
@@ -28,8 +28,7 @@ from urllib.parse import urlencode
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._words import remote_word
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
|
||||
NAME = "jobindex"
|
||||
|
||||
@@ -25,8 +25,8 @@ from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, parse_dt
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import Get, parse_dt, place, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -70,7 +70,7 @@ _SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern",
|
||||
#: The embed's `employment_type` -> the app's employment words.
|
||||
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "consultant": "contract",
|
||||
"temporary": "temporary", "internship": "intern", "intern": "intern",
|
||||
"temporary": "temporary", "internship": "internship", "intern": "internship",
|
||||
"freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"}
|
||||
|
||||
_ASSIGNMENT = "JBL.embed_v2['jobs'] = "
|
||||
@@ -179,22 +179,6 @@ def _published(value: Any) -> Any:
|
||||
return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}")
|
||||
|
||||
|
||||
def _mapped(table: dict[str, str], value: Any, what: str) -> str | None:
|
||||
"""`table`'s word for the host's `value`; None when the host states none
|
||||
or the table does not know it (logged, so the table can grow)."""
|
||||
word = str(value or "").strip()
|
||||
if not word:
|
||||
return None
|
||||
found = table.get(word.lower())
|
||||
if found is None:
|
||||
log.debug("jobylon: unknown %s %r", what, word)
|
||||
return found or None
|
||||
|
||||
|
||||
def _place(text: str) -> Place:
|
||||
return {"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""The embed page -> one row per role. `posted` is the day `published_date`
|
||||
names, which `detail` replaces with the exact time. `location` joins the
|
||||
@@ -210,7 +194,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
|
||||
continue
|
||||
function = str(j.get("function") or "").strip()
|
||||
places = [_place(name) for name in dict.fromkeys(
|
||||
places = [place(name) for name in dict.fromkeys(
|
||||
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
|
||||
row: Row = {
|
||||
"source_id": job_id,
|
||||
@@ -229,9 +213,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split())
|
||||
if summary and summary != "None":
|
||||
row["teaser"] = summary
|
||||
if seniority := _mapped(_SENIORITY, j.get("experience"), "experience"):
|
||||
if seniority := table_word(_SENIORITY, j.get("experience"), log, "experience"):
|
||||
row["seniority"] = seniority
|
||||
if employment := _mapped(_EMPLOYMENT, j.get("employment_type"), "employment_type"):
|
||||
if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
@@ -252,8 +236,8 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
else:
|
||||
raise ValueError("no JobPosting on the role's page")
|
||||
places = []
|
||||
for place in data.get("jobLocation") or []:
|
||||
address = (place or {}).get("address") or {}
|
||||
for loc in data.get("jobLocation") or []:
|
||||
address = (loc or {}).get("address") or {}
|
||||
name = str(address.get("streetAddress") or address.get("addressLocality") or "").strip()
|
||||
if name and name not in places:
|
||||
places.append(name)
|
||||
@@ -263,5 +247,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
}
|
||||
if places:
|
||||
found["location"] = "; ".join(places)
|
||||
found["places"] = [_place(name) for name in places]
|
||||
found["places"] = [place(name) for name in places]
|
||||
return found
|
||||
@@ -24,7 +24,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -40,7 +40,7 @@ READS = Reads(
|
||||
#: text on Lever ("Permanent" is seen too and says nothing about hours).
|
||||
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time",
|
||||
"part-time": "part_time", "part time": "part_time", "parttime": "part_time",
|
||||
"intern": "intern", "internship": "intern", "contract": "contract",
|
||||
"intern": "internship", "internship": "internship", "contract": "contract",
|
||||
"contractor": "contract", "temporary": "temporary", "freelance": "freelance"}
|
||||
|
||||
NAME = "lever"
|
||||
@@ -115,9 +115,7 @@ def _places(categories: dict[str, Any]) -> list[Place]:
|
||||
out: list[Place] = []
|
||||
for name in names:
|
||||
text = str(name or "").strip()
|
||||
if text and text not in (p["text"] for p in out):
|
||||
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
|
||||
else None})
|
||||
add_place(out, place(text))
|
||||
return out
|
||||
|
||||
|
||||
@@ -158,11 +156,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
country = str(j.get("country") or "").strip().upper()
|
||||
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country):
|
||||
row["country"] = country
|
||||
commitment = str(categories.get("commitment") or "").strip()
|
||||
if commitment:
|
||||
if employment := _EMPLOYMENT.get(commitment.lower()):
|
||||
row["employment"] = employment
|
||||
else:
|
||||
log.debug("lever: unknown commitment %r", commitment)
|
||||
if employment := table_word(_EMPLOYMENT, categories.get("commitment"), log, "commitment"):
|
||||
row["employment"] = employment
|
||||
rows.append(row)
|
||||
return rows
|
||||
@@ -35,7 +35,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import NAMES, code_of
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt
|
||||
|
||||
NAME = "oracle"
|
||||
LABEL = "Oracle Recruiting Cloud"
|
||||
@@ -197,8 +197,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
_text(role.get("PrimaryLocationCountry"))),
|
||||
*((_text(s.get("Name")), _text(s.get("CountryCode")))
|
||||
for s in secondary)):
|
||||
if name and name not in (p["text"] for p in places):
|
||||
places.append({"text": name, "remote": {"remote": True, "hybrid": False,
|
||||
add_place(places, {"text": name, "remote": {"remote": True, "hybrid": False,
|
||||
"onsite": False}.get(remote or "")})
|
||||
if code:
|
||||
codes.add(code.upper())
|
||||
@@ -213,7 +212,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
or _text(role.get("Department")) or _text(role.get("Organization"))),
|
||||
"description": "",
|
||||
"teaser": _plain(role.get("ShortDescriptionStr")),
|
||||
"country": next(iter(codes)) if len(codes) == 1 else None,
|
||||
"country": country_if_single(codes),
|
||||
"remote": remote,
|
||||
"salary_text": "", # the resource states no pay
|
||||
})
|
||||
|
||||
@@ -14,7 +14,7 @@ from collections.abc import Callable
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, table_word, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -86,12 +86,7 @@ def _salary_text(salary_info: ET.Element) -> str:
|
||||
|
||||
|
||||
def _seniority(word: str) -> str | None:
|
||||
key = word.strip().lower()
|
||||
if key not in _SENIORITY:
|
||||
if key:
|
||||
log.debug("personio: unknown seniority %r", word)
|
||||
return None
|
||||
return _SENIORITY[key] or None
|
||||
return table_word(_SENIORITY, word, log, "seniority")
|
||||
|
||||
|
||||
def _employment(position: ET.Element) -> str | None:
|
||||
@@ -113,8 +108,7 @@ def _places(position: ET.Element) -> list[Place]:
|
||||
*(o.text for o in position.findall("additionalOffices/office"))]
|
||||
for name in names:
|
||||
text = (name or "").strip()
|
||||
if text and text not in (p["text"] for p in found):
|
||||
found.append({"text": text, "remote": True if "remote" in text.lower() else None})
|
||||
add_place(found, place(text))
|
||||
return found
|
||||
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import name_of
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt
|
||||
|
||||
NAME = "phenom"
|
||||
LABEL = "Phenom"
|
||||
@@ -163,10 +163,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
if not title or not key:
|
||||
continue
|
||||
places: list[Place] = []
|
||||
for place in (job.get("location"), *(job.get("multi_location") or [])):
|
||||
if isinstance(place, str) and place.strip() \
|
||||
and place.strip() not in (p["text"] for p in places):
|
||||
places.append({"text": place.strip(), "remote": None})
|
||||
for text in (job.get("location"), *(job.get("multi_location") or [])):
|
||||
if isinstance(text, str):
|
||||
add_place(places, {"text": text.strip(), "remote": None})
|
||||
tail = _slug_of(title)
|
||||
out.append({
|
||||
"source_id": key,
|
||||
|
||||
@@ -22,8 +22,7 @@ from urllib.parse import urlencode
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._words import remote_word
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
|
||||
|
||||
NAME = "platsbanken"
|
||||
LABEL = "Platsbanken"
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
from ._shared import Row, add_place, country_if_single, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -83,12 +83,7 @@ def _salary_text(salary: dict[str, Any]) -> str:
|
||||
|
||||
|
||||
def _seniority(code: str) -> str | None:
|
||||
key = code.strip().lower()
|
||||
if key not in _SENIORITY:
|
||||
if key:
|
||||
log.debug("recruitee: unknown experience_code %r", code)
|
||||
return None
|
||||
return _SENIORITY[key] or None
|
||||
return table_word(_SENIORITY, code, log, "experience_code")
|
||||
|
||||
|
||||
def _employment(code: str) -> str | None:
|
||||
@@ -112,8 +107,8 @@ def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
|
||||
|
||||
def add(text: str) -> None:
|
||||
text = text.strip()
|
||||
if text and text.lower() not in _NO_PLACE and text not in (p["text"] for p in found):
|
||||
found.append({"text": text, "remote": None})
|
||||
if text.lower() not in _NO_PLACE:
|
||||
add_place(found, {"text": text, "remote": None})
|
||||
|
||||
for loc in offer.get("locations") or []:
|
||||
if not isinstance(loc, dict):
|
||||
@@ -165,7 +160,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
"country": next(iter(codes)) if len(codes) == 1 else None,
|
||||
"country": country_if_single(codes),
|
||||
"seniority": _seniority(str(j.get("experience_code") or "")),
|
||||
"employment": _employment(str(j.get("employment_type_code") or "")),
|
||||
})
|
||||
|
||||
@@ -20,7 +20,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Row, parse_dt, yearly_amount
|
||||
from ._shared import Get, Row, add_place, parse_dt, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -100,8 +100,8 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
continue
|
||||
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
|
||||
places.setdefault(job_id, [])
|
||||
if place and place not in (p["text"] for p in places[job_id]):
|
||||
places[job_id].append(_place(place))
|
||||
if place:
|
||||
add_place(places[job_id], _place(place))
|
||||
roles.setdefault(job_id, {
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
|
||||
@@ -19,7 +19,7 @@ from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -97,10 +97,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None:
|
||||
"""The `id` of the object `job[key]` looked up in `table`; none for a word
|
||||
the table maps to none, and a word it does not know is logged."""
|
||||
word = str((job.get(key) or {}).get("id") or "").strip().lower()
|
||||
if word and word not in table:
|
||||
log.debug("smartrecruiters: unknown %s %r", key, word)
|
||||
return table.get(word) or None
|
||||
return table_word(table, (job.get(key) or {}).get("id"), log, key)
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
|
||||
@@ -59,7 +59,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import NAMES
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, country_if_single, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "successfactors"
|
||||
LABEL = "SuccessFactors"
|
||||
@@ -392,13 +392,12 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
addresses = _addresses(text) or [meta]
|
||||
places: list[Place] = []
|
||||
for address in addresses:
|
||||
if (place := _location(address)) and place not in (p["text"] for p in places):
|
||||
places.append({"text": place, "remote": None})
|
||||
add_place(places, {"text": _location(address), "remote": None})
|
||||
if places:
|
||||
out["places"] = places
|
||||
out["location"] = "; ".join(p["text"] for p in places)
|
||||
codes = {_country_code(address) for address in addresses}
|
||||
if len(codes) == 1 and (code := next(iter(codes))):
|
||||
if code := country_if_single(codes):
|
||||
out["country"] = code
|
||||
return out
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "teamtailor"
|
||||
LABEL = "Teamtailor"
|
||||
@@ -129,9 +129,8 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
for loc in item.findall(f"{_TT}locations/{_TT}location"):
|
||||
city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip()
|
||||
country = (loc.findtext(f"{_TT}country") or "").strip()
|
||||
place = city if country == city else ", ".join(p for p in (city, country) if p)
|
||||
if place and place not in (p["text"] for p in places):
|
||||
places.append({"text": place, "remote": remote})
|
||||
text = city if country == city else ", ".join(p for p in (city, country) if p)
|
||||
add_place(places, {"text": text, "remote": remote})
|
||||
out.append({
|
||||
"source_id": (item.findtext("guid") or url).strip(),
|
||||
"title": title,
|
||||
|
||||
@@ -13,7 +13,7 @@ from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt, path_slug_ok, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -82,8 +82,7 @@ def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
|
||||
if part and part not in parts:
|
||||
parts.append(part)
|
||||
text = ", ".join(parts)
|
||||
if text and text not in (p["text"] for p in found):
|
||||
found.append({"text": text, "remote": remote})
|
||||
add_place(found, {"text": text, "remote": remote})
|
||||
if code := str(loc.get("countryCode") or "").strip().upper():
|
||||
codes.add(code)
|
||||
return found, codes
|
||||
@@ -107,9 +106,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
continue
|
||||
remote = bool(j.get("remote")) or j.get("workplace") == "remote"
|
||||
places, codes = _places(j, remote)
|
||||
kind = str(j.get("type") or "").strip().lower()
|
||||
if kind and kind not in _EMPLOYMENT:
|
||||
log.debug("workable: unknown type %r", kind)
|
||||
dept = j.get("department") or ""
|
||||
if isinstance(dept, list):
|
||||
dept = ", ".join(d for d in dept if d)
|
||||
@@ -124,7 +120,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"description": "",
|
||||
"remote": remote,
|
||||
"salary_text": "", # Workable's list endpoint states no pay at all
|
||||
"country": next(iter(codes)) if len(codes) == 1 else None,
|
||||
"employment": _EMPLOYMENT.get(kind),
|
||||
"country": country_if_single(codes),
|
||||
"employment": table_word(_EMPLOYMENT, j.get("type"), log, "type"),
|
||||
})
|
||||
return out
|
||||
@@ -39,7 +39,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import code_of
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -320,7 +320,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
out["location"] = "; ".join(p["text"] for p in places)
|
||||
if len(places) == 1 and (code := _country_code(posting)):
|
||||
out["country"] = code
|
||||
if employment := _EMPLOYMENT.get(str(posting.get("timeType") or "").strip().lower()):
|
||||
if employment := table_word(_EMPLOYMENT, posting.get("timeType"), log, "timeType"):
|
||||
out["employment"] = employment
|
||||
if posted := parse_dt(posting.get("startDate")):
|
||||
out["posted"] = posted
|
||||
@@ -344,9 +344,8 @@ def _places(posting: dict[str, Any], remote: str | None) -> list[Place]:
|
||||
(LEGO's "Billund") gets it from the role's `country`."""
|
||||
found: list[Place] = []
|
||||
for name in [posting.get("location"), *(posting.get("additionalLocations") or [])]:
|
||||
place = _place(name, remote)
|
||||
if place and place["text"] not in (p["text"] for p in found):
|
||||
found.append(place)
|
||||
if place := _place(name, remote):
|
||||
add_place(found, place)
|
||||
country = str((posting.get("country") or {}).get("descriptor") or "").strip()
|
||||
if len(found) == 1 and country and country.lower() not in found[0]["text"].lower():
|
||||
found[0]["text"] += f", {country}"
|
||||
|
||||
@@ -141,7 +141,7 @@ def test_the_experience_is_mapped_and_a_word_the_table_lacks_is_logged(
|
||||
assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values())
|
||||
rows = {r["source_id"]: r for r in jobylon.parse(
|
||||
(FIXTURES / "jobylon-places.html").read_bytes(), "3143")}
|
||||
assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "intern"
|
||||
assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "internship"
|
||||
|
||||
|
||||
def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None:
|
||||
|
||||
@@ -56,7 +56,7 @@ def test_it_reads_title_company_town_date_and_teaser() -> None:
|
||||
|
||||
|
||||
def test_it_reads_hybrid_and_remote_from_the_town_line() -> None:
|
||||
from plugins.readers._words import remote_word
|
||||
from plugins.readers._shared import remote_word
|
||||
|
||||
assert remote_word("Birkerød, Hybrid position") == "hybrid"
|
||||
assert remote_word("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""The helpers the host modules share, and the one employment vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import _shared # noqa: E402
|
||||
from plugins.readers._shared import ( # noqa: E402
|
||||
EMPLOYMENTS,
|
||||
add_place,
|
||||
country_if_single,
|
||||
place,
|
||||
remote_word,
|
||||
table_word,
|
||||
)
|
||||
|
||||
#: The module-level names a reader keeps its employment words in.
|
||||
_TABLES = ("_EMPLOYMENT", "_CONTRACT", "_SCHEDULE")
|
||||
|
||||
|
||||
def test_every_employment_table_maps_into_the_shared_vocabulary() -> None:
|
||||
found = 0
|
||||
for name, module in readers.READERS.items():
|
||||
for table_name in _TABLES:
|
||||
table = getattr(module, table_name, None)
|
||||
if table is None:
|
||||
continue
|
||||
found += 1
|
||||
values = table.values() if isinstance(table, dict) else [v for _k, v in table]
|
||||
# "" is a known word that names no employment ("permanent").
|
||||
assert set(values) <= {*EMPLOYMENTS, ""}, (name, table_name)
|
||||
assert found >= 15
|
||||
|
||||
|
||||
def test_the_vocabulary_is_the_documented_one() -> None:
|
||||
assert EMPLOYMENTS == ("full_time", "part_time", "contract", "temporary", "internship",
|
||||
"freelance")
|
||||
|
||||
|
||||
def test_place_is_remote_on_the_word_and_otherwise_as_stated() -> None:
|
||||
assert place("Remote (DE)") == {"text": "Remote (DE)", "remote": True}
|
||||
assert place("Berlin, remote") == {"text": "Berlin, remote", "remote": True}
|
||||
assert place("Berlin") == {"text": "Berlin", "remote": None}
|
||||
assert place("Berlin", False) == {"text": "Berlin", "remote": False}
|
||||
assert place("Remote", False)["remote"] is True
|
||||
assert place("Remotely", None)["remote"] is None
|
||||
|
||||
|
||||
def test_add_place_skips_empty_and_repeated_texts_and_keeps_order() -> None:
|
||||
found: list = []
|
||||
for text in ("Berlin", "", "Oslo", "Berlin"):
|
||||
add_place(found, place(text))
|
||||
assert [p["text"] for p in found] == ["Berlin", "Oslo"]
|
||||
|
||||
|
||||
def test_table_word_maps_ignores_case_and_logs_only_unknown_words(caplog) -> None:
|
||||
table = {"full-time": "full_time", "permanent": ""}
|
||||
log = logging.getLogger("plugins.readers.acme")
|
||||
with caplog.at_level(logging.DEBUG, logger=log.name):
|
||||
assert table_word(table, " Full-Time ", log, "type") == "full_time"
|
||||
assert table_word(table, "Permanent", log, "type") is None
|
||||
assert table_word(table, None, log, "type") is None
|
||||
assert not caplog.records
|
||||
assert table_word(table, "Odd", log, "type") is None
|
||||
assert [r.getMessage() for r in caplog.records] == ["acme: unknown type 'Odd'"]
|
||||
|
||||
|
||||
def test_country_if_single_needs_exactly_one_code() -> None:
|
||||
assert country_if_single({"DE"}) == "DE"
|
||||
assert country_if_single({"DE", "SE"}) is None
|
||||
assert country_if_single(set()) is None
|
||||
|
||||
|
||||
def test_remote_word_reads_danish_swedish_and_english() -> None:
|
||||
assert remote_word("Remote, Danmark") == "remote"
|
||||
assert remote_word("Birkerød, Hybrid position") == "hybrid"
|
||||
assert remote_word("Hedehusene") is None
|
||||
assert _shared.remote_word is remote_word
|
||||
+17
-3
@@ -143,7 +143,7 @@ def client(app: FastAPI) -> Iterator[TestClient]:
|
||||
yield c
|
||||
|
||||
|
||||
def test_get_stages_lists_both_with_their_source_model_and_who_answers(
|
||||
def test_get_stages_lists_all_three_with_their_source_model_and_who_answers(
|
||||
client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
answered_by(monkeypatch, "api")
|
||||
res = client.get("/api/stages")
|
||||
@@ -156,6 +156,8 @@ def test_get_stages_lists_both_with_their_source_model_and_who_answers(
|
||||
{"name": "summarise", "on": False, "source": "config",
|
||||
"model": "claude-haiku-4-5-20251001", "backend": "api",
|
||||
"answer": "answered by the Claude API"},
|
||||
{"name": "enrich", "on": True, "source": "config", "model": "", "backend": "",
|
||||
"answer": "reads role pages, asks no model"},
|
||||
]
|
||||
|
||||
|
||||
@@ -164,7 +166,7 @@ def test_who_answers_is_said_for_the_api_claude_code_and_nobody(
|
||||
answers = {}
|
||||
for backend in ("api", "claude_code", ""):
|
||||
answered_by(monkeypatch, backend)
|
||||
[first, second] = client.get("/api/stages").json()
|
||||
[first, second, _enrich] = client.get("/api/stages").json()
|
||||
assert first["backend"] == second["backend"] == backend
|
||||
answers[backend] = first["answer"]
|
||||
|
||||
@@ -191,6 +193,18 @@ def test_put_stores_the_setting_returns_the_same_shape_and_starts_nothing(
|
||||
assert res.json()["on"] is False and store.state["stage:summarise"] == "off"
|
||||
|
||||
|
||||
def test_the_enrich_switch_is_listed_and_stored_like_the_others(
|
||||
client: TestClient, store: Store) -> None:
|
||||
res = client.put("/api/stages/enrich", json={"on": False})
|
||||
|
||||
assert res.status_code == 200
|
||||
assert res.json()["name"] == "enrich" and res.json()["on"] is False
|
||||
assert store.state["stage:enrich"] == "off"
|
||||
listed = {stage["name"]: stage for stage in client.get("/api/stages").json()}
|
||||
assert list(listed) == ["extract", "summarise", "enrich"]
|
||||
assert listed["enrich"]["on"] is False and listed["enrich"]["source"] == "stored"
|
||||
|
||||
|
||||
def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> None:
|
||||
assert client.put("/api/stages/embed", json={"on": True}).status_code == 404
|
||||
assert client.put("/api/stages/extract", json={"on": "sometimes"}).status_code == 422
|
||||
@@ -200,7 +214,7 @@ def test_an_unknown_stage_is_404_and_a_bad_body_is_422(client: TestClient) -> No
|
||||
def test_agent_backend_off_is_named_rather_than_blamed_on_missing_keys(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, store: Store) -> None:
|
||||
off = Config(database=str(tmp_path / "roledesk.db"), agent={"backend": "off"})
|
||||
[first, _] = stages.describe(off, store)
|
||||
[first, *_] = stages.describe(off, store)
|
||||
assert first.backend == "" and "agent.backend" in first.answer
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user