Integration check: wp-p-prep with wp-r1-raw and wp-r2-raw

Resolves tests/test_readers_shared.py, adds MATCH_MODE to the pinpoint,
recruitee and rippling tables, and removes table_word, country_if_single and
remote_word from plugins/readers/_shared.py with their tests and the Helpers
paragraph.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:31:31 +02:00
commit 55757c41e5
51 files changed
+445 -502

No files matched your search

+6 -16
View File
@@ -178,28 +178,18 @@ posting per place, each with its own id, declares `places="copies"`: each
copy is a row with its one place, and `dupes.py` folds them into one role
that keeps each copy's link.
**Helpers.** `._shared` holds what more than one host needs; a module keeps
only its own word tables and uses these:
**Helpers.** `._shared` holds what more than one host needs. A module maps
no word to the app's vocabulary: it hands on the host's own words under
`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them
through the host's table in `app/prep/hosts/<name>.py`.
- `place(text, remote=None)` makes a `Place`: remote when the text has the
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
appends a place unless its text is empty or already in `found`, in the
host's order. A host that states remote itself passes it to `place`, or
builds the dict and passes that to `add_place`.
- `table_word(table, value, log, what)` maps a host's word (stated seniority
or employment) through the module's table, without case. The table's keys
are lowercase. Empty gives None; a known word mapped to `""` (`experienced`,
`permanent`) names no level and gives None; an unknown word gives None and
a debug line naming the host, so the table can grow.
- `EMPLOYMENTS`, the one employment vocabulary: `full_time`, `part_time`,
`contract`, `temporary`, `internship`, `freelance`. Every value of a
module's `_EMPLOYMENT` table is one of these or `""`; the test in
`tests/test_readers_shared.py` holds every module to it. Seniority values
are `app.models.SENIORITIES`.
- `country_if_single(codes)` is the code when the role's places share exactly
one, else None, for a host that states a country per place.
- `remote_word(text)` reads `remote` or `hybrid` from a search provider's
headline or town line (Danish, Swedish, English).
- `EMPLOYMENTS` is a copy of `app.models.EMPLOYMENTS` for the readers'
tests, since a plugin imports nothing from `app`.
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
over as `teaser`, plain text, whether or not the full text comes later. It
-34
View File
@@ -8,7 +8,6 @@ import it without either importing the other. The leading underscore keeps
from __future__ import annotations
import logging
import re
from collections.abc import Callable
from datetime import UTC, datetime
@@ -218,36 +217,3 @@ def add_place(found: list[Place], entry: Place) -> None:
found.append(entry)
def table_word(table: dict[str, str], value: Any, log: logging.Logger, what: str) -> str | None:
"""`table`'s word for a host's `value`, looked up without case. None when
the host states none, when the table maps the word to "" (a known word
that names no level, such as `experienced`), or when the table does not
know it; that last case is logged at debug level so the table can grow."""
word = str(value or "").strip()
if not word:
return None
found = table.get(word.lower())
if found is None:
log.debug("%s: unknown %s %r", log.name.rsplit(".", 1)[-1], what, word)
return found or None
def country_if_single(codes: Any) -> str | None:
"""The one code among `codes` when there is exactly one, else None: a
country is stated for a role only when all its places share it."""
return next(iter(codes)) if len(codes) == 1 else None
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
def remote_word(text: str) -> str | None:
"""`remote`, `hybrid` or None: the word a search provider's headline or
town line says about where the work is done, in Danish, Swedish and
English. A board states its own remote flag and needs none of this."""
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
+1 -2
View File
@@ -24,7 +24,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Key, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "adzuna"
LABEL = "Adzuna"
@@ -124,7 +124,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
"places": places,
"location": town,
"country": country.upper(),
"remote": remote_word(title),
"posted": parse_dt(job.get("created")),
"teaser": _text(job.get("description") or ""),
"salary_min": None if predicted else job.get("salary_min"),
+2 -5
View File
@@ -28,7 +28,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, place
from ._contract import Ask, Fields, Filters, Place, Reads
NAME = "jobindex"
@@ -131,9 +131,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
company = full_title[len(title) + 2:].strip()
area = _AREA.search(description)
town = _text(area.group(1)) if area else decoded["area"]
remote = remote_word(f"{town} {title}")
places: list[Place] = [{"text": town, "remote": True if remote == "remote" else None}] \
if town else []
places: list[Place] = [place(town)] if town else []
out.append({
"source_id": item["id"],
"title": title or full_title,
@@ -142,7 +140,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
"places": places,
"location": town,
"country": "DK",
"remote": remote,
"posted": parse_dt(item["published"]),
"teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
})
+8 -18
View File
@@ -35,7 +35,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES, code_of
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt
from ._shared import Get, Page, Row, add_place, only, parse_dt
NAME = "oracle"
LABEL = "Oracle Recruiting Cloud"
@@ -55,7 +55,7 @@ PATTERNS = (
READS = Reads(
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
complete="stated", places="many", filters=Filters(country="ids", remote="one"),
fields=Fields(posted="trusted", remote="stated", country="stated"),
fields=Fields(posted="trusted", remote="stated"),
robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites/<site>/jobs "
"(checked 2026-09-29)"),
)
@@ -71,11 +71,6 @@ PAGE_SIZE = 200
_HOST = r"[a-z0-9-]+(?:\.[a-z0-9-]+)*\.oraclecloud\.(?:com|eu)"
_SLUG = re.compile(rf"({_HOST})/([A-Za-z0-9_-]+)", re.IGNORECASE)
#: `WorkplaceTypeCode` -> `store._remote`'s words. A role that states none
#: has an empty code, which stays unknown.
_REMOTE = {"ora_remote": "remote", "ora_hybrid": "hybrid", "ora_on_site": "onsite"}
def _parts(slug: str) -> tuple[str, str]:
"""`(host, site number)` of a slug that passed `slug_ok`."""
host, _, site = slug.partition("/")
@@ -189,18 +184,14 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
# The id goes into a URL, and a listing is external content.
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id):
continue
remote = _REMOTE.get(_text(role.get("WorkplaceTypeCode")).lower())
remote = _text(role.get("WorkplaceTypeCode")) or None
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
places: list[Place] = []
codes: set[str] = set()
secondary = [s for s in role.get("secondaryLocations") or [] if isinstance(s, dict)]
for name, code in ((_text(role.get("PrimaryLocation")),
_text(role.get("PrimaryLocationCountry"))),
*((_text(s.get("Name")), _text(s.get("CountryCode")))
for s in secondary)):
add_place(places, {"text": name, "remote": {"remote": True, "hybrid": False,
"onsite": False}.get(remote or "")})
if code:
codes.add(code.upper())
for name in (_text(role.get("PrimaryLocation")),
*(_text(s.get("Name")) for s in secondary)):
add_place(places, {"text": name, "remote": (
True if (remote or "").lower() == "ora_remote" else False if at_office else None)})
out.append({
"source_id": role_id,
"title": title,
@@ -212,7 +203,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
or _text(role.get("Department")) or _text(role.get("Organization"))),
"description": "",
"teaser": _plain(role.get("ShortDescriptionStr")),
"country": country_if_single(codes),
"remote": remote,
"salary_text": "", # the resource states no pay
})
+9 -30
View File
@@ -7,16 +7,13 @@ Personio.
from __future__ import annotations
import logging
import xml.etree.ElementTree as ET
from collections.abc import Callable
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, table_word, yearly_amount
log = logging.getLogger(__name__)
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount
NAME = "personio"
LABEL = "Personio"
@@ -30,19 +27,6 @@ READS = Reads(
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
#: `seniority` -> the app's levels. `experienced` is what most roles say and
#: names no level, so it is known and maps to none; a word not listed is
#: logged at debug level so the table can grow.
_SENIORITY = {"entry-level": "junior", "student": "intern", "executive": "exec",
"experienced": ""}
#: `employmentType` when it is not the usual permanent contract, to the app's
#: employment words; `permanent` says nothing about the hours, so `schedule`
#: decides then.
_CONTRACT = {"temporary": "temporary", "freelance": "contract", "intern": "internship",
"working_student": "internship", "trainee": "internship"}
_SCHEDULE = {"full-time": "full_time", "part-time": "part_time"}
#: Personio's `salaryInformation/type` spellings, to the plain period word
#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through.
_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
@@ -85,18 +69,14 @@ def _salary_text(salary_info: ET.Element) -> str:
salary_info.findtext("currencyCode"), period)
def _seniority(word: str) -> str | None:
return table_word(_SENIORITY, word, log, "seniority")
def _employment(position: ET.Element) -> str | None:
contract = (position.findtext("employmentType") or "").strip().lower()
schedule = (position.findtext("schedule") or "").strip().lower()
if contract in _CONTRACT:
return _CONTRACT[contract]
if contract and contract != "permanent":
log.debug("personio: unknown employmentType %r", contract)
return _SCHEDULE.get(schedule)
"""The host's word for the contract, or, for a `permanent` one (which says
nothing about the hours), the `schedule`."""
contract = (position.findtext("employmentType") or "").strip()
schedule = (position.findtext("schedule") or "").strip()
if contract.lower() in ("", "permanent") and schedule:
return schedule
return contract or None
def _places(position: ET.Element) -> list[Place]:
@@ -148,8 +128,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"posted": parse_dt(position.findtext("createdAt")),
"department": (position.findtext("department") or "").strip(),
"description": strip_html(description),
"remote": any(p["remote"] for p in places) or None,
"seniority": _seniority(position.findtext("seniority") or ""),
"seniority": (position.findtext("seniority") or "").strip() or None,
"employment": _employment(position),
"salary_text": salary_text,
"salary_min": salary_min,
+3 -4
View File
@@ -188,10 +188,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
_JSON_LD = re.compile(r"<script[^>]+application/ld\+json[^>]*>(.*?)</script>", re.S | re.I)
#: Where a page assigns its state, and the words its `checkRemote` says that
#: `store._remote` reads (King's pages say "On-site"; no other was seen).
#: Where a page assigns its state. Its `checkRemote` says "On-site" on King's
#: pages; no other word was seen.
_STATE = "phApp.ddo = "
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
def _job_posting(body: str) -> dict[str, Any]:
@@ -243,6 +242,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
raise ValueError("no description on the role's page")
if posted := parse_dt(posting.get("datePosted")):
out["posted"] = posted
if remote := _REMOTE.get(_text(job.get("checkRemote")).lower()):
if remote := _text(job.get("checkRemote")):
out["remote"] = remote
return out
+4 -26
View File
@@ -18,7 +18,6 @@ from __future__ import annotations
import html
import json
import logging
from collections.abc import Callable
from typing import Any
@@ -27,8 +26,6 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Reads
from ._shared import Row, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
log = logging.getLogger(__name__)
NAME = "pinpoint"
LABEL = "Pinpoint"
PATTERNS = (r"https?://([a-z0-9-]+)\.pinpointhq\.com",)
@@ -39,9 +36,6 @@ READS = Reads(
robots="read: robots.txt allows / (checked 2026-09-29)",
)
#: `workplace_type` -> `store._remote`'s words. Anything else says nothing.
_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
#: `compensation_frequency` -> the multiplier that makes a figure yearly. The
#: spellings are a guess, since no board read so far states pay; a frequency
#: not listed leaves the yearly columns empty, and `salary_text` still says it.
@@ -51,12 +45,6 @@ _YEARLY = {"annually": 1, "annual": 1, "yearly": 1, "year": 1, "monthly": 12, "m
_PERIOD = {"annually": "year", "annual": "year", "yearly": "year", "monthly": "month",
"weekly": "week", "daily": "day", "hourly": "hour"}
#: What `employment_type` (`permanent_full_time`, `fixed_term_full_time`) says,
#: the first fragment found deciding, so a contract type beats the hours.
_EMPLOYMENT = (("intern", "internship"), ("freelance", "contract"), ("contract", "contract"),
("fixed_term", "temporary"), ("temporary", "temporary"),
("part_time", "part_time"), ("full_time", "full_time"))
#: The sections of a posting's text, in the order the page shows them: the
#: description, then these fields, each with its own heading in `<field>_header`.
_SECTIONS = ("key_responsibilities", "skills_knowledge_expertise", "benefits")
@@ -112,16 +100,6 @@ def _place(location: dict[str, Any]) -> str:
return ", ".join(parts)
def _employment(code: str) -> str | None:
code = code.strip().lower()
for fragment, value in _EMPLOYMENT:
if fragment in code:
return value
if code:
log.debug("pinpoint: unknown employment_type %r", code)
return None
def _pay(j: dict[str, Any]) -> tuple[int | None, int | None, str, str]:
"""`(salary_min, salary_max, currency, salary_text)` of a posting whose
pay is shown (`compensation_visible`); the free-text `compensation` is the
@@ -154,21 +132,21 @@ def parse(body: bytes, slug: str) -> list[Row]:
continue
job = j.get("job") or {}
low, high, currency, salary_text = _pay(j)
workplace = str(j.get("workplace_type") or "").strip().lower()
workplace = str(j.get("workplace_type") or "").strip()
place = _place(j.get("location") or {})
out.append({
"source_id": source_id,
"title": title,
"places": [{"text": place, "remote": {"remote": True, "onsite": False,
"on_site": False, "hybrid": False}.get(workplace)}]
"on_site": False, "hybrid": False}.get(workplace.lower())}]
if place else [],
"location": place,
"url": url,
"posted": None, # only the posting's own page has a date
"department": str((job.get("department") or {}).get("name") or "").strip(),
"description": _text(j),
"remote": _REMOTE.get(workplace),
"employment": _employment(str(j.get("employment_type") or "")),
"remote": workplace or None,
"employment": str(j.get("employment_type") or "").strip() or None,
"salary_text": salary_text,
"salary_min": low,
"salary_max": high,
+5 -5
View File
@@ -7,8 +7,8 @@ Checked live on 2026-09-29 and 2026-09-30, moved here from `plugins/search.py`.
above 2000 is refused, so a search reads 2,100 results at most. It states
`total`. A hit has the whole ad's text, so the list has the full text. An ad
may name several workplaces: `workplace_addresses` holds one entry each, and
every one is a place of the role. `workplace_model` is mostly empty, so
`remote` comes from the headline and the model together.
every one is a place of the role. `workplace_model` is mostly empty; its
label is passed on as `remote`, in the host's words.
"""
from __future__ import annotations
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "platsbanken"
LABEL = "Platsbanken"
@@ -113,7 +113,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
places = [place for address in addresses if isinstance(address, dict)
if (place := _place(address))]
model = (hit.get("workplace_model") or {}).get("label") or ""
model = str((hit.get("workplace_model") or {}).get("label") or "").strip()
out.append({
"source_id": str(hit.get("id") or ""),
"title": title,
@@ -122,7 +122,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
"places": places,
"location": places[0]["text"] if places else "",
"country": "SE",
"remote": remote_word(f"{title} {model}"),
"remote": model or None,
"posted": parse_dt(hit.get("publication_date")),
"description": _text((hit.get("description") or {}).get("text") or ""),
})
+8 -43
View File
@@ -8,16 +8,13 @@ Recruitee.
from __future__ import annotations
import json
import logging
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, add_place, country_if_single, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
log = logging.getLogger(__name__)
from ._shared import Row, add_place, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "recruitee"
LABEL = "Recruitee"
@@ -27,23 +24,11 @@ PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",)
READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated",
fields=Fields(pay="figures", posted="trusted", remote="stated",
seniority="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
#: `experience_code` -> the app's levels. `experienced` names no level and is
#: known, so it maps to none; a word not listed is logged at debug level.
_SENIORITY = {"student_school": "intern", "entry_level": "junior", "mid_level": "mid",
"experienced": ""}
#: The words of `employment_type_code` (`fulltime_permanent`), the first that
#: is found deciding, so a contract type beats the hours.
_EMPLOYMENT = (("internship", "internship"), ("intern", "internship"),
("freelance", "contract"), ("contract", "contract"),
("temporary", "temporary"), ("fixed", "temporary"),
("parttime", "part_time"), ("fulltime", "full_time"))
#: What the board writes as a place for a role that is worked anywhere.
_NO_PLACE = ("remote job",)
@@ -82,28 +67,13 @@ def _salary_text(salary: dict[str, Any]) -> str:
salary.get("currency"), period)
def _seniority(code: str) -> str | None:
return table_word(_SENIORITY, code, log, "experience_code")
def _employment(code: str) -> str | None:
words = set(code.strip().lower().replace("-", "_").split("_"))
for word, value in _EMPLOYMENT:
if word in words:
return value
if code.strip():
log.debug("recruitee: unknown employment_type_code %r", code)
return None
def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
"""Every place of an offer and the countries' codes among them. `locations`
def _places(offer: dict[str, Any]) -> list[Place]:
"""Every place of an offer. `locations`
lists them (city, state, country), `location` is the first as one line and
is the fallback for an offer that lists none. The board writes "Remote
job" where an offer has no place: that is not one, and a remote offer with
no other place has the place "Remote"."""
found: list[Place] = []
codes: set[str] = set()
def add(text: str) -> None:
text = text.strip()
@@ -119,15 +89,11 @@ def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
if part and part not in parts:
parts.append(part)
add(", ".join(parts))
if code := str(loc.get("country_code") or "").strip().upper():
codes.add(code)
if not found:
add(str(offer.get("location") or offer.get("city") or ""))
if code := str(offer.get("country_code") or "").strip().upper():
codes.add(code)
if not found and offer.get("remote"):
found.append({"text": "Remote", "remote": True})
return found, codes
return found
def parse(body: bytes, slug: str) -> list[Row]:
@@ -145,7 +111,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
salary_min, salary_max, currency = yearly_amount(
salary.get("min"), salary.get("max"), salary.get("currency"),
salary.get("period"), {"year": 1, "month": 12})
places, codes = _places(j)
places = _places(j)
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
@@ -160,8 +126,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
"country": country_if_single(codes),
"seniority": _seniority(str(j.get("experience_code") or "")),
"employment": _employment(str(j.get("employment_type_code") or "")),
"seniority": str(j.get("experience_code") or "").strip() or None,
"employment": str(j.get("employment_type_code") or "").strip() or None,
})
return rows
+3 -19
View File
@@ -12,7 +12,6 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
@@ -22,8 +21,6 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Row, add_place, parse_dt, yearly_amount
log = logging.getLogger(__name__)
NAME = "rippling"
LABEL = "Rippling"
KEEP_CASE = True
@@ -46,11 +43,6 @@ _SITE = "ats.rippling.com"
#: whole; it is also what discovery's patterns take.
_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
#: The words of `employmentType.label` (`SALARIED_FT`), split at "_".
_EMPLOYMENT = {"FT": "full_time", "PT": "part_time", "INTERN": "internship",
"INTERNSHIP": "internship", "CONTRACT": "contract", "CONTRACTOR": "contract",
"TEMP": "temporary", "TEMPORARY": "temporary", "SEASONAL": "temporary"}
#: A role's id in the board's own links.
_UUID = re.compile(r"[0-9A-Za-z-]{8,64}")
@@ -86,8 +78,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
role and place, so a role in three cities is three entries with one
`uuid` (Rippling's own board: 638 entries, 326 roles); the rows merge
them, the places kept as a list and joined with "; " for `location`. The entry has no description, date
or pay: `detail` reads them. `remote` is True only when every place is a
"Remote (...)" one, since a label says nothing else about the work."""
or pay: `detail` reads them."""
data = json.loads(body)
if not isinstance(data, list):
raise ValueError("not a Rippling board: the answer is not a list")
@@ -114,19 +105,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
for job_id, row in roles.items():
row["places"] = places[job_id]
row["location"] = "; ".join(p["text"] for p in places[job_id])
row["remote"] = True if places[job_id] and all(
p["remote"] for p in places[job_id]) else None
return list(roles.values())
def _employment(kind: Any) -> str | None:
label = str((kind or {}).get("label") or "").strip().upper() if isinstance(kind, dict) else ""
for word in label.split("_"):
if word in _EMPLOYMENT:
return _EMPLOYMENT[word]
if label:
log.debug("rippling: unknown employmentType %r", label)
return None
"""The host's `employmentType.label` (`SALARIED_FT`), as given."""
return (str(kind.get("label") or "").strip() or None) if isinstance(kind, dict) else None
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
+7 -23
View File
@@ -13,15 +13,12 @@ ignores a remote parameter, so remote is not filtered.
from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok, table_word
log = logging.getLogger(__name__)
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
NAME = "smartrecruiters"
LABEL = "SmartRecruiters"
@@ -41,17 +38,6 @@ READS = Reads(
_API = "api.smartrecruiters.com"
#: `experienceLevel.id` -> the app's levels. The three in the middle name no
#: level ("associate" is half of Bosch's roles) and are known, so they map to
#: none; an id not listed is logged at debug level.
_SENIORITY = {"internship": "intern", "entry_level": "junior", "director": "director",
"executive": "exec", "associate": "", "mid_senior_level": "",
"not_applicable": ""}
#: `typeOfEmployment.id` -> the app's employment words.
_EMPLOYMENT = {"permanent": "", "intern": "internship", "temporary": "temporary",
"contract": "contract", "part_time": "part_time", "full_time": "full_time"}
#: The page size asked for, which is also the most the host serves per page.
PAGE_SIZE = 100
@@ -94,10 +80,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None:
"""The `id` of the object `job[key]` looked up in `table`; none for a word
the table maps to none, and a word it does not know is logged."""
return table_word(table, (job.get(key) or {}).get("id"), log, key)
def _word(job: dict[str, Any], key: str) -> str | None:
"""The `id` of the object `job[key]`, as given."""
return str((job.get(key) or {}).get("id") or "").strip() or None
def rows(data: dict[str, Any], slug: str) -> list[Row]:
@@ -122,7 +107,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
loc_text = ", ".join(part.strip() for part in (loc_text or "").split(",")
if part.strip())
place: Place = {"text": loc_text, "remote": location.get("remote")}
country = str(location.get("country") or "").strip().upper()
out.append({
"source_id": job_id,
"title": title,
@@ -134,8 +118,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"description": "",
"remote": location.get("remote"),
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
"country": country if len(country) == 2 else None,
"seniority": _word(j, "experienceLevel", _SENIORITY),
"employment": _word(j, "typeOfEmployment", _EMPLOYMENT),
"country": str(location.get("country") or "").strip() or None,
"seniority": _word(j, "experienceLevel"),
"employment": _word(j, "typeOfEmployment"),
})
return out
+3 -19
View File
@@ -59,7 +59,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES
from ._shared import Get, Page, Row, add_place, country_if_single, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
NAME = "successfactors"
LABEL = "SuccessFactors"
@@ -86,7 +86,7 @@ KEEP_CASE = True
READS = Reads(
scope="board", listing="paged", text="detail", complete="stated", places="many",
filters=Filters(country="one"),
fields=Fields(posted="trusted", country="stated"),
fields=Fields(posted="trusted"),
robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)",
)
@@ -342,19 +342,6 @@ def _posted(value: str) -> datetime | None:
return parse_dt(value)
def _country_code(meta: dict[str, str]) -> str:
"""The address's country as an ISO code: `addressCountry`, or the code that
`_place` reads in a `streetAddress`; '' for neither."""
code = meta.get("addressCountry", "").strip().upper()
if len(code) == 2:
return code
parts = [part.strip() for part in meta.get("streetAddress", "").split(",")]
for i in range(len(parts) - 1, 0, -1):
if len(parts) - i <= 2 and parts[i] in _COUNTRIES:
return parts[i]
return ""
def _location(meta: dict[str, str]) -> str:
"""The address as "City, Country". Vestas and Demant state
`addressLocality` and `addressCountry` (a code), the other sites one
@@ -373,7 +360,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
"""A role's own page: `description` from the block marked
`itemprop="description"`, which Novo Nordisk's pages have without the
microdata around it, and, from a page that has the `JobPosting` microdata,
`posted` (`datePosted`) and every place with the country's code. A page that
`posted` (`datePosted`) and every place. A page that
is not a role's (the site answers 200 for a closed one with its search) has
no description block and gives an empty answer."""
text = get(row["url"]).decode("utf-8", "replace")
@@ -396,9 +383,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if places:
out["places"] = places
out["location"] = "; ".join(p["text"] for p in places)
codes = {_country_code(address) for address in addresses}
if code := country_if_single(codes):
out["country"] = code
return out
+1 -7
View File
@@ -51,12 +51,6 @@ _HOST = re.compile(r"[a-z0-9-]+(\.[a-z0-9-]+)*")
#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`.
_TT = "{https://teamtailor.com/locations}"
#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing
#: about where the role is worked and `temporary` is remote for a spell only,
#: so both stay unknown rather than become "onsite" or "remote".
_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"}
def site(slug: str) -> str:
"""The host a career site is served from: `acme` is acme.teamtailor.com,
and a slug with a dot is a site on the company's own domain,
@@ -141,7 +135,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
"department": ((item.findtext(f"{_TT}department") or "").strip()
or (item.findtext(f"{_TT}role") or "").strip()),
"description": html.unescape(strip_html(item.findtext("description") or "")),
"remote": _REMOTE.get(status),
"remote": status or None,
"salary_text": "", # Teamtailor's feed states no pay at all
})
return out
+10 -21
View File
@@ -8,14 +8,11 @@ board with roles was seen; jobhunt-hq did not read Workable.
from __future__ import annotations
import json
import logging
from collections.abc import Callable
from typing import Any
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt, path_slug_ok, table_word
log = logging.getLogger(__name__)
from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok
NAME = "workable"
LABEL = "Workable"
@@ -28,17 +25,12 @@ PATTERNS = (
READS = Reads(
scope="board", listing="paged", page_size=10, text="none", complete="stated",
places="many",
fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"),
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
_API = "apply.workable.com"
#: `type` -> the app's employment words; a type not listed is logged at debug
#: level so the table can grow.
_EMPLOYMENT = {"full": "full_time", "part": "part_time", "contract": "contract",
"temporary": "temporary", "internship": "internship"}
# No probe: Workable's API sits behind a rate limit that a run of guesses
# trips, and a Workable site links its board plainly anyway.
@@ -67,13 +59,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
return rows(json.loads(body), slug)
def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
"""Every place of a role and the countries' codes among them. `locations`
def _places(job: dict[str, Any], remote: bool) -> list[Place]:
"""Every place of a role. `locations`
lists them (city, region, country; `hidden` ones are stated too), and
`location` is the first alone, the fallback for a role that lists none.
A place is remote when the role is."""
found: list[Place] = []
codes: set[str] = set()
listed = [loc for loc in job.get("locations") or [] if isinstance(loc, dict)]
for loc in listed or [job.get("location") or {}]:
parts: list[str] = []
@@ -83,9 +74,7 @@ def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
parts.append(part)
text = ", ".join(parts)
add_place(found, {"text": text, "remote": remote})
if code := str(loc.get("countryCode") or "").strip().upper():
codes.add(code)
return found, codes
return found
def rows(data: dict[str, Any], slug: str) -> list[Row]:
@@ -104,8 +93,9 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
shortcode = (j.get("shortcode") or "").strip()
if not title or not shortcode:
continue
remote = bool(j.get("remote")) or j.get("workplace") == "remote"
places, codes = _places(j, remote)
workplace = str(j.get("workplace") or "").strip()
remote = bool(j.get("remote")) or workplace == "remote"
places = _places(j, remote)
dept = j.get("department") or ""
if isinstance(dept, list):
dept = ", ".join(d for d in dept if d)
@@ -118,9 +108,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"posted": parse_dt(j.get("published")),
"department": dept.strip(),
"description": "",
"remote": remote,
"remote": workplace or j.get("remote"),
"salary_text": "", # Workable's list endpoint states no pay at all
"country": country_if_single(codes),
"employment": table_word(_EMPLOYMENT, j.get("type"), log, "type"),
"employment": str(j.get("type") or "").strip() or None,
})
return out
+18 -30
View File
@@ -29,7 +29,6 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
@@ -39,9 +38,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import code_of
from ._shared import Get, Page, Row, add_place, only, parse_dt, table_word
log = logging.getLogger(__name__)
from ._shared import Get, Page, Row, add_place, only, parse_dt
NAME = "workday"
LABEL = "Workday"
@@ -50,7 +47,7 @@ KEEP_CASE = True
READS = Reads(
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
complete="stated", places="many", filters=Filters(country="ids", remote="ids"),
fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"),
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots="read: robots.txt allows /<Site>/ (checked 2026-09-29)",
)
@@ -167,7 +164,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
"""The tenant's host ids, read off one page of one role: `country` maps
an ISO code to `<facetParameter>=<id>` in the facet whose values are
countries (most of them, `_COUNTRY_SHARE`; the one with most countries
when two are), and `remote` maps each facet value that `_REMOTE` reads as
when two are), and `remote` maps each facet value that `_REMOTE_FACET` names as
remote ("Remote", "Fully Flexible"), in whatever facet, to its id. A
tenant with neither states no such dimension, and the runner then does
not filter on it."""
@@ -184,7 +181,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
country = found
for v in values:
word = " ".join(str(v.get("descriptor") or "").split())
if v.get("id") and _remote(word) == "remote":
if v.get("id") and word.lower() in _REMOTE_FACET:
remote[word.lower()] = f"{name}={v['id']}"
out: dict[str, dict[str, str]] = {}
if country:
@@ -216,11 +213,14 @@ def _today() -> datetime:
return datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
#: `remoteType` -> `store._remote`'s words. Only some tenants send it, and
#: each words it its own way (Zendesk's "Fully Flexible", Axis' "On-Site"); a
#: word not listed, a bare "No" among them, says nothing and stays unknown.
_REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid",
"in office": "onsite", "on-site": "onsite", "onsite": "onsite"}
#: The `remoteType` words a facet names that mean remote, which the `remote`
#: filter looks for; the row keeps the host's word whatever it is (only some
#: tenants send it, each in its own way: Zendesk's "Fully Flexible", Axis'
#: "On-Site").
_REMOTE_FACET = ("remote", "fully flexible")
#: The words that put a role's places in an office, for their `remote` flag.
_OFFICE = ("hybrid", "in office", "on-site", "onsite")
def _requisition(posting: dict[str, Any], path: str) -> str:
@@ -234,12 +234,10 @@ def _requisition(posting: dict[str, Any], path: str) -> str:
def _remote(value: Any) -> str | None:
return _REMOTE.get(str(value or "").strip().lower())
"""The host's `remoteType`, as given."""
return str(value or "").strip() or None
#: `timeType` -> the app's employment words.
_EMPLOYMENT = {"full time": "full_time", "part time": "part_time"}
#: What `locationsText` says of a role at several places: "2 Locations". The
#: places arrive with the role's own read.
_COUNT = re.compile(r"\d+ locations?", re.I)
@@ -259,8 +257,9 @@ def _place(text: Any, remote: str | None) -> Place | None:
if len(parts) > 1 and "," not in text and all(parts) \
and not parts[0].lower().startswith(("remote", "hybrid", "on-site", "onsite")):
text = ", ".join(reversed(parts))
return {"text": text, "remote": {"remote": True, "hybrid": False, "onsite": False}.get(
remote or "")}
word = (remote or "").lower()
return {"text": text, "remote": True if word in _REMOTE_FACET
else False if word in _OFFICE else None}
def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
@@ -318,9 +317,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
if places := _places(posting, remote):
out["places"] = places
out["location"] = "; ".join(p["text"] for p in places)
if len(places) == 1 and (code := _country_code(posting)):
out["country"] = code
if employment := table_word(_EMPLOYMENT, posting.get("timeType"), log, "timeType"):
if employment := str(posting.get("timeType") or "").strip():
out["employment"] = employment
if posted := parse_dt(posting.get("startDate")):
out["posted"] = posted
@@ -329,15 +326,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
return out
def _country_code(posting: dict[str, Any]) -> str | None:
"""The role's country as an ISO code, from the requisition's location. The
code belongs to the main location only, so it is used for a role with one
place."""
country = (posting.get("jobRequisitionLocation") or {}).get("country") or {}
code = str(country.get("alpha2Code") or "").strip().upper()
return code if len(code) == 2 else None
def _places(posting: dict[str, Any], remote: str | None) -> list[Place]:
"""The main location and the additional ones, each once, which replace the
listing's "2 Locations". A single location that does not name its country