Merge the prep stage with the readers that hand on host words (wp-p-prep, wp-r1-raw, wp-r2-raw)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
3917b31209
117 files changed
+2452
-996
No files matched your search
+31
-1
@@ -8,6 +8,7 @@
|
||||
app agents list|prompt
|
||||
app stages [extract|summarise on|off]
|
||||
app dupes
|
||||
app prep [unmapped] [--remap]
|
||||
app db init
|
||||
|
||||
`app job` queues the work on a queue of its own and drains it here, so a
|
||||
@@ -160,6 +161,13 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
||||
|
||||
prep = sub.add_parser(
|
||||
"prep", help="map the roles' host words and fold the copies, or list the unmapped words")
|
||||
prep.add_argument("action", nargs="?", choices=["unmapped"],
|
||||
help="list the host words no table maps, instead of running")
|
||||
prep.add_argument("--remap", action="store_true",
|
||||
help="map every role again, closed ones included")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
return parser
|
||||
@@ -188,6 +196,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return run_seniority(args)
|
||||
if args.command == "dupes":
|
||||
return run_dupes(args)
|
||||
if args.command == "prep":
|
||||
return run_prep(args)
|
||||
return init_db(args)
|
||||
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
@@ -655,7 +665,7 @@ def run_seniority(args: argparse.Namespace) -> int:
|
||||
|
||||
|
||||
def run_dupes(args: argparse.Namespace) -> int:
|
||||
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
|
||||
"""`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`,
|
||||
now. Prints the groups (kept roles with a copy), the copies the feed
|
||||
hides and the rows whose `dup_of` changed, as JSON."""
|
||||
from app.storage import Store
|
||||
@@ -666,6 +676,26 @@ def run_dupes(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def run_prep(args: argparse.Namespace) -> int:
|
||||
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
|
||||
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
|
||||
host words no table maps, per field and host, most roles first."""
|
||||
from app import prep
|
||||
from app.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
if args.action == "unmapped":
|
||||
found = store.unmapped()
|
||||
if not found:
|
||||
print("every host word is mapped")
|
||||
for w in found:
|
||||
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
|
||||
return 0
|
||||
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
|
||||
return 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from app.storage import LATEST, Store
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||
|
||||
pull.py read every board and search and store the roles they yield
|
||||
prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold
|
||||
extract.py title and description into the gate columns, on Haiku
|
||||
embed.py one vector per open role, for fit
|
||||
score.py every open role scored for every enabled search agent
|
||||
summarise.py about 60 words on each of the best roles, on Haiku
|
||||
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
|
||||
cascade.py pull → prep → embed → score → ... → summarise, what `TASKS`
|
||||
offers, the `pull` job a schedule fires, and that schedule
|
||||
stages.py whether extract and summarise run: the switch, else config.yaml
|
||||
sources.py what a pull reads, changed by hand: boards and searches
|
||||
|
||||
+19
-13
@@ -1,10 +1,13 @@
|
||||
"""The tasks roledesk offers, and the order a pull runs them in.
|
||||
|
||||
pull → embed → score → enrich → embed → score → extract → score → summarise
|
||||
pull → prep → embed → score → enrich → prep → embed → score → extract → score
|
||||
→ summarise
|
||||
|
||||
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
||||
final job of one task starts the next (`step` is the callback it carries).
|
||||
Nothing is alive between two tasks. A stage is addressed by its position in
|
||||
Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what
|
||||
the pull stored and folds the copies before anything reads the roles; after
|
||||
`enrich` it maps the roles their own pages filled. A stage is addressed by its position in
|
||||
the tuple, not its name, since `score` runs twice.
|
||||
|
||||
Every stage is always in the chain, and a stage that is switched off finds
|
||||
@@ -12,8 +15,9 @@ nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull d
|
||||
satisfy, is an extract stage with no batches whose final job moves straight
|
||||
on. The free stages go first: embed and score cost no tokens, and the score
|
||||
they produce is what bounds the rest. Enrich reads the best roles without
|
||||
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
|
||||
the roles it filled; when it filled none, `step` passes over both. Extract
|
||||
text on their own pages (`app/jobs/enrich.py`), and prep, embed and score
|
||||
follow for the roles it filled; when it filled none, `step` passes over all
|
||||
three. Extract
|
||||
reads only the best
|
||||
roles of each enabled agent, as deep and as low as that agent's own `limits`
|
||||
say; the second score then applies the gates to the columns it filled.
|
||||
@@ -27,7 +31,7 @@ having either way.
|
||||
and a cascade is started by one: the `pull` job's only work is to ask for the
|
||||
chain. Its default schedule, `every 6h`, is declared here beside it.
|
||||
|
||||
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
|
||||
`enrich` is a job and a cascade the same way, `enrich → prep → embed → score`, queued
|
||||
behind the `score` that follows a change to the ranking other than a pull's
|
||||
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
|
||||
one agent's taste a little and queues neither.
|
||||
@@ -40,20 +44,20 @@ from dataclasses import replace
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||
from app.jobs import embed, enrich, extract, prep, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||
from desk.tools import agent
|
||||
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
||||
|
||||
STEP = "app.jobs.cascade:step"
|
||||
|
||||
CASCADES: dict[str, tuple[str, ...]] = {
|
||||
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
|
||||
"summarise"),
|
||||
"enrich": ("enrich", "embed", "score"),
|
||||
"pull": ("pull", "prep", "embed", "score", "enrich", "prep", "embed", "score", "extract",
|
||||
"score", "summarise"),
|
||||
"enrich": ("enrich", "prep", "embed", "score"),
|
||||
}
|
||||
|
||||
#: What follows `enrich` for the roles it filled, passed over when it filled none.
|
||||
REFILL = ("embed", "score")
|
||||
REFILL = ("prep", "embed", "score")
|
||||
|
||||
|
||||
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
||||
@@ -103,6 +107,8 @@ STAGES: dict[str, Callable[[str], Task]] = {
|
||||
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
|
||||
"score": lambda cascade: Task(kind="score", finder=_once,
|
||||
label="Scoring roles for every agent"),
|
||||
"prep": lambda cascade: Task(kind="prep", finder=_once,
|
||||
label="Mapping the roles' host words"),
|
||||
"summarise": _summarise,
|
||||
"enrich": lambda cascade: enrich.enrich_task(),
|
||||
}
|
||||
@@ -189,7 +195,7 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
|
||||
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
|
||||
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
|
||||
"""Ask for the enrich cascade, `enrich → prep → embed → score`. Its own work is
|
||||
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
|
||||
if not context.task(chain("enrich")):
|
||||
raise RuntimeError("no queue to start the enrich cascade on")
|
||||
@@ -210,7 +216,7 @@ def enrich_after(queue: Any, score_job: Any) -> Any:
|
||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||
#: is both would be listed twice by `/api/jobs/kinds`.
|
||||
for _name in ("extract", "embed", "score", "summarise"):
|
||||
for _name in ("extract", "embed", "score", "summarise", "prep"):
|
||||
TASKS[_name] = partial(task, _name)
|
||||
|
||||
|
||||
@@ -220,5 +226,5 @@ for _name in ("extract", "embed", "score", "summarise"):
|
||||
# setting: a cadence edited or switched off under Runs survives
|
||||
# every restart (see pipeline/schedule.py).
|
||||
declare("pull", "pull", "every 6h",
|
||||
label="Pull every board, then embed, score, read the best roles' pages, "
|
||||
label="Pull every board, then map, embed, score, read the best roles' pages, "
|
||||
"extract, rescore and summarise")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""`prep`: the stage that maps what a pull stored, `app.prep.run`.
|
||||
|
||||
In the pull cascade it follows the pull, whose settle has closed what is
|
||||
gone, and maps every role the pull changed, then folds the copies across
|
||||
the whole database (`Store.rebuild_dupes`, which the pull's settle used to
|
||||
run). After `enrich` it maps the roles their own pages filled, since only
|
||||
those are due. `solo`, so two cascades never write the same rows at once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app import prep
|
||||
from app.config import Config
|
||||
from app.storage import Store
|
||||
from pipeline import NULL, RunContext, job
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@job("prep", "Mapping the roles' host words", solo=True, resource="cpu")
|
||||
def run_prep(config: Config, store: Store, *, remap: bool = False,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Map the due roles, every open one when the tables changed, and fold
|
||||
the copies. `remap` maps every role, closed ones included."""
|
||||
done = prep.run(store, remap=remap)
|
||||
log.info("prep: %d read, %d written, %d words unmapped, %s copies%s", done.read,
|
||||
done.written, done.unmapped, done.copies, ", tables changed" if done.remapped else "")
|
||||
return done._asdict()
|
||||
+4
-6
@@ -244,10 +244,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
|
||||
After the closes, `Store.rebuild_dupes` recomputes which open roles are
|
||||
copies of one posting, so a closed kept role hands its group on and the
|
||||
roles this run added join theirs; `copies` is how many open roles it
|
||||
leaves hidden as copies.
|
||||
The `prep` stage that follows maps what this run stored and folds the
|
||||
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
||||
and the roles this run added join theirs.
|
||||
"""
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
@@ -284,10 +283,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
release_run(store, run_marker)
|
||||
copies = store.rebuild_dupes().copies
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
||||
|
||||
@@ -649,6 +649,13 @@ SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
||||
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
|
||||
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
|
||||
|
||||
#: The kind of contract a posting states, as `app.prep` maps a host's word.
|
||||
#: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the
|
||||
#: readers' own tests, since a plugin imports nothing from `app`.
|
||||
Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship",
|
||||
"freelance"]
|
||||
EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e)
|
||||
|
||||
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
||||
Permit = Literal["", "sponsors", "no_sponsorship"]
|
||||
PERMITS: tuple[str, ...] = get_args(Permit)
|
||||
@@ -701,6 +708,16 @@ def trusted_posted(host: str) -> bool:
|
||||
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
||||
|
||||
|
||||
def host_word(value: Any) -> str:
|
||||
"""A host's word as `roles.*_host` stores it: stripped text, a bool as
|
||||
'true' or 'false', '' for nothing. What `app.prep` maps."""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def posted_host(raw: dict[str, Any]) -> str:
|
||||
"""The host a role's dates came from, read the way `ats` and `search`
|
||||
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
|
||||
@@ -797,6 +814,8 @@ class Role(Model):
|
||||
seniority_model: str = ""
|
||||
seniority_stated: str = ""
|
||||
seniority_title: str = ""
|
||||
#: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`.
|
||||
employment: str = ""
|
||||
function: str = ""
|
||||
permit: Permit = ""
|
||||
#: ISO 639-1 codes the posting says are required, lowercase.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Prep: every decision about a role that needs a rule or data across roles.
|
||||
|
||||
Readers scrape and hand on the host's own words (`Store.upsert_role` keeps
|
||||
them in `roles.*_host`); prep maps them, once, centrally, between the pull
|
||||
and embedding. One door: `from app.prep import ...`.
|
||||
|
||||
words.py seniority, employment and remote: the host's table, the
|
||||
vocabulary itself, the shared remote words
|
||||
country.py the country from the resolved places, else the host's
|
||||
hosts/ one table file per reader (`hosts.TABLES`)
|
||||
stage.py `run`: the stage over the due roles, `unmapped` and the fold
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.prep.country import country
|
||||
from app.prep.hosts import TABLES
|
||||
from app.prep.stage import STATE, Prepped, company, fingerprint, run
|
||||
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
|
||||
|
||||
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
|
||||
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
|
||||
@@ -0,0 +1,55 @@
|
||||
"""A role's country, from its places, else from the one its host states."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from app import places
|
||||
|
||||
|
||||
def of_host(gaz: places.Gazetteer, stated: str) -> str | None:
|
||||
"""The ISO code a host's country names, in the host's spelling: a code
|
||||
("DK", "dk"), or a name the gazetteer has as an alias of a country
|
||||
("Denmark", "Danmark"), `EU` or Europe by name, or a text that resolves to places of one
|
||||
country. '' for nothing stated, None for a text naming no country."""
|
||||
text = (stated or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
if ("country", text.upper()) in gaz.by_key or text.upper() == "EU":
|
||||
return text.upper()
|
||||
named = {"EU" if gaz.places[p].key == "europe" else gaz.places[p].key
|
||||
for p in gaz.aliases.get(places.fold(text), ())
|
||||
if gaz.places[p].kind == "country"
|
||||
or (gaz.places[p].kind, gaz.places[p].key) == ("continent", "europe")}
|
||||
if len(named) == 1:
|
||||
return named.pop()
|
||||
found = {gaz.country(p) for p in gaz.resolve(text)} - {""}
|
||||
return found.pop() if len(found) == 1 else None
|
||||
|
||||
|
||||
def _in_europe(gaz: places.Gazetteer, code: str) -> bool:
|
||||
if code == "EU":
|
||||
return True
|
||||
europe = gaz.by_key.get(("continent", "europe"))
|
||||
place_id = gaz.by_key.get(("country", code))
|
||||
return europe is not None and place_id is not None and europe in gaz.ancestors(place_id)
|
||||
|
||||
|
||||
def country(gaz: places.Gazetteer, place_ids: Iterable[int], host_country: str = "") -> str:
|
||||
"""The one country the role's resolved places lie in, continents aside;
|
||||
Europe alone is `EU`; else, when they
|
||||
lie in several all in Europe, `EU` (what `country_of` wrote for "Dublin,
|
||||
London" and the country gate reads as Europe); else the host's stated
|
||||
country (`of_host`); else ''."""
|
||||
ids = list(place_ids)
|
||||
# A continent beside a country is the wider unit a line adds ("Home
|
||||
# based - EMEA; London" is GB), as `country_of` had it; alone, Europe
|
||||
# is `EU`.
|
||||
found = {gaz.country(p) for p in ids} - {""}
|
||||
if not found and any(gaz.places[p].key == "europe" for p in ids if p in gaz.places):
|
||||
found = {"EU"}
|
||||
if len(found) == 1:
|
||||
return found.pop()
|
||||
if len(found) > 1 and all(_in_europe(gaz, code) for code in found):
|
||||
return "EU"
|
||||
return of_host(gaz, host_country) or ""
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Each host's own words, one module per reader, named as the reader is.
|
||||
|
||||
A module defines any of three dicts, keyed by the host's word lowercased:
|
||||
|
||||
- `SENIORITY`, values in `app.models.SENIORITIES`
|
||||
- `EMPLOYMENT`, values in `app.models.EMPLOYMENTS`
|
||||
- `REMOTE`, values in `app.models.REMOTE` (remote, hybrid, onsite)
|
||||
|
||||
A value of "" is a word the host uses that names no value ("Experienced",
|
||||
"Permanent"): known, so it is not listed as unmapped.
|
||||
|
||||
A host whose employment word is a code of several parts (`permanent_full_time`,
|
||||
`SALARIED_FT`) may add `MATCH`, `[(fragment, value), ...]`, tried in order
|
||||
after `EMPLOYMENT` with the first hit winning, and must then say how a
|
||||
fragment matches in `MATCH_MODE`: "substring" (the fragment is in the
|
||||
lowercased word) or "token" (it equals one of the word's parts split at `_`
|
||||
and `-`, so `ft` does not match inside `software`). `TABLES` loads every
|
||||
module of this package by name, so a table is added by adding its file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
from typing import Any
|
||||
|
||||
#: The dict a module names, by the field it maps.
|
||||
FIELDS = {"seniority": "SENIORITY", "employment": "EMPLOYMENT", "remote": "REMOTE"}
|
||||
|
||||
#: How a `MATCH` fragment may match.
|
||||
MATCH_MODES = ("substring", "token")
|
||||
|
||||
|
||||
def load() -> dict[str, dict[str, Any]]:
|
||||
"""`host -> field -> {word: value}` for every module of the package, the
|
||||
words lowercased and stripped. A host without a table for a field has
|
||||
no entry for it. A `MATCH` is `host -> "match" -> {"mode": ..., "rules":
|
||||
[[fragment, value], ...]}`; a module with `MATCH` and no valid
|
||||
`MATCH_MODE` fails the import by name."""
|
||||
tables: dict[str, dict[str, Any]] = {}
|
||||
for info in sorted(pkgutil.iter_modules(__path__), key=lambda i: i.name):
|
||||
if info.name.startswith("_"):
|
||||
continue
|
||||
module = importlib.import_module(f"{__name__}.{info.name}")
|
||||
for field, name in FIELDS.items():
|
||||
table = getattr(module, name, None)
|
||||
if isinstance(table, dict):
|
||||
tables.setdefault(info.name, {})[field] = {
|
||||
str(k).strip().lower(): str(v) for k, v in table.items()}
|
||||
rules = getattr(module, "MATCH", None)
|
||||
if rules is not None:
|
||||
mode = getattr(module, "MATCH_MODE", None)
|
||||
if mode not in MATCH_MODES:
|
||||
raise ValueError(f"app/prep/hosts/{info.name}.py has MATCH and its MATCH_MODE "
|
||||
f"is {mode!r}, not one of {', '.join(MATCH_MODES)}")
|
||||
tables.setdefault(info.name, {})["match"] = {
|
||||
"mode": mode, "rules": [[str(f).lower(), str(v)] for f, v in rules]}
|
||||
return tables
|
||||
|
||||
|
||||
TABLES = load()
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Ashby's words, mapped to the app's."""
|
||||
|
||||
#: `workplaceType`.
|
||||
REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"}
|
||||
|
||||
#: `employmentType`.
|
||||
EMPLOYMENT: dict[str, str] = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship",
|
||||
"contract": "contract", "temporary": "temporary"}
|
||||
@@ -0,0 +1,12 @@
|
||||
"""BambooHR's words, mapped to the app's."""
|
||||
|
||||
#: `employmentStatusLabel`. BambooHR's labels are the company's own ("Full-Time",
|
||||
#: "Part-Time", "Contract", "Intern").
|
||||
EMPLOYMENT: dict[str, str] = {
|
||||
"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "contractor": "contract",
|
||||
"temporary": "temporary", "intern": "internship", "internship": "internship",
|
||||
"freelance": "freelance"}
|
||||
|
||||
#: `locationType`. Anything else says nothing.
|
||||
REMOTE: dict[str, str] = {"0": "onsite", "1": "remote", "2": "hybrid"}
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Deel's words, mapped to the app's."""
|
||||
|
||||
#: `workArrangementEnum`.
|
||||
REMOTE: dict[str, str] = {"on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
|
||||
|
||||
#: The name of a `jobEmploymentTypes` entry.
|
||||
EMPLOYMENT: dict[str, str] = {
|
||||
"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "contractor": "contract",
|
||||
"consultant": "contract", "temporary": "temporary", "intern": "internship",
|
||||
"internship": "internship", "freelance": "freelance"}
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Eightfold's words, mapped to the app's."""
|
||||
|
||||
#: `workLocationOption`. Only "onsite" and "hybrid" were seen; the other
|
||||
#: spellings are the obvious ones.
|
||||
REMOTE: dict[str, str] = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Gem's words, mapped to the app's."""
|
||||
|
||||
#: `job.locationType`.
|
||||
REMOTE: dict[str, str] = {"in_office": "onsite", "hybrid": "hybrid", "remote": "remote"}
|
||||
|
||||
#: `job.employmentType`.
|
||||
EMPLOYMENT: dict[str, str] = {
|
||||
"full_time": "full_time", "part_time": "part_time", "contract": "contract",
|
||||
"contractor": "contract", "intern": "internship", "internship": "internship",
|
||||
"temporary": "temporary", "freelance": "freelance"}
|
||||
@@ -0,0 +1,13 @@
|
||||
"""HiBob's words, mapped to the app's."""
|
||||
|
||||
#: `employmentType`, the company's own name for it. "Permanent" says nothing
|
||||
#: about hours and stays unmapped.
|
||||
EMPLOYMENT: dict[str, str] = {
|
||||
"regular full time": "full_time", "full time": "full_time",
|
||||
"full-time": "full_time", "regular part time": "part_time",
|
||||
"part time": "part_time", "part-time": "part_time", "contract": "contract",
|
||||
"contractor": "contract", "temporary": "temporary", "fixed term": "temporary",
|
||||
"intern": "internship", "internship": "internship", "freelance": "freelance"}
|
||||
|
||||
#: `workspaceTypeId`.
|
||||
REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite", "onsite": "onsite"}
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Jibe's words, mapped to the app's."""
|
||||
|
||||
#: `employment_type`.
|
||||
EMPLOYMENT: dict[str, str] = {
|
||||
"full_time": "full_time", "part_time": "part_time", "contractor": "contract",
|
||||
"contract": "contract", "temporary": "temporary", "intern": "internship",
|
||||
"internship": "internship", "freelance": "freelance"}
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Jobylon's words, mapped to the app's."""
|
||||
|
||||
#: The embed's `experience`. Only the coarse levels are mapped; "Experienced"
|
||||
#: sits between "Mid level" and the managerial levels and says too little to
|
||||
#: place, so it stays unmapped.
|
||||
SENIORITY: dict[str, str] = {
|
||||
"student": "intern", "internship": "intern", "trainee": "intern",
|
||||
"entry level": "junior", "entry-level": "junior", "junior": "junior",
|
||||
"graduate": "junior", "mid level": "mid", "mid-level": "mid", "mid": "mid",
|
||||
"senior": "senior", "lead": "lead", "manager": "manager", "director": "director",
|
||||
"executive": "exec", "not applicable": ""}
|
||||
|
||||
#: The embed's `employment_type`.
|
||||
EMPLOYMENT: dict[str, str] = {
|
||||
"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "consultant": "contract",
|
||||
"temporary": "temporary", "internship": "internship", "intern": "internship",
|
||||
"freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"}
|
||||
|
||||
#: `workspace`.
|
||||
REMOTE: dict[str, str] = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Lever's words, mapped to the app's."""
|
||||
|
||||
#: `categories.commitment`. A commitment is free text on Lever ("Permanent" is
|
||||
#: seen too and says nothing about hours).
|
||||
EMPLOYMENT: dict[str, str] = {
|
||||
"full-time": "full_time", "full time": "full_time", "fulltime": "full_time",
|
||||
"part-time": "part_time", "part time": "part_time", "parttime": "part_time",
|
||||
"intern": "internship", "internship": "internship", "contract": "contract",
|
||||
"contractor": "contract", "temporary": "temporary", "freelance": "freelance"}
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Oracle Recruiting Cloud's words, for prep."""
|
||||
|
||||
#: `WorkplaceTypeCode` -> `REMOTE`'s words. A role that states none has an
|
||||
#: empty code, which stays unknown.
|
||||
REMOTE = {"ora_remote": "remote", "ora_hybrid": "hybrid", "ora_on_site": "onsite"}
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Personio's words, for prep."""
|
||||
|
||||
#: `seniority`. `experienced` is what most roles say and names no level.
|
||||
SENIORITY = {"entry-level": "junior", "student": "intern", "executive": "exec",
|
||||
"experienced": ""}
|
||||
|
||||
#: `employment` is the feed's `employmentType`, or, for a `permanent` one
|
||||
#: (which says nothing about the hours), its `schedule`.
|
||||
EMPLOYMENT = {"temporary": "temporary", "freelance": "contract", "intern": "internship",
|
||||
"working_student": "internship", "trainee": "internship",
|
||||
"permanent": "", "full-time": "full_time", "part-time": "part_time"}
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Phenom's words, for prep."""
|
||||
|
||||
#: The words a page's `checkRemote` says (King's pages say "On-site"; no
|
||||
#: other was seen).
|
||||
REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Pinpoint's words, for prep."""
|
||||
|
||||
#: `workplace_type`. Anything else says nothing.
|
||||
REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
|
||||
|
||||
#: What `employment_type` (`permanent_full_time`, `fixed_term_full_time`)
|
||||
#: says, matched by fragment: the first fragment found in the lowercased
|
||||
#: word decides, so a contract type beats the hours. The fixtures hold two
|
||||
#: whole words only, so this is not turned into a whole-word table.
|
||||
MATCH = [("intern", "internship"), ("freelance", "contract"), ("contract", "contract"),
|
||||
("fixed_term", "temporary"), ("temporary", "temporary"),
|
||||
("part_time", "part_time"), ("full_time", "full_time")]
|
||||
MATCH_MODE = "substring"
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Platsbanken's words, for prep."""
|
||||
|
||||
#: `workplace_model.label`, in Swedish; mostly empty. Only "Arbete på plats"
|
||||
#: is in the fixtures; the others are JobTech's other labels as the site
|
||||
#: writes them, and a label not here is listed as unmapped.
|
||||
REMOTE: dict[str, str] = {
|
||||
"arbete på plats": "onsite", "på plats": "onsite",
|
||||
"hybridarbete": "hybrid", "hybrid": "hybrid", "delvis på distans": "hybrid",
|
||||
"delvis distans": "hybrid",
|
||||
"distansarbete": "remote", "arbete på distans": "remote", "helt på distans": "remote",
|
||||
"distans": "remote",
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Recruitee's words, for prep."""
|
||||
|
||||
#: `experience_code`. `experienced` names no level and is known, so it maps
|
||||
#: to none.
|
||||
SENIORITY = {"student_school": "intern", "entry_level": "junior", "mid_level": "mid",
|
||||
"experienced": ""}
|
||||
|
||||
#: The words of `employment_type_code` (`fulltime_permanent`), which is
|
||||
#: lowercased and split at "_" and "-"; the first entry found among its words
|
||||
#: decides, so a contract type beats the hours. The fixtures hold three whole
|
||||
#: codes only, so this is a table of words, not whole codes.
|
||||
MATCH = [("internship", "internship"), ("intern", "internship"),
|
||||
("freelance", "contract"), ("contract", "contract"),
|
||||
("temporary", "temporary"), ("fixed", "temporary"),
|
||||
("parttime", "part_time"), ("fulltime", "full_time")]
|
||||
MATCH_MODE = "token"
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Rippling's words, for prep."""
|
||||
|
||||
#: The words of `employmentType.label` (`SALARIED_FT`), which is split at
|
||||
#: "_" and the first token found here decides. The fixtures hold one whole
|
||||
#: label only, so this is a table of tokens (lowercased), not whole words.
|
||||
MATCH = [("ft", "full_time"), ("pt", "part_time"), ("intern", "internship"),
|
||||
("internship", "internship"), ("contract", "contract"), ("contractor", "contract"),
|
||||
("temp", "temporary"), ("temporary", "temporary"), ("seasonal", "temporary")]
|
||||
MATCH_MODE = "token"
|
||||
@@ -0,0 +1,11 @@
|
||||
"""SmartRecruiters' words, for prep."""
|
||||
|
||||
#: `experienceLevel.id`. The three in the middle name no level ("associate" is
|
||||
#: half of Bosch's roles) and are known, so they map to none.
|
||||
SENIORITY = {"internship": "intern", "entry_level": "junior", "director": "director",
|
||||
"executive": "exec", "associate": "", "mid_senior_level": "",
|
||||
"not_applicable": ""}
|
||||
|
||||
#: `typeOfEmployment.id`.
|
||||
EMPLOYMENT = {"permanent": "", "intern": "internship", "temporary": "temporary",
|
||||
"contract": "contract", "part_time": "part_time", "full_time": "full_time"}
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Teamtailor's words, for prep."""
|
||||
|
||||
#: `remoteStatus`. `none` says nothing about where the role is worked and
|
||||
#: `temporary` is remote for a spell only, so both are known and stay
|
||||
#: unknown rather than become "onsite" or "remote".
|
||||
REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite",
|
||||
"none": "", "temporary": ""}
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Workable's words, for prep."""
|
||||
|
||||
#: `workplace`.
|
||||
REMOTE = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite"}
|
||||
|
||||
#: `type`.
|
||||
EMPLOYMENT = {"full": "full_time", "part": "part_time", "contract": "contract",
|
||||
"temporary": "temporary", "internship": "internship"}
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Workday's words, for prep."""
|
||||
|
||||
#: `remoteType`. Only some tenants send it, and each words it its own way
|
||||
#: (Zendesk's "Fully Flexible", Axis' "On-Site"); a word not listed, a bare
|
||||
#: "No" among them, says nothing and stays unknown.
|
||||
REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid",
|
||||
"in office": "onsite", "on-site": "onsite", "onsite": "onsite"}
|
||||
|
||||
#: `timeType`.
|
||||
EMPLOYMENT = {"full time": "full_time", "part time": "part_time"}
|
||||
@@ -0,0 +1,140 @@
|
||||
"""The prep stage: map what the readers handed on, then fold the copies.
|
||||
|
||||
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
||||
and marks the role due. `run` maps every due role: its company through
|
||||
`company_aliases`, its places against the gazetteer, its level, contract
|
||||
and remote through the host's tables (`app.prep.words`), and its country
|
||||
from the places (`app.prep.country`). When the tables changed since the
|
||||
last run (`fingerprint`, kept in state as `prep:tables`), every open role
|
||||
is mapped again from its stored words, with no new pull. Then the words no
|
||||
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
|
||||
copies across the whole database.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from app import places
|
||||
from app.models import company_key
|
||||
from app.models import company_fold
|
||||
from app.prep import words
|
||||
from app.prep.country import country, of_host
|
||||
from app.prep.hosts import TABLES
|
||||
from app.prep.words import Tables
|
||||
from app.storage import Store
|
||||
|
||||
#: Raised when the rules in this package change the way a stored word maps,
|
||||
#: so every open role is mapped again as it is after a table changes.
|
||||
VERSION = 1
|
||||
|
||||
#: The state key the last run's `fingerprint` is kept under.
|
||||
STATE = "prep:tables"
|
||||
|
||||
|
||||
|
||||
class Prepped(NamedTuple):
|
||||
"""What one run did: roles read, rows written, whether every role was
|
||||
mapped again, words left unmapped, and the open copies after the fold
|
||||
(None when it was not run)."""
|
||||
|
||||
read: int
|
||||
written: int
|
||||
remapped: bool
|
||||
unmapped: int
|
||||
copies: int | None
|
||||
|
||||
|
||||
def fingerprint(tables: Tables = TABLES) -> str:
|
||||
"""A hash of every host table, the shared remote words and `VERSION`."""
|
||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
|
||||
sort_keys=True)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()
|
||||
|
||||
|
||||
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
|
||||
"""The key a role from `name` belongs to, as `Store.resolve_company`
|
||||
has it: the company an alias names, the name's own key first when
|
||||
several share the fold, else `company_key(name)`."""
|
||||
own = company_key(name)
|
||||
keys = folds.get(company_fold(name), ())
|
||||
return own if own in keys or not keys else keys[0]
|
||||
|
||||
|
||||
def _places(raw: Any) -> list[Any]:
|
||||
try:
|
||||
found = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except ValueError:
|
||||
return []
|
||||
return found if isinstance(found, list) else []
|
||||
|
||||
|
||||
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
||||
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
||||
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
||||
host = str(row["host"] or "")
|
||||
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
||||
return {
|
||||
"uid": row["uid"],
|
||||
"due": row["prep_due"],
|
||||
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
|
||||
"seniority_stated": stated,
|
||||
"seniority_title": titled,
|
||||
"employment": words.employment(host, row["employment_host"], tables),
|
||||
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
|
||||
"remote_places": words.remote(
|
||||
host, None, _places(row["raw_places"]), row["place"], tables,
|
||||
title=row["title"] if row["source"] == "search" else ""),
|
||||
"country": country(gaz, place_ids, row["country_host"]),
|
||||
}
|
||||
|
||||
|
||||
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||
or (found["remote"] and found["remote"] != row["remote"]) \
|
||||
or (found["remote_places"] and not row["remote"]) \
|
||||
or (found["country"] and found["country"] != row["country"]) \
|
||||
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
||||
"seniority_title", "employment"))
|
||||
|
||||
|
||||
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
|
||||
tables: Tables = TABLES) -> bool:
|
||||
"""Whether no table, vocabulary or gazetteer knows this host word."""
|
||||
if field == "country":
|
||||
return of_host(gaz, word) is None
|
||||
if field == "remote" and word in ("true", "false"):
|
||||
return False
|
||||
return words.lookup(field, host, word, tables) is None
|
||||
|
||||
|
||||
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
||||
dupes: bool = True) -> Prepped:
|
||||
"""Map the due roles, or every open one when the tables changed, or
|
||||
every role with `remap`; list the unmapped words; fold the copies."""
|
||||
mark = fingerprint(tables)
|
||||
whole = remap or store.state.get(STATE) != mark
|
||||
rows = store.prep_rows("due")
|
||||
if whole:
|
||||
due = {r["uid"] for r in rows}
|
||||
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
|
||||
ids = store.resolve_role_places([r["uid"] for r in rows])
|
||||
gaz = store.gazetteer()
|
||||
folds = store.company_folds()
|
||||
written = []
|
||||
for row in rows:
|
||||
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
|
||||
if _changed(row, found):
|
||||
written.append(found)
|
||||
store.write_prep(written)
|
||||
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
|
||||
store.replace_unmapped(left)
|
||||
if whole:
|
||||
store.state[STATE] = mark
|
||||
copies = store.rebuild_dupes().copies if dupes else None
|
||||
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
||||
copies=copies)
|
||||
@@ -0,0 +1,131 @@
|
||||
"""A host's word to the app's vocabulary: seniority, employment and remote.
|
||||
|
||||
Each field is looked up the same way: the host's table in `hosts`, keyed by
|
||||
the word lowercased, then a word that already is a value of the vocabulary
|
||||
(what readers stored before they handed on the host's words, and what a
|
||||
host with the app's own words sends), then, for remote, the shared word map.
|
||||
An employment word may also match a host's `MATCH` fragments. A word none
|
||||
of these knows is unmapped: it maps to '' and `lookup` says so,
|
||||
so the prep stage can list it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of
|
||||
from app.prep.hosts import TABLES
|
||||
from app.seniority import from_title
|
||||
|
||||
#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`).
|
||||
Tables = Mapping[str, Mapping[str, Any]]
|
||||
|
||||
VOCABULARIES: dict[str, tuple[str, ...]] = {
|
||||
"seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE}
|
||||
|
||||
#: How hosts without a table of their own write remote, after `_squash`:
|
||||
#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office").
|
||||
REMOTE_WORDS: dict[str, str] = {
|
||||
"remote": "remote", "fullyremote": "remote", "remotefirst": "remote",
|
||||
"hybrid": "hybrid",
|
||||
"onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite",
|
||||
"officebased": "onsite", "onpremise": "onsite",
|
||||
}
|
||||
|
||||
|
||||
def _match(rules: Mapping[str, Any], key: str) -> str | None:
|
||||
"""The value of the first `MATCH` fragment `key` holds, by its mode."""
|
||||
parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set()
|
||||
for fragment, value in rules["rules"]:
|
||||
if fragment in parts if rules["mode"] == "token" else fragment in key:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
#: Remote and hybrid in a line of Danish, Swedish or English that `remote_of`
|
||||
#: does not read: "Nordhavn og mulighed for hjemmearbejde", "Arbete på distans".
|
||||
#: Remote first, since "fuldt hjemmearbejde" is remote.
|
||||
_REMOTE_LINE = re.compile(
|
||||
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
|
||||
_HYBRID_LINE = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
|
||||
|
||||
def _line_word(text: str) -> str:
|
||||
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
||||
if found := remote_of(text):
|
||||
return found
|
||||
if _REMOTE_LINE.search(text):
|
||||
return "remote"
|
||||
return "hybrid" if _HYBRID_LINE.search(text) else ""
|
||||
|
||||
|
||||
def _squash(word: str) -> str:
|
||||
return "".join(c for c in word.lower() if c not in " -_")
|
||||
|
||||
|
||||
def lookup(field: str, host: str, word: str,
|
||||
tables: Tables = TABLES) -> str | None:
|
||||
"""The value `word` maps to for `host`, '' for a known word that names
|
||||
none, None for a word nothing knows. An empty word is ''."""
|
||||
key = (word or "").strip().lower()
|
||||
if not key:
|
||||
return ""
|
||||
table = tables.get(host, {})
|
||||
found = table.get(field, {}).get(key)
|
||||
if found is None and field == "employment" and "match" in table:
|
||||
found = _match(table["match"], key)
|
||||
if found is not None:
|
||||
return found
|
||||
if key in VOCABULARIES[field]:
|
||||
return key
|
||||
if field == "remote":
|
||||
return REMOTE_WORDS.get(_squash(key))
|
||||
return None
|
||||
|
||||
|
||||
def seniority(host: str, word: str, title: str,
|
||||
tables: Tables = TABLES) -> tuple[str, str]:
|
||||
"""`(stated, from the title)`: the host's word through its table, and
|
||||
`seniority.from_title`. `roles.seniority` is the model's level, else the
|
||||
stated one, else the title's (`Store.write_prep`)."""
|
||||
return lookup("seniority", host, word, tables) or "", from_title(title)
|
||||
|
||||
|
||||
def employment(host: str, word: str,
|
||||
tables: Tables = TABLES) -> str:
|
||||
return lookup("employment", host, word, tables) or ""
|
||||
|
||||
|
||||
def _place_text(place: Any) -> str:
|
||||
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
|
||||
|
||||
|
||||
def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
|
||||
tables: Tables = TABLES, title: str = "") -> str:
|
||||
"""How the role is worked, one of `REMOTE` or ''.
|
||||
|
||||
`value` is the host's own (a bool, or its word; `host_word`'s 'true' and
|
||||
'false' are bools). True is remote. False says nothing more than the
|
||||
places do: SmartRecruiters, Ashby, Recruitee and Workable send it for
|
||||
every role that is not remote, hybrid ones included. A word goes
|
||||
through the host's table, else the shared map. When the value says
|
||||
nothing, the place line or the place texts are read for a word
|
||||
(`remote_of`: "Hybrid, Berlin", and the Danish and Swedish words), then
|
||||
`title` when given, which the stage gives for a search provider's role
|
||||
only ("Backend Developer - Remote"), and last a place the reader marked
|
||||
remote is remote."""
|
||||
if value is True or value == "true":
|
||||
return "remote"
|
||||
said = value not in (None, False, "false")
|
||||
if said and (found := lookup("remote", host, str(value), tables)):
|
||||
return found
|
||||
places = list(places)
|
||||
if found := _line_word(line or "; ".join(map(_place_text, places))):
|
||||
return found
|
||||
if title and (found := _line_word(title)):
|
||||
return found
|
||||
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
|
||||
return "remote"
|
||||
return ""
|
||||
+39
-2
@@ -19,7 +19,9 @@
|
||||
* Under the Follow form of Company boards, a fold lists what each reader
|
||||
* declares (`/api/readers`): how its list arrives, where the text is, what
|
||||
* it filters at the source, how it states places, and the robots.txt
|
||||
* decision taken when it was built.
|
||||
* decision taken when it was built. A second fold lists the host words no
|
||||
* table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and
|
||||
* host, most roles first; it is left out when there are none.
|
||||
*/
|
||||
(function () {
|
||||
"use strict";
|
||||
@@ -320,6 +322,38 @@
|
||||
$("#board-form").after(fold);
|
||||
}
|
||||
|
||||
/** The host words prep could not map, one row per field and host. */
|
||||
function renderUnmapped(words) {
|
||||
if (!words.length) return;
|
||||
const groups = new Map();
|
||||
for (const w of words) {
|
||||
const key = `${w.field} · ${w.host || "no host"}`;
|
||||
if (!groups.has(key)) groups.set(key, []);
|
||||
groups.get(key).push(w);
|
||||
}
|
||||
const fold = document.createElement("details");
|
||||
fold.className = "prune";
|
||||
fold.id = "unmapped";
|
||||
const summary = document.createElement("summary");
|
||||
summary.textContent = `Host words no table maps (${words.length})`;
|
||||
const list = document.createElement("div");
|
||||
list.className = "srows";
|
||||
for (const [key, found] of groups) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "srow";
|
||||
const name = document.createElement("span");
|
||||
name.className = "name";
|
||||
name.textContent = key;
|
||||
const sub = document.createElement("span");
|
||||
sub.className = "sub";
|
||||
sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · ");
|
||||
row.append(name, sub);
|
||||
list.appendChild(row);
|
||||
}
|
||||
fold.append(summary, list);
|
||||
($("#readers") || $("#board-form")).after(fold);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------ one section shown
|
||||
|
||||
// Sources shows one section: the one the fragment names or contains, or the
|
||||
@@ -360,7 +394,10 @@
|
||||
|
||||
document.addEventListener("DOMContentLoaded", async () => {
|
||||
if (!$("#searches-list")) return;
|
||||
if ($("#board-form")) api("/api/readers").then(renderReaders, () => {});
|
||||
if ($("#board-form")) {
|
||||
api("/api/readers").then(renderReaders, () => {})
|
||||
.then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {});
|
||||
}
|
||||
try {
|
||||
await load();
|
||||
} catch (err) {
|
||||
|
||||
@@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any
|
||||
@@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
|
||||
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
||||
|
||||
|
||||
def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None:
|
||||
"""Replace one role's own places with these texts, unresolved: what a
|
||||
pull stores, for `app.prep` to resolve (`relink_uids`)."""
|
||||
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
|
||||
conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None)
|
||||
for position, text in enumerate(place_texts)])
|
||||
|
||||
|
||||
def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve the own places of these roles, writing only those that now
|
||||
resolve differently. Returns `uid -> place ids` in the host's order for
|
||||
every role of `uids` with places, what prep reads the country from."""
|
||||
held: dict[str, dict[tuple[int, str], set[int | None]]] = {}
|
||||
for start in range(0, len(uids), 5000):
|
||||
chunk = json.dumps(list(uids[start:start + 5000]))
|
||||
for uid, position, text, place_id in conn.execute(
|
||||
"SELECT uid, position, text, place_id FROM role_places WHERE url = '' "
|
||||
"AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid",
|
||||
(chunk,)):
|
||||
held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
|
||||
found: dict[str, list[int]] = {}
|
||||
changed = []
|
||||
for uid, rows_of in held.items():
|
||||
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of}
|
||||
if now != rows_of:
|
||||
changed.append((uid, [text for _, text in sorted(rows_of)]))
|
||||
found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])]
|
||||
write_many(conn, gaz, changed)
|
||||
return found
|
||||
|
||||
|
||||
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
||||
"""`write` for many roles in one statement each way. Returns rows written."""
|
||||
|
||||
+82
-2
@@ -18,7 +18,9 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host
|
||||
from app.models import (
|
||||
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
@@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
teaser TEXT NOT NULL DEFAULT '',
|
||||
seniority_model TEXT NOT NULL DEFAULT '',
|
||||
seniority_stated TEXT NOT NULL DEFAULT '',
|
||||
seniority_title TEXT NOT NULL DEFAULT ''
|
||||
seniority_title TEXT NOT NULL DEFAULT '',
|
||||
seniority_host TEXT NOT NULL DEFAULT '',
|
||||
employment_host TEXT NOT NULL DEFAULT '',
|
||||
remote_host TEXT NOT NULL DEFAULT '',
|
||||
country_host TEXT NOT NULL DEFAULT '',
|
||||
employment TEXT NOT NULL DEFAULT '',
|
||||
prep_due INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
-- The *_host columns hold the host's own words as a reader handed them on
|
||||
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
|
||||
-- them into seniority_stated, employment, remote and country. prep_due is above 0
|
||||
-- on a role whose words, company, title or places changed since prep last
|
||||
-- mapped it (Store.upsert_role), which is what the next prep reads.
|
||||
-- roles.teaser is the short text a list carries, kept until the full text
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
@@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors (
|
||||
embedded_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- A host word no table of app/prep/hosts maps, per field (seniority,
|
||||
-- employment, remote, country) and host: how many open roles carry it and
|
||||
-- when prep first met it. Rewritten by every prep; a word added to a table
|
||||
-- leaves at the next one.
|
||||
CREATE TABLE IF NOT EXISTS unmapped (
|
||||
field TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
roles INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen TEXT NOT NULL,
|
||||
PRIMARY KEY (field, host, word)
|
||||
);
|
||||
|
||||
-- The string map a desk plugin is handed as self.state. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
@@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
-- The roles the next prep maps. Partial: after a prep there are none.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
|
||||
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
|
||||
lambda conn: _add_all_roles(conn)),
|
||||
# A Python step: the host words are read out of raw's JSON.
|
||||
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
|
||||
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"to tag; it reads the agents' reach now", target, key)
|
||||
|
||||
|
||||
_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment")
|
||||
|
||||
_V22 = """
|
||||
CREATE TABLE IF NOT EXISTS unmapped (
|
||||
field TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
roles INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen TEXT NOT NULL,
|
||||
PRIMARY KEY (field, host, word)
|
||||
);
|
||||
DELETE FROM state WHERE key = 'prep:tables';
|
||||
"""
|
||||
|
||||
|
||||
def _add_host_words(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 22's step. The columns and `unmapped`, and the host words
|
||||
of every role filled from its `raw` where it has them. Before this,
|
||||
readers mapped `seniority` and `employment` in their own tables, so what
|
||||
`raw` holds there is mostly a value of the vocabulary already, which
|
||||
`app.prep` takes as it is. `remote` and `country` are the host's as
|
||||
given. The `prep:tables` fingerprint goes, so the first prep maps every
|
||||
open role from these words. Runs twice without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
for column in _V22_COLUMNS:
|
||||
if column not in columns:
|
||||
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
|
||||
if "prep_due" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0")
|
||||
for statement in _statements(_V22):
|
||||
conn.execute(statement)
|
||||
updates = []
|
||||
for uid, raw in conn.execute(
|
||||
"SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.employment') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.remote') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.country') IS NOT NULL").fetchall():
|
||||
try:
|
||||
given = json.loads(raw or "{}")
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(given, dict):
|
||||
updates.append((*(host_word(given.get(k)) for k in (
|
||||
"seniority", "employment", "remote", "country")), uid))
|
||||
conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, "
|
||||
"remote_host = ?, country_host = ? WHERE uid = ?", updates)
|
||||
log.info("migration 22: host words from raw for %d roles", len(updates))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+140
-44
@@ -22,13 +22,12 @@ import numpy as np
|
||||
from desk.models import Item
|
||||
from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
|
||||
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
||||
RoleRow, ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
host_word, posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer, schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
@@ -36,7 +35,17 @@ from app.storage import gazetteer, schema
|
||||
#: says less than `extract` found keeps what `extract` found. `function` is
|
||||
#: not here: it is not a board field (see the note by `keep_filled` below).
|
||||
#: Nor is `seniority`, which is the title's level unless a board states one.
|
||||
_RAW_TEXT = ("place", "country", "currency")
|
||||
_RAW_TEXT = ("place", "currency")
|
||||
|
||||
#: The host's own words a reader hands on in `Item.raw`, stored as
|
||||
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
|
||||
#: rewritten every pull, as the host's level always was; the others are kept
|
||||
#: filled, since a role's own page (`enrich`) may state what its listing
|
||||
#: does not.
|
||||
_HOST_WORDS = ("seniority", "employment", "remote", "country")
|
||||
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
|
||||
#: What prep maps from that a pull rewrites whatever it brings.
|
||||
_PREP_OVERWRITTEN = ("title", "seniority_host")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
@@ -154,14 +163,11 @@ def _iso(moment: datetime) -> str:
|
||||
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _remote(value: Any) -> str:
|
||||
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
||||
unrecognised is '' (unknown), which `extract` may fill later."""
|
||||
if value is True:
|
||||
return "remote"
|
||||
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
||||
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
||||
return text if text in REMOTE else ""
|
||||
def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]:
|
||||
"""One role's own place texts as stored, in the host's order."""
|
||||
return list(dict.fromkeys(r[0] for r in conn.execute(
|
||||
"SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid",
|
||||
(uid,))))
|
||||
|
||||
|
||||
def _like(text: str) -> str:
|
||||
@@ -607,10 +613,19 @@ class Store:
|
||||
"""Store one posting. Returns True when it was new.
|
||||
|
||||
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
||||
every call, and a role seen again is open again. The gate columns come
|
||||
every call, and a role seen again is open again. The raw columns come
|
||||
from `item.raw` under their own names, with `company` falling back to
|
||||
the subscription's and `country` to what `place` names; an empty value
|
||||
never overwrites a filled column.
|
||||
the subscription's; an empty value never overwrites a filled column.
|
||||
|
||||
Nothing is mapped here: that is `app.prep`, which runs after every
|
||||
pull. The host's words go to `seniority_host`, `employment_host`,
|
||||
`remote_host` and `country_host` (`host_word`), `company_key` is
|
||||
written unresolved on a new role only, and the mapped columns
|
||||
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
|
||||
`remote`, `country`) are left as prep last wrote them. `prep_due`
|
||||
is 1 on a new role and grows by one when the title, the company, a
|
||||
host word or the place texts changed or the role reopens, which
|
||||
is what the next prep reads.
|
||||
|
||||
`posted_at` (see `app.models.posted_at`) is set from this pull's own
|
||||
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
|
||||
@@ -627,20 +642,14 @@ class Store:
|
||||
alone for good, whatever the plugin's raw guess now says — see
|
||||
`set_extracted` for the other half of the rule.
|
||||
|
||||
`seniority` is the first filled of three levels. `seniority_model`
|
||||
is extract's (`set_extracted`) and never written here.
|
||||
`seniority_stated` is the host's word as `raw["seniority"]` carries
|
||||
it, already in `SENIORITIES` (anything else is ''), and
|
||||
`seniority_title` is `seniority.from_title`; both are rewritten every
|
||||
pull. The classifier is called here rather than in a plugin, so a
|
||||
role from any source gets it, the way `country` comes from `place`.
|
||||
|
||||
The role's places (`role_places`) are the texts of `raw["places"]`,
|
||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
|
||||
against the gazetteer; `place` is the line shown, the texts joined
|
||||
when `raw` has none. Like `place`, a pull with no place leaves them
|
||||
as they are. `place_fold` is written with `place`, under the same
|
||||
keep-filled rule. `teaser` is kept-filled too.
|
||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
||||
when they differ from the stored ones; prep resolves them. Texts that
|
||||
did not change keep their resolved rows, so the feed's Place filter
|
||||
stays right between a pull and its prep. `place` is the line shown,
|
||||
the texts joined when `raw` has none. Like `place`, a pull with no
|
||||
place leaves them as they are. `place_fold` is written with `place`,
|
||||
under the same keep-filled rule. `teaser` is kept-filled too.
|
||||
|
||||
A role is embedded from its description, else its teaser, else its
|
||||
title (`app.jobs.embed.document`). When a pull brings a better one of
|
||||
@@ -660,12 +669,11 @@ class Store:
|
||||
"source": item.source,
|
||||
"source_id": item.source_id,
|
||||
"subscription_id": subscription.id if subscription else None,
|
||||
"company_key": self.resolve_company(company),
|
||||
"company_key": company_key(company),
|
||||
"company": company,
|
||||
"title": item.title,
|
||||
"title_key": title_key(item.title),
|
||||
"place_key": place_key(place),
|
||||
"remote": _remote(raw.get("remote")),
|
||||
"description": item.body,
|
||||
"feed_text": item.excerpt,
|
||||
"original_url": item.location,
|
||||
@@ -677,30 +685,26 @@ class Store:
|
||||
"created_at": stamp,
|
||||
}
|
||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||
row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS}
|
||||
row["place"] = place
|
||||
row["country"] = row["country"] or country_of(place)
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
stated = str(raw.get("seniority") or "")
|
||||
row["seniority_stated"] = stated if stated in SENIORITIES else ""
|
||||
row["seniority_title"] = from_title(item.title)
|
||||
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
|
||||
row["teaser"] = str(raw.get("teaser") or "")
|
||||
row["place_fold"] = places.fold(row["place"])
|
||||
# A board's own pay line, one per pull like title or published_at —
|
||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||
# must be able to clear it, not leave a stale line behind.
|
||||
row["salary_text"] = str(raw.get("salary_text") or "")
|
||||
row["prep_due"] = 1
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", "teaser", *_RAW_TEXT)
|
||||
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
|
||||
"teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS)
|
||||
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
||||
"salary_text", "seniority_stated", "seniority_title")]
|
||||
"salary_text", "seniority_host")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
@@ -711,11 +715,17 @@ class Store:
|
||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||
"ELSE roles.function END")
|
||||
# The model's level, else the host's, else the title's.
|
||||
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
|
||||
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
|
||||
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
||||
"THEN excluded.place_fold ELSE roles.place_fold END")
|
||||
# One more than before when what prep maps from changed, or the role
|
||||
# reopens, so a prep that read the old values leaves it due
|
||||
# (`Store.prepped`). `roles.` is the stored row here, `excluded.`
|
||||
# what this pull brought.
|
||||
changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN]
|
||||
changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})"
|
||||
for c in ("company", *_KEPT_HOST_WORDS)]
|
||||
updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} "
|
||||
"THEN roles.prep_due + 1 ELSE roles.prep_due END")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
@@ -725,8 +735,11 @@ class Store:
|
||||
conn.execute(
|
||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||
if place_texts:
|
||||
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
|
||||
if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts):
|
||||
gazetteer.write_texts(conn, row["uid"], place_texts)
|
||||
if before is not None:
|
||||
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
||||
(row["uid"],))
|
||||
# `embed` encodes a role once, from the best text it has. When a
|
||||
# later pull brings a better one, as a host that states the text
|
||||
# on the role's own page does, the vector goes, so the next
|
||||
@@ -1253,6 +1266,89 @@ class Store:
|
||||
remotes)
|
||||
return len(countries) + len(remotes)
|
||||
|
||||
# ----------------------------------------------------------------- prep
|
||||
|
||||
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
|
||||
"""What `app.prep` maps a role from, and what it holds now: the roles
|
||||
due (`prep_due` above 0), every open role, or every role."""
|
||||
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
|
||||
return self._rows(
|
||||
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
||||
seniority_host, employment_host, remote_host, country_host,
|
||||
seniority_model, seniority_stated, seniority_title, seniority,
|
||||
employment, remote, country,
|
||||
json_extract(raw, '$.places') AS raw_places
|
||||
FROM roles WHERE {where}""")
|
||||
|
||||
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve these roles' own places against the gazetteer, writing the
|
||||
rows that resolve differently; `uid -> place ids` for each that has
|
||||
places (`gazetteer.relink_uids`)."""
|
||||
with self.transaction() as conn:
|
||||
return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids))
|
||||
|
||||
def company_folds(self) -> dict[str, list[str]]:
|
||||
"""`fold -> company keys` of `company_aliases`, sorted: what
|
||||
`resolve_company` asks one name at a time, for prep's whole batch."""
|
||||
out: dict[str, list[str]] = {}
|
||||
for r in self._rows("SELECT fold, company_key FROM company_aliases "
|
||||
"ORDER BY fold, company_key"):
|
||||
out.setdefault(str(r["fold"]), []).append(str(r["company_key"]))
|
||||
return out
|
||||
|
||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||
`seniority_title`, `employment`, `remote`, `remote_places` and
|
||||
`country`. `seniority` is the model's level, else the stated, else
|
||||
the title's. An empty `remote` or `country` leaves the column as it
|
||||
is, since `extract` fills an empty one. `remote_places`, what the
|
||||
place texts say, only fills an empty column: it is a guess from a
|
||||
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
|
||||
changed the role since it was read. Returns the rows written."""
|
||||
with self.transaction() as conn:
|
||||
return conn.executemany(
|
||||
"""UPDATE roles SET company_key = :company_key,
|
||||
seniority_stated = :seniority_stated,
|
||||
seniority_title = :seniority_title,
|
||||
seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||
employment = :employment,
|
||||
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
|
||||
:remote_places),
|
||||
country = COALESCE(NULLIF(:country, ''), country),
|
||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||
WHERE uid = :uid""", mapped).rowcount
|
||||
|
||||
def host_words(self) -> list[tuple[str, str, str, int]]:
|
||||
"""`(field, host, word, open roles)` for every host word an open role
|
||||
carries, the word lowercased: what prep checks against its tables."""
|
||||
found: list[tuple[str, str, str, int]] = []
|
||||
for field in ("seniority", "employment", "remote", "country"):
|
||||
found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows(
|
||||
f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles
|
||||
WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")]
|
||||
return found
|
||||
|
||||
def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None:
|
||||
"""Make `unmapped` these `(field, host, word, roles)`, keeping the
|
||||
`first_seen` of a word already listed."""
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute(
|
||||
"SELECT field, host, word, first_seen FROM unmapped")}
|
||||
conn.execute("DELETE FROM unmapped")
|
||||
conn.executemany(
|
||||
"INSERT INTO unmapped (field, host, word, roles, first_seen) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
[(*r, seen.get(r[:3], stamp)) for r in rows])
|
||||
|
||||
def unmapped(self) -> list[dict[str, Any]]:
|
||||
"""The host words no table maps, by field and host, most roles first."""
|
||||
return [dict(r) for r in self._rows(
|
||||
"SELECT field, host, word, roles, first_seen FROM unmapped "
|
||||
"ORDER BY field, host, roles DESC, word")]
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
||||
|
||||
@@ -897,6 +897,23 @@ def list_readers() -> list[ReaderOut]:
|
||||
for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))]
|
||||
|
||||
|
||||
class UnmappedOut(BaseModel):
|
||||
"""A host word no table of `app/prep/hosts` maps (`Store.unmapped`)."""
|
||||
|
||||
field: str
|
||||
host: str
|
||||
word: str
|
||||
roles: int
|
||||
first_seen: str
|
||||
|
||||
|
||||
@router.get("/prep/unmapped")
|
||||
def list_unmapped(store: StoreDep) -> list[UnmappedOut]:
|
||||
"""The host words the last prep could not map, per field and host, most
|
||||
roles first: what a table in `app/prep/hosts` should learn."""
|
||||
return [UnmappedOut(**w) for w in store.unmapped()]
|
||||
|
||||
|
||||
class SearchBody(BaseModel):
|
||||
provider: str
|
||||
name: str = ""
|
||||
|
||||
@@ -178,28 +178,18 @@ posting per place, each with its own id, declares `places="copies"`: each
|
||||
copy is a row with its one place, and `dupes.py` folds them into one role
|
||||
that keeps each copy's link.
|
||||
|
||||
**Helpers.** `._shared` holds what more than one host needs; a module keeps
|
||||
only its own word tables and uses these:
|
||||
**Helpers.** `._shared` holds what more than one host needs. A module maps
|
||||
no word to the app's vocabulary: it hands on the host's own words under
|
||||
`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them
|
||||
through the host's table in `app/prep/hosts/<name>.py`.
|
||||
|
||||
- `place(text, remote=None)` makes a `Place`: remote when the text has the
|
||||
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
|
||||
appends a place unless its text is empty or already in `found`, in the
|
||||
host's order. A host that states remote itself passes it to `place`, or
|
||||
builds the dict and passes that to `add_place`.
|
||||
- `table_word(table, value, log, what)` maps a host's word (stated seniority
|
||||
or employment) through the module's table, without case. The table's keys
|
||||
are lowercase. Empty gives None; a known word mapped to `""` (`experienced`,
|
||||
`permanent`) names no level and gives None; an unknown word gives None and
|
||||
a debug line naming the host, so the table can grow.
|
||||
- `EMPLOYMENTS`, the one employment vocabulary: `full_time`, `part_time`,
|
||||
`contract`, `temporary`, `internship`, `freelance`. Every value of a
|
||||
module's `_EMPLOYMENT` table is one of these or `""`; the test in
|
||||
`tests/test_readers_shared.py` holds every module to it. Seniority values
|
||||
are `app.models.SENIORITIES`.
|
||||
- `country_if_single(codes)` is the code when the role's places share exactly
|
||||
one, else None, for a host that states a country per place.
|
||||
- `remote_word(text)` reads `remote` or `hybrid` from a search provider's
|
||||
headline or town line (Danish, Swedish, English).
|
||||
- `EMPLOYMENTS` is a copy of `app.models.EMPLOYMENTS` for the readers'
|
||||
tests, since a plugin imports nothing from `app`.
|
||||
|
||||
**Teaser.** A short text the list carries (an excerpt, a summary) is handed
|
||||
over as `teaser`, plain text, whether or not the full text comes later. It
|
||||
|
||||
@@ -8,7 +8,6 @@ import it without either importing the other. The leading underscore keeps
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
@@ -197,9 +196,9 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) ->
|
||||
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
||||
|
||||
|
||||
#: The employment words a row's `employment` may hold, for every host. A
|
||||
#: host's table maps its own words onto these; a word that names none of them
|
||||
#: (a permanent contract says nothing about hours) maps to "".
|
||||
#: A copy of `app.models.EMPLOYMENTS`, the vocabulary `app.prep` maps a
|
||||
#: host's employment word onto, kept for the readers' tests since a plugin
|
||||
#: imports nothing from `app`. `tests/test_prep.py` holds the two equal.
|
||||
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
|
||||
|
||||
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
|
||||
@@ -218,36 +217,3 @@ def add_place(found: list[Place], entry: Place) -> None:
|
||||
found.append(entry)
|
||||
|
||||
|
||||
def table_word(table: dict[str, str], value: Any, log: logging.Logger, what: str) -> str | None:
|
||||
"""`table`'s word for a host's `value`, looked up without case. None when
|
||||
the host states none, when the table maps the word to "" (a known word
|
||||
that names no level, such as `experienced`), or when the table does not
|
||||
know it; that last case is logged at debug level so the table can grow."""
|
||||
word = str(value or "").strip()
|
||||
if not word:
|
||||
return None
|
||||
found = table.get(word.lower())
|
||||
if found is None:
|
||||
log.debug("%s: unknown %s %r", log.name.rsplit(".", 1)[-1], what, word)
|
||||
return found or None
|
||||
|
||||
|
||||
def country_if_single(codes: Any) -> str | None:
|
||||
"""The one code among `codes` when there is exactly one, else None: a
|
||||
country is stated for a role only when all its places share it."""
|
||||
return next(iter(codes)) if len(codes) == 1 else None
|
||||
|
||||
|
||||
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
||||
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(
|
||||
r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
|
||||
|
||||
|
||||
def remote_word(text: str) -> str | None:
|
||||
"""`remote`, `hybrid` or None: the word a search provider's headline or
|
||||
town line says about where the work is done, in Danish, Swedish and
|
||||
English. A board states its own remote flag and needs none of this."""
|
||||
if _REMOTE.search(text):
|
||||
return "remote"
|
||||
return "hybrid" if _HYBRID.search(text) else None
|
||||
@@ -24,7 +24,7 @@ from urllib.parse import urlencode
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Key, Place, Reads
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "adzuna"
|
||||
LABEL = "Adzuna"
|
||||
@@ -124,7 +124,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
"places": places,
|
||||
"location": town,
|
||||
"country": country.upper(),
|
||||
"remote": remote_word(title),
|
||||
"posted": parse_dt(job.get("created")),
|
||||
"teaser": _text(job.get("description") or ""),
|
||||
"salary_min": None if predicted else job.get("salary_min"),
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -36,13 +36,6 @@ READS = Reads(
|
||||
|
||||
_API = "api.ashbyhq.com"
|
||||
|
||||
#: `workplaceType` -> `store._remote`'s words. `isRemote` is true for hybrid
|
||||
#: roles too, so it is only the fallback when the role has no `workplaceType`.
|
||||
_WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"}
|
||||
|
||||
#: `employmentType` -> the app's employment words.
|
||||
_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "internship",
|
||||
"contract": "contract", "temporary": "temporary"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
@@ -113,12 +106,12 @@ def _salary_text(compensation: dict[str, Any]) -> str:
|
||||
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
||||
|
||||
|
||||
def _places(j: dict[str, Any], workplace: str | None) -> list[Place]:
|
||||
def _places(j: dict[str, Any], workplace: str) -> list[Place]:
|
||||
"""The primary `location`, then each of `secondaryLocations`, in the host's
|
||||
words. The primary is remote or an office as `workplaceType` says; a
|
||||
place that names itself remote is remote."""
|
||||
primary = (j.get("location") or "").strip()
|
||||
named = [(primary, {"remote": True, "hybrid": False, "onsite": False}.get(workplace or ""))]
|
||||
named = [(primary, {"remote": True, "hybrid": False, "onsite": False}.get(workplace.lower()))]
|
||||
named += [(str((s or {}).get("location") or "").strip(), None)
|
||||
for s in j.get("secondaryLocations") or []]
|
||||
out: list[Place] = []
|
||||
@@ -127,10 +120,6 @@ def _places(j: dict[str, Any], workplace: str | None) -> list[Place]:
|
||||
return out
|
||||
|
||||
|
||||
def _employment(value: Any) -> str | None:
|
||||
return table_word(_EMPLOYMENT, value, log, "employmentType")
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
@@ -142,7 +131,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
||||
compensation = j.get("compensation") or {}
|
||||
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
|
||||
workplace = _WORKPLACE.get(str(j.get("workplaceType") or "").strip().lower())
|
||||
workplace = str(j.get("workplaceType") or "").strip()
|
||||
places = _places(j, workplace)
|
||||
row: Row = {
|
||||
"source_id": str(j.get("id") or ""),
|
||||
@@ -153,13 +142,13 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"posted": parse_dt(j.get("publishedAt")),
|
||||
"department": (j.get("department") or j.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": workplace or j.get("isRemote"),
|
||||
"remote": workplace or j.get("isRemote"), # `isRemote` is true for hybrid too
|
||||
"salary_text": _salary_text(compensation),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
}
|
||||
if employment := _employment(j.get("employmentType")):
|
||||
if employment := str(j.get("employmentType") or "").strip():
|
||||
row["employment"] = employment
|
||||
rows.append(row)
|
||||
return rows
|
||||
@@ -23,7 +23,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, label_slug_ok, only, parse_dt, place, table_word
|
||||
from ._shared import Get, label_slug_ok, only, parse_dt, place
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -34,13 +34,6 @@ READS = Reads(
|
||||
robots="read: robots.txt allows /careers (checked 2026-09-29)",
|
||||
)
|
||||
|
||||
#: `employmentStatusLabel` -> the app's employment words. BambooHR's labels are
|
||||
#: the company's own ("Full-Time", "Part-Time", "Contract", "Intern").
|
||||
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "contractor": "contract",
|
||||
"temporary": "temporary", "intern": "internship", "internship": "internship",
|
||||
"freelance": "freelance"}
|
||||
|
||||
NAME = "bamboohr"
|
||||
LABEL = "BambooHR"
|
||||
PATTERNS = (
|
||||
@@ -49,9 +42,6 @@ PATTERNS = (
|
||||
r"https?://(?!(?:app|resources|documentation|help)\.)([a-z0-9-]+)\.bamboohr\.com",
|
||||
)
|
||||
|
||||
#: `locationType` -> `store._remote`'s words. Anything else says nothing.
|
||||
_REMOTE = {"0": "onsite", "1": "remote", "2": "hybrid"}
|
||||
|
||||
#: Sent on the detail read: without it the same URL answers the page's HTML.
|
||||
_JSON = {"Accept": "application/json"}
|
||||
|
||||
@@ -76,10 +66,11 @@ def feed_url(slug: str, ask: Ask) -> str:
|
||||
return f"https://{host(slug)}/careers/list"
|
||||
|
||||
|
||||
def _remote(j: dict[str, Any]) -> str | None:
|
||||
def _remote(j: dict[str, Any]) -> bool | str | None:
|
||||
"""`isRemote` when true, else `locationType` as the host writes it ("0", "1", "2")."""
|
||||
if j.get("isRemote") is True:
|
||||
return "remote"
|
||||
return _REMOTE.get(str(j.get("locationType") if j.get("locationType") is not None else ""))
|
||||
return True
|
||||
return None if j.get("locationType") is None else str(j["locationType"]).strip() or None
|
||||
|
||||
|
||||
def _place(j: dict[str, Any]) -> str:
|
||||
@@ -99,10 +90,6 @@ def _places(j: dict[str, Any]) -> list[Place]:
|
||||
return [place(text)] if text else []
|
||||
|
||||
|
||||
def _employment(j: dict[str, Any]) -> str | None:
|
||||
return table_word(_EMPLOYMENT, j.get("employmentStatusLabel"), log, "employmentStatusLabel")
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""`/careers/list` -> `{"meta": {"totalCount": n}, "result": [...]}`. A
|
||||
role has `id`, `jobOpeningName`, `departmentLabel`, `location` (`city`,
|
||||
@@ -136,7 +123,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"remote": _remote(j),
|
||||
"salary_text": "", # the list states no pay; the detail's `compensation` was null
|
||||
}
|
||||
if employment := _employment(j):
|
||||
if employment := str(j.get("employmentStatusLabel") or "").strip():
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
@@ -160,6 +147,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
if place := _place(opening):
|
||||
out["location"] = place
|
||||
out["places"] = _places(opening)
|
||||
if remote := _remote(opening):
|
||||
if (remote := _remote(opening)) is not None:
|
||||
out["remote"] = remote
|
||||
return out
|
||||
+5
-14
@@ -24,7 +24,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import num, only, parse_dt, pay_line, table_word
|
||||
from ._shared import num, only, parse_dt, pay_line
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,15 +59,6 @@ _PUSH = re.compile(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*")\]\)')
|
||||
_ROW = re.compile(rb"([0-9a-f]*):")
|
||||
_TEXT = re.compile(rb"T([0-9a-f]+),")
|
||||
|
||||
#: Deel's `workArrangementEnum` -> `store._remote`'s words.
|
||||
_REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
|
||||
|
||||
#: The name of a `jobEmploymentTypes` entry -> the app's employment words.
|
||||
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "contractor": "contract",
|
||||
"consultant": "contract", "temporary": "temporary", "intern": "internship",
|
||||
"internship": "internship", "freelance": "freelance"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_SLUG.fullmatch(slug))
|
||||
@@ -121,10 +112,10 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
||||
|
||||
|
||||
def _employment(job: dict[str, Any]) -> str | None:
|
||||
"""The first `jobEmploymentTypes` entry that names a known type."""
|
||||
"""The name of the first `jobEmploymentTypes` entry that has one."""
|
||||
for entry in job.get("jobEmploymentTypes") or []:
|
||||
name = ((entry or {}).get("employmentType") or {}).get("name")
|
||||
if found := table_word(_EMPLOYMENT, name, log, "employment type"):
|
||||
if found := str(name or "").strip():
|
||||
return found
|
||||
return None
|
||||
|
||||
@@ -155,7 +146,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
description = p.get("richtextDescription") or ""
|
||||
if description.startswith("$"):
|
||||
description = stream.get(description[1:], "")
|
||||
arrangement = str(job.get("workArrangementEnum") or "")
|
||||
arrangement = str(job.get("workArrangementEnum") or "").strip()
|
||||
remote = {"REMOTE": True, "HYBRID": False, "ON_SITE": False}.get(arrangement)
|
||||
row: Row = {
|
||||
"source_id": post_id,
|
||||
@@ -166,7 +157,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"posted": parse_dt(published.get("createdAt") or p.get("createdAt")),
|
||||
"department": ", ".join(dict.fromkeys(departments)),
|
||||
"description": html.unescape(strip_html(description)),
|
||||
"remote": _REMOTE.get(str(job.get("workArrangementEnum") or "")),
|
||||
"remote": arrangement or None,
|
||||
"salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")),
|
||||
pay.get("currencyIsoCode"), "")
|
||||
if p.get("isCompensationVisible") else ""),
|
||||
|
||||
@@ -70,10 +70,6 @@ _NAME = re.compile(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9]
|
||||
#: Names a private network uses, never a company's career site.
|
||||
_LOCAL = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
||||
|
||||
#: `workLocationOption` -> `store._remote`'s words. Only "onsite" and "hybrid"
|
||||
#: were seen; the other spellings are the obvious ones.
|
||||
_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
|
||||
|
||||
|
||||
def parts(slug: str) -> tuple[str, str]:
|
||||
"""`(host, domain)` of a slug, lowercased."""
|
||||
@@ -162,7 +158,7 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
"posted": parse_dt(p.get("creationTs") or p.get("postedTs")),
|
||||
"department": str(p.get("department") or "").strip(),
|
||||
"description": "",
|
||||
"remote": _REMOTE.get(str(p.get("workLocationOption") or "").strip().lower()),
|
||||
"remote": str(p.get("workLocationOption") or "").strip() or None,
|
||||
"salary_text": "",
|
||||
})
|
||||
return out
|
||||
|
||||
+4
-12
@@ -27,7 +27,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, Page, add_place, only, parse_dt, table_word
|
||||
from ._shared import Get, Page, add_place, only, parse_dt
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -65,14 +65,6 @@ DETAIL_QUERY = (
|
||||
"job { id locationType employmentType requisitionId teamDisplayName "
|
||||
"department { id extId name } } compensationHtml } }")
|
||||
|
||||
#: `job.locationType` -> `store._remote`'s words.
|
||||
_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
|
||||
|
||||
#: `job.employmentType` -> the app's employment words.
|
||||
_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contract": "contract",
|
||||
"contractor": "contract", "intern": "internship", "internship": "internship",
|
||||
"temporary": "temporary", "freelance": "freelance"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_BOARD.fullmatch(slug))
|
||||
@@ -134,9 +126,9 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
add_place(places, {"text": text, "remote": loc["isRemote"]
|
||||
if isinstance(loc.get("isRemote"), bool) else None})
|
||||
job = p.get("job") or {}
|
||||
remote = _REMOTE.get(str(job.get("locationType") or "").upper())
|
||||
remote: bool | str | None = str(job.get("locationType") or "").strip() or None
|
||||
if remote is None and locations and all(loc.get("isRemote") for loc in locations):
|
||||
remote = "remote"
|
||||
remote = True
|
||||
row: Row = {
|
||||
"source_id": ext_id,
|
||||
"title": title,
|
||||
@@ -149,7 +141,7 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
"remote": remote,
|
||||
"salary_text": "",
|
||||
}
|
||||
if employment := table_word(_EMPLOYMENT, job.get("employmentType"), log, "employmentType"):
|
||||
if employment := str(job.get("employmentType") or "").strip():
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
@@ -23,7 +23,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
|
||||
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -38,21 +38,10 @@ READS = Reads(
|
||||
"app's index page with no rules (checked 2026-09-29)",
|
||||
)
|
||||
|
||||
#: `employmentType`, the company's own name for it -> the app's employment
|
||||
#: words. "Permanent" says nothing about hours and stays unmapped.
|
||||
_EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time",
|
||||
"full-time": "full_time", "regular part time": "part_time",
|
||||
"part time": "part_time", "part-time": "part_time", "contract": "contract",
|
||||
"contractor": "contract", "temporary": "temporary", "fixed term": "temporary",
|
||||
"intern": "internship", "internship": "internship", "freelance": "freelance"}
|
||||
|
||||
NAME = "hibob"
|
||||
LABEL = "HiBob"
|
||||
PATTERNS = (r"https?://([a-z0-9-]+)\.careers\.hibob\.com",)
|
||||
|
||||
#: HiBob's `workspaceTypeId` -> `store._remote`'s words.
|
||||
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite", "onsite": "onsite"}
|
||||
|
||||
#: The texts of an ad, in the order the ad shows them.
|
||||
_SECTIONS = ("description", "responsibilities", "requirements", "benefits")
|
||||
|
||||
@@ -126,13 +115,13 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"posted": parse_dt(ad.get("publishedAt")),
|
||||
"department": str(ad.get("department") or "").strip(),
|
||||
"description": " ".join(html.unescape(text).split()),
|
||||
"remote": _REMOTE.get(str(ad.get("workspaceTypeId") or "").strip().lower()),
|
||||
"remote": str(ad.get("workspaceTypeId") or "").strip() or None,
|
||||
"salary_text": pay_line(lo, hi, currency, period),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": code,
|
||||
}
|
||||
if employment := table_word(_EMPLOYMENT, ad.get("employmentType"), log, "employmentType"):
|
||||
if employment := str(ad.get("employmentType") or "").strip():
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
@@ -38,7 +38,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import Get, Page, add_place, only, parse_dt, place, table_word
|
||||
from ._shared import Get, Page, add_place, only, parse_dt, place
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -49,11 +49,6 @@ READS = Reads(
|
||||
robots="read: robots.txt allows /jobs, with crawl-delay 5 not honoured (checked 2026-09-29)",
|
||||
)
|
||||
|
||||
#: `employment_type` -> the app's employment words.
|
||||
_EMPLOYMENT = {"full_time": "full_time", "part_time": "part_time", "contractor": "contract",
|
||||
"contract": "contract", "temporary": "temporary", "intern": "internship",
|
||||
"internship": "internship", "freelance": "freelance"}
|
||||
|
||||
NAME = "jibe"
|
||||
LABEL = "Jibe"
|
||||
PATTERNS = (
|
||||
@@ -157,10 +152,10 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
}
|
||||
# One code, for a role at one place: over several countries it would
|
||||
# gate the others by the site's own.
|
||||
code = str(j.get("country_code") or "").strip().upper()
|
||||
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code):
|
||||
code = str(j.get("country_code") or "").strip()
|
||||
if len(places) == 1 and code:
|
||||
row["country"] = code
|
||||
if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
|
||||
if employment := str(j.get("employment_type") or "").strip():
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
@@ -28,7 +28,7 @@ from urllib.parse import urlencode
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, place
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
|
||||
NAME = "jobindex"
|
||||
@@ -131,9 +131,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
company = full_title[len(title) + 2:].strip()
|
||||
area = _AREA.search(description)
|
||||
town = _text(area.group(1)) if area else decoded["area"]
|
||||
remote = remote_word(f"{town} {title}")
|
||||
places: list[Place] = [{"text": town, "remote": True if remote == "remote" else None}] \
|
||||
if town else []
|
||||
places: list[Place] = [place(town)] if town else []
|
||||
out.append({
|
||||
"source_id": item["id"],
|
||||
"title": title or full_title,
|
||||
@@ -142,7 +140,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
"places": places,
|
||||
"location": town,
|
||||
"country": "DK",
|
||||
"remote": remote,
|
||||
"posted": parse_dt(item["published"]),
|
||||
"teaser": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
||||
})
|
||||
|
||||
@@ -26,7 +26,7 @@ from typing import Any
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import Get, parse_dt, place, table_word
|
||||
from ._shared import Get, parse_dt, place
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -58,28 +58,10 @@ _SLUG = re.compile(r"[0-9]{1,12}")
|
||||
#: from.
|
||||
_ROLE_PATH = re.compile(r"/jobs/[0-9]+(?:-[A-Za-z0-9_-]*)?/?")
|
||||
|
||||
#: The embed's `experience` -> `app.models.SENIORITIES`. Only the coarse levels
|
||||
#: are mapped; "Experienced" sits between "Mid level" and the managerial
|
||||
#: levels and says too little to place, so it stays unmapped and is logged.
|
||||
_SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern",
|
||||
"entry level": "junior", "entry-level": "junior", "junior": "junior",
|
||||
"graduate": "junior", "mid level": "mid", "mid-level": "mid", "mid": "mid",
|
||||
"senior": "senior", "lead": "lead", "manager": "manager", "director": "director",
|
||||
"executive": "exec", "not applicable": ""}
|
||||
|
||||
#: The embed's `employment_type` -> the app's employment words.
|
||||
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
|
||||
"part time": "part_time", "contract": "contract", "consultant": "contract",
|
||||
"temporary": "temporary", "internship": "internship", "intern": "internship",
|
||||
"freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"}
|
||||
|
||||
_ASSIGNMENT = "JBL.embed_v2['jobs'] = "
|
||||
_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
||||
_TRAILING_COMMA = re.compile(r",(\s*[\]}])")
|
||||
|
||||
#: Jobylon's `workspace` -> `store._remote`'s words.
|
||||
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
|
||||
|
||||
#: The English month names, by their first three letters, in the spellings
|
||||
#: `published_date` uses ("Sept. 25, 2026").
|
||||
_MONTHS = {m: i for i, m in enumerate(
|
||||
@@ -205,7 +187,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"posted": _published(j.get("published_date")),
|
||||
"department": "" if function == "All functions" else function,
|
||||
"description": "",
|
||||
"remote": _REMOTE.get(str(j.get("workspace") or "").strip().lower()),
|
||||
"remote": str(j.get("workspace") or "").strip() or None,
|
||||
"salary_text": "", # the embed states no pay
|
||||
}
|
||||
# `summary` is the text the company gave the listing, the string
|
||||
@@ -213,9 +195,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split())
|
||||
if summary and summary != "None":
|
||||
row["teaser"] = summary
|
||||
if seniority := table_word(_SENIORITY, j.get("experience"), log, "experience"):
|
||||
if seniority := str(j.get("experience") or "").strip():
|
||||
row["seniority"] = seniority
|
||||
if employment := table_word(_EMPLOYMENT, j.get("employment_type"), log, "employment_type"):
|
||||
if employment := str(j.get("employment_type") or "").strip():
|
||||
row["employment"] = employment
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
@@ -17,14 +17,13 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, table_word, yearly_amount
|
||||
from ._shared import add_place, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -36,13 +35,6 @@ READS = Reads(
|
||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
||||
)
|
||||
|
||||
#: `categories.commitment` -> the app's employment words. A commitment is free
|
||||
#: text on Lever ("Permanent" is seen too and says nothing about hours).
|
||||
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time",
|
||||
"part-time": "part_time", "part time": "part_time", "parttime": "part_time",
|
||||
"intern": "internship", "internship": "internship", "contract": "contract",
|
||||
"contractor": "contract", "temporary": "temporary", "freelance": "freelance"}
|
||||
|
||||
NAME = "lever"
|
||||
LABEL = "Lever"
|
||||
RANK = 20
|
||||
@@ -153,10 +145,10 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
}
|
||||
# `country` is one code, for a role at one place; over several it would
|
||||
# gate the others by the first's country.
|
||||
country = str(j.get("country") or "").strip().upper()
|
||||
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country):
|
||||
country = str(j.get("country") or "").strip()
|
||||
if len(places) == 1 and country:
|
||||
row["country"] = country
|
||||
if employment := table_word(_EMPLOYMENT, categories.get("commitment"), log, "commitment"):
|
||||
if employment := str(categories.get("commitment") or "").strip():
|
||||
row["employment"] = employment
|
||||
rows.append(row)
|
||||
return rows
|
||||
@@ -35,7 +35,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import NAMES, code_of
|
||||
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt
|
||||
|
||||
NAME = "oracle"
|
||||
LABEL = "Oracle Recruiting Cloud"
|
||||
@@ -55,7 +55,7 @@ PATTERNS = (
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
|
||||
complete="stated", places="many", filters=Filters(country="ids", remote="one"),
|
||||
fields=Fields(posted="trusted", remote="stated", country="stated"),
|
||||
fields=Fields(posted="trusted", remote="stated"),
|
||||
robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites/<site>/jobs "
|
||||
"(checked 2026-09-29)"),
|
||||
)
|
||||
@@ -71,11 +71,6 @@ PAGE_SIZE = 200
|
||||
_HOST = r"[a-z0-9-]+(?:\.[a-z0-9-]+)*\.oraclecloud\.(?:com|eu)"
|
||||
_SLUG = re.compile(rf"({_HOST})/([A-Za-z0-9_-]+)", re.IGNORECASE)
|
||||
|
||||
#: `WorkplaceTypeCode` -> `store._remote`'s words. A role that states none
|
||||
#: has an empty code, which stays unknown.
|
||||
_REMOTE = {"ora_remote": "remote", "ora_hybrid": "hybrid", "ora_on_site": "onsite"}
|
||||
|
||||
|
||||
def _parts(slug: str) -> tuple[str, str]:
|
||||
"""`(host, site number)` of a slug that passed `slug_ok`."""
|
||||
host, _, site = slug.partition("/")
|
||||
@@ -189,18 +184,14 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
# The id goes into a URL, and a listing is external content.
|
||||
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id):
|
||||
continue
|
||||
remote = _REMOTE.get(_text(role.get("WorkplaceTypeCode")).lower())
|
||||
remote = _text(role.get("WorkplaceTypeCode")) or None
|
||||
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
|
||||
places: list[Place] = []
|
||||
codes: set[str] = set()
|
||||
secondary = [s for s in role.get("secondaryLocations") or [] if isinstance(s, dict)]
|
||||
for name, code in ((_text(role.get("PrimaryLocation")),
|
||||
_text(role.get("PrimaryLocationCountry"))),
|
||||
*((_text(s.get("Name")), _text(s.get("CountryCode")))
|
||||
for s in secondary)):
|
||||
add_place(places, {"text": name, "remote": {"remote": True, "hybrid": False,
|
||||
"onsite": False}.get(remote or "")})
|
||||
if code:
|
||||
codes.add(code.upper())
|
||||
for name in (_text(role.get("PrimaryLocation")),
|
||||
*(_text(s.get("Name")) for s in secondary)):
|
||||
add_place(places, {"text": name, "remote": (
|
||||
True if (remote or "").lower() == "ora_remote" else False if at_office else None)})
|
||||
out.append({
|
||||
"source_id": role_id,
|
||||
"title": title,
|
||||
@@ -212,7 +203,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
or _text(role.get("Department")) or _text(role.get("Organization"))),
|
||||
"description": "",
|
||||
"teaser": _plain(role.get("ShortDescriptionStr")),
|
||||
"country": country_if_single(codes),
|
||||
"remote": remote,
|
||||
"salary_text": "", # the resource states no pay
|
||||
})
|
||||
|
||||
@@ -7,16 +7,13 @@ Personio.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, table_word, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from ._shared import Row, add_place, label_slug_ok, num, only, parse_dt, pay_line, place, yearly_amount
|
||||
|
||||
NAME = "personio"
|
||||
LABEL = "Personio"
|
||||
@@ -30,19 +27,6 @@ READS = Reads(
|
||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
||||
)
|
||||
|
||||
#: `seniority` -> the app's levels. `experienced` is what most roles say and
|
||||
#: names no level, so it is known and maps to none; a word not listed is
|
||||
#: logged at debug level so the table can grow.
|
||||
_SENIORITY = {"entry-level": "junior", "student": "intern", "executive": "exec",
|
||||
"experienced": ""}
|
||||
|
||||
#: `employmentType` when it is not the usual permanent contract, to the app's
|
||||
#: employment words; `permanent` says nothing about the hours, so `schedule`
|
||||
#: decides then.
|
||||
_CONTRACT = {"temporary": "temporary", "freelance": "contract", "intern": "internship",
|
||||
"working_student": "internship", "trainee": "internship"}
|
||||
_SCHEDULE = {"full-time": "full_time", "part-time": "part_time"}
|
||||
|
||||
#: Personio's `salaryInformation/type` spellings, to the plain period word
|
||||
#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through.
|
||||
_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
|
||||
@@ -85,18 +69,14 @@ def _salary_text(salary_info: ET.Element) -> str:
|
||||
salary_info.findtext("currencyCode"), period)
|
||||
|
||||
|
||||
def _seniority(word: str) -> str | None:
|
||||
return table_word(_SENIORITY, word, log, "seniority")
|
||||
|
||||
|
||||
def _employment(position: ET.Element) -> str | None:
|
||||
contract = (position.findtext("employmentType") or "").strip().lower()
|
||||
schedule = (position.findtext("schedule") or "").strip().lower()
|
||||
if contract in _CONTRACT:
|
||||
return _CONTRACT[contract]
|
||||
if contract and contract != "permanent":
|
||||
log.debug("personio: unknown employmentType %r", contract)
|
||||
return _SCHEDULE.get(schedule)
|
||||
"""The host's word for the contract, or, for a `permanent` one (which says
|
||||
nothing about the hours), the `schedule`."""
|
||||
contract = (position.findtext("employmentType") or "").strip()
|
||||
schedule = (position.findtext("schedule") or "").strip()
|
||||
if contract.lower() in ("", "permanent") and schedule:
|
||||
return schedule
|
||||
return contract or None
|
||||
|
||||
|
||||
def _places(position: ET.Element) -> list[Place]:
|
||||
@@ -148,8 +128,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"posted": parse_dt(position.findtext("createdAt")),
|
||||
"department": (position.findtext("department") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": any(p["remote"] for p in places) or None,
|
||||
"seniority": _seniority(position.findtext("seniority") or ""),
|
||||
"seniority": (position.findtext("seniority") or "").strip() or None,
|
||||
"employment": _employment(position),
|
||||
"salary_text": salary_text,
|
||||
"salary_min": salary_min,
|
||||
|
||||
@@ -188,10 +188,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
_JSON_LD = re.compile(r"<script[^>]+application/ld\+json[^>]*>(.*?)</script>", re.S | re.I)
|
||||
|
||||
|
||||
#: Where a page assigns its state, and the words its `checkRemote` says that
|
||||
#: `store._remote` reads (King's pages say "On-site"; no other was seen).
|
||||
#: Where a page assigns its state. Its `checkRemote` says "On-site" on King's
|
||||
#: pages; no other word was seen.
|
||||
_STATE = "phApp.ddo = "
|
||||
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
|
||||
|
||||
|
||||
def _job_posting(body: str) -> dict[str, Any]:
|
||||
@@ -243,6 +242,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
raise ValueError("no description on the role's page")
|
||||
if posted := parse_dt(posting.get("datePosted")):
|
||||
out["posted"] = posted
|
||||
if remote := _REMOTE.get(_text(job.get("checkRemote")).lower()):
|
||||
if remote := _text(job.get("checkRemote")):
|
||||
out["remote"] = remote
|
||||
return out
|
||||
@@ -18,7 +18,6 @@ from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
@@ -27,8 +26,6 @@ from desk.sources import strip_html
|
||||
from ._contract import Ask, Fields, Reads
|
||||
from ._shared import Row, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
NAME = "pinpoint"
|
||||
LABEL = "Pinpoint"
|
||||
PATTERNS = (r"https?://([a-z0-9-]+)\.pinpointhq\.com",)
|
||||
@@ -39,9 +36,6 @@ READS = Reads(
|
||||
robots="read: robots.txt allows / (checked 2026-09-29)",
|
||||
)
|
||||
|
||||
#: `workplace_type` -> `store._remote`'s words. Anything else says nothing.
|
||||
_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
|
||||
|
||||
#: `compensation_frequency` -> the multiplier that makes a figure yearly. The
|
||||
#: spellings are a guess, since no board read so far states pay; a frequency
|
||||
#: not listed leaves the yearly columns empty, and `salary_text` still says it.
|
||||
@@ -51,12 +45,6 @@ _YEARLY = {"annually": 1, "annual": 1, "yearly": 1, "year": 1, "monthly": 12, "m
|
||||
_PERIOD = {"annually": "year", "annual": "year", "yearly": "year", "monthly": "month",
|
||||
"weekly": "week", "daily": "day", "hourly": "hour"}
|
||||
|
||||
#: What `employment_type` (`permanent_full_time`, `fixed_term_full_time`) says,
|
||||
#: the first fragment found deciding, so a contract type beats the hours.
|
||||
_EMPLOYMENT = (("intern", "internship"), ("freelance", "contract"), ("contract", "contract"),
|
||||
("fixed_term", "temporary"), ("temporary", "temporary"),
|
||||
("part_time", "part_time"), ("full_time", "full_time"))
|
||||
|
||||
#: The sections of a posting's text, in the order the page shows them: the
|
||||
#: description, then these fields, each with its own heading in `<field>_header`.
|
||||
_SECTIONS = ("key_responsibilities", "skills_knowledge_expertise", "benefits")
|
||||
@@ -112,16 +100,6 @@ def _place(location: dict[str, Any]) -> str:
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
def _employment(code: str) -> str | None:
|
||||
code = code.strip().lower()
|
||||
for fragment, value in _EMPLOYMENT:
|
||||
if fragment in code:
|
||||
return value
|
||||
if code:
|
||||
log.debug("pinpoint: unknown employment_type %r", code)
|
||||
return None
|
||||
|
||||
|
||||
def _pay(j: dict[str, Any]) -> tuple[int | None, int | None, str, str]:
|
||||
"""`(salary_min, salary_max, currency, salary_text)` of a posting whose
|
||||
pay is shown (`compensation_visible`); the free-text `compensation` is the
|
||||
@@ -154,21 +132,21 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
continue
|
||||
job = j.get("job") or {}
|
||||
low, high, currency, salary_text = _pay(j)
|
||||
workplace = str(j.get("workplace_type") or "").strip().lower()
|
||||
workplace = str(j.get("workplace_type") or "").strip()
|
||||
place = _place(j.get("location") or {})
|
||||
out.append({
|
||||
"source_id": source_id,
|
||||
"title": title,
|
||||
"places": [{"text": place, "remote": {"remote": True, "onsite": False,
|
||||
"on_site": False, "hybrid": False}.get(workplace)}]
|
||||
"on_site": False, "hybrid": False}.get(workplace.lower())}]
|
||||
if place else [],
|
||||
"location": place,
|
||||
"url": url,
|
||||
"posted": None, # only the posting's own page has a date
|
||||
"department": str((job.get("department") or {}).get("name") or "").strip(),
|
||||
"description": _text(j),
|
||||
"remote": _REMOTE.get(workplace),
|
||||
"employment": _employment(str(j.get("employment_type") or "")),
|
||||
"remote": workplace or None,
|
||||
"employment": str(j.get("employment_type") or "").strip() or None,
|
||||
"salary_text": salary_text,
|
||||
"salary_min": low,
|
||||
"salary_max": high,
|
||||
|
||||
@@ -7,8 +7,8 @@ Checked live on 2026-09-29 and 2026-09-30, moved here from `plugins/search.py`.
|
||||
above 2000 is refused, so a search reads 2,100 results at most. It states
|
||||
`total`. A hit has the whole ad's text, so the list has the full text. An ad
|
||||
may name several workplaces: `workplace_addresses` holds one entry each, and
|
||||
every one is a place of the role. `workplace_model` is mostly empty, so
|
||||
`remote` comes from the headline and the model together.
|
||||
every one is a place of the role. `workplace_model` is mostly empty; its
|
||||
label is passed on as `remote`, in the host's words.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt, remote_word
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "platsbanken"
|
||||
LABEL = "Platsbanken"
|
||||
@@ -113,7 +113,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
|
||||
places = [place for address in addresses if isinstance(address, dict)
|
||||
if (place := _place(address))]
|
||||
model = (hit.get("workplace_model") or {}).get("label") or ""
|
||||
model = str((hit.get("workplace_model") or {}).get("label") or "").strip()
|
||||
out.append({
|
||||
"source_id": str(hit.get("id") or ""),
|
||||
"title": title,
|
||||
@@ -122,7 +122,7 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
"places": places,
|
||||
"location": places[0]["text"] if places else "",
|
||||
"country": "SE",
|
||||
"remote": remote_word(f"{title} {model}"),
|
||||
"remote": model or None,
|
||||
"posted": parse_dt(hit.get("publication_date")),
|
||||
"description": _text((hit.get("description") or {}).get("text") or ""),
|
||||
})
|
||||
|
||||
@@ -8,16 +8,13 @@ Recruitee.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Row, add_place, country_if_single, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, table_word, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from ._shared import Row, add_place, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
NAME = "recruitee"
|
||||
LABEL = "Recruitee"
|
||||
@@ -27,23 +24,11 @@ PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",)
|
||||
READS = Reads(
|
||||
scope="board", listing="whole", text="listing", complete="whole",
|
||||
places="many",
|
||||
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated",
|
||||
fields=Fields(pay="figures", posted="trusted", remote="stated",
|
||||
seniority="stated", employment="stated"),
|
||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
||||
)
|
||||
|
||||
#: `experience_code` -> the app's levels. `experienced` names no level and is
|
||||
#: known, so it maps to none; a word not listed is logged at debug level.
|
||||
_SENIORITY = {"student_school": "intern", "entry_level": "junior", "mid_level": "mid",
|
||||
"experienced": ""}
|
||||
|
||||
#: The words of `employment_type_code` (`fulltime_permanent`), the first that
|
||||
#: is found deciding, so a contract type beats the hours.
|
||||
_EMPLOYMENT = (("internship", "internship"), ("intern", "internship"),
|
||||
("freelance", "contract"), ("contract", "contract"),
|
||||
("temporary", "temporary"), ("fixed", "temporary"),
|
||||
("parttime", "part_time"), ("fulltime", "full_time"))
|
||||
|
||||
#: What the board writes as a place for a role that is worked anywhere.
|
||||
_NO_PLACE = ("remote job",)
|
||||
|
||||
@@ -82,28 +67,13 @@ def _salary_text(salary: dict[str, Any]) -> str:
|
||||
salary.get("currency"), period)
|
||||
|
||||
|
||||
def _seniority(code: str) -> str | None:
|
||||
return table_word(_SENIORITY, code, log, "experience_code")
|
||||
|
||||
|
||||
def _employment(code: str) -> str | None:
|
||||
words = set(code.strip().lower().replace("-", "_").split("_"))
|
||||
for word, value in _EMPLOYMENT:
|
||||
if word in words:
|
||||
return value
|
||||
if code.strip():
|
||||
log.debug("recruitee: unknown employment_type_code %r", code)
|
||||
return None
|
||||
|
||||
|
||||
def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
|
||||
"""Every place of an offer and the countries' codes among them. `locations`
|
||||
def _places(offer: dict[str, Any]) -> list[Place]:
|
||||
"""Every place of an offer. `locations`
|
||||
lists them (city, state, country), `location` is the first as one line and
|
||||
is the fallback for an offer that lists none. The board writes "Remote
|
||||
job" where an offer has no place: that is not one, and a remote offer with
|
||||
no other place has the place "Remote"."""
|
||||
found: list[Place] = []
|
||||
codes: set[str] = set()
|
||||
|
||||
def add(text: str) -> None:
|
||||
text = text.strip()
|
||||
@@ -119,15 +89,11 @@ def _places(offer: dict[str, Any]) -> tuple[list[Place], set[str]]:
|
||||
if part and part not in parts:
|
||||
parts.append(part)
|
||||
add(", ".join(parts))
|
||||
if code := str(loc.get("country_code") or "").strip().upper():
|
||||
codes.add(code)
|
||||
if not found:
|
||||
add(str(offer.get("location") or offer.get("city") or ""))
|
||||
if code := str(offer.get("country_code") or "").strip().upper():
|
||||
codes.add(code)
|
||||
if not found and offer.get("remote"):
|
||||
found.append({"text": "Remote", "remote": True})
|
||||
return found, codes
|
||||
return found
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
@@ -145,7 +111,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
salary_min, salary_max, currency = yearly_amount(
|
||||
salary.get("min"), salary.get("max"), salary.get("currency"),
|
||||
salary.get("period"), {"year": 1, "month": 12})
|
||||
places, codes = _places(j)
|
||||
places = _places(j)
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
@@ -160,8 +126,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
"country": country_if_single(codes),
|
||||
"seniority": _seniority(str(j.get("experience_code") or "")),
|
||||
"employment": _employment(str(j.get("employment_type_code") or "")),
|
||||
"seniority": str(j.get("experience_code") or "").strip() or None,
|
||||
"employment": str(j.get("employment_type_code") or "").strip() or None,
|
||||
})
|
||||
return rows
|
||||
@@ -12,7 +12,6 @@ from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
@@ -22,8 +21,6 @@ from desk.sources import strip_html
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Row, add_place, parse_dt, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
NAME = "rippling"
|
||||
LABEL = "Rippling"
|
||||
KEEP_CASE = True
|
||||
@@ -46,11 +43,6 @@ _SITE = "ats.rippling.com"
|
||||
#: whole; it is also what discovery's patterns take.
|
||||
_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
|
||||
|
||||
#: The words of `employmentType.label` (`SALARIED_FT`), split at "_".
|
||||
_EMPLOYMENT = {"FT": "full_time", "PT": "part_time", "INTERN": "internship",
|
||||
"INTERNSHIP": "internship", "CONTRACT": "contract", "CONTRACTOR": "contract",
|
||||
"TEMP": "temporary", "TEMPORARY": "temporary", "SEASONAL": "temporary"}
|
||||
|
||||
#: A role's id in the board's own links.
|
||||
_UUID = re.compile(r"[0-9A-Za-z-]{8,64}")
|
||||
|
||||
@@ -86,8 +78,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
role and place, so a role in three cities is three entries with one
|
||||
`uuid` (Rippling's own board: 638 entries, 326 roles); the rows merge
|
||||
them, the places kept as a list and joined with "; " for `location`. The entry has no description, date
|
||||
or pay: `detail` reads them. `remote` is True only when every place is a
|
||||
"Remote (...)" one, since a label says nothing else about the work."""
|
||||
or pay: `detail` reads them."""
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, list):
|
||||
raise ValueError("not a Rippling board: the answer is not a list")
|
||||
@@ -114,19 +105,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
for job_id, row in roles.items():
|
||||
row["places"] = places[job_id]
|
||||
row["location"] = "; ".join(p["text"] for p in places[job_id])
|
||||
row["remote"] = True if places[job_id] and all(
|
||||
p["remote"] for p in places[job_id]) else None
|
||||
return list(roles.values())
|
||||
|
||||
|
||||
def _employment(kind: Any) -> str | None:
|
||||
label = str((kind or {}).get("label") or "").strip().upper() if isinstance(kind, dict) else ""
|
||||
for word in label.split("_"):
|
||||
if word in _EMPLOYMENT:
|
||||
return _EMPLOYMENT[word]
|
||||
if label:
|
||||
log.debug("rippling: unknown employmentType %r", label)
|
||||
return None
|
||||
"""The host's `employmentType.label` (`SALARIED_FT`), as given."""
|
||||
return (str(kind.get("label") or "").strip() or None) if isinstance(kind, dict) else None
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
|
||||
@@ -13,15 +13,12 @@ ignores a remote parameter, so remote is not filtered.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
|
||||
NAME = "smartrecruiters"
|
||||
LABEL = "SmartRecruiters"
|
||||
@@ -41,17 +38,6 @@ READS = Reads(
|
||||
|
||||
_API = "api.smartrecruiters.com"
|
||||
|
||||
#: `experienceLevel.id` -> the app's levels. The three in the middle name no
|
||||
#: level ("associate" is half of Bosch's roles) and are known, so they map to
|
||||
#: none; an id not listed is logged at debug level.
|
||||
_SENIORITY = {"internship": "intern", "entry_level": "junior", "director": "director",
|
||||
"executive": "exec", "associate": "", "mid_senior_level": "",
|
||||
"not_applicable": ""}
|
||||
|
||||
#: `typeOfEmployment.id` -> the app's employment words.
|
||||
_EMPLOYMENT = {"permanent": "", "intern": "internship", "temporary": "temporary",
|
||||
"contract": "contract", "part_time": "part_time", "full_time": "full_time"}
|
||||
|
||||
#: The page size asked for, which is also the most the host serves per page.
|
||||
PAGE_SIZE = 100
|
||||
|
||||
@@ -94,10 +80,9 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _word(job: dict[str, Any], key: str, table: dict[str, str]) -> str | None:
|
||||
"""The `id` of the object `job[key]` looked up in `table`; none for a word
|
||||
the table maps to none, and a word it does not know is logged."""
|
||||
return table_word(table, (job.get(key) or {}).get("id"), log, key)
|
||||
def _word(job: dict[str, Any], key: str) -> str | None:
|
||||
"""The `id` of the object `job[key]`, as given."""
|
||||
return str((job.get(key) or {}).get("id") or "").strip() or None
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
@@ -122,7 +107,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
loc_text = ", ".join(part.strip() for part in (loc_text or "").split(",")
|
||||
if part.strip())
|
||||
place: Place = {"text": loc_text, "remote": location.get("remote")}
|
||||
country = str(location.get("country") or "").strip().upper()
|
||||
out.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
@@ -134,8 +118,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"description": "",
|
||||
"remote": location.get("remote"),
|
||||
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
|
||||
"country": country if len(country) == 2 else None,
|
||||
"seniority": _word(j, "experienceLevel", _SENIORITY),
|
||||
"employment": _word(j, "typeOfEmployment", _EMPLOYMENT),
|
||||
"country": str(location.get("country") or "").strip() or None,
|
||||
"seniority": _word(j, "experienceLevel"),
|
||||
"employment": _word(j, "typeOfEmployment"),
|
||||
})
|
||||
return out
|
||||
@@ -59,7 +59,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import NAMES
|
||||
from ._shared import Get, Page, Row, add_place, country_if_single, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "successfactors"
|
||||
LABEL = "SuccessFactors"
|
||||
@@ -86,7 +86,7 @@ KEEP_CASE = True
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", text="detail", complete="stated", places="many",
|
||||
filters=Filters(country="one"),
|
||||
fields=Fields(posted="trusted", country="stated"),
|
||||
fields=Fields(posted="trusted"),
|
||||
robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)",
|
||||
)
|
||||
|
||||
@@ -342,19 +342,6 @@ def _posted(value: str) -> datetime | None:
|
||||
return parse_dt(value)
|
||||
|
||||
|
||||
def _country_code(meta: dict[str, str]) -> str:
|
||||
"""The address's country as an ISO code: `addressCountry`, or the code that
|
||||
`_place` reads in a `streetAddress`; '' for neither."""
|
||||
code = meta.get("addressCountry", "").strip().upper()
|
||||
if len(code) == 2:
|
||||
return code
|
||||
parts = [part.strip() for part in meta.get("streetAddress", "").split(",")]
|
||||
for i in range(len(parts) - 1, 0, -1):
|
||||
if len(parts) - i <= 2 and parts[i] in _COUNTRIES:
|
||||
return parts[i]
|
||||
return ""
|
||||
|
||||
|
||||
def _location(meta: dict[str, str]) -> str:
|
||||
"""The address as "City, Country". Vestas and Demant state
|
||||
`addressLocality` and `addressCountry` (a code), the other sites one
|
||||
@@ -373,7 +360,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""A role's own page: `description` from the block marked
|
||||
`itemprop="description"`, which Novo Nordisk's pages have without the
|
||||
microdata around it, and, from a page that has the `JobPosting` microdata,
|
||||
`posted` (`datePosted`) and every place with the country's code. A page that
|
||||
`posted` (`datePosted`) and every place. A page that
|
||||
is not a role's (the site answers 200 for a closed one with its search) has
|
||||
no description block and gives an empty answer."""
|
||||
text = get(row["url"]).decode("utf-8", "replace")
|
||||
@@ -396,9 +383,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
if places:
|
||||
out["places"] = places
|
||||
out["location"] = "; ".join(p["text"] for p in places)
|
||||
codes = {_country_code(address) for address in addresses}
|
||||
if code := country_if_single(codes):
|
||||
out["country"] = code
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -51,12 +51,6 @@ _HOST = re.compile(r"[a-z0-9-]+(\.[a-z0-9-]+)*")
|
||||
#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`.
|
||||
_TT = "{https://teamtailor.com/locations}"
|
||||
|
||||
#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing
|
||||
#: about where the role is worked and `temporary` is remote for a spell only,
|
||||
#: so both stay unknown rather than become "onsite" or "remote".
|
||||
_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"}
|
||||
|
||||
|
||||
def site(slug: str) -> str:
|
||||
"""The host a career site is served from: `acme` is acme.teamtailor.com,
|
||||
and a slug with a dot is a site on the company's own domain,
|
||||
@@ -141,7 +135,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
"department": ((item.findtext(f"{_TT}department") or "").strip()
|
||||
or (item.findtext(f"{_TT}role") or "").strip()),
|
||||
"description": html.unescape(strip_html(item.findtext("description") or "")),
|
||||
"remote": _REMOTE.get(status),
|
||||
"remote": status or None,
|
||||
"salary_text": "", # Teamtailor's feed states no pay at all
|
||||
})
|
||||
return out
|
||||
+10
-21
@@ -8,14 +8,11 @@ board with roles was seen; jobhunt-hq did not read Workable.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Page, Row, add_place, country_if_single, only, parse_dt, path_slug_ok, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok
|
||||
|
||||
NAME = "workable"
|
||||
LABEL = "Workable"
|
||||
@@ -28,17 +25,12 @@ PATTERNS = (
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=10, text="none", complete="stated",
|
||||
places="many",
|
||||
fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"),
|
||||
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
||||
)
|
||||
|
||||
_API = "apply.workable.com"
|
||||
|
||||
#: `type` -> the app's employment words; a type not listed is logged at debug
|
||||
#: level so the table can grow.
|
||||
_EMPLOYMENT = {"full": "full_time", "part": "part_time", "contract": "contract",
|
||||
"temporary": "temporary", "internship": "internship"}
|
||||
|
||||
# No probe: Workable's API sits behind a rate limit that a run of guesses
|
||||
# trips, and a Workable site links its board plainly anyway.
|
||||
|
||||
@@ -67,13 +59,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
|
||||
"""Every place of a role and the countries' codes among them. `locations`
|
||||
def _places(job: dict[str, Any], remote: bool) -> list[Place]:
|
||||
"""Every place of a role. `locations`
|
||||
lists them (city, region, country; `hidden` ones are stated too), and
|
||||
`location` is the first alone, the fallback for a role that lists none.
|
||||
A place is remote when the role is."""
|
||||
found: list[Place] = []
|
||||
codes: set[str] = set()
|
||||
listed = [loc for loc in job.get("locations") or [] if isinstance(loc, dict)]
|
||||
for loc in listed or [job.get("location") or {}]:
|
||||
parts: list[str] = []
|
||||
@@ -83,9 +74,7 @@ def _places(job: dict[str, Any], remote: bool) -> tuple[list[Place], set[str]]:
|
||||
parts.append(part)
|
||||
text = ", ".join(parts)
|
||||
add_place(found, {"text": text, "remote": remote})
|
||||
if code := str(loc.get("countryCode") or "").strip().upper():
|
||||
codes.add(code)
|
||||
return found, codes
|
||||
return found
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
@@ -104,8 +93,9 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
shortcode = (j.get("shortcode") or "").strip()
|
||||
if not title or not shortcode:
|
||||
continue
|
||||
remote = bool(j.get("remote")) or j.get("workplace") == "remote"
|
||||
places, codes = _places(j, remote)
|
||||
workplace = str(j.get("workplace") or "").strip()
|
||||
remote = bool(j.get("remote")) or workplace == "remote"
|
||||
places = _places(j, remote)
|
||||
dept = j.get("department") or ""
|
||||
if isinstance(dept, list):
|
||||
dept = ", ".join(d for d in dept if d)
|
||||
@@ -118,9 +108,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"posted": parse_dt(j.get("published")),
|
||||
"department": dept.strip(),
|
||||
"description": "",
|
||||
"remote": remote,
|
||||
"remote": workplace or j.get("remote"),
|
||||
"salary_text": "", # Workable's list endpoint states no pay at all
|
||||
"country": country_if_single(codes),
|
||||
"employment": table_word(_EMPLOYMENT, j.get("type"), log, "type"),
|
||||
"employment": str(j.get("type") or "").strip() or None,
|
||||
})
|
||||
return out
|
||||
+18
-30
@@ -29,7 +29,6 @@ from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -39,9 +38,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import code_of
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt, table_word
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt
|
||||
|
||||
NAME = "workday"
|
||||
LABEL = "Workday"
|
||||
@@ -50,7 +47,7 @@ KEEP_CASE = True
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
|
||||
complete="stated", places="many", filters=Filters(country="ids", remote="ids"),
|
||||
fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"),
|
||||
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
||||
robots="read: robots.txt allows /<Site>/ (checked 2026-09-29)",
|
||||
)
|
||||
|
||||
@@ -167,7 +164,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
|
||||
"""The tenant's host ids, read off one page of one role: `country` maps
|
||||
an ISO code to `<facetParameter>=<id>` in the facet whose values are
|
||||
countries (most of them, `_COUNTRY_SHARE`; the one with most countries
|
||||
when two are), and `remote` maps each facet value that `_REMOTE` reads as
|
||||
when two are), and `remote` maps each facet value that `_REMOTE_FACET` names as
|
||||
remote ("Remote", "Fully Flexible"), in whatever facet, to its id. A
|
||||
tenant with neither states no such dimension, and the runner then does
|
||||
not filter on it."""
|
||||
@@ -184,7 +181,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
|
||||
country = found
|
||||
for v in values:
|
||||
word = " ".join(str(v.get("descriptor") or "").split())
|
||||
if v.get("id") and _remote(word) == "remote":
|
||||
if v.get("id") and word.lower() in _REMOTE_FACET:
|
||||
remote[word.lower()] = f"{name}={v['id']}"
|
||||
out: dict[str, dict[str, str]] = {}
|
||||
if country:
|
||||
@@ -216,11 +213,14 @@ def _today() -> datetime:
|
||||
return datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
#: `remoteType` -> `store._remote`'s words. Only some tenants send it, and
|
||||
#: each words it its own way (Zendesk's "Fully Flexible", Axis' "On-Site"); a
|
||||
#: word not listed, a bare "No" among them, says nothing and stays unknown.
|
||||
_REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid",
|
||||
"in office": "onsite", "on-site": "onsite", "onsite": "onsite"}
|
||||
#: The `remoteType` words a facet names that mean remote, which the `remote`
|
||||
#: filter looks for; the row keeps the host's word whatever it is (only some
|
||||
#: tenants send it, each in its own way: Zendesk's "Fully Flexible", Axis'
|
||||
#: "On-Site").
|
||||
_REMOTE_FACET = ("remote", "fully flexible")
|
||||
|
||||
#: The words that put a role's places in an office, for their `remote` flag.
|
||||
_OFFICE = ("hybrid", "in office", "on-site", "onsite")
|
||||
|
||||
|
||||
def _requisition(posting: dict[str, Any], path: str) -> str:
|
||||
@@ -234,12 +234,10 @@ def _requisition(posting: dict[str, Any], path: str) -> str:
|
||||
|
||||
|
||||
def _remote(value: Any) -> str | None:
|
||||
return _REMOTE.get(str(value or "").strip().lower())
|
||||
"""The host's `remoteType`, as given."""
|
||||
return str(value or "").strip() or None
|
||||
|
||||
|
||||
#: `timeType` -> the app's employment words.
|
||||
_EMPLOYMENT = {"full time": "full_time", "part time": "part_time"}
|
||||
|
||||
#: What `locationsText` says of a role at several places: "2 Locations". The
|
||||
#: places arrive with the role's own read.
|
||||
_COUNT = re.compile(r"\d+ locations?", re.I)
|
||||
@@ -259,8 +257,9 @@ def _place(text: Any, remote: str | None) -> Place | None:
|
||||
if len(parts) > 1 and "," not in text and all(parts) \
|
||||
and not parts[0].lower().startswith(("remote", "hybrid", "on-site", "onsite")):
|
||||
text = ", ".join(reversed(parts))
|
||||
return {"text": text, "remote": {"remote": True, "hybrid": False, "onsite": False}.get(
|
||||
remote or "")}
|
||||
word = (remote or "").lower()
|
||||
return {"text": text, "remote": True if word in _REMOTE_FACET
|
||||
else False if word in _OFFICE else None}
|
||||
|
||||
|
||||
def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
||||
@@ -318,9 +317,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
if places := _places(posting, remote):
|
||||
out["places"] = places
|
||||
out["location"] = "; ".join(p["text"] for p in places)
|
||||
if len(places) == 1 and (code := _country_code(posting)):
|
||||
out["country"] = code
|
||||
if employment := table_word(_EMPLOYMENT, posting.get("timeType"), log, "timeType"):
|
||||
if employment := str(posting.get("timeType") or "").strip():
|
||||
out["employment"] = employment
|
||||
if posted := parse_dt(posting.get("startDate")):
|
||||
out["posted"] = posted
|
||||
@@ -329,15 +326,6 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
return out
|
||||
|
||||
|
||||
def _country_code(posting: dict[str, Any]) -> str | None:
|
||||
"""The role's country as an ISO code, from the requisition's location. The
|
||||
code belongs to the main location only, so it is used for a role with one
|
||||
place."""
|
||||
country = (posting.get("jobRequisitionLocation") or {}).get("country") or {}
|
||||
code = str(country.get("alpha2Code") or "").strip().upper()
|
||||
return code if len(code) == 2 else None
|
||||
|
||||
|
||||
def _places(posting: dict[str, Any], remote: str | None) -> list[Place]:
|
||||
"""The main location and the additional ones, each once, which replace the
|
||||
listing's "2 Locations". A single location that does not name its country
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
"""`upsert`: a role stored as a pull stores it and then mapped, as the prep
|
||||
stage after the pull maps it. For tests of what the feed, the gates and the
|
||||
pages read, which are prep's values, not the raw ones `upsert_role` keeps."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app import prep
|
||||
from app.storage import Store
|
||||
|
||||
|
||||
def upsert(store: Store, item: Any, subscription: Any = None) -> bool:
|
||||
new = store.upsert_role(item, subscription)
|
||||
prep.run(store, dupes=False)
|
||||
return new
|
||||
@@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
#: no database, so storage may use them. `main` assembles the app from
|
||||
#: everything below it; `cli` and `__main__` are the entry points and may
|
||||
#: reach anything.
|
||||
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli",
|
||||
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "prep", "jobs", "web", "main", "cli",
|
||||
"__main__"]
|
||||
APP = "app"
|
||||
EXTERNAL = {"desk", "pipeline", "auth"}
|
||||
|
||||
+7
-6
@@ -132,10 +132,10 @@ def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
|
||||
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
|
||||
assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/"
|
||||
assert rows[0]["location"] == "France"
|
||||
assert rows[0]["remote"] is True
|
||||
assert rows[0]["remote"] == "remote"
|
||||
assert rows[0]["department"] == "Product"
|
||||
assert rows[1]["location"] == "Berlin, Germany"
|
||||
assert rows[1]["remote"] is False
|
||||
assert rows[1]["remote"] == "hybrid"
|
||||
assert rows[0]["description"] == "" # the list endpoint carries no text
|
||||
assert rows[0]["salary_text"] == "" # Workable's list endpoint states no pay at all
|
||||
|
||||
@@ -173,8 +173,9 @@ def test_personio_builds_the_job_url_and_reads_cdata_description() -> None:
|
||||
assert [r["source_id"] for r in rows] == ["6001", "6002", "6003"]
|
||||
assert rows[0]["url"] == "https://acme.jobs.personio.de/job/6001"
|
||||
assert rows[0]["description"] == "Grow our pipeline."
|
||||
assert rows[0]["remote"] is True # "Remote - Germany"
|
||||
assert rows[1]["remote"] is None # "Munich"
|
||||
assert rows[0]["places"][0]["remote"] is True # "Remote - Germany"
|
||||
assert rows[1]["places"][0]["remote"] is None # "Munich"
|
||||
assert "remote" not in rows[0] # the feed states none
|
||||
|
||||
|
||||
def test_personio_reads_salary_information_when_present() -> None:
|
||||
@@ -291,9 +292,9 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
|
||||
assert country_of(rows[0]["location"]) == "SE"
|
||||
|
||||
|
||||
def test_teamtailor_maps_remote_status_and_falls_back_to_role_for_department() -> None:
|
||||
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||
assert [r["remote"] for r in rows] == ["hybrid", "remote", None] # none says nothing
|
||||
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
|
||||
assert rows[1]["title"] == "Läkare" # trimmed
|
||||
assert rows[1]["department"] == "Software Development"
|
||||
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
||||
|
||||
@@ -25,14 +25,14 @@ def test_the_primary_and_every_secondary_location_are_places() -> None:
|
||||
assert second["places"] == [{"text": "Dublin", "remote": True}]
|
||||
|
||||
|
||||
def test_a_hybrid_role_is_not_remote_although_the_host_flags_it() -> None:
|
||||
def test_remote_is_the_workplace_type_as_the_host_writes_it() -> None:
|
||||
first, second = ashby.parse(BOARD, "acme")
|
||||
assert first["remote"] == "hybrid" and second["remote"] == "remote"
|
||||
assert first["remote"] == "Hybrid" and second["remote"] == "Remote"
|
||||
|
||||
|
||||
def test_the_employment_type_is_mapped() -> None:
|
||||
def test_the_employment_type_is_the_hosts_word() -> None:
|
||||
first, second = ashby.parse(BOARD, "acme")
|
||||
assert (first["employment"], second["employment"]) == ("full_time", "contract")
|
||||
assert (first["employment"], second["employment"]) == ("FullTime", "Contract")
|
||||
|
||||
|
||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
|
||||
|
||||
@@ -47,15 +47,15 @@ def test_a_listing_becomes_rows() -> None:
|
||||
assert second["url"] == "https://configit.bamboohr.com/careers/37"
|
||||
assert second["department"] == "Marketing" and rows[0]["department"] == ""
|
||||
assert second["location"] == "København Ø"
|
||||
assert second["remote"] == "hybrid"
|
||||
assert second["remote"] == "2"
|
||||
assert (second["posted"], second["description"], second["salary_text"]) == (None, "", "")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("flag", "kind", "remote"), [
|
||||
(True, "0", "remote"), (None, "0", "onsite"), (None, "1", "remote"), (None, "2", "hybrid"),
|
||||
(None, None, None), (False, "7", None)])
|
||||
(True, "0", True), (None, "0", "0"), (None, "1", "1"), (None, "2", "2"),
|
||||
(None, None, None), (False, "7", "7")])
|
||||
def test_remote_is_the_flag_then_the_location_type(
|
||||
flag: bool | None, kind: str | None, remote: str | None) -> None:
|
||||
flag: bool | None, kind: str | None, remote: bool | str | None) -> None:
|
||||
body = json.loads(_read("bamboohr.json"))
|
||||
body["result"][0] |= {"isRemote": flag, "locationType": kind}
|
||||
assert bamboohr.parse(json.dumps(body).encode(), "configit")[0]["remote"] == remote
|
||||
@@ -102,7 +102,7 @@ def test_detail_reads_the_text_the_date_and_the_country() -> None:
|
||||
assert found["posted"] == datetime(2026, 9, 23, tzinfo=UTC)
|
||||
assert found["location"] == "København Ø, Denmark"
|
||||
assert found["places"] == [{"text": "København Ø, Denmark", "remote": None}]
|
||||
assert found["remote"] == "hybrid"
|
||||
assert found["remote"] == "2"
|
||||
text = found["description"]
|
||||
assert text.startswith("Our Global Marketing Team is looking for")
|
||||
assert "Skills & Experience" in text and "&" not in text and "<" not in text
|
||||
@@ -144,11 +144,11 @@ def test_the_board_and_its_details_are_read_with_the_json_accept_on_the_detail()
|
||||
def test_the_role_states_one_place_and_the_employment() -> None:
|
||||
rows = bamboohr.parse(_read("bamboohr.json"), "configit")
|
||||
assert rows[1]["places"] == [{"text": "København Ø", "remote": None}]
|
||||
assert rows[1]["employment"] == "full_time"
|
||||
assert rows[1]["employment"] == "Full-Time"
|
||||
body = json.loads(_read("bamboohr.json"))
|
||||
body["result"][0] |= {"employmentStatusLabel": "Seasonal", "location": {"city": None}}
|
||||
first = bamboohr.parse(json.dumps(body).encode(), "configit")[0]
|
||||
assert first["places"] == [] and "employment" not in first
|
||||
assert first["places"] == [] and first["employment"] == "Seasonal"
|
||||
|
||||
|
||||
def test_it_declares_what_it_reads() -> None:
|
||||
|
||||
@@ -32,7 +32,7 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None:
|
||||
assert first["url"] == ("https://jobs.deel.com/klarna/job-details/"
|
||||
"21584046-56d5-44c4-8b18-7e15dc659827/overview")
|
||||
assert first["location"] == "Stockholm" and first["department"] == "Engineering"
|
||||
assert first["remote"] == "onsite"
|
||||
assert first["remote"] == "ON_SITE"
|
||||
assert first["description"].startswith("Klarna, briefly At Klarna")
|
||||
assert "<" not in first["description"] and "$" not in first["description"][:3]
|
||||
assert all(r["description"] for r in rows)
|
||||
@@ -41,18 +41,18 @@ def test_rows_come_from_the_payload_with_descriptions_resolved() -> None:
|
||||
"2026-09-07T12:33:38.484000+00:00"
|
||||
|
||||
|
||||
def test_several_locations_are_joined_and_remote_uses_the_stores_words() -> None:
|
||||
def test_several_locations_are_joined_and_remote_is_the_hosts_word() -> None:
|
||||
rows = {r["source_id"][:8]: r for r in deel.parse(PAGE, "klarna")}
|
||||
assert rows["a8be45ac"]["location"] == "Stockholm; London; Milan; Warsaw"
|
||||
assert rows["f96bb9d4"]["remote"] == "remote"
|
||||
assert rows["f96bb9d4"]["remote"] == "REMOTE"
|
||||
assert rows["f96bb9d4"]["location"] == "Ohio"
|
||||
# Every place is kept, in the host's order; the work arrangement is the
|
||||
# places' remote flag.
|
||||
assert rows["a8be45ac"]["places"] == [
|
||||
{"text": name, "remote": False} for name in ("Stockholm", "London", "Milan", "Warsaw")]
|
||||
assert rows["f96bb9d4"]["places"] == [{"text": "Ohio", "remote": True}]
|
||||
assert rows["f96bb9d4"]["employment"] == "contract" # "Consultant"
|
||||
assert rows["21584046"]["employment"] == "full_time"
|
||||
assert rows["f96bb9d4"]["employment"] == "Consultant"
|
||||
assert rows["21584046"]["employment"] == "Full-time"
|
||||
|
||||
|
||||
def test_pay_is_only_text_since_the_period_is_not_stated() -> None:
|
||||
|
||||
@@ -87,8 +87,8 @@ def test_a_page_becomes_rows() -> None:
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("option", "remote"), [
|
||||
("onsite", "onsite"), ("hybrid", "hybrid"), ("remote", "remote"), (None, None), ("odd", None)])
|
||||
def test_the_work_location_option_maps_to_the_remote_words(
|
||||
("onsite", "onsite"), ("Hybrid", "Hybrid"), (None, None), ("odd", "odd"), (" ", None)])
|
||||
def test_the_work_location_option_is_the_hosts_word(
|
||||
option: str | None, remote: str | None) -> None:
|
||||
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ def test_a_listing_becomes_rows() -> None:
|
||||
assert first["source_id"] == "8080046002" # numeric, from a synced Greenhouse
|
||||
assert first["title"] == "Staff AI Product Engineer (Prague/Brno/Remote EU)"
|
||||
assert first["url"] == "https://jobs.gem.com/productboard/8080046002"
|
||||
assert first["department"] == "Product Engineering" and first["remote"] == "hybrid"
|
||||
assert first["department"] == "Product Engineering" and first["remote"] == "HYBRID"
|
||||
assert first["location"] == "Prague"
|
||||
assert (first["posted"], first["description"], first["salary_text"]) == (None, "", "")
|
||||
assert backend["source_id"] == "am9icG9zdDqWC1V_RM7U6ORvTd1Iu64r" # opaque
|
||||
@@ -64,16 +64,16 @@ def test_a_listing_becomes_rows() -> None:
|
||||
assert backend["places"] == [{"text": "Prague", "remote": False},
|
||||
{"text": "Remote Europe", "remote": True},
|
||||
{"text": "Brno", "remote": False}]
|
||||
assert first["employment"] == "full_time"
|
||||
assert first["employment"] == "FULL_TIME"
|
||||
assert len({r["source_id"] for r in rows}) == 11
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("kind", "flags", "remote"), [
|
||||
("IN_OFFICE", [False], "onsite"), ("HYBRID", [True], "hybrid"), ("REMOTE", [False], "remote"),
|
||||
(None, [True, True], "remote"), (None, [True, False], None), (None, [], None),
|
||||
("SOMETHING_NEW", [False], None)])
|
||||
("IN_OFFICE", [False], "IN_OFFICE"), ("HYBRID", [True], "HYBRID"), ("REMOTE", [False], "REMOTE"),
|
||||
(None, [True, True], True), (None, [True, False], None), (None, [], None),
|
||||
("SOMETHING_NEW", [False], "SOMETHING_NEW")])
|
||||
def test_remote_is_the_location_type_then_the_locations(
|
||||
kind: str | None, flags: list[bool], remote: str | None) -> None:
|
||||
kind: str | None, flags: list[bool], remote: bool | str | None) -> None:
|
||||
posting = {"extId": "1", "title": "Role", "job": {"locationType": kind},
|
||||
"locations": [{"city": "X", "isRemote": f} for f in flags]}
|
||||
assert gem.rows([posting], "acme")[0]["remote"] == remote
|
||||
|
||||
@@ -51,7 +51,7 @@ def test_a_board_of_general_applications_reads_as_it_is() -> None:
|
||||
rows = hibob.parse(DIXA, "dixa")
|
||||
assert [(r["title"], r["location"], r["remote"]) for r in rows] == [
|
||||
("Expression of interest", "Copenhagen, Denmark", "hybrid"),
|
||||
("Expression of interest", "London, United Kingdom", "onsite")]
|
||||
("Expression of interest", "London, United Kingdom", "on_site")]
|
||||
assert rows[1]["description"].startswith("We're always excited")
|
||||
|
||||
|
||||
@@ -153,10 +153,10 @@ def test_each_ad_names_one_site_and_the_host_lists_a_copy_per_site() -> None:
|
||||
assert len({r["source_id"] for r in dixa}) == 2
|
||||
|
||||
|
||||
def test_the_employment_type_is_mapped_and_a_permanent_one_is_left_unmapped() -> None:
|
||||
def test_the_employment_type_is_the_hosts_word() -> None:
|
||||
rows = hibob.parse(BOARD, "liquidia")
|
||||
assert rows[0]["employment"] == "full_time" # "Regular Full Time"
|
||||
assert "employment" not in hibob.parse(DIXA, "dixa")[0] # "Permanent"
|
||||
assert rows[0]["employment"] == "Regular Full Time"
|
||||
assert hibob.parse(DIXA, "dixa")[0]["employment"] == "Permanent"
|
||||
|
||||
|
||||
def test_it_declares_copies_with_a_page_and_a_robots_decision() -> None:
|
||||
|
||||
@@ -57,7 +57,7 @@ def test_a_page_becomes_rows() -> None:
|
||||
"Manchester, United Kingdom"]
|
||||
assert "country" not in several
|
||||
assert first["places"] == [{"text": "Bucharest, Romania", "remote": None}]
|
||||
assert (first["country"], first["employment"]) == ("RO", "full_time")
|
||||
assert (first["country"], first["employment"]) == ("RO", "FULL_TIME")
|
||||
assert all(r["description"] and "<" not in r["description"] and " " not in r["description"]
|
||||
for r in rows)
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models as app_models # noqa: E402
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, jobylon # noqa: E402
|
||||
@@ -36,20 +35,20 @@ def test_rows_come_from_the_embeds_literal() -> None:
|
||||
assert first["location"] == "Stockholm"
|
||||
assert first["places"] == [{"text": "Stockholm", "remote": None}]
|
||||
assert first["department"] == "Accounting & Finance" # `&` read as `&`
|
||||
assert first["remote"] == "hybrid"
|
||||
assert first["remote"] == "Hybrid"
|
||||
assert first["description"] == "" and first["salary_text"] == ""
|
||||
# A day, with no time of day: the embed says "Sept. 25, 2026".
|
||||
assert first["posted"] is not None and first["posted"].isoformat() == \
|
||||
"2026-09-25T00:00:00+00:00"
|
||||
|
||||
|
||||
def test_places_department_and_remote_follow_the_embeds_words() -> None:
|
||||
def test_places_department_and_remote_are_the_embeds_words() -> None:
|
||||
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
||||
assert rows["384238"]["location"] == "Houston; Dallas; Atlanta"
|
||||
assert [p["text"] for p in rows["384238"]["places"]] == ["Houston", "Dallas", "Atlanta"]
|
||||
assert rows["384238"]["department"] == "" # "All functions" names none
|
||||
assert rows["382124"]["remote"] == "remote"
|
||||
assert rows["382860"]["remote"] == "onsite"
|
||||
assert rows["382124"]["remote"] == "Remote"
|
||||
assert rows["382860"]["remote"] == "On-site"
|
||||
assert rows["380347"]["remote"] is None # a role with no workspace
|
||||
|
||||
|
||||
@@ -130,18 +129,14 @@ def test_an_empty_board_is_no_roles() -> None:
|
||||
assert jobylon.parse(EMPTY, "1056") == []
|
||||
|
||||
|
||||
def test_the_experience_is_mapped_and_a_word_the_table_lacks_is_logged(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level("DEBUG", logger="plugins.readers.jobylon")
|
||||
def test_the_experience_and_the_employment_type_are_the_hosts_words() -> None:
|
||||
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
||||
assert rows["384238"]["seniority"] == "mid" # "Mid level"
|
||||
assert "seniority" not in rows["385191"] # "Experienced" says too little
|
||||
assert "unknown experience 'Experienced'" in caplog.text
|
||||
assert rows["385191"]["employment"] == "full_time"
|
||||
assert all(r.get("seniority", "mid") in app_models.SENIORITIES for r in rows.values())
|
||||
assert rows["384238"]["seniority"] == "Mid level"
|
||||
assert rows["385191"]["seniority"] == "Experienced"
|
||||
assert rows["385191"]["employment"] == "Full-time"
|
||||
rows = {r["source_id"]: r for r in jobylon.parse(
|
||||
(FIXTURES / "jobylon-places.html").read_bytes(), "3143")}
|
||||
assert rows["502"]["seniority"] == "intern" and rows["502"]["employment"] == "internship"
|
||||
assert rows["502"]["seniority"] == "Student" and rows["502"]["employment"] == "Internship"
|
||||
|
||||
|
||||
def test_a_summary_is_the_teaser_unless_the_embed_says_none() -> None:
|
||||
|
||||
@@ -28,9 +28,9 @@ def test_the_country_is_only_taken_for_a_role_at_one_place() -> None:
|
||||
assert "country" not in several and one["country"] == "DE"
|
||||
|
||||
|
||||
def test_the_commitment_is_mapped_and_a_permanent_one_is_left_unmapped() -> None:
|
||||
def test_the_commitment_is_the_hosts_word() -> None:
|
||||
several, one = lever.parse(BOARD, "acme")
|
||||
assert several["employment"] == "full_time" and "employment" not in one
|
||||
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
|
||||
|
||||
|
||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
|
||||
|
||||
@@ -59,19 +59,19 @@ def test_the_locations_are_the_primary_and_the_secondary_ones() -> None:
|
||||
assert rows[2]["location"] == "India"
|
||||
|
||||
|
||||
def test_every_place_is_kept_with_its_work_mode_and_the_countrys_code() -> None:
|
||||
def test_every_place_is_kept_with_its_work_mode() -> None:
|
||||
rows = _rows()
|
||||
assert [p["text"] for p in rows[1]["places"]] == [
|
||||
"Malaysia", "Philippines", "Indonesia", "Thailand"]
|
||||
assert [p["remote"] for p in rows[0]["places"]] == [False, False] # hybrid
|
||||
assert [p["remote"] for p in rows[4]["places"]] == [True] # remote
|
||||
assert [p["remote"] for p in rows[3]["places"]] == [None] # states none
|
||||
# One country when the places share one, none over several.
|
||||
assert [r["country"] for r in rows] == [None, None, "IN", "DE", "US"]
|
||||
assert all("country" not in r for r in rows) # prep derives it from the places
|
||||
|
||||
|
||||
def test_remote_follows_the_workplace_type_code() -> None:
|
||||
assert [r["remote"] for r in _rows()] == ["hybrid", "hybrid", "onsite", None, "remote"]
|
||||
def test_remote_is_the_workplace_type_code_as_given() -> None:
|
||||
assert [r["remote"] for r in _rows()] == [
|
||||
"ORA_HYBRID", "ORA_HYBRID", "ORA_ON_SITE", None, "ORA_REMOTE"]
|
||||
|
||||
|
||||
def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
|
||||
|
||||
@@ -3,17 +3,14 @@ No network: the fixtures are saved feeds."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.models import SENIORITIES # noqa: E402
|
||||
from plugins.readers import Ask, personio # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -37,23 +34,12 @@ def test_additional_offices_are_places_of_the_role_each_once() -> None:
|
||||
assert [p["text"] for p in first["places"]] == ["Remote - Germany", "Berlin", "Hamburg"]
|
||||
assert [p["remote"] for p in first["places"]] == [True, None, None]
|
||||
assert first["location"] == "Remote - Germany; Berlin; Hamburg"
|
||||
assert first["remote"] is True # one of the places is remote
|
||||
assert "remote" not in first # the feed states none
|
||||
assert _rows()[1]["places"] == [{"text": "Munich", "remote": None}]
|
||||
assert _rows()[1]["remote"] is None
|
||||
|
||||
|
||||
def test_the_stated_level_and_employment_are_mapped() -> None:
|
||||
def test_the_stated_level_and_employment_are_the_hosts_words() -> None:
|
||||
rows = _rows()
|
||||
assert [r["seniority"] for r in rows] == ["junior", "exec", "intern"]
|
||||
# permanent + full-time; permanent + full-time; working student wins over part-time
|
||||
assert [r["employment"] for r in rows] == ["full_time", "full_time", "internship"]
|
||||
|
||||
|
||||
def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
assert {v for v in personio._SENIORITY.values() if v} <= set(SENIORITIES)
|
||||
with caplog.at_level(logging.DEBUG, logger=personio.__name__):
|
||||
assert personio._seniority("wizard") is None
|
||||
assert personio._seniority("experienced") is None # known, names no level
|
||||
assert personio._seniority("") is None
|
||||
assert [r.getMessage() for r in caplog.records] == ["personio: unknown seniority 'wizard'"]
|
||||
assert [r["seniority"] for r in rows] == ["entry-level", "executive", "student"]
|
||||
# permanent + full-time gives the schedule; a working student is the contract's word
|
||||
assert [r["employment"] for r in rows] == ["full-time", "full-time", "working_student"]
|
||||
@@ -226,7 +226,7 @@ def test_detail_falls_back_to_the_page_state_where_there_is_no_job_posting() ->
|
||||
got = phenom.detail(_get_of("phenom-detail-3.html", []), SLUG, row)
|
||||
assert got["description"].startswith("Craft: Games Design & Production Job Description:")
|
||||
assert "<" not in got["description"] and "&" not in got["description"]
|
||||
assert got["remote"] == "onsite"
|
||||
assert got["remote"] == "On-site"
|
||||
assert "posted" not in got and set(got) <= set(ats.DETAIL_FIELDS)
|
||||
|
||||
|
||||
|
||||
@@ -161,9 +161,6 @@ def test_a_role_has_its_one_place_with_the_work_mode_of_the_role() -> None:
|
||||
assert rows[0]["location"] == "Bucharest"
|
||||
|
||||
|
||||
def test_employment_is_read_from_the_type_and_an_unknown_one_is_left() -> None:
|
||||
def test_employment_is_the_hosts_type_as_given() -> None:
|
||||
rows = pinpoint.parse((FIXTURES / "pinpoint.json").read_bytes(), "xeneta")
|
||||
assert [r["employment"] for r in rows] == ["full_time", "temporary"]
|
||||
assert pinpoint._employment("permanent_part_time") == "part_time"
|
||||
assert pinpoint._employment("internship") == "internship"
|
||||
assert pinpoint._employment("something_new") is None
|
||||
assert [r["employment"] for r in rows] == ["permanent_full_time", "fixed_term_full_time"]
|
||||
@@ -4,17 +4,14 @@ bunq's board, read on 2026-09-30."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.models import SENIORITIES # noqa: E402
|
||||
from plugins.readers import Ask, recruitee # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -27,19 +24,18 @@ def _rows() -> list[dict]:
|
||||
def test_the_declaration_says_what_the_feed_gives() -> None:
|
||||
reads = recruitee.READS
|
||||
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.country == "stated"
|
||||
assert reads.fields.posted == "trusted" and reads.fields.country == "none"
|
||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
||||
assert recruitee.feed_url("acme", Ask()) == "https://acme.recruitee.com/api/offers/"
|
||||
|
||||
|
||||
def test_every_location_is_a_place_and_the_country_is_stated_when_they_share_one() -> None:
|
||||
def test_every_location_is_a_place_and_no_country_is_stated() -> None:
|
||||
first, _remote, berlin = _rows()
|
||||
assert [p["text"] for p in first["places"]] == [
|
||||
"Amsterdam, Noord-Holland, Netherlands", "Sofia, Bulgaria"]
|
||||
assert first["location"] == "Amsterdam, Noord-Holland, Netherlands; Sofia, Bulgaria"
|
||||
assert first["country"] is None # NL and BG
|
||||
assert "country" not in first # prep derives it from the places
|
||||
assert berlin["places"] == [{"text": "Berlin, Germany", "remote": None}]
|
||||
assert berlin["country"] == "DE"
|
||||
|
||||
|
||||
def test_remote_job_is_not_a_place() -> None:
|
||||
@@ -48,28 +44,16 @@ def test_remote_job_is_not_a_place() -> None:
|
||||
assert remote["location"] == "Remote"
|
||||
offer = {"id": 1, "title": "T", "careers_url": "https://a.example/o/t", "location": "Remote job",
|
||||
"remote": False}
|
||||
assert recruitee._places(offer) == ([], set())
|
||||
assert recruitee._places(offer) == []
|
||||
|
||||
|
||||
def test_an_offer_without_a_locations_list_falls_back_to_its_location() -> None:
|
||||
offer = {"location": "Lisbon, Portugal", "country_code": "pt"}
|
||||
assert recruitee._places(offer) == ([{"text": "Lisbon, Portugal", "remote": None}], {"PT"})
|
||||
assert recruitee._places(offer) == [{"text": "Lisbon, Portugal", "remote": None}]
|
||||
|
||||
|
||||
def test_the_stated_level_and_employment_are_mapped() -> None:
|
||||
def test_the_stated_level_and_employment_are_the_hosts_codes() -> None:
|
||||
rows = _rows()
|
||||
assert [r["seniority"] for r in rows] == [None, "junior", "intern"]
|
||||
assert [r["employment"] for r in rows] == ["full_time", "part_time", "internship"]
|
||||
assert recruitee._employment("fulltime_fixed_term") == "temporary"
|
||||
assert recruitee._employment("freelance") == "contract"
|
||||
|
||||
|
||||
def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
assert {v for v in recruitee._SENIORITY.values() if v} <= set(SENIORITIES)
|
||||
with caplog.at_level(logging.DEBUG, logger=recruitee.__name__):
|
||||
assert recruitee._seniority("senior_level") is None
|
||||
assert recruitee._employment("odd_job") is None
|
||||
assert [r.getMessage() for r in caplog.records] == [
|
||||
"recruitee: unknown experience_code 'senior_level'",
|
||||
"recruitee: unknown employment_type_code 'odd_job'"]
|
||||
assert [r["seniority"] for r in rows] == ["experienced", "entry_level", "student_school"]
|
||||
assert [r["employment"] for r in rows] == [
|
||||
"fulltime_permanent", "parttime_permanent", "internship"]
|
||||
+10
-14
@@ -41,21 +41,18 @@ def test_rows_merge_one_entry_per_place_into_one_role() -> None:
|
||||
assert first["description"] == "" and first["salary_text"] == "" and first["posted"] is None
|
||||
|
||||
|
||||
def test_remote_is_true_only_when_every_place_is_a_remote_one() -> None:
|
||||
def test_a_remote_place_is_flagged_and_the_role_states_no_remote() -> None:
|
||||
by_id = {r["source_id"][:8]: r for r in rippling.parse(BOARD, "rippling")}
|
||||
assert by_id["46b4af52"]["remote"] is True and by_id["19ae5b34"]["remote"] is True
|
||||
assert by_id["75ad50c6"]["remote"] is None # an office label says nothing either way
|
||||
mixed = json.dumps([
|
||||
{"uuid": "abcdefgh-1", "name": "A", "workLocation": {"label": "Remote (Germany)"}},
|
||||
{"uuid": "abcdefgh-1", "name": "A", "workLocation": {"label": "Berlin"}}]).encode()
|
||||
assert rippling.parse(mixed, "x")[0]["remote"] is None
|
||||
assert [p["remote"] for p in by_id["46b4af52"]["places"]] == [True]
|
||||
assert [p["remote"] for p in by_id["75ad50c6"]["places"]] == [None] # an office label
|
||||
assert all("remote" not in r for r in by_id.values()) # prep derives it from the places
|
||||
|
||||
|
||||
def test_prisma_board_reads_two_roles() -> None:
|
||||
rows = rippling.parse(PRISMA, "prisma-careers")
|
||||
assert [(r["title"], r["department"], r["location"], r["remote"]) for r in rows] == [
|
||||
("General Applications: Engineering", "Engineering", "Remote (Germany)", True),
|
||||
("General Applications: Product", "Product", "Remote (Germany)", True)]
|
||||
assert [(r["title"], r["department"], r["location"]) for r in rows] == [
|
||||
("General Applications: Engineering", "Engineering", "Remote (Germany)"),
|
||||
("General Applications: Product", "Product", "Remote (Germany)")]
|
||||
|
||||
|
||||
def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None:
|
||||
@@ -92,7 +89,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non
|
||||
assert "<" not in found["description"]
|
||||
assert found["location"] == "San Francisco, CA; Seattle, WA; New York, NY"
|
||||
assert [p["text"] for p in found["places"]] == ["San Francisco, CA", "Seattle, WA", "New York, NY"]
|
||||
assert found["employment"] == "full_time" # employmentType.label SALARIED_FT
|
||||
assert found["employment"] == "SALARIED_FT" # employmentType.label
|
||||
assert found["posted"] is not None and found["posted"].isoformat() == \
|
||||
"2025-04-07T12:29:19.348000-07:00"
|
||||
assert (found["salary_min"], found["salary_max"], found["currency"]) == (216000, 378000, "USD")
|
||||
@@ -203,13 +200,12 @@ def test_discovery_finds_the_board_slug() -> None:
|
||||
("rippling", "prisma-careers"), ("rippling", "Acme"), ("rippling", "beta")]
|
||||
|
||||
|
||||
def test_a_role_at_several_places_keeps_each_once_and_is_remote_only_if_all_are() -> None:
|
||||
def test_a_role_at_several_places_keeps_each_once() -> None:
|
||||
rows = {r["source_id"]: r for r in rippling.parse(BOARD, "rippling")}
|
||||
director = rows[DIRECTOR]
|
||||
assert [p["text"] for p in director["places"]] == [
|
||||
"San Francisco, CA", "Seattle, WA", "New York, NY"]
|
||||
assert director["location"] == "San Francisco, CA; Seattle, WA; New York, NY"
|
||||
assert director["remote"] is None
|
||||
remote = next(r for r in rows.values() if r["places"] == [
|
||||
{"text": "Remote (United States)", "remote": True}])
|
||||
assert remote["remote"] is True
|
||||
assert remote["places"][0]["remote"] is True
|
||||
@@ -4,17 +4,14 @@ No network: the shapes are those of Bosch's board, read on 2026-09-30."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.models import SENIORITIES # noqa: E402
|
||||
from plugins.readers import Ask, smartrecruiters # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -54,38 +51,23 @@ def test_page_reads_by_offset_and_takes_the_ask() -> None:
|
||||
def test_the_place_is_one_clean_line_with_its_remote_flag_and_country() -> None:
|
||||
[row] = _rows(_posting())
|
||||
assert row["places"] == [{"text": "Hatvan, Hungary", "remote": False}] # ", ," gone
|
||||
assert row["location"] == "Hatvan, Hungary" and row["country"] == "HU"
|
||||
assert row["location"] == "Hatvan, Hungary" and row["country"] == "hu"
|
||||
[remote] = _rows(_posting(location={"remote": True, "fullLocation": "Remote", "country": ""}))
|
||||
assert remote["places"] == [{"text": "Remote", "remote": True}] and remote["country"] is None
|
||||
[none] = _rows(_posting(location={}))
|
||||
assert none["places"] == [] and none["location"] == ""
|
||||
|
||||
|
||||
def test_the_stated_level_and_employment_are_mapped() -> None:
|
||||
def test_the_stated_level_and_employment_are_the_hosts_words() -> None:
|
||||
[row] = _rows(_posting(experienceLevel={"id": "internship", "label": "Internship"},
|
||||
typeOfEmployment={"id": "intern", "label": "Intern"}))
|
||||
assert (row["seniority"], row["employment"]) == ("intern", "internship")
|
||||
levels = {"entry_level": "junior", "director": "director", "executive": "exec",
|
||||
"associate": None, "mid_senior_level": None, "not_applicable": None}
|
||||
for word, want in levels.items():
|
||||
[row] = _rows(_posting(experienceLevel={"id": word}))
|
||||
assert row["seniority"] == want, word
|
||||
[row] = _rows(_posting(typeOfEmployment={"id": "permanent"}))
|
||||
assert row["employment"] is None
|
||||
assert (row["seniority"], row["employment"]) == ("internship", "intern")
|
||||
[row] = _rows(_posting(experienceLevel={"id": "mid_senior_level"}))
|
||||
assert row["seniority"] == "mid_senior_level"
|
||||
[row] = _rows(_posting())
|
||||
assert row["seniority"] is None and row["employment"] is None
|
||||
|
||||
|
||||
def test_the_table_speaks_the_apps_vocabulary_and_a_new_word_is_logged(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
assert {v for v in smartrecruiters._SENIORITY.values() if v} <= set(SENIORITIES)
|
||||
with caplog.at_level(logging.DEBUG, logger=smartrecruiters.__name__):
|
||||
[row] = _rows(_posting(experienceLevel={"id": "guru"}))
|
||||
assert row["seniority"] is None
|
||||
assert [r.getMessage() for r in caplog.records] == [
|
||||
"smartrecruiters: unknown experienceLevel 'guru'"]
|
||||
|
||||
|
||||
def test_the_saved_page_still_reads() -> None:
|
||||
data = json.loads((FIXTURES / "smartrecruiters.json").read_bytes())
|
||||
assert [r["location"] for r in smartrecruiters.rows(data, "Acme")] == ["Berlin, Germany",
|
||||
|
||||
@@ -315,7 +315,7 @@ def _detail(name: str, url: str = "https://careers.vestas.com/job/x/1431772533/"
|
||||
return successfactors.detail(get, "careers.vestas.com", {"url": url}) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_detail_reads_the_description_the_date_and_the_country_code_of_an_address() -> None:
|
||||
def test_detail_reads_the_description_the_date_and_the_places_of_an_address() -> None:
|
||||
got = _detail("successfactors-job.html")
|
||||
assert got["description"].startswith("Senior Management | Full-Time | Commercial Are you")
|
||||
assert "<" not in got["description"]
|
||||
@@ -324,7 +324,7 @@ def test_detail_reads_the_description_the_date_and_the_country_code_of_an_addres
|
||||
# The locality and the country's name; the region is cut to "Seou" by the site.
|
||||
assert got["location"] == "Seoul, South Korea"
|
||||
assert got["places"] == [{"text": "Seoul, South Korea", "remote": None}]
|
||||
assert got["country"] == "KR"
|
||||
assert "country" not in got # prep derives it from the places
|
||||
|
||||
|
||||
def test_detail_keeps_every_address_of_a_role_at_several_places() -> None:
|
||||
@@ -333,15 +333,6 @@ def test_detail_keeps_every_address_of_a_role_at_several_places() -> None:
|
||||
assert [p["text"] for p in got["places"]] == [
|
||||
"Seoul, South Korea", "Busan, South Korea", "Aarhus, Denmark"]
|
||||
assert got["location"] == "Seoul, South Korea; Busan, South Korea; Aarhus, Denmark"
|
||||
assert "country" not in got # KR and DK: no single country for the role
|
||||
|
||||
|
||||
def test_detail_states_the_country_when_every_address_is_in_it() -> None:
|
||||
text = _read("successfactors-places-detail.html").replace(
|
||||
'content="Aarhus, DK, 8200"', 'content="Daegu, KR, 41200"')
|
||||
got = successfactors.detail(lambda url, **kw: text.encode(), "careers.vestas.com",
|
||||
{"url": "https://careers.vestas.com/job/x/1/"})
|
||||
assert got["country"] == "KR"
|
||||
|
||||
|
||||
def test_the_listing_place_is_the_roles_only_place_until_its_page_is_read() -> None:
|
||||
@@ -351,11 +342,10 @@ def test_the_listing_place_is_the_roles_only_place_until_its_page_is_read() -> N
|
||||
for r in rows)
|
||||
|
||||
|
||||
def test_detail_reads_the_country_from_a_street_address() -> None:
|
||||
def test_detail_reads_the_place_from_a_street_address() -> None:
|
||||
"""Systematic and most other sites give one `streetAddress`, "City, CC, zip"."""
|
||||
got = _detail("successfactors-job-street.html")
|
||||
assert got["location"] == "Bern, Switzerland"
|
||||
assert got["country"] == "CH"
|
||||
assert got["posted"] == datetime(2026, 9, 28, 2, tzinfo=UTC)
|
||||
assert got["description"].startswith("Senior Solution Engineer Defence")
|
||||
|
||||
|
||||
@@ -5,11 +5,9 @@ No network: the fixture has the shape of Hugging Face's board, read on
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
@@ -49,21 +47,15 @@ def test_every_location_of_a_role_is_a_place_once() -> None:
|
||||
assert [p["text"] for p in first["places"]] == ["Berlin, Germany", "Munich, Bavaria, Germany",
|
||||
"Paris, France"]
|
||||
assert first["location"] == "Berlin, Germany; Munich, Bavaria, Germany; Paris, France"
|
||||
assert first["country"] is None # DE and FR
|
||||
assert "country" not in first # prep derives it from the places
|
||||
assert [p["remote"] for p in first["places"]] == [False] * 3
|
||||
assert remote["places"] == [{"text": "France", "remote": True}]
|
||||
assert remote["country"] == "FR"
|
||||
# A list-less answer (the older shape) still gives its one place.
|
||||
assert oslo["places"] == [{"text": "Oslo, Norway", "remote": False}]
|
||||
assert oslo["country"] == "NO"
|
||||
|
||||
|
||||
def test_the_stated_hours_are_mapped_and_an_unknown_type_is_logged(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
with caplog.at_level(logging.DEBUG, logger=workable.__name__):
|
||||
rows = _rows()
|
||||
assert [r["employment"] for r in rows] == ["full_time", "part_time", None]
|
||||
assert [r.getMessage() for r in caplog.records] == ["workable: unknown type 'volunteer'"]
|
||||
def test_the_stated_hours_are_the_hosts_word() -> None:
|
||||
assert [r["employment"] for r in _rows()] == ["full", "part", "volunteer"]
|
||||
|
||||
|
||||
def test_the_shape_of_a_real_list_answer_is_read() -> None:
|
||||
|
||||
+10
-11
@@ -54,8 +54,8 @@ def test_rows_read_the_listing_of_a_tenant_that_sends_remote_type() -> None:
|
||||
"Associate-Solutions-Architect_R35358")
|
||||
assert first["posted"] == TODAY
|
||||
assert first["description"] == "" and first["department"] == "" and first["salary_text"] == ""
|
||||
# "Fully Flexible" is Zendesk's word for remote.
|
||||
assert [r["remote"] for r in rows] == ["remote", "onsite", "hybrid", "remote", "hybrid"]
|
||||
# The host's own words, "Fully Flexible" among them; prep maps them.
|
||||
assert [r["remote"] for r in rows] == ["Fully Flexible", "In Office", "Hybrid", "Fully Flexible", "Hybrid"]
|
||||
|
||||
|
||||
def test_rows_of_a_tenant_without_remote_type_leave_remote_unknown() -> None:
|
||||
@@ -80,7 +80,7 @@ def test_rows_of_a_site_called_jobs_state_the_requisition_id_and_no_date_past_30
|
||||
assert rows[1]["places"] == [{"text": "Remote US TX", "remote": True}]
|
||||
assert rows[2]["url"].startswith("https://redhat.wd5.myworkdayjobs.com/jobs/job/")
|
||||
assert rows[3]["posted"] is None # "Posted 30+ Days Ago" says only that it is older
|
||||
assert [r["remote"] for r in rows] == ["onsite", "remote", "remote", "hybrid"]
|
||||
assert [r["remote"] for r in rows] == ["Onsite", "Remote", "Remote", "Hybrid"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("text", "expected"), [
|
||||
@@ -275,13 +275,12 @@ def test_detail_reads_the_description_date_remote_and_place_of_the_role() -> Non
|
||||
SLUG, row)
|
||||
assert asked == [f"https://{HOST}/wday/cxs/zendesk/zendesk"
|
||||
"/job/Remote-Mexico-City-Mexico/Associate-Solutions-Architect_R35358"]
|
||||
assert set(got) == {"description", "location", "places", "posted", "remote", "country",
|
||||
"employment"}
|
||||
assert set(got) == {"description", "location", "places", "posted", "remote", "employment"}
|
||||
assert got["posted"] == datetime(2026, 9, 29, tzinfo=UTC)
|
||||
assert got["remote"] == "remote"
|
||||
assert got["remote"] == "Fully Flexible"
|
||||
assert got["location"] == "Remote, Mexico City, Mexico"
|
||||
assert got["places"] == [{"text": "Remote, Mexico City, Mexico", "remote": True}]
|
||||
assert (got["country"], got["employment"]) == ("MX", "full_time")
|
||||
assert got["employment"] == "Full time" and "country" not in got
|
||||
assert got["description"].startswith("Job Description") or len(got["description"]) > 200
|
||||
assert "<" not in got["description"] and " " not in got["description"]
|
||||
assert " " not in got["description"]
|
||||
@@ -296,7 +295,7 @@ def test_detail_names_every_location_of_a_role_listed_as_several() -> None:
|
||||
"/job/Remote-Italy/Telco-Cloud-Consultant_R-059566-2"]
|
||||
assert got["location"] == "Remote Italy; Remote Spain"
|
||||
assert [p["text"] for p in got["places"]] == ["Remote Italy", "Remote Spain"]
|
||||
assert "country" not in got # the code belongs to the main location alone
|
||||
assert "country" not in got
|
||||
|
||||
|
||||
def _detail(**info: Any) -> bytes:
|
||||
@@ -337,9 +336,9 @@ def test_a_country_dash_city_place_is_turned_round() -> None:
|
||||
assert workday._place("12 Locations", None) is None
|
||||
|
||||
|
||||
def test_detail_takes_no_word_for_remote_it_does_not_know() -> None:
|
||||
def test_detail_hands_on_a_word_for_remote_it_does_not_know() -> None:
|
||||
row = workday.rows(_load("workday.json"), SLUG)[0]
|
||||
assert "remote" not in workday.detail(_get_of(_detail(remoteType="No"), []), SLUG, row)
|
||||
assert workday.detail(_get_of(_detail(remoteType="No"), []), SLUG, row)["remote"] == "No"
|
||||
|
||||
|
||||
def test_detail_refuses_a_row_that_is_not_a_role_of_the_board() -> None:
|
||||
@@ -383,7 +382,7 @@ def test_detail_is_read_through_the_getter_ats_binds(
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
row = workday.rows(_load("workday.json"), SLUG)[0]
|
||||
got = workday.detail(ats._getter(object(), "workday", SLUG), SLUG, row) # type: ignore[arg-type]
|
||||
assert got["remote"] == "remote"
|
||||
assert got["remote"] == "Fully Flexible"
|
||||
assert seen[0][:2] == ("workday", SLUG)
|
||||
|
||||
|
||||
|
||||
+12
-11
@@ -13,6 +13,7 @@ from app.models import company_fold
|
||||
from app.storage import Store
|
||||
from app.storage.schema import _add_company_aliases
|
||||
from desk.models import Item
|
||||
from prepped import upsert
|
||||
|
||||
|
||||
def undo_v20(conn: sqlite3.Connection) -> None:
|
||||
@@ -56,10 +57,10 @@ def test_a_company_is_its_own_alias(store: Store) -> None:
|
||||
|
||||
def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
|
||||
store.add_company("Ørsted")
|
||||
store.upsert_role(posting("1", "Orsted"))
|
||||
store.upsert_role(posting("2", "ORSTED A/S"))
|
||||
store.upsert_role(posting("3", "Ørsted"))
|
||||
store.upsert_role(posting("4", "Nobody"))
|
||||
upsert(store, posting("1", "Orsted"))
|
||||
upsert(store, posting("2", "ORSTED A/S"))
|
||||
upsert(store, posting("3", "Ørsted"))
|
||||
upsert(store, posting("4", "Nobody"))
|
||||
assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"]
|
||||
assert [c.key for c in store.companies()] == ["ørsted"]
|
||||
|
||||
@@ -67,7 +68,7 @@ def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
|
||||
def test_an_added_alias_finds_the_company(store: Store) -> None:
|
||||
store.add_company("Ørsted")
|
||||
store.add_alias("ørsted", "Dong Energy")
|
||||
store.upsert_role(posting("1", "Dong Energy A/S"))
|
||||
upsert(store, posting("1", "Dong Energy A/S"))
|
||||
assert company_of(store, "1") == "ørsted"
|
||||
with pytest.raises(KeyError):
|
||||
store.add_alias("nope", "X")
|
||||
@@ -90,15 +91,15 @@ def test_duplicates_are_listed_by_fold_and_not_merged(store: Store) -> None:
|
||||
assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(store.companies()) == 3
|
||||
# The company with the name's own key wins when a role names it.
|
||||
store.upsert_role(posting("1", "Orsted"))
|
||||
upsert(store, posting("1", "Orsted"))
|
||||
assert company_of(store, "1") == "orsted"
|
||||
|
||||
|
||||
def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
|
||||
store.add_company("Ørsted")
|
||||
store.add_company("Orsted")
|
||||
store.upsert_role(posting("1", "Orsted"))
|
||||
store.upsert_role(posting("2", "Ørsted"))
|
||||
upsert(store, posting("1", "Orsted"))
|
||||
upsert(store, posting("2", "Ørsted"))
|
||||
sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted",
|
||||
board="greenhouse")
|
||||
a, b = store.save_agent("A"), store.save_agent("B")
|
||||
@@ -116,7 +117,7 @@ def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
|
||||
assert store.duplicate_companies() == []
|
||||
assert store.company_aliases() == {"ørsted": ["orsted"]}
|
||||
# A later role from the gone spelling lands on the kept company.
|
||||
store.upsert_role(posting("3", "Orsted"))
|
||||
upsert(store, posting("3", "Orsted"))
|
||||
assert company_of(store, "3") == "ørsted"
|
||||
with pytest.raises(ValueError):
|
||||
store.merge_companies("ørsted", "ørsted")
|
||||
@@ -147,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
||||
"VALUES (?, ?, '', '')", (key, name))
|
||||
conn.execute("PRAGMA user_version = 19")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [20, 21]
|
||||
assert reopened.migrations_applied == [20, 21, 22]
|
||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(reopened.companies()) == 3
|
||||
|
||||
@@ -184,7 +185,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
||||
conn.execute("ALTER TABLE companies DROP COLUMN all_roles")
|
||||
conn.execute("PRAGMA user_version = 20")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [21]
|
||||
assert reopened.migrations_applied == [21, 22]
|
||||
assert {c.key: c.all_roles for c in reopened.companies()} == {
|
||||
"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||
params = {s.target: s.params for s in reopened.subscriptions(source="ats")}
|
||||
|
||||
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21]
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22]
|
||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||
|
||||
@@ -250,16 +250,16 @@ class _Context:
|
||||
return t
|
||||
|
||||
|
||||
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
|
||||
def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_enrich() -> None:
|
||||
context = _Context()
|
||||
empty = Tally(results=({"filled": 0},))
|
||||
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
|
||||
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 4},
|
||||
context=context)
|
||||
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
|
||||
assert out["next"] == "extract" and context.made[0].carries["at"] == 8
|
||||
|
||||
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
|
||||
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
|
||||
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
|
||||
{"cascade": "pull", "task": "enrich", "at": 4}, context=context)
|
||||
assert out["next"] == "prep" and context.made[1].carries["at"] == 5
|
||||
|
||||
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
|
||||
context=context)
|
||||
@@ -290,8 +290,10 @@ def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
|
||||
assert started is not None
|
||||
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
|
||||
key=lambda j: j.id)
|
||||
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
|
||||
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
|
||||
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == \
|
||||
["enrich", "prep", "embed", "score"]
|
||||
assert [j.kind for j in jobs if j.kind != FINAL] == \
|
||||
["enrich", enrich.KIND, "prep", "embed", "score"]
|
||||
assert web["asked"] == [f"{BASE}/acme/1"]
|
||||
embedded = next(j for j in jobs if j.kind == "embed")
|
||||
assert embedded.result["embedded"] == 1
|
||||
|
||||
+15
-14
@@ -16,6 +16,7 @@ from app.config import Config
|
||||
from app.jobs import extract
|
||||
from app.models import Limits
|
||||
from app.storage import Store
|
||||
from prepped import upsert
|
||||
|
||||
|
||||
def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item:
|
||||
@@ -77,7 +78,7 @@ def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
|
||||
def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
uid = posting("1").uid
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer(
|
||||
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
|
||||
@@ -100,7 +101,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
||||
def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
ranked(store, item)
|
||||
stub(monkeypatch, lambda roles: [answer(
|
||||
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
|
||||
@@ -116,13 +117,13 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
||||
|
||||
def test_the_next_upsert_keeps_what_extract_filled(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
|
||||
for r in roles])
|
||||
modeltask.run(extract.TASK, config, store)
|
||||
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
|
||||
role = store.role(posting("1").uid)
|
||||
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
|
||||
@@ -131,7 +132,7 @@ def test_the_next_upsert_keeps_what_extract_filled(
|
||||
def test_closed_and_answered_roles_are_not_offered(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for sid in ("open", "closed", "done"):
|
||||
store.upsert_role(posting(sid))
|
||||
upsert(store, posting(sid))
|
||||
ranked(store, posting("open"), posting("closed"), posting("done"))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
|
||||
@@ -147,7 +148,7 @@ def test_closed_and_answered_roles_are_not_offered(
|
||||
|
||||
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
ranked(store, posting("1"))
|
||||
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
|
||||
|
||||
@@ -170,7 +171,7 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
||||
the boards produced. A role nobody ranks is never sent; a batch dealt a
|
||||
uid finishes it."""
|
||||
for sid in ("best", "second", "low", "unscored", "gated"):
|
||||
store.upsert_role(posting(sid))
|
||||
upsert(store, posting(sid))
|
||||
one = store.save_agent("One", "python backend",
|
||||
limits=Limits(extract_top=1, extract_min=0.5))
|
||||
two = store.save_agent("Two", "python backend", enabled=False)
|
||||
@@ -209,7 +210,7 @@ def test_a_value_outside_the_vocabulary_is_dropped() -> None:
|
||||
|
||||
def test_disabled_finds_nothing_and_asks_nobody(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
model = stub(monkeypatch, lambda roles: [])
|
||||
|
||||
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
|
||||
@@ -220,7 +221,7 @@ def test_disabled_finds_nothing_and_asks_nobody(
|
||||
def test_a_role_read_before_without_a_level_is_read_again_for_it(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1", seniority="senior")
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
ranked(store, item)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||
@@ -243,8 +244,8 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
||||
withtext = posting("text", title="Senior Engineer", seniority="mid")
|
||||
noText = posting("teaser", title="Analyst")
|
||||
noText.body = ""
|
||||
store.upsert_role(withtext)
|
||||
store.upsert_role(noText)
|
||||
upsert(store, withtext)
|
||||
upsert(store, noText)
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
|
||||
"WHERE source_id = 'teaser'")
|
||||
@@ -265,7 +266,7 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
||||
def test_an_answer_with_no_level_is_charged_until_given_up(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
item = posting("1")
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
ranked(store, item)
|
||||
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
|
||||
|
||||
@@ -279,7 +280,7 @@ def test_an_answer_with_no_level_is_charged_until_given_up(
|
||||
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
|
||||
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
|
||||
for item in items:
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
one = store.save_agent("One", "python backend")
|
||||
two = store.save_agent("Two", "python backend")
|
||||
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
|
||||
@@ -296,7 +297,7 @@ def test_agreement_per_host(store: Store) -> None:
|
||||
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
|
||||
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
|
||||
("5", "lever", "senior", "")]:
|
||||
store.upsert_role(posting(sid, board=board, seniority=stated))
|
||||
upsert(store, posting(sid, board=board, seniority=stated))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
|
||||
(model, sid))
|
||||
|
||||
+19
-18
@@ -24,6 +24,7 @@ from app.places import Gazetteer, fold, seed
|
||||
from app.storage import Store
|
||||
|
||||
from test_places import LINES, ORACLE_TOKENS
|
||||
from prepped import upsert
|
||||
|
||||
|
||||
def gazetteer() -> Gazetteer:
|
||||
@@ -173,7 +174,7 @@ def test_a_fresh_file_is_seeded(store: Store) -> None:
|
||||
|
||||
|
||||
def test_a_joined_line_is_several_places(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
|
||||
upsert(store, posting("1", place="Copenhagen; London; Warsaw"))
|
||||
uid = posting("1").uid
|
||||
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
|
||||
(2, "Warsaw", "Warsaw")]
|
||||
@@ -183,7 +184,7 @@ def test_a_joined_line_is_several_places(store: Store) -> None:
|
||||
|
||||
|
||||
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Berlin, Munich", places=[
|
||||
upsert(store, posting("1", place="Berlin, Munich", places=[
|
||||
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
|
||||
{"text": "Remote (DE)", "remote": True}]))
|
||||
assert texts(store, posting("1").uid) == [
|
||||
@@ -191,7 +192,7 @@ def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
||||
|
||||
|
||||
def test_places_without_a_line_make_the_line(store: Store) -> None:
|
||||
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
|
||||
upsert(store, posting("1", places=[{"text": "Lund", "remote": None},
|
||||
{"text": "Malmö", "remote": None}]))
|
||||
role = store.role(posting("1").uid)
|
||||
assert role is not None
|
||||
@@ -199,26 +200,26 @@ def test_places_without_a_line_make_the_line(store: Store) -> None:
|
||||
|
||||
|
||||
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Aarhus"))
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1", place="Aarhus"))
|
||||
upsert(store, posting("1"))
|
||||
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
|
||||
store.upsert_role(posting("1", place="Odense"))
|
||||
upsert(store, posting("1", place="Odense"))
|
||||
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
|
||||
|
||||
|
||||
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Aarhus"))
|
||||
upsert(store, posting("1", place="Aarhus"))
|
||||
uid = posting("1").uid
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
||||
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
|
||||
store.upsert_role(posting("1", place="Aalborg"))
|
||||
upsert(store, posting("1", place="Aalborg"))
|
||||
assert [(p.text, p.url) for p in store.role_places(uid)] == [
|
||||
("Aalborg", ""), ("Odense", "https://copy")]
|
||||
|
||||
|
||||
def test_an_extracted_place_is_resolved(store: Store) -> None:
|
||||
store.upsert_role(posting("1"))
|
||||
upsert(store, posting("1"))
|
||||
uid = posting("1").uid
|
||||
store.set_extracted(uid, {"place": "Lund, Sweden"})
|
||||
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
|
||||
@@ -236,7 +237,7 @@ def several(store: Store) -> dict[str, str]:
|
||||
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
|
||||
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
|
||||
for key, line in lines.items():
|
||||
store.upsert_role(posting(key, place=line))
|
||||
upsert(store, posting(key, place=line))
|
||||
return {key: posting(key).uid for key in lines}
|
||||
|
||||
|
||||
@@ -274,7 +275,7 @@ def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
|
||||
|
||||
|
||||
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
|
||||
store.upsert_role(posting("1", country="DK"))
|
||||
upsert(store, posting("1", country="DK"))
|
||||
assert uids(store, "country:DK") == {posting("1").uid}
|
||||
|
||||
|
||||
@@ -312,15 +313,15 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store,
|
||||
|
||||
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
|
||||
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
|
||||
store.upsert_role(posting(str(n), place=line))
|
||||
upsert(store, posting(str(n), place=line))
|
||||
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
|
||||
("jena", "Jena", 1)]
|
||||
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
|
||||
|
||||
|
||||
def test_an_alias_links_what_it_names(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Kbh. K"))
|
||||
store.upsert_role(posting("2", place="Copenhagen"))
|
||||
upsert(store, posting("1", place="Kbh. K"))
|
||||
upsert(store, posting("2", place="Copenhagen"))
|
||||
uid = posting("1").uid
|
||||
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
|
||||
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
|
||||
@@ -333,8 +334,8 @@ def test_an_alias_links_what_it_names(store: Store) -> None:
|
||||
|
||||
|
||||
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Oberkochen; Jena"))
|
||||
store.upsert_role(posting("2", place="Paderborn, Germany"))
|
||||
upsert(store, posting("1", place="Oberkochen; Jena"))
|
||||
upsert(store, posting("2", place="Paderborn, Germany"))
|
||||
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
|
||||
store.add_place("Paderborn", "city", "country:DE")
|
||||
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
|
||||
@@ -355,7 +356,7 @@ def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
|
||||
|
||||
|
||||
def test_relink_places_resolves_every_role_again(store: Store) -> None:
|
||||
store.upsert_role(posting("1", place="Oberkochen"))
|
||||
upsert(store, posting("1", place="Oberkochen"))
|
||||
with sqlite3.connect(store.path) as conn:
|
||||
conn.execute("UPDATE role_places SET place_id = "
|
||||
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
|
||||
@@ -381,7 +382,7 @@ def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tup
|
||||
lines = {}
|
||||
for n, (place, country) in enumerate(LINES):
|
||||
item = posting(f"line{n}", place=place, country=country)
|
||||
store.upsert_role(item)
|
||||
upsert(store, item)
|
||||
lines[item.uid] = (place, country)
|
||||
return store, lines
|
||||
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
"""`app.prep`: the host's words mapped centrally, the stage over the due
|
||||
roles, the unmapped list, the table fingerprint and migration 22. The host
|
||||
tables here are fakes; each real table is held to its host by R1's and R2's
|
||||
tests, and to the vocabularies by the last tests of this file."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import cli, models, prep # noqa: E402
|
||||
from app.prep import hosts, words # noqa: E402
|
||||
from app.prep.country import country, of_host # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from desk.models import Item # noqa: E402
|
||||
from plugins.readers import _shared # noqa: E402
|
||||
|
||||
TABLES: dict[str, Any] = {
|
||||
"acme": {
|
||||
"seniority": {"experienced": "", "sr": "senior"},
|
||||
"employment": {"regular full time": "full_time", "permanent": ""},
|
||||
"remote": {"2": "hybrid", "fully": "remote"},
|
||||
},
|
||||
"codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}},
|
||||
"parts": {"match": {"mode": "substring",
|
||||
"rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}},
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- words
|
||||
|
||||
@pytest.mark.parametrize(("field", "word", "value"), [
|
||||
("seniority", "SR", "senior"),
|
||||
("seniority", " Experienced ", ""), # known, names no level
|
||||
("seniority", "senior", "senior"), # already the vocabulary
|
||||
("seniority", "Grand Poobah", None), # unknown
|
||||
("employment", "Regular Full Time", "full_time"),
|
||||
("employment", "Permanent", ""),
|
||||
("employment", "contract", "contract"),
|
||||
("employment", "Zero hours", None),
|
||||
("remote", "2", "hybrid"),
|
||||
("remote", "On-site", "onsite"), # the shared words
|
||||
("remote", "on_site", "onsite"),
|
||||
("remote", "IN_OFFICE", "onsite"),
|
||||
("remote", "office", "onsite"),
|
||||
("remote", "Hybrid", "hybrid"),
|
||||
("remote", "unspecified", None),
|
||||
("seniority", "", ""),
|
||||
])
|
||||
def test_a_word_goes_through_the_hosts_table_then_the_vocabulary(
|
||||
field: str, word: str, value: str | None) -> None:
|
||||
assert words.lookup(field, "acme", word, TABLES) == value
|
||||
|
||||
|
||||
def test_another_hosts_table_is_not_used() -> None:
|
||||
assert words.lookup("seniority", "other", "sr", TABLES) is None
|
||||
|
||||
|
||||
def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None:
|
||||
assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time"
|
||||
assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time"
|
||||
assert words.lookup("employment", "codes", "software", TABLES) is None
|
||||
assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary"
|
||||
assert words.employment("parts", "permanent_full_time", TABLES) == "full_time"
|
||||
|
||||
|
||||
def test_seniority_is_the_stated_level_and_the_titles() -> None:
|
||||
assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff")
|
||||
assert words.seniority("acme", "", "Engineer", TABLES) == ("", "")
|
||||
|
||||
|
||||
def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None:
|
||||
assert words.remote("acme", True, (), "", TABLES) == "remote"
|
||||
assert words.remote("acme", "true", (), "", TABLES) == "remote"
|
||||
assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote"
|
||||
# False says nothing the places do not: hosts send it for hybrid roles too.
|
||||
assert words.remote("acme", False, (), "Berlin", TABLES) == ""
|
||||
assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid"
|
||||
assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \
|
||||
== "remote"
|
||||
assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}],
|
||||
"", TABLES) == "remote"
|
||||
|
||||
|
||||
def test_a_line_in_danish_or_swedish_and_a_search_title_are_read() -> None:
|
||||
assert words.remote("jobindex", None, (), "Nordhavn og mulighed for hjemmearbejde") \
|
||||
== "hybrid"
|
||||
assert words.remote("jobindex", None, (), "Aarhus, fuldt hjemmearbejde") == "remote"
|
||||
assert words.remote("platsbanken", None, (), "Malmö, arbete på distans") == "remote"
|
||||
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer - Remote") \
|
||||
== "remote"
|
||||
assert words.remote("adzuna", None, (), "Berlin", title="Backend Developer") == ""
|
||||
|
||||
|
||||
def test_a_search_roles_title_counts_and_a_boards_does_not(store: Store) -> None:
|
||||
store.upsert_role(item("1", title="Engineer (remote)", place="Berlin"))
|
||||
store.upsert_role(item("2", title="Engineer (remote)", place="Berlin", host="jobindex",
|
||||
provider="jobindex"))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET source = 'search' WHERE source_id = 'jobindex:2'")
|
||||
prep.run(store, tables=TABLES)
|
||||
assert (row(store, "1")["remote"], row(store, "2", "jobindex")["remote"]) == ("", "remote")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- country
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "roledesk.db")
|
||||
|
||||
|
||||
def ids(store: Store, *texts: str) -> list[int]:
|
||||
gaz = store.gazetteer()
|
||||
return [p for text in texts for p in gaz.resolve(text)]
|
||||
|
||||
|
||||
def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None:
|
||||
gaz = store.gazetteer()
|
||||
assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK"
|
||||
assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU"
|
||||
assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == ""
|
||||
assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB"
|
||||
assert country(gaz, ids(store, "Europe")) == "EU"
|
||||
assert country(gaz, [], "dk") == "DK"
|
||||
assert country(gaz, ids(store, "Remote"), "Sverige") == "SE"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("stated", "code"), [
|
||||
("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"),
|
||||
("", ""), ("Narnia", None)])
|
||||
def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str,
|
||||
code: str | None) -> None:
|
||||
assert of_host(store.gazetteer(), stated) == code
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- stage
|
||||
|
||||
def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme",
|
||||
**raw: Any) -> Item:
|
||||
found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}",
|
||||
title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw={"board": host, "company": "Acme"} | raw)
|
||||
found.source = "ats"
|
||||
return found
|
||||
|
||||
|
||||
def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row:
|
||||
[found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",))
|
||||
return found
|
||||
|
||||
|
||||
def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None:
|
||||
store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2",
|
||||
places=[{"text": "Aarhus", "remote": None}]))
|
||||
raw = row(store, "1")
|
||||
assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \
|
||||
("SR", "Regular Full Time", "2")
|
||||
assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \
|
||||
("", "", "", "")
|
||||
assert raw["prep_due"] == 1
|
||||
assert store.role_places(raw["uid"])[0].place_id is None
|
||||
|
||||
done = prep.run(store, tables=TABLES)
|
||||
|
||||
assert (done.read, done.written, done.remapped) == (1, 1, True)
|
||||
mapped = row(store, "1")
|
||||
assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"],
|
||||
mapped["remote"], mapped["country"], mapped["prep_due"]) == \
|
||||
("senior", "senior", "full_time", "hybrid", "DK", 0)
|
||||
assert store.role_places(mapped["uid"])[0].name == "Aarhus"
|
||||
|
||||
|
||||
def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None:
|
||||
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
|
||||
store.upsert_role(item("2", seniority="sr"))
|
||||
prep.run(store, tables=TABLES)
|
||||
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
|
||||
store.upsert_role(item("2", seniority="experienced"))
|
||||
assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1)
|
||||
# The place stays resolved when its text did not change.
|
||||
assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus"
|
||||
|
||||
store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}]))
|
||||
assert row(store, "1")["prep_due"] == 1
|
||||
assert prep.run(store, tables=TABLES).read == 2
|
||||
assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == ""
|
||||
|
||||
|
||||
def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
|
||||
store.upsert_role(item("1", seniority="sr"))
|
||||
[read] = store.prep_rows("due")
|
||||
store.upsert_role(item("1", seniority="experienced"))
|
||||
store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)])
|
||||
assert row(store, "1")["prep_due"] == 2
|
||||
|
||||
|
||||
def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract(
|
||||
store: Store) -> None:
|
||||
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
|
||||
uid = row(store, "1")["uid"]
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' "
|
||||
"WHERE uid = ?", (uid,))
|
||||
prep.run(store, tables=TABLES)
|
||||
mapped = row(store, "1")
|
||||
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
|
||||
("senior", "lead", "remote")
|
||||
# The host's own statement does overwrite.
|
||||
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2"))
|
||||
prep.run(store, tables=TABLES)
|
||||
assert row(store, "1")["remote"] == "hybrid"
|
||||
|
||||
|
||||
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
||||
store.add_company("Ørsted")
|
||||
store.upsert_role(item("1", company="ORSTED A/S"))
|
||||
assert row(store, "1")["company_key"] != "ørsted"
|
||||
prep.run(store, tables=TABLES)
|
||||
assert row(store, "1")["company_key"] == "ørsted"
|
||||
|
||||
|
||||
def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None:
|
||||
store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified",
|
||||
country="Narnia"))
|
||||
store.upsert_role(item("2", seniority="Grand Poobah", remote=False))
|
||||
store.upsert_role(item("3", host="other", seniority="Grand Poobah"))
|
||||
done = prep.run(store, tables=TABLES)
|
||||
assert done.unmapped == 4
|
||||
listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()}
|
||||
assert listed == {("seniority", "acme", "grand poobah"): 2,
|
||||
("seniority", "other", "grand poobah"): 1,
|
||||
("remote", "acme", "unspecified"): 1,
|
||||
("country", "acme", "narnia"): 1}
|
||||
first = {w["word"]: w["first_seen"] for w in store.unmapped()}
|
||||
|
||||
# A word a table learns leaves the list at the next run; the rest keep
|
||||
# the moment they were first met.
|
||||
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}}
|
||||
prep.run(store, tables=learned)
|
||||
assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \
|
||||
first
|
||||
assert ("seniority", "acme", "grand poobah") not in \
|
||||
{(w["field"], w["host"], w["word"]) for w in store.unmapped()}
|
||||
assert row(store, "1")["seniority"] == "exec"
|
||||
|
||||
|
||||
def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None:
|
||||
store.upsert_role(item("1", seniority="Distinguished"))
|
||||
store.upsert_role(item("2", seniority="Distinguished"))
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'")
|
||||
assert prep.run(store, tables=TABLES).remapped
|
||||
assert not prep.run(store, tables=TABLES).remapped
|
||||
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
|
||||
|
||||
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"distinguished": "staff"}}}
|
||||
done = prep.run(store, tables=learned)
|
||||
assert (done.remapped, done.read) == (True, 1)
|
||||
assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "")
|
||||
# `--remap` maps the closed roles too.
|
||||
assert prep.run(store, tables=learned, remap=True).read == 2
|
||||
assert row(store, "2")["seniority"] == "staff"
|
||||
|
||||
|
||||
def test_prep_folds_the_copies(store: Store) -> None:
|
||||
for n in ("1", "2"):
|
||||
store.upsert_role(item(n, place="Aarhus"))
|
||||
assert prep.run(store, tables=TABLES).copies == 1
|
||||
assert prep.run(store, tables=TABLES, dupes=False).copies is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------- migration
|
||||
|
||||
def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
||||
path = tmp_path / "old.db"
|
||||
store = Store(path)
|
||||
store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True,
|
||||
country="DK"))
|
||||
store.upsert_role(item("2"))
|
||||
store.state[prep.STATE] = "old"
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("DROP INDEX idx_roles_prep_due")
|
||||
for column in ("seniority_host", "employment_host", "remote_host", "country_host",
|
||||
"employment", "prep_due"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
conn.execute("DROP TABLE unmapped")
|
||||
conn.execute("PRAGMA user_version = 21")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [22]
|
||||
one = row(migrated, "1")
|
||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||
assert row(migrated, "2")["remote_host"] == ""
|
||||
assert prep.STATE not in migrated.state
|
||||
# The first prep maps every open role from those words, the vocabulary
|
||||
# values the old readers stored as themselves.
|
||||
prep.run(migrated, tables={})
|
||||
one = row(migrated, "1")
|
||||
assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \
|
||||
("senior", "full_time", "remote", "DK")
|
||||
|
||||
|
||||
# ------------------------------------------------------------ CLI and API
|
||||
|
||||
def test_app_prep_runs_and_lists_the_unmapped_words(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
path = tmp_path / "config.yaml"
|
||||
path.write_text(f"database: {tmp_path / 'roledesk.db'}\n")
|
||||
Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah"))
|
||||
|
||||
assert cli.main(["-c", str(path), "prep"]) == 0
|
||||
assert json.loads(capsys.readouterr().out)["read"] == 1
|
||||
assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0
|
||||
assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"]
|
||||
assert cli.main(["-c", str(path), "prep", "--remap"]) == 0
|
||||
assert json.loads(capsys.readouterr().out)["remapped"] is True
|
||||
|
||||
|
||||
def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None:
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.config import Config, ServerConfig
|
||||
from app.main import create_app
|
||||
|
||||
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
||||
database=tmp_path / "roledesk.db"))
|
||||
app.state.store.upsert_role(item("1", employment="Zero hours"))
|
||||
prep.run(app.state.store, tables=TABLES)
|
||||
with TestClient(app) as client:
|
||||
found = client.get("/api/prep/unmapped").json()
|
||||
assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \
|
||||
[("employment", "acme", "zero hours", 1)]
|
||||
|
||||
|
||||
# ------------------------------------------------------------- the tables
|
||||
|
||||
def test_every_host_table_maps_into_its_vocabulary() -> None:
|
||||
"""Passes with no table at all; R1's and R2's files are held here."""
|
||||
for host, tables in hosts.load().items():
|
||||
for field, table in tables.items():
|
||||
if field == "match":
|
||||
assert table["mode"] in hosts.MATCH_MODES, host
|
||||
values = [value for _, value in table["rules"]]
|
||||
field = "employment"
|
||||
else:
|
||||
assert all(key == key.strip().lower() for key in table), (host, field)
|
||||
values = list(table.values())
|
||||
assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field)
|
||||
|
||||
|
||||
def test_a_match_table_must_say_how_it_matches(tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n')
|
||||
monkeypatch.setattr(hosts, "__path__", [str(tmp_path)])
|
||||
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
|
||||
with pytest.raises(ValueError, match=r"loose\.py has MATCH"):
|
||||
hosts.load()
|
||||
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n')
|
||||
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
|
||||
importlib.invalidate_caches()
|
||||
assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]}
|
||||
|
||||
|
||||
def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None:
|
||||
assert _shared.EMPLOYMENTS == models.EMPLOYMENTS
|
||||
@@ -0,0 +1,37 @@
|
||||
"""The ashby words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.ashby")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['REMOTE', 'EMPLOYMENT'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('onsite', 'onsite')])
|
||||
def test_remote_words(word: str, value: str) -> None:
|
||||
assert hosts.REMOTE[word] == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('fulltime', 'full_time'), ('parttime', 'part_time'), ('intern', 'internship'), ('contract', 'contract'), ('temporary', 'temporary')])
|
||||
def test_employment_words(word: str, value: str) -> None:
|
||||
assert hosts.EMPLOYMENT[word] == value
|
||||
@@ -0,0 +1,37 @@
|
||||
"""The bamboohr words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.bamboohr")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('0', 'onsite'), ('1', 'remote'), ('2', 'hybrid')])
|
||||
def test_remote_words(word: str, value: str) -> None:
|
||||
assert hosts.REMOTE[word] == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('part time', 'part_time'), ('intern', 'internship'), ('contractor', 'contract')])
|
||||
def test_employment_words(word: str, value: str) -> None:
|
||||
assert hosts.EMPLOYMENT[word] == value
|
||||
@@ -0,0 +1,37 @@
|
||||
"""The deel words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.deel")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('on_site', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')])
|
||||
def test_remote_words(word: str, value: str) -> None:
|
||||
assert hosts.REMOTE[word] == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('consultant', 'contract'), ('internship', 'internship')])
|
||||
def test_employment_words(word: str, value: str) -> None:
|
||||
assert hosts.EMPLOYMENT[word] == value
|
||||
@@ -0,0 +1,32 @@
|
||||
"""The eightfold words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.eightfold")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['REMOTE'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('onsite', 'onsite'), ('on_site', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')])
|
||||
def test_remote_words(word: str, value: str) -> None:
|
||||
assert hosts.REMOTE[word] == value
|
||||
@@ -0,0 +1,37 @@
|
||||
"""The gem words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.gem")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('in_office', 'onsite'), ('hybrid', 'hybrid'), ('remote', 'remote')])
|
||||
def test_remote_words(word: str, value: str) -> None:
|
||||
assert hosts.REMOTE[word] == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('full_time', 'full_time'), ('contractor', 'contract'), ('intern', 'internship')])
|
||||
def test_employment_words(word: str, value: str) -> None:
|
||||
assert hosts.EMPLOYMENT[word] == value
|
||||
@@ -0,0 +1,37 @@
|
||||
"""The hibob words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.hibob")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['EMPLOYMENT', 'REMOTE'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('on_site', 'onsite'), ('onsite', 'onsite')])
|
||||
def test_remote_words(word: str, value: str) -> None:
|
||||
assert hosts.REMOTE[word] == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('regular full time', 'full_time'), ('fixed term', 'temporary'), ('intern', 'internship')])
|
||||
def test_employment_words(word: str, value: str) -> None:
|
||||
assert hosts.EMPLOYMENT[word] == value
|
||||
@@ -0,0 +1,32 @@
|
||||
"""The jibe words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.jibe")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['EMPLOYMENT'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('full_time', 'full_time'), ('contractor', 'contract'), ('intern', 'internship')])
|
||||
def test_employment_words(word: str, value: str) -> None:
|
||||
assert hosts.EMPLOYMENT[word] == value
|
||||
@@ -0,0 +1,42 @@
|
||||
"""The jobylon words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.jobylon")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['SENIORITY', 'EMPLOYMENT', 'REMOTE'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('mid level', 'mid'), ('student', 'intern'), ('entry level', 'junior'), ('executive', 'exec'), ('not applicable', '')])
|
||||
def test_seniority_words(word: str, value: str) -> None:
|
||||
assert hosts.SENIORITY[word] == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('consultant', 'contract'), ('summer job', 'temporary')])
|
||||
def test_employment_words(word: str, value: str) -> None:
|
||||
assert hosts.EMPLOYMENT[word] == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('remote', 'remote'), ('hybrid', 'hybrid'), ('on-site', 'onsite')])
|
||||
def test_remote_words(word: str, value: str) -> None:
|
||||
assert hosts.REMOTE[word] == value
|
||||
@@ -0,0 +1,32 @@
|
||||
"""The lever words, mapped to the app's vocabulary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import models # noqa: E402
|
||||
from plugins.readers._shared import EMPLOYMENTS # noqa: E402
|
||||
|
||||
hosts = importlib.import_module("app.prep.hosts.lever")
|
||||
|
||||
VOCABULARY = {"SENIORITY": models.SENIORITIES, "EMPLOYMENT": EMPLOYMENTS, "REMOTE": models.REMOTE}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table", ['EMPLOYMENT'])
|
||||
def test_the_table_keys_are_lowercase_and_the_values_in_the_vocabulary(table: str) -> None:
|
||||
words = getattr(hosts, table)
|
||||
assert all(key == key.strip().lower() for key in words)
|
||||
assert set(words.values()) <= {*VOCABULARY[table], ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("word", "value"), [('full-time', 'full_time'), ('fulltime', 'full_time'), ('intern', 'internship'), ('freelance', 'freelance')])
|
||||
def test_employment_words(word: str, value: str) -> None:
|
||||
assert hosts.EMPLOYMENT[word] == value
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Oracle's word tables for prep: the host's codes, and what they mean."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
oracle = importlib.import_module("app.prep.hosts.oracle")
|
||||
|
||||
|
||||
def test_workplace_type_codes_read() -> None:
|
||||
assert oracle.REMOTE == {"ora_remote": "remote", "ora_hybrid": "hybrid",
|
||||
"ora_on_site": "onsite"}
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Personio's word tables for prep: the feed's own words, and what they mean."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
personio = importlib.import_module("app.prep.hosts.personio")
|
||||
|
||||
|
||||
def test_seniority_words_read() -> None:
|
||||
seniority = personio.SENIORITY
|
||||
assert [seniority[w] for w in ("entry-level", "executive", "student")] == [
|
||||
"junior", "exec", "intern"]
|
||||
assert seniority["experienced"] == "" # known, names no level
|
||||
assert "wizard" not in seniority
|
||||
|
||||
|
||||
def test_employment_words_read_from_the_contract_or_the_schedule() -> None:
|
||||
employment = personio.EMPLOYMENT
|
||||
assert employment["working_student"] == "internship" and employment["freelance"] == "contract"
|
||||
assert employment["permanent"] == ""
|
||||
assert employment["full-time"] == "full_time" and employment["part-time"] == "part_time"
|
||||
Loaded 100 of 117 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user