Prep stage: host words stored raw, mapped centrally in app/prep after every pull

upsert_role stores the host's own words (roles.*_host), the place texts
unresolved when they change, and the company key unresolved on insert, and
marks the role prep_due. app.prep maps seniority, employment, remote and
country through the host tables in app/prep/hosts, resolves places and
company aliases, lists unknown words in `unmapped`, remaps every open role
when the tables' fingerprint changes, and runs rebuild_dupes, which moves
out of the pull's settle. Migration 22 adds the columns and the table and
fills the host words from raw. Cascade: pull, prep, embed, score, enrich,
prep, embed, score, extract, score, summarise. `app prep [unmapped]
[--remap]`, /api/prep/unmapped and a fold on the Sources page.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:19:22 +02:00
1 parent a66aac95cf
commit d04ceae7e0
31 files changed
+1468 -330

No files matched your search

+31 -1
View File
@@ -8,6 +8,7 @@
app agents list|prompt
app stages [extract|summarise on|off]
app dupes
app prep [unmapped] [--remap]
app db init
`app job` queues the work on a queue of its own and drains it here, so a
@@ -160,6 +161,13 @@ def build_parser() -> argparse.ArgumentParser:
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
prep = sub.add_parser(
"prep", help="map the roles' host words and fold the copies, or list the unmapped words")
prep.add_argument("action", nargs="?", choices=["unmapped"],
help="list the host words no table maps, instead of running")
prep.add_argument("--remap", action="store_true",
help="map every role again, closed ones included")
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
return parser
@@ -188,6 +196,8 @@ def main(argv: list[str] | None = None) -> int:
return run_seniority(args)
if args.command == "dupes":
return run_dupes(args)
if args.command == "prep":
return run_prep(args)
return init_db(args)
except (FileNotFoundError, config_mod.ConfigError) as exc:
print(exc, file=sys.stderr)
@@ -655,7 +665,7 @@ def run_seniority(args: argparse.Namespace) -> int:
def run_dupes(args: argparse.Namespace) -> int:
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
"""`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`,
now. Prints the groups (kept roles with a copy), the copies the feed
hides and the rows whose `dup_of` changed, as JSON."""
from app.storage import Store
@@ -666,6 +676,26 @@ def run_dupes(args: argparse.Namespace) -> int:
return 0
def run_prep(args: argparse.Namespace) -> int:
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
host words no table maps, per field and host, most roles first."""
from app import prep
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "unmapped":
found = store.unmapped()
if not found:
print("every host word is mapped")
for w in found:
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
return 0
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from app.storage import LATEST, Store
+2 -1
View File
@@ -2,11 +2,12 @@
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
pull.py read every board and search and store the roles they yield
prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold
extract.py title and description into the gate columns, on Haiku
embed.py one vector per open role, for fit
score.py every open role scored for every enabled search agent
summarise.py about 60 words on each of the best roles, on Haiku
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
cascade.py pull → prep → embed → score → ... → summarise, what `TASKS`
offers, the `pull` job a schedule fires, and that schedule
stages.py whether extract and summarise run: the switch, else config.yaml
sources.py what a pull reads, changed by hand: boards and searches
+19 -13
View File
@@ -1,10 +1,13 @@
"""The tasks roledesk offers, and the order a pull runs them in.
pull → embed → score → enrich → embed → score → extract → score → summarise
pull → prep → embed → score → enrich → prep → embed → score → extract → score
→ summarise
newsdesk's cascade convention: a cascade is a tuple of task names, and the
final job of one task starts the next (`step` is the callback it carries).
Nothing is alive between two tasks. A stage is addressed by its position in
Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what
the pull stored and folds the copies before anything reads the roles; after
`enrich` it maps the roles their own pages filled. A stage is addressed by its position in
the tuple, not its name, since `score` runs twice.
Every stage is always in the chain, and a stage that is switched off finds
@@ -12,8 +15,9 @@ nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull d
satisfy, is an extract stage with no batches whose final job moves straight
on. The free stages go first: embed and score cost no tokens, and the score
they produce is what bounds the rest. Enrich reads the best roles without
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
the roles it filled; when it filled none, `step` passes over both. Extract
text on their own pages (`app/jobs/enrich.py`), and prep, embed and score
follow for the roles it filled; when it filled none, `step` passes over all
three. Extract
reads only the best
roles of each enabled agent, as deep and as low as that agent's own `limits`
say; the second score then applies the gates to the columns it filled.
@@ -27,7 +31,7 @@ having either way.
and a cascade is started by one: the `pull` job's only work is to ask for the
chain. Its default schedule, `every 6h`, is declared here beside it.
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
`enrich` is a job and a cascade the same way, `enrich → prep → embed → score`, queued
behind the `score` that follows a change to the ranking other than a pull's
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
one agent's taste a little and queues neither.
@@ -40,20 +44,20 @@ from dataclasses import replace
from functools import partial
from typing import Any
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from app.jobs import embed, enrich, extract, prep, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
CASCADES: dict[str, tuple[str, ...]] = {
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
"summarise"),
"enrich": ("enrich", "embed", "score"),
"pull": ("pull", "prep", "embed", "score", "enrich", "prep", "embed", "score", "extract",
"score", "summarise"),
"enrich": ("enrich", "prep", "embed", "score"),
}
#: What follows `enrich` for the roles it filled, passed over when it filled none.
REFILL = ("embed", "score")
REFILL = ("prep", "embed", "score")
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
@@ -103,6 +107,8 @@ STAGES: dict[str, Callable[[str], Task]] = {
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
"score": lambda cascade: Task(kind="score", finder=_once,
label="Scoring roles for every agent"),
"prep": lambda cascade: Task(kind="prep", finder=_once,
label="Mapping the roles' host words"),
"summarise": _summarise,
"enrich": lambda cascade: enrich.enrich_task(),
}
@@ -189,7 +195,7 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
"""Ask for the enrich cascade, `enrich → prep → embed → score`. Its own work is
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
if not context.task(chain("enrich")):
raise RuntimeError("no queue to start the enrich cascade on")
@@ -210,7 +216,7 @@ def enrich_after(queue: Any, score_job: Any) -> Any:
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be listed twice by `/api/jobs/kinds`.
for _name in ("extract", "embed", "score", "summarise"):
for _name in ("extract", "embed", "score", "summarise", "prep"):
TASKS[_name] = partial(task, _name)
@@ -220,5 +226,5 @@ for _name in ("extract", "embed", "score", "summarise"):
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then embed, score, read the best roles' pages, "
label="Pull every board, then map, embed, score, read the best roles' pages, "
"extract, rescore and summarise")
+31
View File
@@ -0,0 +1,31 @@
"""`prep`: the stage that maps what a pull stored, `app.prep.run`.
In the pull cascade it follows the pull, whose settle has closed what is
gone, and maps every role the pull changed, then folds the copies across
the whole database (`Store.rebuild_dupes`, which the pull's settle used to
run). After `enrich` it maps the roles their own pages filled, since only
those are due. `solo`, so two cascades never write the same rows at once.
"""
from __future__ import annotations
import logging
from typing import Any
from app import prep
from app.config import Config
from app.storage import Store
from pipeline import NULL, RunContext, job
log = logging.getLogger(__name__)
@job("prep", "Mapping the roles' host words", solo=True, resource="cpu")
def run_prep(config: Config, store: Store, *, remap: bool = False,
context: RunContext = NULL) -> dict[str, Any]:
"""Map the due roles, every open one when the tables changed, and fold
the copies. `remap` maps every role, closed ones included."""
done = prep.run(store, remap=remap)
log.info("prep: %d read, %d written, %d words unmapped, %s copies%s", done.read,
done.written, done.unmapped, done.copies, ", tables changed" if done.remapped else "")
return done._asdict()
+4 -6
View File
@@ -244,10 +244,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
After the closes, `Store.rebuild_dupes` recomputes which open roles are
copies of one posting, so a closed kept role hands its group on and the
roles this run added join theirs; `copies` is how many open roles it
leaves hidden as copies.
The `prep` stage that follows maps what this run stored and folds the
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
and the roles this run added join theirs.
"""
answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
@@ -284,10 +283,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
closed += store.close_unseen("search", stale)
if run_marker:
release_run(store, run_marker)
copies = store.rebuild_dupes().copies
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed,
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
"incomplete": incomplete, "roles": roles, "closed": closed}
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
+19
View File
@@ -649,6 +649,13 @@ SENIORITIES: tuple[str, ...] = get_args(Seniority)
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
#: The kind of contract a posting states, as `app.prep` maps a host's word.
#: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the
#: readers' own tests, since a plugin imports nothing from `app`.
Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship",
"freelance"]
EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e)
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
Permit = Literal["", "sponsors", "no_sponsorship"]
PERMITS: tuple[str, ...] = get_args(Permit)
@@ -701,6 +708,16 @@ def trusted_posted(host: str) -> bool:
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
def host_word(value: Any) -> str:
"""A host's word as `roles.*_host` stores it: stripped text, a bool as
'true' or 'false', '' for nothing. What `app.prep` maps."""
if value is None:
return ""
if isinstance(value, bool):
return "true" if value else "false"
return str(value).strip()
def posted_host(raw: dict[str, Any]) -> str:
"""The host a role's dates came from, read the way `ats` and `search`
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
@@ -797,6 +814,8 @@ class Role(Model):
seniority_model: str = ""
seniority_stated: str = ""
seniority_title: str = ""
#: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`.
employment: str = ""
function: str = ""
permit: Permit = ""
#: ISO 639-1 codes the posting says are required, lowercase.
+22
View File
@@ -0,0 +1,22 @@
"""Prep: every decision about a role that needs a rule or data across roles.
Readers scrape and hand on the host's own words (`Store.upsert_role` keeps
them in `roles.*_host`); prep maps them, once, centrally, between the pull
and embedding. One door: `from app.prep import ...`.
words.py seniority, employment and remote: the host's table, the
vocabulary itself, the shared remote words
country.py the country from the resolved places, else the host's
hosts/ one table file per reader (`hosts.TABLES`)
stage.py `run`: the stage over the due roles, `unmapped` and the fold
"""
from __future__ import annotations
from app.prep.country import country
from app.prep.hosts import TABLES
from app.prep.stage import STATE, Prepped, company, fingerprint, run
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
+55
View File
@@ -0,0 +1,55 @@
"""A role's country, from its places, else from the one its host states."""
from __future__ import annotations
from collections.abc import Iterable
from app import places
def of_host(gaz: places.Gazetteer, stated: str) -> str | None:
"""The ISO code a host's country names, in the host's spelling: a code
("DK", "dk"), or a name the gazetteer has as an alias of a country
("Denmark", "Danmark"), `EU` or Europe by name, or a text that resolves to places of one
country. '' for nothing stated, None for a text naming no country."""
text = (stated or "").strip()
if not text:
return ""
if ("country", text.upper()) in gaz.by_key or text.upper() == "EU":
return text.upper()
named = {"EU" if gaz.places[p].key == "europe" else gaz.places[p].key
for p in gaz.aliases.get(places.fold(text), ())
if gaz.places[p].kind == "country"
or (gaz.places[p].kind, gaz.places[p].key) == ("continent", "europe")}
if len(named) == 1:
return named.pop()
found = {gaz.country(p) for p in gaz.resolve(text)} - {""}
return found.pop() if len(found) == 1 else None
def _in_europe(gaz: places.Gazetteer, code: str) -> bool:
if code == "EU":
return True
europe = gaz.by_key.get(("continent", "europe"))
place_id = gaz.by_key.get(("country", code))
return europe is not None and place_id is not None and europe in gaz.ancestors(place_id)
def country(gaz: places.Gazetteer, place_ids: Iterable[int], host_country: str = "") -> str:
"""The one country the role's resolved places lie in, continents aside;
Europe alone is `EU`; else, when they
lie in several all in Europe, `EU` (what `country_of` wrote for "Dublin,
London" and the country gate reads as Europe); else the host's stated
country (`of_host`); else ''."""
ids = list(place_ids)
# A continent beside a country is the wider unit a line adds ("Home
# based - EMEA; London" is GB), as `country_of` had it; alone, Europe
# is `EU`.
found = {gaz.country(p) for p in ids} - {""}
if not found and any(gaz.places[p].key == "europe" for p in ids if p in gaz.places):
found = {"EU"}
if len(found) == 1:
return found.pop()
if len(found) > 1 and all(_in_europe(gaz, code) for code in found):
return "EU"
return of_host(gaz, host_country) or ""
+61
View File
@@ -0,0 +1,61 @@
"""Each host's own words, one module per reader, named as the reader is.
A module defines any of three dicts, keyed by the host's word lowercased:
- `SENIORITY`, values in `app.models.SENIORITIES`
- `EMPLOYMENT`, values in `app.models.EMPLOYMENTS`
- `REMOTE`, values in `app.models.REMOTE` (remote, hybrid, onsite)
A value of "" is a word the host uses that names no value ("Experienced",
"Permanent"): known, so it is not listed as unmapped.
A host whose employment word is a code of several parts (`permanent_full_time`,
`SALARIED_FT`) may add `MATCH`, `[(fragment, value), ...]`, tried in order
after `EMPLOYMENT` with the first hit winning, and must then say how a
fragment matches in `MATCH_MODE`: "substring" (the fragment is in the
lowercased word) or "token" (it equals one of the word's parts split at `_`
and `-`, so `ft` does not match inside `software`). `TABLES` loads every
module of this package by name, so a table is added by adding its file.
"""
from __future__ import annotations
import importlib
import pkgutil
from typing import Any
#: The dict a module names, by the field it maps.
FIELDS = {"seniority": "SENIORITY", "employment": "EMPLOYMENT", "remote": "REMOTE"}
#: How a `MATCH` fragment may match.
MATCH_MODES = ("substring", "token")
def load() -> dict[str, dict[str, Any]]:
"""`host -> field -> {word: value}` for every module of the package, the
words lowercased and stripped. A host without a table for a field has
no entry for it. A `MATCH` is `host -> "match" -> {"mode": ..., "rules":
[[fragment, value], ...]}`; a module with `MATCH` and no valid
`MATCH_MODE` fails the import by name."""
tables: dict[str, dict[str, Any]] = {}
for info in sorted(pkgutil.iter_modules(__path__), key=lambda i: i.name):
if info.name.startswith("_"):
continue
module = importlib.import_module(f"{__name__}.{info.name}")
for field, name in FIELDS.items():
table = getattr(module, name, None)
if isinstance(table, dict):
tables.setdefault(info.name, {})[field] = {
str(k).strip().lower(): str(v) for k, v in table.items()}
rules = getattr(module, "MATCH", None)
if rules is not None:
mode = getattr(module, "MATCH_MODE", None)
if mode not in MATCH_MODES:
raise ValueError(f"app/prep/hosts/{info.name}.py has MATCH and its MATCH_MODE "
f"is {mode!r}, not one of {', '.join(MATCH_MODES)}")
tables.setdefault(info.name, {})["match"] = {
"mode": mode, "rules": [[str(f).lower(), str(v)] for f, v in rules]}
return tables
TABLES = load()
+12
View File
@@ -0,0 +1,12 @@
"""Platsbanken's words, for prep."""
#: `workplace_model.label`, in Swedish; mostly empty. Only "Arbete på plats"
#: is in the fixtures; the others are JobTech's other labels as the site
#: writes them, and a label not here is listed as unmapped.
REMOTE: dict[str, str] = {
"arbete på plats": "onsite", "på plats": "onsite",
"hybridarbete": "hybrid", "hybrid": "hybrid", "delvis på distans": "hybrid",
"delvis distans": "hybrid",
"distansarbete": "remote", "arbete på distans": "remote", "helt på distans": "remote",
"distans": "remote",
}
+139
View File
@@ -0,0 +1,139 @@
"""The prep stage: map what the readers handed on, then fold the copies.
A pull stores the host's own words and place texts (`Store.upsert_role`)
and marks the role due. `run` maps every due role: its company through
`company_aliases`, its places against the gazetteer, its level, contract
and remote through the host's tables (`app.prep.words`), and its country
from the places (`app.prep.country`). When the tables changed since the
last run (`fingerprint`, kept in state as `prep:tables`), every open role
is mapped again from its stored words, with no new pull. Then the words no
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
copies across the whole database.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, Sequence
from typing import Any, NamedTuple
from app import places
from app.models import company_key
from app.models import company_fold
from app.prep import words
from app.prep.country import country, of_host
from app.prep.hosts import TABLES
from app.prep.words import Tables
from app.storage import Store
#: Raised when the rules in this package change the way a stored word maps,
#: so every open role is mapped again as it is after a table changes.
VERSION = 1
#: The state key the last run's `fingerprint` is kept under.
STATE = "prep:tables"
class Prepped(NamedTuple):
"""What one run did: roles read, rows written, whether every role was
mapped again, words left unmapped, and the open copies after the fold
(None when it was not run)."""
read: int
written: int
remapped: bool
unmapped: int
copies: int | None
def fingerprint(tables: Tables = TABLES) -> str:
"""A hash of every host table, the shared remote words and `VERSION`."""
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
sort_keys=True)
return hashlib.sha1(blob.encode()).hexdigest()
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
"""The key a role from `name` belongs to, as `Store.resolve_company`
has it: the company an alias names, the name's own key first when
several share the fold, else `company_key(name)`."""
own = company_key(name)
keys = folds.get(company_fold(name), ())
return own if own in keys or not keys else keys[0]
def _places(raw: Any) -> list[Any]:
try:
found = json.loads(raw) if isinstance(raw, str) else raw
except ValueError:
return []
return found if isinstance(found, list) else []
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
host = str(row["host"] or "")
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
return {
"uid": row["uid"],
"due": row["prep_due"],
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
"seniority_stated": stated,
"seniority_title": titled,
"employment": words.employment(host, row["employment_host"], tables),
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
"remote_places": words.remote(host, None, _places(row["raw_places"]), row["place"],
tables),
"country": country(gaz, place_ids, row["country_host"]),
}
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
return bool(row["prep_due"]) or level != row["seniority"] \
or (found["remote"] and found["remote"] != row["remote"]) \
or (found["remote_places"] and not row["remote"]) \
or (found["country"] and found["country"] != row["country"]) \
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
"seniority_title", "employment"))
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
tables: Tables = TABLES) -> bool:
"""Whether no table, vocabulary or gazetteer knows this host word."""
if field == "country":
return of_host(gaz, word) is None
if field == "remote" and word in ("true", "false"):
return False
return words.lookup(field, host, word, tables) is None
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
dupes: bool = True) -> Prepped:
"""Map the due roles, or every open one when the tables changed, or
every role with `remap`; list the unmapped words; fold the copies."""
mark = fingerprint(tables)
whole = remap or store.state.get(STATE) != mark
rows = store.prep_rows("due")
if whole:
due = {r["uid"] for r in rows}
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
ids = store.resolve_role_places([r["uid"] for r in rows])
gaz = store.gazetteer()
folds = store.company_folds()
written = []
for row in rows:
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
if _changed(row, found):
written.append(found)
store.write_prep(written)
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
store.replace_unmapped(left)
if whole:
store.state[STATE] = mark
copies = store.rebuild_dupes().copies if dupes else None
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
copies=copies)
+110
View File
@@ -0,0 +1,110 @@
"""A host's word to the app's vocabulary: seniority, employment and remote.
Each field is looked up the same way: the host's table in `hosts`, keyed by
the word lowercased, then a word that already is a value of the vocabulary
(what readers stored before they handed on the host's words, and what a
host with the app's own words sends), then, for remote, the shared word map.
An employment word may also match a host's `MATCH` fragments. A word none
of these knows is unmapped: it maps to '' and `lookup` says so,
so the prep stage can list it.
"""
from __future__ import annotations
import re
from collections.abc import Iterable, Mapping
from typing import Any
from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of
from app.prep.hosts import TABLES
from app.seniority import from_title
#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`).
Tables = Mapping[str, Mapping[str, Any]]
VOCABULARIES: dict[str, tuple[str, ...]] = {
"seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE}
#: How hosts without a table of their own write remote, after `_squash`:
#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office").
REMOTE_WORDS: dict[str, str] = {
"remote": "remote", "fullyremote": "remote", "remotefirst": "remote",
"hybrid": "hybrid",
"onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite",
"officebased": "onsite", "onpremise": "onsite",
}
def _match(rules: Mapping[str, Any], key: str) -> str | None:
"""The value of the first `MATCH` fragment `key` holds, by its mode."""
parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set()
for fragment, value in rules["rules"]:
if fragment in parts if rules["mode"] == "token" else fragment in key:
return str(value)
return None
def _squash(word: str) -> str:
return "".join(c for c in word.lower() if c not in " -_")
def lookup(field: str, host: str, word: str,
tables: Tables = TABLES) -> str | None:
"""The value `word` maps to for `host`, '' for a known word that names
none, None for a word nothing knows. An empty word is ''."""
key = (word or "").strip().lower()
if not key:
return ""
table = tables.get(host, {})
found = table.get(field, {}).get(key)
if found is None and field == "employment" and "match" in table:
found = _match(table["match"], key)
if found is not None:
return found
if key in VOCABULARIES[field]:
return key
if field == "remote":
return REMOTE_WORDS.get(_squash(key))
return None
def seniority(host: str, word: str, title: str,
tables: Tables = TABLES) -> tuple[str, str]:
"""`(stated, from the title)`: the host's word through its table, and
`seniority.from_title`. `roles.seniority` is the model's level, else the
stated one, else the title's (`Store.write_prep`)."""
return lookup("seniority", host, word, tables) or "", from_title(title)
def employment(host: str, word: str,
tables: Tables = TABLES) -> str:
return lookup("employment", host, word, tables) or ""
def _place_text(place: Any) -> str:
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
tables: Tables = TABLES) -> str:
"""How the role is worked, one of `REMOTE` or ''.
`value` is the host's own (a bool, or its word; `host_word`'s 'true' and
'false' are bools). True is remote. False says nothing more than the
places do: SmartRecruiters, Ashby, Recruitee and Workable send it for
every role that is not remote, hybrid ones included. A word goes
through the host's table, else the shared map. When the value says
nothing, the place line or the place texts are read for a word
(`remote_of`: "Hybrid, Berlin"), and last a place the reader marked
remote is remote."""
if value is True or value == "true":
return "remote"
said = value not in (None, False, "false")
if said and (found := lookup("remote", host, str(value), tables)):
return found
places = list(places)
if found := remote_of(line or "; ".join(map(_place_text, places))):
return found
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
return "remote"
return ""
+39 -2
View File
@@ -19,7 +19,9 @@
* Under the Follow form of Company boards, a fold lists what each reader
* declares (`/api/readers`): how its list arrives, where the text is, what
* it filters at the source, how it states places, and the robots.txt
* decision taken when it was built.
* decision taken when it was built. A second fold lists the host words no
* table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and
* host, most roles first; it is left out when there are none.
*/
(function () {
"use strict";
@@ -320,6 +322,38 @@
$("#board-form").after(fold);
}
/** The host words prep could not map, one row per field and host. */
function renderUnmapped(words) {
if (!words.length) return;
const groups = new Map();
for (const w of words) {
const key = `${w.field} · ${w.host || "no host"}`;
if (!groups.has(key)) groups.set(key, []);
groups.get(key).push(w);
}
const fold = document.createElement("details");
fold.className = "prune";
fold.id = "unmapped";
const summary = document.createElement("summary");
summary.textContent = `Host words no table maps (${words.length})`;
const list = document.createElement("div");
list.className = "srows";
for (const [key, found] of groups) {
const row = document.createElement("div");
row.className = "srow";
const name = document.createElement("span");
name.className = "name";
name.textContent = key;
const sub = document.createElement("span");
sub.className = "sub";
sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · ");
row.append(name, sub);
list.appendChild(row);
}
fold.append(summary, list);
($("#readers") || $("#board-form")).after(fold);
}
// ------------------------------------------------------ one section shown
// Sources shows one section: the one the fragment names or contains, or the
@@ -360,7 +394,10 @@
document.addEventListener("DOMContentLoaded", async () => {
if (!$("#searches-list")) return;
if ($("#board-form")) api("/api/readers").then(renderReaders, () => {});
if ($("#board-form")) {
api("/api/readers").then(renderReaders, () => {})
.then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {});
}
try {
await load();
} catch (err) {
+33
View File
@@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here.
from __future__ import annotations
import json
import sqlite3
from collections.abc import Iterable, Mapping, Sequence
from typing import Any
@@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None:
"""Replace one role's own places with these texts, unresolved: what a
pull stores, for `app.prep` to resolve (`relink_uids`)."""
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None)
for position, text in enumerate(place_texts)])
def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer,
uids: Sequence[str]) -> dict[str, list[int]]:
"""Resolve the own places of these roles, writing only those that now
resolve differently. Returns `uid -> place ids` in the host's order for
every role of `uids` with places, what prep reads the country from."""
held: dict[str, dict[tuple[int, str], set[int | None]]] = {}
for start in range(0, len(uids), 5000):
chunk = json.dumps(list(uids[start:start + 5000]))
for uid, position, text, place_id in conn.execute(
"SELECT uid, position, text, place_id FROM role_places WHERE url = '' "
"AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid",
(chunk,)):
held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
found: dict[str, list[int]] = {}
changed = []
for uid, rows_of in held.items():
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of}
if now != rows_of:
changed.append((uid, [text for _, text in sorted(rows_of)]))
found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])]
write_many(conn, gaz, changed)
return found
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
"""`write` for many roles in one statement each way. Returns rows written."""
+82 -2
View File
@@ -18,7 +18,9 @@ import sqlite3
from collections.abc import Callable
from app import places
from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host
from app.models import (
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
)
from app.seniority import from_title
from app.storage import gazetteer
@@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles (
teaser TEXT NOT NULL DEFAULT '',
seniority_model TEXT NOT NULL DEFAULT '',
seniority_stated TEXT NOT NULL DEFAULT '',
seniority_title TEXT NOT NULL DEFAULT ''
seniority_title TEXT NOT NULL DEFAULT '',
seniority_host TEXT NOT NULL DEFAULT '',
employment_host TEXT NOT NULL DEFAULT '',
remote_host TEXT NOT NULL DEFAULT '',
country_host TEXT NOT NULL DEFAULT '',
employment TEXT NOT NULL DEFAULT '',
prep_due INTEGER NOT NULL DEFAULT 0
);
-- The *_host columns hold the host's own words as a reader handed them on
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
-- them into seniority_stated, employment, remote and country. prep_due is above 0
-- on a role whose words, company, title or places changed since prep last
-- mapped it (Store.upsert_role), which is what the next prep reads.
-- roles.teaser is the short text a list carries, kept until the full text
-- arrives and embedded while there is none; a later empty teaser does not
-- wipe it. seniority_model, seniority_stated and seniority_title are the
@@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors (
embedded_at TEXT NOT NULL
);
-- A host word no table of app/prep/hosts maps, per field (seniority,
-- employment, remote, country) and host: how many open roles carry it and
-- when prep first met it. Rewritten by every prep; a word added to a table
-- leaves at the next one.
CREATE TABLE IF NOT EXISTS unmapped (
field TEXT NOT NULL,
host TEXT NOT NULL,
word TEXT NOT NULL,
roles INTEGER NOT NULL DEFAULT 0,
first_seen TEXT NOT NULL,
PRIMARY KEY (field, host, word)
);
-- The string map a desk plugin is handed as self.state. One namespace shared
-- by every plugin, so a plugin prefixes its keys.
CREATE TABLE IF NOT EXISTS state (
@@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
-- The roles the next prep maps. Partial: after a prep there are none.
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
"""
#: The table whose absence marks a file that has never been a roledesk
@@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
lambda conn: _add_all_roles(conn)),
# A Python step: the host words are read out of raw's JSON.
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None:
"to tag; it reads the agents' reach now", target, key)
_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment")
_V22 = """
CREATE TABLE IF NOT EXISTS unmapped (
field TEXT NOT NULL,
host TEXT NOT NULL,
word TEXT NOT NULL,
roles INTEGER NOT NULL DEFAULT 0,
first_seen TEXT NOT NULL,
PRIMARY KEY (field, host, word)
);
DELETE FROM state WHERE key = 'prep:tables';
"""
def _add_host_words(conn: sqlite3.Connection) -> None:
"""Migration 22's step. The columns and `unmapped`, and the host words
of every role filled from its `raw` where it has them. Before this,
readers mapped `seniority` and `employment` in their own tables, so what
`raw` holds there is mostly a value of the vocabulary already, which
`app.prep` takes as it is. `remote` and `country` are the host's as
given. The `prep:tables` fingerprint goes, so the first prep maps every
open role from these words. Runs twice without harm."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
for column in _V22_COLUMNS:
if column not in columns:
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
if "prep_due" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0")
for statement in _statements(_V22):
conn.execute(statement)
updates = []
for uid, raw in conn.execute(
"SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL "
"OR json_extract(raw, '$.employment') IS NOT NULL "
"OR json_extract(raw, '$.remote') IS NOT NULL "
"OR json_extract(raw, '$.country') IS NOT NULL").fetchall():
try:
given = json.loads(raw or "{}")
except ValueError:
continue
if isinstance(given, dict):
updates.append((*(host_word(given.get(k)) for k in (
"seniority", "employment", "remote", "country")), uid))
conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, "
"remote_host = ?, country_host = ? WHERE uid = ?", updates)
log.info("migration 22: host words from raw for %d roles", len(updates))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
+140 -44
View File
@@ -22,13 +22,12 @@ import numpy as np
from desk.models import Item
from app import dupes, places
from app.models import (
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
RoleRow, ScoredRole,
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
posted_at, posted_host, remote_of, title_key,
host_word, posted_at, posted_host, remote_of, title_key,
)
from app.seniority import from_title
from app.storage import gazetteer, schema
#: Text columns a plugin may fill from `Item.raw` under the same name. An
@@ -36,7 +35,17 @@ from app.storage import gazetteer, schema
#: says less than `extract` found keeps what `extract` found. `function` is
#: not here: it is not a board field (see the note by `keep_filled` below).
#: Nor is `seniority`, which is the title's level unless a board states one.
_RAW_TEXT = ("place", "country", "currency")
_RAW_TEXT = ("place", "currency")
#: The host's own words a reader hands on in `Item.raw`, stored as
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
#: rewritten every pull, as the host's level always was; the others are kept
#: filled, since a role's own page (`enrich`) may state what its listing
#: does not.
_HOST_WORDS = ("seniority", "employment", "remote", "country")
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
#: What prep maps from that a pull rewrites whatever it brings.
_PREP_OVERWRITTEN = ("title", "seniority_host")
_RAW_INT = ("salary_min", "salary_max")
#: What `extract` may fill. Only an empty column is written: a board's own
@@ -154,14 +163,11 @@ def _iso(moment: datetime) -> str:
return moment.astimezone(UTC).isoformat(timespec="seconds")
def _remote(value: Any) -> str:
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
unrecognised is '' (unknown), which `extract` may fill later."""
if value is True:
return "remote"
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
return text if text in REMOTE else ""
def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]:
"""One role's own place texts as stored, in the host's order."""
return list(dict.fromkeys(r[0] for r in conn.execute(
"SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid",
(uid,))))
def _like(text: str) -> str:
@@ -607,10 +613,19 @@ class Store:
"""Store one posting. Returns True when it was new.
`first_seen` and `created_at` are written on insert only; `last_seen` on
every call, and a role seen again is open again. The gate columns come
every call, and a role seen again is open again. The raw columns come
from `item.raw` under their own names, with `company` falling back to
the subscription's and `country` to what `place` names; an empty value
never overwrites a filled column.
the subscription's; an empty value never overwrites a filled column.
Nothing is mapped here: that is `app.prep`, which runs after every
pull. The host's words go to `seniority_host`, `employment_host`,
`remote_host` and `country_host` (`host_word`), `company_key` is
written unresolved on a new role only, and the mapped columns
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
`remote`, `country`) are left as prep last wrote them. `prep_due`
is 1 on a new role and grows by one when the title, the company, a
host word or the place texts changed or the role reopens, which
is what the next prep reads.
`posted_at` (see `app.models.posted_at`) is set from this pull's own
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
@@ -627,20 +642,14 @@ class Store:
alone for good, whatever the plugin's raw guess now says — see
`set_extracted` for the other half of the rule.
`seniority` is the first filled of three levels. `seniority_model`
is extract's (`set_extracted`) and never written here.
`seniority_stated` is the host's word as `raw["seniority"]` carries
it, already in `SENIORITIES` (anything else is ''), and
`seniority_title` is `seniority.from_title`; both are rewritten every
pull. The classifier is called here rather than in a plugin, so a
role from any source gets it, the way `country` comes from `place`.
The role's places (`role_places`) are the texts of `raw["places"]`,
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
against the gazetteer; `place` is the line shown, the texts joined
when `raw` has none. Like `place`, a pull with no place leaves them
as they are. `place_fold` is written with `place`, under the same
keep-filled rule. `teaser` is kept-filled too.
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
when they differ from the stored ones; prep resolves them. Texts that
did not change keep their resolved rows, so the feed's Place filter
stays right between a pull and its prep. `place` is the line shown,
the texts joined when `raw` has none. Like `place`, a pull with no
place leaves them as they are. `place_fold` is written with `place`,
under the same keep-filled rule. `teaser` is kept-filled too.
A role is embedded from its description, else its teaser, else its
title (`app.jobs.embed.document`). When a pull brings a better one of
@@ -660,12 +669,11 @@ class Store:
"source": item.source,
"source_id": item.source_id,
"subscription_id": subscription.id if subscription else None,
"company_key": self.resolve_company(company),
"company_key": company_key(company),
"company": company,
"title": item.title,
"title_key": title_key(item.title),
"place_key": place_key(place),
"remote": _remote(raw.get("remote")),
"description": item.body,
"feed_text": item.excerpt,
"original_url": item.location,
@@ -677,30 +685,26 @@ class Store:
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS}
row["place"] = place
row["country"] = row["country"] or country_of(place)
row["remote"] = row["remote"] or remote_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
row["function"] = str(raw.get("function") or "")
stated = str(raw.get("seniority") or "")
row["seniority_stated"] = stated if stated in SENIORITIES else ""
row["seniority_title"] = from_title(item.title)
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
row["teaser"] = str(raw.get("teaser") or "")
row["place_fold"] = places.fold(row["place"])
# A board's own pay line, one per pull like title or published_at —
# not kept-filled like _RAW_TEXT: a board that stops stating pay
# must be able to clear it, not leave a stale line behind.
row["salary_text"] = str(raw.get("salary_text") or "")
row["prep_due"] = 1
keep_filled = ("company", "company_key", "place_key", "remote", "description",
"feed_text", "original_url", "teaser", *_RAW_TEXT)
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
"teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS)
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
for c in (*_RAW_INT, "subscription_id")]
updates += [f"{c}=excluded.{c}"
for c in ("title", "title_key", "published_at", "raw", "last_seen",
"salary_text", "seniority_stated", "seniority_title")]
"salary_text", "seniority_host")]
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
# pull's classifier guess overwrite what extract decided, which is
@@ -711,11 +715,17 @@ class Store:
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
"ELSE roles.function END")
# The model's level, else the host's, else the title's.
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
updates.append("place_fold=CASE WHEN excluded.place != '' "
"THEN excluded.place_fold ELSE roles.place_fold END")
# One more than before when what prep maps from changed, or the role
# reopens, so a prep that read the old values leaves it due
# (`Store.prepped`). `roles.` is the stored row here, `excluded.`
# what this pull brought.
changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN]
changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})"
for c in ("company", *_KEPT_HOST_WORDS)]
updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} "
"THEN roles.prep_due + 1 ELSE roles.prep_due END")
updates.append("closed_at=NULL")
columns = ", ".join(row)
marks = ", ".join(f":{c}" for c in row)
@@ -725,8 +735,11 @@ class Store:
conn.execute(
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
if place_texts:
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts):
gazetteer.write_texts(conn, row["uid"], place_texts)
if before is not None:
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
(row["uid"],))
# `embed` encodes a role once, from the best text it has. When a
# later pull brings a better one, as a host that states the text
# on the role's own page does, the vector goes, so the next
@@ -1253,6 +1266,89 @@ class Store:
remotes)
return len(countries) + len(remotes)
# ----------------------------------------------------------------- prep
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
"""What `app.prep` maps a role from, and what it holds now: the roles
due (`prep_due` above 0), every open role, or every role."""
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
return self._rows(
f"""SELECT uid, prep_due, {_HOST} AS host, title, company, company_key, place,
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country,
json_extract(raw, '$.places') AS raw_places
FROM roles WHERE {where}""")
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
"""Resolve these roles' own places against the gazetteer, writing the
rows that resolve differently; `uid -> place ids` for each that has
places (`gazetteer.relink_uids`)."""
with self.transaction() as conn:
return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids))
def company_folds(self) -> dict[str, list[str]]:
"""`fold -> company keys` of `company_aliases`, sorted: what
`resolve_company` asks one name at a time, for prep's whole batch."""
out: dict[str, list[str]] = {}
for r in self._rows("SELECT fold, company_key FROM company_aliases "
"ORDER BY fold, company_key"):
out.setdefault(str(r["fold"]), []).append(str(r["company_key"]))
return out
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_places` and
`country`. `seniority` is the model's level, else the stated, else
the title's. An empty `remote` or `country` leaves the column as it
is, since `extract` fills an empty one. `remote_places`, what the
place texts say, only fills an empty column: it is a guess from a
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
changed the role since it was read. Returns the rows written."""
with self.transaction() as conn:
return conn.executemany(
"""UPDATE roles SET company_key = :company_key,
seniority_stated = :seniority_stated,
seniority_title = :seniority_title,
seniority = COALESCE(NULLIF(seniority_model, ''),
NULLIF(:seniority_stated, ''), :seniority_title),
employment = :employment,
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
:remote_places),
country = COALESCE(NULLIF(:country, ''), country),
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
WHERE uid = :uid""", mapped).rowcount
def host_words(self) -> list[tuple[str, str, str, int]]:
"""`(field, host, word, open roles)` for every host word an open role
carries, the word lowercased: what prep checks against its tables."""
found: list[tuple[str, str, str, int]] = []
for field in ("seniority", "employment", "remote", "country"):
found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows(
f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles
WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")]
return found
def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None:
"""Make `unmapped` these `(field, host, word, roles)`, keeping the
`first_seen` of a word already listed."""
stamp = self.now()
with self.transaction() as conn:
seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute(
"SELECT field, host, word, first_seen FROM unmapped")}
conn.execute("DELETE FROM unmapped")
conn.executemany(
"INSERT INTO unmapped (field, host, word, roles, first_seen) "
"VALUES (?, ?, ?, ?, ?)",
[(*r, seen.get(r[:3], stamp)) for r in rows])
def unmapped(self) -> list[dict[str, Any]]:
"""The host words no table maps, by field and host, most roles first."""
return [dict(r) for r in self._rows(
"SELECT field, host, word, roles, first_seen FROM unmapped "
"ORDER BY field, host, roles DESC, word")]
# -------------------------------------------------------------- extract
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
+17
View File
@@ -897,6 +897,23 @@ def list_readers() -> list[ReaderOut]:
for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))]
class UnmappedOut(BaseModel):
"""A host word no table of `app/prep/hosts` maps (`Store.unmapped`)."""
field: str
host: str
word: str
roles: int
first_seen: str
@router.get("/prep/unmapped")
def list_unmapped(store: StoreDep) -> list[UnmappedOut]:
"""The host words the last prep could not map, per field and host, most
roles first: what a table in `app/prep/hosts` should learn."""
return [UnmappedOut(**w) for w in store.unmapped()]
class SearchBody(BaseModel):
provider: str
name: str = ""