Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
+31
-1
@@ -8,6 +8,7 @@
|
||||
app agents list|prompt
|
||||
app stages [extract|summarise on|off]
|
||||
app dupes
|
||||
app prep [unmapped] [--remap]
|
||||
app db init
|
||||
|
||||
`app job` queues the work on a queue of its own and drains it here, so a
|
||||
@@ -160,6 +161,13 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
||||
|
||||
prep = sub.add_parser(
|
||||
"prep", help="map the roles' host words and fold the copies, or list the unmapped words")
|
||||
prep.add_argument("action", nargs="?", choices=["unmapped"],
|
||||
help="list the host words no table maps, instead of running")
|
||||
prep.add_argument("--remap", action="store_true",
|
||||
help="map every role again, closed ones included")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
return parser
|
||||
@@ -188,6 +196,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return run_seniority(args)
|
||||
if args.command == "dupes":
|
||||
return run_dupes(args)
|
||||
if args.command == "prep":
|
||||
return run_prep(args)
|
||||
return init_db(args)
|
||||
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
@@ -655,7 +665,7 @@ def run_seniority(args: argparse.Namespace) -> int:
|
||||
|
||||
|
||||
def run_dupes(args: argparse.Namespace) -> int:
|
||||
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
|
||||
"""`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`,
|
||||
now. Prints the groups (kept roles with a copy), the copies the feed
|
||||
hides and the rows whose `dup_of` changed, as JSON."""
|
||||
from app.storage import Store
|
||||
@@ -666,6 +676,26 @@ def run_dupes(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def run_prep(args: argparse.Namespace) -> int:
|
||||
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
|
||||
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
|
||||
host words no table maps, per field and host, most roles first."""
|
||||
from app import prep
|
||||
from app.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
if args.action == "unmapped":
|
||||
found = store.unmapped()
|
||||
if not found:
|
||||
print("every host word is mapped")
|
||||
for w in found:
|
||||
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
|
||||
return 0
|
||||
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
|
||||
return 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from app.storage import LATEST, Store
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||
|
||||
pull.py read every board and search and store the roles they yield
|
||||
prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold
|
||||
extract.py title and description into the gate columns, on Haiku
|
||||
embed.py one vector per open role, for fit
|
||||
score.py every open role scored for every enabled search agent
|
||||
summarise.py about 60 words on each of the best roles, on Haiku
|
||||
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
|
||||
cascade.py pull → prep → embed → score → ... → summarise, what `TASKS`
|
||||
offers, the `pull` job a schedule fires, and that schedule
|
||||
stages.py whether extract and summarise run: the switch, else config.yaml
|
||||
sources.py what a pull reads, changed by hand: boards and searches
|
||||
|
||||
+19
-13
@@ -1,10 +1,13 @@
|
||||
"""The tasks roledesk offers, and the order a pull runs them in.
|
||||
|
||||
pull → embed → score → enrich → embed → score → extract → score → summarise
|
||||
pull → prep → embed → score → enrich → prep → embed → score → extract → score
|
||||
→ summarise
|
||||
|
||||
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
||||
final job of one task starts the next (`step` is the callback it carries).
|
||||
Nothing is alive between two tasks. A stage is addressed by its position in
|
||||
Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what
|
||||
the pull stored and folds the copies before anything reads the roles; after
|
||||
`enrich` it maps the roles their own pages filled. A stage is addressed by its position in
|
||||
the tuple, not its name, since `score` runs twice.
|
||||
|
||||
Every stage is always in the chain, and a stage that is switched off finds
|
||||
@@ -12,8 +15,9 @@ nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull d
|
||||
satisfy, is an extract stage with no batches whose final job moves straight
|
||||
on. The free stages go first: embed and score cost no tokens, and the score
|
||||
they produce is what bounds the rest. Enrich reads the best roles without
|
||||
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
|
||||
the roles it filled; when it filled none, `step` passes over both. Extract
|
||||
text on their own pages (`app/jobs/enrich.py`), and prep, embed and score
|
||||
follow for the roles it filled; when it filled none, `step` passes over all
|
||||
three. Extract
|
||||
reads only the best
|
||||
roles of each enabled agent, as deep and as low as that agent's own `limits`
|
||||
say; the second score then applies the gates to the columns it filled.
|
||||
@@ -27,7 +31,7 @@ having either way.
|
||||
and a cascade is started by one: the `pull` job's only work is to ask for the
|
||||
chain. Its default schedule, `every 6h`, is declared here beside it.
|
||||
|
||||
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
|
||||
`enrich` is a job and a cascade the same way, `enrich → prep → embed → score`, queued
|
||||
behind the `score` that follows a change to the ranking other than a pull's
|
||||
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
|
||||
one agent's taste a little and queues neither.
|
||||
@@ -40,20 +44,20 @@ from dataclasses import replace
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||
from app.jobs import embed, enrich, extract, prep, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||
from desk.tools import agent
|
||||
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
||||
|
||||
STEP = "app.jobs.cascade:step"
|
||||
|
||||
CASCADES: dict[str, tuple[str, ...]] = {
|
||||
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
|
||||
"summarise"),
|
||||
"enrich": ("enrich", "embed", "score"),
|
||||
"pull": ("pull", "prep", "embed", "score", "enrich", "prep", "embed", "score", "extract",
|
||||
"score", "summarise"),
|
||||
"enrich": ("enrich", "prep", "embed", "score"),
|
||||
}
|
||||
|
||||
#: What follows `enrich` for the roles it filled, passed over when it filled none.
|
||||
REFILL = ("embed", "score")
|
||||
REFILL = ("prep", "embed", "score")
|
||||
|
||||
|
||||
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
||||
@@ -103,6 +107,8 @@ STAGES: dict[str, Callable[[str], Task]] = {
|
||||
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
|
||||
"score": lambda cascade: Task(kind="score", finder=_once,
|
||||
label="Scoring roles for every agent"),
|
||||
"prep": lambda cascade: Task(kind="prep", finder=_once,
|
||||
label="Mapping the roles' host words"),
|
||||
"summarise": _summarise,
|
||||
"enrich": lambda cascade: enrich.enrich_task(),
|
||||
}
|
||||
@@ -189,7 +195,7 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
|
||||
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
|
||||
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
|
||||
"""Ask for the enrich cascade, `enrich → prep → embed → score`. Its own work is
|
||||
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
|
||||
if not context.task(chain("enrich")):
|
||||
raise RuntimeError("no queue to start the enrich cascade on")
|
||||
@@ -210,7 +216,7 @@ def enrich_after(queue: Any, score_job: Any) -> Any:
|
||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||
#: is both would be listed twice by `/api/jobs/kinds`.
|
||||
for _name in ("extract", "embed", "score", "summarise"):
|
||||
for _name in ("extract", "embed", "score", "summarise", "prep"):
|
||||
TASKS[_name] = partial(task, _name)
|
||||
|
||||
|
||||
@@ -220,5 +226,5 @@ for _name in ("extract", "embed", "score", "summarise"):
|
||||
# setting: a cadence edited or switched off under Runs survives
|
||||
# every restart (see pipeline/schedule.py).
|
||||
declare("pull", "pull", "every 6h",
|
||||
label="Pull every board, then embed, score, read the best roles' pages, "
|
||||
label="Pull every board, then map, embed, score, read the best roles' pages, "
|
||||
"extract, rescore and summarise")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""`prep`: the stage that maps what a pull stored, `app.prep.run`.
|
||||
|
||||
In the pull cascade it follows the pull, whose settle has closed what is
|
||||
gone, and maps every role the pull changed, then folds the copies across
|
||||
the whole database (`Store.rebuild_dupes`, which the pull's settle used to
|
||||
run). After `enrich` it maps the roles their own pages filled, since only
|
||||
those are due. `solo`, so two cascades never write the same rows at once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app import prep
|
||||
from app.config import Config
|
||||
from app.storage import Store
|
||||
from pipeline import NULL, RunContext, job
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@job("prep", "Mapping the roles' host words", solo=True, resource="cpu")
|
||||
def run_prep(config: Config, store: Store, *, remap: bool = False,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Map the due roles, every open one when the tables changed, and fold
|
||||
the copies. `remap` maps every role, closed ones included."""
|
||||
done = prep.run(store, remap=remap)
|
||||
log.info("prep: %d read, %d written, %d words unmapped, %s copies%s", done.read,
|
||||
done.written, done.unmapped, done.copies, ", tables changed" if done.remapped else "")
|
||||
return done._asdict()
|
||||
+4
-6
@@ -244,10 +244,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
|
||||
After the closes, `Store.rebuild_dupes` recomputes which open roles are
|
||||
copies of one posting, so a closed kept role hands its group on and the
|
||||
roles this run added join theirs; `copies` is how many open roles it
|
||||
leaves hidden as copies.
|
||||
The `prep` stage that follows maps what this run stored and folds the
|
||||
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
||||
and the roles this run added join theirs.
|
||||
"""
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
@@ -284,10 +283,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
release_run(store, run_marker)
|
||||
copies = store.rebuild_dupes().copies
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
||||
|
||||
@@ -649,6 +649,13 @@ SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
||||
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
|
||||
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
|
||||
|
||||
#: The kind of contract a posting states, as `app.prep` maps a host's word.
|
||||
#: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the
|
||||
#: readers' own tests, since a plugin imports nothing from `app`.
|
||||
Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship",
|
||||
"freelance"]
|
||||
EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e)
|
||||
|
||||
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
||||
Permit = Literal["", "sponsors", "no_sponsorship"]
|
||||
PERMITS: tuple[str, ...] = get_args(Permit)
|
||||
@@ -701,6 +708,16 @@ def trusted_posted(host: str) -> bool:
|
||||
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
||||
|
||||
|
||||
def host_word(value: Any) -> str:
|
||||
"""A host's word as `roles.*_host` stores it: stripped text, a bool as
|
||||
'true' or 'false', '' for nothing. What `app.prep` maps."""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def posted_host(raw: dict[str, Any]) -> str:
|
||||
"""The host a role's dates came from, read the way `ats` and `search`
|
||||
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
|
||||
@@ -797,6 +814,8 @@ class Role(Model):
|
||||
seniority_model: str = ""
|
||||
seniority_stated: str = ""
|
||||
seniority_title: str = ""
|
||||
#: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`.
|
||||
employment: str = ""
|
||||
function: str = ""
|
||||
permit: Permit = ""
|
||||
#: ISO 639-1 codes the posting says are required, lowercase.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Prep: every decision about a role that needs a rule or data across roles.
|
||||
|
||||
Readers scrape and hand on the host's own words (`Store.upsert_role` keeps
|
||||
them in `roles.*_host`); prep maps them, once, centrally, between the pull
|
||||
and embedding. One door: `from app.prep import ...`.
|
||||
|
||||
words.py seniority, employment and remote: the host's table, the
|
||||
vocabulary itself, the shared remote words
|
||||
country.py the country from the resolved places, else the host's
|
||||
hosts/ one table file per reader (`hosts.TABLES`)
|
||||
stage.py `run`: the stage over the due roles, `unmapped` and the fold
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.prep.country import country
|
||||
from app.prep.hosts import TABLES
|
||||
from app.prep.stage import STATE, Prepped, company, fingerprint, run
|
||||
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
|
||||
|
||||
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
|
||||
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
|
||||
@@ -0,0 +1,55 @@
|
||||
"""A role's country, from its places, else from the one its host states."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from app import places
|
||||
|
||||
|
||||
def of_host(gaz: places.Gazetteer, stated: str) -> str | None:
|
||||
"""The ISO code a host's country names, in the host's spelling: a code
|
||||
("DK", "dk"), or a name the gazetteer has as an alias of a country
|
||||
("Denmark", "Danmark"), `EU` or Europe by name, or a text that resolves to places of one
|
||||
country. '' for nothing stated, None for a text naming no country."""
|
||||
text = (stated or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
if ("country", text.upper()) in gaz.by_key or text.upper() == "EU":
|
||||
return text.upper()
|
||||
named = {"EU" if gaz.places[p].key == "europe" else gaz.places[p].key
|
||||
for p in gaz.aliases.get(places.fold(text), ())
|
||||
if gaz.places[p].kind == "country"
|
||||
or (gaz.places[p].kind, gaz.places[p].key) == ("continent", "europe")}
|
||||
if len(named) == 1:
|
||||
return named.pop()
|
||||
found = {gaz.country(p) for p in gaz.resolve(text)} - {""}
|
||||
return found.pop() if len(found) == 1 else None
|
||||
|
||||
|
||||
def _in_europe(gaz: places.Gazetteer, code: str) -> bool:
|
||||
if code == "EU":
|
||||
return True
|
||||
europe = gaz.by_key.get(("continent", "europe"))
|
||||
place_id = gaz.by_key.get(("country", code))
|
||||
return europe is not None and place_id is not None and europe in gaz.ancestors(place_id)
|
||||
|
||||
|
||||
def country(gaz: places.Gazetteer, place_ids: Iterable[int], host_country: str = "") -> str:
|
||||
"""The one country the role's resolved places lie in, continents aside;
|
||||
Europe alone is `EU`; else, when they
|
||||
lie in several all in Europe, `EU` (what `country_of` wrote for "Dublin,
|
||||
London" and the country gate reads as Europe); else the host's stated
|
||||
country (`of_host`); else ''."""
|
||||
ids = list(place_ids)
|
||||
# A continent beside a country is the wider unit a line adds ("Home
|
||||
# based - EMEA; London" is GB), as `country_of` had it; alone, Europe
|
||||
# is `EU`.
|
||||
found = {gaz.country(p) for p in ids} - {""}
|
||||
if not found and any(gaz.places[p].key == "europe" for p in ids if p in gaz.places):
|
||||
found = {"EU"}
|
||||
if len(found) == 1:
|
||||
return found.pop()
|
||||
if len(found) > 1 and all(_in_europe(gaz, code) for code in found):
|
||||
return "EU"
|
||||
return of_host(gaz, host_country) or ""
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Each host's own words, one module per reader, named as the reader is.
|
||||
|
||||
A module defines any of three dicts, keyed by the host's word lowercased:
|
||||
|
||||
- `SENIORITY`, values in `app.models.SENIORITIES`
|
||||
- `EMPLOYMENT`, values in `app.models.EMPLOYMENTS`
|
||||
- `REMOTE`, values in `app.models.REMOTE` (remote, hybrid, onsite)
|
||||
|
||||
A value of "" is a word the host uses that names no value ("Experienced",
|
||||
"Permanent"): known, so it is not listed as unmapped.
|
||||
|
||||
A host whose employment word is a code of several parts (`permanent_full_time`,
|
||||
`SALARIED_FT`) may add `MATCH`, `[(fragment, value), ...]`, tried in order
|
||||
after `EMPLOYMENT` with the first hit winning, and must then say how a
|
||||
fragment matches in `MATCH_MODE`: "substring" (the fragment is in the
|
||||
lowercased word) or "token" (it equals one of the word's parts split at `_`
|
||||
and `-`, so `ft` does not match inside `software`). `TABLES` loads every
|
||||
module of this package by name, so a table is added by adding its file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
from typing import Any
|
||||
|
||||
#: The dict a module names, by the field it maps.
|
||||
FIELDS = {"seniority": "SENIORITY", "employment": "EMPLOYMENT", "remote": "REMOTE"}
|
||||
|
||||
#: How a `MATCH` fragment may match.
|
||||
MATCH_MODES = ("substring", "token")
|
||||
|
||||
|
||||
def load() -> dict[str, dict[str, Any]]:
|
||||
"""`host -> field -> {word: value}` for every module of the package, the
|
||||
words lowercased and stripped. A host without a table for a field has
|
||||
no entry for it. A `MATCH` is `host -> "match" -> {"mode": ..., "rules":
|
||||
[[fragment, value], ...]}`; a module with `MATCH` and no valid
|
||||
`MATCH_MODE` fails the import by name."""
|
||||
tables: dict[str, dict[str, Any]] = {}
|
||||
for info in sorted(pkgutil.iter_modules(__path__), key=lambda i: i.name):
|
||||
if info.name.startswith("_"):
|
||||
continue
|
||||
module = importlib.import_module(f"{__name__}.{info.name}")
|
||||
for field, name in FIELDS.items():
|
||||
table = getattr(module, name, None)
|
||||
if isinstance(table, dict):
|
||||
tables.setdefault(info.name, {})[field] = {
|
||||
str(k).strip().lower(): str(v) for k, v in table.items()}
|
||||
rules = getattr(module, "MATCH", None)
|
||||
if rules is not None:
|
||||
mode = getattr(module, "MATCH_MODE", None)
|
||||
if mode not in MATCH_MODES:
|
||||
raise ValueError(f"app/prep/hosts/{info.name}.py has MATCH and its MATCH_MODE "
|
||||
f"is {mode!r}, not one of {', '.join(MATCH_MODES)}")
|
||||
tables.setdefault(info.name, {})["match"] = {
|
||||
"mode": mode, "rules": [[str(f).lower(), str(v)] for f, v in rules]}
|
||||
return tables
|
||||
|
||||
|
||||
TABLES = load()
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Platsbanken's words, for prep."""
|
||||
|
||||
#: `workplace_model.label`, in Swedish; mostly empty. Only "Arbete på plats"
|
||||
#: is in the fixtures; the others are JobTech's other labels as the site
|
||||
#: writes them, and a label not here is listed as unmapped.
|
||||
REMOTE: dict[str, str] = {
|
||||
"arbete på plats": "onsite", "på plats": "onsite",
|
||||
"hybridarbete": "hybrid", "hybrid": "hybrid", "delvis på distans": "hybrid",
|
||||
"delvis distans": "hybrid",
|
||||
"distansarbete": "remote", "arbete på distans": "remote", "helt på distans": "remote",
|
||||
"distans": "remote",
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
"""The prep stage: map what the readers handed on, then fold the copies.
|
||||
|
||||
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
||||
and marks the role due. `run` maps every due role: its company through
|
||||
`company_aliases`, its places against the gazetteer, its level, contract
|
||||
and remote through the host's tables (`app.prep.words`), and its country
|
||||
from the places (`app.prep.country`). When the tables changed since the
|
||||
last run (`fingerprint`, kept in state as `prep:tables`), every open role
|
||||
is mapped again from its stored words, with no new pull. Then the words no
|
||||
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
|
||||
copies across the whole database.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from app import places
|
||||
from app.models import company_key
|
||||
from app.models import company_fold
|
||||
from app.prep import words
|
||||
from app.prep.country import country, of_host
|
||||
from app.prep.hosts import TABLES
|
||||
from app.prep.words import Tables
|
||||
from app.storage import Store
|
||||
|
||||
#: Raised when the rules in this package change the way a stored word maps,
|
||||
#: so every open role is mapped again as it is after a table changes.
|
||||
VERSION = 1
|
||||
|
||||
#: The state key the last run's `fingerprint` is kept under.
|
||||
STATE = "prep:tables"
|
||||
|
||||
|
||||
|
||||
class Prepped(NamedTuple):
|
||||
"""What one run did: roles read, rows written, whether every role was
|
||||
mapped again, words left unmapped, and the open copies after the fold
|
||||
(None when it was not run)."""
|
||||
|
||||
read: int
|
||||
written: int
|
||||
remapped: bool
|
||||
unmapped: int
|
||||
copies: int | None
|
||||
|
||||
|
||||
def fingerprint(tables: Tables = TABLES) -> str:
|
||||
"""A hash of every host table, the shared remote words and `VERSION`."""
|
||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
|
||||
sort_keys=True)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()
|
||||
|
||||
|
||||
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
|
||||
"""The key a role from `name` belongs to, as `Store.resolve_company`
|
||||
has it: the company an alias names, the name's own key first when
|
||||
several share the fold, else `company_key(name)`."""
|
||||
own = company_key(name)
|
||||
keys = folds.get(company_fold(name), ())
|
||||
return own if own in keys or not keys else keys[0]
|
||||
|
||||
|
||||
def _places(raw: Any) -> list[Any]:
|
||||
try:
|
||||
found = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except ValueError:
|
||||
return []
|
||||
return found if isinstance(found, list) else []
|
||||
|
||||
|
||||
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
||||
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
||||
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
||||
host = str(row["host"] or "")
|
||||
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
||||
return {
|
||||
"uid": row["uid"],
|
||||
"due": row["prep_due"],
|
||||
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
|
||||
"seniority_stated": stated,
|
||||
"seniority_title": titled,
|
||||
"employment": words.employment(host, row["employment_host"], tables),
|
||||
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
|
||||
"remote_places": words.remote(host, None, _places(row["raw_places"]), row["place"],
|
||||
tables),
|
||||
"country": country(gaz, place_ids, row["country_host"]),
|
||||
}
|
||||
|
||||
|
||||
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||
or (found["remote"] and found["remote"] != row["remote"]) \
|
||||
or (found["remote_places"] and not row["remote"]) \
|
||||
or (found["country"] and found["country"] != row["country"]) \
|
||||
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
||||
"seniority_title", "employment"))
|
||||
|
||||
|
||||
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
|
||||
tables: Tables = TABLES) -> bool:
|
||||
"""Whether no table, vocabulary or gazetteer knows this host word."""
|
||||
if field == "country":
|
||||
return of_host(gaz, word) is None
|
||||
if field == "remote" and word in ("true", "false"):
|
||||
return False
|
||||
return words.lookup(field, host, word, tables) is None
|
||||
|
||||
|
||||
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
||||
dupes: bool = True) -> Prepped:
|
||||
"""Map the due roles, or every open one when the tables changed, or
|
||||
every role with `remap`; list the unmapped words; fold the copies."""
|
||||
mark = fingerprint(tables)
|
||||
whole = remap or store.state.get(STATE) != mark
|
||||
rows = store.prep_rows("due")
|
||||
if whole:
|
||||
due = {r["uid"] for r in rows}
|
||||
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
|
||||
ids = store.resolve_role_places([r["uid"] for r in rows])
|
||||
gaz = store.gazetteer()
|
||||
folds = store.company_folds()
|
||||
written = []
|
||||
for row in rows:
|
||||
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
|
||||
if _changed(row, found):
|
||||
written.append(found)
|
||||
store.write_prep(written)
|
||||
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
|
||||
store.replace_unmapped(left)
|
||||
if whole:
|
||||
store.state[STATE] = mark
|
||||
copies = store.rebuild_dupes().copies if dupes else None
|
||||
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
||||
copies=copies)
|
||||
@@ -0,0 +1,110 @@
|
||||
"""A host's word to the app's vocabulary: seniority, employment and remote.
|
||||
|
||||
Each field is looked up the same way: the host's table in `hosts`, keyed by
|
||||
the word lowercased, then a word that already is a value of the vocabulary
|
||||
(what readers stored before they handed on the host's words, and what a
|
||||
host with the app's own words sends), then, for remote, the shared word map.
|
||||
An employment word may also match a host's `MATCH` fragments. A word none
|
||||
of these knows is unmapped: it maps to '' and `lookup` says so,
|
||||
so the prep stage can list it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of
|
||||
from app.prep.hosts import TABLES
|
||||
from app.seniority import from_title
|
||||
|
||||
#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`).
|
||||
Tables = Mapping[str, Mapping[str, Any]]
|
||||
|
||||
VOCABULARIES: dict[str, tuple[str, ...]] = {
|
||||
"seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE}
|
||||
|
||||
#: How hosts without a table of their own write remote, after `_squash`:
|
||||
#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office").
|
||||
REMOTE_WORDS: dict[str, str] = {
|
||||
"remote": "remote", "fullyremote": "remote", "remotefirst": "remote",
|
||||
"hybrid": "hybrid",
|
||||
"onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite",
|
||||
"officebased": "onsite", "onpremise": "onsite",
|
||||
}
|
||||
|
||||
|
||||
def _match(rules: Mapping[str, Any], key: str) -> str | None:
|
||||
"""The value of the first `MATCH` fragment `key` holds, by its mode."""
|
||||
parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set()
|
||||
for fragment, value in rules["rules"]:
|
||||
if fragment in parts if rules["mode"] == "token" else fragment in key:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def _squash(word: str) -> str:
|
||||
return "".join(c for c in word.lower() if c not in " -_")
|
||||
|
||||
|
||||
def lookup(field: str, host: str, word: str,
|
||||
tables: Tables = TABLES) -> str | None:
|
||||
"""The value `word` maps to for `host`, '' for a known word that names
|
||||
none, None for a word nothing knows. An empty word is ''."""
|
||||
key = (word or "").strip().lower()
|
||||
if not key:
|
||||
return ""
|
||||
table = tables.get(host, {})
|
||||
found = table.get(field, {}).get(key)
|
||||
if found is None and field == "employment" and "match" in table:
|
||||
found = _match(table["match"], key)
|
||||
if found is not None:
|
||||
return found
|
||||
if key in VOCABULARIES[field]:
|
||||
return key
|
||||
if field == "remote":
|
||||
return REMOTE_WORDS.get(_squash(key))
|
||||
return None
|
||||
|
||||
|
||||
def seniority(host: str, word: str, title: str,
|
||||
tables: Tables = TABLES) -> tuple[str, str]:
|
||||
"""`(stated, from the title)`: the host's word through its table, and
|
||||
`seniority.from_title`. `roles.seniority` is the model's level, else the
|
||||
stated one, else the title's (`Store.write_prep`)."""
|
||||
return lookup("seniority", host, word, tables) or "", from_title(title)
|
||||
|
||||
|
||||
def employment(host: str, word: str,
|
||||
tables: Tables = TABLES) -> str:
|
||||
return lookup("employment", host, word, tables) or ""
|
||||
|
||||
|
||||
def _place_text(place: Any) -> str:
|
||||
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
|
||||
|
||||
|
||||
def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
|
||||
tables: Tables = TABLES) -> str:
|
||||
"""How the role is worked, one of `REMOTE` or ''.
|
||||
|
||||
`value` is the host's own (a bool, or its word; `host_word`'s 'true' and
|
||||
'false' are bools). True is remote. False says nothing more than the
|
||||
places do: SmartRecruiters, Ashby, Recruitee and Workable send it for
|
||||
every role that is not remote, hybrid ones included. A word goes
|
||||
through the host's table, else the shared map. When the value says
|
||||
nothing, the place line or the place texts are read for a word
|
||||
(`remote_of`: "Hybrid, Berlin"), and last a place the reader marked
|
||||
remote is remote."""
|
||||
if value is True or value == "true":
|
||||
return "remote"
|
||||
said = value not in (None, False, "false")
|
||||
if said and (found := lookup("remote", host, str(value), tables)):
|
||||
return found
|
||||
places = list(places)
|
||||
if found := remote_of(line or "; ".join(map(_place_text, places))):
|
||||
return found
|
||||
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
|
||||
return "remote"
|
||||
return ""
|
||||
+39
-2
@@ -19,7 +19,9 @@
|
||||
* Under the Follow form of Company boards, a fold lists what each reader
|
||||
* declares (`/api/readers`): how its list arrives, where the text is, what
|
||||
* it filters at the source, how it states places, and the robots.txt
|
||||
* decision taken when it was built.
|
||||
* decision taken when it was built. A second fold lists the host words no
|
||||
* table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and
|
||||
* host, most roles first; it is left out when there are none.
|
||||
*/
|
||||
(function () {
|
||||
"use strict";
|
||||
@@ -320,6 +322,38 @@
|
||||
$("#board-form").after(fold);
|
||||
}
|
||||
|
||||
/** The host words prep could not map, one row per field and host. */
|
||||
function renderUnmapped(words) {
|
||||
if (!words.length) return;
|
||||
const groups = new Map();
|
||||
for (const w of words) {
|
||||
const key = `${w.field} · ${w.host || "no host"}`;
|
||||
if (!groups.has(key)) groups.set(key, []);
|
||||
groups.get(key).push(w);
|
||||
}
|
||||
const fold = document.createElement("details");
|
||||
fold.className = "prune";
|
||||
fold.id = "unmapped";
|
||||
const summary = document.createElement("summary");
|
||||
summary.textContent = `Host words no table maps (${words.length})`;
|
||||
const list = document.createElement("div");
|
||||
list.className = "srows";
|
||||
for (const [key, found] of groups) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "srow";
|
||||
const name = document.createElement("span");
|
||||
name.className = "name";
|
||||
name.textContent = key;
|
||||
const sub = document.createElement("span");
|
||||
sub.className = "sub";
|
||||
sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · ");
|
||||
row.append(name, sub);
|
||||
list.appendChild(row);
|
||||
}
|
||||
fold.append(summary, list);
|
||||
($("#readers") || $("#board-form")).after(fold);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------ one section shown
|
||||
|
||||
// Sources shows one section: the one the fragment names or contains, or the
|
||||
@@ -360,7 +394,10 @@
|
||||
|
||||
document.addEventListener("DOMContentLoaded", async () => {
|
||||
if (!$("#searches-list")) return;
|
||||
if ($("#board-form")) api("/api/readers").then(renderReaders, () => {});
|
||||
if ($("#board-form")) {
|
||||
api("/api/readers").then(renderReaders, () => {})
|
||||
.then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {});
|
||||
}
|
||||
try {
|
||||
await load();
|
||||
} catch (err) {
|
||||
|
||||
@@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any
|
||||
@@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
|
||||
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
||||
|
||||
|
||||
def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None:
|
||||
"""Replace one role's own places with these texts, unresolved: what a
|
||||
pull stores, for `app.prep` to resolve (`relink_uids`)."""
|
||||
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
|
||||
conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None)
|
||||
for position, text in enumerate(place_texts)])
|
||||
|
||||
|
||||
def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve the own places of these roles, writing only those that now
|
||||
resolve differently. Returns `uid -> place ids` in the host's order for
|
||||
every role of `uids` with places, what prep reads the country from."""
|
||||
held: dict[str, dict[tuple[int, str], set[int | None]]] = {}
|
||||
for start in range(0, len(uids), 5000):
|
||||
chunk = json.dumps(list(uids[start:start + 5000]))
|
||||
for uid, position, text, place_id in conn.execute(
|
||||
"SELECT uid, position, text, place_id FROM role_places WHERE url = '' "
|
||||
"AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid",
|
||||
(chunk,)):
|
||||
held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
|
||||
found: dict[str, list[int]] = {}
|
||||
changed = []
|
||||
for uid, rows_of in held.items():
|
||||
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of}
|
||||
if now != rows_of:
|
||||
changed.append((uid, [text for _, text in sorted(rows_of)]))
|
||||
found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])]
|
||||
write_many(conn, gaz, changed)
|
||||
return found
|
||||
|
||||
|
||||
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
||||
"""`write` for many roles in one statement each way. Returns rows written."""
|
||||
|
||||
+82
-2
@@ -18,7 +18,9 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host
|
||||
from app.models import (
|
||||
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
@@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
teaser TEXT NOT NULL DEFAULT '',
|
||||
seniority_model TEXT NOT NULL DEFAULT '',
|
||||
seniority_stated TEXT NOT NULL DEFAULT '',
|
||||
seniority_title TEXT NOT NULL DEFAULT ''
|
||||
seniority_title TEXT NOT NULL DEFAULT '',
|
||||
seniority_host TEXT NOT NULL DEFAULT '',
|
||||
employment_host TEXT NOT NULL DEFAULT '',
|
||||
remote_host TEXT NOT NULL DEFAULT '',
|
||||
country_host TEXT NOT NULL DEFAULT '',
|
||||
employment TEXT NOT NULL DEFAULT '',
|
||||
prep_due INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
-- The *_host columns hold the host's own words as a reader handed them on
|
||||
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
|
||||
-- them into seniority_stated, employment, remote and country. prep_due is above 0
|
||||
-- on a role whose words, company, title or places changed since prep last
|
||||
-- mapped it (Store.upsert_role), which is what the next prep reads.
|
||||
-- roles.teaser is the short text a list carries, kept until the full text
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
@@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors (
|
||||
embedded_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- A host word no table of app/prep/hosts maps, per field (seniority,
|
||||
-- employment, remote, country) and host: how many open roles carry it and
|
||||
-- when prep first met it. Rewritten by every prep; a word added to a table
|
||||
-- leaves at the next one.
|
||||
CREATE TABLE IF NOT EXISTS unmapped (
|
||||
field TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
roles INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen TEXT NOT NULL,
|
||||
PRIMARY KEY (field, host, word)
|
||||
);
|
||||
|
||||
-- The string map a desk plugin is handed as self.state. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
@@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
-- The roles the next prep maps. Partial: after a prep there are none.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
|
||||
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
|
||||
lambda conn: _add_all_roles(conn)),
|
||||
# A Python step: the host words are read out of raw's JSON.
|
||||
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
|
||||
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"to tag; it reads the agents' reach now", target, key)
|
||||
|
||||
|
||||
_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment")
|
||||
|
||||
_V22 = """
|
||||
CREATE TABLE IF NOT EXISTS unmapped (
|
||||
field TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
roles INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen TEXT NOT NULL,
|
||||
PRIMARY KEY (field, host, word)
|
||||
);
|
||||
DELETE FROM state WHERE key = 'prep:tables';
|
||||
"""
|
||||
|
||||
|
||||
def _add_host_words(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 22's step. The columns and `unmapped`, and the host words
|
||||
of every role filled from its `raw` where it has them. Before this,
|
||||
readers mapped `seniority` and `employment` in their own tables, so what
|
||||
`raw` holds there is mostly a value of the vocabulary already, which
|
||||
`app.prep` takes as it is. `remote` and `country` are the host's as
|
||||
given. The `prep:tables` fingerprint goes, so the first prep maps every
|
||||
open role from these words. Runs twice without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
for column in _V22_COLUMNS:
|
||||
if column not in columns:
|
||||
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
|
||||
if "prep_due" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0")
|
||||
for statement in _statements(_V22):
|
||||
conn.execute(statement)
|
||||
updates = []
|
||||
for uid, raw in conn.execute(
|
||||
"SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.employment') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.remote') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.country') IS NOT NULL").fetchall():
|
||||
try:
|
||||
given = json.loads(raw or "{}")
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(given, dict):
|
||||
updates.append((*(host_word(given.get(k)) for k in (
|
||||
"seniority", "employment", "remote", "country")), uid))
|
||||
conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, "
|
||||
"remote_host = ?, country_host = ? WHERE uid = ?", updates)
|
||||
log.info("migration 22: host words from raw for %d roles", len(updates))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+140
-44
@@ -22,13 +22,12 @@ import numpy as np
|
||||
from desk.models import Item
|
||||
from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
|
||||
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
||||
RoleRow, ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
host_word, posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer, schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
@@ -36,7 +35,17 @@ from app.storage import gazetteer, schema
|
||||
#: says less than `extract` found keeps what `extract` found. `function` is
|
||||
#: not here: it is not a board field (see the note by `keep_filled` below).
|
||||
#: Nor is `seniority`, which is the title's level unless a board states one.
|
||||
_RAW_TEXT = ("place", "country", "currency")
|
||||
_RAW_TEXT = ("place", "currency")
|
||||
|
||||
#: The host's own words a reader hands on in `Item.raw`, stored as
|
||||
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
|
||||
#: rewritten every pull, as the host's level always was; the others are kept
|
||||
#: filled, since a role's own page (`enrich`) may state what its listing
|
||||
#: does not.
|
||||
_HOST_WORDS = ("seniority", "employment", "remote", "country")
|
||||
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
|
||||
#: What prep maps from that a pull rewrites whatever it brings.
|
||||
_PREP_OVERWRITTEN = ("title", "seniority_host")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
@@ -154,14 +163,11 @@ def _iso(moment: datetime) -> str:
|
||||
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _remote(value: Any) -> str:
|
||||
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
||||
unrecognised is '' (unknown), which `extract` may fill later."""
|
||||
if value is True:
|
||||
return "remote"
|
||||
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
||||
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
||||
return text if text in REMOTE else ""
|
||||
def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]:
|
||||
"""One role's own place texts as stored, in the host's order."""
|
||||
return list(dict.fromkeys(r[0] for r in conn.execute(
|
||||
"SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid",
|
||||
(uid,))))
|
||||
|
||||
|
||||
def _like(text: str) -> str:
|
||||
@@ -607,10 +613,19 @@ class Store:
|
||||
"""Store one posting. Returns True when it was new.
|
||||
|
||||
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
||||
every call, and a role seen again is open again. The gate columns come
|
||||
every call, and a role seen again is open again. The raw columns come
|
||||
from `item.raw` under their own names, with `company` falling back to
|
||||
the subscription's and `country` to what `place` names; an empty value
|
||||
never overwrites a filled column.
|
||||
the subscription's; an empty value never overwrites a filled column.
|
||||
|
||||
Nothing is mapped here: that is `app.prep`, which runs after every
|
||||
pull. The host's words go to `seniority_host`, `employment_host`,
|
||||
`remote_host` and `country_host` (`host_word`), `company_key` is
|
||||
written unresolved on a new role only, and the mapped columns
|
||||
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
|
||||
`remote`, `country`) are left as prep last wrote them. `prep_due`
|
||||
is 1 on a new role and grows by one when the title, the company, a
|
||||
host word or the place texts changed or the role reopens, which
|
||||
is what the next prep reads.
|
||||
|
||||
`posted_at` (see `app.models.posted_at`) is set from this pull's own
|
||||
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
|
||||
@@ -627,20 +642,14 @@ class Store:
|
||||
alone for good, whatever the plugin's raw guess now says — see
|
||||
`set_extracted` for the other half of the rule.
|
||||
|
||||
`seniority` is the first filled of three levels. `seniority_model`
|
||||
is extract's (`set_extracted`) and never written here.
|
||||
`seniority_stated` is the host's word as `raw["seniority"]` carries
|
||||
it, already in `SENIORITIES` (anything else is ''), and
|
||||
`seniority_title` is `seniority.from_title`; both are rewritten every
|
||||
pull. The classifier is called here rather than in a plugin, so a
|
||||
role from any source gets it, the way `country` comes from `place`.
|
||||
|
||||
The role's places (`role_places`) are the texts of `raw["places"]`,
|
||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
|
||||
against the gazetteer; `place` is the line shown, the texts joined
|
||||
when `raw` has none. Like `place`, a pull with no place leaves them
|
||||
as they are. `place_fold` is written with `place`, under the same
|
||||
keep-filled rule. `teaser` is kept-filled too.
|
||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
||||
when they differ from the stored ones; prep resolves them. Texts that
|
||||
did not change keep their resolved rows, so the feed's Place filter
|
||||
stays right between a pull and its prep. `place` is the line shown,
|
||||
the texts joined when `raw` has none. Like `place`, a pull with no
|
||||
place leaves them as they are. `place_fold` is written with `place`,
|
||||
under the same keep-filled rule. `teaser` is kept-filled too.
|
||||
|
||||
A role is embedded from its description, else its teaser, else its
|
||||
title (`app.jobs.embed.document`). When a pull brings a better one of
|
||||
@@ -660,12 +669,11 @@ class Store:
|
||||
"source": item.source,
|
||||
"source_id": item.source_id,
|
||||
"subscription_id": subscription.id if subscription else None,
|
||||
"company_key": self.resolve_company(company),
|
||||
"company_key": company_key(company),
|
||||
"company": company,
|
||||
"title": item.title,
|
||||
"title_key": title_key(item.title),
|
||||
"place_key": place_key(place),
|
||||
"remote": _remote(raw.get("remote")),
|
||||
"description": item.body,
|
||||
"feed_text": item.excerpt,
|
||||
"original_url": item.location,
|
||||
@@ -677,30 +685,26 @@ class Store:
|
||||
"created_at": stamp,
|
||||
}
|
||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||
row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS}
|
||||
row["place"] = place
|
||||
row["country"] = row["country"] or country_of(place)
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
stated = str(raw.get("seniority") or "")
|
||||
row["seniority_stated"] = stated if stated in SENIORITIES else ""
|
||||
row["seniority_title"] = from_title(item.title)
|
||||
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
|
||||
row["teaser"] = str(raw.get("teaser") or "")
|
||||
row["place_fold"] = places.fold(row["place"])
|
||||
# A board's own pay line, one per pull like title or published_at —
|
||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||
# must be able to clear it, not leave a stale line behind.
|
||||
row["salary_text"] = str(raw.get("salary_text") or "")
|
||||
row["prep_due"] = 1
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", "teaser", *_RAW_TEXT)
|
||||
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
|
||||
"teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS)
|
||||
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
||||
"salary_text", "seniority_stated", "seniority_title")]
|
||||
"salary_text", "seniority_host")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
@@ -711,11 +715,17 @@ class Store:
|
||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||
"ELSE roles.function END")
|
||||
# The model's level, else the host's, else the title's.
|
||||
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
|
||||
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
|
||||
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
||||
"THEN excluded.place_fold ELSE roles.place_fold END")
|
||||
# One more than before when what prep maps from changed, or the role
|
||||
# reopens, so a prep that read the old values leaves it due
|
||||
# (`Store.prepped`). `roles.` is the stored row here, `excluded.`
|
||||
# what this pull brought.
|
||||
changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN]
|
||||
changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})"
|
||||
for c in ("company", *_KEPT_HOST_WORDS)]
|
||||
updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} "
|
||||
"THEN roles.prep_due + 1 ELSE roles.prep_due END")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
@@ -725,8 +735,11 @@ class Store:
|
||||
conn.execute(
|
||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||
if place_texts:
|
||||
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
|
||||
if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts):
|
||||
gazetteer.write_texts(conn, row["uid"], place_texts)
|
||||
if before is not None:
|
||||
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
||||
(row["uid"],))
|
||||
# `embed` encodes a role once, from the best text it has. When a
|
||||
# later pull brings a better one, as a host that states the text
|
||||
# on the role's own page does, the vector goes, so the next
|
||||
@@ -1253,6 +1266,89 @@ class Store:
|
||||
remotes)
|
||||
return len(countries) + len(remotes)
|
||||
|
||||
# ----------------------------------------------------------------- prep
|
||||
|
||||
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
|
||||
"""What `app.prep` maps a role from, and what it holds now: the roles
|
||||
due (`prep_due` above 0), every open role, or every role."""
|
||||
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
|
||||
return self._rows(
|
||||
f"""SELECT uid, prep_due, {_HOST} AS host, title, company, company_key, place,
|
||||
seniority_host, employment_host, remote_host, country_host,
|
||||
seniority_model, seniority_stated, seniority_title, seniority,
|
||||
employment, remote, country,
|
||||
json_extract(raw, '$.places') AS raw_places
|
||||
FROM roles WHERE {where}""")
|
||||
|
||||
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve these roles' own places against the gazetteer, writing the
|
||||
rows that resolve differently; `uid -> place ids` for each that has
|
||||
places (`gazetteer.relink_uids`)."""
|
||||
with self.transaction() as conn:
|
||||
return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids))
|
||||
|
||||
def company_folds(self) -> dict[str, list[str]]:
|
||||
"""`fold -> company keys` of `company_aliases`, sorted: what
|
||||
`resolve_company` asks one name at a time, for prep's whole batch."""
|
||||
out: dict[str, list[str]] = {}
|
||||
for r in self._rows("SELECT fold, company_key FROM company_aliases "
|
||||
"ORDER BY fold, company_key"):
|
||||
out.setdefault(str(r["fold"]), []).append(str(r["company_key"]))
|
||||
return out
|
||||
|
||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||
`seniority_title`, `employment`, `remote`, `remote_places` and
|
||||
`country`. `seniority` is the model's level, else the stated, else
|
||||
the title's. An empty `remote` or `country` leaves the column as it
|
||||
is, since `extract` fills an empty one. `remote_places`, what the
|
||||
place texts say, only fills an empty column: it is a guess from a
|
||||
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
|
||||
changed the role since it was read. Returns the rows written."""
|
||||
with self.transaction() as conn:
|
||||
return conn.executemany(
|
||||
"""UPDATE roles SET company_key = :company_key,
|
||||
seniority_stated = :seniority_stated,
|
||||
seniority_title = :seniority_title,
|
||||
seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||
employment = :employment,
|
||||
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
|
||||
:remote_places),
|
||||
country = COALESCE(NULLIF(:country, ''), country),
|
||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||
WHERE uid = :uid""", mapped).rowcount
|
||||
|
||||
def host_words(self) -> list[tuple[str, str, str, int]]:
|
||||
"""`(field, host, word, open roles)` for every host word an open role
|
||||
carries, the word lowercased: what prep checks against its tables."""
|
||||
found: list[tuple[str, str, str, int]] = []
|
||||
for field in ("seniority", "employment", "remote", "country"):
|
||||
found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows(
|
||||
f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles
|
||||
WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")]
|
||||
return found
|
||||
|
||||
def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None:
|
||||
"""Make `unmapped` these `(field, host, word, roles)`, keeping the
|
||||
`first_seen` of a word already listed."""
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute(
|
||||
"SELECT field, host, word, first_seen FROM unmapped")}
|
||||
conn.execute("DELETE FROM unmapped")
|
||||
conn.executemany(
|
||||
"INSERT INTO unmapped (field, host, word, roles, first_seen) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
[(*r, seen.get(r[:3], stamp)) for r in rows])
|
||||
|
||||
def unmapped(self) -> list[dict[str, Any]]:
|
||||
"""The host words no table maps, by field and host, most roles first."""
|
||||
return [dict(r) for r in self._rows(
|
||||
"SELECT field, host, word, roles, first_seen FROM unmapped "
|
||||
"ORDER BY field, host, roles DESC, word")]
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
||||
|
||||
@@ -897,6 +897,23 @@ def list_readers() -> list[ReaderOut]:
|
||||
for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))]
|
||||
|
||||
|
||||
class UnmappedOut(BaseModel):
|
||||
"""A host word no table of `app/prep/hosts` maps (`Store.unmapped`)."""
|
||||
|
||||
field: str
|
||||
host: str
|
||||
word: str
|
||||
roles: int
|
||||
first_seen: str
|
||||
|
||||
|
||||
@router.get("/prep/unmapped")
|
||||
def list_unmapped(store: StoreDep) -> list[UnmappedOut]:
|
||||
"""The host words the last prep could not map, per field and host, most
|
||||
roles first: what a table in `app/prep/hosts` should learn."""
|
||||
return [UnmappedOut(**w) for w in store.unmapped()]
|
||||
|
||||
|
||||
class SearchBody(BaseModel):
|
||||
provider: str
|
||||
name: str = ""
|
||||
|
||||
Reference in new issue
Block a user