Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
@@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Any
|
||||
@@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
|
||||
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
||||
|
||||
|
||||
def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None:
|
||||
"""Replace one role's own places with these texts, unresolved: what a
|
||||
pull stores, for `app.prep` to resolve (`relink_uids`)."""
|
||||
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
|
||||
conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None)
|
||||
for position, text in enumerate(place_texts)])
|
||||
|
||||
|
||||
def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve the own places of these roles, writing only those that now
|
||||
resolve differently. Returns `uid -> place ids` in the host's order for
|
||||
every role of `uids` with places, what prep reads the country from."""
|
||||
held: dict[str, dict[tuple[int, str], set[int | None]]] = {}
|
||||
for start in range(0, len(uids), 5000):
|
||||
chunk = json.dumps(list(uids[start:start + 5000]))
|
||||
for uid, position, text, place_id in conn.execute(
|
||||
"SELECT uid, position, text, place_id FROM role_places WHERE url = '' "
|
||||
"AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid",
|
||||
(chunk,)):
|
||||
held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
|
||||
found: dict[str, list[int]] = {}
|
||||
changed = []
|
||||
for uid, rows_of in held.items():
|
||||
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of}
|
||||
if now != rows_of:
|
||||
changed.append((uid, [text for _, text in sorted(rows_of)]))
|
||||
found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])]
|
||||
write_many(conn, gaz, changed)
|
||||
return found
|
||||
|
||||
|
||||
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
||||
"""`write` for many roles in one statement each way. Returns rows written."""
|
||||
|
||||
+82
-2
@@ -18,7 +18,9 @@ import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app import places
|
||||
from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host
|
||||
from app.models import (
|
||||
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer
|
||||
|
||||
@@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
teaser TEXT NOT NULL DEFAULT '',
|
||||
seniority_model TEXT NOT NULL DEFAULT '',
|
||||
seniority_stated TEXT NOT NULL DEFAULT '',
|
||||
seniority_title TEXT NOT NULL DEFAULT ''
|
||||
seniority_title TEXT NOT NULL DEFAULT '',
|
||||
seniority_host TEXT NOT NULL DEFAULT '',
|
||||
employment_host TEXT NOT NULL DEFAULT '',
|
||||
remote_host TEXT NOT NULL DEFAULT '',
|
||||
country_host TEXT NOT NULL DEFAULT '',
|
||||
employment TEXT NOT NULL DEFAULT '',
|
||||
prep_due INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
-- The *_host columns hold the host's own words as a reader handed them on
|
||||
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
|
||||
-- them into seniority_stated, employment, remote and country. prep_due is above 0
|
||||
-- on a role whose words, company, title or places changed since prep last
|
||||
-- mapped it (Store.upsert_role), which is what the next prep reads.
|
||||
-- roles.teaser is the short text a list carries, kept until the full text
|
||||
-- arrives and embedded while there is none; a later empty teaser does not
|
||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||
@@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors (
|
||||
embedded_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- A host word no table of app/prep/hosts maps, per field (seniority,
|
||||
-- employment, remote, country) and host: how many open roles carry it and
|
||||
-- when prep first met it. Rewritten by every prep; a word added to a table
|
||||
-- leaves at the next one.
|
||||
CREATE TABLE IF NOT EXISTS unmapped (
|
||||
field TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
roles INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen TEXT NOT NULL,
|
||||
PRIMARY KEY (field, host, word)
|
||||
);
|
||||
|
||||
-- The string map a desk plugin is handed as self.state. One namespace shared
|
||||
-- by every plugin, so a plugin prefixes its keys.
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
@@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||
-- The roles the next prep maps. Partial: after a prep there are none.
|
||||
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
|
||||
"""
|
||||
|
||||
#: The table whose absence marks a file that has never been a roledesk
|
||||
@@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
|
||||
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
|
||||
lambda conn: _add_all_roles(conn)),
|
||||
# A Python step: the host words are read out of raw's JSON.
|
||||
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
|
||||
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"to tag; it reads the agents' reach now", target, key)
|
||||
|
||||
|
||||
_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment")
|
||||
|
||||
_V22 = """
|
||||
CREATE TABLE IF NOT EXISTS unmapped (
|
||||
field TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
roles INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen TEXT NOT NULL,
|
||||
PRIMARY KEY (field, host, word)
|
||||
);
|
||||
DELETE FROM state WHERE key = 'prep:tables';
|
||||
"""
|
||||
|
||||
|
||||
def _add_host_words(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 22's step. The columns and `unmapped`, and the host words
|
||||
of every role filled from its `raw` where it has them. Before this,
|
||||
readers mapped `seniority` and `employment` in their own tables, so what
|
||||
`raw` holds there is mostly a value of the vocabulary already, which
|
||||
`app.prep` takes as it is. `remote` and `country` are the host's as
|
||||
given. The `prep:tables` fingerprint goes, so the first prep maps every
|
||||
open role from these words. Runs twice without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
for column in _V22_COLUMNS:
|
||||
if column not in columns:
|
||||
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
|
||||
if "prep_due" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0")
|
||||
for statement in _statements(_V22):
|
||||
conn.execute(statement)
|
||||
updates = []
|
||||
for uid, raw in conn.execute(
|
||||
"SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.employment') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.remote') IS NOT NULL "
|
||||
"OR json_extract(raw, '$.country') IS NOT NULL").fetchall():
|
||||
try:
|
||||
given = json.loads(raw or "{}")
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(given, dict):
|
||||
updates.append((*(host_word(given.get(k)) for k in (
|
||||
"seniority", "employment", "remote", "country")), uid))
|
||||
conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, "
|
||||
"remote_host = ?, country_host = ? WHERE uid = ?", updates)
|
||||
log.info("migration 22: host words from raw for %d roles", len(updates))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+140
-44
@@ -22,13 +22,12 @@ import numpy as np
|
||||
from desk.models import Item
|
||||
from app import dupes, places
|
||||
from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
|
||||
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
||||
RoleRow, ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||
posted_at, posted_host, remote_of, title_key,
|
||||
host_word, posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.seniority import from_title
|
||||
from app.storage import gazetteer, schema
|
||||
|
||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||
@@ -36,7 +35,17 @@ from app.storage import gazetteer, schema
|
||||
#: says less than `extract` found keeps what `extract` found. `function` is
|
||||
#: not here: it is not a board field (see the note by `keep_filled` below).
|
||||
#: Nor is `seniority`, which is the title's level unless a board states one.
|
||||
_RAW_TEXT = ("place", "country", "currency")
|
||||
_RAW_TEXT = ("place", "currency")
|
||||
|
||||
#: The host's own words a reader hands on in `Item.raw`, stored as
|
||||
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
|
||||
#: rewritten every pull, as the host's level always was; the others are kept
|
||||
#: filled, since a role's own page (`enrich`) may state what its listing
|
||||
#: does not.
|
||||
_HOST_WORDS = ("seniority", "employment", "remote", "country")
|
||||
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
|
||||
#: What prep maps from that a pull rewrites whatever it brings.
|
||||
_PREP_OVERWRITTEN = ("title", "seniority_host")
|
||||
_RAW_INT = ("salary_min", "salary_max")
|
||||
|
||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||
@@ -154,14 +163,11 @@ def _iso(moment: datetime) -> str:
|
||||
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _remote(value: Any) -> str:
|
||||
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
||||
unrecognised is '' (unknown), which `extract` may fill later."""
|
||||
if value is True:
|
||||
return "remote"
|
||||
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
||||
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
||||
return text if text in REMOTE else ""
|
||||
def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]:
|
||||
"""One role's own place texts as stored, in the host's order."""
|
||||
return list(dict.fromkeys(r[0] for r in conn.execute(
|
||||
"SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid",
|
||||
(uid,))))
|
||||
|
||||
|
||||
def _like(text: str) -> str:
|
||||
@@ -607,10 +613,19 @@ class Store:
|
||||
"""Store one posting. Returns True when it was new.
|
||||
|
||||
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
||||
every call, and a role seen again is open again. The gate columns come
|
||||
every call, and a role seen again is open again. The raw columns come
|
||||
from `item.raw` under their own names, with `company` falling back to
|
||||
the subscription's and `country` to what `place` names; an empty value
|
||||
never overwrites a filled column.
|
||||
the subscription's; an empty value never overwrites a filled column.
|
||||
|
||||
Nothing is mapped here: that is `app.prep`, which runs after every
|
||||
pull. The host's words go to `seniority_host`, `employment_host`,
|
||||
`remote_host` and `country_host` (`host_word`), `company_key` is
|
||||
written unresolved on a new role only, and the mapped columns
|
||||
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
|
||||
`remote`, `country`) are left as prep last wrote them. `prep_due`
|
||||
is 1 on a new role and grows by one when the title, the company, a
|
||||
host word or the place texts changed or the role reopens, which
|
||||
is what the next prep reads.
|
||||
|
||||
`posted_at` (see `app.models.posted_at`) is set from this pull's own
|
||||
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
|
||||
@@ -627,20 +642,14 @@ class Store:
|
||||
alone for good, whatever the plugin's raw guess now says — see
|
||||
`set_extracted` for the other half of the rule.
|
||||
|
||||
`seniority` is the first filled of three levels. `seniority_model`
|
||||
is extract's (`set_extracted`) and never written here.
|
||||
`seniority_stated` is the host's word as `raw["seniority"]` carries
|
||||
it, already in `SENIORITIES` (anything else is ''), and
|
||||
`seniority_title` is `seniority.from_title`; both are rewritten every
|
||||
pull. The classifier is called here rather than in a plugin, so a
|
||||
role from any source gets it, the way `country` comes from `place`.
|
||||
|
||||
The role's places (`role_places`) are the texts of `raw["places"]`,
|
||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
|
||||
against the gazetteer; `place` is the line shown, the texts joined
|
||||
when `raw` has none. Like `place`, a pull with no place leaves them
|
||||
as they are. `place_fold` is written with `place`, under the same
|
||||
keep-filled rule. `teaser` is kept-filled too.
|
||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
||||
when they differ from the stored ones; prep resolves them. Texts that
|
||||
did not change keep their resolved rows, so the feed's Place filter
|
||||
stays right between a pull and its prep. `place` is the line shown,
|
||||
the texts joined when `raw` has none. Like `place`, a pull with no
|
||||
place leaves them as they are. `place_fold` is written with `place`,
|
||||
under the same keep-filled rule. `teaser` is kept-filled too.
|
||||
|
||||
A role is embedded from its description, else its teaser, else its
|
||||
title (`app.jobs.embed.document`). When a pull brings a better one of
|
||||
@@ -660,12 +669,11 @@ class Store:
|
||||
"source": item.source,
|
||||
"source_id": item.source_id,
|
||||
"subscription_id": subscription.id if subscription else None,
|
||||
"company_key": self.resolve_company(company),
|
||||
"company_key": company_key(company),
|
||||
"company": company,
|
||||
"title": item.title,
|
||||
"title_key": title_key(item.title),
|
||||
"place_key": place_key(place),
|
||||
"remote": _remote(raw.get("remote")),
|
||||
"description": item.body,
|
||||
"feed_text": item.excerpt,
|
||||
"original_url": item.location,
|
||||
@@ -677,30 +685,26 @@ class Store:
|
||||
"created_at": stamp,
|
||||
}
|
||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||
row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS}
|
||||
row["place"] = place
|
||||
row["country"] = row["country"] or country_of(place)
|
||||
row["remote"] = row["remote"] or remote_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
row["function"] = str(raw.get("function") or "")
|
||||
stated = str(raw.get("seniority") or "")
|
||||
row["seniority_stated"] = stated if stated in SENIORITIES else ""
|
||||
row["seniority_title"] = from_title(item.title)
|
||||
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
|
||||
row["teaser"] = str(raw.get("teaser") or "")
|
||||
row["place_fold"] = places.fold(row["place"])
|
||||
# A board's own pay line, one per pull like title or published_at —
|
||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||
# must be able to clear it, not leave a stale line behind.
|
||||
row["salary_text"] = str(raw.get("salary_text") or "")
|
||||
row["prep_due"] = 1
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
"feed_text", "original_url", "teaser", *_RAW_TEXT)
|
||||
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
|
||||
"teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS)
|
||||
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||
for c in (*_RAW_INT, "subscription_id")]
|
||||
updates += [f"{c}=excluded.{c}"
|
||||
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
||||
"salary_text", "seniority_stated", "seniority_title")]
|
||||
"salary_text", "seniority_host")]
|
||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||
# pull's classifier guess overwrite what extract decided, which is
|
||||
@@ -711,11 +715,17 @@ class Store:
|
||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||
"ELSE roles.function END")
|
||||
# The model's level, else the host's, else the title's.
|
||||
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
|
||||
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
|
||||
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
||||
"THEN excluded.place_fold ELSE roles.place_fold END")
|
||||
# One more than before when what prep maps from changed, or the role
|
||||
# reopens, so a prep that read the old values leaves it due
|
||||
# (`Store.prepped`). `roles.` is the stored row here, `excluded.`
|
||||
# what this pull brought.
|
||||
changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN]
|
||||
changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})"
|
||||
for c in ("company", *_KEPT_HOST_WORDS)]
|
||||
updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} "
|
||||
"THEN roles.prep_due + 1 ELSE roles.prep_due END")
|
||||
updates.append("closed_at=NULL")
|
||||
columns = ", ".join(row)
|
||||
marks = ", ".join(f":{c}" for c in row)
|
||||
@@ -725,8 +735,11 @@ class Store:
|
||||
conn.execute(
|
||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||
if place_texts:
|
||||
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
|
||||
if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts):
|
||||
gazetteer.write_texts(conn, row["uid"], place_texts)
|
||||
if before is not None:
|
||||
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
||||
(row["uid"],))
|
||||
# `embed` encodes a role once, from the best text it has. When a
|
||||
# later pull brings a better one, as a host that states the text
|
||||
# on the role's own page does, the vector goes, so the next
|
||||
@@ -1253,6 +1266,89 @@ class Store:
|
||||
remotes)
|
||||
return len(countries) + len(remotes)
|
||||
|
||||
# ----------------------------------------------------------------- prep
|
||||
|
||||
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
|
||||
"""What `app.prep` maps a role from, and what it holds now: the roles
|
||||
due (`prep_due` above 0), every open role, or every role."""
|
||||
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
|
||||
return self._rows(
|
||||
f"""SELECT uid, prep_due, {_HOST} AS host, title, company, company_key, place,
|
||||
seniority_host, employment_host, remote_host, country_host,
|
||||
seniority_model, seniority_stated, seniority_title, seniority,
|
||||
employment, remote, country,
|
||||
json_extract(raw, '$.places') AS raw_places
|
||||
FROM roles WHERE {where}""")
|
||||
|
||||
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
|
||||
"""Resolve these roles' own places against the gazetteer, writing the
|
||||
rows that resolve differently; `uid -> place ids` for each that has
|
||||
places (`gazetteer.relink_uids`)."""
|
||||
with self.transaction() as conn:
|
||||
return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids))
|
||||
|
||||
def company_folds(self) -> dict[str, list[str]]:
|
||||
"""`fold -> company keys` of `company_aliases`, sorted: what
|
||||
`resolve_company` asks one name at a time, for prep's whole batch."""
|
||||
out: dict[str, list[str]] = {}
|
||||
for r in self._rows("SELECT fold, company_key FROM company_aliases "
|
||||
"ORDER BY fold, company_key"):
|
||||
out.setdefault(str(r["fold"]), []).append(str(r["company_key"]))
|
||||
return out
|
||||
|
||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||
`seniority_title`, `employment`, `remote`, `remote_places` and
|
||||
`country`. `seniority` is the model's level, else the stated, else
|
||||
the title's. An empty `remote` or `country` leaves the column as it
|
||||
is, since `extract` fills an empty one. `remote_places`, what the
|
||||
place texts say, only fills an empty column: it is a guess from a
|
||||
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
|
||||
changed the role since it was read. Returns the rows written."""
|
||||
with self.transaction() as conn:
|
||||
return conn.executemany(
|
||||
"""UPDATE roles SET company_key = :company_key,
|
||||
seniority_stated = :seniority_stated,
|
||||
seniority_title = :seniority_title,
|
||||
seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||
employment = :employment,
|
||||
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
|
||||
:remote_places),
|
||||
country = COALESCE(NULLIF(:country, ''), country),
|
||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||
WHERE uid = :uid""", mapped).rowcount
|
||||
|
||||
def host_words(self) -> list[tuple[str, str, str, int]]:
|
||||
"""`(field, host, word, open roles)` for every host word an open role
|
||||
carries, the word lowercased: what prep checks against its tables."""
|
||||
found: list[tuple[str, str, str, int]] = []
|
||||
for field in ("seniority", "employment", "remote", "country"):
|
||||
found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows(
|
||||
f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles
|
||||
WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")]
|
||||
return found
|
||||
|
||||
def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None:
|
||||
"""Make `unmapped` these `(field, host, word, roles)`, keeping the
|
||||
`first_seen` of a word already listed."""
|
||||
stamp = self.now()
|
||||
with self.transaction() as conn:
|
||||
seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute(
|
||||
"SELECT field, host, word, first_seen FROM unmapped")}
|
||||
conn.execute("DELETE FROM unmapped")
|
||||
conn.executemany(
|
||||
"INSERT INTO unmapped (field, host, word, roles, first_seen) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
[(*r, seen.get(r[:3], stamp)) for r in rows])
|
||||
|
||||
def unmapped(self) -> list[dict[str, Any]]:
|
||||
"""The host words no table maps, by field and host, most roles first."""
|
||||
return [dict(r) for r in self._rows(
|
||||
"SELECT field, host, word, roles, first_seen FROM unmapped "
|
||||
"ORDER BY field, host, roles DESC, word")]
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
||||
|
||||
Reference in new issue
Block a user