Prep stage: host words stored raw, mapped centrally in app/prep after every pull
upsert_role stores the host's own words (roles.*_host), the place texts unresolved when they change, and the company key unresolved on insert, and marks the role prep_due. app.prep maps seniority, employment, remote and country through the host tables in app/prep/hosts, resolves places and company aliases, lists unknown words in `unmapped`, remaps every open role when the tables' fingerprint changes, and runs rebuild_dupes, which moves out of the pull's settle. Migration 22 adds the columns and the table and fills the host words from raw. Cascade: pull, prep, embed, score, enrich, prep, embed, score, extract, score, summarise. `app prep [unmapped] [--remap]`, /api/prep/unmapped and a fold on the Sources page. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a66aac95cf
commit
d04ceae7e0
31 files changed
+1468
-330
No files matched your search
+31
-1
@@ -8,6 +8,7 @@
|
|||||||
app agents list|prompt
|
app agents list|prompt
|
||||||
app stages [extract|summarise on|off]
|
app stages [extract|summarise on|off]
|
||||||
app dupes
|
app dupes
|
||||||
|
app prep [unmapped] [--remap]
|
||||||
app db init
|
app db init
|
||||||
|
|
||||||
`app job` queues the work on a queue of its own and drains it here, so a
|
`app job` queues the work on a queue of its own and drains it here, so a
|
||||||
@@ -160,6 +161,13 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
|
|
||||||
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
|
||||||
|
|
||||||
|
prep = sub.add_parser(
|
||||||
|
"prep", help="map the roles' host words and fold the copies, or list the unmapped words")
|
||||||
|
prep.add_argument("action", nargs="?", choices=["unmapped"],
|
||||||
|
help="list the host words no table maps, instead of running")
|
||||||
|
prep.add_argument("--remap", action="store_true",
|
||||||
|
help="map every role again, closed ones included")
|
||||||
|
|
||||||
db = sub.add_parser("db", help="database housekeeping")
|
db = sub.add_parser("db", help="database housekeeping")
|
||||||
db.add_argument("action", choices=["init"])
|
db.add_argument("action", choices=["init"])
|
||||||
return parser
|
return parser
|
||||||
@@ -188,6 +196,8 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
return run_seniority(args)
|
return run_seniority(args)
|
||||||
if args.command == "dupes":
|
if args.command == "dupes":
|
||||||
return run_dupes(args)
|
return run_dupes(args)
|
||||||
|
if args.command == "prep":
|
||||||
|
return run_prep(args)
|
||||||
return init_db(args)
|
return init_db(args)
|
||||||
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
||||||
print(exc, file=sys.stderr)
|
print(exc, file=sys.stderr)
|
||||||
@@ -655,7 +665,7 @@ def run_seniority(args: argparse.Namespace) -> int:
|
|||||||
|
|
||||||
|
|
||||||
def run_dupes(args: argparse.Namespace) -> int:
|
def run_dupes(args: argparse.Namespace) -> int:
|
||||||
"""`app dupes`: what the end of every pull runs, `Store.rebuild_dupes`,
|
"""`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`,
|
||||||
now. Prints the groups (kept roles with a copy), the copies the feed
|
now. Prints the groups (kept roles with a copy), the copies the feed
|
||||||
hides and the rows whose `dup_of` changed, as JSON."""
|
hides and the rows whose `dup_of` changed, as JSON."""
|
||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
@@ -666,6 +676,26 @@ def run_dupes(args: argparse.Namespace) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def run_prep(args: argparse.Namespace) -> int:
|
||||||
|
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
|
||||||
|
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
|
||||||
|
host words no table maps, per field and host, most roles first."""
|
||||||
|
from app import prep
|
||||||
|
from app.storage import Store
|
||||||
|
|
||||||
|
config = config_mod.load(args.config)
|
||||||
|
store = Store(config.database)
|
||||||
|
if args.action == "unmapped":
|
||||||
|
found = store.unmapped()
|
||||||
|
if not found:
|
||||||
|
print("every host word is mapped")
|
||||||
|
for w in found:
|
||||||
|
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
|
||||||
|
return 0
|
||||||
|
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def init_db(args: argparse.Namespace) -> int:
|
def init_db(args: argparse.Namespace) -> int:
|
||||||
"""Create the file, or bring it to the current schema."""
|
"""Create the file, or bring it to the current schema."""
|
||||||
from app.storage import LATEST, Store
|
from app.storage import LATEST, Store
|
||||||
|
|||||||
@@ -2,11 +2,12 @@
|
|||||||
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||||
|
|
||||||
pull.py read every board and search and store the roles they yield
|
pull.py read every board and search and store the roles they yield
|
||||||
|
prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold
|
||||||
extract.py title and description into the gate columns, on Haiku
|
extract.py title and description into the gate columns, on Haiku
|
||||||
embed.py one vector per open role, for fit
|
embed.py one vector per open role, for fit
|
||||||
score.py every open role scored for every enabled search agent
|
score.py every open role scored for every enabled search agent
|
||||||
summarise.py about 60 words on each of the best roles, on Haiku
|
summarise.py about 60 words on each of the best roles, on Haiku
|
||||||
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
|
cascade.py pull → prep → embed → score → ... → summarise, what `TASKS`
|
||||||
offers, the `pull` job a schedule fires, and that schedule
|
offers, the `pull` job a schedule fires, and that schedule
|
||||||
stages.py whether extract and summarise run: the switch, else config.yaml
|
stages.py whether extract and summarise run: the switch, else config.yaml
|
||||||
sources.py what a pull reads, changed by hand: boards and searches
|
sources.py what a pull reads, changed by hand: boards and searches
|
||||||
|
|||||||
+19
-13
@@ -1,10 +1,13 @@
|
|||||||
"""The tasks roledesk offers, and the order a pull runs them in.
|
"""The tasks roledesk offers, and the order a pull runs them in.
|
||||||
|
|
||||||
pull → embed → score → enrich → embed → score → extract → score → summarise
|
pull → prep → embed → score → enrich → prep → embed → score → extract → score
|
||||||
|
→ summarise
|
||||||
|
|
||||||
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
newsdesk's cascade convention: a cascade is a tuple of task names, and the
|
||||||
final job of one task starts the next (`step` is the callback it carries).
|
final job of one task starts the next (`step` is the callback it carries).
|
||||||
Nothing is alive between two tasks. A stage is addressed by its position in
|
Nothing is alive between two tasks. `prep` (`app/jobs/prep.py`) maps what
|
||||||
|
the pull stored and folds the copies before anything reads the roles; after
|
||||||
|
`enrich` it maps the roles their own pages filled. A stage is addressed by its position in
|
||||||
the tuple, not its name, since `score` runs twice.
|
the tuple, not its name, since `score` runs twice.
|
||||||
|
|
||||||
Every stage is always in the chain, and a stage that is switched off finds
|
Every stage is always in the chain, and a stage that is switched off finds
|
||||||
@@ -12,8 +15,9 @@ nothing: a stage switched off (`stages.stage_on`), or an `on_ingest` this pull d
|
|||||||
satisfy, is an extract stage with no batches whose final job moves straight
|
satisfy, is an extract stage with no batches whose final job moves straight
|
||||||
on. The free stages go first: embed and score cost no tokens, and the score
|
on. The free stages go first: embed and score cost no tokens, and the score
|
||||||
they produce is what bounds the rest. Enrich reads the best roles without
|
they produce is what bounds the rest. Enrich reads the best roles without
|
||||||
text on their own pages (`app/jobs/enrich.py`), and embed and score follow for
|
text on their own pages (`app/jobs/enrich.py`), and prep, embed and score
|
||||||
the roles it filled; when it filled none, `step` passes over both. Extract
|
follow for the roles it filled; when it filled none, `step` passes over all
|
||||||
|
three. Extract
|
||||||
reads only the best
|
reads only the best
|
||||||
roles of each enabled agent, as deep and as low as that agent's own `limits`
|
roles of each enabled agent, as deep and as low as that agent's own `limits`
|
||||||
say; the second score then applies the gates to the columns it filled.
|
say; the second score then applies the gates to the columns it filled.
|
||||||
@@ -27,7 +31,7 @@ having either way.
|
|||||||
and a cascade is started by one: the `pull` job's only work is to ask for the
|
and a cascade is started by one: the `pull` job's only work is to ask for the
|
||||||
chain. Its default schedule, `every 6h`, is declared here beside it.
|
chain. Its default schedule, `every 6h`, is declared here beside it.
|
||||||
|
|
||||||
`enrich` is a job and a cascade the same way, `enrich → embed → score`, queued
|
`enrich` is a job and a cascade the same way, `enrich → prep → embed → score`, queued
|
||||||
behind the `score` that follows a change to the ranking other than a pull's
|
behind the `score` that follows a change to the ranking other than a pull's
|
||||||
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
|
(`enrich_after`): an agent saved, a company's affinity changed. A vote moves
|
||||||
one agent's taste a little and queues neither.
|
one agent's taste a little and queues neither.
|
||||||
@@ -40,20 +44,20 @@ from dataclasses import replace
|
|||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.jobs import embed, enrich, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
from app.jobs import embed, enrich, extract, prep, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||||
from desk.tools import agent
|
from desk.tools import agent
|
||||||
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
||||||
|
|
||||||
STEP = "app.jobs.cascade:step"
|
STEP = "app.jobs.cascade:step"
|
||||||
|
|
||||||
CASCADES: dict[str, tuple[str, ...]] = {
|
CASCADES: dict[str, tuple[str, ...]] = {
|
||||||
"pull": ("pull", "embed", "score", "enrich", "embed", "score", "extract", "score",
|
"pull": ("pull", "prep", "embed", "score", "enrich", "prep", "embed", "score", "extract",
|
||||||
"summarise"),
|
"score", "summarise"),
|
||||||
"enrich": ("enrich", "embed", "score"),
|
"enrich": ("enrich", "prep", "embed", "score"),
|
||||||
}
|
}
|
||||||
|
|
||||||
#: What follows `enrich` for the roles it filled, passed over when it filled none.
|
#: What follows `enrich` for the roles it filled, passed over when it filled none.
|
||||||
REFILL = ("embed", "score")
|
REFILL = ("prep", "embed", "score")
|
||||||
|
|
||||||
|
|
||||||
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
def _once(config: Any, store: Any) -> list[dict[str, Any]]:
|
||||||
@@ -103,6 +107,8 @@ STAGES: dict[str, Callable[[str], Task]] = {
|
|||||||
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
|
"embed": lambda cascade: Task(kind="embed", finder=_once, label="Embedding roles"),
|
||||||
"score": lambda cascade: Task(kind="score", finder=_once,
|
"score": lambda cascade: Task(kind="score", finder=_once,
|
||||||
label="Scoring roles for every agent"),
|
label="Scoring roles for every agent"),
|
||||||
|
"prep": lambda cascade: Task(kind="prep", finder=_once,
|
||||||
|
label="Mapping the roles' host words"),
|
||||||
"summarise": _summarise,
|
"summarise": _summarise,
|
||||||
"enrich": lambda cascade: enrich.enrich_task(),
|
"enrich": lambda cascade: enrich.enrich_task(),
|
||||||
}
|
}
|
||||||
@@ -189,7 +195,7 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
|||||||
|
|
||||||
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
|
@job("enrich", "Reading the best roles on their own pages", solo=True, resource="http")
|
||||||
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
def run_enrich(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||||
"""Ask for the enrich cascade, `enrich → embed → score`. Its own work is
|
"""Ask for the enrich cascade, `enrich → prep → embed → score`. Its own work is
|
||||||
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
|
nil, as the `pull` job's. Under `NULL` there is no queue to take it."""
|
||||||
if not context.task(chain("enrich")):
|
if not context.task(chain("enrich")):
|
||||||
raise RuntimeError("no queue to start the enrich cascade on")
|
raise RuntimeError("no queue to start the enrich cascade on")
|
||||||
@@ -210,7 +216,7 @@ def enrich_after(queue: Any, score_job: Any) -> Any:
|
|||||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||||
#: is both would be listed twice by `/api/jobs/kinds`.
|
#: is both would be listed twice by `/api/jobs/kinds`.
|
||||||
for _name in ("extract", "embed", "score", "summarise"):
|
for _name in ("extract", "embed", "score", "summarise", "prep"):
|
||||||
TASKS[_name] = partial(task, _name)
|
TASKS[_name] = partial(task, _name)
|
||||||
|
|
||||||
|
|
||||||
@@ -220,5 +226,5 @@ for _name in ("extract", "embed", "score", "summarise"):
|
|||||||
# setting: a cadence edited or switched off under Runs survives
|
# setting: a cadence edited or switched off under Runs survives
|
||||||
# every restart (see pipeline/schedule.py).
|
# every restart (see pipeline/schedule.py).
|
||||||
declare("pull", "pull", "every 6h",
|
declare("pull", "pull", "every 6h",
|
||||||
label="Pull every board, then embed, score, read the best roles' pages, "
|
label="Pull every board, then map, embed, score, read the best roles' pages, "
|
||||||
"extract, rescore and summarise")
|
"extract, rescore and summarise")
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""`prep`: the stage that maps what a pull stored, `app.prep.run`.
|
||||||
|
|
||||||
|
In the pull cascade it follows the pull, whose settle has closed what is
|
||||||
|
gone, and maps every role the pull changed, then folds the copies across
|
||||||
|
the whole database (`Store.rebuild_dupes`, which the pull's settle used to
|
||||||
|
run). After `enrich` it maps the roles their own pages filled, since only
|
||||||
|
those are due. `solo`, so two cascades never write the same rows at once.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app import prep
|
||||||
|
from app.config import Config
|
||||||
|
from app.storage import Store
|
||||||
|
from pipeline import NULL, RunContext, job
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@job("prep", "Mapping the roles' host words", solo=True, resource="cpu")
|
||||||
|
def run_prep(config: Config, store: Store, *, remap: bool = False,
|
||||||
|
context: RunContext = NULL) -> dict[str, Any]:
|
||||||
|
"""Map the due roles, every open one when the tables changed, and fold
|
||||||
|
the copies. `remap` maps every role, closed ones included."""
|
||||||
|
done = prep.run(store, remap=remap)
|
||||||
|
log.info("prep: %d read, %d written, %d words unmapped, %s copies%s", done.read,
|
||||||
|
done.written, done.unmapped, done.copies, ", tables changed" if done.remapped else "")
|
||||||
|
return done._asdict()
|
||||||
+4
-6
@@ -244,10 +244,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
roles reopened by that batch, after the switch closed them, so every
|
roles reopened by that batch, after the switch closed them, so every
|
||||||
switched-off one is closed again here.
|
switched-off one is closed again here.
|
||||||
|
|
||||||
After the closes, `Store.rebuild_dupes` recomputes which open roles are
|
The `prep` stage that follows maps what this run stored and folds the
|
||||||
copies of one posting, so a closed kept role hands its group on and the
|
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
||||||
roles this run added join theirs; `copies` is how many open roles it
|
and the roles this run added join theirs.
|
||||||
leaves hidden as copies.
|
|
||||||
"""
|
"""
|
||||||
answered = searched = failed = incomplete = closed = 0
|
answered = searched = failed = incomplete = closed = 0
|
||||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||||
@@ -284,10 +283,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
closed += store.close_unseen("search", stale)
|
closed += store.close_unseen("search", stale)
|
||||||
if run_marker:
|
if run_marker:
|
||||||
release_run(store, run_marker)
|
release_run(store, run_marker)
|
||||||
copies = store.rebuild_dupes().copies
|
|
||||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||||
return {"boards": answered, "searches": searched, "failed": failed,
|
return {"boards": answered, "searches": searched, "failed": failed,
|
||||||
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
|
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||||
|
|
||||||
|
|
||||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
||||||
|
|||||||
@@ -649,6 +649,13 @@ SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
|||||||
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
|
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
|
||||||
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
|
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
|
||||||
|
|
||||||
|
#: The kind of contract a posting states, as `app.prep` maps a host's word.
|
||||||
|
#: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the
|
||||||
|
#: readers' own tests, since a plugin imports nothing from `app`.
|
||||||
|
Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship",
|
||||||
|
"freelance"]
|
||||||
|
EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e)
|
||||||
|
|
||||||
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
||||||
Permit = Literal["", "sponsors", "no_sponsorship"]
|
Permit = Literal["", "sponsors", "no_sponsorship"]
|
||||||
PERMITS: tuple[str, ...] = get_args(Permit)
|
PERMITS: tuple[str, ...] = get_args(Permit)
|
||||||
@@ -701,6 +708,16 @@ def trusted_posted(host: str) -> bool:
|
|||||||
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
||||||
|
|
||||||
|
|
||||||
|
def host_word(value: Any) -> str:
|
||||||
|
"""A host's word as `roles.*_host` stores it: stripped text, a bool as
|
||||||
|
'true' or 'false', '' for nothing. What `app.prep` maps."""
|
||||||
|
if value is None:
|
||||||
|
return ""
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return "true" if value else "false"
|
||||||
|
return str(value).strip()
|
||||||
|
|
||||||
|
|
||||||
def posted_host(raw: dict[str, Any]) -> str:
|
def posted_host(raw: dict[str, Any]) -> str:
|
||||||
"""The host a role's dates came from, read the way `ats` and `search`
|
"""The host a role's dates came from, read the way `ats` and `search`
|
||||||
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
|
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
|
||||||
@@ -797,6 +814,8 @@ class Role(Model):
|
|||||||
seniority_model: str = ""
|
seniority_model: str = ""
|
||||||
seniority_stated: str = ""
|
seniority_stated: str = ""
|
||||||
seniority_title: str = ""
|
seniority_title: str = ""
|
||||||
|
#: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`.
|
||||||
|
employment: str = ""
|
||||||
function: str = ""
|
function: str = ""
|
||||||
permit: Permit = ""
|
permit: Permit = ""
|
||||||
#: ISO 639-1 codes the posting says are required, lowercase.
|
#: ISO 639-1 codes the posting says are required, lowercase.
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
"""Prep: every decision about a role that needs a rule or data across roles.
|
||||||
|
|
||||||
|
Readers scrape and hand on the host's own words (`Store.upsert_role` keeps
|
||||||
|
them in `roles.*_host`); prep maps them, once, centrally, between the pull
|
||||||
|
and embedding. One door: `from app.prep import ...`.
|
||||||
|
|
||||||
|
words.py seniority, employment and remote: the host's table, the
|
||||||
|
vocabulary itself, the shared remote words
|
||||||
|
country.py the country from the resolved places, else the host's
|
||||||
|
hosts/ one table file per reader (`hosts.TABLES`)
|
||||||
|
stage.py `run`: the stage over the due roles, `unmapped` and the fold
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.prep.country import country
|
||||||
|
from app.prep.hosts import TABLES
|
||||||
|
from app.prep.stage import STATE, Prepped, company, fingerprint, run
|
||||||
|
from app.prep.words import REMOTE_WORDS, VOCABULARIES, employment, lookup, remote, seniority
|
||||||
|
|
||||||
|
__all__ = ["REMOTE_WORDS", "STATE", "TABLES", "VOCABULARIES", "Prepped", "company", "country",
|
||||||
|
"employment", "fingerprint", "lookup", "remote", "run", "seniority"]
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""A role's country, from its places, else from the one its host states."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
|
from app import places
|
||||||
|
|
||||||
|
|
||||||
|
def of_host(gaz: places.Gazetteer, stated: str) -> str | None:
|
||||||
|
"""The ISO code a host's country names, in the host's spelling: a code
|
||||||
|
("DK", "dk"), or a name the gazetteer has as an alias of a country
|
||||||
|
("Denmark", "Danmark"), `EU` or Europe by name, or a text that resolves to places of one
|
||||||
|
country. '' for nothing stated, None for a text naming no country."""
|
||||||
|
text = (stated or "").strip()
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
if ("country", text.upper()) in gaz.by_key or text.upper() == "EU":
|
||||||
|
return text.upper()
|
||||||
|
named = {"EU" if gaz.places[p].key == "europe" else gaz.places[p].key
|
||||||
|
for p in gaz.aliases.get(places.fold(text), ())
|
||||||
|
if gaz.places[p].kind == "country"
|
||||||
|
or (gaz.places[p].kind, gaz.places[p].key) == ("continent", "europe")}
|
||||||
|
if len(named) == 1:
|
||||||
|
return named.pop()
|
||||||
|
found = {gaz.country(p) for p in gaz.resolve(text)} - {""}
|
||||||
|
return found.pop() if len(found) == 1 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _in_europe(gaz: places.Gazetteer, code: str) -> bool:
|
||||||
|
if code == "EU":
|
||||||
|
return True
|
||||||
|
europe = gaz.by_key.get(("continent", "europe"))
|
||||||
|
place_id = gaz.by_key.get(("country", code))
|
||||||
|
return europe is not None and place_id is not None and europe in gaz.ancestors(place_id)
|
||||||
|
|
||||||
|
|
||||||
|
def country(gaz: places.Gazetteer, place_ids: Iterable[int], host_country: str = "") -> str:
|
||||||
|
"""The one country the role's resolved places lie in, continents aside;
|
||||||
|
Europe alone is `EU`; else, when they
|
||||||
|
lie in several all in Europe, `EU` (what `country_of` wrote for "Dublin,
|
||||||
|
London" and the country gate reads as Europe); else the host's stated
|
||||||
|
country (`of_host`); else ''."""
|
||||||
|
ids = list(place_ids)
|
||||||
|
# A continent beside a country is the wider unit a line adds ("Home
|
||||||
|
# based - EMEA; London" is GB), as `country_of` had it; alone, Europe
|
||||||
|
# is `EU`.
|
||||||
|
found = {gaz.country(p) for p in ids} - {""}
|
||||||
|
if not found and any(gaz.places[p].key == "europe" for p in ids if p in gaz.places):
|
||||||
|
found = {"EU"}
|
||||||
|
if len(found) == 1:
|
||||||
|
return found.pop()
|
||||||
|
if len(found) > 1 and all(_in_europe(gaz, code) for code in found):
|
||||||
|
return "EU"
|
||||||
|
return of_host(gaz, host_country) or ""
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
"""Each host's own words, one module per reader, named as the reader is.
|
||||||
|
|
||||||
|
A module defines any of three dicts, keyed by the host's word lowercased:
|
||||||
|
|
||||||
|
- `SENIORITY`, values in `app.models.SENIORITIES`
|
||||||
|
- `EMPLOYMENT`, values in `app.models.EMPLOYMENTS`
|
||||||
|
- `REMOTE`, values in `app.models.REMOTE` (remote, hybrid, onsite)
|
||||||
|
|
||||||
|
A value of "" is a word the host uses that names no value ("Experienced",
|
||||||
|
"Permanent"): known, so it is not listed as unmapped.
|
||||||
|
|
||||||
|
A host whose employment word is a code of several parts (`permanent_full_time`,
|
||||||
|
`SALARIED_FT`) may add `MATCH`, `[(fragment, value), ...]`, tried in order
|
||||||
|
after `EMPLOYMENT` with the first hit winning, and must then say how a
|
||||||
|
fragment matches in `MATCH_MODE`: "substring" (the fragment is in the
|
||||||
|
lowercased word) or "token" (it equals one of the word's parts split at `_`
|
||||||
|
and `-`, so `ft` does not match inside `software`). `TABLES` loads every
|
||||||
|
module of this package by name, so a table is added by adding its file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import pkgutil
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
#: The dict a module names, by the field it maps.
|
||||||
|
FIELDS = {"seniority": "SENIORITY", "employment": "EMPLOYMENT", "remote": "REMOTE"}
|
||||||
|
|
||||||
|
#: How a `MATCH` fragment may match.
|
||||||
|
MATCH_MODES = ("substring", "token")
|
||||||
|
|
||||||
|
|
||||||
|
def load() -> dict[str, dict[str, Any]]:
|
||||||
|
"""`host -> field -> {word: value}` for every module of the package, the
|
||||||
|
words lowercased and stripped. A host without a table for a field has
|
||||||
|
no entry for it. A `MATCH` is `host -> "match" -> {"mode": ..., "rules":
|
||||||
|
[[fragment, value], ...]}`; a module with `MATCH` and no valid
|
||||||
|
`MATCH_MODE` fails the import by name."""
|
||||||
|
tables: dict[str, dict[str, Any]] = {}
|
||||||
|
for info in sorted(pkgutil.iter_modules(__path__), key=lambda i: i.name):
|
||||||
|
if info.name.startswith("_"):
|
||||||
|
continue
|
||||||
|
module = importlib.import_module(f"{__name__}.{info.name}")
|
||||||
|
for field, name in FIELDS.items():
|
||||||
|
table = getattr(module, name, None)
|
||||||
|
if isinstance(table, dict):
|
||||||
|
tables.setdefault(info.name, {})[field] = {
|
||||||
|
str(k).strip().lower(): str(v) for k, v in table.items()}
|
||||||
|
rules = getattr(module, "MATCH", None)
|
||||||
|
if rules is not None:
|
||||||
|
mode = getattr(module, "MATCH_MODE", None)
|
||||||
|
if mode not in MATCH_MODES:
|
||||||
|
raise ValueError(f"app/prep/hosts/{info.name}.py has MATCH and its MATCH_MODE "
|
||||||
|
f"is {mode!r}, not one of {', '.join(MATCH_MODES)}")
|
||||||
|
tables.setdefault(info.name, {})["match"] = {
|
||||||
|
"mode": mode, "rules": [[str(f).lower(), str(v)] for f, v in rules]}
|
||||||
|
return tables
|
||||||
|
|
||||||
|
|
||||||
|
TABLES = load()
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
"""Platsbanken's words, for prep."""
|
||||||
|
|
||||||
|
#: `workplace_model.label`, in Swedish; mostly empty. Only "Arbete på plats"
|
||||||
|
#: is in the fixtures; the others are JobTech's other labels as the site
|
||||||
|
#: writes them, and a label not here is listed as unmapped.
|
||||||
|
REMOTE: dict[str, str] = {
|
||||||
|
"arbete på plats": "onsite", "på plats": "onsite",
|
||||||
|
"hybridarbete": "hybrid", "hybrid": "hybrid", "delvis på distans": "hybrid",
|
||||||
|
"delvis distans": "hybrid",
|
||||||
|
"distansarbete": "remote", "arbete på distans": "remote", "helt på distans": "remote",
|
||||||
|
"distans": "remote",
|
||||||
|
}
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
"""The prep stage: map what the readers handed on, then fold the copies.
|
||||||
|
|
||||||
|
A pull stores the host's own words and place texts (`Store.upsert_role`)
|
||||||
|
and marks the role due. `run` maps every due role: its company through
|
||||||
|
`company_aliases`, its places against the gazetteer, its level, contract
|
||||||
|
and remote through the host's tables (`app.prep.words`), and its country
|
||||||
|
from the places (`app.prep.country`). When the tables changed since the
|
||||||
|
last run (`fingerprint`, kept in state as `prep:tables`), every open role
|
||||||
|
is mapped again from its stored words, with no new pull. Then the words no
|
||||||
|
table knows are listed in `unmapped`, and `Store.rebuild_dupes` folds the
|
||||||
|
copies across the whole database.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from typing import Any, NamedTuple
|
||||||
|
|
||||||
|
from app import places
|
||||||
|
from app.models import company_key
|
||||||
|
from app.models import company_fold
|
||||||
|
from app.prep import words
|
||||||
|
from app.prep.country import country, of_host
|
||||||
|
from app.prep.hosts import TABLES
|
||||||
|
from app.prep.words import Tables
|
||||||
|
from app.storage import Store
|
||||||
|
|
||||||
|
#: Raised when the rules in this package change the way a stored word maps,
|
||||||
|
#: so every open role is mapped again as it is after a table changes.
|
||||||
|
VERSION = 1
|
||||||
|
|
||||||
|
#: The state key the last run's `fingerprint` is kept under.
|
||||||
|
STATE = "prep:tables"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class Prepped(NamedTuple):
|
||||||
|
"""What one run did: roles read, rows written, whether every role was
|
||||||
|
mapped again, words left unmapped, and the open copies after the fold
|
||||||
|
(None when it was not run)."""
|
||||||
|
|
||||||
|
read: int
|
||||||
|
written: int
|
||||||
|
remapped: bool
|
||||||
|
unmapped: int
|
||||||
|
copies: int | None
|
||||||
|
|
||||||
|
|
||||||
|
def fingerprint(tables: Tables = TABLES) -> str:
|
||||||
|
"""A hash of every host table, the shared remote words and `VERSION`."""
|
||||||
|
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS},
|
||||||
|
sort_keys=True)
|
||||||
|
return hashlib.sha1(blob.encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def company(folds: Mapping[str, Sequence[str]], name: str) -> str:
|
||||||
|
"""The key a role from `name` belongs to, as `Store.resolve_company`
|
||||||
|
has it: the company an alias names, the name's own key first when
|
||||||
|
several share the fold, else `company_key(name)`."""
|
||||||
|
own = company_key(name)
|
||||||
|
keys = folds.get(company_fold(name), ())
|
||||||
|
return own if own in keys or not keys else keys[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _places(raw: Any) -> list[Any]:
|
||||||
|
try:
|
||||||
|
found = json.loads(raw) if isinstance(raw, str) else raw
|
||||||
|
except ValueError:
|
||||||
|
return []
|
||||||
|
return found if isinstance(found, list) else []
|
||||||
|
|
||||||
|
|
||||||
|
def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str, Sequence[str]],
|
||||||
|
place_ids: Sequence[int], tables: Tables = TABLES) -> dict[str, Any]:
|
||||||
|
"""One role as `Store.write_prep` takes it, from a `Store.prep_rows` row."""
|
||||||
|
host = str(row["host"] or "")
|
||||||
|
stated, titled = words.seniority(host, row["seniority_host"], row["title"], tables)
|
||||||
|
return {
|
||||||
|
"uid": row["uid"],
|
||||||
|
"due": row["prep_due"],
|
||||||
|
"company_key": company(folds, row["company"]) if row["company"] else row["company_key"],
|
||||||
|
"seniority_stated": stated,
|
||||||
|
"seniority_title": titled,
|
||||||
|
"employment": words.employment(host, row["employment_host"], tables),
|
||||||
|
"remote": words.remote(host, row["remote_host"] or None, (), "", tables),
|
||||||
|
"remote_places": words.remote(host, None, _places(row["raw_places"]), row["place"],
|
||||||
|
tables),
|
||||||
|
"country": country(gaz, place_ids, row["country_host"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
||||||
|
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||||
|
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||||
|
or (found["remote"] and found["remote"] != row["remote"]) \
|
||||||
|
or (found["remote_places"] and not row["remote"]) \
|
||||||
|
or (found["country"] and found["country"] != row["country"]) \
|
||||||
|
or any(found[c] != row[c] for c in ("company_key", "seniority_stated",
|
||||||
|
"seniority_title", "employment"))
|
||||||
|
|
||||||
|
|
||||||
|
def unknown(field: str, host: str, word: str, gaz: places.Gazetteer,
|
||||||
|
tables: Tables = TABLES) -> bool:
|
||||||
|
"""Whether no table, vocabulary or gazetteer knows this host word."""
|
||||||
|
if field == "country":
|
||||||
|
return of_host(gaz, word) is None
|
||||||
|
if field == "remote" and word in ("true", "false"):
|
||||||
|
return False
|
||||||
|
return words.lookup(field, host, word, tables) is None
|
||||||
|
|
||||||
|
|
||||||
|
def run(store: Store, *, remap: bool = False, tables: Tables = TABLES,
|
||||||
|
dupes: bool = True) -> Prepped:
|
||||||
|
"""Map the due roles, or every open one when the tables changed, or
|
||||||
|
every role with `remap`; list the unmapped words; fold the copies."""
|
||||||
|
mark = fingerprint(tables)
|
||||||
|
whole = remap or store.state.get(STATE) != mark
|
||||||
|
rows = store.prep_rows("due")
|
||||||
|
if whole:
|
||||||
|
due = {r["uid"] for r in rows}
|
||||||
|
rows += [r for r in store.prep_rows("all" if remap else "open") if r["uid"] not in due]
|
||||||
|
ids = store.resolve_role_places([r["uid"] for r in rows])
|
||||||
|
gaz = store.gazetteer()
|
||||||
|
folds = store.company_folds()
|
||||||
|
written = []
|
||||||
|
for row in rows:
|
||||||
|
found = map_role(row, gaz, folds, ids.get(row["uid"], ()), tables)
|
||||||
|
if _changed(row, found):
|
||||||
|
written.append(found)
|
||||||
|
store.write_prep(written)
|
||||||
|
left = [w for w in store.host_words() if unknown(w[0], w[1], w[2], gaz, tables)]
|
||||||
|
store.replace_unmapped(left)
|
||||||
|
if whole:
|
||||||
|
store.state[STATE] = mark
|
||||||
|
copies = store.rebuild_dupes().copies if dupes else None
|
||||||
|
return Prepped(read=len(rows), written=len(written), remapped=whole, unmapped=len(left),
|
||||||
|
copies=copies)
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
"""A host's word to the app's vocabulary: seniority, employment and remote.
|
||||||
|
|
||||||
|
Each field is looked up the same way: the host's table in `hosts`, keyed by
|
||||||
|
the word lowercased, then a word that already is a value of the vocabulary
|
||||||
|
(what readers stored before they handed on the host's words, and what a
|
||||||
|
host with the app's own words sends), then, for remote, the shared word map.
|
||||||
|
An employment word may also match a host's `MATCH` fragments. A word none
|
||||||
|
of these knows is unmapped: it maps to '' and `lookup` says so,
|
||||||
|
so the prep stage can list it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from collections.abc import Iterable, Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.models import EMPLOYMENTS, REMOTE, SENIORITIES, remote_of
|
||||||
|
from app.prep.hosts import TABLES
|
||||||
|
from app.seniority import from_title
|
||||||
|
|
||||||
|
#: `host -> field -> {word: value}`, plus `"match"` (`hosts.load`).
|
||||||
|
Tables = Mapping[str, Mapping[str, Any]]
|
||||||
|
|
||||||
|
VOCABULARIES: dict[str, tuple[str, ...]] = {
|
||||||
|
"seniority": SENIORITIES, "employment": EMPLOYMENTS, "remote": REMOTE}
|
||||||
|
|
||||||
|
#: How hosts without a table of their own write remote, after `_squash`:
|
||||||
|
#: case, spaces, `-` and `_` gone ("On-site", "on_site", "in office").
|
||||||
|
REMOTE_WORDS: dict[str, str] = {
|
||||||
|
"remote": "remote", "fullyremote": "remote", "remotefirst": "remote",
|
||||||
|
"hybrid": "hybrid",
|
||||||
|
"onsite": "onsite", "office": "onsite", "inoffice": "onsite", "inperson": "onsite",
|
||||||
|
"officebased": "onsite", "onpremise": "onsite",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _match(rules: Mapping[str, Any], key: str) -> str | None:
|
||||||
|
"""The value of the first `MATCH` fragment `key` holds, by its mode."""
|
||||||
|
parts = set(re.split(r"[_-]", key)) if rules["mode"] == "token" else set()
|
||||||
|
for fragment, value in rules["rules"]:
|
||||||
|
if fragment in parts if rules["mode"] == "token" else fragment in key:
|
||||||
|
return str(value)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _squash(word: str) -> str:
|
||||||
|
return "".join(c for c in word.lower() if c not in " -_")
|
||||||
|
|
||||||
|
|
||||||
|
def lookup(field: str, host: str, word: str,
|
||||||
|
tables: Tables = TABLES) -> str | None:
|
||||||
|
"""The value `word` maps to for `host`, '' for a known word that names
|
||||||
|
none, None for a word nothing knows. An empty word is ''."""
|
||||||
|
key = (word or "").strip().lower()
|
||||||
|
if not key:
|
||||||
|
return ""
|
||||||
|
table = tables.get(host, {})
|
||||||
|
found = table.get(field, {}).get(key)
|
||||||
|
if found is None and field == "employment" and "match" in table:
|
||||||
|
found = _match(table["match"], key)
|
||||||
|
if found is not None:
|
||||||
|
return found
|
||||||
|
if key in VOCABULARIES[field]:
|
||||||
|
return key
|
||||||
|
if field == "remote":
|
||||||
|
return REMOTE_WORDS.get(_squash(key))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def seniority(host: str, word: str, title: str,
|
||||||
|
tables: Tables = TABLES) -> tuple[str, str]:
|
||||||
|
"""`(stated, from the title)`: the host's word through its table, and
|
||||||
|
`seniority.from_title`. `roles.seniority` is the model's level, else the
|
||||||
|
stated one, else the title's (`Store.write_prep`)."""
|
||||||
|
return lookup("seniority", host, word, tables) or "", from_title(title)
|
||||||
|
|
||||||
|
|
||||||
|
def employment(host: str, word: str,
|
||||||
|
tables: Tables = TABLES) -> str:
|
||||||
|
return lookup("employment", host, word, tables) or ""
|
||||||
|
|
||||||
|
|
||||||
|
def _place_text(place: Any) -> str:
|
||||||
|
return str((place.get("text") if isinstance(place, Mapping) else place) or "")
|
||||||
|
|
||||||
|
|
||||||
|
def remote(host: str, value: Any, places: Iterable[Any] = (), line: str = "",
|
||||||
|
tables: Tables = TABLES) -> str:
|
||||||
|
"""How the role is worked, one of `REMOTE` or ''.
|
||||||
|
|
||||||
|
`value` is the host's own (a bool, or its word; `host_word`'s 'true' and
|
||||||
|
'false' are bools). True is remote. False says nothing more than the
|
||||||
|
places do: SmartRecruiters, Ashby, Recruitee and Workable send it for
|
||||||
|
every role that is not remote, hybrid ones included. A word goes
|
||||||
|
through the host's table, else the shared map. When the value says
|
||||||
|
nothing, the place line or the place texts are read for a word
|
||||||
|
(`remote_of`: "Hybrid, Berlin"), and last a place the reader marked
|
||||||
|
remote is remote."""
|
||||||
|
if value is True or value == "true":
|
||||||
|
return "remote"
|
||||||
|
said = value not in (None, False, "false")
|
||||||
|
if said and (found := lookup("remote", host, str(value), tables)):
|
||||||
|
return found
|
||||||
|
places = list(places)
|
||||||
|
if found := remote_of(line or "; ".join(map(_place_text, places))):
|
||||||
|
return found
|
||||||
|
if any(isinstance(p, Mapping) and p.get("remote") is True for p in places):
|
||||||
|
return "remote"
|
||||||
|
return ""
|
||||||
+39
-2
@@ -19,7 +19,9 @@
|
|||||||
* Under the Follow form of Company boards, a fold lists what each reader
|
* Under the Follow form of Company boards, a fold lists what each reader
|
||||||
* declares (`/api/readers`): how its list arrives, where the text is, what
|
* declares (`/api/readers`): how its list arrives, where the text is, what
|
||||||
* it filters at the source, how it states places, and the robots.txt
|
* it filters at the source, how it states places, and the robots.txt
|
||||||
* decision taken when it was built.
|
* decision taken when it was built. A second fold lists the host words no
|
||||||
|
* table in `app/prep/hosts` maps (`/api/prep/unmapped`), per field and
|
||||||
|
* host, most roles first; it is left out when there are none.
|
||||||
*/
|
*/
|
||||||
(function () {
|
(function () {
|
||||||
"use strict";
|
"use strict";
|
||||||
@@ -320,6 +322,38 @@
|
|||||||
$("#board-form").after(fold);
|
$("#board-form").after(fold);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The host words prep could not map, one row per field and host. */
|
||||||
|
function renderUnmapped(words) {
|
||||||
|
if (!words.length) return;
|
||||||
|
const groups = new Map();
|
||||||
|
for (const w of words) {
|
||||||
|
const key = `${w.field} · ${w.host || "no host"}`;
|
||||||
|
if (!groups.has(key)) groups.set(key, []);
|
||||||
|
groups.get(key).push(w);
|
||||||
|
}
|
||||||
|
const fold = document.createElement("details");
|
||||||
|
fold.className = "prune";
|
||||||
|
fold.id = "unmapped";
|
||||||
|
const summary = document.createElement("summary");
|
||||||
|
summary.textContent = `Host words no table maps (${words.length})`;
|
||||||
|
const list = document.createElement("div");
|
||||||
|
list.className = "srows";
|
||||||
|
for (const [key, found] of groups) {
|
||||||
|
const row = document.createElement("div");
|
||||||
|
row.className = "srow";
|
||||||
|
const name = document.createElement("span");
|
||||||
|
name.className = "name";
|
||||||
|
name.textContent = key;
|
||||||
|
const sub = document.createElement("span");
|
||||||
|
sub.className = "sub";
|
||||||
|
sub.textContent = found.map((w) => `${w.word} ${w.roles}`).join(" · ");
|
||||||
|
row.append(name, sub);
|
||||||
|
list.appendChild(row);
|
||||||
|
}
|
||||||
|
fold.append(summary, list);
|
||||||
|
($("#readers") || $("#board-form")).after(fold);
|
||||||
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------ one section shown
|
// ------------------------------------------------------ one section shown
|
||||||
|
|
||||||
// Sources shows one section: the one the fragment names or contains, or the
|
// Sources shows one section: the one the fragment names or contains, or the
|
||||||
@@ -360,7 +394,10 @@
|
|||||||
|
|
||||||
document.addEventListener("DOMContentLoaded", async () => {
|
document.addEventListener("DOMContentLoaded", async () => {
|
||||||
if (!$("#searches-list")) return;
|
if (!$("#searches-list")) return;
|
||||||
if ($("#board-form")) api("/api/readers").then(renderReaders, () => {});
|
if ($("#board-form")) {
|
||||||
|
api("/api/readers").then(renderReaders, () => {})
|
||||||
|
.then(() => api("/api/prep/unmapped")).then(renderUnmapped, () => {});
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
await load();
|
await load();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ own rows (`url = ''`) are rewritten here.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -86,6 +87,38 @@ def write(conn: sqlite3.Connection, gaz: places.Gazetteer, uid: str,
|
|||||||
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
conn.executemany(_INSERT, rows(gaz, uid, place_texts))
|
||||||
|
|
||||||
|
|
||||||
|
def write_texts(conn: sqlite3.Connection, uid: str, place_texts: Sequence[str]) -> None:
|
||||||
|
"""Replace one role's own places with these texts, unresolved: what a
|
||||||
|
pull stores, for `app.prep` to resolve (`relink_uids`)."""
|
||||||
|
conn.execute("DELETE FROM role_places WHERE uid = ? AND url = ''", (uid,))
|
||||||
|
conn.executemany(_INSERT, [(uid, position, text, places.fold(text), None)
|
||||||
|
for position, text in enumerate(place_texts)])
|
||||||
|
|
||||||
|
|
||||||
|
def relink_uids(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||||
|
uids: Sequence[str]) -> dict[str, list[int]]:
|
||||||
|
"""Resolve the own places of these roles, writing only those that now
|
||||||
|
resolve differently. Returns `uid -> place ids` in the host's order for
|
||||||
|
every role of `uids` with places, what prep reads the country from."""
|
||||||
|
held: dict[str, dict[tuple[int, str], set[int | None]]] = {}
|
||||||
|
for start in range(0, len(uids), 5000):
|
||||||
|
chunk = json.dumps(list(uids[start:start + 5000]))
|
||||||
|
for uid, position, text, place_id in conn.execute(
|
||||||
|
"SELECT uid, position, text, place_id FROM role_places WHERE url = '' "
|
||||||
|
"AND uid IN (SELECT value FROM json_each(?)) ORDER BY uid, position, rowid",
|
||||||
|
(chunk,)):
|
||||||
|
held.setdefault(uid, {}).setdefault((position, text), set()).add(place_id)
|
||||||
|
found: dict[str, list[int]] = {}
|
||||||
|
changed = []
|
||||||
|
for uid, rows_of in held.items():
|
||||||
|
now = {key: set(gaz.resolve(key[1]) or (None,)) for key in rows_of}
|
||||||
|
if now != rows_of:
|
||||||
|
changed.append((uid, [text for _, text in sorted(rows_of)]))
|
||||||
|
found[uid] = [p for key in sorted(now) for p in gaz.resolve(key[1])]
|
||||||
|
write_many(conn, gaz, changed)
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
def write_many(conn: sqlite3.Connection, gaz: places.Gazetteer,
|
||||||
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
by_uid: Iterable[tuple[str, Sequence[str]]]) -> int:
|
||||||
"""`write` for many roles in one statement each way. Returns rows written."""
|
"""`write` for many roles in one statement each way. Returns rows written."""
|
||||||
|
|||||||
+82
-2
@@ -18,7 +18,9 @@ import sqlite3
|
|||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
|
|
||||||
from app import places
|
from app import places
|
||||||
from app.models import SENIORITIES, company_fold, company_key, country_of, posted_at, posted_host
|
from app.models import (
|
||||||
|
SENIORITIES, company_fold, company_key, country_of, host_word, posted_at, posted_host,
|
||||||
|
)
|
||||||
from app.seniority import from_title
|
from app.seniority import from_title
|
||||||
from app.storage import gazetteer
|
from app.storage import gazetteer
|
||||||
|
|
||||||
@@ -107,8 +109,19 @@ CREATE TABLE IF NOT EXISTS roles (
|
|||||||
teaser TEXT NOT NULL DEFAULT '',
|
teaser TEXT NOT NULL DEFAULT '',
|
||||||
seniority_model TEXT NOT NULL DEFAULT '',
|
seniority_model TEXT NOT NULL DEFAULT '',
|
||||||
seniority_stated TEXT NOT NULL DEFAULT '',
|
seniority_stated TEXT NOT NULL DEFAULT '',
|
||||||
seniority_title TEXT NOT NULL DEFAULT ''
|
seniority_title TEXT NOT NULL DEFAULT '',
|
||||||
|
seniority_host TEXT NOT NULL DEFAULT '',
|
||||||
|
employment_host TEXT NOT NULL DEFAULT '',
|
||||||
|
remote_host TEXT NOT NULL DEFAULT '',
|
||||||
|
country_host TEXT NOT NULL DEFAULT '',
|
||||||
|
employment TEXT NOT NULL DEFAULT '',
|
||||||
|
prep_due INTEGER NOT NULL DEFAULT 0
|
||||||
);
|
);
|
||||||
|
-- The *_host columns hold the host's own words as a reader handed them on
|
||||||
|
-- (app.models.host_word: a bool is 'true' or 'false'), and app.prep maps
|
||||||
|
-- them into seniority_stated, employment, remote and country. prep_due is above 0
|
||||||
|
-- on a role whose words, company, title or places changed since prep last
|
||||||
|
-- mapped it (Store.upsert_role), which is what the next prep reads.
|
||||||
-- roles.teaser is the short text a list carries, kept until the full text
|
-- roles.teaser is the short text a list carries, kept until the full text
|
||||||
-- arrives and embedded while there is none; a later empty teaser does not
|
-- arrives and embedded while there is none; a later empty teaser does not
|
||||||
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
-- wipe it. seniority_model, seniority_stated and seniority_title are the
|
||||||
@@ -352,6 +365,19 @@ CREATE TABLE IF NOT EXISTS vectors (
|
|||||||
embedded_at TEXT NOT NULL
|
embedded_at TEXT NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- A host word no table of app/prep/hosts maps, per field (seniority,
|
||||||
|
-- employment, remote, country) and host: how many open roles carry it and
|
||||||
|
-- when prep first met it. Rewritten by every prep; a word added to a table
|
||||||
|
-- leaves at the next one.
|
||||||
|
CREATE TABLE IF NOT EXISTS unmapped (
|
||||||
|
field TEXT NOT NULL,
|
||||||
|
host TEXT NOT NULL,
|
||||||
|
word TEXT NOT NULL,
|
||||||
|
roles INTEGER NOT NULL DEFAULT 0,
|
||||||
|
first_seen TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (field, host, word)
|
||||||
|
);
|
||||||
|
|
||||||
-- The string map a desk plugin is handed as self.state. One namespace shared
|
-- The string map a desk plugin is handed as self.state. One namespace shared
|
||||||
-- by every plugin, so a plugin prefixes its keys.
|
-- by every plugin, so a plugin prefixes its keys.
|
||||||
CREATE TABLE IF NOT EXISTS state (
|
CREATE TABLE IF NOT EXISTS state (
|
||||||
@@ -389,6 +415,8 @@ CREATE INDEX IF NOT EXISTS idx_roles_dup_of
|
|||||||
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
CREATE INDEX IF NOT EXISTS idx_role_places_uid ON role_places(uid);
|
||||||
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
CREATE INDEX IF NOT EXISTS idx_role_places_place ON role_places(place_id, uid);
|
||||||
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
CREATE INDEX IF NOT EXISTS idx_places_parent ON places(parent_id);
|
||||||
|
-- The roles the next prep maps. Partial: after a prep there are none.
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_roles_prep_due ON roles(prep_due) WHERE prep_due > 0;
|
||||||
"""
|
"""
|
||||||
|
|
||||||
#: The table whose absence marks a file that has never been a roledesk
|
#: The table whose absence marks a file that has never been a roledesk
|
||||||
@@ -567,6 +595,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
|||||||
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
|
(21, "companies.all_roles: a company tag that reads its boards with the agents' wide "
|
||||||
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
|
"reach, given to every favourite and taking over a board's {\"reach\": \"all\"}",
|
||||||
lambda conn: _add_all_roles(conn)),
|
lambda conn: _add_all_roles(conn)),
|
||||||
|
# A Python step: the host words are read out of raw's JSON.
|
||||||
|
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
|
||||||
|
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: What a file created from `SCHEMA` today is already at.
|
#: What a file created from `SCHEMA` today is already at.
|
||||||
@@ -833,6 +864,55 @@ def _add_all_roles(conn: sqlite3.Connection) -> None:
|
|||||||
"to tag; it reads the agents' reach now", target, key)
|
"to tag; it reads the agents' reach now", target, key)
|
||||||
|
|
||||||
|
|
||||||
|
_V22_COLUMNS = ("seniority_host", "employment_host", "remote_host", "country_host", "employment")
|
||||||
|
|
||||||
|
_V22 = """
|
||||||
|
CREATE TABLE IF NOT EXISTS unmapped (
|
||||||
|
field TEXT NOT NULL,
|
||||||
|
host TEXT NOT NULL,
|
||||||
|
word TEXT NOT NULL,
|
||||||
|
roles INTEGER NOT NULL DEFAULT 0,
|
||||||
|
first_seen TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (field, host, word)
|
||||||
|
);
|
||||||
|
DELETE FROM state WHERE key = 'prep:tables';
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _add_host_words(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Migration 22's step. The columns and `unmapped`, and the host words
|
||||||
|
of every role filled from its `raw` where it has them. Before this,
|
||||||
|
readers mapped `seniority` and `employment` in their own tables, so what
|
||||||
|
`raw` holds there is mostly a value of the vocabulary already, which
|
||||||
|
`app.prep` takes as it is. `remote` and `country` are the host's as
|
||||||
|
given. The `prep:tables` fingerprint goes, so the first prep maps every
|
||||||
|
open role from these words. Runs twice without harm."""
|
||||||
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||||
|
for column in _V22_COLUMNS:
|
||||||
|
if column not in columns:
|
||||||
|
conn.execute(f"ALTER TABLE roles ADD COLUMN {column} TEXT NOT NULL DEFAULT ''")
|
||||||
|
if "prep_due" not in columns:
|
||||||
|
conn.execute("ALTER TABLE roles ADD COLUMN prep_due INTEGER NOT NULL DEFAULT 0")
|
||||||
|
for statement in _statements(_V22):
|
||||||
|
conn.execute(statement)
|
||||||
|
updates = []
|
||||||
|
for uid, raw in conn.execute(
|
||||||
|
"SELECT uid, raw FROM roles WHERE json_extract(raw, '$.seniority') IS NOT NULL "
|
||||||
|
"OR json_extract(raw, '$.employment') IS NOT NULL "
|
||||||
|
"OR json_extract(raw, '$.remote') IS NOT NULL "
|
||||||
|
"OR json_extract(raw, '$.country') IS NOT NULL").fetchall():
|
||||||
|
try:
|
||||||
|
given = json.loads(raw or "{}")
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if isinstance(given, dict):
|
||||||
|
updates.append((*(host_word(given.get(k)) for k in (
|
||||||
|
"seniority", "employment", "remote", "country")), uid))
|
||||||
|
conn.executemany("UPDATE roles SET seniority_host = ?, employment_host = ?, "
|
||||||
|
"remote_host = ?, country_host = ? WHERE uid = ?", updates)
|
||||||
|
log.info("migration 22: host words from raw for %d roles", len(updates))
|
||||||
|
|
||||||
|
|
||||||
def user_version(conn: sqlite3.Connection) -> int:
|
def user_version(conn: sqlite3.Connection) -> int:
|
||||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||||
|
|
||||||
|
|||||||
+140
-44
@@ -22,13 +22,12 @@ import numpy as np
|
|||||||
from desk.models import Item
|
from desk.models import Item
|
||||||
from app import dupes, places
|
from app import dupes, places
|
||||||
from app.models import (
|
from app.models import (
|
||||||
AFFINITY, HAND_REASONS, LEVELS, REMOTE, SENIORITIES, STATUSES, Agent, Application,
|
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
||||||
RoleRow, ScoredRole,
|
RoleRow, ScoredRole,
|
||||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||||
posted_at, posted_host, remote_of, title_key,
|
host_word, posted_at, posted_host, remote_of, title_key,
|
||||||
)
|
)
|
||||||
from app.seniority import from_title
|
|
||||||
from app.storage import gazetteer, schema
|
from app.storage import gazetteer, schema
|
||||||
|
|
||||||
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
#: Text columns a plugin may fill from `Item.raw` under the same name. An
|
||||||
@@ -36,7 +35,17 @@ from app.storage import gazetteer, schema
|
|||||||
#: says less than `extract` found keeps what `extract` found. `function` is
|
#: says less than `extract` found keeps what `extract` found. `function` is
|
||||||
#: not here: it is not a board field (see the note by `keep_filled` below).
|
#: not here: it is not a board field (see the note by `keep_filled` below).
|
||||||
#: Nor is `seniority`, which is the title's level unless a board states one.
|
#: Nor is `seniority`, which is the title's level unless a board states one.
|
||||||
_RAW_TEXT = ("place", "country", "currency")
|
_RAW_TEXT = ("place", "currency")
|
||||||
|
|
||||||
|
#: The host's own words a reader hands on in `Item.raw`, stored as
|
||||||
|
#: `<name>_host` (`host_word`) for `app.prep` to map. `seniority_host` is
|
||||||
|
#: rewritten every pull, as the host's level always was; the others are kept
|
||||||
|
#: filled, since a role's own page (`enrich`) may state what its listing
|
||||||
|
#: does not.
|
||||||
|
_HOST_WORDS = ("seniority", "employment", "remote", "country")
|
||||||
|
_KEPT_HOST_WORDS = ("employment_host", "remote_host", "country_host")
|
||||||
|
#: What prep maps from that a pull rewrites whatever it brings.
|
||||||
|
_PREP_OVERWRITTEN = ("title", "seniority_host")
|
||||||
_RAW_INT = ("salary_min", "salary_max")
|
_RAW_INT = ("salary_min", "salary_max")
|
||||||
|
|
||||||
#: What `extract` may fill. Only an empty column is written: a board's own
|
#: What `extract` may fill. Only an empty column is written: a board's own
|
||||||
@@ -154,14 +163,11 @@ def _iso(moment: datetime) -> str:
|
|||||||
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
return moment.astimezone(UTC).isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
|
||||||
def _remote(value: Any) -> str:
|
def _own_texts(conn: sqlite3.Connection, uid: str) -> list[str]:
|
||||||
"""A plugin's remote flag as one of `REMOTE`. `True` is 'remote'; anything
|
"""One role's own place texts as stored, in the host's order."""
|
||||||
unrecognised is '' (unknown), which `extract` may fill later."""
|
return list(dict.fromkeys(r[0] for r in conn.execute(
|
||||||
if value is True:
|
"SELECT text FROM role_places WHERE uid = ? AND url = '' ORDER BY position, rowid",
|
||||||
return "remote"
|
(uid,))))
|
||||||
text = str(value or "").strip().lower().replace("-", "").replace(" ", "")
|
|
||||||
text = {"onsite": "onsite", "office": "onsite", "inoffice": "onsite"}.get(text, text)
|
|
||||||
return text if text in REMOTE else ""
|
|
||||||
|
|
||||||
|
|
||||||
def _like(text: str) -> str:
|
def _like(text: str) -> str:
|
||||||
@@ -607,10 +613,19 @@ class Store:
|
|||||||
"""Store one posting. Returns True when it was new.
|
"""Store one posting. Returns True when it was new.
|
||||||
|
|
||||||
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
||||||
every call, and a role seen again is open again. The gate columns come
|
every call, and a role seen again is open again. The raw columns come
|
||||||
from `item.raw` under their own names, with `company` falling back to
|
from `item.raw` under their own names, with `company` falling back to
|
||||||
the subscription's and `country` to what `place` names; an empty value
|
the subscription's; an empty value never overwrites a filled column.
|
||||||
never overwrites a filled column.
|
|
||||||
|
Nothing is mapped here: that is `app.prep`, which runs after every
|
||||||
|
pull. The host's words go to `seniority_host`, `employment_host`,
|
||||||
|
`remote_host` and `country_host` (`host_word`), `company_key` is
|
||||||
|
written unresolved on a new role only, and the mapped columns
|
||||||
|
(`seniority_stated`, `seniority_title`, `seniority`, `employment`,
|
||||||
|
`remote`, `country`) are left as prep last wrote them. `prep_due`
|
||||||
|
is 1 on a new role and grows by one when the title, the company, a
|
||||||
|
host word or the place texts changed or the role reopens, which
|
||||||
|
is what the next prep reads.
|
||||||
|
|
||||||
`posted_at` (see `app.models.posted_at`) is set from this pull's own
|
`posted_at` (see `app.models.posted_at`) is set from this pull's own
|
||||||
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
|
`published_at` and `stamp`; on an update the `ON CONFLICT` clause below
|
||||||
@@ -627,20 +642,14 @@ class Store:
|
|||||||
alone for good, whatever the plugin's raw guess now says — see
|
alone for good, whatever the plugin's raw guess now says — see
|
||||||
`set_extracted` for the other half of the rule.
|
`set_extracted` for the other half of the rule.
|
||||||
|
|
||||||
`seniority` is the first filled of three levels. `seniority_model`
|
|
||||||
is extract's (`set_extracted`) and never written here.
|
|
||||||
`seniority_stated` is the host's word as `raw["seniority"]` carries
|
|
||||||
it, already in `SENIORITIES` (anything else is ''), and
|
|
||||||
`seniority_title` is `seniority.from_title`; both are rewritten every
|
|
||||||
pull. The classifier is called here rather than in a plugin, so a
|
|
||||||
role from any source gets it, the way `country` comes from `place`.
|
|
||||||
|
|
||||||
The role's places (`role_places`) are the texts of `raw["places"]`,
|
The role's places (`role_places`) are the texts of `raw["places"]`,
|
||||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), each resolved
|
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
||||||
against the gazetteer; `place` is the line shown, the texts joined
|
when they differ from the stored ones; prep resolves them. Texts that
|
||||||
when `raw` has none. Like `place`, a pull with no place leaves them
|
did not change keep their resolved rows, so the feed's Place filter
|
||||||
as they are. `place_fold` is written with `place`, under the same
|
stays right between a pull and its prep. `place` is the line shown,
|
||||||
keep-filled rule. `teaser` is kept-filled too.
|
the texts joined when `raw` has none. Like `place`, a pull with no
|
||||||
|
place leaves them as they are. `place_fold` is written with `place`,
|
||||||
|
under the same keep-filled rule. `teaser` is kept-filled too.
|
||||||
|
|
||||||
A role is embedded from its description, else its teaser, else its
|
A role is embedded from its description, else its teaser, else its
|
||||||
title (`app.jobs.embed.document`). When a pull brings a better one of
|
title (`app.jobs.embed.document`). When a pull brings a better one of
|
||||||
@@ -660,12 +669,11 @@ class Store:
|
|||||||
"source": item.source,
|
"source": item.source,
|
||||||
"source_id": item.source_id,
|
"source_id": item.source_id,
|
||||||
"subscription_id": subscription.id if subscription else None,
|
"subscription_id": subscription.id if subscription else None,
|
||||||
"company_key": self.resolve_company(company),
|
"company_key": company_key(company),
|
||||||
"company": company,
|
"company": company,
|
||||||
"title": item.title,
|
"title": item.title,
|
||||||
"title_key": title_key(item.title),
|
"title_key": title_key(item.title),
|
||||||
"place_key": place_key(place),
|
"place_key": place_key(place),
|
||||||
"remote": _remote(raw.get("remote")),
|
|
||||||
"description": item.body,
|
"description": item.body,
|
||||||
"feed_text": item.excerpt,
|
"feed_text": item.excerpt,
|
||||||
"original_url": item.location,
|
"original_url": item.location,
|
||||||
@@ -677,30 +685,26 @@ class Store:
|
|||||||
"created_at": stamp,
|
"created_at": stamp,
|
||||||
}
|
}
|
||||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||||
|
row |= {f"{name}_host": host_word(raw.get(name)) for name in _HOST_WORDS}
|
||||||
row["place"] = place
|
row["place"] = place
|
||||||
row["country"] = row["country"] or country_of(place)
|
|
||||||
row["remote"] = row["remote"] or remote_of(place)
|
|
||||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||||
row["function"] = str(raw.get("function") or "")
|
row["function"] = str(raw.get("function") or "")
|
||||||
stated = str(raw.get("seniority") or "")
|
|
||||||
row["seniority_stated"] = stated if stated in SENIORITIES else ""
|
|
||||||
row["seniority_title"] = from_title(item.title)
|
|
||||||
row["seniority"] = row["seniority_stated"] or row["seniority_title"]
|
|
||||||
row["teaser"] = str(raw.get("teaser") or "")
|
row["teaser"] = str(raw.get("teaser") or "")
|
||||||
row["place_fold"] = places.fold(row["place"])
|
row["place_fold"] = places.fold(row["place"])
|
||||||
# A board's own pay line, one per pull like title or published_at —
|
# A board's own pay line, one per pull like title or published_at —
|
||||||
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
# not kept-filled like _RAW_TEXT: a board that stops stating pay
|
||||||
# must be able to clear it, not leave a stale line behind.
|
# must be able to clear it, not leave a stale line behind.
|
||||||
row["salary_text"] = str(raw.get("salary_text") or "")
|
row["salary_text"] = str(raw.get("salary_text") or "")
|
||||||
|
row["prep_due"] = 1
|
||||||
|
|
||||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
keep_filled = ("company", "place_key", "description", "feed_text", "original_url",
|
||||||
"feed_text", "original_url", "teaser", *_RAW_TEXT)
|
"teaser", *_RAW_TEXT, *_KEPT_HOST_WORDS)
|
||||||
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
updates = [f"{c}=COALESCE(NULLIF(excluded.{c}, ''), roles.{c})" for c in keep_filled]
|
||||||
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
updates += [f"{c}=COALESCE(excluded.{c}, roles.{c})"
|
||||||
for c in (*_RAW_INT, "subscription_id")]
|
for c in (*_RAW_INT, "subscription_id")]
|
||||||
updates += [f"{c}=excluded.{c}"
|
updates += [f"{c}=excluded.{c}"
|
||||||
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
for c in ("title", "title_key", "published_at", "raw", "last_seen",
|
||||||
"salary_text", "seniority_stated", "seniority_title")]
|
"salary_text", "seniority_host")]
|
||||||
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
updates.append("posted_at=MIN(excluded.posted_at, roles.posted_at)")
|
||||||
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
# Not `keep_filled`'s COALESCE(NULLIF(...)): that would let a fresh
|
||||||
# pull's classifier guess overwrite what extract decided, which is
|
# pull's classifier guess overwrite what extract decided, which is
|
||||||
@@ -711,11 +715,17 @@ class Store:
|
|||||||
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
updates.append("function=CASE WHEN roles.extracted_at IS NULL "
|
||||||
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
"THEN COALESCE(NULLIF(excluded.function, ''), roles.function) "
|
||||||
"ELSE roles.function END")
|
"ELSE roles.function END")
|
||||||
# The model's level, else the host's, else the title's.
|
|
||||||
updates.append("seniority=COALESCE(NULLIF(roles.seniority_model, ''), "
|
|
||||||
"NULLIF(excluded.seniority_stated, ''), excluded.seniority_title)")
|
|
||||||
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
updates.append("place_fold=CASE WHEN excluded.place != '' "
|
||||||
"THEN excluded.place_fold ELSE roles.place_fold END")
|
"THEN excluded.place_fold ELSE roles.place_fold END")
|
||||||
|
# One more than before when what prep maps from changed, or the role
|
||||||
|
# reopens, so a prep that read the old values leaves it due
|
||||||
|
# (`Store.prepped`). `roles.` is the stored row here, `excluded.`
|
||||||
|
# what this pull brought.
|
||||||
|
changed = [f"roles.{c} IS NOT excluded.{c}" for c in _PREP_OVERWRITTEN]
|
||||||
|
changed += [f"(excluded.{c} != '' AND roles.{c} IS NOT excluded.{c})"
|
||||||
|
for c in ("company", *_KEPT_HOST_WORDS)]
|
||||||
|
updates.append(f"prep_due=CASE WHEN roles.closed_at IS NOT NULL OR {' OR '.join(changed)} "
|
||||||
|
"THEN roles.prep_due + 1 ELSE roles.prep_due END")
|
||||||
updates.append("closed_at=NULL")
|
updates.append("closed_at=NULL")
|
||||||
columns = ", ".join(row)
|
columns = ", ".join(row)
|
||||||
marks = ", ".join(f":{c}" for c in row)
|
marks = ", ".join(f":{c}" for c in row)
|
||||||
@@ -725,8 +735,11 @@ class Store:
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
f"INSERT INTO roles ({columns}) VALUES ({marks}) "
|
||||||
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
f"ON CONFLICT(uid) DO UPDATE SET {', '.join(updates)}", row)
|
||||||
if place_texts:
|
if place_texts and (before is None or _own_texts(conn, row["uid"]) != place_texts):
|
||||||
gazetteer.write(conn, self._gazetteer(conn), row["uid"], place_texts)
|
gazetteer.write_texts(conn, row["uid"], place_texts)
|
||||||
|
if before is not None:
|
||||||
|
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
||||||
|
(row["uid"],))
|
||||||
# `embed` encodes a role once, from the best text it has. When a
|
# `embed` encodes a role once, from the best text it has. When a
|
||||||
# later pull brings a better one, as a host that states the text
|
# later pull brings a better one, as a host that states the text
|
||||||
# on the role's own page does, the vector goes, so the next
|
# on the role's own page does, the vector goes, so the next
|
||||||
@@ -1253,6 +1266,89 @@ class Store:
|
|||||||
remotes)
|
remotes)
|
||||||
return len(countries) + len(remotes)
|
return len(countries) + len(remotes)
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------- prep
|
||||||
|
|
||||||
|
def prep_rows(self, scope: Literal["due", "open", "all"] = "due") -> list[sqlite3.Row]:
|
||||||
|
"""What `app.prep` maps a role from, and what it holds now: the roles
|
||||||
|
due (`prep_due` above 0), every open role, or every role."""
|
||||||
|
where = {"due": "prep_due > 0", "open": "closed_at IS NULL", "all": "1"}[scope]
|
||||||
|
return self._rows(
|
||||||
|
f"""SELECT uid, prep_due, {_HOST} AS host, title, company, company_key, place,
|
||||||
|
seniority_host, employment_host, remote_host, country_host,
|
||||||
|
seniority_model, seniority_stated, seniority_title, seniority,
|
||||||
|
employment, remote, country,
|
||||||
|
json_extract(raw, '$.places') AS raw_places
|
||||||
|
FROM roles WHERE {where}""")
|
||||||
|
|
||||||
|
def resolve_role_places(self, uids: Sequence[str]) -> dict[str, list[int]]:
|
||||||
|
"""Resolve these roles' own places against the gazetteer, writing the
|
||||||
|
rows that resolve differently; `uid -> place ids` for each that has
|
||||||
|
places (`gazetteer.relink_uids`)."""
|
||||||
|
with self.transaction() as conn:
|
||||||
|
return gazetteer.relink_uids(conn, self._gazetteer(conn), list(uids))
|
||||||
|
|
||||||
|
def company_folds(self) -> dict[str, list[str]]:
|
||||||
|
"""`fold -> company keys` of `company_aliases`, sorted: what
|
||||||
|
`resolve_company` asks one name at a time, for prep's whole batch."""
|
||||||
|
out: dict[str, list[str]] = {}
|
||||||
|
for r in self._rows("SELECT fold, company_key FROM company_aliases "
|
||||||
|
"ORDER BY fold, company_key"):
|
||||||
|
out.setdefault(str(r["fold"]), []).append(str(r["company_key"]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||||
|
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||||
|
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||||
|
`seniority_title`, `employment`, `remote`, `remote_places` and
|
||||||
|
`country`. `seniority` is the model's level, else the stated, else
|
||||||
|
the title's. An empty `remote` or `country` leaves the column as it
|
||||||
|
is, since `extract` fills an empty one. `remote_places`, what the
|
||||||
|
place texts say, only fills an empty column: it is a guess from a
|
||||||
|
line, weaker than what `extract` read in the posting. `prep_due` goes back to 0 only when no pull
|
||||||
|
changed the role since it was read. Returns the rows written."""
|
||||||
|
with self.transaction() as conn:
|
||||||
|
return conn.executemany(
|
||||||
|
"""UPDATE roles SET company_key = :company_key,
|
||||||
|
seniority_stated = :seniority_stated,
|
||||||
|
seniority_title = :seniority_title,
|
||||||
|
seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||||
|
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||||
|
employment = :employment,
|
||||||
|
remote = COALESCE(NULLIF(:remote, ''), NULLIF(remote, ''),
|
||||||
|
:remote_places),
|
||||||
|
country = COALESCE(NULLIF(:country, ''), country),
|
||||||
|
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||||
|
WHERE uid = :uid""", mapped).rowcount
|
||||||
|
|
||||||
|
def host_words(self) -> list[tuple[str, str, str, int]]:
|
||||||
|
"""`(field, host, word, open roles)` for every host word an open role
|
||||||
|
carries, the word lowercased: what prep checks against its tables."""
|
||||||
|
found: list[tuple[str, str, str, int]] = []
|
||||||
|
for field in ("seniority", "employment", "remote", "country"):
|
||||||
|
found += [(field, str(r[0]), str(r[1]), int(r[2])) for r in self._rows(
|
||||||
|
f"""SELECT {_HOST}, LOWER({field}_host), COUNT(*) FROM roles
|
||||||
|
WHERE closed_at IS NULL AND {field}_host != '' GROUP BY 1, 2""")]
|
||||||
|
return found
|
||||||
|
|
||||||
|
def replace_unmapped(self, rows: Iterable[tuple[str, str, str, int]]) -> None:
|
||||||
|
"""Make `unmapped` these `(field, host, word, roles)`, keeping the
|
||||||
|
`first_seen` of a word already listed."""
|
||||||
|
stamp = self.now()
|
||||||
|
with self.transaction() as conn:
|
||||||
|
seen = {(r[0], r[1], r[2]): r[3] for r in conn.execute(
|
||||||
|
"SELECT field, host, word, first_seen FROM unmapped")}
|
||||||
|
conn.execute("DELETE FROM unmapped")
|
||||||
|
conn.executemany(
|
||||||
|
"INSERT INTO unmapped (field, host, word, roles, first_seen) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?)",
|
||||||
|
[(*r, seen.get(r[:3], stamp)) for r in rows])
|
||||||
|
|
||||||
|
def unmapped(self) -> list[dict[str, Any]]:
|
||||||
|
"""The host words no table maps, by field and host, most roles first."""
|
||||||
|
return [dict(r) for r in self._rows(
|
||||||
|
"SELECT field, host, word, roles, first_seen FROM unmapped "
|
||||||
|
"ORDER BY field, host, roles DESC, word")]
|
||||||
|
|
||||||
# -------------------------------------------------------------- extract
|
# -------------------------------------------------------------- extract
|
||||||
|
|
||||||
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
def _top_scope(self, stage: Literal["extract", "summarise"]) -> tuple[str, list[Any]]:
|
||||||
|
|||||||
@@ -897,6 +897,23 @@ def list_readers() -> list[ReaderOut]:
|
|||||||
for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))]
|
for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))]
|
||||||
|
|
||||||
|
|
||||||
|
class UnmappedOut(BaseModel):
|
||||||
|
"""A host word no table of `app/prep/hosts` maps (`Store.unmapped`)."""
|
||||||
|
|
||||||
|
field: str
|
||||||
|
host: str
|
||||||
|
word: str
|
||||||
|
roles: int
|
||||||
|
first_seen: str
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/prep/unmapped")
|
||||||
|
def list_unmapped(store: StoreDep) -> list[UnmappedOut]:
|
||||||
|
"""The host words the last prep could not map, per field and host, most
|
||||||
|
roles first: what a table in `app/prep/hosts` should learn."""
|
||||||
|
return [UnmappedOut(**w) for w in store.unmapped()]
|
||||||
|
|
||||||
|
|
||||||
class SearchBody(BaseModel):
|
class SearchBody(BaseModel):
|
||||||
provider: str
|
provider: str
|
||||||
name: str = ""
|
name: str = ""
|
||||||
|
|||||||
@@ -197,9 +197,9 @@ def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) ->
|
|||||||
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
||||||
|
|
||||||
|
|
||||||
#: The employment words a row's `employment` may hold, for every host. A
|
#: A copy of `app.models.EMPLOYMENTS`, the vocabulary `app.prep` maps a
|
||||||
#: host's table maps its own words onto these; a word that names none of them
|
#: host's employment word onto, kept for the readers' tests since a plugin
|
||||||
#: (a permanent contract says nothing about hours) maps to "".
|
#: imports nothing from `app`. `tests/test_prep.py` holds the two equal.
|
||||||
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
|
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
|
||||||
|
|
||||||
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
|
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
"""`upsert`: a role stored as a pull stores it and then mapped, as the prep
|
||||||
|
stage after the pull maps it. For tests of what the feed, the gates and the
|
||||||
|
pages read, which are prep's values, not the raw ones `upsert_role` keeps."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app import prep
|
||||||
|
from app.storage import Store
|
||||||
|
|
||||||
|
|
||||||
|
def upsert(store: Store, item: Any, subscription: Any = None) -> bool:
|
||||||
|
new = store.upsert_role(item, subscription)
|
||||||
|
prep.run(store, dupes=False)
|
||||||
|
return new
|
||||||
@@ -31,7 +31,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
|||||||
#: no database, so storage may use them. `main` assembles the app from
|
#: no database, so storage may use them. `main` assembles the app from
|
||||||
#: everything below it; `cli` and `__main__` are the entry points and may
|
#: everything below it; `cli` and `__main__` are the entry points and may
|
||||||
#: reach anything.
|
#: reach anything.
|
||||||
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "jobs", "web", "main", "cli",
|
LAYERS = ["models", "places", "dupes", "seniority", "config", "storage", "prep", "jobs", "web", "main", "cli",
|
||||||
"__main__"]
|
"__main__"]
|
||||||
APP = "app"
|
APP = "app"
|
||||||
EXTERNAL = {"desk", "pipeline", "auth"}
|
EXTERNAL = {"desk", "pipeline", "auth"}
|
||||||
|
|||||||
+12
-11
@@ -13,6 +13,7 @@ from app.models import company_fold
|
|||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
from app.storage.schema import _add_company_aliases
|
from app.storage.schema import _add_company_aliases
|
||||||
from desk.models import Item
|
from desk.models import Item
|
||||||
|
from prepped import upsert
|
||||||
|
|
||||||
|
|
||||||
def undo_v20(conn: sqlite3.Connection) -> None:
|
def undo_v20(conn: sqlite3.Connection) -> None:
|
||||||
@@ -56,10 +57,10 @@ def test_a_company_is_its_own_alias(store: Store) -> None:
|
|||||||
|
|
||||||
def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
|
def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
|
||||||
store.add_company("Ørsted")
|
store.add_company("Ørsted")
|
||||||
store.upsert_role(posting("1", "Orsted"))
|
upsert(store, posting("1", "Orsted"))
|
||||||
store.upsert_role(posting("2", "ORSTED A/S"))
|
upsert(store, posting("2", "ORSTED A/S"))
|
||||||
store.upsert_role(posting("3", "Ørsted"))
|
upsert(store, posting("3", "Ørsted"))
|
||||||
store.upsert_role(posting("4", "Nobody"))
|
upsert(store, posting("4", "Nobody"))
|
||||||
assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"]
|
assert [company_of(store, s) for s in "1234"] == ["ørsted"] * 3 + ["nobody"]
|
||||||
assert [c.key for c in store.companies()] == ["ørsted"]
|
assert [c.key for c in store.companies()] == ["ørsted"]
|
||||||
|
|
||||||
@@ -67,7 +68,7 @@ def test_a_role_from_a_spelling_lands_on_the_company(store: Store) -> None:
|
|||||||
def test_an_added_alias_finds_the_company(store: Store) -> None:
|
def test_an_added_alias_finds_the_company(store: Store) -> None:
|
||||||
store.add_company("Ørsted")
|
store.add_company("Ørsted")
|
||||||
store.add_alias("ørsted", "Dong Energy")
|
store.add_alias("ørsted", "Dong Energy")
|
||||||
store.upsert_role(posting("1", "Dong Energy A/S"))
|
upsert(store, posting("1", "Dong Energy A/S"))
|
||||||
assert company_of(store, "1") == "ørsted"
|
assert company_of(store, "1") == "ørsted"
|
||||||
with pytest.raises(KeyError):
|
with pytest.raises(KeyError):
|
||||||
store.add_alias("nope", "X")
|
store.add_alias("nope", "X")
|
||||||
@@ -90,15 +91,15 @@ def test_duplicates_are_listed_by_fold_and_not_merged(store: Store) -> None:
|
|||||||
assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
assert store.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||||
assert len(store.companies()) == 3
|
assert len(store.companies()) == 3
|
||||||
# The company with the name's own key wins when a role names it.
|
# The company with the name's own key wins when a role names it.
|
||||||
store.upsert_role(posting("1", "Orsted"))
|
upsert(store, posting("1", "Orsted"))
|
||||||
assert company_of(store, "1") == "orsted"
|
assert company_of(store, "1") == "orsted"
|
||||||
|
|
||||||
|
|
||||||
def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
|
def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
|
||||||
store.add_company("Ørsted")
|
store.add_company("Ørsted")
|
||||||
store.add_company("Orsted")
|
store.add_company("Orsted")
|
||||||
store.upsert_role(posting("1", "Orsted"))
|
upsert(store, posting("1", "Orsted"))
|
||||||
store.upsert_role(posting("2", "Ørsted"))
|
upsert(store, posting("2", "Ørsted"))
|
||||||
sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted",
|
sub = store.add_subscription("ats", "orsted-board", label="Orsted", company="Orsted",
|
||||||
board="greenhouse")
|
board="greenhouse")
|
||||||
a, b = store.save_agent("A"), store.save_agent("B")
|
a, b = store.save_agent("A"), store.save_agent("B")
|
||||||
@@ -116,7 +117,7 @@ def test_a_merge_moves_roles_subscriptions_and_ratings(store: Store) -> None:
|
|||||||
assert store.duplicate_companies() == []
|
assert store.duplicate_companies() == []
|
||||||
assert store.company_aliases() == {"ørsted": ["orsted"]}
|
assert store.company_aliases() == {"ørsted": ["orsted"]}
|
||||||
# A later role from the gone spelling lands on the kept company.
|
# A later role from the gone spelling lands on the kept company.
|
||||||
store.upsert_role(posting("3", "Orsted"))
|
upsert(store, posting("3", "Orsted"))
|
||||||
assert company_of(store, "3") == "ørsted"
|
assert company_of(store, "3") == "ørsted"
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
store.merge_companies("ørsted", "ørsted")
|
store.merge_companies("ørsted", "ørsted")
|
||||||
@@ -147,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
|||||||
"VALUES (?, ?, '', '')", (key, name))
|
"VALUES (?, ?, '', '')", (key, name))
|
||||||
conn.execute("PRAGMA user_version = 19")
|
conn.execute("PRAGMA user_version = 19")
|
||||||
reopened = Store(path)
|
reopened = Store(path)
|
||||||
assert reopened.migrations_applied == [20, 21]
|
assert reopened.migrations_applied == [20, 21, 22]
|
||||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||||
assert len(reopened.companies()) == 3
|
assert len(reopened.companies()) == 3
|
||||||
|
|
||||||
@@ -184,7 +185,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
|||||||
conn.execute("ALTER TABLE companies DROP COLUMN all_roles")
|
conn.execute("ALTER TABLE companies DROP COLUMN all_roles")
|
||||||
conn.execute("PRAGMA user_version = 20")
|
conn.execute("PRAGMA user_version = 20")
|
||||||
reopened = Store(path)
|
reopened = Store(path)
|
||||||
assert reopened.migrations_applied == [21]
|
assert reopened.migrations_applied == [21, 22]
|
||||||
assert {c.key: c.all_roles for c in reopened.companies()} == {
|
assert {c.key: c.all_roles for c in reopened.companies()} == {
|
||||||
"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||||
params = {s.target: s.params for s in reopened.subscriptions(source="ats")}
|
params = {s.target: s.params for s in reopened.subscriptions(source="ats")}
|
||||||
|
|||||||
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [18, 19, 20, 21]
|
assert migrated.migrations_applied == [18, 19, 20, 21, 22]
|
||||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||||
|
|||||||
@@ -250,16 +250,16 @@ class _Context:
|
|||||||
return t
|
return t
|
||||||
|
|
||||||
|
|
||||||
def test_nothing_filled_passes_over_the_embed_and_score_behind_enrich() -> None:
|
def test_nothing_filled_passes_over_the_prep_embed_and_score_behind_enrich() -> None:
|
||||||
context = _Context()
|
context = _Context()
|
||||||
empty = Tally(results=({"filled": 0},))
|
empty = Tally(results=({"filled": 0},))
|
||||||
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 3},
|
out = cascade.step(None, None, empty, {"cascade": "pull", "task": "enrich", "at": 4},
|
||||||
context=context)
|
context=context)
|
||||||
assert out["next"] == "extract" and context.made[0].carries["at"] == 6
|
assert out["next"] == "extract" and context.made[0].carries["at"] == 8
|
||||||
|
|
||||||
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
|
out = cascade.step(None, None, Tally(results=({"filled": 2}, {"filled": 0})),
|
||||||
{"cascade": "pull", "task": "enrich", "at": 3}, context=context)
|
{"cascade": "pull", "task": "enrich", "at": 4}, context=context)
|
||||||
assert out["next"] == "embed" and context.made[1].carries["at"] == 4
|
assert out["next"] == "prep" and context.made[1].carries["at"] == 5
|
||||||
|
|
||||||
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
|
out = cascade.step(None, None, empty, {"cascade": "enrich", "task": "enrich", "at": 0},
|
||||||
context=context)
|
context=context)
|
||||||
@@ -290,8 +290,10 @@ def test_the_enrich_job_reads_the_pages_then_embeds_and_scores_those_roles(
|
|||||||
assert started is not None
|
assert started is not None
|
||||||
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
|
jobs = sorted(queue.jobs.of_cascade(queue.jobs.last(enrich.KIND).cascade_id), # type: ignore[union-attr]
|
||||||
key=lambda j: j.id)
|
key=lambda j: j.id)
|
||||||
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == ["enrich", "embed", "score"]
|
assert [j.result.get("task") for j in jobs if j.kind == FINAL] == \
|
||||||
assert [j.kind for j in jobs if j.kind != FINAL] == ["enrich", enrich.KIND, "embed", "score"]
|
["enrich", "prep", "embed", "score"]
|
||||||
|
assert [j.kind for j in jobs if j.kind != FINAL] == \
|
||||||
|
["enrich", enrich.KIND, "prep", "embed", "score"]
|
||||||
assert web["asked"] == [f"{BASE}/acme/1"]
|
assert web["asked"] == [f"{BASE}/acme/1"]
|
||||||
embedded = next(j for j in jobs if j.kind == "embed")
|
embedded = next(j for j in jobs if j.kind == "embed")
|
||||||
assert embedded.result["embedded"] == 1
|
assert embedded.result["embedded"] == 1
|
||||||
|
|||||||
+15
-14
@@ -16,6 +16,7 @@ from app.config import Config
|
|||||||
from app.jobs import extract
|
from app.jobs import extract
|
||||||
from app.models import Limits
|
from app.models import Limits
|
||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
|
from prepped import upsert
|
||||||
|
|
||||||
|
|
||||||
def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item:
|
def posting(source_id: str, title: str = "Backend Engineer", **raw: object) -> Item:
|
||||||
@@ -77,7 +78,7 @@ def stub(monkeypatch: pytest.MonkeyPatch, reply: Any) -> Model:
|
|||||||
def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
ranked(store, posting("1"))
|
ranked(store, posting("1"))
|
||||||
stub(monkeypatch, lambda roles: [answer(
|
stub(monkeypatch, lambda roles: [answer(
|
||||||
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
|
r["uid"], place="Berlin", country="de", remote="hybrid", seniority="senior",
|
||||||
@@ -100,7 +101,7 @@ def test_a_batch_writes_the_columns_and_stamps_extracted_at(
|
|||||||
def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
|
item = posting("1", place="Berlin", remote="hybrid", salary_min=70000)
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
ranked(store, item)
|
ranked(store, item)
|
||||||
stub(monkeypatch, lambda roles: [answer(
|
stub(monkeypatch, lambda roles: [answer(
|
||||||
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
|
r["uid"], place="Munich", remote="", salary_min=None, salary_max=90000,
|
||||||
@@ -116,13 +117,13 @@ def test_an_empty_answer_never_blanks_and_a_board_field_is_never_replaced(
|
|||||||
|
|
||||||
def test_the_next_upsert_keeps_what_extract_filled(
|
def test_the_next_upsert_keeps_what_extract_filled(
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
ranked(store, posting("1"))
|
ranked(store, posting("1"))
|
||||||
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
|
stub(monkeypatch, lambda roles: [answer(r["uid"], country="DE", seniority="mid")
|
||||||
for r in roles])
|
for r in roles])
|
||||||
modeltask.run(extract.TASK, config, store)
|
modeltask.run(extract.TASK, config, store)
|
||||||
|
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
|
|
||||||
role = store.role(posting("1").uid)
|
role = store.role(posting("1").uid)
|
||||||
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
|
assert role is not None and (role.country, role.seniority) == ("DE", "mid")
|
||||||
@@ -131,7 +132,7 @@ def test_the_next_upsert_keeps_what_extract_filled(
|
|||||||
def test_closed_and_answered_roles_are_not_offered(
|
def test_closed_and_answered_roles_are_not_offered(
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
for sid in ("open", "closed", "done"):
|
for sid in ("open", "closed", "done"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
ranked(store, posting("open"), posting("closed"), posting("done"))
|
ranked(store, posting("open"), posting("closed"), posting("done"))
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
|
conn.execute("UPDATE roles SET closed_at = '2026-09-22T00:00:00+00:00' "
|
||||||
@@ -147,7 +148,7 @@ def test_closed_and_answered_roles_are_not_offered(
|
|||||||
|
|
||||||
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
def test_an_unknown_uid_is_ignored_and_the_role_left_out_is_charged(
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
ranked(store, posting("1"))
|
ranked(store, posting("1"))
|
||||||
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
|
stub(monkeypatch, lambda roles: [answer("not-in-the-batch", country="DE")])
|
||||||
|
|
||||||
@@ -170,7 +171,7 @@ def test_only_the_best_of_each_agent_above_the_floor_is_read(
|
|||||||
the boards produced. A role nobody ranks is never sent; a batch dealt a
|
the boards produced. A role nobody ranks is never sent; a batch dealt a
|
||||||
uid finishes it."""
|
uid finishes it."""
|
||||||
for sid in ("best", "second", "low", "unscored", "gated"):
|
for sid in ("best", "second", "low", "unscored", "gated"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
one = store.save_agent("One", "python backend",
|
one = store.save_agent("One", "python backend",
|
||||||
limits=Limits(extract_top=1, extract_min=0.5))
|
limits=Limits(extract_top=1, extract_min=0.5))
|
||||||
two = store.save_agent("Two", "python backend", enabled=False)
|
two = store.save_agent("Two", "python backend", enabled=False)
|
||||||
@@ -209,7 +210,7 @@ def test_a_value_outside_the_vocabulary_is_dropped() -> None:
|
|||||||
|
|
||||||
def test_disabled_finds_nothing_and_asks_nobody(
|
def test_disabled_finds_nothing_and_asks_nobody(
|
||||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
model = stub(monkeypatch, lambda roles: [])
|
model = stub(monkeypatch, lambda roles: [])
|
||||||
|
|
||||||
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
|
result = modeltask.run(extract.TASK, Config(database=str(tmp_path / "x.db")), store)
|
||||||
@@ -220,7 +221,7 @@ def test_disabled_finds_nothing_and_asks_nobody(
|
|||||||
def test_a_role_read_before_without_a_level_is_read_again_for_it(
|
def test_a_role_read_before_without_a_level_is_read_again_for_it(
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
item = posting("1", seniority="senior")
|
item = posting("1", seniority="senior")
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
ranked(store, item)
|
ranked(store, item)
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
conn.execute("UPDATE roles SET extracted_at = '2026-09-22T00:00:00+00:00', "
|
||||||
@@ -243,8 +244,8 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
|||||||
withtext = posting("text", title="Senior Engineer", seniority="mid")
|
withtext = posting("text", title="Senior Engineer", seniority="mid")
|
||||||
noText = posting("teaser", title="Analyst")
|
noText = posting("teaser", title="Analyst")
|
||||||
noText.body = ""
|
noText.body = ""
|
||||||
store.upsert_role(withtext)
|
upsert(store, withtext)
|
||||||
store.upsert_role(noText)
|
upsert(store, noText)
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
|
conn.execute("UPDATE roles SET teaser = 'Join our analytics team' "
|
||||||
"WHERE source_id = 'teaser'")
|
"WHERE source_id = 'teaser'")
|
||||||
@@ -265,7 +266,7 @@ def test_the_prompt_carries_the_hints_and_the_teaser_when_there_is_no_text(
|
|||||||
def test_an_answer_with_no_level_is_charged_until_given_up(
|
def test_an_answer_with_no_level_is_charged_until_given_up(
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
item = posting("1")
|
item = posting("1")
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
ranked(store, item)
|
ranked(store, item)
|
||||||
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
|
stub(monkeypatch, lambda roles: [answer(r["uid"], seniority="") for r in roles])
|
||||||
|
|
||||||
@@ -279,7 +280,7 @@ def test_an_answer_with_no_level_is_charged_until_given_up(
|
|||||||
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
|
def test_each_agents_best_come_first_then_the_rest(store: Store) -> None:
|
||||||
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
|
items = [posting(s) for s in ("a1", "a2", "a3", "b1")]
|
||||||
for item in items:
|
for item in items:
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
one = store.save_agent("One", "python backend")
|
one = store.save_agent("One", "python backend")
|
||||||
two = store.save_agent("Two", "python backend")
|
two = store.save_agent("Two", "python backend")
|
||||||
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
|
store.replace_scores(one.id, [(items[0].uid, 0.9, ""), (items[1].uid, 0.8, ""),
|
||||||
@@ -296,7 +297,7 @@ def test_agreement_per_host(store: Store) -> None:
|
|||||||
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
|
("1", "personio", "senior", "senior"), ("2", "personio", "junior", "mid"),
|
||||||
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
|
("3", "lever", "mid", "mid"), ("4", "lever", "", "senior"),
|
||||||
("5", "lever", "senior", "")]:
|
("5", "lever", "senior", "")]:
|
||||||
store.upsert_role(posting(sid, board=board, seniority=stated))
|
upsert(store, posting(sid, board=board, seniority=stated))
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
|
conn.execute("UPDATE roles SET seniority_model = ? WHERE source_id = ?",
|
||||||
(model, sid))
|
(model, sid))
|
||||||
|
|||||||
+19
-18
@@ -24,6 +24,7 @@ from app.places import Gazetteer, fold, seed
|
|||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
|
|
||||||
from test_places import LINES, ORACLE_TOKENS
|
from test_places import LINES, ORACLE_TOKENS
|
||||||
|
from prepped import upsert
|
||||||
|
|
||||||
|
|
||||||
def gazetteer() -> Gazetteer:
|
def gazetteer() -> Gazetteer:
|
||||||
@@ -173,7 +174,7 @@ def test_a_fresh_file_is_seeded(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_a_joined_line_is_several_places(store: Store) -> None:
|
def test_a_joined_line_is_several_places(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", place="Copenhagen; London; Warsaw"))
|
upsert(store, posting("1", place="Copenhagen; London; Warsaw"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
|
assert texts(store, uid) == [(0, "Copenhagen", "Copenhagen"), (1, "London", "London"),
|
||||||
(2, "Warsaw", "Warsaw")]
|
(2, "Warsaw", "Warsaw")]
|
||||||
@@ -183,7 +184,7 @@ def test_a_joined_line_is_several_places(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", place="Berlin, Munich", places=[
|
upsert(store, posting("1", place="Berlin, Munich", places=[
|
||||||
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
|
{"text": "Berlin", "remote": False}, {"text": "Munich", "remote": False},
|
||||||
{"text": "Remote (DE)", "remote": True}]))
|
{"text": "Remote (DE)", "remote": True}]))
|
||||||
assert texts(store, posting("1").uid) == [
|
assert texts(store, posting("1").uid) == [
|
||||||
@@ -191,7 +192,7 @@ def test_the_places_of_the_reader_contract_come_first(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_places_without_a_line_make_the_line(store: Store) -> None:
|
def test_places_without_a_line_make_the_line(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", places=[{"text": "Lund", "remote": None},
|
upsert(store, posting("1", places=[{"text": "Lund", "remote": None},
|
||||||
{"text": "Malmö", "remote": None}]))
|
{"text": "Malmö", "remote": None}]))
|
||||||
role = store.role(posting("1").uid)
|
role = store.role(posting("1").uid)
|
||||||
assert role is not None
|
assert role is not None
|
||||||
@@ -199,26 +200,26 @@ def test_places_without_a_line_make_the_line(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
|
def test_a_pull_without_a_place_keeps_the_places(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", place="Aarhus"))
|
upsert(store, posting("1", place="Aarhus"))
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
|
assert texts(store, posting("1").uid) == [(0, "Aarhus", "Aarhus")]
|
||||||
store.upsert_role(posting("1", place="Odense"))
|
upsert(store, posting("1", place="Odense"))
|
||||||
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
|
assert texts(store, posting("1").uid) == [(0, "Odense", "Odense")]
|
||||||
|
|
||||||
|
|
||||||
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
|
def test_a_copys_place_survives_the_roles_own_pull(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", place="Aarhus"))
|
upsert(store, posting("1", place="Aarhus"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
with sqlite3.connect(store.path) as conn:
|
with sqlite3.connect(store.path) as conn:
|
||||||
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
conn.execute("INSERT INTO role_places (uid, position, text, fold, place_id, url) "
|
||||||
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
|
"VALUES (?, 0, 'Odense', 'odense', NULL, 'https://copy')", (uid,))
|
||||||
store.upsert_role(posting("1", place="Aalborg"))
|
upsert(store, posting("1", place="Aalborg"))
|
||||||
assert [(p.text, p.url) for p in store.role_places(uid)] == [
|
assert [(p.text, p.url) for p in store.role_places(uid)] == [
|
||||||
("Aalborg", ""), ("Odense", "https://copy")]
|
("Aalborg", ""), ("Odense", "https://copy")]
|
||||||
|
|
||||||
|
|
||||||
def test_an_extracted_place_is_resolved(store: Store) -> None:
|
def test_an_extracted_place_is_resolved(store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
store.set_extracted(uid, {"place": "Lund, Sweden"})
|
store.set_extracted(uid, {"place": "Lund, Sweden"})
|
||||||
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
|
assert texts(store, uid) == [(0, "Lund, Sweden", "Lund")]
|
||||||
@@ -236,7 +237,7 @@ def several(store: Store) -> dict[str, str]:
|
|||||||
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
|
"malmo": "Malmö", "lund": "Lund, Skåne", "billund": "Billund",
|
||||||
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
|
"zeiss": "Oberkochen; Jena", "remote": "Remote (SE)", "ny": "New York, NY"}
|
||||||
for key, line in lines.items():
|
for key, line in lines.items():
|
||||||
store.upsert_role(posting(key, place=line))
|
upsert(store, posting(key, place=line))
|
||||||
return {key: posting(key).uid for key in lines}
|
return {key: posting(key).uid for key in lines}
|
||||||
|
|
||||||
|
|
||||||
@@ -274,7 +275,7 @@ def test_a_token_that_names_nothing_is_refused(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
|
def test_a_country_given_without_a_place_still_counts(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", country="DK"))
|
upsert(store, posting("1", country="DK"))
|
||||||
assert uids(store, "country:DK") == {posting("1").uid}
|
assert uids(store, "country:DK") == {posting("1").uid}
|
||||||
|
|
||||||
|
|
||||||
@@ -312,15 +313,15 @@ def test_a_place_gate_matches_over_the_gazetteer(store: Store,
|
|||||||
|
|
||||||
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
|
def test_unresolved_texts_are_listed_most_roles_first(store: Store) -> None:
|
||||||
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
|
for n, line in enumerate(("Oberkochen; Jena", "Oberkochen", "Remote", "Aalen; Berlin")):
|
||||||
store.upsert_role(posting(str(n), place=line))
|
upsert(store, posting(str(n), place=line))
|
||||||
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
|
assert store.unresolved_places() == [("oberkochen", "Oberkochen", 2), ("aalen", "Aalen", 1),
|
||||||
("jena", "Jena", 1)]
|
("jena", "Jena", 1)]
|
||||||
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
|
assert store.unresolved_places(limit=1) == [("oberkochen", "Oberkochen", 2)]
|
||||||
|
|
||||||
|
|
||||||
def test_an_alias_links_what_it_names(store: Store) -> None:
|
def test_an_alias_links_what_it_names(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", place="Kbh. K"))
|
upsert(store, posting("1", place="Kbh. K"))
|
||||||
store.upsert_role(posting("2", place="Copenhagen"))
|
upsert(store, posting("2", place="Copenhagen"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
|
assert store.unresolved_places() == [("kbh k", "Kbh. K", 1)]
|
||||||
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
|
assert store.add_place_alias("Kbh", "place:copenhagen") == 1
|
||||||
@@ -333,8 +334,8 @@ def test_an_alias_links_what_it_names(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
|
def test_a_place_added_by_hand_is_resolved_and_found(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", place="Oberkochen; Jena"))
|
upsert(store, posting("1", place="Oberkochen; Jena"))
|
||||||
store.upsert_role(posting("2", place="Paderborn, Germany"))
|
upsert(store, posting("2", place="Paderborn, Germany"))
|
||||||
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
|
oberkochen = store.add_place("Oberkochen", "city", "country:DE")
|
||||||
store.add_place("Paderborn", "city", "country:DE")
|
store.add_place("Paderborn", "city", "country:DE")
|
||||||
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
|
assert texts(store, posting("1").uid) == [(0, "Oberkochen", "Oberkochen"), (1, "Jena", "")]
|
||||||
@@ -355,7 +356,7 @@ def test_an_alias_for_no_single_place_is_refused(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_relink_places_resolves_every_role_again(store: Store) -> None:
|
def test_relink_places_resolves_every_role_again(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", place="Oberkochen"))
|
upsert(store, posting("1", place="Oberkochen"))
|
||||||
with sqlite3.connect(store.path) as conn:
|
with sqlite3.connect(store.path) as conn:
|
||||||
conn.execute("UPDATE role_places SET place_id = "
|
conn.execute("UPDATE role_places SET place_id = "
|
||||||
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
|
"(SELECT id FROM places WHERE kind = 'city' AND key = 'berlin')")
|
||||||
@@ -381,7 +382,7 @@ def real(tmp_path_factory: pytest.TempPathFactory) -> tuple[Store, dict[str, tup
|
|||||||
lines = {}
|
lines = {}
|
||||||
for n, (place, country) in enumerate(LINES):
|
for n, (place, country) in enumerate(LINES):
|
||||||
item = posting(f"line{n}", place=place, country=country)
|
item = posting(f"line{n}", place=place, country=country)
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
lines[item.uid] = (place, country)
|
lines[item.uid] = (place, country)
|
||||||
return store, lines
|
return store, lines
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,358 @@
|
|||||||
|
"""`app.prep`: the host's words mapped centrally, the stage over the due
|
||||||
|
roles, the unmapped list, the table fingerprint and migration 22. The host
|
||||||
|
tables here are fakes; each real table is held to its host by R1's and R2's
|
||||||
|
tests, and to the vocabularies by the last tests of this file."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from app import cli, models, prep # noqa: E402
|
||||||
|
from app.prep import hosts, words # noqa: E402
|
||||||
|
from app.prep.country import country, of_host # noqa: E402
|
||||||
|
from app.storage import Store # noqa: E402
|
||||||
|
from desk.models import Item # noqa: E402
|
||||||
|
from plugins.readers import _shared # noqa: E402
|
||||||
|
|
||||||
|
TABLES: dict[str, Any] = {
|
||||||
|
"acme": {
|
||||||
|
"seniority": {"experienced": "", "sr": "senior"},
|
||||||
|
"employment": {"regular full time": "full_time", "permanent": ""},
|
||||||
|
"remote": {"2": "hybrid", "fully": "remote"},
|
||||||
|
},
|
||||||
|
"codes": {"match": {"mode": "token", "rules": [["ft", "full_time"], ["pt", "part_time"]]}},
|
||||||
|
"parts": {"match": {"mode": "substring",
|
||||||
|
"rules": [["fixed_term", "temporary"], ["full_time", "full_time"]]}},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------- words
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("field", "word", "value"), [
|
||||||
|
("seniority", "SR", "senior"),
|
||||||
|
("seniority", " Experienced ", ""), # known, names no level
|
||||||
|
("seniority", "senior", "senior"), # already the vocabulary
|
||||||
|
("seniority", "Grand Poobah", None), # unknown
|
||||||
|
("employment", "Regular Full Time", "full_time"),
|
||||||
|
("employment", "Permanent", ""),
|
||||||
|
("employment", "contract", "contract"),
|
||||||
|
("employment", "Zero hours", None),
|
||||||
|
("remote", "2", "hybrid"),
|
||||||
|
("remote", "On-site", "onsite"), # the shared words
|
||||||
|
("remote", "on_site", "onsite"),
|
||||||
|
("remote", "IN_OFFICE", "onsite"),
|
||||||
|
("remote", "office", "onsite"),
|
||||||
|
("remote", "Hybrid", "hybrid"),
|
||||||
|
("remote", "unspecified", None),
|
||||||
|
("seniority", "", ""),
|
||||||
|
])
|
||||||
|
def test_a_word_goes_through_the_hosts_table_then_the_vocabulary(
|
||||||
|
field: str, word: str, value: str | None) -> None:
|
||||||
|
assert words.lookup(field, "acme", word, TABLES) == value
|
||||||
|
|
||||||
|
|
||||||
|
def test_another_hosts_table_is_not_used() -> None:
|
||||||
|
assert words.lookup("seniority", "other", "sr", TABLES) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_match_table_matches_tokens_or_substrings_by_its_mode() -> None:
|
||||||
|
assert words.employment("codes", "SALARIED_FT", TABLES) == "full_time"
|
||||||
|
assert words.employment("codes", "SALARIED-PT", TABLES) == "part_time"
|
||||||
|
assert words.lookup("employment", "codes", "software", TABLES) is None
|
||||||
|
assert words.employment("parts", "fixed_term_full_time", TABLES) == "temporary"
|
||||||
|
assert words.employment("parts", "permanent_full_time", TABLES) == "full_time"
|
||||||
|
|
||||||
|
|
||||||
|
def test_seniority_is_the_stated_level_and_the_titles() -> None:
|
||||||
|
assert words.seniority("acme", "sr", "Staff Engineer", TABLES) == ("senior", "staff")
|
||||||
|
assert words.seniority("acme", "", "Engineer", TABLES) == ("", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_remote_from_a_bool_a_word_the_place_line_then_a_place_marked_remote() -> None:
|
||||||
|
assert words.remote("acme", True, (), "", TABLES) == "remote"
|
||||||
|
assert words.remote("acme", "true", (), "", TABLES) == "remote"
|
||||||
|
assert words.remote("acme", "fully", (), "Berlin", TABLES) == "remote"
|
||||||
|
# False says nothing the places do not: hosts send it for hybrid roles too.
|
||||||
|
assert words.remote("acme", False, (), "Berlin", TABLES) == ""
|
||||||
|
assert words.remote("acme", "false", (), "Hybrid, Berlin", TABLES) == "hybrid"
|
||||||
|
assert words.remote("acme", None, [{"text": "Berlin", "remote": True}], "", TABLES) \
|
||||||
|
== "remote"
|
||||||
|
assert words.remote("acme", "unspecified", [{"text": "Remote - Europe", "remote": None}],
|
||||||
|
"", TABLES) == "remote"
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------- country
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def store(tmp_path: Path) -> Store:
|
||||||
|
return Store(tmp_path / "roledesk.db")
|
||||||
|
|
||||||
|
|
||||||
|
def ids(store: Store, *texts: str) -> list[int]:
|
||||||
|
gaz = store.gazetteer()
|
||||||
|
return [p for text in texts for p in gaz.resolve(text)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_country_is_the_one_the_places_lie_in(store: Store) -> None:
|
||||||
|
gaz = store.gazetteer()
|
||||||
|
assert country(gaz, ids(store, "Aarhus", "Copenhagen"), "SE") == "DK"
|
||||||
|
assert country(gaz, ids(store, "Copenhagen", "Malmö"), "") == "EU"
|
||||||
|
assert country(gaz, ids(store, "Berlin", "Boston, MA"), "") == ""
|
||||||
|
assert country(gaz, ids(store, "Home based - EMEA", "London, UK")) == "GB"
|
||||||
|
assert country(gaz, ids(store, "Europe")) == "EU"
|
||||||
|
assert country(gaz, [], "dk") == "DK"
|
||||||
|
assert country(gaz, ids(store, "Remote"), "Sverige") == "SE"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("stated", "code"), [
|
||||||
|
("DK", "DK"), ("hu", "HU"), ("Denmark", "DK"), ("Danmark", "DK"), ("EU", "EU"),
|
||||||
|
("", ""), ("Narnia", None)])
|
||||||
|
def test_a_hosts_country_is_read_in_its_own_spelling(store: Store, stated: str,
|
||||||
|
code: str | None) -> None:
|
||||||
|
assert of_host(store.gazetteer(), stated) == code
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------- stage
|
||||||
|
|
||||||
|
def item(source_id: str, *, title: str = "Backend Engineer", host: str = "acme",
|
||||||
|
**raw: Any) -> Item:
|
||||||
|
found = Item(source_id=f"{host}:{source_id}", location=f"https://x.example/{source_id}",
|
||||||
|
title=title, body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||||
|
raw={"board": host, "company": "Acme"} | raw)
|
||||||
|
found.source = "ats"
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def row(store: Store, source_id: str, host: str = "acme") -> sqlite3.Row:
|
||||||
|
[found] = store._rows("SELECT * FROM roles WHERE source_id = ?", (f"{host}:{source_id}",))
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_pull_stores_the_host_words_and_prep_maps_them(store: Store) -> None:
|
||||||
|
store.upsert_role(item("1", seniority=" SR ", employment="Regular Full Time", remote="2",
|
||||||
|
places=[{"text": "Aarhus", "remote": None}]))
|
||||||
|
raw = row(store, "1")
|
||||||
|
assert (raw["seniority_host"], raw["employment_host"], raw["remote_host"]) == \
|
||||||
|
("SR", "Regular Full Time", "2")
|
||||||
|
assert (raw["seniority"], raw["employment"], raw["remote"], raw["country"]) == \
|
||||||
|
("", "", "", "")
|
||||||
|
assert raw["prep_due"] == 1
|
||||||
|
assert store.role_places(raw["uid"])[0].place_id is None
|
||||||
|
|
||||||
|
done = prep.run(store, tables=TABLES)
|
||||||
|
|
||||||
|
assert (done.read, done.written, done.remapped) == (1, 1, True)
|
||||||
|
mapped = row(store, "1")
|
||||||
|
assert (mapped["seniority_stated"], mapped["seniority"], mapped["employment"],
|
||||||
|
mapped["remote"], mapped["country"], mapped["prep_due"]) == \
|
||||||
|
("senior", "senior", "full_time", "hybrid", "DK", 0)
|
||||||
|
assert store.role_places(mapped["uid"])[0].name == "Aarhus"
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_a_role_whose_words_changed_is_due_again(store: Store) -> None:
|
||||||
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
|
||||||
|
store.upsert_role(item("2", seniority="sr"))
|
||||||
|
prep.run(store, tables=TABLES)
|
||||||
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Aarhus", "remote": None}]))
|
||||||
|
store.upsert_role(item("2", seniority="experienced"))
|
||||||
|
assert (row(store, "1")["prep_due"], row(store, "2")["prep_due"]) == (0, 1)
|
||||||
|
# The place stays resolved when its text did not change.
|
||||||
|
assert store.role_places(row(store, "1")["uid"])[0].name == "Aarhus"
|
||||||
|
|
||||||
|
store.upsert_role(item("1", seniority="sr", places=[{"text": "Odense", "remote": None}]))
|
||||||
|
assert row(store, "1")["prep_due"] == 1
|
||||||
|
assert prep.run(store, tables=TABLES).read == 2
|
||||||
|
assert row(store, "1")["country"] == "DK" and row(store, "2")["seniority"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_prep_that_read_old_words_leaves_the_role_due(store: Store) -> None:
|
||||||
|
store.upsert_role(item("1", seniority="sr"))
|
||||||
|
[read] = store.prep_rows("due")
|
||||||
|
store.upsert_role(item("1", seniority="experienced"))
|
||||||
|
store.write_prep([prep.stage.map_role(read, store.gazetteer(), {}, [], TABLES)])
|
||||||
|
assert row(store, "1")["prep_due"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_model_level_wins_and_a_place_guess_never_overwrites_extract(
|
||||||
|
store: Store) -> None:
|
||||||
|
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin"))
|
||||||
|
uid = row(store, "1")["uid"]
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET seniority_model = 'lead', remote = 'remote' "
|
||||||
|
"WHERE uid = ?", (uid,))
|
||||||
|
prep.run(store, tables=TABLES)
|
||||||
|
mapped = row(store, "1")
|
||||||
|
assert (mapped["seniority_title"], mapped["seniority"], mapped["remote"]) == \
|
||||||
|
("senior", "lead", "remote")
|
||||||
|
# The host's own statement does overwrite.
|
||||||
|
store.upsert_role(item("1", title="Senior Engineer", place="Hybrid, Berlin", remote="2"))
|
||||||
|
prep.run(store, tables=TABLES)
|
||||||
|
assert row(store, "1")["remote"] == "hybrid"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
||||||
|
store.add_company("Ørsted")
|
||||||
|
store.upsert_role(item("1", company="ORSTED A/S"))
|
||||||
|
assert row(store, "1")["company_key"] != "ørsted"
|
||||||
|
prep.run(store, tables=TABLES)
|
||||||
|
assert row(store, "1")["company_key"] == "ørsted"
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_words_are_listed_per_field_and_host(store: Store) -> None:
|
||||||
|
store.upsert_role(item("1", seniority="Grand Poobah", remote="unspecified",
|
||||||
|
country="Narnia"))
|
||||||
|
store.upsert_role(item("2", seniority="Grand Poobah", remote=False))
|
||||||
|
store.upsert_role(item("3", host="other", seniority="Grand Poobah"))
|
||||||
|
done = prep.run(store, tables=TABLES)
|
||||||
|
assert done.unmapped == 4
|
||||||
|
listed = {(w["field"], w["host"], w["word"]): w["roles"] for w in store.unmapped()}
|
||||||
|
assert listed == {("seniority", "acme", "grand poobah"): 2,
|
||||||
|
("seniority", "other", "grand poobah"): 1,
|
||||||
|
("remote", "acme", "unspecified"): 1,
|
||||||
|
("country", "acme", "narnia"): 1}
|
||||||
|
first = {w["word"]: w["first_seen"] for w in store.unmapped()}
|
||||||
|
|
||||||
|
# A word a table learns leaves the list at the next run; the rest keep
|
||||||
|
# the moment they were first met.
|
||||||
|
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"grand poobah": "exec"}}}
|
||||||
|
prep.run(store, tables=learned)
|
||||||
|
assert {w["word"]: w["first_seen"] for w in store.unmapped()} == \
|
||||||
|
first
|
||||||
|
assert ("seniority", "acme", "grand poobah") not in \
|
||||||
|
{(w["field"], w["host"], w["word"]) for w in store.unmapped()}
|
||||||
|
assert row(store, "1")["seniority"] == "exec"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_changed_table_maps_every_open_role_again_without_a_pull(store: Store) -> None:
|
||||||
|
store.upsert_role(item("1", seniority="Distinguished"))
|
||||||
|
store.upsert_role(item("2", seniority="Distinguished"))
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET closed_at = '2026-09-21' WHERE source_id = 'acme:2'")
|
||||||
|
assert prep.run(store, tables=TABLES).remapped
|
||||||
|
assert not prep.run(store, tables=TABLES).remapped
|
||||||
|
assert store.state[prep.STATE] == prep.fingerprint(TABLES)
|
||||||
|
|
||||||
|
learned = TABLES | {"acme": TABLES["acme"] | {"seniority": {"distinguished": "staff"}}}
|
||||||
|
done = prep.run(store, tables=learned)
|
||||||
|
assert (done.remapped, done.read) == (True, 1)
|
||||||
|
assert (row(store, "1")["seniority"], row(store, "2")["seniority"]) == ("staff", "")
|
||||||
|
# `--remap` maps the closed roles too.
|
||||||
|
assert prep.run(store, tables=learned, remap=True).read == 2
|
||||||
|
assert row(store, "2")["seniority"] == "staff"
|
||||||
|
|
||||||
|
|
||||||
|
def test_prep_folds_the_copies(store: Store) -> None:
|
||||||
|
for n in ("1", "2"):
|
||||||
|
store.upsert_role(item(n, place="Aarhus"))
|
||||||
|
assert prep.run(store, tables=TABLES).copies == 1
|
||||||
|
assert prep.run(store, tables=TABLES, dupes=False).copies is None
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- migration
|
||||||
|
|
||||||
|
def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "old.db"
|
||||||
|
store = Store(path)
|
||||||
|
store.upsert_role(item("1", seniority="senior", employment="full_time", remote=True,
|
||||||
|
country="DK"))
|
||||||
|
store.upsert_role(item("2"))
|
||||||
|
store.state[prep.STATE] = "old"
|
||||||
|
with sqlite3.connect(path) as conn:
|
||||||
|
conn.execute("DROP INDEX idx_roles_prep_due")
|
||||||
|
for column in ("seniority_host", "employment_host", "remote_host", "country_host",
|
||||||
|
"employment", "prep_due"):
|
||||||
|
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||||
|
conn.execute("DROP TABLE unmapped")
|
||||||
|
conn.execute("PRAGMA user_version = 21")
|
||||||
|
|
||||||
|
migrated = Store(path)
|
||||||
|
|
||||||
|
assert migrated.migrations_applied == [22]
|
||||||
|
one = row(migrated, "1")
|
||||||
|
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||||
|
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||||
|
assert row(migrated, "2")["remote_host"] == ""
|
||||||
|
assert prep.STATE not in migrated.state
|
||||||
|
# The first prep maps every open role from those words, the vocabulary
|
||||||
|
# values the old readers stored as themselves.
|
||||||
|
prep.run(migrated, tables={})
|
||||||
|
one = row(migrated, "1")
|
||||||
|
assert (one["seniority"], one["employment"], one["remote"], one["country"]) == \
|
||||||
|
("senior", "full_time", "remote", "DK")
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ CLI and API
|
||||||
|
|
||||||
|
def test_app_prep_runs_and_lists_the_unmapped_words(
|
||||||
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||||
|
path = tmp_path / "config.yaml"
|
||||||
|
path.write_text(f"database: {tmp_path / 'roledesk.db'}\n")
|
||||||
|
Store(tmp_path / "roledesk.db").upsert_role(item("1", seniority="Grand Poobah"))
|
||||||
|
|
||||||
|
assert cli.main(["-c", str(path), "prep"]) == 0
|
||||||
|
assert json.loads(capsys.readouterr().out)["read"] == 1
|
||||||
|
assert cli.main(["-c", str(path), "prep", "unmapped"]) == 0
|
||||||
|
assert capsys.readouterr().out.split() == ["seniority", "acme", "1", "grand", "poobah"]
|
||||||
|
assert cli.main(["-c", str(path), "prep", "--remap"]) == 0
|
||||||
|
assert json.loads(capsys.readouterr().out)["remapped"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_api_lists_the_unmapped_words(tmp_path: Path) -> None:
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from app.config import Config, ServerConfig
|
||||||
|
from app.main import create_app
|
||||||
|
|
||||||
|
app = create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
||||||
|
database=tmp_path / "roledesk.db"))
|
||||||
|
app.state.store.upsert_role(item("1", employment="Zero hours"))
|
||||||
|
prep.run(app.state.store, tables=TABLES)
|
||||||
|
with TestClient(app) as client:
|
||||||
|
found = client.get("/api/prep/unmapped").json()
|
||||||
|
assert [(w["field"], w["host"], w["word"], w["roles"]) for w in found] == \
|
||||||
|
[("employment", "acme", "zero hours", 1)]
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- the tables
|
||||||
|
|
||||||
|
def test_every_host_table_maps_into_its_vocabulary() -> None:
|
||||||
|
"""Passes with no table at all; R1's and R2's files are held here."""
|
||||||
|
for host, tables in hosts.load().items():
|
||||||
|
for field, table in tables.items():
|
||||||
|
if field == "match":
|
||||||
|
assert table["mode"] in hosts.MATCH_MODES, host
|
||||||
|
values = [value for _, value in table["rules"]]
|
||||||
|
field = "employment"
|
||||||
|
else:
|
||||||
|
assert all(key == key.strip().lower() for key in table), (host, field)
|
||||||
|
values = list(table.values())
|
||||||
|
assert set(values) <= {*words.VOCABULARIES[field], ""}, (host, field)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_match_table_must_say_how_it_matches(tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\n')
|
||||||
|
monkeypatch.setattr(hosts, "__path__", [str(tmp_path)])
|
||||||
|
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
|
||||||
|
with pytest.raises(ValueError, match=r"loose\.py has MATCH"):
|
||||||
|
hosts.load()
|
||||||
|
(tmp_path / "loose.py").write_text('MATCH = [("ft", "full_time")]\nMATCH_MODE = "token"\n')
|
||||||
|
monkeypatch.delitem(sys.modules, "app.prep.hosts.loose", raising=False)
|
||||||
|
importlib.invalidate_caches()
|
||||||
|
assert hosts.load()["loose"]["match"] == {"mode": "token", "rules": [["ft", "full_time"]]}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_readers_copy_of_the_employment_words_is_the_apps() -> None:
|
||||||
|
assert _shared.EMPLOYMENTS == models.EMPLOYMENTS
|
||||||
+10
-8
@@ -24,6 +24,7 @@ from app.models import Application # noqa: E402
|
|||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import _runner # noqa: E402
|
from plugins.readers import _runner # noqa: E402
|
||||||
from plugins.ats import ATSSource # noqa: E402
|
from plugins.ats import ATSSource # noqa: E402
|
||||||
|
from app import prep # noqa: E402
|
||||||
from app.jobs import cascade, pull # noqa: E402
|
from app.jobs import cascade, pull # noqa: E402
|
||||||
from app.storage import Store # noqa: E402
|
from app.storage import Store # noqa: E402
|
||||||
|
|
||||||
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
|||||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||||
|
|
||||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
||||||
"roles": 1, "closed": 1, "copies": 0}
|
"roles": 1, "closed": 1}
|
||||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
||||||
|
|
||||||
|
|
||||||
@@ -163,13 +164,14 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
|||||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||||
|
|
||||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
||||||
"roles": 0, "closed": 0, "copies": 0}
|
"roles": 0, "closed": 0}
|
||||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||||
|
|
||||||
|
|
||||||
def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
|
def test_the_prep_after_the_settle_rebuilds_the_copies(store: Store) -> None:
|
||||||
"""The settle's last step: a board's repeats of one job are one role,
|
"""The prep that follows the settle folds: a board's repeats of one job
|
||||||
and when the kept one closes the next copy is kept in its place."""
|
are one role, and when the kept one closes the next copy is kept in its
|
||||||
|
place."""
|
||||||
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
||||||
company="Acme", board="greenhouse")
|
company="Acme", board="greenhouse")
|
||||||
for job_id in ("1", "2", "3"):
|
for job_id in ("1", "2", "3"):
|
||||||
@@ -182,8 +184,8 @@ def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
|
|||||||
class FakeTally:
|
class FakeTally:
|
||||||
results = ({"roles": 3, "run": "run-1"},)
|
results = ({"roles": 3, "run": "run-1"},)
|
||||||
|
|
||||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||||
assert result["copies"] == 2
|
assert prep.run(store).copies == 2
|
||||||
[kept] = store.roles_page()
|
[kept] = store.roles_page()
|
||||||
assert kept.copies == 2
|
assert kept.copies == 2
|
||||||
|
|
||||||
@@ -191,7 +193,7 @@ def test_close_boards_rebuilds_the_copies_after_closing(store: Store) -> None:
|
|||||||
store.state["ats:ids:greenhouse:acme"] = json.dumps(
|
store.state["ats:ids:greenhouse:acme"] = json.dumps(
|
||||||
[f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone])
|
[f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone])
|
||||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||||
assert (result["closed"], result["copies"]) == (1, 1)
|
assert (result["closed"], prep.run(store).copies) == (1, 1)
|
||||||
[after] = store.roles_page()
|
[after] = store.roles_page()
|
||||||
assert after.uid != kept.uid and after.copies == 1
|
assert after.uid != kept.uid and after.copies == 1
|
||||||
|
|
||||||
|
|||||||
+19
-16
@@ -24,6 +24,7 @@ from app.config import CALIBRATION, AgentSeed, Config, ConfigError # noqa: E402
|
|||||||
from app.jobs import cascade, embed, score # noqa: E402
|
from app.jobs import cascade, embed, score # noqa: E402
|
||||||
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
|
from app.models import Candidate, Gate, Limits, Prompt, Weights # noqa: E402
|
||||||
from app.storage import Store # noqa: E402
|
from app.storage import Store # noqa: E402
|
||||||
|
from prepped import upsert # noqa: E402
|
||||||
|
|
||||||
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
NOW = datetime(2026, 9, 23, 9, 0, tzinfo=UTC)
|
||||||
|
|
||||||
@@ -97,7 +98,7 @@ def role(store: Store, source_id: str, title: str = "Backend Engineer", *,
|
|||||||
title=title, body=body, published_at=published_at,
|
title=title, body=body, published_at=published_at,
|
||||||
raw={"company": company, **raw})
|
raw={"company": company, **raw})
|
||||||
item.source = "ats"
|
item.source = "ats"
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
return item.uid
|
return item.uid
|
||||||
|
|
||||||
|
|
||||||
@@ -1023,7 +1024,7 @@ def test_the_shipped_starter_searches_validate() -> None:
|
|||||||
|
|
||||||
# ---------------------------------------------------------------- cascade
|
# ---------------------------------------------------------------- cascade
|
||||||
|
|
||||||
def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
def test_the_pull_job_runs_pull_prep_embed_score_extract_score_then_summarise(
|
||||||
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
store: Store, config: Config, fake: FakeEmbedder, tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||||
@@ -1053,14 +1054,15 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
|
|||||||
assert batch is not None
|
assert batch is not None
|
||||||
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
jobs = sorted(queue.jobs.of_cascade(batch.cascade_id), key=lambda j: j.id)
|
||||||
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
stages = [j.result.get("task") for j in jobs if j.kind == FINAL]
|
||||||
# Enrich found no board with role pages, so the embed and score behind it
|
# Enrich found no board with role pages, so the prep, embed and score
|
||||||
# are passed over.
|
# behind it are passed over.
|
||||||
assert stages == ["pull", "embed", "score", "enrich", "extract", "score", "summarise"]
|
assert stages == ["pull", "prep", "embed", "score", "enrich", "extract", "score",
|
||||||
|
"summarise"]
|
||||||
work = [j.kind for j in jobs if j.kind != FINAL]
|
work = [j.kind for j in jobs if j.kind != FINAL]
|
||||||
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
# The `pull` job and the chain it asked for are one cascade, so the Jobs
|
||||||
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
# tab shows a scheduled pull as one run. Extract and summarise are off and
|
||||||
# find nothing; score runs twice, before and after extract.
|
# find nothing; score runs twice, before and after extract.
|
||||||
assert work == ["pull", "pull_batch", "embed", "score", "score"]
|
assert work == ["pull", "pull_batch", "prep", "embed", "score", "score"]
|
||||||
assert all(j.state == "done" for j in jobs)
|
assert all(j.state == "done" for j in jobs)
|
||||||
assert len(store.scores(agent.id)) == 1
|
assert len(store.scores(agent.id)) == 1
|
||||||
# The board was stamped with the run the `pull` job claimed, and the
|
# The board was stamped with the run the `pull` job claimed, and the
|
||||||
@@ -1109,11 +1111,12 @@ def test_the_model_batches_are_isolated_with_a_timeout() -> None:
|
|||||||
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
def test_follows_walks_the_cascade_by_position_and_stops_at_its_end() -> None:
|
||||||
"""By position, not by name: `score` is in the chain twice, once before
|
"""By position, not by name: `score` is in the chain twice, once before
|
||||||
extract and once after, and the second must lead on to summarise."""
|
extract and once after, and the second must lead on to summarise."""
|
||||||
assert cascade.CASCADES["pull"] == ("pull", "embed", "score", "enrich", "embed", "score",
|
assert cascade.CASCADES["pull"] == ("pull", "prep", "embed", "score", "enrich", "prep",
|
||||||
"extract", "score", "summarise")
|
"embed", "score", "extract", "score", "summarise")
|
||||||
assert [cascade.follows("pull", at) for at in range(9)] == \
|
assert [cascade.follows("pull", at) for at in range(11)] == \
|
||||||
["embed", "score", "enrich", "embed", "score", "extract", "score", "summarise", ""]
|
["prep", "embed", "score", "enrich", "prep", "embed", "score", "extract", "score",
|
||||||
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 9) == ""
|
"summarise", ""]
|
||||||
|
assert cascade.follows("pull", -1) == "" and cascade.follows("pull", 11) == ""
|
||||||
assert cascade.follows("nope", 0) == ""
|
assert cascade.follows("nope", 0) == ""
|
||||||
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
assert cascade.chain("pull").carries == {"cascade": "pull", "task": "pull", "at": 0}
|
||||||
|
|
||||||
@@ -1126,10 +1129,10 @@ def test_step_hands_the_position_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||||||
made.append(t)
|
made.append(t)
|
||||||
return t
|
return t
|
||||||
|
|
||||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 5},
|
|
||||||
context=Context())
|
|
||||||
assert out == {"task": "score", "next": "extract"}
|
|
||||||
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 6}
|
|
||||||
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
|
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 7},
|
||||||
context=Context())
|
context=Context())
|
||||||
assert out["next"] == "summarise" and made[1].carries["at"] == 8
|
assert out == {"task": "score", "next": "extract"}
|
||||||
|
assert made[0].carries == {"cascade": "pull", "task": "extract", "at": 8}
|
||||||
|
out = cascade.step(None, None, Tally(), {"cascade": "pull", "task": "score", "at": 9},
|
||||||
|
context=Context())
|
||||||
|
assert out["next"] == "summarise" and made[1].carries["at"] == 10
|
||||||
@@ -20,6 +20,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from desk.models import Item # noqa: E402
|
from desk.models import Item # noqa: E402
|
||||||
|
from app import prep # noqa: E402
|
||||||
from plugins import readers, search # noqa: E402
|
from plugins import readers, search # noqa: E402
|
||||||
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
||||||
from plugins.search import SearchSource # noqa: E402
|
from plugins.search import SearchSource # noqa: E402
|
||||||
@@ -28,6 +29,8 @@ from app.jobs import pull # noqa: E402
|
|||||||
from pipeline import Tally # noqa: E402
|
from pipeline import Tally # noqa: E402
|
||||||
from app.models import country_of, remote_of # noqa: E402
|
from app.models import country_of, remote_of # noqa: E402
|
||||||
from app.storage import Store # noqa: E402
|
from app.storage import Store # noqa: E402
|
||||||
|
from prepped import upsert # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||||
|
|
||||||
@@ -108,9 +111,9 @@ def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) ->
|
|||||||
it.source = "ats"
|
it.source = "ats"
|
||||||
return it
|
return it
|
||||||
|
|
||||||
store.upsert_role(item("a", place="Copenhagen, Denmark"))
|
upsert(store, item("a", place="Copenhagen, Denmark"))
|
||||||
store.upsert_role(item("b", place="Remote", country="EU"))
|
upsert(store, item("b", place="Remote", country="EU"))
|
||||||
store.upsert_role(item("c", place="Somewhere"))
|
upsert(store, item("c", place="Somewhere"))
|
||||||
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
|
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
|
||||||
|
|
||||||
|
|
||||||
@@ -281,7 +284,9 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo
|
|||||||
def _pull(store: Store) -> dict[str, Any]:
|
def _pull(store: Store) -> dict[str, Any]:
|
||||||
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||||
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||||
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
closed = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||||
|
prep.run(store)
|
||||||
|
return closed
|
||||||
|
|
||||||
|
|
||||||
def test_a_complete_search_closes_what_it_stopped_listing(
|
def test_a_complete_search_closes_what_it_stopped_listing(
|
||||||
|
|||||||
+125
-124
@@ -19,6 +19,7 @@ from app.models import (
|
|||||||
)
|
)
|
||||||
from app.storage.store import pack
|
from app.storage.store import pack
|
||||||
from app.storage import LATEST, Store
|
from app.storage import LATEST, Store
|
||||||
|
from prepped import upsert
|
||||||
|
|
||||||
|
|
||||||
class Clock:
|
class Clock:
|
||||||
@@ -69,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
|||||||
"application_events", "agents", "scores", "state"} <= tables
|
"application_events", "agents", "scores", "state"} <= tables
|
||||||
assert "idx_roles_cluster" in indexes
|
assert "idx_roles_cluster" in indexes
|
||||||
assert store.migrations_applied == []
|
assert store.migrations_applied == []
|
||||||
assert LATEST == 21
|
assert LATEST == 22
|
||||||
# A second open finds it current and applies nothing.
|
# A second open finds it current and applies nothing.
|
||||||
assert Store(path).migrations_applied == []
|
assert Store(path).migrations_applied == []
|
||||||
|
|
||||||
@@ -186,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21]
|
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22]
|
||||||
acme = migrated.company("acme")
|
acme = migrated.company("acme")
|
||||||
assert acme is not None
|
assert acme is not None
|
||||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||||
@@ -236,9 +237,9 @@ def _downgrade_to_v4(path: Path) -> None:
|
|||||||
def test_v6_moves_the_votes_and_affinities_to_the_default_agent(tmp_path: Path) -> None:
|
def test_v6_moves_the_votes_and_affinities_to_the_default_agent(tmp_path: Path) -> None:
|
||||||
path = tmp_path / "v4.db"
|
path = tmp_path / "v4.db"
|
||||||
store = Store(path)
|
store = Store(path)
|
||||||
store.upsert_role(posting("liked"))
|
upsert(store, posting("liked"))
|
||||||
store.upsert_role(posting("passed"))
|
upsert(store, posting("passed"))
|
||||||
store.upsert_role(posting("none"))
|
upsert(store, posting("none"))
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
store.add_company("Other", key="other")
|
store.add_company("Other", key="other")
|
||||||
# The disabled agent has the lower id; the enabled one is the default.
|
# The disabled agent has the lower id; the enabled one is the default.
|
||||||
@@ -274,7 +275,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it(
|
|||||||
tmp_path: Path) -> None:
|
tmp_path: Path) -> None:
|
||||||
path = tmp_path / "v4.db"
|
path = tmp_path / "v4.db"
|
||||||
store = Store(path)
|
store = Store(path)
|
||||||
store.upsert_role(posting("liked"))
|
upsert(store, posting("liked"))
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
_downgrade_to_v4(path)
|
_downgrade_to_v4(path)
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
@@ -292,7 +293,7 @@ def test_v6_makes_an_agent_when_there_is_something_to_move_and_none_to_take_it(
|
|||||||
def test_v6_makes_no_agent_when_there_is_nothing_to_move(tmp_path: Path) -> None:
|
def test_v6_makes_no_agent_when_there_is_nothing_to_move(tmp_path: Path) -> None:
|
||||||
path = tmp_path / "v4.db"
|
path = tmp_path / "v4.db"
|
||||||
store = Store(path)
|
store = Store(path)
|
||||||
store.upsert_role(posting("plain"))
|
upsert(store, posting("plain"))
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
_downgrade_to_v4(path)
|
_downgrade_to_v4(path)
|
||||||
|
|
||||||
@@ -322,8 +323,8 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
|
|||||||
old_date = datetime(2020, 1, 1, tzinfo=UTC)
|
old_date = datetime(2020, 1, 1, tzinfo=UTC)
|
||||||
trusted = posting("1", board="greenhouse", published_at=old_date)
|
trusted = posting("1", board="greenhouse", published_at=old_date)
|
||||||
untrusted = posting("2", published_at=old_date)
|
untrusted = posting("2", published_at=old_date)
|
||||||
store.upsert_role(trusted)
|
upsert(store, trusted)
|
||||||
store.upsert_role(untrusted)
|
upsert(store, untrusted)
|
||||||
before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid)
|
before_trusted, before_untrusted = store.role(trusted.uid), store.role(untrusted.uid)
|
||||||
assert before_trusted is not None and before_untrusted is not None
|
assert before_trusted is not None and before_untrusted is not None
|
||||||
|
|
||||||
@@ -354,7 +355,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
|
|||||||
given = posting("3", place="Bogota, CO", country="US") # the board said so
|
given = posting("3", place="Bogota, CO", country="US") # the board said so
|
||||||
extracted = posting("4", place="Bogota, CO")
|
extracted = posting("4", place="Bogota, CO")
|
||||||
for item in (wrong, several, given, extracted):
|
for item in (wrong, several, given, extracted):
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
_undo_v14(conn)
|
_undo_v14(conn)
|
||||||
conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'")
|
conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'")
|
||||||
@@ -375,7 +376,7 @@ def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
|
|||||||
def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: Path) -> None:
|
def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: Path) -> None:
|
||||||
path = tmp_path / "v13.db"
|
path = tmp_path / "v13.db"
|
||||||
store = Store(path)
|
store = Store(path)
|
||||||
store.upsert_role(posting("a"))
|
upsert(store, posting("a"))
|
||||||
uid = posting("a").uid
|
uid = posting("a").uid
|
||||||
backend = store.save_agent("Backend")
|
backend = store.save_agent("Backend")
|
||||||
blank = store.save_agent("Blank")
|
blank = store.save_agent("Blank")
|
||||||
@@ -391,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21]
|
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22]
|
||||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||||
Prompt(name="Backend", text="python backend")]
|
Prompt(name="Backend", text="python backend")]
|
||||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||||
@@ -415,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21]
|
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22]
|
||||||
acme = migrated.company("acme")
|
acme = migrated.company("acme")
|
||||||
assert acme is not None
|
assert acme is not None
|
||||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||||
@@ -433,7 +434,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
|||||||
unknown = posting("unknown")
|
unknown = posting("unknown")
|
||||||
unknown.title = "Backend Engineer"
|
unknown.title = "Backend Engineer"
|
||||||
for item in (plain, extracted, kept, unknown):
|
for item in (plain, extracted, kept, unknown):
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
_undo_v17(conn)
|
_undo_v17(conn)
|
||||||
conn.execute("UPDATE roles SET seniority = ''")
|
conn.execute("UPDATE roles SET seniority = ''")
|
||||||
@@ -444,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21]
|
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22]
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||||
"SELECT uid, place_fold, seniority FROM roles")}
|
"SELECT uid, place_fold, seniority FROM roles")}
|
||||||
@@ -492,8 +493,8 @@ def test_prompts_are_saved_replaced_reordered_and_deleted(store: Store) -> None:
|
|||||||
|
|
||||||
def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage(
|
def test_an_agent_whose_prompts_are_all_empty_feeds_neither_model_stage(
|
||||||
store: Store) -> None:
|
store: Store) -> None:
|
||||||
store.upsert_role(posting("a"))
|
upsert(store, posting("a"))
|
||||||
store.upsert_role(posting("b"))
|
upsert(store, posting("b"))
|
||||||
a, b = posting("a").uid, posting("b").uid
|
a, b = posting("a").uid, posting("b").uid
|
||||||
top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
top = Limits(extract_top=1, extract_min=0, summarise_top=1, summarise_min=0)
|
||||||
blank = store.save_agent("Blank", limits=top,
|
blank = store.save_agent("Blank", limits=top,
|
||||||
@@ -585,11 +586,11 @@ def test_posted_at_rejects_a_future_or_absurdly_old_date() -> None:
|
|||||||
|
|
||||||
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
|
def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
|
||||||
store: Store, clock: Clock) -> None:
|
store: Store, clock: Clock) -> None:
|
||||||
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is True
|
assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is True
|
||||||
first = store.role(posting().uid)
|
first = store.role(posting().uid)
|
||||||
assert first is not None
|
assert first is not None
|
||||||
clock.tick()
|
clock.tick()
|
||||||
assert store.upsert_role(posting(company="Acme GmbH", place="Berlin")) is False
|
assert upsert(store, posting(company="Acme GmbH", place="Berlin")) is False
|
||||||
again = store.role(posting().uid)
|
again = store.role(posting().uid)
|
||||||
assert again is not None
|
assert again is not None
|
||||||
assert again.first_seen == first.first_seen == first.created_at
|
assert again.first_seen == first.first_seen == first.created_at
|
||||||
@@ -601,18 +602,18 @@ def test_upsert_twice_keeps_first_seen_and_bumps_last_seen(
|
|||||||
def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None:
|
def test_a_vector_made_without_the_text_goes_when_the_text_arrives(store: Store) -> None:
|
||||||
listed = posting()
|
listed = posting()
|
||||||
listed.body = ""
|
listed.body = ""
|
||||||
store.upsert_role(listed)
|
upsert(store, listed)
|
||||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||||
assert store.roles_to_embed("m", 10) == []
|
assert store.roles_to_embed("m", 10) == []
|
||||||
|
|
||||||
store.upsert_role(listed) # listed again, still without text
|
upsert(store, listed) # listed again, still without text
|
||||||
assert store.roles_to_embed("m", 10) == []
|
assert store.roles_to_embed("m", 10) == []
|
||||||
store.upsert_role(posting()) # the role's own page was read
|
upsert(store, posting()) # the role's own page was read
|
||||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||||
|
|
||||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||||
store.upsert_role(listed) # later pulls send no text
|
upsert(store, listed) # later pulls send no text
|
||||||
store.upsert_role(posting()) # or the same text again
|
upsert(store, posting()) # or the same text again
|
||||||
assert store.roles_to_embed("m", 10) == []
|
assert store.roles_to_embed("m", 10) == []
|
||||||
|
|
||||||
|
|
||||||
@@ -624,9 +625,9 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
|
|||||||
listed.body = ""
|
listed.body = ""
|
||||||
bare = posting()
|
bare = posting()
|
||||||
bare.body = ""
|
bare.body = ""
|
||||||
store.upsert_role(bare) # the title alone
|
upsert(store, bare) # the title alone
|
||||||
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
|
store.put_vectors("m", [(bare.uid, np.ones(4, dtype=np.float32))])
|
||||||
store.upsert_role(listed) # a teaser beats the title
|
upsert(store, listed) # a teaser beats the title
|
||||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||||
role = store.role(listed.uid)
|
role = store.role(listed.uid)
|
||||||
assert role is not None and role.teaser == "Build the payments core."
|
assert role is not None and role.teaser == "Build the payments core."
|
||||||
@@ -634,21 +635,21 @@ def test_a_teaser_is_kept_and_its_vector_goes_when_a_better_text_arrives(
|
|||||||
assert store.roles_page()[0].excerpt == "Build the payments core."
|
assert store.roles_page()[0].excerpt == "Build the payments core."
|
||||||
|
|
||||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||||
store.upsert_role(bare) # a pull without it keeps it
|
upsert(store, bare) # a pull without it keeps it
|
||||||
assert store.roles_to_embed("m", 10) == []
|
assert store.roles_to_embed("m", 10) == []
|
||||||
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
|
assert store.role(listed.uid).teaser == "Build the payments core." # type: ignore[union-attr]
|
||||||
store.upsert_role(posting(teaser="Build the payments core.")) # the text beats it
|
upsert(store, posting(teaser="Build the payments core.")) # the text beats it
|
||||||
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
assert [r.uid for r in store.roles_to_embed("m", 10)] == [listed.uid]
|
||||||
role = store.role(listed.uid)
|
role = store.role(listed.uid)
|
||||||
assert role is not None and document(role).endswith("We build things.")
|
assert role is not None and document(role).endswith("We build things.")
|
||||||
|
|
||||||
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
store.put_vectors("m", [(listed.uid, np.ones(4, dtype=np.float32))])
|
||||||
store.upsert_role(posting(teaser="Another teaser.")) # a new teaser under the text
|
upsert(store, posting(teaser="Another teaser.")) # a new teaser under the text
|
||||||
assert store.roles_to_embed("m", 10) == []
|
assert store.roles_to_embed("m", 10) == []
|
||||||
|
|
||||||
|
|
||||||
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
|
def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> None:
|
||||||
store.upsert_role(posting("1")) # "Senior Backend Engineer"
|
upsert(store, posting("1")) # "Senior Backend Engineer"
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
|
|
||||||
def levels() -> tuple[str, str, str, str]:
|
def levels() -> tuple[str, str, str, str]:
|
||||||
@@ -658,12 +659,12 @@ def test_the_level_is_the_model_then_the_host_then_the_title(store: Store) -> No
|
|||||||
role.seniority_title)
|
role.seniority_title)
|
||||||
|
|
||||||
assert levels() == ("senior", "", "", "senior")
|
assert levels() == ("senior", "", "", "senior")
|
||||||
store.upsert_role(posting("1", seniority="lead"))
|
upsert(store, posting("1", seniority="lead"))
|
||||||
assert levels() == ("lead", "", "lead", "senior")
|
assert levels() == ("lead", "", "lead", "senior")
|
||||||
store.upsert_role(posting("1", seniority="Teamleitung")) # not in the vocabulary
|
upsert(store, posting("1", seniority="Teamleitung")) # not in the vocabulary
|
||||||
assert levels() == ("senior", "", "", "senior")
|
assert levels() == ("senior", "", "", "senior")
|
||||||
store.set_extracted(uid, {"seniority": "staff"})
|
store.set_extracted(uid, {"seniority": "staff"})
|
||||||
store.upsert_role(posting("1", seniority="lead"))
|
upsert(store, posting("1", seniority="lead"))
|
||||||
assert levels() == ("staff", "staff", "lead", "senior")
|
assert levels() == ("staff", "staff", "lead", "senior")
|
||||||
|
|
||||||
|
|
||||||
@@ -675,7 +676,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
|||||||
stated = posting("stated", seniority="lead")
|
stated = posting("stated", seniority="lead")
|
||||||
extracted = posting("extracted")
|
extracted = posting("extracted")
|
||||||
for item in (joined, detailed, stated, extracted):
|
for item in (joined, detailed, stated, extracted):
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
_undo_v19(conn)
|
_undo_v19(conn)
|
||||||
# A role whose own page was read is pulled with an empty place in raw.
|
# A role whose own page was read is pulled with an empty place in raw.
|
||||||
@@ -686,7 +687,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [19, 20, 21]
|
assert migrated.migrations_applied == [19, 20, 21, 22]
|
||||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||||
assert migrated.role_places(stated.uid) == []
|
assert migrated.role_places(stated.uid) == []
|
||||||
@@ -703,7 +704,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
|||||||
|
|
||||||
|
|
||||||
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
||||||
store.upsert_role(posting(place="Berlin", remote=True, salary_min="80000",
|
upsert(store, posting(place="Berlin", remote=True, salary_min="80000",
|
||||||
salary_max=95000, currency="EUR", department="Eng"))
|
salary_max=95000, currency="EUR", department="Eng"))
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None
|
assert role is not None
|
||||||
@@ -713,7 +714,7 @@ def test_upsert_unpacks_raw_and_keeps_filled_columns(store: Store) -> None:
|
|||||||
assert role.description == "We build things."
|
assert role.description == "We build things."
|
||||||
assert role.original_url.endswith("/123")
|
assert role.original_url.endswith("/123")
|
||||||
# A later fetch saying less does not erase what was known.
|
# A later fetch saying less does not erase what was known.
|
||||||
store.upsert_role(posting())
|
upsert(store, posting())
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None
|
assert role is not None
|
||||||
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
|
assert (role.place, role.remote, role.salary_min) == ("Berlin", "remote", 80000)
|
||||||
@@ -725,16 +726,16 @@ def test_upsert_replaces_salary_text_including_with_empty(store: Store) -> None:
|
|||||||
every pull, and a pull that states no pay at all clears it rather than
|
every pull, and a pull that states no pay at all clears it rather than
|
||||||
leaving a stale line behind — the opposite of the keep-filled rule
|
leaving a stale line behind — the opposite of the keep-filled rule
|
||||||
`test_upsert_unpacks_raw_and_keeps_filled_columns` checks just above."""
|
`test_upsert_unpacks_raw_and_keeps_filled_columns` checks just above."""
|
||||||
store.upsert_role(posting(salary_text="EUR 70,000 to 90,000 per year"))
|
upsert(store, posting(salary_text="EUR 70,000 to 90,000 per year"))
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None and role.salary_text == "EUR 70,000 to 90,000 per year"
|
assert role is not None and role.salary_text == "EUR 70,000 to 90,000 per year"
|
||||||
|
|
||||||
store.upsert_role(posting(salary_text="EUR 75,000 to 95,000 per year"))
|
upsert(store, posting(salary_text="EUR 75,000 to 95,000 per year"))
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None and role.salary_text == "EUR 75,000 to 95,000 per year"
|
assert role is not None and role.salary_text == "EUR 75,000 to 95,000 per year"
|
||||||
|
|
||||||
# The board stops stating pay: the stored line is cleared, not kept.
|
# The board stops stating pay: the stored line is cleared, not kept.
|
||||||
store.upsert_role(posting())
|
upsert(store, posting())
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None and role.salary_text == ""
|
assert role is not None and role.salary_text == ""
|
||||||
|
|
||||||
@@ -743,10 +744,10 @@ def test_salary_state_is_derived_from_the_columns_and_the_text(store: Store) ->
|
|||||||
"""Not a stored column (docs/decisions.md): 'normalised' needs only
|
"""Not a stored column (docs/decisions.md): 'normalised' needs only
|
||||||
`salary_min` or `salary_max`, whatever `salary_text` says beside it;
|
`salary_min` or `salary_max`, whatever `salary_text` says beside it;
|
||||||
'stated' is `salary_text` alone; neither is ''."""
|
'stated' is `salary_text` alone; neither is ''."""
|
||||||
store.upsert_role(posting("1", salary_text="EUR 70,000 to 90,000 per year",
|
upsert(store, posting("1", salary_text="EUR 70,000 to 90,000 per year",
|
||||||
salary_min=70000, salary_max=90000, currency="EUR"))
|
salary_min=70000, salary_max=90000, currency="EUR"))
|
||||||
store.upsert_role(posting("2", salary_text="USD 45 to 60 per hour"))
|
upsert(store, posting("2", salary_text="USD 45 to 60 per hour"))
|
||||||
store.upsert_role(posting("3"))
|
upsert(store, posting("3"))
|
||||||
by_id = {r.source_id: r for r in store.roles()}
|
by_id = {r.source_id: r for r in store.roles()}
|
||||||
assert by_id["1"].salary_state == "normalised"
|
assert by_id["1"].salary_state == "normalised"
|
||||||
assert by_id["2"].salary_state == "stated"
|
assert by_id["2"].salary_state == "stated"
|
||||||
@@ -760,27 +761,27 @@ def test_upsert_sets_posted_at_from_a_trusted_host_and_keeps_the_earliest(
|
|||||||
that finally reports the true, earlier date must still win — an edit on
|
that finally reports the true, earlier date must still win — an edit on
|
||||||
the board only ever moves published_at later, never earlier, so the
|
the board only ever moves published_at later, never earlier, so the
|
||||||
earliest posted_at ever computed is the one to keep."""
|
earliest posted_at ever computed is the one to keep."""
|
||||||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 20,
|
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 20,
|
||||||
tzinfo=UTC)))
|
tzinfo=UTC)))
|
||||||
first = store.role(posting().uid)
|
first = store.role(posting().uid)
|
||||||
assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00"
|
assert first is not None and first.posted_at == "2026-09-20T00:00:00+00:00"
|
||||||
|
|
||||||
clock.tick()
|
clock.tick()
|
||||||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 1, 1,
|
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 1, 1,
|
||||||
tzinfo=UTC)))
|
tzinfo=UTC)))
|
||||||
earlier = store.role(posting().uid)
|
earlier = store.role(posting().uid)
|
||||||
assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00"
|
assert earlier is not None and earlier.posted_at == "2026-01-01T00:00:00+00:00"
|
||||||
|
|
||||||
clock.tick()
|
clock.tick()
|
||||||
# A later, LATER published_at (an edit) must not push posted_at forward.
|
# A later, LATER published_at (an edit) must not push posted_at forward.
|
||||||
store.upsert_role(posting(board="greenhouse", published_at=datetime(2026, 9, 25,
|
upsert(store, posting(board="greenhouse", published_at=datetime(2026, 9, 25,
|
||||||
tzinfo=UTC)))
|
tzinfo=UTC)))
|
||||||
still_earlier = store.role(posting().uid)
|
still_earlier = store.role(posting().uid)
|
||||||
assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00"
|
assert still_earlier is not None and still_earlier.posted_at == "2026-01-01T00:00:00+00:00"
|
||||||
|
|
||||||
|
|
||||||
def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None:
|
def test_upsert_without_a_trusted_host_keeps_posted_at_at_first_seen(store: Store) -> None:
|
||||||
store.upsert_role(posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
|
upsert(store, posting(published_at=datetime(2020, 1, 1, tzinfo=UTC)))
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None
|
assert role is not None
|
||||||
assert role.posted_at == role.first_seen
|
assert role.posted_at == role.first_seen
|
||||||
@@ -791,7 +792,7 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No
|
|||||||
only set it before extract has run, and once extract has answered — even
|
only set it before extract has run, and once extract has answered — even
|
||||||
with a different function than the pull's guess — no later pull may put
|
with a different function than the pull's guess — no later pull may put
|
||||||
the guess back. The no-flip property across pull, extract, pull."""
|
the guess back. The no-flip property across pull, extract, pull."""
|
||||||
store.upsert_role(posting(function="engineering"))
|
upsert(store, posting(function="engineering"))
|
||||||
uid = posting().uid
|
uid = posting().uid
|
||||||
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
|
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
|
||||||
|
|
||||||
@@ -801,28 +802,28 @@ def test_function_is_the_classifiers_guess_not_a_board_field(store: Store) -> No
|
|||||||
# A later pull with the classifier's old (now wrong) guess must not flip
|
# A later pull with the classifier's old (now wrong) guess must not flip
|
||||||
# it back — the "board wins" COALESCE that every other raw field gets
|
# it back — the "board wins" COALESCE that every other raw field gets
|
||||||
# would otherwise put "engineering" straight back on every pull.
|
# would otherwise put "engineering" straight back on every pull.
|
||||||
store.upsert_role(posting(function="engineering"))
|
upsert(store, posting(function="engineering"))
|
||||||
assert store.role(uid).function == "sales" # type: ignore[union-attr]
|
assert store.role(uid).function == "sales" # type: ignore[union-attr]
|
||||||
|
|
||||||
|
|
||||||
def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None:
|
def test_function_is_frozen_once_extracted_even_with_no_answer(store: Store) -> None:
|
||||||
store.upsert_role(posting(function="engineering"))
|
upsert(store, posting(function="engineering"))
|
||||||
uid = posting().uid
|
uid = posting().uid
|
||||||
store.set_extracted(uid, {}) # extract answered this role, but not function
|
store.set_extracted(uid, {}) # extract answered this role, but not function
|
||||||
assert store.role(uid).extracted_at is not None # type: ignore[union-attr]
|
assert store.role(uid).extracted_at is not None # type: ignore[union-attr]
|
||||||
|
|
||||||
# extracted_at alone freezes it, whatever a later pull's guess says.
|
# extracted_at alone freezes it, whatever a later pull's guess says.
|
||||||
store.upsert_role(posting(function="sales"))
|
upsert(store, posting(function="sales"))
|
||||||
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
|
assert store.role(uid).function == "engineering" # type: ignore[union-attr]
|
||||||
|
|
||||||
|
|
||||||
def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None:
|
def test_function_is_set_from_raw_before_extract_like_any_other_field(store: Store) -> None:
|
||||||
store.upsert_role(posting(function="engineering"))
|
upsert(store, posting(function="engineering"))
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None and role.function == "engineering"
|
assert role is not None and role.function == "engineering"
|
||||||
# Before extract, a fresh guess still overwrites the old one, the same
|
# Before extract, a fresh guess still overwrites the old one, the same
|
||||||
# "board wins" behaviour as place, country and the rest.
|
# "board wins" behaviour as place, country and the rest.
|
||||||
store.upsert_role(posting(function="sales"))
|
upsert(store, posting(function="sales"))
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None and role.function == "sales"
|
assert role is not None and role.function == "sales"
|
||||||
|
|
||||||
@@ -834,33 +835,33 @@ def _folded(store: Store, uid: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def test_place_fold_is_written_with_place(store: Store) -> None:
|
def test_place_fold_is_written_with_place(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", place="København K"))
|
upsert(store, posting("1", place="København K"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
assert _folded(store, uid) == "kobenhavn k"
|
assert _folded(store, uid) == "kobenhavn k"
|
||||||
store.upsert_role(posting("1")) # says less: both kept
|
upsert(store, posting("1")) # says less: both kept
|
||||||
assert (store.role(uid).place, _folded(store, uid)) == ( # type: ignore[union-attr]
|
assert (store.role(uid).place, _folded(store, uid)) == ( # type: ignore[union-attr]
|
||||||
"København K", "kobenhavn k")
|
"København K", "kobenhavn k")
|
||||||
store.upsert_role(posting("1", place="Kgs. Lyngby"))
|
upsert(store, posting("1", place="Kgs. Lyngby"))
|
||||||
assert _folded(store, uid) == "kgs lyngby"
|
assert _folded(store, uid) == "kgs lyngby"
|
||||||
|
|
||||||
store.upsert_role(posting("2"))
|
upsert(store, posting("2"))
|
||||||
store.set_extracted(posting("2").uid, {"place": "Malmö, Sweden"})
|
store.set_extracted(posting("2").uid, {"place": "Malmö, Sweden"})
|
||||||
assert _folded(store, posting("2").uid) == "malmo sweden"
|
assert _folded(store, posting("2").uid) == "malmo sweden"
|
||||||
|
|
||||||
|
|
||||||
def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> None:
|
def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> None:
|
||||||
store.upsert_role(posting("1")) # "Senior Backend Engineer"
|
upsert(store, posting("1")) # "Senior Backend Engineer"
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
assert store.role(uid).seniority == "senior" # type: ignore[union-attr]
|
assert store.role(uid).seniority == "senior" # type: ignore[union-attr]
|
||||||
retitled = posting("1")
|
retitled = posting("1")
|
||||||
retitled.title = "Backend Engineer"
|
retitled.title = "Backend Engineer"
|
||||||
store.upsert_role(retitled) # the title no longer says
|
upsert(store, retitled) # the title no longer says
|
||||||
assert store.role(uid).seniority == "" # type: ignore[union-attr]
|
assert store.role(uid).seniority == "" # type: ignore[union-attr]
|
||||||
|
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
store.set_extracted(uid, {"seniority": "lead"}) # extract overwrites the title's
|
store.set_extracted(uid, {"seniority": "lead"}) # extract overwrites the title's
|
||||||
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
||||||
store.upsert_role(posting("1")) # and a pull does not put it back
|
upsert(store, posting("1")) # and a pull does not put it back
|
||||||
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
||||||
store.set_extracted(uid, {})
|
store.set_extracted(uid, {})
|
||||||
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
assert store.role(uid).seniority == "lead" # type: ignore[union-attr]
|
||||||
@@ -868,14 +869,14 @@ def test_seniority_is_the_titles_until_extract_reads_the_role(store: Store) -> N
|
|||||||
|
|
||||||
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
|
def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
|
||||||
store: Store) -> None:
|
store: Store) -> None:
|
||||||
store.upsert_role(posting("1", seniority="junior"))
|
upsert(store, posting("1", seniority="junior"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
role = store.role(uid)
|
role = store.role(uid)
|
||||||
assert role is not None
|
assert role is not None
|
||||||
assert (role.seniority, role.seniority_stated, role.seniority_title,
|
assert (role.seniority, role.seniority_stated, role.seniority_title,
|
||||||
role.seniority_model) == ("junior", "junior", "senior", "")
|
role.seniority_model) == ("junior", "junior", "senior", "")
|
||||||
store.set_extracted(uid, {"seniority": "staff"})
|
store.set_extracted(uid, {"seniority": "staff"})
|
||||||
store.upsert_role(posting("1", seniority="junior"))
|
upsert(store, posting("1", seniority="junior"))
|
||||||
role = store.role(uid)
|
role = store.role(uid)
|
||||||
assert role is not None
|
assert role is not None
|
||||||
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
|
assert (role.seniority, role.seniority_stated, role.seniority_model) == \
|
||||||
@@ -884,18 +885,18 @@ def test_a_level_the_board_states_wins_over_the_title_and_extract_over_both(
|
|||||||
|
|
||||||
def test_company_falls_back_to_the_subscription(store: Store) -> None:
|
def test_company_falls_back_to_the_subscription(store: Store) -> None:
|
||||||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
|
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="Greenhouse")
|
||||||
store.upsert_role(posting(), sub)
|
upsert(store, posting(), sub)
|
||||||
role = store.role(posting().uid)
|
role = store.role(posting().uid)
|
||||||
assert role is not None
|
assert role is not None
|
||||||
assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id)
|
assert (role.company, role.company_key, role.subscription_id) == ("Acme", "acme", sub.id)
|
||||||
|
|
||||||
|
|
||||||
def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
|
def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
|
||||||
store.upsert_role(posting("1", company="Acme", place="Berlin"))
|
upsert(store, posting("1", company="Acme", place="Berlin"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
store.upsert_role(posting("2", company="Acme", place="Munich"))
|
upsert(store, posting("2", company="Acme", place="Munich"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
store.upsert_role(posting("3", company="Other", place="Berlin"))
|
upsert(store, posting("3", company="Other", place="Berlin"))
|
||||||
assert [r.source_id for r in store.roles()] == ["3", "2", "1"]
|
assert [r.source_id for r in store.roles()] == ["3", "2", "1"]
|
||||||
assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"]
|
assert [r.source_id for r in store.roles(company="ACME")] == ["2", "1"]
|
||||||
assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"]
|
assert [r.source_id for r in store.roles(place="berlin")] == ["3", "1"]
|
||||||
@@ -905,9 +906,9 @@ def test_roles_newest_first_and_filtered(store: Store, clock: Clock) -> None:
|
|||||||
def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None:
|
def test_roles_newest_first_is_by_posted_at_not_first_seen(store: Store, clock: Clock) -> None:
|
||||||
"""A role from a trusted board found today but posted months ago must not
|
"""A role from a trusted board found today but posted months ago must not
|
||||||
outrank one the app has genuinely known about for longer."""
|
outrank one the app has genuinely known about for longer."""
|
||||||
store.upsert_role(posting("recent")) # untrusted: posted_at=first_seen
|
upsert(store, posting("recent")) # untrusted: posted_at=first_seen
|
||||||
clock.tick()
|
clock.tick()
|
||||||
store.upsert_role(posting("old", board="greenhouse",
|
upsert(store, posting("old", board="greenhouse",
|
||||||
published_at=datetime(2020, 1, 1, tzinfo=UTC)))
|
published_at=datetime(2020, 1, 1, tzinfo=UTC)))
|
||||||
assert [r.source_id for r in store.roles()] == ["recent", "old"]
|
assert [r.source_id for r in store.roles()] == ["recent", "old"]
|
||||||
assert [r.source_id for r in store.roles(days=30)] == ["recent"]
|
assert [r.source_id for r in store.roles(days=30)] == ["recent"]
|
||||||
@@ -933,7 +934,7 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index(
|
|||||||
can instead pick `idx_roles_closed` and sort every open role, which on
|
can instead pick `idx_roles_closed` and sort every open role, which on
|
||||||
the live database (15,558 of 15,815 roles open) measured over 30x slower
|
the live database (15,558 of 15,815 roles open) measured over 30x slower
|
||||||
than the plan this asserts (see docs/decisions.md)."""
|
than the plan this asserts (see docs/decisions.md)."""
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
sql, _ = store._roles_from(
|
sql, _ = store._roles_from(
|
||||||
company=None, place=None, open_only=True, q=None, function=None, remote=None,
|
company=None, place=None, open_only=True, q=None, function=None, remote=None,
|
||||||
days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any",
|
days=None, hide_muted=False, include_read=True, tags=(), tag_mode="any",
|
||||||
@@ -955,10 +956,10 @@ def test_the_newest_feed_query_uses_the_partial_posted_at_index(
|
|||||||
|
|
||||||
|
|
||||||
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
|
def test_roles_filters_q_function_remote_and_days(store: Store, clock: Clock) -> None:
|
||||||
store.upsert_role(posting("1", company="Acme", place="Berlin",
|
upsert(store, posting("1", company="Acme", place="Berlin",
|
||||||
function="engineering", remote=True))
|
function="engineering", remote=True))
|
||||||
clock.tick(hours=48)
|
clock.tick(hours=48)
|
||||||
store.upsert_role(posting("2", company="Acme", place="Munich",
|
upsert(store, posting("2", company="Acme", place="Munich",
|
||||||
function="sales"))
|
function="sales"))
|
||||||
assert [r.source_id for r in store.roles(q="senior engineer")] == ["2", "1"]
|
assert [r.source_id for r in store.roles(q="senior engineer")] == ["2", "1"]
|
||||||
assert [r.source_id for r in store.roles(q="backend nonsense")] == []
|
assert [r.source_id for r in store.roles(q="backend nonsense")] == []
|
||||||
@@ -977,12 +978,12 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No
|
|||||||
"""`pay` filters in SQL, the same as the other filters `_roles_from`
|
"""`pay` filters in SQL, the same as the other filters `_roles_from`
|
||||||
builds, so it narrows what a `limit` is taken from rather than the page
|
builds, so it narrows what a `limit` is taken from rather than the page
|
||||||
already built (docs/decisions.md, "filtering before LIMIT")."""
|
already built (docs/decisions.md, "filtering before LIMIT")."""
|
||||||
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
|
upsert(store, posting("normalised", salary_min=70000, salary_max=90000,
|
||||||
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
|
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
|
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
store.upsert_role(posting("none"))
|
upsert(store, posting("none"))
|
||||||
assert {r.source_id for r in store.roles(pay="stated")} == {"normalised", "stated"}
|
assert {r.source_id for r in store.roles(pay="stated")} == {"normalised", "stated"}
|
||||||
assert [r.source_id for r in store.roles(pay="normalised")] == ["normalised"]
|
assert [r.source_id for r in store.roles(pay="normalised")] == ["normalised"]
|
||||||
assert [r.source_id for r in store.roles(pay="stated_only")] == ["stated"]
|
assert [r.source_id for r in store.roles(pay="stated_only")] == ["stated"]
|
||||||
@@ -993,9 +994,9 @@ def test_roles_filters_by_pay_before_the_limit(store: Store, clock: Clock) -> No
|
|||||||
|
|
||||||
|
|
||||||
def test_roles_filter_by_company_keys(store: Store) -> None:
|
def test_roles_filter_by_company_keys(store: Store) -> None:
|
||||||
store.upsert_role(posting("1", company="Acme"))
|
upsert(store, posting("1", company="Acme"))
|
||||||
store.upsert_role(posting("2", company="Acme Labs"))
|
upsert(store, posting("2", company="Acme Labs"))
|
||||||
store.upsert_role(posting("3", company="Other"))
|
upsert(store, posting("3", company="Other"))
|
||||||
assert {r.source_id for r in store.roles(company_keys=["acme"])} == {"1"}
|
assert {r.source_id for r in store.roles(company_keys=["acme"])} == {"1"}
|
||||||
assert {r.source_id for r in store.roles(company_keys=["acme", "other"])} == {"1", "3"}
|
assert {r.source_id for r in store.roles(company_keys=["acme", "other"])} == {"1", "3"}
|
||||||
assert store.count_roles(company_keys=["acme-labs"]) == 1
|
assert store.count_roles(company_keys=["acme-labs"]) == 1
|
||||||
@@ -1008,7 +1009,7 @@ def test_roles_filter_by_place_tokens_on_whole_words(store: Store) -> None:
|
|||||||
("offices", "Copenhagen; London; Warsaw", ""), ("aarhus", "Aarhus C", "DK"),
|
("offices", "Copenhagen; London; Warsaw", ""), ("aarhus", "Aarhus C", "DK"),
|
||||||
("malmo", "Malmö", ""), ("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
|
("malmo", "Malmö", ""), ("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
|
||||||
("berlin", "Berlin", "")):
|
("berlin", "Berlin", "")):
|
||||||
store.upsert_role(posting(sid, place=place, country=country))
|
upsert(store, posting(sid, place=place, country=country))
|
||||||
|
|
||||||
def found(*tokens: str) -> set[str]:
|
def found(*tokens: str) -> set[str]:
|
||||||
return {r.source_id for r in store.roles(places=tokens)}
|
return {r.source_id for r in store.roles(places=tokens)}
|
||||||
@@ -1030,7 +1031,7 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store
|
|||||||
("lead", "Tech Lead"), ("none", "Engineer")):
|
("lead", "Tech Lead"), ("none", "Engineer")):
|
||||||
item = posting(sid)
|
item = posting(sid)
|
||||||
item.title = title
|
item.title = title
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
|
|
||||||
def found(*levels: str, unknown: bool = True) -> set[str]:
|
def found(*levels: str, unknown: bool = True) -> set[str]:
|
||||||
return {r.source_id for r in store.roles(seniority=levels, seniority_unknown=unknown)}
|
return {r.source_id for r in store.roles(seniority=levels, seniority_unknown=unknown)}
|
||||||
@@ -1046,9 +1047,9 @@ def test_roles_filter_by_seniority_and_keep_the_unknown_unless_told(store: Store
|
|||||||
|
|
||||||
|
|
||||||
def test_roles_pay_unknown_lets_the_roles_with_no_pay_through(store: Store) -> None:
|
def test_roles_pay_unknown_lets_the_roles_with_no_pay_through(store: Store) -> None:
|
||||||
store.upsert_role(posting("normalised", salary_min=70000, currency="EUR"))
|
upsert(store, posting("normalised", salary_min=70000, currency="EUR"))
|
||||||
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
|
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
|
||||||
store.upsert_role(posting("none"))
|
upsert(store, posting("none"))
|
||||||
assert {r.source_id for r in store.roles(pay="normalised", pay_unknown=True)} == \
|
assert {r.source_id for r in store.roles(pay="normalised", pay_unknown=True)} == \
|
||||||
{"normalised", "none"}
|
{"normalised", "none"}
|
||||||
assert {r.source_id for r in store.roles(pay="stated", pay_unknown=True)} == \
|
assert {r.source_id for r in store.roles(pay="stated", pay_unknown=True)} == \
|
||||||
@@ -1066,7 +1067,7 @@ def test_role_facets_count_each_facet_with_its_own_filter_left_out(store: Store)
|
|||||||
("e", "Senior Engineer", {"company": "Other"})):
|
("e", "Senior Engineer", {"company": "Other"})):
|
||||||
item = posting(sid, **raw)
|
item = posting(sid, **raw)
|
||||||
item.title = title
|
item.title = title
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
|
|
||||||
facets = store.role_facets(seniority=["junior"], seniority_unknown=False,
|
facets = store.role_facets(seniority=["junior"], seniority_unknown=False,
|
||||||
pay="normalised")
|
pay="normalised")
|
||||||
@@ -1087,10 +1088,10 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store)
|
|||||||
# One key, three spellings: the company row's name first, then the roles'
|
# One key, three spellings: the company row's name first, then the roles'
|
||||||
# by how many carry each.
|
# by how many carry each.
|
||||||
store.add_company("Ørsted Group", key="ørsted")
|
store.add_company("Ørsted Group", key="ørsted")
|
||||||
store.upsert_role(posting("1", company="Ørsted A/S", place="Gentofte"))
|
upsert(store, posting("1", company="Ørsted A/S", place="Gentofte"))
|
||||||
store.upsert_role(posting("2", company="Ørsted", place="Gentofte"))
|
upsert(store, posting("2", company="Ørsted", place="Gentofte"))
|
||||||
store.upsert_role(posting("3", company="Ørsted", place="Malmö"))
|
upsert(store, posting("3", company="Ørsted", place="Malmö"))
|
||||||
store.upsert_role(posting("4", company="Hidden", place="Malmö"))
|
upsert(store, posting("4", company="Hidden", place="Malmö"))
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
store.set_affinity(agent.id, "hidden", -1)
|
store.set_affinity(agent.id, "hidden", -1)
|
||||||
store.set_affinity(agent.id, "ørsted", 2)
|
store.set_affinity(agent.id, "ørsted", 2)
|
||||||
@@ -1099,17 +1100,17 @@ def test_open_companies_and_places_are_what_the_rail_suggests_from(store: Store)
|
|||||||
assert sorted(store.open_companies()) == [
|
assert sorted(store.open_companies()) == [
|
||||||
("hidden", ["Hidden"], 1, 0), ("ørsted", spellings, 3, 0)]
|
("hidden", ["Hidden"], 1, 0), ("ørsted", spellings, 3, 0)]
|
||||||
assert store.open_companies(agent.id) == [("ørsted", spellings, 3, 2)]
|
assert store.open_companies(agent.id) == [("ørsted", spellings, 3, 2)]
|
||||||
assert sorted(store.open_places()) == [("Gentofte", "", 2), ("Malmö", "SE", 2)]
|
assert sorted(store.open_places()) == [("Gentofte", "DK", 2), ("Malmö", "SE", 2)]
|
||||||
|
|
||||||
|
|
||||||
def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None:
|
def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock) -> None:
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
store.set_affinity(agent.id, "acme", -1)
|
store.set_affinity(agent.id, "acme", -1)
|
||||||
store.upsert_role(posting("other", company="Other"))
|
upsert(store, posting("other", company="Other"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
for i in range(3):
|
for i in range(3):
|
||||||
store.upsert_role(posting(f"a{i}", company="Acme"))
|
upsert(store, posting(f"a{i}", company="Acme"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
# Acme's roles are the three newest, so a plain limit=1 shows only Acme;
|
# Acme's roles are the three newest, so a plain limit=1 shows only Acme;
|
||||||
# hide_muted has to reach past all three to find "other" rather than
|
# hide_muted has to reach past all three to find "other" rather than
|
||||||
@@ -1128,7 +1129,7 @@ def test_roles_hide_muted_excludes_before_the_limit(store: Store, clock: Clock)
|
|||||||
def test_roles_page_by_offset_and_count_the_same_filter(
|
def test_roles_page_by_offset_and_count_the_same_filter(
|
||||||
store: Store, clock: Clock) -> None:
|
store: Store, clock: Clock) -> None:
|
||||||
for sid in ("1", "2", "3", "4"):
|
for sid in ("1", "2", "3", "4"):
|
||||||
store.upsert_role(posting(sid, company="Acme" if sid != "4" else "Other"))
|
upsert(store, posting(sid, company="Acme" if sid != "4" else "Other"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
assert [r.source_id for r in store.roles(limit=2, offset=1)] == ["3", "2"]
|
assert [r.source_id for r in store.roles(limit=2, offset=1)] == ["3", "2"]
|
||||||
assert [r.source_id for r in store.roles(limit=2, offset=3)] == ["1"]
|
assert [r.source_id for r in store.roles(limit=2, offset=3)] == ["1"]
|
||||||
@@ -1142,7 +1143,7 @@ def test_roles_page_by_offset_and_count_the_same_filter(
|
|||||||
|
|
||||||
def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> None:
|
def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> None:
|
||||||
for sid in ("up", "down", "fine", "none"):
|
for sid in ("up", "down", "fine", "none"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
role = store.set_vote(agent.id, uid["up"], 1)
|
role = store.set_vote(agent.id, uid["up"], 1)
|
||||||
@@ -1179,7 +1180,7 @@ def test_a_vote_is_stamped_cleared_and_filtered(store: Store, clock: Clock) -> N
|
|||||||
|
|
||||||
|
|
||||||
def test_votes_are_per_agent(store: Store) -> None:
|
def test_votes_are_per_agent(store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
one = store.save_agent("One")
|
one = store.save_agent("One")
|
||||||
two = store.save_agent("Two")
|
two = store.save_agent("Two")
|
||||||
@@ -1209,7 +1210,7 @@ def test_votes_are_per_agent(store: Store) -> None:
|
|||||||
def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
|
def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
|
||||||
store: Store, clock: Clock) -> None:
|
store: Store, clock: Clock) -> None:
|
||||||
for sid in ("1", "2", "3"):
|
for sid in ("1", "2", "3"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
@@ -1226,7 +1227,7 @@ def test_roles_with_an_agent_but_not_by_score_are_newest_first_with_votes(
|
|||||||
|
|
||||||
def test_read_state_is_set_cleared_and_a_vote_marks_read(
|
def test_read_state_is_set_cleared_and_a_vote_marks_read(
|
||||||
store: Store, clock: Clock) -> None:
|
store: Store, clock: Clock) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
role = store.role(uid)
|
role = store.role(uid)
|
||||||
assert role is not None and role.read_at is None
|
assert role is not None and role.read_at is None
|
||||||
@@ -1252,8 +1253,8 @@ def test_read_state_is_set_cleared_and_a_vote_marks_read(
|
|||||||
|
|
||||||
|
|
||||||
def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) -> None:
|
def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
store.upsert_role(posting("2"))
|
upsert(store, posting("2"))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
store.set_read(uid["1"])
|
store.set_read(uid["1"])
|
||||||
|
|
||||||
@@ -1263,7 +1264,7 @@ def test_roles_include_read_excludes_a_read_role_before_the_limit(store: Store)
|
|||||||
|
|
||||||
|
|
||||||
def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None:
|
def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
long_tag = "x" * 50
|
long_tag = "x" * 50
|
||||||
|
|
||||||
@@ -1283,9 +1284,9 @@ def test_set_tags_normalises_dedupes_and_caps(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_roles_filters_by_tags_any_and_all(store: Store) -> None:
|
def test_roles_filters_by_tags_any_and_all(store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
store.upsert_role(posting("2"))
|
upsert(store, posting("2"))
|
||||||
store.upsert_role(posting("3"))
|
upsert(store, posting("3"))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
store.set_tags(uid["1"], ["python", "backend"])
|
store.set_tags(uid["1"], ["python", "backend"])
|
||||||
store.set_tags(uid["2"], ["python", "frontend"])
|
store.set_tags(uid["2"], ["python", "frontend"])
|
||||||
@@ -1305,9 +1306,9 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) ->
|
|||||||
store.add_company("Hidden", key="hidden")
|
store.add_company("Hidden", key="hidden")
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
store.set_affinity(agent.id, "hidden", -1)
|
store.set_affinity(agent.id, "hidden", -1)
|
||||||
store.upsert_role(posting("1", company="Acme"))
|
upsert(store, posting("1", company="Acme"))
|
||||||
store.upsert_role(posting("2", company="Acme"))
|
upsert(store, posting("2", company="Acme"))
|
||||||
store.upsert_role(posting("3", company="Hidden"))
|
upsert(store, posting("3", company="Hidden"))
|
||||||
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
|
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
|
||||||
store.set_tags(uid["1"], ["python", "backend"])
|
store.set_tags(uid["1"], ["python", "backend"])
|
||||||
store.set_tags(uid["2"], ["python"])
|
store.set_tags(uid["2"], ["python"])
|
||||||
@@ -1320,7 +1321,7 @@ def test_tag_counts_orders_by_count_then_tag_and_respects_muted(store: Store) ->
|
|||||||
|
|
||||||
|
|
||||||
def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Store) -> None:
|
def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Store) -> None:
|
||||||
store.upsert_role(posting("a"))
|
upsert(store, posting("a"))
|
||||||
uid = store.roles()[0].uid
|
uid = store.roles()[0].uid
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
store.delete_agent(agent.id)
|
store.delete_agent(agent.id)
|
||||||
@@ -1330,7 +1331,7 @@ def test_scores_for_an_agent_deleted_mid_run_are_dropped_not_an_error(store: Sto
|
|||||||
|
|
||||||
def test_a_vote_moves_the_served_score_and_the_order(store: Store) -> None:
|
def test_a_vote_moves_the_served_score_and_the_order(store: Store) -> None:
|
||||||
for sid in ("a", "b", "c"):
|
for sid in ("a", "b", "c"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
other = store.save_agent("B")
|
other = store.save_agent("B")
|
||||||
@@ -1354,13 +1355,13 @@ def test_mark_closed_closes_what_the_board_no_longer_lists(store: Store) -> None
|
|||||||
sub = store.add_subscription("ats", "greenhouse:acme")
|
sub = store.add_subscription("ats", "greenhouse:acme")
|
||||||
other = store.add_subscription("ats", "greenhouse:other")
|
other = store.add_subscription("ats", "greenhouse:other")
|
||||||
for source_id in ("1", "2", "3"):
|
for source_id in ("1", "2", "3"):
|
||||||
store.upsert_role(posting(source_id), sub)
|
upsert(store, posting(source_id), sub)
|
||||||
store.upsert_role(posting("9"), other)
|
upsert(store, posting("9"), other)
|
||||||
assert store.mark_closed("ats", ["1", "3"], sub.id) == 1
|
assert store.mark_closed("ats", ["1", "3"], sub.id) == 1
|
||||||
assert {r.source_id for r in store.roles()} == {"1", "3", "9"}
|
assert {r.source_id for r in store.roles()} == {"1", "3", "9"}
|
||||||
assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"}
|
assert {r.source_id for r in store.roles(open_only=False)} == {"1", "2", "3", "9"}
|
||||||
# Listed again: open again.
|
# Listed again: open again.
|
||||||
store.upsert_role(posting("2"), sub)
|
upsert(store, posting("2"), sub)
|
||||||
assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"}
|
assert {r.source_id for r in store.roles()} == {"1", "2", "3", "9"}
|
||||||
|
|
||||||
|
|
||||||
@@ -1368,7 +1369,7 @@ def test_an_item_without_a_source_is_refused(store: Store) -> None:
|
|||||||
item = posting()
|
item = posting()
|
||||||
item.source = ""
|
item.source = ""
|
||||||
with pytest.raises(ValueError, match="no source"):
|
with pytest.raises(ValueError, match="no source"):
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- companies
|
# --------------------------------------------------------------- companies
|
||||||
@@ -1472,7 +1473,7 @@ def test_the_default_agent_is_the_lowest_enabled_id(store: Store) -> None:
|
|||||||
def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None:
|
def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None:
|
||||||
for sid, place, country in (("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
|
for sid, place, country in (("lund", "Lund", "SE"), ("billund", "Billund", "DK"),
|
||||||
("kbh", "København K", ""), ("none", "", "")):
|
("kbh", "København K", ""), ("none", "", "")):
|
||||||
store.upsert_role(posting(sid, place=place, country=country))
|
upsert(store, posting(sid, place=place, country=country))
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
|
|
||||||
@@ -1490,8 +1491,8 @@ def test_a_place_gate_matches_whole_words_and_takes_tokens(store: Store) -> None
|
|||||||
def test_open_role_counts_and_company_boards(store: Store) -> None:
|
def test_open_role_counts_and_company_boards(store: Store) -> None:
|
||||||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
store.upsert_role(posting("1", company="Acme", place="Berlin"), sub)
|
upsert(store, posting("1", company="Acme", place="Berlin"), sub)
|
||||||
store.upsert_role(posting("2", company="Acme", place="Munich"), sub)
|
upsert(store, posting("2", company="Acme", place="Munich"), sub)
|
||||||
store.mark_closed("ats", ["1"], sub.id)
|
store.mark_closed("ats", ["1"], sub.id)
|
||||||
assert store.open_role_counts() == {"acme": 1}
|
assert store.open_role_counts() == {"acme": 1}
|
||||||
assert store.company_boards() == {"acme": "greenhouse"}
|
assert store.company_boards() == {"acme": "greenhouse"}
|
||||||
@@ -1524,7 +1525,7 @@ def test_a_clash_on_replace_leaves_the_old_board_and_its_roles(store: Store) ->
|
|||||||
store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme")
|
store.set_company_board("acme", "greenhouse:acme", host="greenhouse", name="Acme")
|
||||||
_, beta, _ = store.set_company_board("beta", "lever:beta", host="lever", name="Beta")
|
_, beta, _ = store.set_company_board("beta", "lever:beta", host="lever", name="Beta")
|
||||||
assert beta is not None
|
assert beta is not None
|
||||||
store.upsert_role(posting("1", company="Beta"), beta)
|
upsert(store, posting("1", company="Beta"), beta)
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
store.set_company_board("beta", "greenhouse:acme", host="greenhouse",
|
store.set_company_board("beta", "greenhouse:acme", host="greenhouse",
|
||||||
fields={"ats": "greenhouse"})
|
fields={"ats": "greenhouse"})
|
||||||
@@ -1539,8 +1540,8 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles(
|
|||||||
name="Acme")
|
name="Acme")
|
||||||
assert old is not None
|
assert old is not None
|
||||||
store.update_subscription(old.id, fetch=False, target_id="gh-1")
|
store.update_subscription(old.id, fetch=False, target_id="gh-1")
|
||||||
store.upsert_role(posting("1", company="Acme"), old)
|
upsert(store, posting("1", company="Acme"), old)
|
||||||
store.upsert_role(posting("2", company="Acme"), old)
|
upsert(store, posting("2", company="Acme"), old)
|
||||||
|
|
||||||
company, new, closed = store.set_company_board("acme", "ashby:acme", host="ashby",
|
company, new, closed = store.set_company_board("acme", "ashby:acme", host="ashby",
|
||||||
fields={"ats": "ashby"})
|
fields={"ats": "ashby"})
|
||||||
@@ -1550,7 +1551,7 @@ def test_replacing_a_board_rewrites_it_in_place_and_closes_the_old_roles(
|
|||||||
"ashby:acme", "ashby", "", 1, 1)
|
"ashby:acme", "ashby", "", 1, 1)
|
||||||
assert company.ats == "ashby"
|
assert company.ats == "ashby"
|
||||||
# The same board again closes nothing.
|
# The same board again closes nothing.
|
||||||
store.upsert_role(posting("3", company="Acme"), new)
|
upsert(store, posting("3", company="Acme"), new)
|
||||||
assert store.set_company_board("acme", "ashby:acme", host="ashby")[2] == 0
|
assert store.set_company_board("acme", "ashby:acme", host="ashby")[2] == 0
|
||||||
assert [r.source_id for r in store.roles()] == ["3"]
|
assert [r.source_id for r in store.roles()] == ["3"]
|
||||||
|
|
||||||
@@ -1560,7 +1561,7 @@ def test_detaching_a_board_hides_it_closes_its_roles_and_keeps_the_row(
|
|||||||
_, sub, _ = store.set_company_board("acme", "greenhouse:acme", host="greenhouse",
|
_, sub, _ = store.set_company_board("acme", "greenhouse:acme", host="greenhouse",
|
||||||
name="Acme")
|
name="Acme")
|
||||||
assert sub is not None
|
assert sub is not None
|
||||||
store.upsert_role(posting("1", company="Acme"), sub)
|
upsert(store, posting("1", company="Acme"), sub)
|
||||||
company, detached, closed = store.set_company_board("acme", "")
|
company, detached, closed = store.set_company_board("acme", "")
|
||||||
assert closed == 1 and company.key == "acme"
|
assert closed == 1 and company.key == "acme"
|
||||||
assert detached is not None
|
assert detached is not None
|
||||||
@@ -1627,7 +1628,7 @@ def test_events_are_append_only(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_an_application_may_point_at_a_role(store: Store) -> None:
|
def test_an_application_may_point_at_a_role(store: Store) -> None:
|
||||||
store.upsert_role(posting())
|
upsert(store, posting())
|
||||||
saved = store.save_application(Application(role_uid=posting().uid, status="applied"))
|
saved = store.save_application(Application(role_uid=posting().uid, status="applied"))
|
||||||
assert saved.role_uid == posting().uid
|
assert saved.role_uid == posting().uid
|
||||||
assert [e.status for e in store.application_events(saved.id)] == ["applied"]
|
assert [e.status for e in store.application_events(saved.id)] == ["applied"]
|
||||||
@@ -1636,7 +1637,7 @@ def test_an_application_may_point_at_a_role(store: Store) -> None:
|
|||||||
# -------------------------------------------------- summary and posting text
|
# -------------------------------------------------- summary and posting text
|
||||||
|
|
||||||
def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
|
def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
|
||||||
store.upsert_role(posting())
|
upsert(store, posting())
|
||||||
role = store.set_article_text(posting().uid, "The whole posting.")
|
role = store.set_article_text(posting().uid, "The whole posting.")
|
||||||
assert role is not None
|
assert role is not None
|
||||||
assert role.article_text == "The whole posting." and role.article_at == store.now()
|
assert role.article_text == "The whole posting." and role.article_at == store.now()
|
||||||
@@ -1644,7 +1645,7 @@ def test_the_posting_text_is_stored_and_stamped(store: Store) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_a_summary_is_written_once(store: Store) -> None:
|
def test_a_summary_is_written_once(store: Store) -> None:
|
||||||
store.upsert_role(posting())
|
upsert(store, posting())
|
||||||
uid = posting().uid
|
uid = posting().uid
|
||||||
assert store.set_summary(uid, "First.") is True
|
assert store.set_summary(uid, "First.") is True
|
||||||
assert store.set_summary(uid, "Second.") is False
|
assert store.set_summary(uid, "Second.") is False
|
||||||
@@ -1655,7 +1656,7 @@ def test_a_summary_is_written_once(store: Store) -> None:
|
|||||||
|
|
||||||
def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer(
|
def test_a_summarys_tags_land_with_it_and_not_on_a_rejected_late_answer(
|
||||||
store: Store) -> None:
|
store: Store) -> None:
|
||||||
store.upsert_role(posting())
|
upsert(store, posting())
|
||||||
uid = posting().uid
|
uid = posting().uid
|
||||||
assert store.set_summary(uid, "First.", ["Python", "backend"]) is True
|
assert store.set_summary(uid, "First.", ["Python", "backend"]) is True
|
||||||
role = store.role(uid)
|
role = store.role(uid)
|
||||||
@@ -1675,7 +1676,7 @@ def test_summarise_waits_on_the_top_of_each_enabled_agent_by_its_own_limits(
|
|||||||
agent that hid it. An agent saved before v8 has `limits` of '{}', which
|
agent that hid it. An agent saved before v8 has `limits` of '{}', which
|
||||||
reads as the defaults."""
|
reads as the defaults."""
|
||||||
for sid in "abcd":
|
for sid in "abcd":
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
a, b, c, d = (posting(s).uid for s in "abcd")
|
a, b, c, d = (posting(s).uid for s in "abcd")
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
|
conn.execute("UPDATE roles SET closed_at = ? WHERE uid = ?", (store.now(), d))
|
||||||
|
|||||||
+55
-54
@@ -23,6 +23,7 @@ from app.web.api import _score_after_change
|
|||||||
from app.web.pages import SECTIONS
|
from app.web.pages import SECTIONS
|
||||||
from auth import AuthConfig
|
from auth import AuthConfig
|
||||||
from desk.models import Item
|
from desk.models import Item
|
||||||
|
from prepped import upsert
|
||||||
|
|
||||||
|
|
||||||
class Clock:
|
class Clock:
|
||||||
@@ -75,9 +76,9 @@ def client(app: FastAPI) -> Iterator[TestClient]:
|
|||||||
|
|
||||||
def test_list_roles_filters_by_query_place_and_function(
|
def test_list_roles_filters_by_query_place_and_function(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering"))
|
upsert(store, posting("1", company="Acme", place="Berlin", function="engineering"))
|
||||||
store.upsert_role(posting("2", company="Acme", place="Munich", function="sales"))
|
upsert(store, posting("2", company="Acme", place="Munich", function="sales"))
|
||||||
store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering"))
|
upsert(store, posting("3", company="Other", place="Berlin", function="engineering"))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
|
|
||||||
res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"})
|
res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"})
|
||||||
@@ -100,7 +101,7 @@ def test_list_roles_filters_by_company_keys_places_and_seniority(
|
|||||||
("berlin", "Other", "Berlin", "Senior Engineer")):
|
("berlin", "Other", "Berlin", "Senior Engineer")):
|
||||||
item = posting(sid, company=company, place=place)
|
item = posting(sid, company=company, place=place)
|
||||||
item.title = title
|
item.title = title
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
|
|
||||||
def shown(**params: Any) -> set[str]:
|
def shown(**params: Any) -> set[str]:
|
||||||
@@ -133,7 +134,7 @@ def test_role_facets_count_each_facet_under_the_other_filters(
|
|||||||
("c", "Engineer", {"salary_text": "EUR 50 per hour"})):
|
("c", "Engineer", {"salary_text": "EUR 50 per hour"})):
|
||||||
item = posting(sid, **raw)
|
item = posting(sid, **raw)
|
||||||
item.title = title
|
item.title = title
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
|
|
||||||
res = client.get("/api/roles/facets",
|
res = client.get("/api/roles/facets",
|
||||||
params={"seniority": ["senior"], "seniority_unknown": "false",
|
params={"seniority": ["senior"], "seniority_unknown": "false",
|
||||||
@@ -152,7 +153,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden(
|
|||||||
for sid, company in (("1", "Ørsted"), ("2", "Ørsted"), ("3", "Data Minds"),
|
for sid, company in (("1", "Ørsted"), ("2", "Ørsted"), ("3", "Data Minds"),
|
||||||
("4", "Big Data Corp"), ("5", "Big Data Corp"), ("6", "Big Data Corp"),
|
("4", "Big Data Corp"), ("5", "Big Data Corp"), ("6", "Big Data Corp"),
|
||||||
("7", "Metadata Ltd"), ("8", "Hidden Data")):
|
("7", "Metadata Ltd"), ("8", "Hidden Data")):
|
||||||
store.upsert_role(posting(sid, company=company))
|
upsert(store, posting(sid, company=company))
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
store.set_affinity(agent.id, "hidden-data", -1)
|
store.set_affinity(agent.id, "hidden-data", -1)
|
||||||
store.set_affinity(agent.id, "ørsted", 2)
|
store.set_affinity(agent.id, "ørsted", 2)
|
||||||
@@ -177,7 +178,7 @@ def test_suggest_companies_folds_the_name_and_leaves_out_the_hidden(
|
|||||||
def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None:
|
def test_suggest_places_and_read_labels_back(client: TestClient, store: Store) -> None:
|
||||||
for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"),
|
for sid, place in (("1", "Malmö"), ("2", "Malmö"), ("3", "Lund"),
|
||||||
("4", "København K"), ("5", "Kolding")):
|
("4", "København K"), ("5", "Kolding")):
|
||||||
store.upsert_role(posting(sid, place=place))
|
upsert(store, posting(sid, place=place))
|
||||||
|
|
||||||
res = client.get("/api/suggest/places", params={"q": "ska"})
|
res = client.get("/api/suggest/places", params={"q": "ska"})
|
||||||
assert res.status_code == 200
|
assert res.status_code == 200
|
||||||
@@ -200,10 +201,10 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store)
|
|||||||
"""`salary_text` and `salary_state` (app/models.py) are on every role the
|
"""`salary_text` and `salary_state` (app/models.py) are on every role the
|
||||||
API serves, and `pay` (`Store.PAY`) filters the feed by it, in SQL before
|
API serves, and `pay` (`Store.PAY`) filters the feed by it, in SQL before
|
||||||
`limit` like the other filters."""
|
`limit` like the other filters."""
|
||||||
store.upsert_role(posting("normalised", salary_min=70000, salary_max=90000,
|
upsert(store, posting("normalised", salary_min=70000, salary_max=90000,
|
||||||
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
|
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
|
||||||
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
|
upsert(store, posting("stated", salary_text="USD 45 to 60 per hour"))
|
||||||
store.upsert_role(posting("none"))
|
upsert(store, posting("none"))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
|
|
||||||
items = {r["uid"]: r for r in client.get("/api/roles").json()["items"]}
|
items = {r["uid"]: r for r in client.get("/api/roles").json()["items"]}
|
||||||
@@ -223,7 +224,7 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store)
|
|||||||
|
|
||||||
def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
|
def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
|
||||||
for i in range(3):
|
for i in range(3):
|
||||||
store.upsert_role(posting(str(i)))
|
upsert(store, posting(str(i)))
|
||||||
assert len(client.get("/api/roles", params={"limit": 1}).json()["items"]) == 1
|
assert len(client.get("/api/roles", params={"limit": 1}).json()["items"]) == 1
|
||||||
assert client.get("/api/roles", params={"limit": 0}).status_code == 422
|
assert client.get("/api/roles", params={"limit": 0}).status_code == 422
|
||||||
assert client.get("/api/roles", params={"limit": 1001}).status_code == 422
|
assert client.get("/api/roles", params={"limit": 1001}).status_code == 422
|
||||||
@@ -231,8 +232,8 @@ def test_list_roles_limit_is_bounded(client: TestClient, store: Store) -> None:
|
|||||||
|
|
||||||
def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Store) -> None:
|
def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Store) -> None:
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
store.upsert_role(posting("1", company="Acme"))
|
upsert(store, posting("1", company="Acme"))
|
||||||
store.upsert_role(posting("2", company="Other"))
|
upsert(store, posting("2", company="Other"))
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
other = store.save_agent("B")
|
other = store.save_agent("B")
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
@@ -265,10 +266,10 @@ def test_muted_is_excluded_before_the_limit_not_after(
|
|||||||
store.clock = clock
|
store.clock = clock
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
store.set_affinity(store.save_agent("A").id, "acme", -1)
|
store.set_affinity(store.save_agent("A").id, "acme", -1)
|
||||||
store.upsert_role(posting("other", company="Other"))
|
upsert(store, posting("other", company="Other"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
for i in range(3):
|
for i in range(3):
|
||||||
store.upsert_role(posting(f"a{i}", company="Acme"))
|
upsert(store, posting(f"a{i}", company="Acme"))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
|
|
||||||
@@ -282,7 +283,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
|
|||||||
clock = Clock()
|
clock = Clock()
|
||||||
store.clock = clock
|
store.clock = clock
|
||||||
for i in range(5):
|
for i in range(5):
|
||||||
store.upsert_role(posting(str(i)))
|
upsert(store, posting(str(i)))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
|
|
||||||
@@ -298,7 +299,7 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
|
|||||||
|
|
||||||
def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) -> None:
|
def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store) -> None:
|
||||||
for sid in ("liked", "passed", "fine", "open"):
|
for sid in ("liked", "passed", "fine", "open"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
|
|
||||||
@@ -340,7 +341,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
|
|||||||
|
|
||||||
|
|
||||||
def test_votes_are_per_agent_over_the_api(client: TestClient, store: Store) -> None:
|
def test_votes_are_per_agent_over_the_api(client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
one = store.save_agent("One")
|
one = store.save_agent("One")
|
||||||
two = store.save_agent("Two")
|
two = store.save_agent("Two")
|
||||||
@@ -364,7 +365,7 @@ def test_a_vote_or_an_affinity_needs_an_agent_to_belong_to(
|
|||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
"""Nothing to vote as: a 422 with a sentence, not a vote that lands
|
"""Nothing to vote as: a 422 with a sentence, not a vote that lands
|
||||||
nowhere. Reading is fine, everything is simply unjudged and neutral."""
|
nowhere. Reading is fine, everything is simply unjudged and neutral."""
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
|
|
||||||
@@ -382,7 +383,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
|
|||||||
clock = Clock()
|
clock = Clock()
|
||||||
store.clock = clock
|
store.clock = clock
|
||||||
for sid in ("1", "2", "3"):
|
for sid in ("1", "2", "3"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
clock.tick()
|
clock.tick()
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
@@ -403,7 +404,7 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
|
|||||||
def test_a_vote_moves_a_role_in_the_ranking_at_once(
|
def test_a_vote_moves_a_role_in_the_ranking_at_once(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
for sid in ("a", "b", "c"):
|
for sid in ("a", "b", "c"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
label = {v: k for k, v in uid.items()}
|
label = {v: k for k, v in uid.items()}
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
@@ -424,7 +425,7 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once(
|
|||||||
|
|
||||||
|
|
||||||
def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: Store) -> None:
|
def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
|
|
||||||
res = client.post(f"/api/roles/{uid}/read")
|
res = client.post(f"/api/roles/{uid}/read")
|
||||||
@@ -440,8 +441,8 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S
|
|||||||
|
|
||||||
|
|
||||||
def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None:
|
def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
store.upsert_role(posting("2"))
|
upsert(store, posting("2"))
|
||||||
ids = {r.source_id: r.uid for r in store.roles()}
|
ids = {r.source_id: r.uid for r in store.roles()}
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
client.post(f"/api/roles/{uid}/read")
|
client.post(f"/api/roles/{uid}/read")
|
||||||
@@ -455,8 +456,8 @@ def test_list_roles_filters_by_include_read(client: TestClient, store: Store) ->
|
|||||||
|
|
||||||
|
|
||||||
def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store) -> None:
|
def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
store.upsert_role(posting("2"))
|
upsert(store, posting("2"))
|
||||||
uid = {r.source_id: r.uid for r in store.roles()}
|
uid = {r.source_id: r.uid for r in store.roles()}
|
||||||
store.set_tags(uid["1"], ["python", "backend"])
|
store.set_tags(uid["1"], ["python", "backend"])
|
||||||
store.set_tags(uid["2"], ["python", "frontend"])
|
store.set_tags(uid["2"], ["python", "frontend"])
|
||||||
@@ -476,8 +477,8 @@ def test_filters_lists_tag_counts_and_respects_show_muted(
|
|||||||
store.add_company("Hidden", key="hidden")
|
store.add_company("Hidden", key="hidden")
|
||||||
agent = store.save_agent("A") # the default agent, whose hidden companies count
|
agent = store.save_agent("A") # the default agent, whose hidden companies count
|
||||||
store.set_affinity(agent.id, "hidden", -1)
|
store.set_affinity(agent.id, "hidden", -1)
|
||||||
store.upsert_role(posting("1", company="Acme"))
|
upsert(store, posting("1", company="Acme"))
|
||||||
store.upsert_role(posting("2", company="Hidden"))
|
upsert(store, posting("2", company="Hidden"))
|
||||||
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
|
uid = {r.source_id: r.uid for r in store.roles(hide_muted=False)}
|
||||||
store.set_tags(uid["1"], ["python"])
|
store.set_tags(uid["1"], ["python"])
|
||||||
store.set_tags(uid["2"], ["python"])
|
store.set_tags(uid["2"], ["python"])
|
||||||
@@ -491,7 +492,7 @@ def test_filters_lists_tag_counts_and_respects_show_muted(
|
|||||||
|
|
||||||
|
|
||||||
def test_one_role_and_its_read_page(client: TestClient, store: Store) -> None:
|
def test_one_role_and_its_read_page(client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1", company="Acme", place="Berlin"))
|
upsert(store, posting("1", company="Acme", place="Berlin"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
|
|
||||||
res = client.get(f"/api/roles/{uid}")
|
res = client.get(f"/api/roles/{uid}")
|
||||||
@@ -524,7 +525,7 @@ def test_companies_lists_board_and_open_role_count(
|
|||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
||||||
store.add_company("Acme", key="acme", ats="greenhouse", country="DK")
|
store.add_company("Acme", key="acme", ats="greenhouse", country="DK")
|
||||||
store.upsert_role(posting("1", company="Acme"), sub)
|
upsert(store, posting("1", company="Acme"), sub)
|
||||||
store.add_company("Other", key="other", domain="other.example") # no board found
|
store.add_company("Other", key="other", domain="other.example") # no board found
|
||||||
store.add_company("Novo", key="novo", ats="taleo", # followed, not readable
|
store.add_company("Novo", key="novo", ats="taleo", # followed, not readable
|
||||||
careers_url="https://novo.example/careers")
|
careers_url="https://novo.example/careers")
|
||||||
@@ -561,7 +562,7 @@ def test_judging_a_company_known_only_from_a_search_creates_its_row(
|
|||||||
for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")):
|
for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")):
|
||||||
item = posting(f"n{i}", company=name)
|
item = posting(f"n{i}", company=name)
|
||||||
item.source = "search"
|
item.source = "search"
|
||||||
store.upsert_role(item, search)
|
upsert(store, item, search)
|
||||||
key = "novo-nordisk"
|
key = "novo-nordisk"
|
||||||
assert store.company(key) is None
|
assert store.company(key) is None
|
||||||
|
|
||||||
@@ -712,7 +713,7 @@ def test_the_watchlist_endpoints_404_for_an_unknown_company(client: TestClient)
|
|||||||
|
|
||||||
def test_save_creates_an_application_with_a_saved_event(
|
def test_save_creates_an_application_with_a_saved_event(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1", company="Acme", place="Berlin"))
|
upsert(store, posting("1", company="Acme", place="Berlin"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
|
|
||||||
res = client.post("/api/applications", json={"role_uid": uid, "status": "saved"})
|
res = client.post("/api/applications", json={"role_uid": uid, "status": "saved"})
|
||||||
@@ -744,7 +745,7 @@ def test_save_an_unknown_role_is_refused(client: TestClient) -> None:
|
|||||||
|
|
||||||
def test_set_status_validates_and_writes_an_event(
|
def test_set_status_validates_and_writes_an_event(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1", company="Acme"))
|
upsert(store, posting("1", company="Acme"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
saved = client.post("/api/applications", json={"role_uid": uid}).json()
|
saved = client.post("/api/applications", json={"role_uid": uid}).json()
|
||||||
|
|
||||||
@@ -761,8 +762,8 @@ def test_set_status_validates_and_writes_an_event(
|
|||||||
|
|
||||||
def test_unsave_drops_a_saved_application_and_refuses_one_that_moved_on(
|
def test_unsave_drops_a_saved_application_and_refuses_one_that_moved_on(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1", company="Acme"))
|
upsert(store, posting("1", company="Acme"))
|
||||||
store.upsert_role(posting("2", company="Acme"))
|
upsert(store, posting("2", company="Acme"))
|
||||||
first = client.post("/api/applications", json={"role_uid": posting("1").uid}).json()
|
first = client.post("/api/applications", json={"role_uid": posting("1").uid}).json()
|
||||||
second = client.post("/api/applications", json={"role_uid": posting("2").uid}).json()
|
second = client.post("/api/applications", json={"role_uid": posting("2").uid}).json()
|
||||||
|
|
||||||
@@ -1069,7 +1070,7 @@ def test_a_duplicate_or_empty_prompt_name_is_a_422_sentence(
|
|||||||
|
|
||||||
def test_the_role_endpoint_serves_the_agents_score_and_its_prompt(
|
def test_the_role_endpoint_serves_the_agents_score_and_its_prompt(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
agent = store.save_agent("A", prompts=[])
|
agent = store.save_agent("A", prompts=[])
|
||||||
store.replace_scores(agent.id, [(uid, 0.8, "", "Forward deployed")])
|
store.replace_scores(agent.id, [(uid, 0.8, "", "Forward deployed")])
|
||||||
@@ -1088,7 +1089,7 @@ def test_the_role_endpoint_serves_the_agents_score_and_its_prompt(
|
|||||||
|
|
||||||
def test_deleting_an_agent_takes_its_votes_and_affinities_with_it(
|
def test_deleting_an_agent_takes_its_votes_and_affinities_with_it(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
store.add_company("Acme", key="acme")
|
store.add_company("Acme", key="acme")
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
agent = client.post("/api/agents", json={"name": "A"}).json()
|
agent = client.post("/api/agents", json={"name": "A"}).json()
|
||||||
@@ -1127,7 +1128,7 @@ def test_an_agents_limits_are_saved_listed_and_checked(client: TestClient) -> No
|
|||||||
def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
|
def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
for sid in ("low", "high", "gated"):
|
for sid in ("low", "high", "gated"):
|
||||||
store.upsert_role(posting(sid))
|
upsert(store, posting(sid))
|
||||||
uids = {r.source_id: r.uid for r in store.roles()}
|
uids = {r.source_id: r.uid for r in store.roles()}
|
||||||
agent = store.save_agent("A")
|
agent = store.save_agent("A")
|
||||||
store.replace_scores(agent.id, [(uids["low"], 0.2, ""), (uids["high"], 0.8, ""),
|
store.replace_scores(agent.id, [(uids["low"], 0.2, ""), (uids["high"], 0.8, ""),
|
||||||
@@ -1152,7 +1153,7 @@ def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
|
|||||||
item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid",
|
item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid",
|
||||||
function="engineering")
|
function="engineering")
|
||||||
item.body = long_description
|
item.body = long_description
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
uid = item.uid
|
uid = item.uid
|
||||||
|
|
||||||
row = client.get("/api/roles").json()["items"][0]
|
row = client.get("/api/roles").json()["items"][0]
|
||||||
@@ -1184,7 +1185,7 @@ def test_list_roles_page_stays_small_even_with_long_postings(
|
|||||||
for n in range(60):
|
for n in range(60):
|
||||||
item = posting(str(n), company="Acme")
|
item = posting(str(n), company="Acme")
|
||||||
item.body = big
|
item.body = big
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?",
|
conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?",
|
||||||
(big, json.dumps({"blob": big}), big))
|
(big, json.dumps({"blob": big}), big))
|
||||||
@@ -1197,7 +1198,7 @@ def test_list_roles_page_stays_small_even_with_long_postings(
|
|||||||
|
|
||||||
|
|
||||||
def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None:
|
def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None:
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
store.set_summary(uid, "A short, hand-written summary.")
|
store.set_summary(uid, "A short, hand-written summary.")
|
||||||
row = client.get("/api/roles").json()["items"][0]
|
row = client.get("/api/roles").json()["items"][0]
|
||||||
@@ -1221,9 +1222,9 @@ def test_sources_lists_searches_and_boards_with_their_countries(
|
|||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||||
board="greenhouse")
|
board="greenhouse")
|
||||||
store.upsert_role(posting("1", company="Acme", place="Copenhagen"), board)
|
upsert(store, posting("1", company="Acme", place="Copenhagen"), board)
|
||||||
store.upsert_role(posting("2", company="Acme", place="Boston, MA"), board)
|
upsert(store, posting("2", company="Acme", place="Boston, MA"), board)
|
||||||
store.upsert_role(posting("3", company="Acme", place="Aarhus"), board)
|
upsert(store, posting("3", company="Acme", place="Aarhus"), board)
|
||||||
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
|
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
|
||||||
params={"q": "backend", "area": 1})
|
params={"q": "backend", "area": 1})
|
||||||
|
|
||||||
@@ -1244,7 +1245,7 @@ def test_switching_a_board_off_closes_its_roles_and_on_queues_a_pull(
|
|||||||
client: TestClient, store: Store, queue: Any) -> None:
|
client: TestClient, store: Store, queue: Any) -> None:
|
||||||
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
|
||||||
board="greenhouse")
|
board="greenhouse")
|
||||||
store.upsert_role(posting("1", company="Acme"), board)
|
upsert(store, posting("1", company="Acme"), board)
|
||||||
|
|
||||||
off = client.post(f"/api/sources/{board.id}/switch", json={"on": False}).json()
|
off = client.post(f"/api/sources/{board.id}/switch", json={"on": False}).json()
|
||||||
assert off["closed"] == 1 and off["source"]["on"] is False
|
assert off["closed"] == 1 and off["source"]["on"] is False
|
||||||
@@ -1345,7 +1346,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
|
|||||||
body = res.json()
|
body = res.json()
|
||||||
assert (body["found"], body["closed"], body["source"]["target"]) == (0, 0, "greenhouse:acme")
|
assert (body["found"], body["closed"], body["source"]["target"]) == (0, 0, "greenhouse:acme")
|
||||||
sub_id = body["source"]["id"]
|
sub_id = body["source"]["id"]
|
||||||
store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id))
|
upsert(store, posting("1", company="Acme"), store.subscription(sub_id))
|
||||||
[row] = client.get("/api/companies").json()
|
[row] = client.get("/api/companies").json()
|
||||||
assert (row["board"], row["subscription_id"], row["ats"], row["ats_slug"]) == (
|
assert (row["board"], row["subscription_id"], row["ats"], row["ats_slug"]) == (
|
||||||
"greenhouse", sub_id, "greenhouse", "acme")
|
"greenhouse", sub_id, "greenhouse", "acme")
|
||||||
@@ -1418,7 +1419,7 @@ def test_a_company_from_a_search_only_is_no_longer_one_once_it_has_a_reason(
|
|||||||
search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex")
|
search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex")
|
||||||
item = posting("1", company="Novo")
|
item = posting("1", company="Novo")
|
||||||
item.source = "search"
|
item.source = "search"
|
||||||
store.upsert_role(item, search)
|
upsert(store, item, search)
|
||||||
client.post("/api/companies/novo/affinity", json={"affinity": 1})
|
client.post("/api/companies/novo/affinity", json={"affinity": 1})
|
||||||
[row] = client.get("/api/companies").json()
|
[row] = client.get("/api/companies").json()
|
||||||
assert (row["from_search"], row["by_hand"]) == (True, False)
|
assert (row["from_search"], row["by_hand"]) == (True, False)
|
||||||
@@ -1514,14 +1515,14 @@ def test_a_company_is_deleted_unless_it_is_read(
|
|||||||
).json()["source"]["id"]
|
).json()["source"]["id"]
|
||||||
res = client.delete("/api/companies/acme")
|
res = client.delete("/api/companies/acme")
|
||||||
assert res.status_code == 409 and "greenhouse:acme" in res.json()["detail"]
|
assert res.status_code == 409 and "greenhouse:acme" in res.json()["detail"]
|
||||||
store.upsert_role(posting("1", company="Acme"), store.subscription(sub_id))
|
upsert(store, posting("1", company="Acme"), store.subscription(sub_id))
|
||||||
client.delete("/api/companies/acme/board")
|
client.delete("/api/companies/acme/board")
|
||||||
assert client.delete("/api/companies/acme").status_code == 204
|
assert client.delete("/api/companies/acme").status_code == 204
|
||||||
|
|
||||||
# Open roles hold a company even where no board is on: they would be
|
# Open roles hold a company even where no board is on: they would be
|
||||||
# orphaned from the list.
|
# orphaned from the list.
|
||||||
store.add_company("Beta", key="beta")
|
store.add_company("Beta", key="beta")
|
||||||
store.upsert_role(posting("2", company="Beta"))
|
upsert(store, posting("2", company="Beta"))
|
||||||
res = client.delete("/api/companies/beta")
|
res = client.delete("/api/companies/beta")
|
||||||
assert res.status_code == 409 and "open roles" in res.json()["detail"]
|
assert res.status_code == 409 and "open roles" in res.json()["detail"]
|
||||||
assert store.company("beta") is not None
|
assert store.company("beta") is not None
|
||||||
@@ -1562,7 +1563,7 @@ def test_the_posting_text_is_fetched_once(
|
|||||||
from desk.tools import scrape
|
from desk.tools import scrape
|
||||||
from desk.tools.article import ArticleContent
|
from desk.tools.article import ArticleContent
|
||||||
|
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
uid = posting("1").uid
|
uid = posting("1").uid
|
||||||
asked: list[str] = []
|
asked: list[str] = []
|
||||||
|
|
||||||
@@ -1608,8 +1609,8 @@ def test_summarise_queues_one_role_and_the_job_writes_it(
|
|||||||
monkeypatch.setitem(JOBS, "summarise_batch",
|
monkeypatch.setitem(JOBS, "summarise_batch",
|
||||||
replace(JOBS["summarise_batch"], isolate=False, timeout=0.0))
|
replace(JOBS["summarise_batch"], isolate=False, timeout=0.0))
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
store.upsert_role(posting("1"))
|
upsert(store, posting("1"))
|
||||||
store.upsert_role(posting("2"))
|
upsert(store, posting("2"))
|
||||||
uid, bare = posting("1").uid, posting("2").uid
|
uid, bare = posting("1").uid, posting("2").uid
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
conn.execute("UPDATE roles SET description = '' WHERE uid = ?", (bare,))
|
conn.execute("UPDATE roles SET description = '' WHERE uid = ?", (bare,))
|
||||||
@@ -1656,7 +1657,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
|
|||||||
board.body = "The whole posting."
|
board.body = "The whole posting."
|
||||||
search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex")
|
search = posting("search", company="Acme", place="Ballerup, Denmark", provider="jobindex")
|
||||||
for item in (board, search, posting("alone", company="Other")):
|
for item in (board, search, posting("alone", company="Other")):
|
||||||
store.upsert_role(item)
|
upsert(store, item)
|
||||||
store.rebuild_dupes()
|
store.rebuild_dupes()
|
||||||
|
|
||||||
page = client.get("/api/roles").json()
|
page = client.get("/api/roles").json()
|
||||||
@@ -1675,7 +1676,7 @@ def test_a_kept_role_lists_its_copies_and_a_copy_names_its_kept_role(
|
|||||||
|
|
||||||
def test_suggest_companies_matches_through_an_alias(client: TestClient, store: Store) -> None:
|
def test_suggest_companies_matches_through_an_alias(client: TestClient, store: Store) -> None:
|
||||||
store.add_company("Ørsted")
|
store.add_company("Ørsted")
|
||||||
store.upsert_role(posting("1", company="Orsted"))
|
upsert(store, posting("1", company="Orsted"))
|
||||||
store.add_alias("ørsted", "Dong Energy")
|
store.add_alias("ørsted", "Dong Energy")
|
||||||
|
|
||||||
for q in ("orsted", "ørsted", "dong"):
|
for q in ("orsted", "ørsted", "dong"):
|
||||||
|
|||||||
Reference in new issue
Block a user