Merge main into wp-review-contract

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:57:46 +02:00
commit fe2a1b0347
36 files changed
+1741 -596

No files matched your search

+1 -1
View File
@@ -6,7 +6,7 @@
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
app searches add|list|remove
app agents list|prompt
app stages [extract|summarise on|off]
app stages [extract|summarise|enrich on|off]
app dupes
app prep [unmapped|unlinked] [--remap]
app db init
+21 -3
View File
@@ -16,8 +16,9 @@ the starter agents; `config.example.yaml` documents every key.
from __future__ import annotations
import os
import sys
from pathlib import Path
from typing import Any, Literal
from typing import Any
import yaml
from pydantic import Field, ValidationError, field_validator, model_validator
@@ -165,12 +166,29 @@ class SearchSeed(Section):
"""A saved search as `config.yaml` lists it: one `search` subscription,
imported once like the agents. `params` are the provider's own, see
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
`where` and `pages` for adzuna."""
`where` and `pages` for adzuna; `q` and `region` for platsbanken. The provider is any reader of
`scope="search"`, so a new one needs no edit here."""
name: str
provider: Literal["jobindex", "adzuna"]
provider: str
params: dict[str, Any] = Field(default_factory=dict)
@field_validator("provider")
@classmethod
def _a_search_provider(cls, provider: str) -> str:
# `plugins` is a top-level folder beside `app` and is not on the path
# when this module is imported first, so it is reached the way
# `app.models` reaches it (`_declared_posted`), and only when a seed
# is validated.
root = str(Path(__file__).resolve().parents[1])
if root not in sys.path:
sys.path.insert(0, root)
from plugins.readers import PROVIDERS
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}, not {provider!r}")
return provider
class Config(Section):
"""The whole file."""
+10 -2
View File
@@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
Under `NULL` there is no queue to take the chain, so this raises rather
than report a pull that never happened; `app job pull` claims the run and
starts the chain on a queue of its own instead of calling this.
starts the chain on a queue of its own instead of calling this. Anything
that raises once the run is claimed, building the chain or handing it
over, releases the claim first, or the next pull would wait
`pull.ABANDONED` for a run that never started.
"""
run = pull.claim_run(store)
if not run:
since = pull.running(store)
context.notice(already_running(since))
return {"skipped": pull.ALREADY_RUNNING, "running": since}
if not context.task(pull_chain(run)):
try:
started = context.task(pull_chain(run))
except Exception:
pull.release_run(store, run)
raise
if not started:
pull.release_run(store, run)
raise RuntimeError("no queue to start the pull cascade on")
return {"started": " > ".join(CASCADES["pull"]), "run": run}
+282 -107
View File
@@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see
`plugins/readers/_runner.py`).
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
person acted on: those with a vote or an application (`_close_unlisted`). A
filtered read, or one of a reader that declares `empty="suspect"`, that lists
under a tenth of what the same read listed last time is recorded by the
runner as a failure, not as a list to close against.
`app.jobs.reach`, or its subscription's `params`) lists only the roles in
reach, so its close keeps open the roles a person acted on: those with a vote
or an application (`_close_unlisted`). When the reach wanted remote roles and
the reader could not ask for them (`ats:remote_unasked:<target>`), the read
says nothing of the remote roles either, so they stay open too and close only
once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
that lists under a tenth of what the same read listed last time is recorded
by the runner as a failure, not as a list to close against (`_runner.read`).
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
a sweep an earlier pull began) closes a role only when two finished sweeps in a
@@ -33,14 +36,21 @@ not swept clears them.
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
the roles matching a word, which says nothing of the roles it did not list,
so it closes nothing; only an unfiltered or a place-filtered complete read
closes.
so it closes by list nothing: a role of such a board closes once the board
has not listed it for `STALE_DAYS`, the rule searches have.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
roles only once no search has seen them for `STALE_DAYS`. A board
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
closes all of its roles, and its next answer reopens what it lists.
roles only once no search has seen them for `STALE_DAYS`. A board or a
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
`search:failing:<target>`) closes all of its roles, and its next answer
reopens what it lists. A target that raises in its batch after its read, a
row that does not store, is recorded as failing the same way and the batch
goes on with the next one (`run_batch`).
The settle says what needs a look in one notice (`settled`): the targets
that did not answer or were read incomplete, and the boards that closed more
than `CLOSE_NOTICE` roles or more than half of theirs.
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
set only while the previous run is settled, in one statement), so the
@@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta
from functools import partial
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
from app.config import Config
from app.jobs.reach import reaches
@@ -91,6 +101,19 @@ STALE_DAYS = 21
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
FAILING = "ats:failing:"
#: The same for a search, kept by the `search` plugin.
SEARCH_FAILING = "search:failing:"
#: A board that closes more roles than this in one settle is named in the
#: settle's notice, and so is one that closes more than half of its open
#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
#: one is an ordinary day.
CLOSE_NOTICE = 50
CLOSE_SHARE_MIN = 10
#: How many targets a notice names in its text; its detail names them all.
NOTICE_NAMES = 10
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
@@ -142,27 +165,38 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
`close_boards` compares against. `run` is a marker the caller claimed
already (the `pull` job, `app job pull`); without one the finder claims
its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it."""
seed_searches(config, store)
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
if not any(found for _, found in targets):
if run:
release_run(store, run)
return []
run = run or claim_run(store)
if not run:
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
reach_of = reaches(store, subs["ats"])
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
for batch in batches:
if batch["source"] == "ats":
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
return batches
nothing to read is settled here, since no settle will see a batch of it,
and so is a run whose finder raises: no batch and no settle would follow,
and the next pull would wait `ABANDONED` for a claim nobody holds."""
held = run
try:
seed_searches(config, store)
subs = {source: store.subscriptions(source=source, fetchable=True)
for source in SOURCES}
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
if not any(found for _, found in targets):
if run:
release_run(store, run)
return []
run = run or claim_run(store)
if not run:
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
held = run
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
reach_of = reaches(store, subs["ats"])
batches: list[dict[str, Any]] = [
{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
for batch in batches:
if batch["source"] == "ats":
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
return batches
except Exception:
if held:
release_run(store, held)
raise
@job("pull_batch", "Reading boards", resource="http")
@@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs. `reach` is `target ->` the
reach the run computed for a board (`app.jobs.reach`), handed to the
plugin as its `reach` option.
plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
which the runner's 10% guard accepts the answer it kept refusing.
Each target is guarded on its own. A read that fails is the plugin's to
record; anything raised after it, a row `upsert_role` refuses or a bug
past the read, is recorded here under `<source>:failing:<target>` with
its reason, and the batch goes on with the next target. Otherwise one
bad row would cost the targets after it in the batch their read on every
pull, since batches are stable, and leave no trace on any of them. A
Kill (`Cancelled`) still ends the batch.
"""
from plugins.ats import ATSSource
from plugins.readers import _runner
from plugins.search import SearchSource
# Fetchable now, not when the pull was queued: a board switched off on the
@@ -200,24 +244,39 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
if sub is None:
continue
context.check()
options = dict(config.sources.get(source) or {}) | {"run": run}
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
plugin = SearchSource(options)
else:
# A board's params are its filter; a reader's key is kept with
# the searches' (`sources.search.<name>`).
options["boards"] = [{"url": target, "label": sub.company or sub.label,
"params": sub.params}]
options["keys"] = dict(config.sources.get("search") or {})
options["reach"] = (reach or {}).get(target)
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
item.source = source
store.upsert_role(item, sub)
stored += 1
options = dict(config.sources.get(source) or {}) | {"run": run,
"stale_days": STALE_DAYS}
items: Any = None
try:
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": target, "label": sub.label,
"params": sub.params}]
plugin = SearchSource(options)
else:
# A board's params are its filter; a reader's key is kept with
# the searches' (`sources.search.<name>`).
options["boards"] = [{"url": target, "label": sub.company or sub.label,
"params": sub.params}]
options["keys"] = dict(config.sources.get("search") or {})
options["reach"] = (reach or {}).get(target)
plugin = ATSSource(options)
plugin.state = store.state
items = plugin.fetch()
for item in items:
item.source = source
store.upsert_role(item, sub)
stored += 1
except Cancelled:
raise
except Exception as exc:
log.warning("pull: %s failed after its read (%s); on to the next target",
target, exc, exc_info=True)
_runner.record_failure(store.state, f"{source}:failing:{target}", exc)
finally:
# The plugin's generator holds its HTTP client until it is closed.
if items is not None and hasattr(items, "close"):
items.close()
return {"roles": stored, "run": run}
@@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str:
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened.
stopped listing a role from, tally what happened, and say what needs a
look.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
@@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only
when that list was complete, and otherwise by `STALE_DAYS`. A board that
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
search that has failed on every fetch since `STALE_DAYS` ago closes all
its roles (`failing_boards`). How a read that answered closes is
`_close_read`'s.
The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have
replaced. A settle with no batch behind it (a refused pull, nothing to
read) closes nothing by list or by age. Last, the run is marked settled,
which lets the next pull start.
read) closes nothing by list or by age and says nothing. Last, the run is
marked settled, which lets the next pull start.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
switched-off one is closed again here; those closes are asked for and not
named in the notice.
The counts go into the result (`unanswered` counts boards and searches
whose stamp is not this run's), and one notice (`settled`) names every
target unanswered, every board read incomplete other than one mid-sweep,
and every board that closed more than `CLOSE_NOTICE` roles or more than
half the open roles it had, when there is any. The result alone was
seen by nobody: a settle closed 12,588 roles with nothing on screen.
The `prep` stage that follows maps what this run stored and folds the
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
and the roles this run added join theirs.
"""
answered = searched = failed = incomplete = closed = 0
answered = searched = incomplete = closed = 0
unanswered: list[str] = []
partial: list[str] = []
many: list[tuple[str, int, int]] = []
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store)
run_marker = _run_of(tally, context)
open_before = _open_counts(store) if run_marker else {}
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
@@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
failed += 1
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
if since and since < stale:
closed += store.close_subscription(sub.id)
continue
if source == "search":
searched += 1
unanswered.append(sub.target)
since = failing.get(sub.target, ("", ""))[0]
shut = store.close_subscription(sub.id) if since and since < stale else 0
else:
answered += 1
if not store.state.get(f"{source}:complete:{sub.target}", ""):
if source == "ats":
incomplete += 1
continue
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
continue
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
if store.state.get(f"{source}:swept:{sub.target}", ""):
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
continue
store.state.pop(f"{source}:missing:{sub.target}", None)
if is_filtered:
closed += _close_unlisted(store, source, seen_ids, sub.id)
else:
closed += store.mark_closed(source, seen_ids, sub.id)
if source == "search":
searched += 1
else:
answered += 1
read = _close_read(store, source, sub, stale)
if read is None:
if source == "ats":
incomplete += 1
if not store.state.get(f"ats:sweep:{sub.target}"):
partial.append(sub.target)
continue
shut = read
closed += shut
before = open_before.get(sub.id, 0)
if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
many.append((sub.target, shut, before))
if searched:
closed += store.close_unseen("search", stale)
if run_marker:
if unanswered or partial or many:
context.notice(settled(unanswered, partial, many, failing,
boards=answered, searches=searched, closed=closed))
release_run(store, run_marker)
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed,
return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
"incomplete": incomplete, "roles": roles, "closed": closed}
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
"""Close what one target's read of this run says has closed, and return
how many; None when the read was incomplete, which closes nothing.
A read narrowed by terms closes only the roles it has not listed for
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
two sweeps in a row missed (`_close_swept`); any other complete read
clears that record and closes what it did not list, all of it unfiltered
(`Store.mark_closed`), filtered sparing the roles acted on
(`_close_unlisted`). A filtered read that could not ask for remote roles
(`remote_unasked`) spares the remote ones as well, and closes those by
age instead."""
state, target = store.state, sub.target
if not state.get(f"{source}:complete:{target}", ""):
return None
seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
if state.get(f"{source}:narrowed:{target}", ""):
return _close_aged(store, source, sub.id, seen_ids, stale)
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
if spare_remote else 0)
if state.get(f"{source}:swept:{target}", ""):
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
spare_remote=spare_remote)
state.pop(f"{source}:missing:{target}", None)
if is_filtered:
return aged + _close_unlisted(store, source, seen_ids, sub.id,
spare_remote=spare_remote)
return store.mark_closed(source, seen_ids, sub.id)
def _open_counts(store: Store) -> dict[int, int]:
"""`subscription id -> open roles`, before the settle closes any, for the
share a board closed."""
with store.transaction() as conn:
return {int(row[0]): int(row[1]) for row in conn.execute(
"""SELECT subscription_id, count(*) FROM roles
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
GROUP BY subscription_id""")}
def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
closed: int) -> Notice:
"""The notice of a settle that has something to look at: the targets
that did not answer (with the reason their failing record gives), the
boards read incomplete, and the boards that closed many roles, each
as `closed of open`, with the totals. The text names the first
`NOTICE_NAMES` of each, the detail all of them, one a line."""
def named(target: str) -> str:
reason = failing.get(target, ("", ""))[1]
return f"{target} ({reason})" if reason else target
groups = [("Not answering", [named(t) for t in unanswered]),
("Read incomplete", list(incomplete)),
("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
text = [f"The pull read {boards} boards and {searches} searches and closed "
f"{closed:,} roles."]
detail: list[str] = []
for title, names in groups:
if not names:
continue
shown = ", ".join(names[:NOTICE_NAMES])
more = len(names) - NOTICE_NAMES
text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
else "."))
detail += [f"{title}: {name}" for name in names]
return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
#: The roles a partial read never closes by its list: one with a vote (any,
#: neutral included) or an application (which holds the notes).
_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)"""
#: The remote roles a filtered read that could not ask for them leaves alone.
_NOT_REMOTE = " AND remote != 'remote'"
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
*, spare_remote: bool = False) -> int:
"""`Store.mark_closed` for a filtered read: close every open role of the
board it did not list, except a role someone acted on, one with a vote
(any, neutral included) or an application (which holds the notes). A
board it did not list, except a role someone acted on (`_ACTED_ON`). A
filtered read cannot tell a role gone from a role outside its filter, and
a role acted on stays until the board is read without the filter or
fails for `STALE_DAYS`."""
fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
well, since the read could not ask for them."""
with store.transaction() as conn:
return conn.execute(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?))
AND uid NOT IN (SELECT uid FROM role_votes)
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)""",
f"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
+ (_NOT_REMOTE if spare_remote else ""),
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
stale: str, *, remote_only: bool = False) -> int:
"""Close the open roles of one board that its read did not list and that
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
writes on every listing): the rule for what a read cannot speak for. A
read narrowed by terms closes all such roles, as a search does; with
`remote_only`, a filtered read that could not ask for remote closes only
its remote roles so, sparing those acted on as its list does."""
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
with store.transaction() as conn:
return conn.execute(
f"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND last_seen < ?
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
(store.now(), source, subscription_id, stale,
json.dumps(list(seen_ids)))).rowcount
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
filtered: bool) -> int:
filtered: bool, *, spare_remote: bool = False) -> int:
"""The close of a board whose complete read finished a sweep across
pulls: its open roles missing from `seen_ids` that also are in
`<source>:missing:<target>`, the ones the previous finished sweep missed.
@@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
missed may only have shifted back past the cursor when roles closed
between pulls, so one sweep is not evidence that it is gone. A filtered
read keeps its exceptions as `_close_unlisted` does: a role acted on
stays open, and stays in `missing`."""
stays open, and stays in `missing`, and so does a remote role with
`spare_remote`."""
key = f"{source}:missing:{sub.target}"
try:
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
except (TypeError, ValueError):
earlier = set()
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
AND uid NOT IN (SELECT role_uid FROM applications
WHERE role_uid IS NOT NULL)""" if filtered else ""
acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
with store.transaction() as conn:
missed = [row[0] for row in conn.execute(
"""SELECT source_id FROM roles
@@ -365,16 +536,20 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
"""`target -> (since, reason)` for every `ats` board whose last fetch
failed: when the failures began, and what the latest one said."""
"""`target -> (since, reason)` for every `ats` board and every search
whose last fetch failed: when the failures began, and what the latest
one said. No board host is also a search provider, so a board's target
(`host:slug`) and a search's (`provider:name`) never collide in one
dict."""
out: dict[str, tuple[str, str]] = {}
for key, value in store.state_items(FAILING).items():
try:
record = json.loads(value)
except ValueError:
continue
out[key[len(FAILING):]] = (str(record.get("since") or ""),
str(record.get("reason") or ""))
for prefix in (FAILING, SEARCH_FAILING):
for key, value in store.state_items(prefix).items():
try:
record = json.loads(value)
except ValueError:
continue
out[key[len(prefix):]] = (str(record.get("since") or ""),
str(record.get("reason") or ""))
return out
+13 -7
View File
@@ -9,6 +9,8 @@ good for a given search. For each agent:
column filled for at least one, is gated ("none of
country DK / remote remote")
prefer gates `weight` when the role matches, 0 when it does not
exclude gates a role that matches one of the values keeps its row, with
score 0 and the value ("title account executive")
A gate rejects on evidence, never on absence: a role whose column is empty
is not gated by it. `extract` is what fills an empty column, and it reads
@@ -29,7 +31,8 @@ roles out of extract's reach for good.
Every term is 0..1. The score is the weighted sum over the sum of the weights
(the agent's four and its prefer gates'), so it stays 0..1. A favourite
company (`affinity = 2`) skips the `any` group but not the `require` gates, and
company (`affinity = 2`) skips the `any` group but not the `require` or
`exclude` gates, and
is scored like any other company with its affinity term at 1: nothing puts
its roles above a better fit. A hidden one (`-1`) is not scored at all. A
role with no vector is scored without `fit` and `taste`, unless `fit` is all
@@ -156,14 +159,17 @@ def _reason(gate: Gate) -> str:
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
"""Why this role is gated, as reasons: every require gate it fails, and
the `any` group when it matches none of them. A gate with an empty column
(state -1) neither passes nor fails, so the group fails only when some
member had evidence. A favourite company's roles skip the `any` group but
not the require gates, so a star widens a search to a company's roles
within e.g. Europe, not to the whole world."""
"""Why this role is gated, as reasons: every require gate it fails, every
exclude gate it matches, and the `any` group when it matches none of them.
A gate with an empty column (state -1) neither passes nor fails, so the
group fails only when some member had evidence. A favourite company's
roles skip the `any` group but not the require and exclude gates, so a
star widens a search to a company's roles within e.g. Europe, not to the
whole world, and not to the titles the agent leaves out."""
states = list(zip(gates, candidate.gates, strict=True))
reasons = [_reason(g) for g, state in states if g.mode == "require" and state == 0]
reasons += [f"{g.field} {g.values[state - 1]}" for g, state in states
if g.mode == "exclude" and state > 0]
group = [(g, state) for g, state in states if g.mode == "any"]
if candidate.affinity != 2 and group and not any(state == 1 for _, state in group) \
and any(state == 0 for _, state in group):
+6 -3
View File
@@ -11,11 +11,14 @@ finders ask it (`extract.extract_task`, `summarise.summarise_task`).
`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`)
asks no model and is on by default (`enrich.enabled`); its switch is
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
`stage:enrich`, set the same way as the others: by the Runs page
(`PUT /api/stages/enrich`) and by `app stages enrich on|off`.
`describe` is what `GET /api/stages` and `app stages` show: for each stage
with a switch, the model stages and `enrich`, whether it is on, where that comes from, the model it uses, and who
would answer.
would answer. The model is the one a call is made with: `agent.model` when
set, since it overrides the per-task model everywhere (`desk/modeltask.py`),
else the stage's own.
"""
from __future__ import annotations
@@ -98,7 +101,7 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
backend, said = answer(config)
return Stage(name=name, on=stage_on(config, store, name),
source="config" if stored(store, name) is None else "stored",
model=getattr(getattr(config, name), "model", ""), backend=backend,
model=config.agent.model or getattr(config, name).model, backend=backend,
answer=said)
+12 -5
View File
@@ -1167,21 +1167,26 @@ class Application(Model):
#: The role columns a gate can name. `company` matches on the company key;
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; `title`
#: matches a value as whole words anywhere in the title, folded as `title_key`
#: folds it, so `account executive` finds "Enterprise Account Executive, UK"
#: and `data engineer` does not find "Data Engineering Manager"; the rest are
#: case-insensitive equality, except `salary_min` (at least the value) and
#: `languages` (every language the role requires is one of the values).
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company"]
"currency", "permit", "languages", "company", "title"]
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
GateMode = Literal["require", "any", "prefer"]
GateMode = Literal["require", "any", "prefer", "exclude"]
class Gate(Model):
"""One condition of an agent. `require` gates a role that fails it (the
row is kept with the reason); the `any` gates together are one group a
role must match at least one of ("in Denmark, or remote"); `prefer` adds
`weight` to its score when it matches. A gate rejects on evidence only: a
`weight` to its score when it matches; `exclude` gates a role that matches
any of its values ("title account executive"), a favourite's too. A gate
rejects on evidence only: a
role whose column is empty is not rejected, since `extract` fills the
column later for the roles that rank, and only for those."""
@@ -1297,7 +1302,9 @@ class Agent(Model):
class Candidate(Model):
"""What scoring reads about one open role for one agent: its posted
date (for `recency`), its company's affinity for that agent, and per
gate `1` match, `0` no match, `-1` the column is empty."""
gate `1` match, `0` no match, `-1` the column is empty. An `exclude`
gate's match is the position of the first value it matched, from 1, so
the reason can name it."""
uid: str
posted_at: str
+13 -4
View File
@@ -39,8 +39,8 @@
"HU", "IE", "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
"IS", "LI", "NO", "GB", "CH", "EU"];
const FIELDS = ["place", "country", "remote", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company"];
const MODES = [["require", "must"], ["any", "one of"], ["prefer", "prefer"]];
"currency", "permit", "languages", "company", "title"];
const MODES = [["require", "must"], ["any", "one of"], ["prefer", "prefer"], ["exclude", "not"]];
const HYBRID_WEIGHT = 0.05;
/** @type {object[]} every agent, as the server lists them */
@@ -227,11 +227,14 @@
const must = of("require").map(phrase);
const prefer = of("prefer").map((g) => phrase(g).replace(/^with /, ""));
if (where.hybrid) prefer.unshift("hybrid");
const not = of("exclude").map(phrase);
const sentences = [];
const any = whereSentence(where, of("any"));
if (any) sentences.push(any);
if (must.length) sentences.push(`Only roles ${must.join(" and ")}.`);
if (prefer.length) sentences.push(`Prefers ${prefer.join("; ")}.`);
if (where.titles) sentences.push(`Leaves out titles with ${orList(where.titles.values.map((t) => `“${t}”`))}.`);
if (not.length) sentences.push(`Leaves out roles ${not.join(" or ")}.`);
return sentences.join(" ");
}
@@ -497,7 +500,7 @@
// that is Europe and the countries line exactly, and only with Remote
// ticked; any other must on country is a row, and Remote reads anywhere.
function distribute(gates) {
const where = { countries: null, places: null, remote: null, europe: null, hybrid: null };
const where = { countries: null, places: null, remote: null, europe: null, hybrid: null, titles: null };
const rest = [];
const musts = [];
const plain = (g) => g.weight === undefined || g.weight === 1;
@@ -512,6 +515,7 @@
else if (g.field === "remote" && g.mode === "any" && plain(g) && only(g, "remote")) slot = "remote";
else if (g.field === "place" && g.mode === "any" && plain(g)) slot = "places";
else if (g.field === "remote" && g.mode === "prefer" && only(g, "hybrid")) slot = "hybrid";
else if (g.field === "title" && g.mode === "exclude" && plain(g)) slot = "titles";
if (slot && !where[slot]) where[slot] = g;
else rest.push(g);
}
@@ -561,7 +565,9 @@
const { where, rest } = distribute(row ? row.gates : []);
const countries = countryInput(where.countries ? where.countries.values : []);
const places = wordInput(where.places ? where.places.values : [], "Add a town");
const titles = wordInput(where.titles ? where.titles.values : [], "account executive");
const terms = wordInput(row ? row.fetch_terms || [] : [], "machine learning, data science");
$("#a-titles", f).replaceWith(titles.node);
$("#a-terms", f).replaceWith(terms.node);
$("#a-countries", f).replaceWith(countries.node);
$("#a-towns", f).replaceWith(places.node);
@@ -601,7 +607,8 @@
});
// The gates the Where block says, in the order the starter agents have
// them: the must on country, the lines, then hybrid.
// them: the must on country, the lines, then hybrid; and the titles left
// out, the block under it.
function whereGates() {
const out = [];
const within = countries.values();
@@ -615,6 +622,8 @@
if ($("#a-hybrid", f).checked) {
out.push({ field: "remote", values: ["hybrid"], mode: "prefer", weight: Number($("#a-hybrid-weight", f).value) });
}
const left = titles.values();
if (left.length) out.push({ field: "title", values: left, mode: "exclude" });
return out;
}
// The gates Save sends: the Where block's, then the rows. The sentence
+2 -1
View File
@@ -1223,7 +1223,8 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
border-radius: .3rem; padding: .35rem .5rem;
}
/* The fieldsets that hold rows: a column, the Add button under the rows. */
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.terms,
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
.agent-form fieldset.terms,
.agent-form fieldset.rules {
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
border: 1px solid var(--line); border-radius: .3rem;
+3 -2
View File
@@ -59,7 +59,7 @@
const company = companies.get(application.company_key);
cell(tr, "Company", company ? company.name : application.company_key);
const select = statusSelect(application.status);
select.addEventListener("change", () => setStatus(application.id, select));
select.addEventListener("change", () => setStatus(application.id, select, application.status));
cell(tr, "Status", select);
const saved = cell(tr, "Saved", savedOn(application.created_at));
saved.title = application.created_at;
@@ -69,7 +69,7 @@
count(applications.length, "tracked application");
}
async function setStatus(id, select) {
async function setStatus(id, select, was) {
select.disabled = true;
try {
await api(`/api/applications/${encodeURIComponent(id)}/status`, {
@@ -77,6 +77,7 @@
});
render(await api("/api/applications"));
} catch (err) {
select.value = was; // the change was refused, so the row still holds `was`
select.disabled = false;
}
}
+4 -4
View File
@@ -217,8 +217,8 @@
// Replacing a different board closes its roles (see set_board in
// app/jobs/sources.py); ask first, naming both boards and the count,
// the same as Detach and a removed search already do.
if (company.board && company.board !== board
&& !window.confirm(`Replace ${company.name}'s board (${company.board}) `
if (company.target && company.target !== board
&& !window.confirm(`Replace ${company.name}'s board (${company.target}) `
+ `with ${board}? Its ${count(company.open_roles, "open role")} close.`)) {
return;
}
@@ -429,8 +429,8 @@
// editor does, so it asks first too.
const existing = [...companies.values()]
.find((c) => c.name.toLowerCase() === name.toLowerCase());
if (existing && existing.board && existing.board !== board
&& !window.confirm(`Replace ${existing.name}'s board (${existing.board}) `
if (existing && existing.target && existing.target !== board
&& !window.confirm(`Replace ${existing.name}'s board (${existing.target}) `
+ `with ${board}? Its ${count(existing.open_roles, "open role")} close.`)) {
return;
}
+11 -4
View File
@@ -109,10 +109,17 @@
...(stage.model ? [el("span", "badge", stage.model)] : []));
row.append(name, stageSwitch(stage, () => renderStages(stages)));
const sub = el("span", "sub");
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
const agents = el("a", "", "Search agents");
agents.href = "/settings/agents";
sub.append(agents, stage.source === "config"
if (stage.name === "enrich") {
/* Global, unlike the model stages: `enrich.top` and `enrich.min_score`. */
sub.append(`${DOES[stage.name]}. How many it reads is set for every agent alike by `
+ "enrich.top and enrich.min_score in config.yaml");
} else {
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
const agents = el("a", "", "Search agents");
agents.href = "/settings/agents";
sub.append(agents);
}
sub.append(stage.source === "config"
? ". Following config.yaml until this switch is used."
: ".");
row.appendChild(sub);
+3 -2
View File
@@ -864,7 +864,8 @@
is the only time this page ever sees a role's `summary` or
`description` directly. */
function excerptOf(role) {
const text = (role.summary || role.description || "").replace(/\s+/g, " ").trim();
/* The server's order (`_EXCERPT_SOURCE`): summary, description, teaser. */
const text = (role.summary || role.description || role.teaser || "").replace(/\s+/g, " ").trim();
if (text.length <= 200) return text;
const cut = text.slice(0, 200);
const space = cut.lastIndexOf(" ");
@@ -1458,7 +1459,7 @@
async function fetchText(button) {
const uid = state.readingUid;
const role = await busy(button, "Fetching…",
() => api(`/api/roles/${encodeURIComponent(uid)}/text`, { method: "POST" }));
() => api(roleUrl(uid).replace(/(\?|$)/, "/text$1"), { method: "POST" }));
if (role) merge(role);
}
+1 -1
View File
@@ -176,7 +176,7 @@
text: n.text,
fix: n.fix,
detail: n.detail,
link: { label: "all messages", href: "/messages" },
link: { label: "all messages", href: "/settings/sources#messages" },
};
if (PROBLEM.has(n.level)) t.count = n.count;
if (n.level === "good") t.action = { label: "reload", run: () => location.reload() };
+16 -4
View File
@@ -219,6 +219,13 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
if gate.field == "place":
match, params = _places_sql(values, gaz, table="r")
return "r.place = ''", match, params
if gate.field == "title":
# Whole words of the folded title: " account executive " inside
# " enterprise account executive uk ". A fold drops `%` and `_`, so a
# value cannot smuggle a LIKE wildcard in.
words = [f"% {title_key(v)} %" for v in values if title_key(v)]
match = " OR ".join("(' ' || r.title_key || ' ') LIKE ?" for _ in words) or "0"
return "r.title_key = ''", f"({match})", words
if gate.field == "company":
keys = [company_key(v) for v in values]
return ("r.company_key = ''",
@@ -1830,10 +1837,15 @@ class Store:
params: list[Any] = []
gaz = self.gazetteer()
for n, gate in enumerate(gates):
empty, match, values = _gate_sql(gate, gaz)
columns.append(
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
params += values
# An exclude gate says which of its values matched: one WHEN each.
parts = ([gate.model_copy(update={"values": [v]}) for v in gate.values]
if gate.mode == "exclude" else [gate])
whens = []
for k, part in enumerate(parts, 1):
empty, match, values = _gate_sql(part, gaz)
whens.append(f"WHEN {match} THEN {k}")
params += values
columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}")
select = ", ".join(["r.uid", "r.posted_at",
"COALESCE(ac.affinity, 0) AS affinity", *columns])
rows = self._rows(
+8
View File
@@ -49,6 +49,14 @@
<span class="hint">This one ranks rather than filters: a hybrid role gains a little.</span>
<p id="a-where-sum" class="where-sum" role="status"></p>
</fieldset>
<fieldset class="titles">
<legend>Which titles to leave out?</legend>
<div class="where-line">
<span>Leave out titles with</span>
<div id="a-titles"></div>
</div>
<span class="hint">Separate phrases with a comma or Enter. A phrase matches whole words anywhere in the title, whatever the case: <code>account executive</code> takes "Enterprise Account Executive, UK", and <code>data engineer</code> leaves "Data Engineering Manager" alone. A shorter phrase takes more: <code>sales</code> takes Sales Engineer too. It applies to liked companies as well. A role left out keeps its row, and Show gated on Roles lists it with the phrase that took it.</span>
</fieldset>
<fieldset class="terms">
<legend>What big boards send</legend>
<div class="where-line">
+30 -8
View File
@@ -98,7 +98,13 @@ def _feed_filters(
raise HTTPException(422, f"{name} is one of {', '.join(RUNGS)}, not {rung!r}.")
if track and track not in TRACKS:
raise HTTPException(422, f"track is one of {', '.join(TRACKS)}, not {track!r}.")
levels = levels_between(level_from, level_to, track) if level_from or level_to or track else []
levels: tuple[str, ...] = ()
if level_from or level_to or track:
# The store reads an empty list as "no filter", so a range that holds no
# level must be refused here or it would list every role.
levels = levels_between(level_from, level_to, track)
if not levels:
raise HTTPException(422, "That seniority range holds no level.")
return {
"q": q or None, "company": company or None, "place": place or None,
"function": function or None, "remote": remote or None, "days": days,
@@ -359,14 +365,21 @@ def _known_role(store: Store, uid: str) -> Role:
@router.post("/roles/{uid}/text")
def fetch_text(uid: str, store: StoreDep) -> Role:
def fetch_text(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole:
"""Fetch the posting page once and keep it as plain text, for a role whose
description is a teaser (`jobs.posting.fetch_posting`); a Jobindex role
is read from the page its "Se jobbet" link leads to."""
is read from the page its "Se jobbet" link leads to. The role comes back as
`agent` sees it, as `GET /api/roles/{uid}` does: the page merges the answer
into the open role, and one without the vote would blank the reader's."""
agent_id = _agent_id(store, agent)
try:
return jobs.posting.fetch_posting(store, uid)
jobs.posting.fetch_posting(store, uid)
except jobs.posting.PostingError as exc:
raise HTTPException(exc.status, str(exc)) from exc
role = store.role(uid, agent_id)
if role is None:
raise HTTPException(404, f"no such role: {uid}")
return role
@router.post("/roles/{uid}/summarise", status_code=202)
@@ -436,12 +449,17 @@ class CompanyOut(BaseModel):
with no board shown that is not `from_search`, and `hand_reason` says why
it is read by hand (`app.models.HAND_REASONS`). `all_countries` and
`all_roles` are the company's two fetch overrides: its big boards read
with the agents' wide reach, and with no role-type narrowing."""
with the agents' wide reach, and with no role-type narrowing. `target` is
the attached board in full, `host:slug`, where `board` is its host alone;
the page compares it with the board typed to know whether a replacement
closes anything. `ats_slug` cannot stand in for it, since the careers site
may run on another host than the board."""
key: str
name: str
notes: str = ""
board: str
target: str = ""
subscription_id: int | None = None
on: bool | None = None
affinity: int
@@ -500,6 +518,7 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]:
sub = subs.get(sub_id) if sub_id is not None else None
from_search = jobs.sources.known_from_search_only(c, ever_boarded)
out.append(CompanyOut(key=c.key, name=c.name, notes=c.notes, board=host,
target=target,
subscription_id=sub_id, on=on, affinity=c.affinity,
open_roles=counts.get(c.key, 0), ats=c.ats,
ats_slug=c.ats_slug, careers_url=c.careers_url,
@@ -865,9 +884,12 @@ def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]:
sub = store.subscription(sub_id)
if sub is None or sub.source != "ats":
raise HTTPException(404, f"no such board: {sub_id}")
codes = sorted({c.strip().upper() for c in body.countries if c.strip()})
if unknown := [c for c in codes if not places.known(f"country:{c}") or c == "EU"]:
raise HTTPException(422, f"Not a country code: {', '.join(unknown)}.")
named = sorted({c.strip() for c in body.countries if c.strip()})
tokens = {c: places.parse(f"country:{c}") for c in named}
if unknown := [c for c, t in tokens.items() if not places.known(str(t)) or t.value == "EU"]:
raise HTTPException(422, f"Not a country: {', '.join(unknown)}.")
# The ISO code is what the reach compares against, not the name or alias typed.
codes = sorted({t.value for t in tokens.values()})
if body.mode == "own" and not (codes or body.remote):
raise HTTPException(422, "Name a country or remote, or take the agents' reach.")
params = {k: v for k, v in sub.params.items() if k not in jobs.reach.OVERRIDE_KEYS}