Merge main into wp-review-contract
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
fe2a1b0347
36 files changed
+1741
-596
No files matched your search
+1
-1
@@ -6,7 +6,7 @@
|
||||
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
|
||||
app searches add|list|remove
|
||||
app agents list|prompt
|
||||
app stages [extract|summarise on|off]
|
||||
app stages [extract|summarise|enrich on|off]
|
||||
app dupes
|
||||
app prep [unmapped|unlinked] [--remap]
|
||||
app db init
|
||||
|
||||
+21
-3
@@ -16,8 +16,9 @@ the starter agents; `config.example.yaml` documents every key.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from pydantic import Field, ValidationError, field_validator, model_validator
|
||||
@@ -165,12 +166,29 @@ class SearchSeed(Section):
|
||||
"""A saved search as `config.yaml` lists it: one `search` subscription,
|
||||
imported once like the agents. `params` are the provider's own, see
|
||||
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
|
||||
`where` and `pages` for adzuna."""
|
||||
`where` and `pages` for adzuna; `q` and `region` for platsbanken. The provider is any reader of
|
||||
`scope="search"`, so a new one needs no edit here."""
|
||||
|
||||
name: str
|
||||
provider: Literal["jobindex", "adzuna"]
|
||||
provider: str
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("provider")
|
||||
@classmethod
|
||||
def _a_search_provider(cls, provider: str) -> str:
|
||||
# `plugins` is a top-level folder beside `app` and is not on the path
|
||||
# when this module is imported first, so it is reached the way
|
||||
# `app.models` reaches it (`_declared_posted`), and only when a seed
|
||||
# is validated.
|
||||
root = str(Path(__file__).resolve().parents[1])
|
||||
if root not in sys.path:
|
||||
sys.path.insert(0, root)
|
||||
from plugins.readers import PROVIDERS
|
||||
|
||||
if provider not in PROVIDERS:
|
||||
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}, not {provider!r}")
|
||||
return provider
|
||||
|
||||
|
||||
class Config(Section):
|
||||
"""The whole file."""
|
||||
|
||||
+10
-2
@@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
|
||||
Under `NULL` there is no queue to take the chain, so this raises rather
|
||||
than report a pull that never happened; `app job pull` claims the run and
|
||||
starts the chain on a queue of its own instead of calling this.
|
||||
starts the chain on a queue of its own instead of calling this. Anything
|
||||
that raises once the run is claimed, building the chain or handing it
|
||||
over, releases the claim first, or the next pull would wait
|
||||
`pull.ABANDONED` for a run that never started.
|
||||
"""
|
||||
run = pull.claim_run(store)
|
||||
if not run:
|
||||
since = pull.running(store)
|
||||
context.notice(already_running(since))
|
||||
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
||||
if not context.task(pull_chain(run)):
|
||||
try:
|
||||
started = context.task(pull_chain(run))
|
||||
except Exception:
|
||||
pull.release_run(store, run)
|
||||
raise
|
||||
if not started:
|
||||
pull.release_run(store, run)
|
||||
raise RuntimeError("no queue to start the pull cascade on")
|
||||
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
||||
|
||||
+282
-107
@@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see
|
||||
`plugins/readers/_runner.py`).
|
||||
|
||||
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
|
||||
person acted on: those with a vote or an application (`_close_unlisted`). A
|
||||
filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
||||
under a tenth of what the same read listed last time is recorded by the
|
||||
runner as a failure, not as a list to close against.
|
||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in
|
||||
reach, so its close keeps open the roles a person acted on: those with a vote
|
||||
or an application (`_close_unlisted`). When the reach wanted remote roles and
|
||||
the reader could not ask for them (`ats:remote_unasked:<target>`), the read
|
||||
says nothing of the remote roles either, so they stay open too and close only
|
||||
once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
|
||||
that lists under a tenth of what the same read listed last time is recorded
|
||||
by the runner as a failure, not as a list to close against (`_runner.read`).
|
||||
|
||||
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
||||
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
||||
@@ -33,14 +36,21 @@ not swept clears them.
|
||||
|
||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||
the roles matching a word, which says nothing of the roles it did not list,
|
||||
so it closes nothing; only an unfiltered or a place-filtered complete read
|
||||
closes.
|
||||
so it closes by list nothing: a role of such a board closes once the board
|
||||
has not listed it for `STALE_DAYS`, the rule searches have.
|
||||
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
||||
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
||||
closes all of its roles, and its next answer reopens what it lists.
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board or a
|
||||
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
|
||||
`search:failing:<target>`) closes all of its roles, and its next answer
|
||||
reopens what it lists. A target that raises in its batch after its read, a
|
||||
row that does not store, is recorded as failing the same way and the batch
|
||||
goes on with the next one (`run_batch`).
|
||||
|
||||
The settle says what needs a look in one notice (`settled`): the targets
|
||||
that did not answer or were read incomplete, and the boards that closed more
|
||||
than `CLOSE_NOTICE` roles or more than half of theirs.
|
||||
|
||||
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
||||
set only while the previous run is settled, in one statement), so the
|
||||
@@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, Tally, Task, job
|
||||
from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs.reach import reaches
|
||||
@@ -91,6 +101,19 @@ STALE_DAYS = 21
|
||||
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
||||
FAILING = "ats:failing:"
|
||||
|
||||
#: The same for a search, kept by the `search` plugin.
|
||||
SEARCH_FAILING = "search:failing:"
|
||||
|
||||
#: A board that closes more roles than this in one settle is named in the
|
||||
#: settle's notice, and so is one that closes more than half of its open
|
||||
#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
|
||||
#: one is an ordinary day.
|
||||
CLOSE_NOTICE = 50
|
||||
CLOSE_SHARE_MIN = 10
|
||||
|
||||
#: How many targets a notice names in its text; its detail names them all.
|
||||
NOTICE_NAMES = 10
|
||||
|
||||
#: The `state` key that records the starter searches were imported.
|
||||
SEARCHES_SEEDED = "searches:seeded"
|
||||
|
||||
@@ -142,27 +165,38 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
|
||||
`close_boards` compares against. `run` is a marker the caller claimed
|
||||
already (the `pull` job, `app job pull`); without one the finder claims
|
||||
its own, and queues nothing when another run is in flight. A run with
|
||||
nothing to read is settled here, since no settle will see a batch of it."""
|
||||
seed_searches(config, store)
|
||||
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
|
||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||
if not any(found for _, found in targets):
|
||||
if run:
|
||||
release_run(store, run)
|
||||
return []
|
||||
run = run or claim_run(store)
|
||||
if not run:
|
||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||
running(store))
|
||||
return []
|
||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||
reach_of = reaches(store, subs["ats"])
|
||||
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
for batch in batches:
|
||||
if batch["source"] == "ats":
|
||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||
return batches
|
||||
nothing to read is settled here, since no settle will see a batch of it,
|
||||
and so is a run whose finder raises: no batch and no settle would follow,
|
||||
and the next pull would wait `ABANDONED` for a claim nobody holds."""
|
||||
held = run
|
||||
try:
|
||||
seed_searches(config, store)
|
||||
subs = {source: store.subscriptions(source=source, fetchable=True)
|
||||
for source in SOURCES}
|
||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||
if not any(found for _, found in targets):
|
||||
if run:
|
||||
release_run(store, run)
|
||||
return []
|
||||
run = run or claim_run(store)
|
||||
if not run:
|
||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||
running(store))
|
||||
return []
|
||||
held = run
|
||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||
reach_of = reaches(store, subs["ats"])
|
||||
batches: list[dict[str, Any]] = [
|
||||
{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
for batch in batches:
|
||||
if batch["source"] == "ats":
|
||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||
return batches
|
||||
except Exception:
|
||||
if held:
|
||||
release_run(store, held)
|
||||
raise
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
@@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
plugin as its `run` option, so its stamps belong to that run whatever the
|
||||
stored marker says by the time the batch runs. `reach` is `target ->` the
|
||||
reach the run computed for a board (`app.jobs.reach`), handed to the
|
||||
plugin as its `reach` option.
|
||||
plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
|
||||
which the runner's 10% guard accepts the answer it kept refusing.
|
||||
|
||||
Each target is guarded on its own. A read that fails is the plugin's to
|
||||
record; anything raised after it, a row `upsert_role` refuses or a bug
|
||||
past the read, is recorded here under `<source>:failing:<target>` with
|
||||
its reason, and the batch goes on with the next target. Otherwise one
|
||||
bad row would cost the targets after it in the batch their read on every
|
||||
pull, since batches are stable, and leave no trace on any of them. A
|
||||
Kill (`Cancelled`) still ends the batch.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.readers import _runner
|
||||
from plugins.search import SearchSource
|
||||
|
||||
# Fetchable now, not when the pull was queued: a board switched off on the
|
||||
@@ -200,24 +244,39 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
if sub is None:
|
||||
continue
|
||||
context.check()
|
||||
options = dict(config.sources.get(source) or {}) | {"run": run}
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
# A board's params are its filter; a reader's key is kept with
|
||||
# the searches' (`sources.search.<name>`).
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
options["reach"] = (reach or {}).get(target)
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
options = dict(config.sources.get(source) or {}) | {"run": run,
|
||||
"stale_days": STALE_DAYS}
|
||||
items: Any = None
|
||||
try:
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": target, "label": sub.label,
|
||||
"params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
# A board's params are its filter; a reader's key is kept with
|
||||
# the searches' (`sources.search.<name>`).
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
options["reach"] = (reach or {}).get(target)
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
items = plugin.fetch()
|
||||
for item in items:
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
except Cancelled:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.warning("pull: %s failed after its read (%s); on to the next target",
|
||||
target, exc, exc_info=True)
|
||||
_runner.record_failure(store.state, f"{source}:failing:{target}", exc)
|
||||
finally:
|
||||
# The plugin's generator holds its HTTP client until it is closed.
|
||||
if items is not None and hasattr(items, "close"):
|
||||
items.close()
|
||||
return {"roles": stored, "run": run}
|
||||
|
||||
|
||||
@@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str:
|
||||
def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""The run's settle: close every board this run answered in full and
|
||||
stopped listing a role from, and tally what happened.
|
||||
stopped listing a role from, tally what happened, and say what needs a
|
||||
look.
|
||||
|
||||
Reads `self.state` rather than the batch jobs' own results, because a
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
@@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed by its list, and a board
|
||||
that answered incompletely is not either. A search closes by its list only
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
||||
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
|
||||
search that has failed on every fetch since `STALE_DAYS` ago closes all
|
||||
its roles (`failing_boards`). How a read that answered closes is
|
||||
`_close_read`'s.
|
||||
|
||||
The stamps are compared with this run's own marker, the one its batches
|
||||
carried, never with the stored `pull:run`, which a later run may have
|
||||
replaced. A settle with no batch behind it (a refused pull, nothing to
|
||||
read) closes nothing by list or by age. Last, the run is marked settled,
|
||||
which lets the next pull start.
|
||||
read) closes nothing by list or by age and says nothing. Last, the run is
|
||||
marked settled, which lets the next pull start.
|
||||
|
||||
A board or search switched off while its batch was in flight had its
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
switched-off one is closed again here; those closes are asked for and not
|
||||
named in the notice.
|
||||
|
||||
The counts go into the result (`unanswered` counts boards and searches
|
||||
whose stamp is not this run's), and one notice (`settled`) names every
|
||||
target unanswered, every board read incomplete other than one mid-sweep,
|
||||
and every board that closed more than `CLOSE_NOTICE` roles or more than
|
||||
half the open roles it had, when there is any. The result alone was
|
||||
seen by nobody: a settle closed 12,588 roles with nothing on screen.
|
||||
|
||||
The `prep` stage that follows maps what this run stored and folds the
|
||||
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
||||
and the roles this run added join theirs.
|
||||
"""
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
answered = searched = incomplete = closed = 0
|
||||
unanswered: list[str] = []
|
||||
partial: list[str] = []
|
||||
many: list[tuple[str, int, int]] = []
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
failing = failing_boards(store)
|
||||
run_marker = _run_of(tally, context)
|
||||
open_before = _open_counts(store) if run_marker else {}
|
||||
for source in SOURCES:
|
||||
for sub in store.subscriptions(source=source):
|
||||
if not (sub.subscribed and sub.fetch):
|
||||
@@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
for sub in store.subscriptions(source=source, fetchable=True):
|
||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||
if not (stamp and stamp == run_marker):
|
||||
failed += 1
|
||||
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
||||
if since and since < stale:
|
||||
closed += store.close_subscription(sub.id)
|
||||
continue
|
||||
if source == "search":
|
||||
searched += 1
|
||||
unanswered.append(sub.target)
|
||||
since = failing.get(sub.target, ("", ""))[0]
|
||||
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
||||
else:
|
||||
answered += 1
|
||||
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
continue
|
||||
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
|
||||
if store.state.get(f"{source}:swept:{sub.target}", ""):
|
||||
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
|
||||
continue
|
||||
store.state.pop(f"{source}:missing:{sub.target}", None)
|
||||
if is_filtered:
|
||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
||||
else:
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if source == "search":
|
||||
searched += 1
|
||||
else:
|
||||
answered += 1
|
||||
read = _close_read(store, source, sub, stale)
|
||||
if read is None:
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||
partial.append(sub.target)
|
||||
continue
|
||||
shut = read
|
||||
closed += shut
|
||||
before = open_before.get(sub.id, 0)
|
||||
if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
|
||||
many.append((sub.target, shut, before))
|
||||
if searched:
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
if unanswered or partial or many:
|
||||
context.notice(settled(unanswered, partial, many, failing,
|
||||
boards=answered, searches=searched, closed=closed))
|
||||
release_run(store, run_marker)
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
||||
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||
"""Close what one target's read of this run says has closed, and return
|
||||
how many; None when the read was incomplete, which closes nothing.
|
||||
|
||||
A read narrowed by terms closes only the roles it has not listed for
|
||||
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
||||
two sweeps in a row missed (`_close_swept`); any other complete read
|
||||
clears that record and closes what it did not list, all of it unfiltered
|
||||
(`Store.mark_closed`), filtered sparing the roles acted on
|
||||
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
||||
(`remote_unasked`) spares the remote ones as well, and closes those by
|
||||
age instead."""
|
||||
state, target = store.state, sub.target
|
||||
if not state.get(f"{source}:complete:{target}", ""):
|
||||
return None
|
||||
seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
|
||||
if state.get(f"{source}:narrowed:{target}", ""):
|
||||
return _close_aged(store, source, sub.id, seen_ids, stale)
|
||||
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
||||
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
||||
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
||||
if spare_remote else 0)
|
||||
if state.get(f"{source}:swept:{target}", ""):
|
||||
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
||||
spare_remote=spare_remote)
|
||||
state.pop(f"{source}:missing:{target}", None)
|
||||
if is_filtered:
|
||||
return aged + _close_unlisted(store, source, seen_ids, sub.id,
|
||||
spare_remote=spare_remote)
|
||||
return store.mark_closed(source, seen_ids, sub.id)
|
||||
|
||||
|
||||
def _open_counts(store: Store) -> dict[int, int]:
|
||||
"""`subscription id -> open roles`, before the settle closes any, for the
|
||||
share a board closed."""
|
||||
with store.transaction() as conn:
|
||||
return {int(row[0]): int(row[1]) for row in conn.execute(
|
||||
"""SELECT subscription_id, count(*) FROM roles
|
||||
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
||||
GROUP BY subscription_id""")}
|
||||
|
||||
|
||||
def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
|
||||
failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
|
||||
closed: int) -> Notice:
|
||||
"""The notice of a settle that has something to look at: the targets
|
||||
that did not answer (with the reason their failing record gives), the
|
||||
boards read incomplete, and the boards that closed many roles, each
|
||||
as `closed of open`, with the totals. The text names the first
|
||||
`NOTICE_NAMES` of each, the detail all of them, one a line."""
|
||||
def named(target: str) -> str:
|
||||
reason = failing.get(target, ("", ""))[1]
|
||||
return f"{target} ({reason})" if reason else target
|
||||
|
||||
groups = [("Not answering", [named(t) for t in unanswered]),
|
||||
("Read incomplete", list(incomplete)),
|
||||
("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
|
||||
text = [f"The pull read {boards} boards and {searches} searches and closed "
|
||||
f"{closed:,} roles."]
|
||||
detail: list[str] = []
|
||||
for title, names in groups:
|
||||
if not names:
|
||||
continue
|
||||
shown = ", ".join(names[:NOTICE_NAMES])
|
||||
more = len(names) - NOTICE_NAMES
|
||||
text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
|
||||
else "."))
|
||||
detail += [f"{title}: {name}" for name in names]
|
||||
return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
|
||||
text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
|
||||
|
||||
|
||||
#: The roles a partial read never closes by its list: one with a vote (any,
|
||||
#: neutral included) or an application (which holds the notes).
|
||||
_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)"""
|
||||
|
||||
#: The remote roles a filtered read that could not ask for them leaves alone.
|
||||
_NOT_REMOTE = " AND remote != 'remote'"
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
|
||||
*, spare_remote: bool = False) -> int:
|
||||
"""`Store.mark_closed` for a filtered read: close every open role of the
|
||||
board it did not list, except a role someone acted on, one with a vote
|
||||
(any, neutral included) or an application (which holds the notes). A
|
||||
board it did not list, except a role someone acted on (`_ACTED_ON`). A
|
||||
filtered read cannot tell a role gone from a role outside its filter, and
|
||||
a role acted on stays until the board is read without the filter or
|
||||
fails for `STALE_DAYS`."""
|
||||
fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
|
||||
well, since the read could not ask for them."""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))
|
||||
AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)""",
|
||||
f"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
|
||||
+ (_NOT_REMOTE if spare_remote else ""),
|
||||
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
||||
stale: str, *, remote_only: bool = False) -> int:
|
||||
"""Close the open roles of one board that its read did not list and that
|
||||
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
||||
writes on every listing): the rule for what a read cannot speak for. A
|
||||
read narrowed by terms closes all such roles, as a search does; with
|
||||
`remote_only`, a filtered read that could not ask for remote closes only
|
||||
its remote roles so, sparing those acted on as its list does."""
|
||||
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
f"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND last_seen < ?
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
||||
(store.now(), source, subscription_id, stale,
|
||||
json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
filtered: bool) -> int:
|
||||
filtered: bool, *, spare_remote: bool = False) -> int:
|
||||
"""The close of a board whose complete read finished a sweep across
|
||||
pulls: its open roles missing from `seen_ids` that also are in
|
||||
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
||||
@@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
missed may only have shifted back past the cursor when roles closed
|
||||
between pulls, so one sweep is not evidence that it is gone. A filtered
|
||||
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
||||
stays open, and stays in `missing`."""
|
||||
stays open, and stays in `missing`, and so does a remote role with
|
||||
`spare_remote`."""
|
||||
key = f"{source}:missing:{sub.target}"
|
||||
try:
|
||||
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
||||
except (TypeError, ValueError):
|
||||
earlier = set()
|
||||
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)""" if filtered else ""
|
||||
acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
|
||||
with store.transaction() as conn:
|
||||
missed = [row[0] for row in conn.execute(
|
||||
"""SELECT source_id FROM roles
|
||||
@@ -365,16 +536,20 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||
failed: when the failures began, and what the latest one said."""
|
||||
"""`target -> (since, reason)` for every `ats` board and every search
|
||||
whose last fetch failed: when the failures began, and what the latest
|
||||
one said. No board host is also a search provider, so a board's target
|
||||
(`host:slug`) and a search's (`provider:name`) never collide in one
|
||||
dict."""
|
||||
out: dict[str, tuple[str, str]] = {}
|
||||
for key, value in store.state_items(FAILING).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
for prefix in (FAILING, SEARCH_FAILING):
|
||||
for key, value in store.state_items(prefix).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(prefix):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
return out
|
||||
|
||||
|
||||
|
||||
+13
-7
@@ -9,6 +9,8 @@ good for a given search. For each agent:
|
||||
column filled for at least one, is gated ("none of
|
||||
country DK / remote remote")
|
||||
prefer gates `weight` when the role matches, 0 when it does not
|
||||
exclude gates a role that matches one of the values keeps its row, with
|
||||
score 0 and the value ("title account executive")
|
||||
|
||||
A gate rejects on evidence, never on absence: a role whose column is empty
|
||||
is not gated by it. `extract` is what fills an empty column, and it reads
|
||||
@@ -29,7 +31,8 @@ roles out of extract's reach for good.
|
||||
|
||||
Every term is 0..1. The score is the weighted sum over the sum of the weights
|
||||
(the agent's four and its prefer gates'), so it stays 0..1. A favourite
|
||||
company (`affinity = 2`) skips the `any` group but not the `require` gates, and
|
||||
company (`affinity = 2`) skips the `any` group but not the `require` or
|
||||
`exclude` gates, and
|
||||
is scored like any other company with its affinity term at 1: nothing puts
|
||||
its roles above a better fit. A hidden one (`-1`) is not scored at all. A
|
||||
role with no vector is scored without `fit` and `taste`, unless `fit` is all
|
||||
@@ -156,14 +159,17 @@ def _reason(gate: Gate) -> str:
|
||||
|
||||
|
||||
def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
|
||||
"""Why this role is gated, as reasons: every require gate it fails, and
|
||||
the `any` group when it matches none of them. A gate with an empty column
|
||||
(state -1) neither passes nor fails, so the group fails only when some
|
||||
member had evidence. A favourite company's roles skip the `any` group but
|
||||
not the require gates, so a star widens a search to a company's roles
|
||||
within e.g. Europe, not to the whole world."""
|
||||
"""Why this role is gated, as reasons: every require gate it fails, every
|
||||
exclude gate it matches, and the `any` group when it matches none of them.
|
||||
A gate with an empty column (state -1) neither passes nor fails, so the
|
||||
group fails only when some member had evidence. A favourite company's
|
||||
roles skip the `any` group but not the require and exclude gates, so a
|
||||
star widens a search to a company's roles within e.g. Europe, not to the
|
||||
whole world, and not to the titles the agent leaves out."""
|
||||
states = list(zip(gates, candidate.gates, strict=True))
|
||||
reasons = [_reason(g) for g, state in states if g.mode == "require" and state == 0]
|
||||
reasons += [f"{g.field} {g.values[state - 1]}" for g, state in states
|
||||
if g.mode == "exclude" and state > 0]
|
||||
group = [(g, state) for g, state in states if g.mode == "any"]
|
||||
if candidate.affinity != 2 and group and not any(state == 1 for _, state in group) \
|
||||
and any(state == 0 for _, state in group):
|
||||
|
||||
+6
-3
@@ -11,11 +11,14 @@ finders ask it (`extract.extract_task`, `summarise.summarise_task`).
|
||||
|
||||
`enrich` (reading the best roles on their own pages, `app/jobs/enrich.py`)
|
||||
asks no model and is on by default (`enrich.enabled`); its switch is
|
||||
`stage:enrich`, set the same way through `PUT /api/stages/enrich`.
|
||||
`stage:enrich`, set the same way as the others: by the Runs page
|
||||
(`PUT /api/stages/enrich`) and by `app stages enrich on|off`.
|
||||
|
||||
`describe` is what `GET /api/stages` and `app stages` show: for each stage
|
||||
with a switch, the model stages and `enrich`, whether it is on, where that comes from, the model it uses, and who
|
||||
would answer.
|
||||
would answer. The model is the one a call is made with: `agent.model` when
|
||||
set, since it overrides the per-task model everywhere (`desk/modeltask.py`),
|
||||
else the stage's own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -98,7 +101,7 @@ def describe_one(config: Config, store: Store, name: str) -> Stage:
|
||||
backend, said = answer(config)
|
||||
return Stage(name=name, on=stage_on(config, store, name),
|
||||
source="config" if stored(store, name) is None else "stored",
|
||||
model=getattr(getattr(config, name), "model", ""), backend=backend,
|
||||
model=config.agent.model or getattr(config, name).model, backend=backend,
|
||||
answer=said)
|
||||
|
||||
|
||||
|
||||
+12
-5
@@ -1167,21 +1167,26 @@ class Application(Model):
|
||||
|
||||
#: The role columns a gate can name. `company` matches on the company key;
|
||||
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
|
||||
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are
|
||||
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; `title`
|
||||
#: matches a value as whole words anywhere in the title, folded as `title_key`
|
||||
#: folds it, so `account executive` finds "Enterprise Account Executive, UK"
|
||||
#: and `data engineer` does not find "Data Engineering Manager"; the rest are
|
||||
#: case-insensitive equality, except `salary_min` (at least the value) and
|
||||
#: `languages` (every language the role requires is one of the values).
|
||||
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
|
||||
"currency", "permit", "languages", "company"]
|
||||
"currency", "permit", "languages", "company", "title"]
|
||||
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
|
||||
|
||||
GateMode = Literal["require", "any", "prefer"]
|
||||
GateMode = Literal["require", "any", "prefer", "exclude"]
|
||||
|
||||
|
||||
class Gate(Model):
|
||||
"""One condition of an agent. `require` gates a role that fails it (the
|
||||
row is kept with the reason); the `any` gates together are one group a
|
||||
role must match at least one of ("in Denmark, or remote"); `prefer` adds
|
||||
`weight` to its score when it matches. A gate rejects on evidence only: a
|
||||
`weight` to its score when it matches; `exclude` gates a role that matches
|
||||
any of its values ("title account executive"), a favourite's too. A gate
|
||||
rejects on evidence only: a
|
||||
role whose column is empty is not rejected, since `extract` fills the
|
||||
column later for the roles that rank, and only for those."""
|
||||
|
||||
@@ -1297,7 +1302,9 @@ class Agent(Model):
|
||||
class Candidate(Model):
|
||||
"""What scoring reads about one open role for one agent: its posted
|
||||
date (for `recency`), its company's affinity for that agent, and per
|
||||
gate `1` match, `0` no match, `-1` the column is empty."""
|
||||
gate `1` match, `0` no match, `-1` the column is empty. An `exclude`
|
||||
gate's match is the position of the first value it matched, from 1, so
|
||||
the reason can name it."""
|
||||
|
||||
uid: str
|
||||
posted_at: str
|
||||
|
||||
+13
-4
@@ -39,8 +39,8 @@
|
||||
"HU", "IE", "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
|
||||
"IS", "LI", "NO", "GB", "CH", "EU"];
|
||||
const FIELDS = ["place", "country", "remote", "seniority", "function", "salary_min",
|
||||
"currency", "permit", "languages", "company"];
|
||||
const MODES = [["require", "must"], ["any", "one of"], ["prefer", "prefer"]];
|
||||
"currency", "permit", "languages", "company", "title"];
|
||||
const MODES = [["require", "must"], ["any", "one of"], ["prefer", "prefer"], ["exclude", "not"]];
|
||||
const HYBRID_WEIGHT = 0.05;
|
||||
|
||||
/** @type {object[]} every agent, as the server lists them */
|
||||
@@ -227,11 +227,14 @@
|
||||
const must = of("require").map(phrase);
|
||||
const prefer = of("prefer").map((g) => phrase(g).replace(/^with /, ""));
|
||||
if (where.hybrid) prefer.unshift("hybrid");
|
||||
const not = of("exclude").map(phrase);
|
||||
const sentences = [];
|
||||
const any = whereSentence(where, of("any"));
|
||||
if (any) sentences.push(any);
|
||||
if (must.length) sentences.push(`Only roles ${must.join(" and ")}.`);
|
||||
if (prefer.length) sentences.push(`Prefers ${prefer.join("; ")}.`);
|
||||
if (where.titles) sentences.push(`Leaves out titles with ${orList(where.titles.values.map((t) => `“${t}”`))}.`);
|
||||
if (not.length) sentences.push(`Leaves out roles ${not.join(" or ")}.`);
|
||||
return sentences.join(" ");
|
||||
}
|
||||
|
||||
@@ -497,7 +500,7 @@
|
||||
// that is Europe and the countries line exactly, and only with Remote
|
||||
// ticked; any other must on country is a row, and Remote reads anywhere.
|
||||
function distribute(gates) {
|
||||
const where = { countries: null, places: null, remote: null, europe: null, hybrid: null };
|
||||
const where = { countries: null, places: null, remote: null, europe: null, hybrid: null, titles: null };
|
||||
const rest = [];
|
||||
const musts = [];
|
||||
const plain = (g) => g.weight === undefined || g.weight === 1;
|
||||
@@ -512,6 +515,7 @@
|
||||
else if (g.field === "remote" && g.mode === "any" && plain(g) && only(g, "remote")) slot = "remote";
|
||||
else if (g.field === "place" && g.mode === "any" && plain(g)) slot = "places";
|
||||
else if (g.field === "remote" && g.mode === "prefer" && only(g, "hybrid")) slot = "hybrid";
|
||||
else if (g.field === "title" && g.mode === "exclude" && plain(g)) slot = "titles";
|
||||
if (slot && !where[slot]) where[slot] = g;
|
||||
else rest.push(g);
|
||||
}
|
||||
@@ -561,7 +565,9 @@
|
||||
const { where, rest } = distribute(row ? row.gates : []);
|
||||
const countries = countryInput(where.countries ? where.countries.values : []);
|
||||
const places = wordInput(where.places ? where.places.values : [], "Add a town");
|
||||
const titles = wordInput(where.titles ? where.titles.values : [], "account executive");
|
||||
const terms = wordInput(row ? row.fetch_terms || [] : [], "machine learning, data science");
|
||||
$("#a-titles", f).replaceWith(titles.node);
|
||||
$("#a-terms", f).replaceWith(terms.node);
|
||||
$("#a-countries", f).replaceWith(countries.node);
|
||||
$("#a-towns", f).replaceWith(places.node);
|
||||
@@ -601,7 +607,8 @@
|
||||
});
|
||||
|
||||
// The gates the Where block says, in the order the starter agents have
|
||||
// them: the must on country, the lines, then hybrid.
|
||||
// them: the must on country, the lines, then hybrid; and the titles left
|
||||
// out, the block under it.
|
||||
function whereGates() {
|
||||
const out = [];
|
||||
const within = countries.values();
|
||||
@@ -615,6 +622,8 @@
|
||||
if ($("#a-hybrid", f).checked) {
|
||||
out.push({ field: "remote", values: ["hybrid"], mode: "prefer", weight: Number($("#a-hybrid-weight", f).value) });
|
||||
}
|
||||
const left = titles.values();
|
||||
if (left.length) out.push({ field: "title", values: left, mode: "exclude" });
|
||||
return out;
|
||||
}
|
||||
// The gates Save sends: the Where block's, then the rows. The sentence
|
||||
|
||||
+2
-1
@@ -1223,7 +1223,8 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
border-radius: .3rem; padding: .35rem .5rem;
|
||||
}
|
||||
/* The fieldsets that hold rows: a column, the Add button under the rows. */
|
||||
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.terms,
|
||||
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
|
||||
.agent-form fieldset.terms,
|
||||
.agent-form fieldset.rules {
|
||||
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
|
||||
border: 1px solid var(--line); border-radius: .3rem;
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
const company = companies.get(application.company_key);
|
||||
cell(tr, "Company", company ? company.name : application.company_key);
|
||||
const select = statusSelect(application.status);
|
||||
select.addEventListener("change", () => setStatus(application.id, select));
|
||||
select.addEventListener("change", () => setStatus(application.id, select, application.status));
|
||||
cell(tr, "Status", select);
|
||||
const saved = cell(tr, "Saved", savedOn(application.created_at));
|
||||
saved.title = application.created_at;
|
||||
@@ -69,7 +69,7 @@
|
||||
count(applications.length, "tracked application");
|
||||
}
|
||||
|
||||
async function setStatus(id, select) {
|
||||
async function setStatus(id, select, was) {
|
||||
select.disabled = true;
|
||||
try {
|
||||
await api(`/api/applications/${encodeURIComponent(id)}/status`, {
|
||||
@@ -77,6 +77,7 @@
|
||||
});
|
||||
render(await api("/api/applications"));
|
||||
} catch (err) {
|
||||
select.value = was; // the change was refused, so the row still holds `was`
|
||||
select.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,8 +217,8 @@
|
||||
// Replacing a different board closes its roles (see set_board in
|
||||
// app/jobs/sources.py); ask first, naming both boards and the count,
|
||||
// the same as Detach and a removed search already do.
|
||||
if (company.board && company.board !== board
|
||||
&& !window.confirm(`Replace ${company.name}'s board (${company.board}) `
|
||||
if (company.target && company.target !== board
|
||||
&& !window.confirm(`Replace ${company.name}'s board (${company.target}) `
|
||||
+ `with ${board}? Its ${count(company.open_roles, "open role")} close.`)) {
|
||||
return;
|
||||
}
|
||||
@@ -429,8 +429,8 @@
|
||||
// editor does, so it asks first too.
|
||||
const existing = [...companies.values()]
|
||||
.find((c) => c.name.toLowerCase() === name.toLowerCase());
|
||||
if (existing && existing.board && existing.board !== board
|
||||
&& !window.confirm(`Replace ${existing.name}'s board (${existing.board}) `
|
||||
if (existing && existing.target && existing.target !== board
|
||||
&& !window.confirm(`Replace ${existing.name}'s board (${existing.target}) `
|
||||
+ `with ${board}? Its ${count(existing.open_roles, "open role")} close.`)) {
|
||||
return;
|
||||
}
|
||||
|
||||
+11
-4
@@ -109,10 +109,17 @@
|
||||
...(stage.model ? [el("span", "badge", stage.model)] : []));
|
||||
row.append(name, stageSwitch(stage, () => renderStages(stages)));
|
||||
const sub = el("span", "sub");
|
||||
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
|
||||
const agents = el("a", "", "Search agents");
|
||||
agents.href = "/settings/agents";
|
||||
sub.append(agents, stage.source === "config"
|
||||
if (stage.name === "enrich") {
|
||||
/* Global, unlike the model stages: `enrich.top` and `enrich.min_score`. */
|
||||
sub.append(`${DOES[stage.name]}. How many it reads is set for every agent alike by `
|
||||
+ "enrich.top and enrich.min_score in config.yaml");
|
||||
} else {
|
||||
sub.append(`${DOES[stage.name]}. How deep it reads is set per search agent, under `);
|
||||
const agents = el("a", "", "Search agents");
|
||||
agents.href = "/settings/agents";
|
||||
sub.append(agents);
|
||||
}
|
||||
sub.append(stage.source === "config"
|
||||
? ". Following config.yaml until this switch is used."
|
||||
: ".");
|
||||
row.appendChild(sub);
|
||||
|
||||
+3
-2
@@ -864,7 +864,8 @@
|
||||
is the only time this page ever sees a role's `summary` or
|
||||
`description` directly. */
|
||||
function excerptOf(role) {
|
||||
const text = (role.summary || role.description || "").replace(/\s+/g, " ").trim();
|
||||
/* The server's order (`_EXCERPT_SOURCE`): summary, description, teaser. */
|
||||
const text = (role.summary || role.description || role.teaser || "").replace(/\s+/g, " ").trim();
|
||||
if (text.length <= 200) return text;
|
||||
const cut = text.slice(0, 200);
|
||||
const space = cut.lastIndexOf(" ");
|
||||
@@ -1458,7 +1459,7 @@
|
||||
async function fetchText(button) {
|
||||
const uid = state.readingUid;
|
||||
const role = await busy(button, "Fetching…",
|
||||
() => api(`/api/roles/${encodeURIComponent(uid)}/text`, { method: "POST" }));
|
||||
() => api(roleUrl(uid).replace(/(\?|$)/, "/text$1"), { method: "POST" }));
|
||||
if (role) merge(role);
|
||||
}
|
||||
|
||||
|
||||
@@ -176,7 +176,7 @@
|
||||
text: n.text,
|
||||
fix: n.fix,
|
||||
detail: n.detail,
|
||||
link: { label: "all messages", href: "/messages" },
|
||||
link: { label: "all messages", href: "/settings/sources#messages" },
|
||||
};
|
||||
if (PROBLEM.has(n.level)) t.count = n.count;
|
||||
if (n.level === "good") t.action = { label: "reload", run: () => location.reload() };
|
||||
|
||||
+16
-4
@@ -219,6 +219,13 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
if gate.field == "place":
|
||||
match, params = _places_sql(values, gaz, table="r")
|
||||
return "r.place = ''", match, params
|
||||
if gate.field == "title":
|
||||
# Whole words of the folded title: " account executive " inside
|
||||
# " enterprise account executive uk ". A fold drops `%` and `_`, so a
|
||||
# value cannot smuggle a LIKE wildcard in.
|
||||
words = [f"% {title_key(v)} %" for v in values if title_key(v)]
|
||||
match = " OR ".join("(' ' || r.title_key || ' ') LIKE ?" for _ in words) or "0"
|
||||
return "r.title_key = ''", f"({match})", words
|
||||
if gate.field == "company":
|
||||
keys = [company_key(v) for v in values]
|
||||
return ("r.company_key = ''",
|
||||
@@ -1830,10 +1837,15 @@ class Store:
|
||||
params: list[Any] = []
|
||||
gaz = self.gazetteer()
|
||||
for n, gate in enumerate(gates):
|
||||
empty, match, values = _gate_sql(gate, gaz)
|
||||
columns.append(
|
||||
f"CASE WHEN {empty} THEN -1 WHEN {match} THEN 1 ELSE 0 END AS g{n}")
|
||||
params += values
|
||||
# An exclude gate says which of its values matched: one WHEN each.
|
||||
parts = ([gate.model_copy(update={"values": [v]}) for v in gate.values]
|
||||
if gate.mode == "exclude" else [gate])
|
||||
whens = []
|
||||
for k, part in enumerate(parts, 1):
|
||||
empty, match, values = _gate_sql(part, gaz)
|
||||
whens.append(f"WHEN {match} THEN {k}")
|
||||
params += values
|
||||
columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}")
|
||||
select = ", ".join(["r.uid", "r.posted_at",
|
||||
"COALESCE(ac.affinity, 0) AS affinity", *columns])
|
||||
rows = self._rows(
|
||||
|
||||
@@ -49,6 +49,14 @@
|
||||
<span class="hint">This one ranks rather than filters: a hybrid role gains a little.</span>
|
||||
<p id="a-where-sum" class="where-sum" role="status"></p>
|
||||
</fieldset>
|
||||
<fieldset class="titles">
|
||||
<legend>Which titles to leave out?</legend>
|
||||
<div class="where-line">
|
||||
<span>Leave out titles with</span>
|
||||
<div id="a-titles"></div>
|
||||
</div>
|
||||
<span class="hint">Separate phrases with a comma or Enter. A phrase matches whole words anywhere in the title, whatever the case: <code>account executive</code> takes "Enterprise Account Executive, UK", and <code>data engineer</code> leaves "Data Engineering Manager" alone. A shorter phrase takes more: <code>sales</code> takes Sales Engineer too. It applies to liked companies as well. A role left out keeps its row, and Show gated on Roles lists it with the phrase that took it.</span>
|
||||
</fieldset>
|
||||
<fieldset class="terms">
|
||||
<legend>What big boards send</legend>
|
||||
<div class="where-line">
|
||||
|
||||
+30
-8
@@ -98,7 +98,13 @@ def _feed_filters(
|
||||
raise HTTPException(422, f"{name} is one of {', '.join(RUNGS)}, not {rung!r}.")
|
||||
if track and track not in TRACKS:
|
||||
raise HTTPException(422, f"track is one of {', '.join(TRACKS)}, not {track!r}.")
|
||||
levels = levels_between(level_from, level_to, track) if level_from or level_to or track else []
|
||||
levels: tuple[str, ...] = ()
|
||||
if level_from or level_to or track:
|
||||
# The store reads an empty list as "no filter", so a range that holds no
|
||||
# level must be refused here or it would list every role.
|
||||
levels = levels_between(level_from, level_to, track)
|
||||
if not levels:
|
||||
raise HTTPException(422, "That seniority range holds no level.")
|
||||
return {
|
||||
"q": q or None, "company": company or None, "place": place or None,
|
||||
"function": function or None, "remote": remote or None, "days": days,
|
||||
@@ -359,14 +365,21 @@ def _known_role(store: Store, uid: str) -> Role:
|
||||
|
||||
|
||||
@router.post("/roles/{uid}/text")
|
||||
def fetch_text(uid: str, store: StoreDep) -> Role:
|
||||
def fetch_text(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole:
|
||||
"""Fetch the posting page once and keep it as plain text, for a role whose
|
||||
description is a teaser (`jobs.posting.fetch_posting`); a Jobindex role
|
||||
is read from the page its "Se jobbet" link leads to."""
|
||||
is read from the page its "Se jobbet" link leads to. The role comes back as
|
||||
`agent` sees it, as `GET /api/roles/{uid}` does: the page merges the answer
|
||||
into the open role, and one without the vote would blank the reader's."""
|
||||
agent_id = _agent_id(store, agent)
|
||||
try:
|
||||
return jobs.posting.fetch_posting(store, uid)
|
||||
jobs.posting.fetch_posting(store, uid)
|
||||
except jobs.posting.PostingError as exc:
|
||||
raise HTTPException(exc.status, str(exc)) from exc
|
||||
role = store.role(uid, agent_id)
|
||||
if role is None:
|
||||
raise HTTPException(404, f"no such role: {uid}")
|
||||
return role
|
||||
|
||||
|
||||
@router.post("/roles/{uid}/summarise", status_code=202)
|
||||
@@ -436,12 +449,17 @@ class CompanyOut(BaseModel):
|
||||
with no board shown that is not `from_search`, and `hand_reason` says why
|
||||
it is read by hand (`app.models.HAND_REASONS`). `all_countries` and
|
||||
`all_roles` are the company's two fetch overrides: its big boards read
|
||||
with the agents' wide reach, and with no role-type narrowing."""
|
||||
with the agents' wide reach, and with no role-type narrowing. `target` is
|
||||
the attached board in full, `host:slug`, where `board` is its host alone;
|
||||
the page compares it with the board typed to know whether a replacement
|
||||
closes anything. `ats_slug` cannot stand in for it, since the careers site
|
||||
may run on another host than the board."""
|
||||
|
||||
key: str
|
||||
name: str
|
||||
notes: str = ""
|
||||
board: str
|
||||
target: str = ""
|
||||
subscription_id: int | None = None
|
||||
on: bool | None = None
|
||||
affinity: int
|
||||
@@ -500,6 +518,7 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]:
|
||||
sub = subs.get(sub_id) if sub_id is not None else None
|
||||
from_search = jobs.sources.known_from_search_only(c, ever_boarded)
|
||||
out.append(CompanyOut(key=c.key, name=c.name, notes=c.notes, board=host,
|
||||
target=target,
|
||||
subscription_id=sub_id, on=on, affinity=c.affinity,
|
||||
open_roles=counts.get(c.key, 0), ats=c.ats,
|
||||
ats_slug=c.ats_slug, careers_url=c.careers_url,
|
||||
@@ -865,9 +884,12 @@ def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]:
|
||||
sub = store.subscription(sub_id)
|
||||
if sub is None or sub.source != "ats":
|
||||
raise HTTPException(404, f"no such board: {sub_id}")
|
||||
codes = sorted({c.strip().upper() for c in body.countries if c.strip()})
|
||||
if unknown := [c for c in codes if not places.known(f"country:{c}") or c == "EU"]:
|
||||
raise HTTPException(422, f"Not a country code: {', '.join(unknown)}.")
|
||||
named = sorted({c.strip() for c in body.countries if c.strip()})
|
||||
tokens = {c: places.parse(f"country:{c}") for c in named}
|
||||
if unknown := [c for c, t in tokens.items() if not places.known(str(t)) or t.value == "EU"]:
|
||||
raise HTTPException(422, f"Not a country: {', '.join(unknown)}.")
|
||||
# The ISO code is what the reach compares against, not the name or alias typed.
|
||||
codes = sorted({t.value for t in tokens.values()})
|
||||
if body.mode == "own" and not (codes or body.remote):
|
||||
raise HTTPException(422, "Name a country or remote, or take the agents' reach.")
|
||||
params = {k: v for k, v in sub.params.items() if k not in jobs.reach.OVERRIDE_KEYS}
|
||||
|
||||
Reference in new issue
Block a user