Compare commits

...
16 Commits
Author SHA1 Message Date
WolfandClaude Fable 5.1 920457d326 Record that the form's gates are the four parts alone, and strike the two decided items from open.md
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:34:12 +02:00
WolfandClaude Fable 5.1 2262cda8f1 Merge the agent form without Advanced rules: the gates are the four parts of the fetch gate, a stored gate the form cannot say is named and kept (wp-advanced)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:34:11 +02:00
WolfandClaude Fable 5.1 1913101061 Drop Advanced › Other rules from the agent form, so an agent's gates are the fetch gate's four parts: a stored gate the parts cannot say is named in a note under the fetch gate and sent back unchanged on Save, and the card still says it in words
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:33:34 +02:00
WolfandClaude Fable 5.1 849adb197d Close by age what an incomplete board read did not list and what a filtered read spares, so no role stays open for good, as ADR 1.3 has it
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:27:09 +02:00
WolfandClaude Fable 5.1 136047023e Merge the runner dropping and counting rows without an id, title or link, so no reader skips a listing (wp-rows)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:05:15 +02:00
WolfandClaude Fable 5.1 911dfe8584 Let the runner drop and count a row without an id, a title or a link, as ADR Plugins 3.8 has it, in one place for whole, paged and search reads: the count is stamped per target, logged, named on the board's row, and over 10% of the rows fails the read; the 28 readers hand every listing over instead of skipping it
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:04:33 +02:00
WolfandClaude Fable 5.1 8b875b377b Record the world read, the remote mark, the one-country fold, the judge's languages and the end of must-haves in decisions.md, and cut open.md to what is still open
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:03:59 +02:00
WolfandClaude Fable 5.1 c6730062fd Pin that Phenom is asked for the agents' world when they want remote, with or without All locations
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:03:09 +02:00
WolfandClaude Fable 5.1 49929372c7 Mark a role whose title or text says remote while no field does, in prep with no model, and let it meet a gate on remote until extract has read it (migration 35, roles.remote_text)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 14:02:46 +02:00
WolfandClaude Fable 5.1 f141cffc3c Pin the per-office fold within one country: two offices in two countries are two roles, and a role in several Danish cities leads with the agent's town
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 13:58:41 +02:00
WolfandClaude Fable 5.1 02ea191b53 Merge the removal of must-haves: the judge reads CV, languages, prompts, likes and dislikes, the CV match is uncapped, migration 34 drops the columns (wp-musts)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 13:57:16 +02:00
WolfandClaude Fable 5.1 1976f6a353 Ask a host that cannot filter on remote for the agents' world where it takes the world in one read, and keep the per-office fold within one country, as ADR 1.2 and 1.4 have it
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 13:56:01 +02:00
WolfandClaude Fable 5.1 2159c7925f Remove the must-haves the form dropped on 2026-10-03: the agent's must text, the judge's verdicts and the career cap they set, with migration 34 dropping their columns and the judge's format at 6 so every role is judged again
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 13:55:59 +02:00
WolfandClaude Fable 5.1 0c4f3c9bbe Add to the flow ADR what the review found missing: the gate's unknown location, prep, the remote read on hosts without a remote filter, the per-office fold within one country, the embedding and its calibration, the models and their inputs, the feed's read state and Show gated, and must-haves taken out
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 13:49:46 +02:00
WolfandClaude Fable 5.1 38c31c830b Rewrite the plugins ADR to the rules the runner needed and the target where the code falls short: cursors and stalls, the runner dropping and counting bad rows, the guard thresholds, remote not asked, a search never closing by list, provider params declared, what the runner keeps per target
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 13:36:15 +02:00
WolfandClaude Fable 5.1 4a9bcb2b1c Let the judge read the person's languages beside the CV, as ADR 4.2 has it: a language a posting requires above the level spoken lowers skills and is a gap, and a change to them re-judges the cut
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 13:33:32 +02:00
94 changed files with 1245 additions and 869 deletions

No files matched your search

+13 -3
View File
@@ -111,6 +111,12 @@ def same_posting(a: Copy, b: Copy) -> bool:
return not ta or not tb or bool(ta & tb)
def _same_country(a: Copy, b: Copy) -> bool:
"""Whether two rows can be in one country: equal, or either unknown."""
ca, cb = _country(a), _country(b)
return not ca or not cb or ca == cb
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
"""The sort key that picks the kept copy of a group, smallest first: a
board over a search provider, then one with a description, then the
@@ -193,12 +199,16 @@ def fold(rows: Iterable[Copy]) -> Folded:
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
again, in `keep_order`: a row joins the first group whose members are all
of its host, `similar` to it and remote as it is or not. Only a row with
of its host, in its country (equal, or either unknown, as `same_posting`
reads it), `similar` to it and remote as it is or not. Only a row with
a link can be folded, as its link is what the kept role keeps. Every row
of a merged group points at its first row, the rows that pointed at a
folded one included.
A remote listing and an office listing stay two roles: the kept role
One country, as ADR Fetch prep and score 1.4 has it: a job posted once
per office in several countries is several roles, one per country, which
keeps a different opening with an alike text (a federal sales role in
the US and one in Australia) from merging. A remote listing and an office listing stay two roles: the kept role
takes a copy's places and not its remote, so the fetch gate, which reads
a remote role by its area, would reject the remote listing with an
office role elsewhere, and the office one with a remote role elsewhere."""
@@ -216,7 +226,7 @@ def fold(rows: Iterable[Copy]) -> Folded:
for row in sorted(part, key=keep_order):
for group in groups:
if all(m.host == row.host and m.remote == row.remote
and similar(m.text, row.text) for m in group):
and _same_country(m, row) and similar(m.text, row.text) for m in group):
group.append(row)
break
else:
+25 -70
View File
@@ -7,18 +7,14 @@ to three strengths and gaps. A model's rating is an estimate, so the scale is
coarse. The total is `app.models.cv_score`, the three numbers under the
agent's `cv_*` weights times 10, so 0 to 100, worked out when a role is served
and not stored, so the weights are no part of `input_key`; it is what the
feed's CV match sorts by. The model also reports a verdict on each of the
agent's must-haves (one per line of its must text, numbered M1..Mn in the
prompt); it does not apply them. The rule is code's: a "broken" one makes
career count as at most `app.models.BROKEN_CAP` (4), when the role is
served. Culture and location are left out: the first is guessed from ad text,
the second is what the fetch gate does.
feed's CV match sorts by. Culture and location are left out: the first is
guessed from ad text, the second is what the fetch gate does.
The work is pairs, `(agent, role)`: an enabled agent with a CV, with or
without a prompt, and an open role in its cut (`scores.cut`, see `app.jobs.score`: the best by
fit, the best by CV closeness that those leave out, and the liked and saved)
with some text, that has no judgement yet, or one read from another CV,
prompts, like, avoid or must text or role text than today's (`input_key`).
prompts, languages, like or avoid text or role text than today's (`input_key`).
The cut is the only selection: the pull judges it after extract and the
rescore that follows, so it reads the cut as recomputed. A pair is named by the composite key `"<agent_id>:<uid>"`,
which is what `desk.modeltask` hands round as a uid, so `desk/` is untouched.
@@ -51,9 +47,7 @@ from pipeline import NULL, RunContext, Task, job
from pipeline.task import Batch
from app.config import Config
from app.jobs import stages
from app.models import (
LANGUAGE_LEVELS, MUST_VERDICTS, Agent, CvLanguage, cv_languages, must_lines,
)
from app.models import LANGUAGE_LEVELS, Agent, CvLanguage, _language_name, cv_languages
from app.storage import Store
#: How much of a posting goes in one item: summarise's 8000, since the
@@ -66,22 +60,28 @@ EXCERPT = 8000
#: 3: what the person likes (`like_text`) beside what they do not, and the
#: career rubric reading both.
#: 4: the three numbers 0 to 10 rather than 0 to 100.
FORMAT = 4
#: 5: the person's languages, read into skills.
#: 6: no must-haves: the profile is the CV, languages, likes and dislikes.
FORMAT = 6
#: How many strengths and gaps are kept, and how long each may be.
MOST, LONGEST = 3, 300
SYSTEM = """You judge job postings for one person: how well each fits their CV and what they want.
Below the rules is the person: their CV, what they want, what they like and
what they do not like (the work, the workplace, situations), and their
must-haves and deal-breakers when they gave any. Each item is one posting with a key, its title, company, place
and text. Answer one judgement per item.
Below the rules is the person: their CV, the languages they speak and how
well, what they want, what they like and what they do not like (the work,
the workplace, situations). Each item is one posting with a key, its title,
company, place and text. Answer one judgement per item.
Score three dimensions, each a whole number from 0 to 10:
- skills: 8-10 the core requirements are the person's primary skills; 6-7
most requirements match, with 1-2 gaps that are learnable; 4-5 a partial
match, significant upskilling needed; 0-3 a fundamental mismatch.
match, significant upskilling needed; 0-3 a fundamental mismatch. A
language the posting requires counts as a requirement: one the person
speaks below the level asked, or not at all when their languages are
listed, lowers skills and is a gap. A language the posting does not
mention changes nothing.
- experience: match on the function and nature of the work performed, not the
literal job title, and on its level. 8-10 direct experience in the same
domain and role type; 6-7 related, with transferable skills clear; 4-5
@@ -93,16 +93,6 @@ Score three dimensions, each a whole number from 0 to 10:
whether the tasks and the setting would energise them, not only whether
they could do them.
Their must-haves and deal-breakers are numbered M1, M2 and so on. Give a
verdict on every one of them, in `musts`, for each item:
- "met": the posting text says it is met.
- "broken": only when the posting text clearly contradicts it.
- "unclear": the text does not say. Never infer from silence: a posting that
is silent on a must-have is "unclear", not "broken".
Judge a must-have by its own words, not by how well the role fits otherwise.
Answer musts as [] when there are none. The verdicts do not change the three
scores, which follow their rubrics; name a broken must-have as a gap.
Then 1 to 3 strengths and 1 to 3 gaps, each one short sentence grounded in
the posting text. State the gaps honestly: a prestigious role that fits
poorly scores low.
@@ -126,21 +116,8 @@ SCHEMA: dict[str, Any] = {
"career": {"type": "integer"},
"strengths": {"type": "array", "items": {"type": "string"}},
"gaps": {"type": "array", "items": {"type": "string"}},
"musts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {"type": "string"},
"verdict": {"type": "string", "enum": list(MUST_VERDICTS)},
},
"required": ["id", "verdict"],
"additionalProperties": False,
},
},
},
"required": ["key", "skills", "experience", "career", "strengths", "gaps",
"musts"],
"required": ["key", "skills", "experience", "career", "strengths", "gaps"],
"additionalProperties": False,
},
},
@@ -181,10 +158,11 @@ def _text(row: Any) -> str:
def input_key(agent: Agent, text: str) -> str:
"""A hash of everything a judgement reads: the CV, the prompts, the
like, avoid and must texts, and the role's text as sent. A judgement stored
under another key is read again."""
languages, the like and avoid texts, and the role's text as sent. A judgement
stored under another key is read again."""
blob = json.dumps([FORMAT, agent.cv_text, [[p.name, p.text] for p in agent.prompts],
agent.like_text, agent.avoid_text, agent.must_text, text], ensure_ascii=False)
[[l.language, l.level] for l in agent.cv_languages],
agent.like_text, agent.avoid_text, text], ensure_ascii=False)
return hashlib.sha1(blob.encode()).hexdigest()[:16]
@@ -230,6 +208,9 @@ def pending(store: Store) -> int:
def system(agent: Agent) -> str:
"""The rubric, and the person as this agent describes them."""
parts = [SYSTEM, "", "The person.", "", f"<cv>\n{agent.cv_text.strip()}\n</cv>"]
if agent.cv_languages:
spoken = ", ".join(f"{_language_name(l.language)} {l.level}" for l in agent.cv_languages)
parts.append(f"\n<languages title=\"the languages they speak, and how well\">\n{spoken}\n</languages>")
wants = "\n".join(f"- {p.name}: {' '.join(p.text.split())}"
for p in agent.prompts if p.text.strip())
if wants:
@@ -240,9 +221,6 @@ def system(agent: Agent) -> str:
if agent.avoid_text.strip():
parts.append("\n<dislikes title=\"what they do not like\">\n"
f"{agent.avoid_text.strip()}\n</dislikes>")
if musts := must_lines(agent.must_text):
lines = "\n".join(f"M{n}: {line}" for n, line in enumerate(musts, start=1))
parts.append(f"\n<must title=\"must-haves and deal-breakers\">\n{lines}\n</must>")
return "\n".join(parts)
@@ -290,25 +268,6 @@ def _lines(value: Any) -> list[str]:
return [line for line in found if line][:MOST]
def _musts(agent: Agent | None, value: Any) -> list[dict[str, str]]:
"""The verdicts on the agent's must-haves as they are stored: one
`{id, text, verdict}` per line of its must text, in order. A must-have
the model left out, or gave a verdict outside "met", "broken", "unclear",
is "unclear": nothing is broken by silence. An id that is not one of the
agent's is ignored, the first answer for an id counts."""
said: dict[str, str] = {}
for one in value if isinstance(value, list) else []:
if isinstance(one, dict):
said.setdefault(str(one.get("id") or "").strip().upper(),
str(one.get("verdict") or "").strip().lower())
found = []
for n, line in enumerate(must_lines(agent.must_text if agent else ""), start=1):
verdict = said.get(f"M{n}", "")
found.append({"id": f"M{n}", "text": line,
"verdict": verdict if verdict in MUST_VERDICTS else "unclear"})
return found
def apply_batch(store: Store, answer: dict[str, Any],
batch: dict[str, Any] | None = None,
config: Config | None = None) -> JudgeResult:
@@ -317,7 +276,6 @@ def apply_batch(store: Store, answer: dict[str, Any],
all three numbers; the pair stays waiting."""
keys: dict[str, str] = dict((batch or {}).get("keys") or {})
stats = JudgeResult()
agents: dict[int, Agent | None] = {}
for one in answer.get("judgements") or []:
key = str(one.get("key") or "")
pair = pair_of(key)
@@ -326,11 +284,8 @@ def apply_batch(store: Store, answer: dict[str, Any],
stats.rejected.append(key or "?")
continue
skills, experience, career = (int(m) for m in marks if m is not None)
if pair[0] not in agents:
agents[pair[0]] = store.agent(pair[0])
if not store.set_judgement(pair[0], pair[1], keys.pop(key), skills, experience, career,
_lines(one.get("strengths")), _lines(one.get("gaps")),
_musts(agents[pair[0]], one.get("musts"))):
_lines(one.get("strengths")), _lines(one.get("gaps"))):
stats.rejected.append(key)
continue
stats.judged += 1
+17 -3
View File
@@ -10,7 +10,10 @@ both; nothing here decides how a board is read, so the words never disagree
with the pull as long as they read the keys the pull writes.
A word with a number carries it after a colon ("without text: 12"), and the
filter matches the phrase before the colon (`WORDS`).
filter matches the phrase before the colon (`WORDS`). One word is outside
`WORDS`: the listings the last read dropped for want of an id, a title or a
link ("3 listings dropped: no title or link", `_runner.valid`), shown on the
board's row and not offered by the filter.
"""
from __future__ import annotations
@@ -49,7 +52,7 @@ WORDS: dict[str, str] = {
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
"narrowed", "detailfailed")
"narrowed", "detailfailed", "dropped")
class Texts(NamedTuple):
@@ -84,7 +87,8 @@ def texts(store: Store) -> dict[int, Texts]:
def words(module: ModuleType, state: Mapping[str, str], target: str,
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
line: str = "", counts: Texts | None = None) -> list[str]:
"""The limits of one board, in the order of `WORDS`, those that apply.
"""The limits of one board, in the order of `WORDS`, those that apply, the
rows the last read dropped after "read incomplete".
`state` holds the board's keys (`state_of`), `size` its kept size,
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
@@ -105,6 +109,8 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
out.append("failing")
if state.get(key("ok")) and not state.get(key("complete")):
out.append("read incomplete")
if dropped := _count(state.get(key("dropped"))):
out.append(f"{dropped} listing{'' if dropped == 1 else 's'} dropped: no title or link")
if state.get(key("swept")) == "1" or key("sweep") in state:
out.append("swept")
if state.get(key("filtered")) == "1":
@@ -130,6 +136,14 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
return out
def _count(value: str | None) -> int:
"""A number kept in state, 0 for none or one that does not parse."""
try:
return max(0, int(value or 0))
except ValueError:
return 0
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
"""Whether the reach leaves the board unread by place: its reader cannot
filter, or the board is under `SMALL` roles (the line's two "read whole"
+21 -14
View File
@@ -37,7 +37,12 @@ not swept clears them.
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
the roles matching a word, which says nothing of the roles it did not list,
so it closes by list nothing: a role of such a board closes once the board
has not listed it for `STALE_DAYS`, the rule searches have.
has not listed it for `STALE_DAYS`, the rule searches have. The same age
rule is the floor under every board (ADR Fetch prep and score 1.3): an
incomplete read (a ceiling, a page cap without a sweep) and a filtered read,
which spares the roles acted on and the remote ones it could not ask for,
close by age what no read has listed for `STALE_DAYS` (`_close_aged`), so no
role stays open for good.
A search never closes by its list, even one stamped complete because it
read as many results as its provider stated: a search is never a company's
@@ -365,7 +370,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
incomplete += 1
if not store.state.get(f"ats:sweep:{sub.target}"):
partial.append(sub.target)
continue
# An incomplete read closes nothing by its list; what no
# read has listed for STALE_DAYS still goes.
read = _close_aged(store, source, sub.id, json.loads(
store.state.get(f"{source}:ids:{sub.target}", "[]")), stale)
shut = read
closed += shut
before = open_before.get(sub.id, 0)
@@ -394,8 +402,9 @@ def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
clears that record and closes what it did not list, all of it unfiltered
(`Store.mark_closed`), filtered sparing the roles acted on
(`_close_unlisted`). A filtered read that could not ask for remote roles
(`remote_unasked`) spares the remote ones as well, and closes those by
age instead."""
(`remote_unasked`) spares the remote ones as well. What a filtered read
spares closes by age (`_close_aged`), as on an incomplete read
(`close_boards`)."""
state, target = store.state, sub.target
if not state.get(f"{source}:complete:{target}", ""):
return None
@@ -404,8 +413,7 @@ def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
return _close_aged(store, source, sub.id, seen_ids, stale)
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
if spare_remote else 0)
aged = _close_aged(store, source, sub.id, seen_ids, stale) if is_filtered else 0
if state.get(f"{source}:swept:{target}", ""):
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
spare_remote=spare_remote)
@@ -484,20 +492,19 @@ def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
stale: str, *, remote_only: bool = False) -> int:
stale: str) -> int:
"""Close the open roles of one board that its read did not list and that
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
writes on every listing): the rule for what a read cannot speak for. A
read narrowed by terms closes all such roles, as a search does; with
`remote_only`, a filtered read that could not ask for remote closes only
its remote roles so, sparing those acted on as its list does."""
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
writes on every listing): the rule for what a read cannot speak for, a
read narrowed by terms, an incomplete one, or what a filtered read
spares. A role acted on is not spared here: three weeks unlisted is the
one evidence a filtered or incomplete read gives."""
with store.transaction() as conn:
return conn.execute(
f"""UPDATE roles SET closed_at = ?
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
AND last_seen < ?
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
AND source_id NOT IN (SELECT value FROM json_each(?))""",
(store.now(), source, subscription_id, stale,
json.dumps(list(seen_ids)))).rowcount
+39 -8
View File
@@ -283,9 +283,18 @@ class Plan:
return self._big(sub)
def _big(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach of `sub` as a board of `SMALL` roles or more."""
"""The reach of `sub` as a board of `SMALL` roles or more. A host
that cannot filter on remote while the agents want remote roles is
asked for the wide reach, the agents' world (ADR Fetch prep and
score 1.2): asked for "DK, SE" alone it would never list a remote
role filed under Germany, and the gate sorts the rest out. Only
where the host takes the world in one read, a list of countries or
their ids: on a host that takes one country a request, Europe is
the whole board, so the location read stays and remote is not asked."""
if company_key(sub.company or sub.label) in self.all_countries:
return self.wide, "countries"
if self.base.remote and not _filters_remote(sub) and _takes_world(sub, self.wide):
return self.wide, "world"
return self.base, "agents"
def narrows_roles(self, sub: Subscription) -> bool:
@@ -305,15 +314,34 @@ class Plan:
if why == "unknown":
found, why = self._big(sub)
out = found.params()
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
if self.terms and why in ("countries", "agents", "world") and self.narrows_roles(sub):
out = (out or {}) | {"terms": list(self.terms)}
if why == "countries":
if why in ("countries", "world"):
out["wide"] = True
if sub.target not in self.sizes:
out = (out or {}) | {"unsized": True, "small": SMALL}
return out
def _filters_remote(sub: Subscription) -> bool:
"""Whether `sub`'s host declares a remote filter at all. A host that
declares one by ids may still lack the facet on a site, which the
runner finds out when it asks (`_runner.ask_for`)."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
return module is None or module.READS.filters.remote != "none"
def _takes_world(sub: Subscription, wide: Reach) -> bool:
"""Whether `sub`'s host can be asked for the wide reach in one read: a
country filter that takes a list (`many`, `ids`), or one value a request
with the countries within `MAX_SERIES`; never everywhere."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None or wide.everywhere or not wide.countries:
return False
reach = module.READS.filters.country
return reach in ("many", "ids") or (reach == "one" and len(wide.countries) <= _runner.MAX_SERIES)
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
"""Each board's size as kept (`ats:size:<target>`); a board never sized
has none."""
@@ -381,7 +409,7 @@ def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str
#: Why a board is read whole, as the Companies list says it.
_WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
"override": "set here", "countries": "all locations (everywhere)",
"agents": "an agent takes any place"}
"world": "the agents' world (everywhere)", "agents": "an agent takes any place"}
def _codes(codes: Iterable[str]) -> list[str]:
@@ -486,6 +514,9 @@ def line(found: Plan, sub: Subscription) -> str:
if why == "countries":
named = found.wide_names or f"{len(found.wide.countries)} countries"
return f"all locations ({named}){unasked} · {roles}"
if why == "world":
named = found.wide_names or f"{len(found.wide.countries)} countries"
return f"follows agents' world ({named}){unasked} · {roles}"
if why == "override":
return f"set here: {_places(ask)}{unasked} · {roles}"
return f"follows agents: {_places(ask)}{unasked} · {roles}"
@@ -494,14 +525,14 @@ def line(found: Plan, sub: Subscription) -> str:
what = "all locations (everywhere)" if why == "countries" else _WHOLE[why]
return f"{what} · {roles}"
return f"all places, too many countries to ask · {roles}"
if why in ("agents", "countries") and given.everywhere:
if why in ("agents", "countries", "world") and given.everywhere:
return f"{_WHOLE[why]}: read whole"
if why in ("agents", "countries", "override") and not module.READS.filters.declared():
if why in ("agents", "countries", "world", "override") and not module.READS.filters.declared():
return f"{module.LABEL} cannot filter: read whole"
reads = module.READS.filters
if why in ("agents", "countries") and reads.country == "one" \
if why in ("agents", "countries", "world") and reads.country == "one" \
and len(given.countries) > _runner.MAX_SERIES:
return "too many countries to ask: read whole"
if why in ("agents", "countries"):
if why in ("agents", "countries", "world"):
return f"{module.LABEL} cannot ask for these: read whole"
return f"{_WHOLE[why]}: read whole"
+9 -62
View File
@@ -1008,79 +1008,30 @@ def language_flag(required: list[RoleLanguage],
return "", ""
#: What career counts as at most, on the judge's 0 to 10, when a posting
#: breaks one of the agent's must-haves: the foot of the rubric's 4-5 band,
#: a decent job that does not build toward the person's goals. The model
#: reports whether each must-have is met; this cap is applied in code, when a
#: role is served (`cv_score`, `Store.judgement`, `store._cv_sql`).
BROKEN_CAP = 4
#: A must-have's verdict, as the judge answers it.
MUST_VERDICTS = ("met", "broken", "unclear")
_BULLET = re.compile(r"^\s*(?:[-*\u2022]+|\(?\d+[.)])\s+")
def must_lines(text: str) -> list[str]:
"""The agent's must-haves text read as one must-have per line: a leading
bullet (`-`, `*`, `•`) or number (`1.`, `2)`) is dropped, so are empty
lines."""
found = []
for line in text.splitlines():
line = " ".join(_BULLET.sub("", line, count=1).split())
if line:
found.append(line)
return found
def capped_career(career: int, broken: int = 0) -> int:
"""`career` as it counts: at most `BROKEN_CAP` when any must-have is
broken."""
return min(career, BROKEN_CAP) if broken else career
def cv_score(skills: int, experience: int, career: int,
weights: Weights | None = None, broken: int = 0) -> float | None:
weights: Weights | None = None) -> float | None:
"""The one number of a CV match, 0 to 100 for sorting and display: the
judge's three numbers, each 0 to 10, averaged by the agent's
`cv_skills`, `cv_experience` and `cv_career` weights (30, 25 and 30 by
default), career as `capped_career` has it, times 10. None when all
three weights are 0."""
default), times 10. None when all three weights are 0."""
w = weights or Weights()
total = w.cv_skills + w.cv_experience + w.cv_career
if not total:
return None
career = capped_career(career, broken)
return round(10 * (w.cv_skills * skills + w.cv_experience * experience
+ w.cv_career * career) / total, 1)
class Must(Model):
"""One must-have of an agent as the judge read it against a role: its
`id` (`M1`, by line), the line's `text` and the `verdict`."""
id: str
text: str = ""
verdict: Literal["met", "broken", "unclear"] = "unclear"
class Judgement(Model):
"""What `judge` read of one role for one agent (`role_judgements`): the
three dimensions 0 to 10 as the model gave them, `cv_score`
(`cv_score`, 0 to 100) and 1 to 3 strengths and gaps grounded in the
posting.
`musts` is the verdict on each must-have and `broken` how many are
broken; any makes `career_capped` the lower of `career` and
`BROKEN_CAP`. `cv_score` and `career_capped` are not read from the
table: `Store.judgement` sets them from the agent's weights; `cv_score`
is None when all three are 0."""
posting. `cv_score` is not read from the table: `Store.judgement` sets
it from the agent's weights; it is None when all three are 0."""
skills: int
experience: int
career: int
career_capped: int | None = None
broken: int = 0
musts: Annotated[list[Must], BeforeValidator(lambda v: _json(v, "[]"))] = []
cv_score: float | None = None
strengths: JsonStrings = []
gaps: JsonStrings = []
@@ -1574,14 +1525,11 @@ class Agent(Model):
are the union over the enabled agents, and an agent with none means no
board is narrowed by words. They decide what is fetched, not what ranks.
`cv_text`, `cv_languages`, `like_text`, `avoid_text` and `must_text` are
what `judge` reads a role against: the CV, the languages it states with
a level, what the person likes and does not like (the work, the
workplace, situations; both feed the career score), and their
must-haves and deal-breakers (set through the API only, not on the
form; soft, read by the model only; a hard rule is a gate). An agent without
a CV is not judged, and one without languages flags none
(`language_flag`)."""
`cv_text`, `cv_languages`, `like_text` and `avoid_text` are what `judge`
reads a role against: the CV, the languages it states with a level, and
what the person likes and does not like (the work, the workplace,
situations; both feed the career score). An agent without a CV is not
judged, and one without languages flags none (`language_flag`)."""
id: int
name: str
@@ -1594,7 +1542,6 @@ class Agent(Model):
cv_languages: CvLanguages = []
like_text: str = ""
avoid_text: str = ""
must_text: str = ""
enabled: bool = True
created_at: str
+6 -2
View File
@@ -37,8 +37,9 @@ from app.storage import Store
#: Raised when the rules in this package change the way a stored word maps,
#: so every open role is mapped again as it is after a table changes. 3: the
#: remote area, and remote as the model's before the host's word. 4: the
#: deadline, the host's before the model's.
VERSION = 4
#: deadline, the host's before the model's. 5: `remote_text`, the title or
#: text saying remote while no field does.
VERSION = 5
#: The state key the last run's `fingerprint` is kept under.
STATE = "prep:tables"
@@ -61,6 +62,7 @@ def fingerprint(tables: Tables = TABLES) -> str:
"""A hash of every host table, the shared remote words, the gazetteer the
code describes and `VERSION`."""
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
"remote_text": [words.REMOTE_TEXT, words.REMOTE_TEXT_NOT],
"places": repr(places.seed())}, sort_keys=True)
return hashlib.sha1(blob.encode()).hexdigest()
@@ -114,6 +116,7 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
"seniority_title": titled,
"employment": words.employment(host, row["employment_host"], tables),
"remote": remote,
"remote_text": int(words.remote_in_text(row["title"], row["text"] or "")),
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
if remote == "remote" else ()),
"country": country(gaz, place_ids, row["country_host"]),
@@ -125,6 +128,7 @@ def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
return bool(row["prep_due"]) or level != row["seniority"] \
or found["remote"] != row["remote"] \
or found["remote_text"] != row["remote_text"] \
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
or (found["country"] and found["country"] != row["country"]) \
or found["deadline"] != row["deadline"] \
+33
View File
@@ -58,6 +58,39 @@ _NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\
re.IGNORECASE)
#: Phrases in a title or text that say the role is worked remotely, and the
#: phrases that take one back. Both are read on the lowercased text with its
#: hyphens and runs of spaces squashed to one space. A match marks the role as
#: possibly remote (`remote_in_text`); it decides nothing by itself.
REMOTE_TEXT: tuple[str, ...] = (
"fully remote", "100% remote", "100 % remote", "remote first", "remote work",
"remote working", "work remotely", "working remotely", "remote position", "remote role",
"remote job", "remote team", "remote friendly", "remote possible", "remote option",
"remote ok", "work from home", "working from home", "work from anywhere", "home based",
"home office", "homeoffice", "hjemmearbejde", "fjernarbejde", "distansarbete",
)
REMOTE_TEXT_NOT: tuple[str, ...] = (
"no remote", "not remote", "not a remote", "non remote", "not be remote", "cannot be remote",
"no home office", "kein homeoffice", "not available remotely", "not open to remote",
"remote work is not", "remote working is not", "is not a remote", "remote sensing",
"remote control", "remote monitoring",
)
_REMOTE_TEXT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT))
_REMOTE_TEXT_NOT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT_NOT))
_SQUASHED = re.compile(r"[\s-]+")
def remote_in_text(title: str, text: str) -> bool:
"""Whether the title or the text says the role is worked remotely: a
phrase of `REMOTE_TEXT` left once those of `REMOTE_TEXT_NOT` are taken
out. Cheap evidence, no model: a role marked so passes a location gate
on remote until extract has read it (ADR Fetch prep and score 1.1), so a
false mark costs a cut slot at most, and a missed one nothing new."""
folded = _SQUASHED.sub(" ", f"{title or ''} {text or ''}".lower())
return bool(_REMOTE_TEXT.search(_REMOTE_TEXT_NOT.sub(" ", folded)))
def _line_word(text: str) -> str:
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
if found := remote_of(text):
+55 -87
View File
@@ -9,13 +9,15 @@
* ADR's order: the world (the scope select), the location (a short list of
* lines a role passes by matching any one of: countries, towns, remote), the
* role exclusion (the titles to leave out) and the role fetch (the fetch
* terms), over a row per remaining rule under Advanced; a gate the four parts
* cannot say lands in a row, so a load and a save lose nothing. A gate is a
* field, values and a mode (must, one of, not), with no weight. After it come
* the analytics filter (the best N by prompt closeness and the best M by CV
* closeness, each with a minimum level), the want feed score weights (prompt
* closeness, newness, company like, job like), the CV match weights and the
* size of summarise. Edit turns a card
* terms), and nothing else: seniority, pay and the like are filters on Roles.
* A gate is a field, values and a mode (must, one of, not), with no weight.
* A stored gate the four parts cannot say, which only the config or the API
* writes, is named in a note under the fetch gate and sent back as it came,
* so a load and a save lose nothing. After the fetch gate come the analytics
* filter (the best N by prompt closeness and the best M by CV closeness, each
* with a minimum level), the want feed score weights (prompt closeness,
* newness, company like, job like), the CV match weights and the size of
* summarise. Edit turns a card
* into the form from the page's `<template>`, in place, and New agent puts a
* blank card on top in the same state. One card edits at a time, so the
* form's ids stay unique. Votes, liked companies and scores are the agent's,
@@ -30,10 +32,7 @@
* text (typed, or read from a .md/.txt file by the browser), the languages it
* states as rows of a code and a level, and the two soft texts, what the
* person likes and what they do not like (the work, the workplace,
* situations), which the judge reads for the career score. The must-have text
* is no longer on the form: a save leaves `must_text` out, so the stored one
* is kept (it is set through the API, and still caps career at 4 in code
* when the judge reports one broken).
* situations), which the judge reads for the career score.
* Every save sends the CV, languages and the two texts, and the card's switch
* re-posts the row as the server gave it, so neither drops them.
*
@@ -64,9 +63,8 @@
const NEW = "new";
// Europe as the starter agents spell it (static/remote.js).
const EUROPE = rh.EUROPE;
const FIELDS = ["place", "country", "remote", "remote_from", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company", "title"];
const MODES = [["require", "must"], ["any", "one of"], ["exclude", "not"]];
// A gate's mode in the words of the note on the gates the form keeps.
const MODES = { require: "must", any: "one of", exclude: "not" };
/** @type {object[]} every agent, as the server lists them */
let agents = [];
@@ -266,8 +264,8 @@
}
// The region and place in words, one sentence led by the region, then the lines a
// role may match, with the one-of rows among them since they are the same
// group: "Within Europe: keeps roles anywhere in Denmark, in Malmö, Lund or
// role may match, with any other one-of gates among them since they are the
// same group: "Within Europe: keeps roles anywhere in Denmark, in Malmö, Lund or
// Helsingborg, or remote, and leaves out the rest." The lines add up, which
// the first wording ("Roles in ...") left the owner to guess.
function whereSentence(where, others) {
@@ -384,7 +382,7 @@
// ---------------------------------------------------------------- the form
// Tell the form something changed that no field typed: a chip or a row
// added or removed. It marks the form dirty and redraws the gates view.
// added or removed. It marks the form dirty and redraws the gates' sentence.
function changed(node) {
node.dispatchEvent(new Event("input", { bubbles: true }));
}
@@ -559,9 +557,9 @@
return box;
}
// A language as a row, after the rule rows: a code, a level, Remove. The
// code is two letters, which the server turns into the language; its name
// is the row's title where the browser knows it.
// A language as a row: a code, a level, Remove. The code is two letters,
// which the server turns into the language; its name is the row's title
// where the browser knows it.
function languageRow(language, remove) {
const row = el("div", "rule-row");
const code = el("input");
@@ -587,32 +585,32 @@
return { node: row, read: () => ({ language: code.value.trim().toLowerCase(), level: level.value }) };
}
// A gate as a row: field, mode, values and Remove. Values untouched since the load are sent as they came, so a
// value with a comma in it survives a save.
function ruleRow(gate, remove) {
const row = el("div", "rule-row");
const field = select(FIELDS.map((f) => [f, f.replace("_", " ")]), gate.field, "Field");
const mode = select(MODES, gate.mode, "Mode");
const values = el("input");
values.type = "text";
values.required = true;
values.spellcheck = false;
values.placeholder = "senior, staff";
values.value = gate.values.join(", ");
values.setAttribute("aria-label", "Values, separated by commas");
const loaded = values.value;
const drop = linky("Remove");
drop.addEventListener("click", () => remove(row));
row.append(field, mode, values, drop);
// The gates the fetch gate's four parts say, in the order the starter
// agents have them: the world, the location lines and the titles left out,
// then `kept`, the stored gates the parts cannot say, as they came. `parts`
// holds what the form reads: `scope` (europe, countries, everywhere) and
// its `scoped` countries, the `within` countries, the `towns`, `remote`
// (none, scope, from) and its `from` areas, and the `titles`.
function formGates(parts, kept = []) {
const out = [];
const { scope, scoped, within, towns: lines, remote, from, titles } = parts;
if (scope === "europe") out.push({ field: "country", values: rh.europeWith(within), mode: "require" });
if (scope === "countries" && scoped.length) out.push({ field: "country", values: scoped, mode: "require" });
if (within.length) out.push({ field: "country", values: within, mode: "any" });
if (remote === "scope") out.push({ field: "remote", values: ["remote"], mode: "any" });
if (remote === "from" && from.length) out.push({ field: "remote_from", values: from, mode: "any" });
if (lines.length) out.push({ field: "place", values: lines, mode: "any" });
if (titles.length) out.push({ field: "title", values: titles, mode: "exclude" });
return out.concat(kept);
}
function read() {
const parts = values.value === loaded
? gate.values
: values.value.split(",").map((v) => v.trim()).filter(Boolean)
.map((v) => (field.value === "salary_min" ? Number(v) : v));
return { field: field.value, values: parts, mode: mode.value };
}
return { node: row, read };
// The note on the gates the form keeps: "The form cannot edit these rules,
// and Save keeps them as stored: must: with seniority senior; not: at Acme."
function keptNote(kept) {
if (!kept.length) return "";
const said = kept.map((g) => `${MODES[g.mode] || g.mode}: ${phrase(g)}`).join("; ");
return `The form cannot edit ${kept.length > 1 ? "these rules" : "this rule"}, and Save keeps `
+ `${kept.length > 1 ? "them" : "it"} as stored: ${said}.`;
}
function form(row) {
@@ -713,51 +711,18 @@
remoteFrom.node.hidden = remoteMode.value !== "from";
};
offer();
$("details.advanced", f).open = rest.length > 0;
const kept = $("#a-kept", f);
kept.textContent = keptNote(rest);
kept.hidden = !rest.length;
const rules = [];
const dropRule = (node) => {
rules.splice(rules.findIndex((r) => r.node === node), 1);
node.remove();
changed(f);
};
const addRule = (gate) => {
const entry = ruleRow(gate, dropRule);
rules.push(entry);
$("#a-rules", f).appendChild(entry.node);
return entry;
};
rest.forEach(addRule);
$("#a-add-rule", f).addEventListener("click", () => {
$("select", addRule({ field: "seniority", values: [], mode: "require" }).node).focus();
changed(f);
});
// The gates the fetch gate's three parts say, in the order the starter
// agents have them: the region, the place lines, and the titles left out.
function whereGates() {
const out = [];
const within = countries.values();
const towns = places.values();
const from = remoteFrom.values();
if (scope.value === "europe") out.push({ field: "country", values: rh.europeWith(within), mode: "require" });
if (scope.value === "countries" && scoped.values().length) {
out.push({ field: "country", values: scoped.values(), mode: "require" });
}
if (within.length) out.push({ field: "country", values: within, mode: "any" });
if (remoteMode.value === "scope") out.push({ field: "remote", values: ["remote"], mode: "any" });
if (remoteMode.value === "from" && from.length) out.push({ field: "remote_from", values: from, mode: "any" });
if (towns.length) out.push({ field: "place", values: towns, mode: "any" });
const left = titles.values();
if (left.length) out.push({ field: "title", values: left, mode: "exclude" });
return out;
}
// The gates Save sends: the fetch gate's, then the rows. The sentence
// under it says them all, as the card will, rows included.
const gates = () => whereGates().concat(rules.map((r) => r.read()));
// The gates Save sends: the four parts', then the kept ones. The sentence
// under them says them all, as the card will, the kept ones included.
const gates = () => formGates({
scope: scope.value, scoped: scoped.values(), within: countries.values(), towns: places.values(),
remote: remoteMode.value, from: remoteFrom.values(), titles: titles.values(),
}, rest);
const view = () => {
offer();
$("#a-gates", f).value = JSON.stringify(gates(), null, 2);
$("#a-where-sum", f).textContent = rulesLine(gates()) || "Passes every role.";
};
view();
@@ -981,6 +946,9 @@
}
}
// For tests/js/agents.test.mjs: what Save sends and how it is said.
window.agentGates = { formGates, keptNote, rulesLine };
document.addEventListener("DOMContentLoaded", async () => {
if (!$("#agents-list")) return;
$("#a-new").addEventListener("click", () => open(null));
+3 -17
View File
@@ -978,14 +978,6 @@ body[data-reader="open"] .company-strip { display: none; }
.reader-cv .cv-total { margin: 0 0 .45rem; font-variant-numeric: tabular-nums; }
.reader-cv .cv-total b { font-size: 1.25rem; }
.reader-cv .cv-total span { display: inline-block; margin-left: .4rem; color: var(--dim); font-size: .88rem; }
/* The must-haves, one line each with its verdict's mark. */
.cv-musts { list-style: none; margin: 0 0 .55rem; padding: 0; font-size: .92rem; line-height: 1.4; }
.cv-musts li { display: flex; gap: .45rem; }
.cv-musts li + li { margin-top: .15rem; }
.cv-musts .mark { flex: none; width: 1rem; text-align: center; font-weight: 700; }
.cv-musts .must-met .mark { color: var(--good); }
.cv-musts .must-broken .mark { color: var(--error); }
.cv-musts .must-unclear .mark { color: var(--faint); }
.cv-lists { display: grid; grid-template-columns: 1fr 1fr; gap: .5rem 1.2rem; }
.cv-lists ul { margin: 0; padding-left: 1.1rem; font-size: .92rem; line-height: 1.45; }
.cv-lists li + li { margin-top: .2rem; }
@@ -1308,8 +1300,6 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
background: var(--bg); border: 1px solid var(--line);
border-radius: .3rem; padding: .35rem .5rem; width: 100%;
}
/* Prose in a textarea is read in the page face; JSON is in the mono one. */
.agent-form textarea.code { font-family: var(--mono); font-size: .85rem; }
.agent-form .hint { color: var(--dim); font-size: .8rem; overflow-wrap: anywhere; }
.weights { display: flex; gap: .75rem; flex-wrap: wrap; border: 1px solid var(--line); border-radius: .3rem; }
/* A label that wraps to two lines pushes its box down; the boxes stay in line. */
@@ -1324,14 +1314,13 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
}
/* The fieldsets that hold rows: a column, the Add button under the rows. */
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
.agent-form fieldset.terms,
.agent-form fieldset.rules {
.agent-form fieldset.terms {
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
border: 1px solid var(--line); border-radius: .3rem;
}
.agent-form fieldset > button { align-self: flex-start; }
/* A prompt or a rule as a row of a list, after `.crow`: the name and Remove
on line one, the text spanning the row under them. A rule row wraps like
/* A prompt or a language as a row of a list, after `.crow`: the name and Remove
on line one, the text spanning the row under them. A language row wraps like
`.crow .board-edit`, so a phone takes it in two lines rather than scrolling. */
.prompt-row {
display: grid; grid-template-columns: 1fr auto; align-items: center;
@@ -1361,9 +1350,6 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
.agent-form .chips input[type="text"] { flex: 1 1 6rem; width: auto; min-width: 0; }
.agent-form .chips select { flex: 0 1 12rem; min-width: 0; max-width: 100%; }
.chips .tag span { color: var(--faint); }
.agent-form details summary { cursor: pointer; color: var(--dim); font-size: .9rem; }
.agent-form details[open] summary { margin-bottom: .5rem; }
.agent-form .advanced-body { display: flex; flex-direction: column; gap: .6rem; }
/* ================================================================ */
/* companies and sources — the Sources page's sections, */
+12 -10
View File
@@ -1,7 +1,8 @@
/* Where a remote role may be worked from, as the pages say it: the words of
* the badge, the picks of the agent's "Remote, open to someone in" chips, and
* how an agent's gates share out between the fetch gate's blocks and Other rules. No
* DOM in it, so `tests/js/remote.test.mjs` can walk it without a browser.
* how an agent's gates share out between the fetch gate's blocks and the rest
* the form keeps as stored. No DOM in it, so `tests/js/remote.test.mjs` can
* walk it without a browser.
*
* The area is a place token in three spellings, one per layer: `remote:<place
* token>` on the Roles rail and in the address, `<place token>` in a
@@ -61,18 +62,19 @@
}
/** Share out an agent's gates between the fetch gate's blocks (region,
place, titles to leave out) and Other rules. Each line takes the first
gate it can say whole; the rest are rows. A gate has a field, values and
a mode (require, any, exclude) and no weight; a gate with a mode the
form does not know, such as an old prefer, stays a row.
place, titles to leave out) and the rest, which the form cannot edit
and keeps as stored. Each line takes the first gate it can say whole.
A gate has a field, values and a mode (require, any, exclude) and no
weight; a gate with a mode the form does not know, such as an old
prefer, is in the rest.
The scope line is the first plain require gate on country: `europe`
when it is exactly Europe and the countries line (`europeWith`),
`countries` with its codes as chips otherwise, `everywhere` when there
is none. Any other require gate on country is a row. The remote line is
`scope` for the one-of on `remote: [remote]` (remote, within the
scope), `from` for a one-of on `remote_from`, else `none`; when both
are there `remote_from` wins and the plain one stays a row. */
is none. Any other require gate on country is in the rest. The remote
line is `scope` for the one-of on `remote: [remote]` (remote, within
the scope), `from` for a one-of on `remote_from`, else `none`; when
both are there `remote_from` wins and the plain one is in the rest. */
function distribute(gates) {
const where = { scope: null, countries: null, places: null, remote: { mode: "none", values: [] }, titles: null };
const rest = [];
+4 -18
View File
@@ -1744,34 +1744,20 @@
try { return languageNames?.of(code) || code; } catch (err) { return code; }
}
const MUST_MARKS = { met: ["✓", "met"], broken: ["✗", "broken"], unclear: ["?", "unclear"] };
/* The agent's `judge` reading against its CV: the total and its three
numbers, each of 10, career as the server counted it (capped at 4 when a must-have
is broken, with what the model said), the must-haves with a verdict
each, then the strengths and gaps it named. A box as the summary is,
since it is the same kind of thing: a model's reading, not the posting. */
numbers, each of 10, then the strengths and gaps it named. A box as the
summary is, since it is the same kind of thing: a model's reading, not
the posting. */
function cvMatch(r) {
const j = r.judgement;
if (!j) return "";
const list = (items, none) => (items?.length
? `<ul>${items.map((x) => `<li>${esc(x)}</li>`).join("")}</ul>`
: `<p class="muted small">${none}</p>`);
const counted = j.career_capped ?? j.career;
const career = counted < j.career
? `career ${counted}/10 (capped: breaks a must-have; model said ${j.career}/10)`
: `career ${j.career}/10`;
const musts = Array.isArray(j.musts) && j.musts.length
? `<ul class="cv-musts">${j.musts.map((m) => {
const [mark, word] = MUST_MARKS[m.verdict] || MUST_MARKS.unclear;
return `<li class="must-${word}"><span class="mark" role="img"
aria-label="${word}" title="${word}">${mark}</span><span>${esc(m.text)}</span></li>`;
}).join("")}</ul>` : "";
return `<section class="reader-summary reader-cv">
<h2 class="reader-label">CV match</h2>
<p class="cv-total"><b>${Math.round(j.cv_score)}</b> of 100
<span>skills ${j.skills}/10 · experience ${j.experience}/10 · ${career}</span></p>
${musts}
<span>skills ${j.skills}/10 · experience ${j.experience}/10 · career ${j.career}/10</span></p>
<div class="cv-lists">
<div><h3 class="reader-label">Strengths</h3>${list(j.strengths, "None named.")}</div>
<div><h3 class="reader-label">Gaps</h3>${list(j.gaps, "None named.")}</div>
+43 -16
View File
@@ -55,6 +55,10 @@ CREATE TABLE IF NOT EXISTS roles (
country TEXT NOT NULL DEFAULT '',
remote TEXT NOT NULL DEFAULT ''
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
-- 1 when the title or text says the role is worked remotely while no
-- field does (prep, app.prep.words.remote_in_text): the location gate
-- lets such a role through until extract has read it.
remote_text INTEGER NOT NULL DEFAULT 0,
salary_min INTEGER,
salary_max INTEGER,
currency TEXT NOT NULL DEFAULT '',
@@ -328,9 +332,8 @@ END;
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
-- level}), like_text (what they like: the work, the workplace, situations),
-- avoid_text (what they do not like) and must_text (must-haves and
-- deal-breakers, set through the API only) are what judge reads a role
-- level}), like_text (what they like: the work, the workplace, situations)
-- and avoid_text (what they do not like) are what judge reads a role
-- against; an agent with no CV is not judged.
CREATE TABLE IF NOT EXISTS agents (
id INTEGER PRIMARY KEY,
@@ -344,7 +347,6 @@ CREATE TABLE IF NOT EXISTS agents (
cv_text TEXT NOT NULL DEFAULT '',
cv_languages TEXT NOT NULL DEFAULT '[]',
avoid_text TEXT NOT NULL DEFAULT '',
must_text TEXT NOT NULL DEFAULT '',
like_text TEXT NOT NULL DEFAULT ''
);
@@ -435,11 +437,8 @@ CREATE TABLE IF NOT EXISTS agent_cv_vectors (
-- 100 until migration 33), cv_score their weighted total times 10 at the
-- default weights (not read: app.models.cv_score is worked out when a role
-- is served), strengths and gaps JSON lists. input_key
-- is a hash of the CV, prompts, avoid text and the role's text it read: a
-- different one is read again. musts is a JSON list of {id, text, verdict},
-- the model's verdict on each of the agent's must-haves (one per line of its
-- must text), broken how many are "broken": career counts as at most 4 then
-- (app.models.BROKEN_CAP), applied when the role is served. tries counts
-- is a hash of the CV, prompts, languages, like and avoid texts and the
-- role's text it read: a different one is read again. tries counts
-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A
-- row with judged_at NULL holds only tries.
CREATE TABLE IF NOT EXISTS role_judgements (
@@ -451,8 +450,6 @@ CREATE TABLE IF NOT EXISTS role_judgements (
cv_score REAL,
strengths TEXT NOT NULL DEFAULT '[]',
gaps TEXT NOT NULL DEFAULT '[]',
musts TEXT NOT NULL DEFAULT '[]',
broken INTEGER NOT NULL DEFAULT 0,
input_key TEXT NOT NULL DEFAULT '',
tries INTEGER NOT NULL DEFAULT 0,
judged_at TEXT,
@@ -763,6 +760,12 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
# A Python step: JSON limits, and a count for the log.
(33, "the judge on 0 to 10: every judgement cleared to be made again, and summarise_top "
"50 becomes 30", lambda conn: _judge_out_of_ten(conn)),
# A Python step: it drops the columns only where they are there. The
# judge reads every role again (judge.FORMAT).
(34, "agents.must_text, role_judgements.musts and broken dropped: the profile is the "
"CV, languages, likes and dislikes", lambda conn: _drop_musts(conn)),
(35, "roles.remote_text: the text says remote while no field does, read by prep for the "
"location gate", lambda conn: _add_remote_text(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -1012,6 +1015,15 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
def _add_remote_text(conn: sqlite3.Connection) -> None:
"""Migration 35's step: `roles.remote_text`, 0 until the next prep,
which maps every open role again since its rules changed. Runs twice
without harm."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
if "remote_text" not in columns:
conn.execute("ALTER TABLE roles ADD COLUMN remote_text INTEGER NOT NULL DEFAULT 0")
def _add_remote_area(conn: sqlite3.Connection) -> None:
"""Migration 27's step: `roles.remote_model` empty and
`roles.remote_where_model` NULL for every role, so `extract` reads the
@@ -1203,12 +1215,14 @@ def _judge_out_of_ten(conn: sqlite3.Connection) -> None:
old number is shown on the new scale. The rows stay, as rows of no
judgement, and the judge reads every pair again. An agent whose
`summarise_top` is 50, migration 32's default, takes 30, the new one.
Runs twice without harm."""
Runs twice without harm, also after migration 34 dropped the verdicts."""
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
verdicts = ", musts = '[]', broken = 0" if {"musts", "broken"} <= columns else ""
cleared = conn.execute(
"""UPDATE role_judgements
SET skills = NULL, experience = NULL, career = NULL, cv_score = NULL,
strengths = '[]', gaps = '[]', musts = '[]', broken = 0,
input_key = '', tries = 0, judged_at = NULL""").rowcount
f"""UPDATE role_judgements
SET skills = NULL, experience = NULL, career = NULL, cv_score = NULL,
strengths = '[]', gaps = '[]'{verdicts},
input_key = '', tries = 0, judged_at = NULL""").rowcount
lowered = conn.execute(
"""UPDATE agents SET limits = json_set(limits, '$.summarise_top', 30)
WHERE json_valid(limits) AND json_type(limits) = 'object'
@@ -1217,6 +1231,19 @@ def _judge_out_of_ten(conn: sqlite3.Connection) -> None:
"summarise_top 50 set to 30", cleared, lowered)
def _drop_musts(conn: sqlite3.Connection) -> None:
"""Migration 34's step: the must-haves are gone, the agent's text
(`agents.must_text`) and the judge's verdicts on it
(`role_judgements.musts`, `broken`), and with them the cap on career.
Runs twice without harm."""
for table, columns in (("agents", ("must_text",)),
("role_judgements", ("musts", "broken"))):
present = {r[1] for r in conn.execute(f"PRAGMA table_info({table})")}
for column in columns:
if column in present:
conn.execute(f"ALTER TABLE {table} DROP COLUMN {column}")
def _add_all_roles(conn: sqlite3.Connection) -> None:
"""Migration 21's step. The column; the tag for every company that is a
favourite of any agent, since favourite no longer widens what a pull
+32 -39
View File
@@ -11,7 +11,7 @@ from __future__ import annotations
import json
import sqlite3
from collections import Counter
from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
from contextlib import closing, contextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -26,7 +26,7 @@ from app.models import (
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
BROKEN_CAP, capped_career, language_flag, role_languages,
language_flag, role_languages,
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
)
@@ -140,7 +140,7 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
#: The agent's `judge` readings beside every role, the same way: `j.skills`,
#: `j.experience`, `j.career` and `j.broken` are NULL for a role not judged and with no
#: `j.experience` and `j.career` are NULL for a role not judged and with no
#: agent.
_WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? "
"AND j.judged_at IS NOT NULL")
@@ -150,19 +150,15 @@ _WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agen
def _cv_sql(weights: Weights | None) -> str:
"""`cv_score` as SQL for one agent's weights, 0 to 100 from the judge's
three numbers 0 to 10, unrounded (`app.models.cv_score` rounds it): NULL
for a role not judged, and for all three weights 0. Career
counts as at most `BROKEN_CAP` when the judgement holds a broken
must-have (`j.broken`), as `app.models.capped_career` has it. The
numbers are the model's validated, finite floats, so they are written in
rather than bound, the select's placeholders being then no concern."""
for a role not judged, and for all three weights 0. The weights are
validated, finite floats, so they are written in rather than bound, the
select's placeholders being then no concern."""
w = weights or Weights()
total = w.cv_skills + w.cv_experience + w.cv_career
if not total:
return "NULL"
career = (f"(CASE WHEN j.broken > 0 AND j.career > {BROKEN_CAP} THEN {BROKEN_CAP} "
"ELSE j.career END)")
return (f"(10.0 * ({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience "
f"+ {w.cv_career!r} * {career}) / {total!r})")
f"+ {w.cv_career!r} * j.career) / {total!r})")
def _by_cv(expression: str) -> str:
@@ -359,6 +355,12 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
return ("r.languages IN ('', '[]')",
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
f"WHERE LOWER(value) NOT IN ({marks}))", values)
if gate.field == "remote" and "remote" in values:
# A role whose text says remote while no field does is possibly
# remote (`remote_text`, prep): it meets a gate on remote until
# extract has read it, and is not empty for it.
return (f"({column} = '' AND r.remote_text = 0)",
f"(LOWER({column}) IN ({marks}) OR ({column} = '' AND r.remote_text = 1))", values)
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
@@ -1629,8 +1631,9 @@ class Store:
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
seniority_host, employment_host, remote_host, country_host,
seniority_model, seniority_stated, seniority_title, seniority,
employment, remote, country, remote_model, remote_where_model,
deadline_host, deadline_model, deadline,
employment, remote, remote_text, country, remote_model,
remote_where_model, deadline_host, deadline_model, deadline,
SUBSTR(COALESCE(NULLIF(description, ''), teaser), 1, 6000) AS text,
json_extract(raw, '$.places') AS raw_places,
(SELECT json_group_array(place_id) FROM (
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
@@ -1656,8 +1659,8 @@ class Store:
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
"""Write what prep mapped, one dict per role with `uid`, `due` (the
`prep_due` it read), `company_key`, `seniority_stated`,
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
`seniority_title`, `employment`, `remote`, `remote_text`, `remote_area`,
`country` and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
the title's. `remote` is written as prep composed it (the model's,
else the host's word, else the place line) and `remote_area`, place
ids, replaces the role's `role_remote`. An empty `country` leaves the
@@ -1673,6 +1676,7 @@ class Store:
NULLIF(:seniority_stated, ''), :seniority_title),
employment = :employment,
remote = :remote,
remote_text = :remote_text,
country = COALESCE(NULLIF(:country, ''), country),
deadline = :deadline,
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
@@ -2021,33 +2025,28 @@ class Store:
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
experience: int, career: int, strengths: Sequence[str],
gaps: Sequence[str], musts: Sequence[Mapping[str, str]] = ()) -> bool:
gaps: Sequence[str]) -> bool:
"""Store `judge`'s reading of one role for one agent, replacing the
one before: the three numbers as the model gave them, the verdict on
each of the agent's must-haves (`musts`, `{id, text, verdict}`) and how
many are broken, `cv_score` (`app.models.cv_score`, at the default
weights and with the cap: the table's column is not read, see
`judgement`) and its tries back at 0. False for an unknown role or
agent."""
broken = sum(1 for m in musts if m.get("verdict") == "broken")
one before: the three numbers as the model gave them, `cv_score`
(`app.models.cv_score`, at the default weights: the table's column is
not read, see `judgement`) and its tries back at 0. False for an
unknown role or agent."""
try:
with self.transaction() as conn:
conn.execute(
"""INSERT INTO role_judgements (uid, agent_id, skills, experience, career,
cv_score, strengths, gaps, musts, broken,
cv_score, strengths, gaps,
input_key, tries, judged_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,0,?)
VALUES (?,?,?,?,?,?,?,?,?,0,?)
ON CONFLICT(uid, agent_id) DO UPDATE SET
skills=excluded.skills, experience=excluded.experience,
career=excluded.career, cv_score=excluded.cv_score,
strengths=excluded.strengths, gaps=excluded.gaps,
musts=excluded.musts, broken=excluded.broken,
input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""",
(uid, agent_id, skills, experience, career,
cv_score(skills, experience, career, broken=broken),
cv_score(skills, experience, career),
json.dumps(list(strengths), ensure_ascii=False),
json.dumps(list(gaps), ensure_ascii=False),
json.dumps([dict(m) for m in musts], ensure_ascii=False), broken,
input_key, self.now()))
except sqlite3.IntegrityError:
return False
@@ -2075,8 +2074,7 @@ class Store:
def judgement(self, agent_id: int | None, uid: str) -> Judgement | None:
"""The agent's judgement of one role, None when it has none, its
`cv_score` the three numbers under the agent's weights as they are
now, not as stored, and `career_capped` the model's career as the
broken must-haves leave it (`BROKEN_CAP`)."""
now, not as stored."""
if agent_id is None:
return None
row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? "
@@ -2085,9 +2083,8 @@ class Store:
return None
found = Judgement.from_row(row)
agent = self.agent(agent_id)
found.career_capped = capped_career(found.career, found.broken)
found.cv_score = cv_score(found.skills, found.experience, found.career,
agent.weights if agent else None, found.broken)
agent.weights if agent else None)
return found
def set_article_text(self, uid: str, text: str) -> Role | None:
@@ -2209,14 +2206,13 @@ class Store:
cv_text: str | None = None,
cv_languages: Sequence[CvLanguage] | None = None,
avoid_text: str | None = None,
must_text: str | None = None,
like_text: str | None = None) -> Agent:
"""Create an agent, or with `agent_id` replace that one's fields and
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
named after the agent. Without `fetch_terms` an update keeps the
stored ones and a new agent has none, so a caller that does not know
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
`like_text`, `avoid_text` and `must_text` are kept the same way. A prompt whose text is
`like_text` and `avoid_text` are kept the same way. A prompt whose text is
unchanged keeps its cached vector, whatever its name or place, so an
edit re-embeds only the prompts it changed. A save gives every role
`judge` gave up on for the agent its tries back, since what it is
@@ -2237,7 +2233,6 @@ class Store:
(limits or Limits()).model_dump_json(), int(enabled), terms,
None if cv_text is None else cv_text.strip(), spoken,
None if avoid_text is None else avoid_text.strip(),
None if must_text is None else must_text.strip(),
None if like_text is None else like_text.strip())
try:
with self.transaction() as conn:
@@ -2245,10 +2240,9 @@ class Store:
agent_id = conn.execute(
"""INSERT INTO agents (name, gates, weights, limits, enabled,
fetch_terms, cv_text, cv_languages, avoid_text,
must_text, like_text, created_at)
like_text, created_at)
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),
COALESCE(?, ''),?)""",
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),?)""",
(*fields, self.now())).lastrowid
elif not conn.execute(
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
@@ -2256,7 +2250,6 @@ class Store:
cv_text=COALESCE(?, cv_text),
cv_languages=COALESCE(?, cv_languages),
avoid_text=COALESCE(?, avoid_text),
must_text=COALESCE(?, must_text),
like_text=COALESCE(?, like_text)
WHERE id=?""",
(*fields, agent_id)).rowcount:
+2 -15
View File
@@ -74,6 +74,7 @@
</div>
<span class="hint">Fetch terms, separated by a comma or Enter: <code>machine learning, data science</code> is two terms, and <code>machine learning</code> is searched as one. A host that takes a query sends only roles with one of the terms anywhere in the ad, but only for boards of 300 or more roles; smaller boards and other hosts are read whole. The terms are ignored after the fetch, since prompt closeness is the finer filter. Empty means no narrowing.</span>
<p id="a-where-sum" class="where-sum" role="status"></p>
<p id="a-kept" class="hint" hidden></p>
</fieldset>
<fieldset class="terms cv">
<legend>Your CV</legend>
@@ -94,7 +95,7 @@
<textarea id="a-like-text" rows="4" placeholder="Building a team from scratch, close work with the product side, small companies, a say in the roadmap"></textarea></label>
<label>What I don't like
<textarea id="a-avoid-text" rows="4" placeholder="Constant on-call, sales targets, large consultancies, open-plan offices"></textarea></label>
<span class="hint">The work, the workplace, situations; read by the judge for the career rating. Rules that must hold go under the fetch gate or Other rules instead.</span>
<span class="hint">The work, the workplace, situations; read by the judge for the career rating. Rules that must hold go under the fetch gate instead.</span>
</fieldset>
<fieldset class="weights limits">
<legend>Analytics filter</legend>
@@ -124,20 +125,6 @@
<label>Summarise the top <input type="number" id="a-summarise-top" min="0" step="5" value="30"> of each feed order</label>
</fieldset>
<span class="hint">Summarise writes a short summary and tags for the top N of What I'm looking for and the top N of What fits me, at most twice N roles, and for every role you liked or saved. It is no part of any score.</span>
<details class="advanced">
<summary>Advanced</summary>
<div class="advanced-body">
<fieldset class="rules">
<legend>Other rules</legend>
<div id="a-rules" class="rule-rows"></div>
<button type="button" id="a-add-rule" class="secondary">Add rule</button>
<span class="hint">Must: a role must meet it. One of: a role must meet one of these or a Place line. Not: a role that meets it is left out. Values are separated by commas; salary is a number.</span>
</fieldset>
<label>Rules as sent
<textarea id="a-gates" class="code" rows="6" spellcheck="false" readonly>[]</textarea>
<span class="hint">Read only: what Save sends as the agent's gates, from the fetch gate and Other rules.</span></label>
</div>
</details>
<div class="agent-actions">
<button type="submit">Save and score</button>
<button type="button" id="a-cancel" class="secondary">Cancel</button>
+2 -4
View File
@@ -1251,7 +1251,7 @@ class AgentBody(BaseModel):
it is stored as one prompt named after the agent. `fetch_terms` narrow
what big boards send (`app/jobs/reach.py`); `[]` narrows nothing, and
leaving the key out keeps an existing agent's. `cv_text`,
`cv_languages`, `like_text`, `avoid_text` and `must_text` are what `judge` reads a
`cv_languages`, `like_text` and `avoid_text` are what `judge` reads a
role against (`app/jobs/judge.py`), kept the same way when left out or
null."""
@@ -1267,7 +1267,6 @@ class AgentBody(BaseModel):
cv_languages: CvLanguages | None = None
like_text: str | None = None
avoid_text: str | None = None
must_text: str | None = None
enabled: bool = True
@@ -1307,8 +1306,7 @@ def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent:
body.enabled, agent_id=body.id, limits=body.limits,
prompts=body.prompts, fetch_terms=body.fetch_terms,
cv_text=body.cv_text, cv_languages=body.cv_languages,
like_text=body.like_text, avoid_text=body.avoid_text,
must_text=body.must_text)
like_text=body.like_text, avoid_text=body.avoid_text)
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
except KeyError:
+10 -2
View File
@@ -16,7 +16,7 @@ What was decided for roledesk and the reason for each. The target design is the
**A group is rows that all match each other, never a chain.** A row with no place matches Belgium and the Netherlands, and joining on one match merged both on the live data. Rows are taken sharpest first (a town, then only a country, then neither) and join a group only when they match every member.
**One posting per place is folded too, on every host.** A host that lists one job once per office gives rows of the same company and title whose towns never meet; `dupes.fold` merges those of one host when their texts share at least `SAME_TEXT` (30%) of their words, and the kept role keeps each copy's place and link. It applies to every host, not only to readers that declare `places="copies"`, because most such rows came from hosts that do not declare it. A remote listing and an office listing stay two roles: the kept role takes a copy's places and not its remote, and the fetch gate reads a remote role by its area, so a remote listing folded behind an office elsewhere was rejected with it, and an office listing behind a remote role elsewhere likewise.
**One posting per place is folded too, on every host.** A host that lists one job once per office gives rows of the same company and title whose towns never meet; `dupes.fold` merges those of one host when their texts share at least `SAME_TEXT` (30%) of their words, and the kept role keeps each copy's place and link. It applies to every host, not only to readers that declare `places="copies"`, because most such rows came from hosts that do not declare it. It keeps to one country, as ADR 1.4 has it, since the same rule across countries merged different openings whose texts happened to be alike (a federal sales role in the US behind one in Australia); a job posted once per country is then one role per country, which costs a few rows for a pan-European posting and hides nothing. A remote listing and an office listing stay two roles: the kept role takes a copy's places and not its remote, and the fetch gate reads a remote role by its area, so a remote listing folded behind an office elsewhere was rejected with it, and an office listing behind a remote role elsewhere likewise.
**A copy is stored as `roles.dup_of` and rebuilt from scratch after every pull.** Every reader of open roles adds `dup_of IS NULL`, which keeps `idx_roles_posted` usable where a window function over the filtered rows would scan them all. Where a group changed, the kept role takes the earliest `read_at` and each agent's latest vote from its copies; an unchanged group is left alone, so marking a role unread again sticks. The per-board counts on Sources still count copies, since they report what a board holds.
@@ -70,6 +70,12 @@ What was decided for roledesk and the reason for each. The target design is the
**A read that stalls is incomplete, and a sweep's cursor may be a pair.** A reader that sees the host send a page again hands it over empty with a cursor, and the runner ends the read incomplete; a cursor of None there read as the last page, and on a host that states no total as a complete read that closed what the pages had not listed. With a stated total a page of ids already read is not a stall, since a resumed sweep starts on one when the list grew at the top. A cursor is kept across pulls when it survives JSON apart from a tuple coming back as a list, so Workday's and SuccessFactors' sweeps resume.
**A host that cannot filter on remote is asked for the agents' world where it takes the world in one read** (ADR Fetch prep and score 1.2). Asked for "DK, SE" alone such a host never listed a remote role filed under Germany, and 18 boards fetched no new remote role outside those two countries. Phenom and the Workday tenants without a remote facet take a list of countries, so Europe is one request and the gate sorts the rest out. SmartRecruiters, SuccessFactors and Eightfold take one country a request, where Europe is the whole board (Bosch 49 pages, a SuccessFactors site 10 roles a page), so there the location read stays and the row says remote was not asked.
**A role whose text says remote while no field does is possibly remote, read in prep with no model** (ADR 1.1 and 1.5). A role outside the agent's locations whose remote is unstated could never "turn out to be remote": extract reads only the cut, and the cut needs the gate. 5,016 open roles were rejected that way, 805 of them saying "remote" in their text. A phrase table ("fully remote", "work from home", the Danish and Swedish words) less its denials ("no remote", "remote sensing") marks `remote_text`; the gate's remote member takes the mark as a match while `remote` is empty, and once extract has answered, prep composes `remote` from the model and the mark no longer counts. A false mark costs at most a cut slot.
**Age is the floor under every board's closing** (ADR Fetch prep and score 1.3). An incomplete read (a ceiling, a page cap without a sweep) closed nothing ever, and a filtered read spared the roles acted on from its list and from age alike, so such roles stayed open for good. Now what no read has listed for 21 days closes under an incomplete read, and what a filtered read spares closes by age too; a role acted on is spared from the list close, which is guesswork, not from three weeks of absence. The alternative was to strike the sentence from the ADR, which would have left "may stay open for good" in the design.
**Switching a board or search off closes its open roles.** Only a pull that reads a target closes its roles, so a target no longer read would keep them open for good. Switched back on, the next pull reopens what it still lists. A batch in flight when the switch happens reopens them, so the settle closes every switched-off target once more.
**A destructive switch asks first; a reopening one queues a pull.** Switching a board or search off, replacing a board and detaching one each close roles in a tap, so the page asks first when there are roles to lose, naming the count. Switching a target on, following or setting a board, setting All locations or All roles on and saving a search each queue `pull` at once instead of waiting up to six hours for the schedule. The pull is queued in the API route, not in the page's script, so a script calling the endpoint gets what a click gets; `claim_run`'s refusal is not an error, since the run in flight or the next one reads the change.
@@ -182,6 +188,8 @@ What was decided for roledesk and the reason for each. The target design is the
**`extract_batch`, `judge_batch` and `summarise_batch` run isolated, with a timeout.** `context.check()` runs only between the calls `modeltask.one` makes, not inside one, so a call that hangs was unkillable short of restarting the server; a process boundary lets a Kill, or the timeout, end it. One call is bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for judge and summarise): a batch still running after that is stuck, not slow. The cost is that an isolated job's context has no working `stop`, `task` or `fan_out` (`context_for` in `pipeline/child.py`), so a refusal or an account-level failure inside a batch cannot spare the task's queued siblings.
**The profile is the CV, the languages, the prompts, the likes and the dislikes; must-haves are gone** (ADR 4.2). The must-have text with a verdict per line and a career cap at 4 was the harder form of what likes and dislikes now say softly; it left the form on 2026-10-03 and its machinery followed on 2026-10-06 (migration 34). The judge reads the languages with their levels, since a role that asks for fluent Danish against conversational Danish is a skills gap the CV text alone did not show; a change to the languages re-judges the cut like a change to the CV.
**Judge reads a role against a written profile, after the `/rank` of github.com/MadsLorentzen/ai-job-search.** Prompt closeness compares topics; it cannot tell a step down, years of experience or a deal-breaker, which a model reading both texts can. The profile is the CV, its languages with a level, and two soft texts, "What I like" (`like_text`) and "What I don't like" (`avoid_text`), each covering the work, the workplace and situations with no split into tasks versus environment, since the prompts already say what the owner wants. A deal-breaker is a hard rule and belongs in the fetch gate, so the form has no must-haves box; `must_text` is set only through the API. Culture is left out because it is guessed from ad text, and location because the fetch gate does it.
**The model reports facts and code applies the rules.** For each must-have line the model answers "met", "broken" only when the posting clearly contradicts it, or "unclear" when the text does not say, never inferred from silence; a must-have it leaves out counts as unclear. The cap is code's: a broken must-have makes career count as at most `BROKEN_CAP` (4) when a role is served (`app/models.py`), so the stored career stays the model's and the reader shows both. The CV match weights are applied when a role is served too, so changing them needs no model call. The verdicts store each line's text, so the reader still shows what a judgement was made against after the text is edited. The `input_key` carries `judge.FORMAT` (4) and the texts, so a change to the prompt's contract makes every judgement again.
@@ -236,7 +244,7 @@ What was decided for roledesk and the reason for each. The target design is the
**Search agents are cards, edited in place.** One person may run several agents: their own hunt and one that watches roles for someone else. An agent is defined by what it looks for, so its prompts are the card's body and the weights a small line under it. Edit turns that card into the form and New puts a blank one on top, because a form at the bottom of the page was out of sight of the card it changed. Job boards and Watchlist companies are edited the same way, one open at a time.
**A gate the form cannot say lands in a row under Advanced, so a load and a save lose nothing.** The form asks the fetch gate in its four parts by name rather than as the stored `require` and `any` sets with ISO codes, which the owner could not read; any other gate is a row under Other rules. Spellings of a town that differ by accents only share one chip, and both are stored.
**The form's gates are the four parts of the fetch gate, and nothing else** (ADR Fetch prep and score 5.2 and Settings). Free rows of field, mode and values over twelve fields sat under Advanced, first to keep a rule an older form had saved; a row on seniority or pay was a permanent cut on fields half the roles leave empty, which the rail's filters do with a click and undo. The rows are gone; a stored gate the form cannot say is named under the fetch gate and kept as it is on save, so nothing is lost silently.
**Viewing as on Roles is a select, also with one agent.** The owner looked for the select to switch agent and found it gone after the second agent was switched off. It lists the enabled agents, since an agent that is off has no fresh scores.
+6 -36
View File
@@ -1,48 +1,18 @@
# Open
What is known and not yet fixed, checked against the code on 2026-10-05, the day of the architecture review. What was decided is in `decisions.md`, the target design in `documentation/`, the unbuilt readers in `plan-hosts-wave-2.md`. Counts are of the live database on that day.
What is known and not yet fixed, checked against the code on 2026-10-06, after the architecture review of 2026-10-05. What was decided is in `decisions.md`, the target design in `documentation/`, the unbuilt readers in `plan-hosts-wave-2.md`. Counts are of the live database on that day.
## For the owner to decide
Points where the code and `ADR Fetch prep and score` disagree, or where the ADR is silent and roles are lost.
- **Remote roles on hosts that cannot filter on remote.** On 18 boards (SmartRecruiters, Phenom, SuccessFactors, Eightfold, and Workday tenants without a remote facet) the read for "DK, SE, remote" goes out as "DK, SE". The row says "remote not asked" and the remote roles already held are kept, but a new remote role elsewhere in the world gate is never fetched. ADR 1.2 ("a host that cannot filter on a part sends the board whole for that part") reads as reading such boards whole; Bosch is then about 50 requests a pull.
- **A role outside the agent's locations with remote unstated is never read by extract.** ADR 4.1 says a role can "turn out to be remote", but extract reads only the cut and the cut requires the gate. 5,016 open roles are rejected for location with remote unknown, and 805 of them say "remote" in their text. A word scan in prep costs no tokens and would let those through to extract.
- **Copies across countries.** ADR 1.4 requires the same country and overlapping towns. `dupes.fold` also merges one host's listings of one title across towns and countries when 30% of their words are shared: 2,125 open copies, about 1,550 in another country than the kept role, some of them different openings (Datadog's US federal role behind the Canberra one). The gate reads the kept role through its copies' places, so no office listing is hidden. Remote and office listings are kept apart since 2026-10-05; 176 remote listings still sit behind a remote kept role in another country, whose area alone the world gate reads.
- **Judge and languages.** ADR 4.2 names the agent's languages as an input. The judge does not read them (they only set the flag on a row), and editing them re-judges nothing. The ADR's list also omits the prompts and the must-haves, which the judge reads and which re-judge.
- **Must-haves.** A broken must-have caps career at 4, but `must_text` is set only through the API: it is not on the form and not in the ADR's Settings.
- **Show gated and Advanced rules.** ADR 5 says no order shows a role the gate rejects and that seniority and pay are filters. Show gated lists the rejected roles with the reason, on purpose; an Advanced row can gate on any of twelve fields.
- **Closing by age on boards.** ADR 1.3 closes a role no read has listed for 21 days. That holds for searches and for boards narrowed by terms. A board read that stays incomplete (a ceiling, the page cap without a sweep) closes nothing ever, and on a filtered board a role with any vote or an application never closes. Since closing is not valued, striking the rule from the ADR is the cheaper way to agree.
- **Remote on hosts that take one country a request.** SmartRecruiters, SuccessFactors and Eightfold (14 boards, about 15,000 roles) are still asked for DK, SE alone when the agents want remote, since Europe there is the whole board. Reading them whole is the only way to their remote roles elsewhere.
- **Remote listings across countries.** A job posted remote once per country is now one role per country (the fold keeps to one country), which lists a pan-European remote posting several times.
- **A search cut at its page cap leaves no record.** In the last pull 4 of 26 searches stopped at a cap (Jobbank and freehire at 1,000 rows, two Jobindex searches at 200). Nothing on a page says so, and Jobbank's feed is not sorted by date, so new ads can lie past the cap.
- **A row without a title, link or id is dropped by its reader, unrecorded,** while its id still counts toward completeness. `ADR Plugins` says both "never drops rows" and "title and url are required"; the runner could validate, log and mark the read incomplete.
- **The world gate is not sent to a host when an agent sets no location.** Every big board is then read whole. Nothing is lost.
## What the ADRs do not say
## What the ADRs still do not say
What someone rebuilding from `ADR Fetch prep and score` would have to guess, by section.
`ADR Fetch prep and score` was brought up to the code on 2026-10-06; left for the owner's read as a whole.
- **Prep has no section**, though the title names it: a host's words mapped through a table per host, a place text resolved against a gazetteer of towns, regions, countries and continents with regions across borders, remote and the remote area read from the place words, the seniority ladder, and that a text nothing maps is listed and passes the gate.
- **1.1 Fetch gate.** Every role is stored whatever the gates say, and each agent applies its gate at scoring. An unknown location is no resolved place and no country, which includes an unresolved text and a wider region alone. A role in several places needs one inside. The world gate reads a remote role by its remote area. Exclusion matches whole words of the title.
- **1.2 Reading a board.** The union is per agent the location gate, or the world gate when no location is set, then over the agents. What happens when a host filters countries but not remote. A board's own saved params win over the agents'. With no enabled agent every board is read whole. A search reads 5 pages by default and 10 at most.
- **1.3 Closing.** "Filtered read" is not defined. Switching a board or search off closes its roles; a closed role listed again reopens.
- **1.4 Copies.** The second rule, one host's listings of one title merged when their texts are alike. Groups never chain. Votes and read state move to the kept copy. How titles and company names are normalised.
- **2.1 Closeness.** The embedding model (`intfloat/multilingual-e5-small` with its query and passage prefixes) and what is embedded (title and the first 2,000 characters of the text, else the teaser). The CV is cut into chunks of at most 1,200 characters. A level is a linear clip between two cosines measured per model (prompt 0.75 to 0.88, CV 0.77 to 0.90, job like -0.14 to 0.19). With several prompts each is scaled and the best counts. The bonus divides by all four weights also when one has left the sum.
- **2.2 Filter.** The bullets carry the "and" and the "or" by layout only. A liked or saved role must still pass the gate. A like is this agent's; a save under any agent counts.
- **3 Fetch text.** Only hosts with a role page are read, Jobindex never in a pull, and the caps (200 a board, 1,000 a run, retry after 7 days) are in `ADR Plugins` only.
- **4 Model analytics.** The model of each call and the batch sizes (20, 5, 10). Extract reads the first 2,500 and last 1,500 characters and also fills pay, currency, function and work permit; it gives up after 3 unanswered tries. The judge reads up to 8,000 characters, rates career against the prompts, likes and dislikes, and the CV match is 10 times the weighted mean, 0 to 100. Summarise writes about 60 words and 3 to 8 tags, once. A stage that is off, or has no model to ask, is skipped without a notice; only Settings › Sources › Runs says so. A cut role without text is neither judged nor summarised.
- **5 Feed.** Open roles and kept copies only. Opening or voting marks a role read. The list of filters, 60 rows a page, that ★ also likes, and what a role without an embedding scores. Newest is a third order, where the glossary and the flow say two.
- **Settings.** The agent's On/Off switch and the pull's schedule (every 6 h).
Wording to settle: 2.1 "over the sum of the want feed score weights" against 5.1.3 "its weight leaves the sum"; 4 "once extract has completed the ingestion" against 4.1 "a role that enters is extracted on the next pull"; 4.1's field list against "raw and processed fields for e.g. salary"; "model analytics score" is never defined (the number is the CV match); 1.3 against `ADR Plugins` 3.2, which lets a search declare `stated`, and 2.1, which says an incomplete read closes nothing; the title is "Architecture" while the other ADR calls it by its file name.
`ADR Plugins`, read lightly since it is not final:
- **The API block already differs from the code:** `empty` is "trusted" or "suspect", not None; `Page` is a plain tuple; `Reads` has an unused `title="provisional"`; a provider must define `allowed`; a reader may import `desk`.
- **Rules the code needed and the ADR lacks:** a cursor must survive JSON, where a tuple returns as a list; a reader that sees a page come again hands it over empty with a cursor; a wrong-shaped answer is an error, never an empty list.
- **Rules the code does not keep:** "at most 8 series" is enforced only with terms or more than 8 countries; the 10% guard starts at 1 role, not 10, for filtered, narrowed and suspect reads; a 429 after page one starts a sweep; a module that breaks the contract stops the import at the first problem, and with it the app.
- **"Never maps a word"** is broken by `Place.remote`, which makes every reader decide whether "hybrid" is office, and by the national providers that state their country (Jobindex DK, Platsbanken SE).
- **"Nothing else changes"** holds for a host. A provider with a new param or a key touches the Sources form, its script, the CLI, the API body and `sources.search_params`.
- Cases 4 and 5 are handled after the plugin, so "upstream" reads the wrong way round. The constants (0.5 s, 100 pages, 96 MiB, 21 and 7 days) will go stale faster than the rules.
`ADR Plugins` was rewritten on 2026-10-06 and names a target the code has not reached: a broken plugin left out instead of stopping the app; providers declaring their params with label, kind and choices, so form, CLI and API follow from the module; detail state kept for a search's roles as for a board's; a national provider declaring its implied country; page markers for hosts on a company's own domain; a contract test that names no reader.
## Structure
+40 -27
View File
@@ -1,10 +1,10 @@
# Architecture
# ADR Fetch prep and score
The target design of roledesk's ranking: how a role gets from a board to the Roles feed for one search agent, and the key decisions behind it. What gets built is tested against this document.
- **Fetch gate**: the agent's hard rules, a yes or no per role, in four parts: world, location, role exclusion and role fetch.
- **Input analytics score** and **input analytics filter**: what decides the **cut**, the roles that pass the analytics filter and get fetch text and the `model analytics score`.
- **Feed scores**: the two orders of the feed, the want feed score and the match feed score.
- **Input analytics score** and **input analytics filter**: what decides the **cut**, the roles that pass the analytics filter and get fetch text and the `model analytics score`, which is the `CV match` (4.2.1).
- **Feed scores**: the two scores that order the feed, the want feed score and the match feed score; Newest is the third order.
- **Fetch text**: the second HTTP read of a role's own page, without a model.
- **Extract**, **judge**, **summarise**: the model calls (Haiku).
- **Affinity**: an agent's setting for a company: hidden, neutral, liked or favourite.
@@ -17,6 +17,7 @@ The target design of roledesk's ranking: how a role gets from a board to the Rol
world · location · role exclusion · role fetch, with All locations and All roles as overrides
company boards: whole under 300 roles, else filtered at the host where it can
search providers: their own saved query
then prep, no tokens: host words, places, remote, country; the copies folded
│
▼
2 Input analytics score, no tokens ─ prompt closeness, CV closeness, affinity → the cut
@@ -28,7 +29,7 @@ The target design of roledesk's ranking: how a role gets from a board to the Rol
4 Model analytics score ─ extract the cut → fetch gate and cut again → judge the cut
│
▼
5 Feed ─ every role that passes the fetch gate, in two orders;
5 Feed ─ every role that passes the fetch gate, in three orders;
filters and saved filters narrow it
│
▼
@@ -51,7 +52,7 @@ The stricter location filter. A role passes the `location fetch gate` when it ma
#### 1.1.3 Role exclusion gate
A list of titles to leave out: phrases that reject a role by its title. No host can filter by exclusion, so this gate is applied to every role after fetch.
A list of titles to leave out: phrases that reject a role by its title, matched on whole words. No host can filter by exclusion, so this gate is applied to every role after fetch.
#### 1.1.4 Role fetch gate
@@ -64,26 +65,34 @@ The fetch terms: words a big board's host must match before it sends a role. The
- A search provider is read by its own saved query, which replaces the `role fetch gate` (1.1.4).
- A hidden company never passes.
The gate rejects on evidence only. A role with an unknown location passes until extract has read it.
The gate rejects on evidence only. A role with an unknown location passes until extract has read it. A location is unknown when no place of the role resolved and no country is stated, an unresolved place text included. A place that holds the agent's places (Øresund or Europe for Denmark) is no evidence against the role, unless the role also names a place elsewhere. A role in several places passes when one is inside. A hybrid or onsite role at an unknown location passes. A role whose text says remote while its fields do not (1.5) passes the location gate until extract has read it. The world gate reads a remote role by its remote area.
### 1.2 Reading a board
- A company board with fewer than 300 roles is read whole.
- A company board with 300 or more roles is asked for the fetch gate at the host: the world and location gates, and the fetch terms, where the host can filter on them. A host that cannot filter on a part sends the board whole for that part.
- A board's size is measured once, when it is added or rechecked, not on every read.
- One fetch serves all enabled agents. The host is asked for the union of their world and location gates and the union of their fetch terms; one enabled agent without fetch terms means no terms. Each agent then applies its own gate to what arrived.
- One fetch serves all enabled agents. The host is asked for, per agent, its location gate, or its world gate when it sets no location, united over the agents, and for the union of their fetch terms; one enabled agent without fetch terms means no terms. Each agent then applies its own gate to what arrived.
- A host that filters on countries but not on remote is asked for the world gate's countries when an agent wants remote roles, so a remote role listed under any country in the world gate still arrives; the read says that remote was not asked. This holds where the host takes the world in one read; on a host that takes one country a request, the world is the whole board, so the location gate is asked and remote roles elsewhere are not fetched.
- A board's own saved params win over the agents' gates on the dimensions they set. With no enabled agent every board is read whole.
### 1.3 Closing
- A role closes when a complete read of its board no longer lists it.
- A read that is not complete closes nothing: one narrowed by fetch terms, a search provider's, a partial or failed read. Roles on such boards close by age. This is accepted because ranking relies on recency, not on closing.
- A read that is not complete closes nothing: one narrowed by fetch terms, a search provider's, a partial or failed read. Roles on such boards close by age. This is accepted because ranking relies on recency, not on closing. A read filtered by location is complete for what it asked: it closes the roles inside its filter that it no longer lists.
- Switching a board or search off closes its roles. A closed role that a read lists again reopens. A search is read 5 pages deep, 10 at most.
- A role that no read has listed for 21 days closes. A board that has failed on every read for 21 days closes its roles.
- A liked or saved role is never closed by a filtered read.
### 1.4 Copies
- One posting listed several times (by a search provider and by the company's board, or by one board under several ids) is one role. Two listings are one posting when they share company, title and country, and their towns overlap; a listing without a town matches any town.
- The kept copy is the company board's over a search provider's, then the one with text, then the earliest posted. Only the kept copy is scored, read by a model and shown; it lists the others.
- The kept copy is the company board's over a search provider's, then the one with text, then the earliest posted. Only the kept copy is scored, read by a model and shown; it lists the others. Votes and read state move to the kept copy.
- A host that posts one job once per office gives listings of one title whose towns never meet. Those of one host are one role when their texts share 30% of their words, within one country, remote and office kept apart; the kept copy carries every office's place.
### 1.5 Prep
After fetch, without a model: a host's words for seniority, employment and remote map through a table per host; each place text resolves against a gazetteer of towns, regions, countries and continents, where a region may lie in two countries; remote and the remote area are read from the place words; the country comes from the resolved places; a role whose title or text says remote ("fully remote", "work from home", not "no remote") is marked as possibly remote for the gate (1.1). A text nothing maps is listed under Unmapped for the owner to add, and until then the role's location counts as unknown.
## 2 Input analytics score and filter
@@ -94,17 +103,18 @@ Each role is embedded and scored semantically:
- **Prompt closeness** is its level against the best of the agent's prompts.
- **CV closeness** is its level against the closest part of the CV.
Both are calibrated levels from 0 to 1. A company's affinity adds to both levels the share it has in the `want feed score`: the company like weight times the company like's rise above neutral (5.1.1), over the sum of the want feed score weights. On the start weights a liked company adds about 0.056, a favourite about 0.111. The bonus counts before the count and the minimum apply.
Both are calibrated levels from 0 to 1. A company's affinity adds to both levels the share it has in the `want feed score`: the company like weight times the company like's rise above neutral (5.1.1), over the sum of all four want feed score weights, also when one of them has left the sum. On the start weights a liked company adds about 0.056, a favourite about 0.111. The bonus counts before the count and the minimum apply.
The embedding is `intfloat/multilingual-e5-small`, with its query and passage prefixes, over the title and the first 2,000 characters of the text, else the teaser. The CV is cut into chunks of at most 1,200 characters and the closest chunk counts. A level is a linear clip of the cosine between two bounds measured per model: prompt closeness 0.75 to 0.88, CV closeness 0.77 to 0.90, job like -0.14 to 0.19. With several prompts each is scaled on its own and the best counts.
#### 2.2 Input analytics filter
The `closeness`es are the main input for the `input analytics filter`, which determines which roles get `fetch text` and the `model analytics score`. A role passes the `input analytics filter` when it:
- passes the `fetch gate` (1.1.1 to 1.1.3)
- passes the `fetch gate` (1.1.1 to 1.1.3), and
- is among the best N by `prompt closeness`, each at least X (500 and 0.6 to start), or
- is among the best M by `CV closeness` that the first list leaves out, each at least Y (150 and 0.6 to start), or
- is liked or saved not matter the `closeness`.
- is liked by this agent or saved under any agent, whatever its `closeness`.
The two lists are kept apart because they answer different questions (what I want, what I am), and a merged number lets the stronger side crowd the other out. For a broad agent the count stops each list; for a narrow agent the minimum stops it, so roles that do not fit are not read just to fill the count.
@@ -114,43 +124,42 @@ An agent without a CV has no CV list and no judge. An agent without prompts has
## 3 Fetch text
Some job boards have one API for the listings and another for the job specs. `fetch text` reads the role's own page for every role in the cut that came without text. Once the full text is fetched, the role is embedded again and its levels recomputed. A role that enters the cut after this recompute gets its text on the next pull.
Some job boards have one API for the listings and another for the job specs. `fetch text` reads the role's own page for every role in the cut that came without text, on hosts that have a role page, at most 200 a board and 1,000 a run, a failed read retried after 7 days; Jobindex is never read in a pull, since each read counts as a click there. Once the full text is fetched, the role is embedded again, its levels recomputed, and extract reads it again if it had read the title alone. A role that enters the cut after this recompute gets its text on the next pull.
## 4 Model analytics
All roles in the cut go through the detailed analysis and metadata extraction. Extract and judge are two calls: `extract` needs no CV, it is faster, and it is the final step of the ingestion, so it decides what finally passes the `analytics filter`.
All roles in the cut go through the detailed analysis and metadata extraction. Extract and judge are two calls: `extract` needs no CV, it is faster, and it is the final step of the ingestion, so it decides what finally passes the `analytics filter`. All three calls go to Haiku (`claude-haiku-4-5`), extract 20 roles a call, judge 5, summarise 10. A stage that is switched off, or has no model to ask, is skipped, and the Runs page says so. A cut role without text is neither judged nor summarised.
### 4.1 Extract
Often information is only in the free text, e.g. that a role in a specific location is actually remote. So every role has raw and processed fields for e.g. salary, location, remote and seniority. Extract reads what is hidden in the free text. A field the board states wins over what extract reads, except seniority and remote, where extract wins.
**Extract** reads the cut first. It fills location, remote area, seniority, languages and deadline.
**Extract** reads the cut first, the first 2,500 and last 1,500 characters of each role. It fills location, remote and remote area, seniority, languages, deadline, pay and currency, function and work permit. A role the model leaves unanswered three times is given up.
The fetch gate and the cut are then applied again with what extract filled. A role can drop out, such as one that turns out to be outside the agent's locations, or enter, such as one that turns out to be remote. A role that enters is extracted on the next pull.
The fetch gate and the cut are then applied again with what extract filled. A role can drop out, such as one that turns out to be outside the agent's locations, or enter, such as one that turns out to be remote. A role that enters is judged in this pull and extracted on the next.
### 4.2 Judge
Once extract has completed the ingestion, `judge` reads each role in the cut against the agent's CV, languages, likes and dislikes. It outputs:
Once extract has completed the ingestion, `judge` reads each role in the cut, up to 8,000 characters of it, against the agent's CV, languages, prompts, likes and dislikes. It outputs:
- the `CV match`
- one to three strengths and gaps
- a verdict on each must-have
Each role is extracted once. A judgement is redone only when its input changes: a change to the CV, a prompt, or the like or dislike text re-judges the whole cut (about an hour for 650 roles); a changed posting re-judges that role only.
Each role is judged once. A judgement is redone only when its input changes: a change to the CV, the languages, a prompt, or the like or dislike text re-judges the whole cut (about an hour for 650 roles); a changed posting re-judges that role only.
#### 4.2.1 CV match
Judge rates how closely the job spec matches the experience in the CV on three dimensions:
- skills,
- experience,
- career,
- skills, where a language the posting requires above the level spoken counts as a missing skill,
- experience,
- career, rated against the prompts, likes and dislikes rather than the CV,
each from 0 to 10. A model's rating is an estimate, not a measurement, so a coarse scale serves better than 0 to 100. The **CV match** is the three numbers weighted as given in the agent settings (30/25/30 to start). A must-have the role breaks caps career at 4. Roles with the same CV match are ordered by CV closeness.
each from 0 to 10. A model's rating is an estimate, not a measurement, so a coarse scale serves better than 0 to 100. The **CV match** is the three numbers weighted as given in the agent settings (30/25/30 to start), times 10, so 0 to 100. Roles with the same CV match are ordered by CV closeness.
## 5 Feed
The feed shows every role that passes the `world fetch gate`, the `location fetch gate` and the `role exclusion gate` (1.1.1 to 1.1.3), loaded as you scroll, in every order, newest first included. No order shows a role the fetch gate rejects. A hidden company's roles are never shown.
The feed shows every open role, kept copies only, that passes the `world fetch gate`, the `location fetch gate` and the `role exclusion gate` (1.1.1 to 1.1.3), loaded as you scroll, 60 at a time, in every order, newest first included. No order shows a role the fetch gate rejects. A hidden company's roles are never shown. Opening a role or voting on it marks it read; Show read, on by default, hides the read ones when switched off. Saving a role (★) also likes it.
### 5.1 Feed scores
@@ -161,6 +170,7 @@ The `want feed score` consists of `prompt closeness`, newness, company likes and
- `newness` counts from the posting date, the earlier of when the app first saw the role and the date the board states (only for boards whose dates are reliable), and halves every 14 days.
- `company like` is 1/3 for a neutral company, 2/3 for a liked one and 1 for a favourite.
- `job like` is described in 5.1.3.
- A role without an embedding yet is scored without `prompt closeness` and `job like`; their weights leave the sum.
#### 5.1.2 Match feed score, "What fits me"
@@ -185,16 +195,17 @@ The user likes or passes on roles in the feed, per agent. Each role has an embed
- The Roles page says that the feed is already filtered by the agent's fetch gate, location included.
- The company view says what is filtered: which parts of the fetch gate apply to this company, for example "every role in Europe, not filtered by country" for an All locations company. Looking up a company never silently hides roles elsewhere.
- Show gated lists the roles the fetch gate rejects, each with the reason, so what is kept out can be checked. It is the one way a rejected role is shown.
## 6 After the ranking
Summarise is not part of any score. It runs after the feed scores, on the top N of the `want feed score` and the top N of the `match feed score` (N set per agent, 30 to start, so at most 60 roles), and on every role liked or saved.
Summarise is not part of any score. It runs after the feed scores, on the top N of the `want feed score` and the top N of the `match feed score` (N set per agent, 30 to start, so at most 60 roles), and on every role liked or saved. It writes about 60 words and 3 to 8 tags, once per role, and needs the role's text.
## Settings
Per agent, on the form:
- name and prompts
- name, an On/Off switch, and prompts
- the fetch gate: world (the outer countries), location (countries, towns, remote), role exclusion (titles to leave out), role fetch (fetch terms)
- CV, languages, likes and dislikes
- the analytics filter: count and minimum by prompt closeness, count and minimum by CV closeness
@@ -202,3 +213,5 @@ Per agent, on the form:
- summarise size
Per agent, per company: affinity (hidden, neutral, liked, favourite). Per company, for every agent: All locations and All roles.
The pull runs every 6 hours.
+74 -48
View File
@@ -5,104 +5,112 @@ What a plugin has to deliver so that the flow in `ADR Fetch prep and score` gets
## Terms
- **Source**: a site that lists roles.
- **Plugin** (reader): the code for one kind of source. There are two kinds:
- **Plugin**: one module for one kind of source. There are two kinds:
- a **host**, which reads a company's own list on an ATS (Workday, Greenhouse, ...). That list is a **board**, attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
- a **provider**, which runs a **search**, a saved query on a site that lists roles across companies (Jobindex, Jobnet, ...).
- **Runner**: the shared code that calls a plugin. It asks, pages, retries, guards and stamps, so no plugin repeats that.
- **Runner**: the shared code that calls a plugin. It asks, pages, retries, guards, validates and stamps, so no plugin repeats that. It has one driver per kind, for boards and for searches, which differ only in what they store.
- **Row**: one role as a plugin hands it over, in the host's own words.
- **Declaration**: what a plugin states it can do (`READS`). The runner acts on the declaration, never on guesses.
- **Read**: one pull's requests to one target, and what they yielded. A read is **complete** when it listed every role the target has, else **incomplete**; the runner stamps which.
## 1 What a plugin delivers
A plugin turns one source into rows. It:
- hands over every role the source lists for what it was asked, as one row per role the host lists;
- hands over every role the source lists for what it was asked, one row per listing, as the host lists them;
- asks the host only for the filters it was handed, by building them into the host's query;
- keeps the host's own words for places, remote, seniority, employment and country;
- declares truthfully what the source can do: how the list arrives, where the text is, how completeness is known, what it can filter and which fields it states.
- declares truthfully what the source can do: how the list arrives, where the text is, how completeness is known, what it can filter and which fields it states;
- raises on anything it cannot read: an error page, an answer of another shape, a page that is not the page it asked for. An empty list is a valid answer only when the host says the list is empty.
A plugin never:
- filters, drops or reorders rows itself;
- filters, drops or reorders rows; a listing it cannot make a row of (no id, no title or no link) it hands over as far as it can, and the runner drops and counts it (3.8);
- maps a word to the app's vocabulary;
- merges, splits or multiplies rows;
- decides closing, retries or failure;
- reads robots.txt at run time.
- reads robots.txt at run time;
- logs in, runs a page's script, or uses a key taken from a site's script.
## 2 The five cases
A source can present roles in five ways that need handling. Three are the plugin's job and two are handled upstream.
A source can present roles in five ways that need handling. Three are the plugin's job and two are the app's, after the plugin.
| # | Case | Who handles it |
|---|---|---|
| 1 | The list arrives in pages | Plugin, with the runner |
| 2 | The list has no text; each role's text is on its own page | Plugin, with the fetch text stage |
| 3 | One role states several places | Plugin |
| 4 | One job is listed several times | Upstream |
| 5 | Facts such as remote are only in the free text | Upstream |
| 4 | One job is listed several times | The app |
| 5 | Facts such as remote are only in the free text | The app |
### 2.1 Pagination (plugin, with the runner)
The plugin says how to ask for one page and what the page holds. The runner does the rest.
- The plugin declares `listing="paged"`, the page size and the host's **ceiling**, the offset past which the host answers nothing whatever is asked (Workday 2,000).
- `page` asks for one page from a cursor and returns the page's rows, the raw results with their ids, the total the host states if any, and the next cursor, or none after the last page.
- The runner reads the pages one after another, 0.5 s apart, at most 100 pages a pull. A board above 100 pages is **swept**: each pull continues from the saved cursor.
- **Completeness**: a read is complete when the distinct ids read equal the stated total (`complete="stated"`), or when it reaches an empty page (`complete="end"`). A read that reaches the ceiling is incomplete, is not swept and closes nothing; only a filter brings such a board under the ceiling.
- `page` asks for one page from a cursor and returns the page's rows, the raw results with their ids, the total the host states if any, and the next cursor, or none after the last page. The first call gets cursor none. A cursor is any JSON value; a tuple comes back as a list when the runner hands it over again (a sweep), and a plugin reads it by position.
- The runner reads the pages one after another, 0.5 s apart, at most 100 pages a pull. A board above 100 pages is **swept**: each pull continues from the saved cursor, series and ids, and the sweep is finished when the read reaches the end.
- **Completeness**: a read is complete when the distinct ids read equal the stated total (`complete="stated"`), or when the plugin hands over the last page with no cursor (`complete="end"`: an empty page, or a short one where the host has no empty page). A read that reaches the ceiling is incomplete, is not swept and closes nothing; only a filter brings such a board under the ceiling.
- **Stalls**: a page that brings no new id while the plugin hands over a cursor is a stall when the host states no total, or when the page came empty: the host ignores the cursor, or sent a page again. A plugin that sees a page come again (the same first id as the page before) hands it over with no results and a cursor that is not none, never none, since none would read as the last page. A stalled read is incomplete. With a stated total, a page of ids already read is not a stall, since a resumed sweep starts on one when roles were added at the top since.
- A role of a swept board closes only when two finished sweeps in a row missed it, since roles closing between pulls shift later roles back past the cursor.
- A 429 after the first page ends the read. What was read is kept, and the read is incomplete.
- A 429 after the first page ends the read. What was read is kept, the read is incomplete, and a board continues from there in the next pull as a sweep.
- A plugin whose list comes in one answer declares `listing="whole"` and `complete="whole"`; a whole answer that parses is complete. One whose list comes in one answer that states a total it does not fill raises.
### 2.2 List without text (plugin, with the fetch text stage)
Some hosts list title, place and date only, and state the text on each role's own page.
- The plugin declares `text="detail"` and supplies `detail`, which reads one role's page and returns the fields it found. A list that carries a short excerpt hands it over as `teaser` and declares `teaser=True`.
- The plugin does not decide which roles are read. The fetch text stage calls `detail` after scoring, only for roles in the cut (`ADR Fetch prep and score`, section 3), at most 200 per board and 1,000 per run.
- The plugin does not decide which roles are read. The fetch text stage calls `detail` after scoring, only for roles in the cut (`ADR Fetch prep and score`, section 3), at most 200 per board and 1,000 per run, 0.5 s apart.
- `detail` may fill only description, location, places, teaser, remote, country, employment, pay figures, currency and the posting date. Title, link, department and the pay line come from the list, which states them on every pull.
- A role read in detail is sent by later pulls without the list's place and remote, so what its own page said stays.
- A 404 or 410 counts as read. Another failure is retried on a later run. A read that gave no text is retried after 7 days. A 429 ends the reads of that board for the run.
- A role read in detail is sent by later pulls without the list's place and remote, so what its own page said stays. This holds for a search's roles as for a board's.
- A 404 or 410 counts as read. Another failure is retried on a later run, after the roles never tried. A read that gave no text is retried after 7 days. A 429 ends the reads of that target for the run.
- Completeness, and so closing, is decided by the list alone.
### 2.3 One role, several places (plugin)
- A row carries `places`: every place the host states for the role, in the host's order and the host's words, each marked remote, office or unsaid.
- A row carries `places`: every place the host states for the role, in the host's order and the host's words, each marked remote, office or unsaid as the host marks that place. The host's own word for how the role is worked ("Hybrid", "Remote", "On-site") goes on the row as `remote`, unmapped.
- A plugin never makes one row per place, never drops a place and never joins places into one text.
- A host that states no place declares `places="none"` and hands over an empty list.
- Resolving a place text to a town, region or country is the app's job.
### 2.4 One job listed several times (upstream)
### 2.4 One job listed several times (the app)
One job can come as several rows: one per office, each with its own id; several requisition ids for the same job; or a search provider and the company's board listing the same role.
- The plugin hands every listing over as its own row, with its own id and link. A host that publishes one posting per place declares `places="copies"`.
- Folding them into one role, and choosing the copy that is kept, happens upstream (`ADR Fetch prep and score`, section 1.4).
- Folding them into one role, and choosing the copy that is kept, happens in the app (`ADR Fetch prep and score`, section 1.4).
- Two listings under one id are one row; a host that lists one id once per place hands over one row with all the places.
### 2.5 Facts in the free text (upstream)
### 2.5 Facts in the free text (the app)
- The plugin passes on only what the host states in a field: remote, country, seniority, employment, pay and date, in the host's words, and declares each as `stated` or `none`.
- Mapping the host's words to the app's vocabulary happens upstream, through a table per host.
- Facts found only in the prose, e.g. a Copenhagen role that is actually remote, are read upstream: from the place words first, then by extract (`ADR Fetch prep and score`, section 4.1).
- The plugin passes on only what the host states in a field: remote, country, seniority, employment, pay and date, in the host's words, and declares each as `stated` or `none`. A provider that covers one country states that country on every row and declares it `stated`.
- Mapping the host's words to the app's vocabulary happens in the app, through a table per host.
- Facts found only in the prose, e.g. a Copenhagen role that is actually remote, are read in the app: from the place words first, then by extract (`ADR Fetch prep and score`, section 4.1).
## 3 Further rules
### 3.1 Filtering at the host
- The plugin declares per dimension (country, place, remote, query) whether the host filters it: `none`, `one` (one value a request), `many` (a list in one request) or `ids` (the host's own ids, looked up with `facets`).
- The runner hands the plugin an `Ask` that holds only declared dimensions. With `one`, it runs one series of requests per value and unions the rows by id. A read costs at most 8 series. Above that the runner narrows less: a read for the location gate keeps the places and drops the fetch terms; a read of an All locations company keeps the terms and drops the place part; more than 8 terms alone are dropped.
- "In these countries, or remote" cannot be asked in one request, because two facets intersect. The runner makes remote a series of its own, so a plugin never gets remote and a country in the same `Ask`.
- The runner hands the plugin an `Ask` that holds only declared dimensions. With `one`, it runs one series of requests per value and unions the rows by id. A read costs at most 8 series. Above that the runner narrows less: more than 8 countries on a `one` host means a read without countries; when terms times the location series pass 8, a read for the location gate keeps the places and drops the terms, and a read of an All locations company keeps the terms and drops the place part; more than 8 terms alone are dropped.
- "In these countries, or remote" cannot be asked in one request, because two facets intersect. The runner makes remote a series of its own, so a plugin never gets remote and a country in the same `Ask`. A host that cannot filter on remote is asked for the countries alone, and the read is stamped as not having asked for remote, which the app shows and spares the remote roles it holds from closing.
- A host with `query="one"` also takes the fetch terms, one term a series, handed over as `query`.
- A host's facet ids are looked up at most once a day per board.
- Which boards are filtered at all, and by what, is decided upstream (`ADR Fetch prep and score`, section 1.2).
- A host's facet ids are looked up at most once a day per board. A value the host's facets do not know is dropped; a read for which nothing remains asks nothing, lists nothing and closes nothing.
- A board's own saved params win over the agents' reach on the dimensions they set.
- Which boards are filtered at all, and by what, is decided in the app (`ADR Fetch prep and score`, section 1.2).
### 3.2 Completeness and closing
The plugin declares how completeness is known (`stated`, `end`, `whole` for a one-answer list, `never`). The runner stamps each read as complete or not. Closing follows `ADR Fetch prep and score`, section 1.3. A search is never a company's whole list, so it declares `never` or `stated`.
The plugin declares how completeness is known (`stated`, `end`, `whole` for a one-answer list, `never`). The runner stamps each read as complete or not, as filtered (a location filter), as narrowed (fetch terms or a provider's query), and with the ids it listed. Closing follows `ADR Fetch prep and score`, section 1.3: a search never closes a role by its list, whatever it declares, so a provider's `stated` only says how far a read got.
### 3.3 Failures
- A plugin lets every error through: HTTP errors, timeouts, parse errors. The runner logs, records and decides.
- A read that lists under 10% of what the same read listed last time, once that was at least 10 roles, counts as failed. This catches a host that answers an empty list for a mistyped board; such a host declares `empty="suspect"`.
- A plugin lets every error through: HTTP errors, timeouts, parse errors, an answer of another shape. The runner logs, records and decides. An error on a later page fails the whole read; what a 429 read is kept (2.1).
- A read that lists under 10% of what the same read listed last time counts as failed. For a whole read of a board this holds once the last count was at least 10 roles; a filtered or narrowed read, and a read of a host that answers an empty list for an unknown slug (`empty="suspect"`), is held from a count of 1. "The same read" is one with the same filters: a changed filter starts the count again.
- A board that fails on every read for 21 days closes its roles, and a suspect answer that has stood that long is accepted.
- A failing target is named on its company's row and in the pull's notice, with the reason.
### 3.4 Fetching
@@ -114,13 +122,15 @@ Every request goes through the runner's `get`:
- 10 s to connect, 30 s between bytes, 60 s for the whole request;
- at most 96 MiB a response. A plugin may name a smaller fallback feed, read once when the main one is over the cap.
The figures in this document are the defaults as they stand; the rules are what counts.
### 3.5 Searches
A provider has saved params instead of a board. It declares which filter dimension each param fills, cleans a param it would not accept with a reason a person can read, and receives the saved params in `Ask.params`. A search row carries `company`, the employer's name. A provider has no slug patterns, probes or careers page, and takes no fetch terms: its query replaces them. A provider that needs an API key declares it, and the runner hands it the key.
A provider has saved params instead of a board. It declares each param: the filter dimension it fills, its label, its kind (text, a choice from a table, a list of country codes) and the table of choices where there is one, so the form, the command line and the API offer a provider's params from the declaration alone. It cleans a param it would not accept with a reason a person can read, and receives the saved params in `Ask.params`. A search row carries `company`, the employer's name. A provider has no slug patterns, probes or careers page, and takes no fetch terms: its query replaces them. A provider that needs an API key declares it, and the runner hands it the key; a search reads 5 pages by default and 10 at most.
### 3.6 Discovery
A host can be found from a careers page link: its slug patterns are regexes over a URL, with the slug in group 1, or several groups joined by `/`. A host may offer a cheap probe URL that answers 2xx only for a real board, and a check of the probe's body where 2xx is not enough. The slug is lowercased unless the host's paths are case-sensitive.
A host can be found from a careers page link: its slug patterns are regexes over a URL, with the slug in group 1, or several groups joined by `/`. A host may offer a cheap probe URL that answers 2xx only for a real board, and a check of the probe's body where 2xx is not enough. The slug is lowercased unless the host's paths are case-sensitive. A host whose boards live on a company's own domain names the page markers that identify it, and the path discovery confirms with.
### 3.7 robots.txt
@@ -128,18 +138,22 @@ Whoever builds a plugin reads the robots.txt of every host it requests, for the
### 3.8 Rows
- `source_id` is the host's id, unique on the board or search. `title` and `url` (http or https) are required.
- `source_id` is the host's id, unique on the board or search. `title` and `url` (http or https) are required. The runner, not the plugin, drops a row that lacks one of them; the read's record counts the rows dropped, and a board whose dropped rows pass 10% of its list counts as failed, since a changed page shape looks like that.
- `description` and `teaser` are plain text, never HTML.
- `posted` is an aware datetime or none. `posted="trusted"` declares it the posting date; `listed` declares a date the host moves, e.g. on an edit.
- `posted` is an aware datetime or none. `posted="trusted"` declares it the posting date; `listed` declares a date the host moves, e.g. on an edit. A plugin whose only date comes from the role's page or from a field the host moves declares `listed`.
- Pay figures are yearly integers with an ISO 4217 currency. A monthly figure is multiplied by 12 where the host states the period. An hourly, daily or weekly figure is kept as a pay line only.
- `country` is ISO 3166 alpha-2, and only where the host states it.
- `country` is ISO 3166 alpha-2, and only where the host states it (or the provider covers that one country, 2.5).
- Any other key the host states is kept as raw, so a new field needs no change in the runner.
### 3.9 What the runner keeps per target
So that a plugin keeps nothing: the last read's marker, whether it was complete, filtered, narrowed and asked for remote, the ids it listed, the sweep in progress, the roles read and failed in detail, the facet ids and when they were read, the board's size and when it was measured, the last count the guard compares with, and the failure record with its first date and reason. All of it is shown or summarised on the board's row.
---
## API
A plugin is one module, named after its source, importing only the shared plugin helpers, the standard library, `httpx` and `pydantic`. It registers by existing: a module that breaks the contract fails at import, naming every problem.
A plugin is one module, named after its source, importing only the shared plugin helpers, the standard library, `httpx`, `pydantic` and the shared model base. It registers by existing: a module that breaks the contract is refused at import, every problem named; the other plugins load, and the Sources page names the refused one.
### Module attributes
@@ -149,7 +163,11 @@ LABEL = "Workday" # shown in the source select
PATTERNS = (r"...",) # host only: discovery regexes over a URL, IGNORECASE
RANK = 100 # optional: order in the select and in probes, lower first
KEEP_CASE = False # optional: True keeps a discovered slug's case
PARAMS = {"q": "query"} # provider only: saved param -> filter dimension it fills
MARKERS = ("...",) # optional, host on a company's own domain: page markers
PARAMS = { # provider only: saved param -> what it is
"q": Param(fills="query", label="Query", kind="text"),
"region": Param(fills="place", label="Region", kind="choice", choices=REGIONS),
}
READS = Reads(
scope="board", # "board" | "search"
@@ -167,7 +185,7 @@ READS = Reads(
# pay "figures" | "line" | "none"; posted "trusted" | "listed" | "none";
# the rest "stated" | "none"
key=None, # Key(name, fields, env) when an API key is needed
empty=None, # "suspect" when an unknown slug answers an empty list
empty="trusted", # "suspect" when an unknown slug answers an empty list
robots="read: ... (checked 2026-09-30)", # required, with the date
)
```
@@ -175,10 +193,12 @@ READS = Reads(
### Functions
```python
# Every plugin
def allowed(slug: str) -> Callable[[str], bool] # the host names a request may reach
# Every host
def slug_ok(slug: str) -> bool # full match; refuses anything that adds a path,
# query, fragment, port or user to a URL
def allowed(slug: str) -> Callable[[str], bool] # the host names a request may reach
def careers_url(slug: str) -> str # the page a person opens
# listing="whole"
@@ -213,18 +233,20 @@ def probe_slugs(guess: str) -> tuple[str, ...] # slugs a guessed company name
```python
get(url, *, method="GET", payload=None, headers=None) -> bytes
# POST sends payload as JSON. Raises httpx.HTTPStatusError on non-2xx,
# httpx.HTTPError on timeout or connection failure, ValueError on a refused
# host, an oversized body or a broken redirect. A plugin lets these through.
# httpx.HTTPError on timeout or connection failure, and the runner's own
# errors on a refused host, an oversized body or a broken redirect. A
# plugin lets these through.
Ask(countries=(), places=(), remote=None, remote_ids=(), query=None, terms=(), params={})
Ask(countries=(), places=(), remote=None, remote_ids=(), query="", terms=(), params={})
# only declared dimensions are set; with "one" a series carries one value;
# with "ids" countries and places hold host ids; never remote together with
# a country or place; terms are handed over one a series as query, so a
# plugin reads query only; params holds a search's saved params and a key under "key"
Page(decoded, results, total, cursor)
Page = (decoded, results, total, cursor)
# decoded: handed to rows; results: raw dicts with an "id", counted by
# distinct id; total: stated by the host, or None; cursor: next page, or None
# distinct id; total: stated by the host, or None; cursor: next page, or
# None after the last; any JSON value, a tuple returning as a list
Place = {"text": str, "remote": bool | None}
# the host's words; True remote, False office, None unsaid
@@ -237,20 +259,24 @@ Row = {
"country": str, "seniority": str, "employment": str,
"company": str, # search rows only
}
Param(fills, label, kind, choices=None)
# fills: "query" | "country" | "place" | "remote" | "pages" | "" (the host's own);
# kind: "text" | "choice" | "countries" | "number"; choices: {value: label}
```
### Helpers the plugins share
- `place(text, remote=None)` makes a `Place`, remote when the text says remote; `add_place(found, entry)` appends one unless it is empty or already there.
- `html_text(html)` turns HTML into plain text; `parse_dt(value)` makes an aware datetime; `yearly_amount(...)` makes a yearly figure; `is_web_url(url)` checks a link.
- `html_text(html)` turns HTML into plain text, one line per paragraph, with list items and headings marked; `parse_dt(value)` makes an aware datetime; `yearly_amount(...)` makes a yearly figure and `pay_line(...)` a pay line; `is_web_url(url)` checks a link.
- `path_slug_ok(slug)` and `label_slug_ok(slug)` check a slug that is one URL path segment or one DNS label; `only(host)` is the usual `allowed`.
- A table of the country names hosts write.
- A table of the country names hosts write, both ways (`code_of`, `name_of`).
### Adding a plugin
1. Write the module with its declaration and functions.
2. Add a table of the host's words for seniority, employment and remote, when the host states them in its own words.
3. Add fixtures from real answers, and tests. A contract test checks each declared field against the fixtures: a field declared `stated` must show on some row, a field declared `none` on none.
3. Add fixtures from real answers, and tests. A contract test, shared by every plugin and naming none, checks each declared field against the fixtures: a field declared `stated` must show on some row, a field declared `none` on none; required row keys, plain text, aware dates, ISO codes and integer pay; that results carry ids and cursors survive JSON; that a declared filter reaches the request.
4. Record the robots.txt decision in the declaration.
Nothing else changes: the list of plugins, the source select, discovery and the filters a board is read with all follow from the module.
Nothing else changes: the list of plugins, the source select, discovery, a provider's form, the filters a board is read with and what its row says all follow from the module.
+14 -6
View File
@@ -17,8 +17,9 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is read independently; one that fails is logged,
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
whose read lists under a tenth of what it listed last time (`_runner.read`'s
guard), until it has failed that way for the pull's `stale_days`. A board's
whose read lists under a tenth of what it listed last time, or drops more
than a tenth of the rows its reader handed over (`_runner.read`'s guards),
until it has failed that way for the pull's `stale_days`. A board's
filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`) win, narrowed to what its reader declares
@@ -343,8 +344,10 @@ SIZE_PAGES = 100
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
"""How many roles board `target` lists, unfiltered and without terms: a
whole host's listing, the total a `complete="stated"` host gives on page
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
as `_fetch_rows` does, so a board that does not answer is refused."""
1, else the listing read to the end, at most `SIZE_PAGES` pages. A read
counts the rows the runner dropped as well (`_listed`), as a stated total
does. Raises as `_fetch_rows` does, so a board that does not answer is
refused."""
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
if client is None:
@@ -358,7 +361,12 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
if isinstance(total, int):
return total
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
return _listed(_runner.read_listing(module, slug, get, limits=limits))
def _listed(listing: _runner.Listing) -> int:
"""The listings a read found: the ids it kept and the rows it dropped."""
return len(listing.ids) + listing.dropped
def keep_size(state: Any, target: str, size: int, *, today: str = "",
@@ -462,7 +470,7 @@ class ATSSource(Source):
[compound(i) for i in listing.ids])
self.state.pop(failing, None)
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
keep_size(self.state, target, len(listing.ids), reset=False)
keep_size(self.state, target, _listed(listing), reset=False)
if detail_sets is not None:
# A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
+7 -2
View File
@@ -179,7 +179,7 @@ ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
own id for the role, unique on the board), `title`, `url` (http or https,
checked with `_shared.is_web_url`) and `places`; `location` (the host's
which the runner checks with `_shared.is_web_url`) and `places`; `location` (the host's
one-line place, kept while storage reads it), `posted` (an aware datetime
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
see `_shared.html_text`), `remote` (True, False, a word or None) and
@@ -187,7 +187,12 @@ see `_shared.html_text`), `remote` (True, False, a word or None) and
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
every stated key in `raw`, so a new field needs no change there.
every stated key in `raw`, so a new field needs no change there. A reader
never drops a listing: one it cannot make a whole row of (no id, no title or
no link) it hands over with what it has, an empty string for what is
missing, and the runner drops it and counts it (`_runner.valid`). A reader
leaves out only what is not a listing of the board: an entry that is not an
object, a second copy of an id, a role of another board.
**Places.** A role has a list of places: `places` is `[Place(text=...,
remote=...)]`, one entry per place the host states, in the host's order, in
+2
View File
@@ -205,6 +205,8 @@ class Row(TypedDict, total=False):
`company` is the employer's name on a search row, which lists many
(`plugins/search.py` stores it); a board row carries none, the board is
the company. Any other stated key is kept in `raw` by the runner.
`source_id`, `title` and `url` come empty from a listing that lacks
them: the runner drops such a row and counts it (`_runner.valid`).
"""
source_id: Required[str]
+98 -20
View File
@@ -4,10 +4,12 @@ the reader declares in `READS`.
A reader says how its host answers; this module reads it: the guarded fetch
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
to (`ask_for`) and its series (`series`), the listing whole or paged, one
series per `one`-filter value unioned by id (`read_listing`), completeness,
the sweep of an oversized paged board across pulls, the 10% guard on every
read that is not a sweep (`read`), the per-role detail reads
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
series per `one`-filter value unioned by id (`read_listing`), the rows it
keeps (`valid`: a row without an id, a title or a link is dropped and
counted here, never by the reader), completeness, the sweep of an oversized
paged board across pulls, the two 10% guards (`read`: on what a read that
is not a sweep listed, and on what any read dropped), the per-role detail
reads (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
`plugins/search.py` are thin `Source` classes over it: they pass their own
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
@@ -30,6 +32,8 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
drifts when roles close between pulls, so `app.jobs.pull` closes
a role of such a board only when two finished sweeps in a row
missed it (it keeps the ids the last one missed in `missing`)
dropped the rows the last stamped read dropped (`Listing.dropped`), as a
number, "" for none; `app.jobs.limits` names it on the board's row
"""
from __future__ import annotations
@@ -52,7 +56,7 @@ import httpx
from desk.sources import USER_AGENT
from ._contract import Ask, Key, Place, Reads
from ._shared import Get, Row
from ._shared import Get, Row, is_web_url
log = logging.getLogger(__name__)
@@ -96,7 +100,8 @@ SUSPECT_SHARE = 0.1
#: The last count from which the guard holds every read. A read that is
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
#: a count of 1, as it was before every read was; a whole read of a board
#: that listed a handful of roles can drop to none by itself.
#: that listed a handful of roles can drop to none by itself. It is also the
#: rows a read must hand over before the share it dropped can fail it.
SUSPECT_MIN = 10
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
@@ -113,8 +118,8 @@ class TooLarge(ValueError):
class Suspect(Exception):
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
(see `read`)."""
"""A read that listed under `SUSPECT_SHARE` of what it listed last time,
or dropped more than that share of its rows (see `read`)."""
class MissingKey(ValueError):
@@ -147,7 +152,9 @@ class Listing:
and roles closing meanwhile may have shifted an offset-paged host's later
roles back past the cursor, so `ids` can miss a role that is still open.
`remote_unasked` says a filtered read was meant to ask for remote roles
as well and could not, so its `ids` say nothing of the remote roles."""
as well and could not, so its `ids` say nothing of the remote roles.
`handed` counts the rows the reader handed over this pull, `dropped`
those of them `valid` refused, which are in neither `rows` nor `ids`."""
rows: list[Row]
complete: bool
@@ -158,6 +165,8 @@ class Listing:
narrowed: bool = False
swept: bool = False
remote_unasked: bool = False
handed: int = 0
dropped: int = 0
# --------------------------------------------------------------- modules
@@ -524,8 +533,42 @@ def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
def valid(row: Mapping[str, Any]) -> bool:
"""Whether the runner keeps a row: a non-empty `source_id`, a non-empty
`title` and a web URL (`is_web_url`). A reader hands over every listing
as far as it can make it out; a row without one of the three cannot be
stored or opened, and the runner drops it and counts it."""
try:
link = is_web_url(str(row.get("url") or ""))
except (httpx.InvalidURL, ValueError): # a string httpx cannot read as a URL
link = False
return (link and bool(str(row.get("source_id") or "").strip())
and bool(str(row.get("title") or "").strip()))
@dataclass(slots=True)
class _Tally:
"""The rows a read was handed and dropped, across its series and pages."""
handed: int = 0
dropped: int = 0
def _keep(found: Iterable[Row], rows: dict[str, Row], tally: _Tally) -> None:
"""The rows a reader's `parse` or `rows` returned, into `rows` by
`source_id`, the first of an id kept; those `valid` refuses are counted
in `tally` and left out. The one place a row is checked, for the whole
and the paged listing alike."""
for row in found:
tally.handed += 1
if not valid(row):
tally.dropped += 1
continue
rows.setdefault(row["source_id"], row)
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
limits: Limits) -> dict[str, Row]:
limits: Limits, tally: _Tally) -> dict[str, Row]:
"""A whole listing, one request a series, unioned by `source_id`. A body
over the cap gets one retry at the reader's `fallback_url`."""
rows: dict[str, Row] = {}
@@ -541,8 +584,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
module.NAME, slug)
body = get(module.fallback_url(slug))
for row in module.parse(body, slug):
rows.setdefault(row["source_id"], row)
_keep(module.parse(body, slug), rows, tally)
return rows
@@ -580,7 +622,29 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
only: a search's places are what the search is, as its query is.
`remote` says the reach or the params wanted remote roles; when no series
asked for them (a reader without a remote filter, or one whose facets
state none beside places) the listing is `remote_unasked`."""
state none beside places) the listing is `remote_unasked`.
Every row `rows` or `parse` returns goes through `valid`: one without a
`source_id`, a title or a web URL is dropped here, counted in
`Listing.dropped` and logged once for the read. Completeness counts the
distinct result ids `page` returns beside its rows, not the rows kept, so
a dropped row still counts as read; a whole listing is complete whenever
it parses, whatever it dropped."""
tally = _Tally()
listing = _listing(module, slug, get, ask, limits=limits, sweep=sweep,
can_sweep=can_sweep, facets=facets, remote=remote, tally=tally)
listing.handed, listing.dropped = tally.handed, tally.dropped
if tally.dropped:
log.warning("readers: %s:%s: dropped %d of %d listings: no id, title or link",
module.NAME, slug, tally.dropped, tally.handed)
return listing
def _listing(module: ModuleType, slug: str, get: Get, ask: Ask | None, *,
limits: Limits | None, sweep: Mapping[str, Any] | None, can_sweep: bool,
facets: Mapping[str, Mapping[str, str]] | None, remote: bool,
tally: _Tally) -> Listing:
"""`read_listing` before its count of dropped rows, which `tally` keeps."""
limits = limits or Limits()
ask = ask or Ask()
reads: Reads = module.READS
@@ -595,7 +659,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
"nothing asked, not closing", module.NAME, slug)
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
if not hasattr(module, "page"):
whole_rows = _whole(module, slug, get, asks, limits)
whole_rows = _whole(module, slug, get, asks, limits, tally)
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
key, by_words, remote_unasked=unasked)
@@ -632,8 +696,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
pages += 1
before = len(seen)
seen |= _result_ids(results)
for row in module.rows(data, slug):
rows.setdefault(row["source_id"], row)
_keep(module.rows(data, slug), rows, tally)
if cursor is None or (isinstance(total, int) and len(seen) >= total):
ended = True
break
@@ -747,13 +810,26 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
that really shrank is accepted once the target has been failing for
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
stamp then resets the count; a Recheck resets it at once
(`plugins.ats.keep_size`)."""
(`plugins.ats.keep_size`).
A read, a sweep's included, that dropped more than `SUSPECT_SHARE` of
the rows its reader handed over (`valid`), once it was handed at least
`SUSPECT_MIN`, raises `Suspect` the same way: a page whose shape changed
parses to rows without a title or a link, and the roles it hid would
close. It gives way after `limits.stale_days` of failing as well."""
limits = limits or Limits()
ask = ask or Ask()
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
remote=remote)
if listing.handed >= SUSPECT_MIN and listing.dropped > SUSPECT_SHARE * listing.handed:
why = (f"dropped {listing.dropped} of {listing.handed} listings, over "
f"{SUSPECT_SHARE:.0%}: no id, title or link")
if not _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
raise Suspect(why)
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
module.NAME, slug, why, limits.stale_days)
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
last = _load(state, f"{prefix}:count:{target}")
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
@@ -788,9 +864,10 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
`complete`, `ids` (the compound ids, what the pull closes against),
`filtered`, `narrowed` (a read by terms, which the pull never closes
against), whether it finished a sweep from earlier pulls (`swept`),
whether a filtered read left remote unasked (`remote_unasked`), the
sweep to continue or none, and the count for the 10% guard once a read
is not a sweep. A read that asked nothing keeps the count it had."""
whether a filtered read left remote unasked (`remote_unasked`), the rows
it dropped (`dropped`, "" for none), the sweep to continue or none, and
the count for the 10% guard once a read is not a sweep. A read that
asked nothing keeps the count it had."""
state[f"{prefix}:ok:{target}"] = run
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
@@ -798,6 +875,7 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
state[f"{prefix}:dropped:{target}"] = str(listing.dropped) if listing.dropped else ""
if listing.sweep is not None:
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
return
+1 -3
View File
@@ -24,7 +24,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Key, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, only, parse_dt
NAME = "adzuna"
LABEL = "Adzuna"
@@ -111,8 +111,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
for job in decoded["results"]:
title = _text(job.get("title") or "")
url = (job.get("redirect_url") or "").strip()
if not title or not is_web_url(url):
continue
predicted = str(job.get("salary_is_predicted") or "0") == "1"
town = ((job.get("location") or {}).get("display_name") or "").strip()
places: list[Place] = [{"text": town, "remote": None}] if town else []
+1 -3
View File
@@ -14,7 +14,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
log = logging.getLogger(__name__)
@@ -126,8 +126,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in data.get("jobs") or []:
title = (j.get("title") or "").strip()
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
if not title or not is_web_url(url):
continue
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
compensation = j.get("compensation") or {}
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
-2
View File
@@ -106,8 +106,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in listed:
title = (j.get("jobOpeningName") or "").strip()
source_id = str(j.get("id") or "")
if not title or not source_id:
continue
places = _places(j)
row: Row = {
"source_id": source_id,
-2
View File
@@ -132,8 +132,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for p in postings:
title = (p.get("title") or "").strip()
post_id = str(p.get("id") or "")
if not title or not post_id:
continue
job = p.get("job") or {}
places = [name for loc in job.get("jobLocations") or []
if (name := ((loc.get("location") or {}).get("name") or "").strip())]
+1 -3
View File
@@ -47,7 +47,7 @@ from urllib.parse import quote
from ._contract import Ask, Fields, Filters, Place, Reads, Row
from ._countries import name_of
from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place
from ._shared import Get, Page, add_place, html_text, only, parse_dt, place
READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
@@ -151,8 +151,6 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
source_id = str(p.get("id") or "")
path = str(p.get("positionUrl") or "").strip()
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
if not title or not source_id or not is_web_url(url):
continue
places = _places(p.get("locations") or p.get("standardizedLocations"))
out.append({
"source_id": source_id,
+1 -3
View File
@@ -37,7 +37,7 @@ from urllib.parse import urlencode, urlsplit, urlunsplit
from desk.sources import USER_AGENT
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line,
from ._shared import (Get, Page, Row, html_text, num, only, parse_dt, pay_line,
yearly_amount)
NAME = "freehire"
@@ -170,8 +170,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
title = " ".join(html.unescape(str(hit.get("title") or "")).split())
source_id = str(hit.get("public_slug") or "").strip()
url = _url(str(hit.get("url") or ""))
if not title or not source_id or not is_web_url(url):
continue
mode = str(hit.get("work_mode") or "").strip()
places = _places(hit, mode)
countries = [str(c) for c in hit.get("countries") or [] if c]
-2
View File
@@ -116,8 +116,6 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
for p in postings:
title = (p.get("title") or "").strip()
ext_id = str(p.get("extId") or "")
if not title or not ext_id:
continue
locations = p.get("locations") or []
places: list[Place] = []
for loc in locations:
+1 -3
View File
@@ -13,7 +13,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place
NAME = "greenhouse"
LABEL = "Greenhouse"
@@ -149,8 +149,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in data.get("jobs") or []:
title = (j.get("title") or "").strip()
url = (j.get("absolute_url") or "").strip()
if not title or not is_web_url(url):
continue
departments = j.get("departments") or []
ranges = j.get("pay_input_ranges") or []
salary_min, salary_max, currency = _salary(ranges)
-2
View File
@@ -90,8 +90,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
for ad in data.get("jobAdDetails") or []:
title = (ad.get("title") or "").strip()
ad_id = str(ad.get("id") or "").strip()
if not title or not ad_id:
continue
site = str(ad.get("site") or "").strip()
country = str(ad.get("country") or "").strip()
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
+1 -3
View File
@@ -30,7 +30,7 @@ from typing import Any
from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, Page, html_text, is_web_url, only, parse_dt
from ._shared import Get, Page, html_text, only, parse_dt
READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="end",
@@ -106,8 +106,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
for item in items:
title = (item.findtext("title") or "").strip()
url = (item.findtext("link") or "").strip()
if not title or not is_web_url(url):
continue
out.append({
"source_id": (item.findtext("guid") or url).strip(),
"title": title,
-2
View File
@@ -127,8 +127,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
j = job.get("data") or {}
title = (j.get("title") or "").strip()
role = str(j.get("slug") or j.get("req_id") or "")
if not title or not role:
continue
categories = j.get("categories") or []
text = "\n".join(p for p in (html_text(j.get("description")),
html_text(j.get("qualifications"))) if p)
+3 -5
View File
@@ -36,7 +36,7 @@ from typing import Any
from urllib.parse import urlencode, urlsplit
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt, place
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt, place
NAME = "jobbank"
LABEL = "Akademikernes Jobbank (Denmark)"
@@ -132,8 +132,8 @@ def _query(ask: Ask, number: int) -> tuple[str, str]:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search; the cursor is the page number. The next page
follows a full page (counted before any item is dropped, so a dropped item
does not pass for the end) while the ceiling is not reached."""
follows a full page (counted before the runner drops an item, so a
dropped item does not pass for the end) while the ceiling is not reached."""
number = int(cursor or 1)
url, remote = _query(ask, number)
try:
@@ -184,8 +184,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = []
for item in decoded["items"]:
link, title = item["link"], item["title"]
if not title or not item["id"] or not is_web_url(link):
continue
line = _LINE.fullmatch(item["description"])
kind = line.group("kind").strip() if line else ""
company, regions, cut = _regions(line.group("rest")) if line else ("", [], False)
+3 -5
View File
@@ -117,9 +117,9 @@ def _query(ask: Ask, number: int) -> str:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search, newest first; the cursor is the page number.
The next page follows a full page (counted before any item is dropped, so
a dropped item does not pass for the end) while the 200-result ceiling is
not reached."""
The next page follows a full page (counted before the runner drops an
item, so a dropped item does not pass for the end) while the 200-result
ceiling is not reached."""
number = int(cursor or 1)
root = ET.fromstring(get(_query(ask, number)))
channel = root.find("channel")
@@ -143,8 +143,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = []
for item in decoded["items"]:
link, full_title = item["link"], item["title"]
if not full_title or not is_web_url(link):
continue
description = item["description"]
h4 = _H4.search(description)
title = _text(h4.group(1)) if h4 else full_title
-2
View File
@@ -138,8 +138,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
for ad in decoded.get("jobAds") or []:
ident = str(ad.get("jobAdId") or "").strip()
title = _text(ad.get("title"))
if not ident or not title:
continue
link = str(ad.get("jobAdUrl") or "").strip()
town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
places: list[Place] = [place(town)] if town else []
+1 -3
View File
@@ -176,8 +176,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
title = (j.get("title") or "").strip()
job_id = str(j.get("id") or "").strip()
path = str(j.get("url") or "").strip()
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
continue
function = str(j.get("function") or "").strip()
places = [place(name) for name in dict.fromkeys(
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
@@ -186,7 +184,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"title": title,
"places": places,
"location": "; ".join(pl["text"] for pl in places),
"url": f"https://{_EMP}{path}",
"url": f"https://{_EMP}{path}" if _ROLE_PATH.fullmatch(path) else "",
"posted": _published(j.get("published_date")),
"department": "" if function == "All functions" else function,
"description": "",
+1 -3
View File
@@ -22,7 +22,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
log = logging.getLogger(__name__)
@@ -123,8 +123,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in data:
title = (j.get("text") or "").strip()
url = (j.get("hostedUrl") or "").strip()
if not title or not is_web_url(url):
continue
categories = j.get("categories") or {}
description = j.get("description") or j.get("descriptionPlain") or ""
salary_range = j.get("salaryRange") or {}
+4 -4
View File
@@ -198,9 +198,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
for role in search.get("requisitionList") or []:
role_id = str(role.get("Id") or "").strip()
title = _text(role.get("Title"))
# The id goes into a URL, and a listing is external content.
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id):
continue
# The id goes into a URL, and a listing is external content: an id
# of other characters gets no link, and the runner drops the row.
safe = bool(re.fullmatch(r"[A-Za-z0-9_-]+", role_id))
remote = _text(role.get("WorkplaceTypeCode")) or None
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
places: list[Place] = []
@@ -214,7 +214,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
"title": title,
"places": places,
"location": "; ".join(p["text"] for p in places),
"url": f"{_base(slug)}/job/{role_id}",
"url": f"{_base(slug)}/job/{role_id}" if safe else "",
"posted": parse_dt(role.get("PostedDate")),
"department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction"))
or _text(role.get("Department")) or _text(role.get("Organization"))),
-2
View File
@@ -104,8 +104,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for position in root.findall("position"):
job_id = (position.findtext("id") or "").strip()
title = (position.findtext("name") or "").strip()
if not job_id or not title:
continue
description = " ".join(
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
salary_info = position.find("salaryInformation")
-2
View File
@@ -168,8 +168,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
for job in search.get("data", {}).get("jobs") or []:
key = _key(job)
title = _text(job.get("title"))
if not title or not key:
continue
places: list[Place] = []
for text in (job.get("location"), *(job.get("multi_location") or [])):
if isinstance(text, str):
+1 -3
View File
@@ -22,7 +22,7 @@ from typing import Any
from ._contract import Ask, Fields, Reads
from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
from ._shared import HEADING, Row, html_text, label_slug_ok, num, only, pay_line, yearly_amount
NAME = "pinpoint"
LABEL = "Pinpoint"
@@ -121,8 +121,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
title = (j.get("title") or "").strip()
url = (j.get("url") or "").strip()
source_id = str(j.get("id") or "")
if not title or not source_id or not is_web_url(url):
continue
job = j.get("job") or {}
low, high, currency, salary_text = _pay(j)
workplace = str(j.get("workplace_type") or "").strip()
+1 -3
View File
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, html_text, only, parse_dt
NAME = "platsbanken"
LABEL = "Platsbanken"
@@ -108,8 +108,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
for hit in decoded.get("hits") or []:
title = _text(hit.get("headline") or "")
url = (hit.get("webpage_url") or "").strip()
if not title or not is_web_url(url):
continue
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
places = [place for address in addresses if isinstance(address, dict)
if (place := _place(address))]
+1 -3
View File
@@ -13,7 +13,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "recruitee"
LABEL = "Recruitee"
@@ -105,8 +105,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in data.get("offers") or []:
title = (j.get("title") or "").strip()
url = (j.get("careers_url") or "").strip()
if not title or not is_web_url(url):
continue
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
salary_min, salary_max, currency = yearly_amount(
salary.get("min"), salary.get("max"), salary.get("currency"),
+14 -10
View File
@@ -82,27 +82,31 @@ def parse(body: bytes, slug: str) -> list[Row]:
raise ValueError("not a Rippling board: the answer is not a list")
roles: dict[str, Row] = {}
places: dict[str, list[Place]] = {}
for entry in data:
for index, entry in enumerate(data):
job_id = str(entry.get("uuid") or "").strip()
title = (entry.get("name") or "").strip()
if not title or not _UUID.fullmatch(job_id):
continue
# The id goes into a URL: one of other characters gets no link, and
# the runner drops the row. An entry without an id is a role of its
# own, not merged with the others that lack one.
key = job_id or f"#{index}"
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
places.setdefault(job_id, [])
places.setdefault(key, [])
if place:
add_place(places[job_id], _place(place))
roles.setdefault(job_id, {
add_place(places[key], _place(place))
roles.setdefault(key, {
"source_id": job_id,
"title": title,
"url": f"https://{_SITE}/{slug}/jobs/{job_id}",
"url": f"https://{_SITE}/{slug}/jobs/{job_id}" if _UUID.fullmatch(job_id) else "",
"posted": None,
"department": str(((entry.get("department") or {}).get("label")) or "").strip(),
"description": "",
"salary_text": "", # the board's list states no pay
})
for job_id, row in roles.items():
row["places"] = places[job_id]
row["location"] = "; ".join(p["text"] for p in places[job_id])
if title and not roles[key]["title"]:
roles[key]["title"] = title # an entry of the role that has one
for key, row in roles.items():
row["places"] = places[key]
row["location"] = "; ".join(p["text"] for p in places[key])
return list(roles.values())
-2
View File
@@ -115,8 +115,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
for j in data.get("content") or []:
title = (j.get("name") or "").strip()
job_id = str(j.get("id") or "")
if not title or not job_id:
continue
location = j.get("location") or {}
loc_text = (location.get("fullLocation")
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
+25 -16
View File
@@ -63,7 +63,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
NAME = "successfactors"
LABEL = "SuccessFactors"
@@ -247,8 +247,9 @@ def _tile_fields(chunk: str) -> tuple[str, str, str]:
def _entries(text: str) -> list[dict[str, Any]]:
"""The roles of one search page, in the order shown, whichever layout it is
in and before the prefix filter: `id`, `href` (as written, relative),
`title`, `place`, `date` and `department`. A role is listed once, and a row
without a link to a role is not one."""
`title`, `place`, `date` and `department`. A role is listed once. The id
and the title are in the role's link, so a row without one is handed over
with an empty `id`, `href` and `title`, for the runner to drop and count."""
table = _ROW.split(text)[1:]
chunks = table or _TILE.split(text)[1:]
out: list[dict[str, Any]] = []
@@ -256,10 +257,12 @@ def _entries(text: str) -> list[dict[str, Any]]:
for chunk in chunks:
if table:
chunk = chunk.split("</tr>", 1)[0]
if not (link := _LINK.search(chunk)):
continue
href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3))
if job_id in seen:
if link := _LINK.search(chunk):
href, job_id, title = (html.unescape(link.group(1)), link.group(2),
_text(link.group(3)))
else:
href = job_id = title = ""
if job_id and job_id in seen:
continue
seen.add(job_id)
place, date, department = (_table_fields if table else _tile_fields)(chunk)
@@ -283,12 +286,14 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
if not _SEARCH_PAGE.search(text):
raise ValueError("not a SuccessFactors search page")
found = _entries(text)
ids = [entry["id"] for entry in found if entry["id"]]
total = _total(text)
if found and found[0]["id"] == previous:
first = ids[0] if ids else None
if first is not None and first == previous:
return text, [], total, (start, previous)
more = bool(found) and (total is None or start + len(found) < total)
return (text, [{"id": entry["id"]} for entry in found], total,
(start + len(found), found[0]["id"]) if more else None)
return (text, [{"id": job_id} for job_id in ids], total,
(start + len(found), first) if more else None)
def rows(decoded: str, slug: str) -> list[Row]:
@@ -296,17 +301,21 @@ def rows(decoded: str, slug: str) -> list[Row]:
slug's prefix when it has one; the id in it is the requisition's `source_id`.
The table and the tile layout give a title, a location, a date and, where
the site has the column, a department; nothing else about a role is on
the list, so its description and pay are empty."""
the list, so its description and pay are empty.
A role linked on another host, or outside the prefix, is another board's
(another site, or another brand of a shared site), and is left out: it is
no role of this company. A row with no link or no title is handed over,
and the runner drops it."""
host, prefix = _parts(slug)
base = f"https://{host}/"
out = []
for entry in _entries(decoded):
url = urljoin(base, entry["href"])
if urlsplit(url).hostname != host or not is_web_url(url):
url = urljoin(base, entry["href"]) if entry["href"] else ""
if url and urlsplit(url).hostname != host:
continue
if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"):
continue
if not entry["title"]:
if url and prefix and not urlsplit(url).path.lower().startswith(
f"{prefix.lower()}/job/"):
continue
out.append({
"source_id": entry["id"],
+1 -3
View File
@@ -15,7 +15,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
NAME = "teamtailor"
LABEL = "Teamtailor"
@@ -115,8 +115,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
for item in items:
title = (item.findtext("title") or "").strip()
url = (item.findtext("link") or "").strip()
if not title or not is_web_url(url):
continue
status = (item.findtext("remoteStatus") or "").strip()
remote = {"fully": True, "hybrid": False, "onsite": False}.get(status)
places: list[Place] = []
-2
View File
@@ -102,8 +102,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
for j in data.get("results") or []:
title = (j.get("title") or "").strip()
shortcode = (j.get("shortcode") or "").strip()
if not title or not shortcode:
continue
workplace = str(j.get("workplace") or "").strip()
remote = bool(j.get("remote")) or workplace == "remote"
places = _places(j, remote)
+6 -6
View File
@@ -124,8 +124,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
postings = data.get("jobPostings") or []
# Every posting is counted, a path-less one (LEGO lists one that has only
# `bulletFields`) by its position, so the parser dropping it does not
# leave the board short of its total.
# `bulletFields`) by its position, so the runner dropping its row does
# not leave the board short of its total.
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
following = offset + len(postings)
more = bool(postings) and (total is None or following < total)
@@ -274,13 +274,13 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
`bulletFields`, which lead with the requisition id, and on some tenants
`remoteType`. No description, no date and no country: `detail` has
those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the
path where not; a posting with no title or no path is skipped."""
path where not. A posting with no path has neither an id nor a link, and
the runner drops it."""
out = []
for p in decoded.get("jobPostings") or []:
title = (p.get("title") or "").strip()
path = (p.get("externalPath") or "").strip()
if not title or not _ROLE_PATH.fullmatch(path):
continue
path = path if _ROLE_PATH.fullmatch(path) else ""
remote = _remote(p.get("remoteType"))
place = _place(p.get("locationsText"), remote)
out.append({
@@ -288,7 +288,7 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
"title": title,
"places": [place] if place else [],
"location": place["text"] if place else "",
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}", # type: ignore[index]
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}" if path else "", # type: ignore[index]
"posted": _posted(p.get("postedOn") or ""),
"department": "",
"description": "",
+2 -1
View File
@@ -19,7 +19,8 @@ roles close once no search has seen them for `STALE_DAYS` (see
`app/jobs/pull.py`).
In a pull a search is read by `_runner.read` under the prefix `search`, as a
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
board is under `ats`: the runner drops a row without an id, a title or a
link, the 10% guards hold it, `_runner.stamp` records it,
and a search that fails or is suspect is kept as failing under
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
the settle reads as it reads a failing board's.
+85
View File
@@ -0,0 +1,85 @@
// static/agents.js without a browser: the gates Save sends from the fetch
// gate's four parts, and the stored gates the parts cannot say, which the
// form names in a note and sends back as they came.
//
// node --test tests/js/*.test.mjs
import { test } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import vm from "node:vm";
// Only what agents.js touches before DOMContentLoaded.
const window = { api: async () => [], count: (n, noun) => `${n} ${noun}` };
const document = { addEventListener() {} };
Object.assign(globalThis, { window, document });
for (const file of ["remote.js", "agents.js"]) {
vm.runInThisContext(readFileSync(new URL(`../../app/static/${file}`, import.meta.url), "utf8"));
}
const rh = window.remoteHelpers;
const { formGates, keptNote, rulesLine } = window.agentGates;
/** The four parts as the form shows them after loading `gates`, and what it keeps. */
function load(gates) {
const { where, rest } = rh.distribute(gates);
const parts = {
scope: where.scope.kind, scoped: where.scope.values,
within: where.countries ? where.countries.values : [],
towns: where.places ? where.places.values : [],
remote: where.remote.mode, from: where.remote.values,
titles: where.titles ? where.titles.values : [],
};
return { parts, rest };
}
const europe = { field: "country", values: rh.europeWith(["DK"]), mode: "require" };
const dk = { field: "country", values: ["DK"], mode: "any" };
const remote = { field: "remote", values: ["remote"], mode: "any" };
const towns = { field: "place", values: ["Malmö", "Lund"], mode: "any" };
const titles = { field: "title", values: ["account executive", "sales"], mode: "exclude" };
const level = { field: "seniority", values: ["senior", "staff"], mode: "require" };
const acme = { field: "company", values: ["acme"], mode: "exclude" };
test("a save sends the four parts back as they were loaded", () => {
const agents = [
[europe, dk, remote, towns, titles],
[{ field: "country", values: ["DK", "SE"], mode: "require" },
{ field: "remote_from", values: ["country:DK", "country:EU"], mode: "any" }, titles],
[dk],
[],
];
for (const gates of agents) {
const { parts, rest } = load(gates);
assert.deepEqual(rest, []);
assert.deepEqual(formGates(parts, rest), gates);
}
});
test("each part sends its own gate, and an empty part none", () => {
const parts = { scope: "everywhere", scoped: [], within: [], towns: [], remote: "none", from: [], titles: [] };
assert.deepEqual(formGates(parts), []);
// A remote line open to no area yet, or chosen countries with none picked, say nothing.
assert.deepEqual(formGates({ ...parts, remote: "from", scope: "countries" }), []);
assert.deepEqual(formGates({ ...parts, scope: "europe", within: ["US"] }), [
{ field: "country", values: rh.europeWith(["US"]), mode: "require" },
{ field: "country", values: ["US"], mode: "any" },
]);
});
test("a stored gate the parts cannot say is kept across a save, named in the note and on the card", () => {
const gates = [europe, dk, remote, titles, level, acme];
const { parts, rest } = load(gates);
assert.deepEqual(rest, [level, acme]);
assert.deepEqual(formGates(parts, rest), gates);
assert.equal(keptNote(rest),
"The form cannot edit these rules, and Save keeps them as stored: "
+ "must: with seniority senior, staff; not: at acme.");
const words = rulesLine(gates);
assert.match(words, /Only roles with seniority senior, staff\./);
assert.match(words, /Leaves out roles at acme\./);
});
test("an agent with nothing to keep has no note", () => {
assert.equal(keptNote([]), "");
assert.equal(keptNote([level]), "The form cannot edit this rule, and Save keeps it as stored: must: with seniority senior, staff.");
});
+8 -7
View File
@@ -1,5 +1,6 @@
// static/remote.js without a browser: the badge, the suggestions the agent's
// remote line offers, and how gates share out between Where and Other rules.
// remote line offers, and how gates share out between the fetch gate and the
// rest the form keeps as stored.
//
// node --test tests/js/*.test.mjs
@@ -89,14 +90,14 @@ test("any other require gate on country is the scope with its codes as chips", (
assert.deepEqual(rest, []);
});
test("a second require gate on country stays a row", () => {
test("a second require gate on country is kept as stored", () => {
const second = must("NO");
const { where, rest } = rh.distribute([europe, second]);
assert.equal(where.scope.kind, "europe");
assert.deepEqual(rest, [second]);
});
test("the titles to leave out are the exclude gate on title, and any other exclude is a row", () => {
test("the titles to leave out are the exclude gate on title, and any other exclude is kept as stored", () => {
const titles = { field: "title", values: ["account executive"], mode: "exclude" };
const level = { field: "seniority", values: ["junior"], mode: "exclude" };
const { where, rest } = rh.distribute([titles, level]);
@@ -104,7 +105,7 @@ test("the titles to leave out are the exclude gate on title, and any other exclu
assert.deepEqual(rest, [level]);
});
test("a gate with a mode the form does not know stays a row and takes no Where line", () => {
test("a gate with a mode the form does not know is kept as stored and takes no Where line", () => {
const prefer = { field: "remote", values: ["hybrid"], mode: "prefer" };
const { where, rest } = rh.distribute([prefer]);
assert.deepEqual(where.remote, { mode: "none", values: [] });
@@ -112,7 +113,7 @@ test("a gate with a mode the form does not know stays a row and takes no Where l
assert.deepEqual(rest, [prefer]);
});
test("a require gate on another field is a row and leaves the scope everywhere", () => {
test("a require gate on another field is kept as stored and leaves the scope everywhere", () => {
const level = { field: "seniority", values: ["senior"], mode: "require" };
const { where, rest } = rh.distribute([level]);
assert.equal(where.scope.kind, "everywhere");
@@ -129,7 +130,7 @@ test("remote, three modes: the plain gate is within the scope", () => {
assert.deepEqual(named.rest, []);
});
test("a remote_from gate wins over the plain one, which stays a row", () => {
test("a remote_from gate wins over the plain one, which is kept as stored", () => {
for (const gates of [[older, from("country:DK")], [from("country:DK"), older]]) {
const { where, rest } = rh.distribute(gates);
assert.deepEqual(where.remote, { mode: "from", values: ["country:DK"] });
@@ -137,7 +138,7 @@ test("a remote_from gate wins over the plain one, which stays a row", () => {
}
});
test("a second remote_from gate or a must stays a row", () => {
test("a second remote_from gate or a must is kept as stored", () => {
const second = from("country:SE");
const required = { field: "remote_from", values: ["country:DK"], mode: "require" };
const { where, rest } = rh.distribute([from("country:DK"), second, required]);
+15 -8
View File
@@ -24,18 +24,24 @@ from plugins.readers import _runner, smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
def _parsed(host: str, slug: str = "acme") -> list[ats.Row]:
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
body = path.read_bytes()
return ats._PARSERS[host](body, slug)
return ats._PARSERS[host](path.read_bytes(), slug)
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
"""The rows of a host's fixture the runner keeps (`_runner.valid`)."""
return [row for row in _parsed(host, slug) if _runner.valid(row)]
# ------------------------------------------------------------------ hosts
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
# The javascript: and data: rows are handed over and the runner drops
# them: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in _parsed("greenhouse")] == [
"1001", "1002", "1003", "1004", "1005", "1006"]
rows = _rows("greenhouse")
# The javascript: and data: rows are refused, same as the four real ones
# are kept: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
first = rows[0]
assert first["title"] == "Senior Backend Engineer" # trimmed
@@ -269,10 +275,11 @@ TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
# the javascript: link is refused like on the other hosts
# the javascript: link is handed over and refused by the runner
assert [r["source_id"] for r in rows] == [
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"]
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"]
assert [_runner.valid(r) for r in rows] == [True, True, True, False]
first = rows[0]
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
assert first["department"] == "Engineering"
@@ -294,7 +301,7 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them
assert rows[1]["title"] == "Läkare" # trimmed
assert rows[1]["department"] == "Software Development"
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
+8 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, bamboohr # noqa: E402
from plugins.readers import Ask, _runner, bamboohr # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -84,11 +84,16 @@ def test_a_page_that_is_not_a_listing_raises_and_an_empty_board_is_no_rows() ->
bamboohr.parse(body, "configit")
def test_a_role_without_a_title_or_an_id_is_left_out() -> None:
def test_a_role_without_a_title_or_an_id_reaches_the_runner_and_is_dropped_there() -> None:
body = json.loads(_read("bamboohr.json"))
body["result"][0]["jobOpeningName"] = " "
body["result"][1]["id"] = None
assert bamboohr.parse(json.dumps(body).encode(), "configit") == []
raw = json.dumps(body).encode()
rows = bamboohr.parse(raw, "configit")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
listing = _runner.read_listing(bamboohr, "configit", lambda url, **kw: raw)
assert (listing.rows, listing.ids, listing.complete) == ([], [], True)
assert (listing.handed, listing.dropped) == (2, 2)
def test_detail_reads_the_text_the_date_and_the_country() -> None:
+5 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, deel # noqa: E402
from plugins.readers import Ask, _runner, deel # noqa: E402
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
URL = "https://jobs.deel.com/klarna"
@@ -89,10 +89,12 @@ def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
def test_a_posting_without_a_title_or_id_is_skipped() -> None:
def test_a_posting_without_a_title_or_id_is_handed_over_for_the_runner_to_drop() -> None:
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
{"id": "c", "title": "Cook", "job": {}}]}
assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"]
rows = deel.parse(_page_with(body), "acme")
assert [r["source_id"] for r in rows] == ["a", "", "c"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
# ------------------------------------------------------------ the fetch
+5 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, eightfold # noqa: E402
from plugins.readers import Ask, _runner, eightfold # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "jobs.ericsson.com/ericsson.com"
@@ -93,10 +93,12 @@ def test_the_work_location_option_is_the_hosts_word(
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
def test_a_position_without_a_name_an_id_or_a_path_is_left_out() -> None:
def test_a_position_without_a_name_an_id_or_a_path_is_handed_over_for_the_runner_to_drop() -> None:
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
assert eightfold.rows(bad, SLUG) == []
rows = eightfold.rows(bad, SLUG)
assert len(rows) == 4 and not any(map(_runner.valid, rows))
assert [r["url"] for r in rows[2:]] == ["", ""] # no link built off the host
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
+4 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, gem # noqa: E402
from plugins.readers import Ask, _runner, gem # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
API = "https://jobs.gem.com/api/public/graphql"
@@ -79,8 +79,9 @@ def test_remote_is_the_location_type_then_the_locations(
assert gem.rows([posting], "acme")[0]["remote"] == remote
def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None:
assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == []
def test_a_posting_without_a_title_or_an_ext_id_is_handed_over_for_the_runner_to_drop() -> None:
rows = gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
def test_the_board_is_one_post_without_origin_or_referer() -> None:
+5 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, hibob # noqa: E402
from plugins.readers import Ask, _runner, hibob # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hibob.json").read_bytes()
@@ -87,10 +87,12 @@ def test_the_place_leaves_the_country_off_where_the_site_names_it() -> None:
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
def test_an_ad_without_a_title_or_id_is_skipped_and_an_empty_board_is_empty() -> None:
def test_an_ad_without_a_title_or_id_is_handed_over_and_an_empty_board_is_empty() -> None:
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
{"id": "c", "title": "Cook"}]}).encode()
assert [r["source_id"] for r in hibob.parse(body, "acme")] == ["c"]
rows = hibob.parse(body, "acme")
assert [r["source_id"] for r in rows] == ["a", "", "c"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
+5 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, hrmanager # noqa: E402
from plugins.readers import Ask, _runner, hrmanager # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
@@ -52,12 +52,14 @@ def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_f
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None:
def test_an_item_without_a_title_or_a_web_link_is_handed_over_for_the_runner_to_drop() -> None:
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
b"</channel></rss>")
assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"]
rows = hrmanager.parse(body, "x")
assert [r["source_id"] for r in rows] == ["1", "2", "3"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["3"]
# ------------------------------------------------------------- paging
+4 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, jibe # noqa: E402
from plugins.readers import Ask, _runner, jibe # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SITE = "jobs.booking.com"
@@ -72,9 +72,10 @@ def test_the_location_falls_back_to_city_and_country() -> None:
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
def test_a_role_without_a_title_or_a_slug_is_handed_over_for_the_runner_to_drop() -> None:
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
assert jibe.rows({"jobs": jobs}, SITE) == []
rows = jibe.rows({"jobs": jobs}, SITE)
assert len(rows) == 3 and not any(map(_runner.valid, rows))
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
+5 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, jobylon # noqa: E402
from plugins.readers import Ask, _runner, jobylon # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "jobylon.html").read_bytes()
@@ -65,12 +65,14 @@ def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None:
def test_a_role_whose_link_is_not_a_role_page_gets_no_link_and_is_dropped() -> None:
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
b"{id: '3', title: 'C', url: '/companies/3/'},"
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"]
rows = jobylon.parse(body, "9")
assert [r["url"] for r in rows[:3]] == ["", "", ""]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["4"]
# ---------------------------------------------------------------- detail
+5 -4
View File
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import oracle # noqa: E402
from plugins.readers import _runner, oracle # noqa: E402
from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -84,14 +84,15 @@ def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_left_out() -> None:
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_dropped_by_the_runner() -> None:
data = _listing()
roles = data["items"][0]["requisitionList"]
roles[0]["Title"] = " "
roles[1]["Id"] = "1/../../x"
roles[2]["Id"] = ""
assert [r["source_id"] for r in oracle.parse(json.dumps(data).encode(), SLUG)] == \
["40975", "31680"]
rows = oracle.parse(json.dumps(data).encode(), SLUG)
assert rows[1]["url"] == "" # the id builds no link
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["40975", "31680"]
def test_a_listing_without_the_expand_has_no_roles() -> None:
+5 -3
View File
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import phenom # noqa: E402
from plugins.readers import _runner, phenom # noqa: E402
from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -101,13 +101,15 @@ def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
def test_a_role_without_title_or_id_is_left_out() -> None:
def test_a_role_without_title_or_id_is_dropped_by_the_runner() -> None:
data = _answer()
jobs = data["refineSearch"]["data"]["jobs"]
jobs[0]["title"] = ""
for key in ("jobSeqNo", "jobId", "reqId"):
jobs[1].pop(key, None)
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
rows = phenom.parse(json.dumps(data).encode(), SLUG)
assert len(rows) == len(IDS)
assert [r["source_id"] for r in rows if _runner.valid(r)] == IDS[2:]
# ---------------------------------------------------------- the request
+4 -3
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import pinpoint # noqa: E402
from plugins.readers import _runner, pinpoint # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -86,11 +86,12 @@ def test_an_unknown_pay_frequency_keeps_the_text_and_leaves_the_columns() -> Non
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_left_out() -> None:
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_dropped_by_the_runner() -> None:
body = json.loads(_body())
body["data"][0]["title"] = ""
body["data"][1]["url"] = "javascript:alert(1)"
assert pinpoint.parse(json.dumps(body).encode(), "xeneta") == []
rows = pinpoint.parse(json.dumps(body).encode(), "xeneta")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
+6 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import rippling # noqa: E402
from plugins.readers import _runner, rippling # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "rippling.json").read_bytes()
@@ -61,11 +61,14 @@ def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
def test_an_entry_without_a_title_or_a_plain_id_is_skipped() -> None:
def test_an_entry_without_a_title_or_a_plain_id_is_dropped_by_the_runner() -> None:
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
{"uuid": "../../x", "name": "C"},
{"uuid": "abcdefgh-2", "name": "D"}]).encode()
assert [r["source_id"] for r in rippling.parse(body, "acme")] == ["abcdefgh-2"]
rows = rippling.parse(body, "acme")
assert [r["source_id"] for r in rows] == ["abcdefgh-1", "", "../../x", "abcdefgh-2"]
assert rows[2]["url"] == "" # the id builds no link
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["abcdefgh-2"]
# ---------------------------------------------------------------- detail
+3 -2
View File
@@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins.readers import Ask, teamtailor # noqa: E402
from plugins.readers import Ask, _runner, teamtailor # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -40,7 +40,8 @@ def test_page_takes_the_ask_and_reads_the_same_url() -> None:
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
first, second, third = _rows()
first, second, third, refused = _rows()
assert not _runner.valid(refused) # javascript:, for the runner
# "Sverige" (a city that is a country) once, Stockholm's twin once.
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
{"text": "Stockholm, Sweden", "remote": False}]
+12 -3
View File
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import workday # noqa: E402
from plugins.readers import _runner, workday # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "zendesk.wd1/zendesk"
@@ -96,7 +96,7 @@ def test_posted_is_read_only_where_the_text_is_exact(text: str, expected: dateti
assert workday._posted(text) == expected
def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
def test_a_posting_without_a_title_or_a_path_reaches_the_runner_and_is_dropped() -> None:
page = {"jobPostings": [
{"bulletFields": ["0000036369"]}, # LEGO's, no title
{"title": "No path", "bulletFields": ["R1"]},
@@ -105,7 +105,16 @@ def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
"postedOn": "Posted Today", "bulletFields": ["R4"]},
]}
assert [r["title"] for r in workday.rows(page, SLUG)] == ["Whole"]
rows = workday.rows(page, SLUG)
assert len(rows) == 5
assert [(r["source_id"], r["url"]) for r in rows[:2]] == [("", ""), ("", "")]
assert [r["title"] for r in rows if _runner.valid(r)] == ["Whole"]
# Through the runner: four dropped and counted, and the read complete,
# since every posting counts by its position towards the stated total.
body = json.dumps(page | {"total": 5}).encode()
listing = _runner.read_listing(workday, SLUG, lambda url, **kw: body)
assert [r["title"] for r in listing.rows] == ["Whole"]
assert (listing.handed, listing.dropped, listing.complete) == (5, 4, True)
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
+2 -2
View File
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
"VALUES (?, ?, '', '')", (key, name))
conn.execute("PRAGMA user_version = 19")
reopened = Store(path)
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
assert len(reopened.companies()) == 3
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
conn.execute("PRAGMA user_version = 20")
reopened = Store(path)
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
# The tag of 21 is All countries and All roles since 24.
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
+2 -34
View File
@@ -91,37 +91,6 @@ def test_best_cv_match_puts_the_judged_first_by_cv_score(
"order": "best"}).status_code == 422
def test_a_broken_must_have_caps_career_in_the_order_and_the_reader(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
# c: 9 on all three, but one must-have broken: career counts as 4.
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
{"id": "M2", "text": "Remote", "verdict": "broken"}]
store.set_judgement(owner.id, uid["c"], "k", 9, 9, 9, [], [], musts)
capped = round(10 * (30 * 9 + 25 * 9 + 30 * 4) / 85, 1)
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["c"], capped), (uid["b"], 60.0)]
# With career weighted up the capped value drops under b's 60.
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=owner.cv_text,
weights=Weights(cv_skills=0, cv_experience=0, cv_career=1))
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["b"], 60.0), (uid["c"], 40.0)]
one = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()
j = one["judgement"]
assert (j["career"], j["career_capped"], j["broken"]) == (9, 4, 1)
assert j["musts"] == musts and one["cv_score"] == j["cv_score"] == 40.0
def test_a_met_or_unclear_must_have_leaves_career_as_the_model_gave_it(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
{"id": "M2", "text": "Remote", "verdict": "unclear"}]
store.set_judgement(owner.id, uid["c"], "k", 9, 9, 9, [], [], musts)
j = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()["judgement"]
assert (j["career"], j["career_capped"], j["broken"], j["cv_score"]) == (9, 9, 0, 90.0)
def test_the_agents_weights_make_the_cv_score_at_read_time(
client: TestClient, store: Store) -> None:
owner, uid = setup(store)
@@ -218,15 +187,14 @@ def test_an_agent_without_languages_flags_nothing(client: TestClient, store: Sto
def test_the_cv_fields_round_trip_and_are_kept_when_left_out(client: TestClient) -> None:
body = {"name": "Data", "cv_text": "VP Data.", "like_text": "Teams.", "avoid_text": "Reporting.",
"must_text": "Permanent.",
"cv_languages": [{"language": "da", "level": "native"},
{"language": "EN", "level": "fluent"}]}
made = client.post("/api/agents", json=body).json()
assert made["cv_languages"] == [{"language": "da", "level": "native"},
{"language": "en", "level": "fluent"}]
kept = client.post("/api/agents", json={"id": made["id"], "name": "Data"}).json()
assert (kept["cv_text"], kept["like_text"], kept["avoid_text"], kept["must_text"]) == \
("VP Data.", "Teams.", "Reporting.", "Permanent.")
assert (kept["cv_text"], kept["like_text"], kept["avoid_text"]) == \
("VP Data.", "Teams.", "Reporting.")
assert len(kept["cv_languages"]) == 2
nulls = client.post("/api/agents", json={"id": made["id"], "name": "Data",
"cv_text": None, "cv_languages": None}).json()
+46 -42
View File
@@ -16,7 +16,7 @@ from pydantic import ValidationError
from app import prep
from app.config import Config
from app.jobs import extract
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag, must_lines
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag
from app.storage import Store
from desk import modeltask
from desk.models import Item
@@ -44,24 +44,21 @@ def test_an_agent_keeps_its_cv_fields_when_a_save_leaves_them_out(store: Store)
cv_languages=[CvLanguage(language="DA", level="native"),
CvLanguage(language="en", level="fluent"),
CvLanguage(language="da", level="basic")],
like_text=" Building teams. ", avoid_text="Pure reporting.",
must_text="Permanent contract.")
like_text=" Building teams. ", avoid_text="Pure reporting.")
assert made.cv_text == "VP Data."
assert [(lang.language, lang.level) for lang in made.cv_languages] == \
[("da", "native"), ("en", "fluent")]
assert (made.like_text, made.avoid_text, made.must_text) == \
("Building teams.", "Pure reporting.", "Permanent contract.")
assert (made.like_text, made.avoid_text) == ("Building teams.", "Pure reporting.")
kept = store.save_agent("Data", "data leadership", agent_id=made.id)
assert (kept.cv_text, kept.like_text, kept.avoid_text, kept.must_text) == \
("VP Data.", "Building teams.", "Pure reporting.", "Permanent contract.")
assert (kept.cv_text, kept.like_text, kept.avoid_text) == \
("VP Data.", "Building teams.", "Pure reporting.")
assert len(kept.cv_languages) == 2
cleared = store.save_agent("Data", "data leadership", agent_id=made.id, cv_text="",
cv_languages=[], like_text="", avoid_text="",
must_text="")
assert (cleared.cv_text, cleared.cv_languages, cleared.like_text, cleared.avoid_text,
cleared.must_text) == ("", [], "", "", "")
cv_languages=[], like_text="", avoid_text="")
assert (cleared.cv_text, cleared.cv_languages, cleared.like_text, cleared.avoid_text) == \
("", [], "", "")
def test_a_cv_language_is_a_code_and_a_level_of_the_ladder() -> None:
@@ -92,25 +89,24 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
conn.execute("DROP TABLE role_judgements")
for column in ("languages_required", "deadline_host", "deadline_model", "deadline"):
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
for column in ("cv_text", "cv_languages", "avoid_text", "must_text"):
for column in ("cv_text", "cv_languages", "avoid_text"):
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
conn.execute("PRAGMA user_version = 27")
migrated = Store(path)
assert migrated.migrations_applied == [28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [28, 29, 30, 31, 32, 33, 34, 35]
rows = {r["source_id"]: r for r in migrated._rows(
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
assert rows["a"]["languages_required"] is None and rows["b"]["deadline_host"] == ""
[data] = migrated.agents()
assert (data.cv_text, data.cv_languages, data.avoid_text, data.must_text) == \
("", [], "", "")
assert (data.cv_text, data.cv_languages, data.avoid_text) == ("", [], "")
prep.run(migrated, remap=True, dupes=False)
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 27")
assert Store(path).migrations_applied == [28, 29, 30, 31, 32, 33]
assert Store(path).migrations_applied == [28, 29, 30, 31, 32, 33, 34, 35]
# ---------------------------------------------------------------- deadline
@@ -231,7 +227,6 @@ def test_the_language_flag() -> None:
def test_the_cv_score_weighs_skills_and_career_above_experience() -> None:
assert cv_score(10, 10, 10) == 100.0
assert cv_score(8, 6, 4) == round(10 * (240 + 150 + 120) / 85, 1)
assert cv_score(8, 6, 9, broken=1) == cv_score(8, 6, 4)
def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
@@ -244,20 +239,36 @@ def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
assert [lang.language for lang in store.role(item.uid).languages_required] == ["da"]
def test_v29_adds_the_musts_and_broken_columns_and_runs_twice(tmp_path: Path) -> None:
path = tmp_path / "v28.db"
Store(path)
def test_v34_drops_the_must_haves_and_runs_twice(tmp_path: Path) -> None:
"""A file from before migration 34 holds the agent's must text and the
judge's verdicts on it; the columns go, the rest of the agent and the
judgement stay."""
path = tmp_path / "v33.db"
store = Store(path)
item = posting("1")
upsert(store, item)
agent = store.save_agent("Data", "data leadership", cv_text="CV.", like_text="Teams.")
assert store.set_judgement(agent.id, item.uid, "key", 8, 7, 9, ["Good."], ["Bad."])
with sqlite3.connect(path) as conn:
for column in ("musts", "broken"):
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
conn.execute("PRAGMA user_version = 28")
conn.execute("ALTER TABLE agents ADD COLUMN must_text TEXT NOT NULL DEFAULT ''")
conn.execute("ALTER TABLE role_judgements ADD COLUMN musts TEXT NOT NULL DEFAULT '[]'")
conn.execute("ALTER TABLE role_judgements ADD COLUMN broken INTEGER NOT NULL DEFAULT 0")
conn.execute("UPDATE agents SET must_text = 'Remote.'")
conn.execute("UPDATE role_judgements SET musts = ?, broken = 1",
(json.dumps([{"id": "M1", "text": "Remote.", "verdict": "broken"}]),))
conn.execute("PRAGMA user_version = 33")
assert Store(path).migrations_applied == [29, 30, 31, 32, 33]
with sqlite3.connect(path) as conn:
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
assert {"musts", "broken"} <= columns
conn.execute("PRAGMA user_version = 28")
assert Store(path).migrations_applied == [29, 30, 31, 32, 33]
for _ in range(2): # runs twice without harm
migrated = Store(path)
assert migrated.migrations_applied == [34, 35]
with sqlite3.connect(path) as conn:
assert "must_text" not in {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
assert not {"musts", "broken"} & {
r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
conn.execute("PRAGMA user_version = 33")
assert migrated.agent(agent.id).like_text == "Teams." # type: ignore[union-attr]
found = migrated.judgement(agent.id, item.uid)
assert found is not None and (found.career, found.cv_score) == (9, cv_score(8, 7, 9))
def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
@@ -269,12 +280,12 @@ def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [30, 31, 32, 33]
assert migrated.migrations_applied == [30, 31, 32, 33, 34, 35]
[data] = migrated.agents()
assert (data.like_text, data.avoid_text) == ("", "Reporting.")
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 29")
assert Store(path).migrations_applied == [30, 31, 32, 33]
assert Store(path).migrations_applied == [30, 31, 32, 33, 34, 35]
def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
@@ -290,8 +301,7 @@ def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
old = store.save_agent("Old", "data leadership", cv_text="CV.")
own = store.save_agent("Own", "data leadership")
bare = store.save_agent("Bare", "data leadership")
musts = [{"id": "M1", "text": "Remote", "verdict": "broken"}]
assert store.set_judgement(old.id, item.uid, "key", 80, 70, 90, ["Good."], ["Bad."], musts)
assert store.set_judgement(old.id, item.uid, "key", 80, 70, 90, ["Good."], ["Bad."])
store.judge_missed([(own.id, item.uid)])
with sqlite3.connect(path) as conn:
conn.execute("UPDATE agents SET limits = json_set(limits, '$.summarise_top', 50) "
@@ -303,22 +313,16 @@ def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
for _ in range(2): # runs twice without harm
migrated = Store(path)
assert migrated.migrations_applied == [33]
assert migrated.migrations_applied == [33, 34, 35]
assert migrated.judgement(old.id, item.uid) is None
with sqlite3.connect(path) as conn:
rows = conn.execute(
"SELECT skills, experience, career, cv_score, strengths, gaps, musts, broken, "
"SELECT skills, experience, career, cv_score, strengths, gaps, "
"input_key, tries, judged_at FROM role_judgements").fetchall()
assert rows == [(None, None, None, None, "[]", "[]", "[]", 0, "", 0, None)] * 2
assert rows == [(None, None, None, None, "[]", "[]", "", 0, None)] * 2
conn.execute("PRAGMA user_version = 32")
assert {a.name: a.limits.summarise_top for a in migrated.agents()} == {
"Old": 30, "Own": 12, "Bare": 30}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT limits FROM agents WHERE id = ?",
(bare.id,)).fetchone() == ("{}",)
def test_the_musts_text_is_one_must_have_per_line() -> None:
assert must_lines("- A\n* B c\n\u2022 D\n1. E\n2) F\n \nG\n-H") == [
"A", "B c", "D", "E", "F", "G", "-H"]
assert must_lines("") == []
+1 -1
View File
@@ -282,7 +282,7 @@ def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Pat
conn.execute("DROP TABLE agent_cv_vectors")
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
conn.execute("PRAGMA user_version = 30")
assert Store(path).migrations_applied == [31, 32, 33]
assert Store(path).migrations_applied == [31, 32, 33, 34, 35]
step = {v: s for v, _, s in MIGRATIONS}[31]
with sqlite3.connect(path) as conn:
assert callable(step)
+14 -2
View File
@@ -239,14 +239,26 @@ def test_folding_needs_the_same_title_a_link_and_texts_that_can_be_one_job() ->
assert fold([pc("1", "Berlin", a), pc("2", "Munich", "")]).placed == {"2"}
def test_the_per_place_fold_keeps_to_one_country() -> None:
"""A federal sales role posted in Canberra and in Washington has alike
text and is two openings; Berlin and Munich are one job in two offices.
A row with no country joins either."""
text = "Sell to federal agencies, run the public sector pipeline and partners."
got = fold([replace(pc("au", "Canberra", text), country="AU"),
replace(pc("us", "Washington", text), country="US"),
pc("b", "Berlin", text), pc("m", "Munich", text),
replace(pc("n", "", text), country="")])
assert got.dup_of == {"au": None, "us": None, "b": None, "m": "b", "n": "au"}
def test_a_remote_listing_is_not_folded_into_an_office_one() -> None:
"""The kept role takes a copy's places, not its remote, so the gate would
read "Remote, Germany" as an office role in Vienna and reject it for an
agent that takes remote roles. Two remote listings still fold."""
text = "We build the payment platform for our bank customers in Kotlin."
office = replace(pc("v", "Vienna", text), country="AT")
remote = replace(pc("r", "Remote, Germany", text), remote=True)
other = replace(pc("s", "Remote, Spain", text), country="ES", remote=True)
remote = replace(pc("r", "Remote, Berlin", text), remote=True)
other = replace(pc("s", "Remote, Hamburg", text), remote=True)
got = fold([office, remote, other])
assert got.dup_of == {"v": None, "r": None, "s": "r"}
assert got.placed == {"s"}
+22 -19
View File
@@ -103,10 +103,11 @@ def test_berlin_and_munich_of_one_host_fold_into_one_role_with_two_places(
assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0)
def test_a_place_copy_in_europe_meets_a_europe_gate_for_its_kept_role(store: Store) -> None:
"""OpenAI posts one role in San Francisco and in Dublin. The kept role
holds both places, so a Europe gate and the Place filter find it through
Dublin, though `roles.place` and `roles.country` are San Francisco's."""
def test_a_posting_per_office_in_two_countries_is_two_roles(store: Store) -> None:
"""OpenAI posts one role in San Francisco and in Dublin. The fold keeps
to one country (ADR 1.4), so each is a role of its own: a Europe gate
takes Dublin and rejects San Francisco, and the Place filter finds
Dublin alone."""
from app import prep
from app.models import Gate
sf = role("sf", company="OpenAI", place="San Francisco", country="US", board="ashby")
@@ -114,10 +115,11 @@ def test_a_place_copy_in_europe_meets_a_europe_gate_for_its_kept_role(store: Sto
for item in (sf, dublin):
store.upsert_role(item)
prep.run(store)
kept = kept_of(store, sf, dublin)
assert dup_of(store) == {sf.uid: None, dublin.uid: None}
agent = store.save_agent("A", gates=[Gate(field="country", values=["FR", "IE", "EU"])])
assert {c.uid: c.gates for c in store.score_candidates(agent.id, agent.gates)} == {kept: [1]}
assert {r.uid for r in store.roles(places=["country:IE"], by_score=False)} == {kept}
assert {c.uid: c.gates for c in store.score_candidates(agent.id, agent.gates)} == {
sf.uid: [0], dublin.uid: [1]}
assert {r.uid for r in store.roles(places=["country:IE"], by_score=False)} == {dublin.uid}
def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None:
@@ -279,7 +281,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
with sqlite3.connect(path) as conn:
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
@@ -305,30 +307,31 @@ def test_switching_a_board_off_shows_its_roles_copies_at_once(store: Store) -> N
def test_a_role_in_several_cities_leads_with_the_one_nearest_the_agent(store: Store) -> None:
"""Mistral posts one role per city. For an agent kept to Europe the kept
Montréal role leads with Paris, in the row and the reader, and opens the
Paris copy; without an agent the host's order stands."""
"""A Danish company posts one role per city. For an agent whose towns
hold Copenhagen the kept Aarhus role leads with Copenhagen, in the row
and the reader, and opens the Copenhagen copy; without an agent the
host's order stands."""
from app import prep
from app.models import Gate
items = [role(c, company="Mistral AI", place=p, board="ashby") for c, p in (
("mtl", "Montréal"), ("nyc", "New York"), ("par", "Paris"))]
items = [role(c, company="Danske", place=p, board="ashby") for c, p in (
("aar", "Aarhus"), ("bil", "Billund"), ("cph", "Copenhagen"))]
for item in items:
store.upsert_role(item)
prep.run(store)
kept = kept_of(store, *items)
agent = store.save_agent("A", gates=[
Gate(field="country", values=["FR", "DK", "EU"], mode="require"),
Gate(field="country", values=["DK"], mode="any")])
Gate(field="place", values=["copenhagen"], mode="any")])
got = store.role(kept, agent.id)
assert got is not None
own = next(p.text for p in got.places if not p.url)
assert own != "Paris"
assert (got.lead_place, got.lead_url) == ("Paris", "https://jobs.example/par")
assert got.places[0].text == "Paris"
assert own != "Copenhagen"
assert (got.lead_place, got.lead_url) == ("Copenhagen", "https://jobs.example/cph")
assert got.places[0].text == "Copenhagen"
(row,) = store.roles_page(agent=agent.id, by_score=False)
assert (row.place, row.lead_place, row.lead_url) == ("Paris +2", "Paris", got.lead_url)
assert row.place_texts[0] == "Paris" and len(row.place_texts) == 3
assert (row.place, row.lead_place, row.lead_url) == ("Copenhagen +2", "Copenhagen", got.lead_url)
assert row.place_texts[0] == "Copenhagen" and len(row.place_texts) == 3
plain = store.role(kept)
assert plain is not None
+1 -1
View File
@@ -520,7 +520,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
conn.execute("PRAGMA user_version = 22")
migrated = Store(path)
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
assert migrated.agents()[0].fetch_terms == []
+17 -53
View File
@@ -14,7 +14,7 @@ import pytest
from app.config import Config
from app.jobs import judge, stages
from app.models import CvLanguage, Prompt
from app.models import CvLanguage, Prompt, Weights
from app.storage import Store
from desk import modeltask
from desk.models import Item
@@ -76,8 +76,7 @@ def config(tmp_path: Path) -> Config:
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
cv_text=cv, like_text="Building teams.",
avoid_text="Pure reporting.",
must_text="A permanent contract.", **kwargs)
avoid_text="Pure reporting.", **kwargs)
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
@@ -109,59 +108,18 @@ def test_a_run_writes_the_judgement_and_its_total(
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
[system] = model.systems
assert CV in system and "Pure reporting." in system and "A permanent contract." in system
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
assert CV in system and "Pure reporting." in system
assert "- AI lead: Head of AI" in system
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
in system
assert '<dislikes title="what they do not like">\nPure reporting.' in system
assert "<avoid" not in system
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
[uid] = ranked(store, owner, "1")
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [{**one, "career": 8, "musts": [
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
for one in judged(items)]
model = stub(monkeypatch, reply)
modeltask.run(judge.TASK, config, store)
[system] = model.systems
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
found = store.judgement(owner.id, uid)
assert found is not None
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
("M3", "Danish team", "unclear")]
assert (found.career, found.career_capped, found.broken) == (8, 4, 1)
assert found.cv_score == round(10 * (30 * 8 + 25 * 7 + 30 * 4) / 85, 1)
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
owner = hunter(store)
[uid] = ranked(store, owner, "1")
stub(monkeypatch) # answers without any `musts`
modeltask.run(judge.TASK, config, store)
found = store.judgement(owner.id, uid)
assert found is not None
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
assert (found.career, found.career_capped, found.broken) == (9, 9, 0)
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
def test_the_schema_asks_for_three_numbers_strengths_and_gaps() -> None:
item = judge.SCHEMA["properties"]["judgements"]["items"]
assert "musts" in item["required"]
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
assert verdict["enum"] == ["met", "broken", "unclear"]
assert "career 0-3" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
assert item["required"] == ["key", "skills", "experience", "career", "strengths", "gaps"]
assert "must-have" not in judge.SYSTEM
assert "0 to 10" in judge.SYSTEM and "80-100" not in judge.SYSTEM
@@ -190,7 +148,7 @@ def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) ->
def hashlib_old_key(agent: Any, text: str) -> str:
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
agent.avoid_text, text], ensure_ascii=False)
return hashlib.sha1(blob.encode()).hexdigest()[:16]
@@ -259,12 +217,18 @@ def test_a_judgement_is_read_again_only_when_what_it_read_changed(
with store.transaction() as conn:
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
# The languages are not part of what it read; the must text is.
# The languages and the dislikes are part of what it read; the weights
# are not.
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
cv_languages=[CvLanguage(language="da", level="native")])
weights=Weights(cv_skills=50))
assert judge.pending(store) == 0
spoken = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
cv_languages=[CvLanguage(language="da", level="native")])
assert judge.pending(store) == 1
assert '<languages title="the languages they speak, and how well">\nDanish native' \
in judge.system(spoken)
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
must_text="Remote two days a week.")
avoid_text="Long commutes.")
assert judge.pending(store) == 1
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
+21
View File
@@ -116,6 +116,16 @@ def test_roles_without_text_and_text_reads_that_failed() -> None:
assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
@pytest.mark.parametrize(("dropped", "found"), [
("3", ["3 listings dropped: no title or link"]),
("1", ["1 listing dropped: no title or link"]),
("", []), ("x", [])])
def test_the_listings_the_last_read_dropped_are_named(dropped: str, found: list[str]) -> None:
"""The rows the runner dropped (`_runner.valid`), after "read incomplete"."""
assert words(fake(), state(dropped=dropped)) == found
assert words(fake(), state(dropped=dropped, complete=""))[1:] == found
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
assert words(fake(), state(), reach.SMALL) == []
@@ -199,6 +209,17 @@ def test_companies_carry_their_limits_and_the_boards_own_open_roles(
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
def test_a_board_that_dropped_listings_says_so_on_its_row(
client: TestClient, store: Store) -> None:
store.add_subscription("ats", "lever:acme", company="Acme", board="lever")
store.add_company("Acme", key="acme", ats="lever")
store.state["ats:ok:lever:acme"] = "run"
store.state["ats:complete:lever:acme"] = "1"
store.state["ats:dropped:lever:acme"] = "3"
[row] = client.get("/api/companies").json()
assert row["limits"][0] == "3 listings dropped: no title or link"
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
store.add_company("Novo", key="novo", ats="taleo")
[row] = client.get("/api/companies").json()
+6 -8
View File
@@ -51,8 +51,8 @@ def _read(store: Store, target: str) -> tuple[Ask, str]:
@pytest.mark.parametrize("countries, roles, line", [
(False, False, "follows agents: DK, SE · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
(False, True, "follows agents: DK, SE · remote not asked on Phenom · all roles"),
(False, False, "follows agents' world (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
(False, True, "follows agents' world (Europe) · remote not asked on Phenom · all roles"),
(True, False, "all locations (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
(True, True, "all locations (Europe) · remote not asked on Phenom · all roles"),
], ids=["neither", "all roles alone", "all locations alone", "both"])
@@ -62,11 +62,9 @@ def test_the_four_combinations(store: Store, countries: bool, roles: bool, line:
store.set_all_roles("philips", roles)
ask, said = _read(store, PHILIPS)
assert said == line
# Phenom asks for countries, many in one request, and not for remote.
if countries:
assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30
else:
assert ask.countries == ("DK", "SE")
# Phenom asks for countries, many in one request, and not for remote: so
# the agents want remote, it is asked for their world either way.
assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30
# Terms go to every big board, and never to a company set to All roles.
assert ask.terms == (() if roles else ("data", "engineer", "product"))
@@ -122,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
conn.execute("PRAGMA user_version = 23")
migrated = Store(path)
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
+26 -2
View File
@@ -224,7 +224,31 @@ def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
("senior", "lead", "onsite")
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
def test_remote_in_the_text_is_marked_and_a_denial_is_not() -> None:
yes = ("Fully-remote within the EU, we meet twice a year.",
"Work from home or from our Berlin office, as you like.",
"Dette er en stilling med mulighed for fjernarbejde.")
no = ("No remote work: this role is on site in Munich.",
"Remote sensing analyst for our satellite team.",
"A great office in Copenhagen with free lunch.")
assert all(words.remote_in_text("Engineer", text) for text in yes)
assert not any(words.remote_in_text("Engineer", text) for text in no)
assert words.remote_in_text("Backend Developer, remote-first team", "")
def test_prep_marks_a_role_whose_text_says_remote(store: Store) -> None:
said = item("1", title="Engineer", place="Berlin")
said.body = "Fully remote within Germany, with a Berlin office."
silent = item("2", title="Engineer", place="Berlin")
silent.body = "On site in Berlin."
store.upsert_role(said)
store.upsert_role(silent)
prep.run(store, tables=TABLES)
assert (row(store, "1")["remote"], row(store, "1")["remote_text"]) == ("", 1)
assert (row(store, "2")["remote"], row(store, "2")["remote_text"]) == ("", 0)
store.add_company("Ørsted")
store.upsert_role(item("1", company="ORSTED A/S"))
assert row(store, "1")["company_key"] != "ørsted"
@@ -316,7 +340,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
migrated = Store(path)
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
one = row(migrated, "1")
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
one["country_host"]) == ("senior", "full_time", "true", "DK")
+38
View File
@@ -250,6 +250,44 @@ def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
def _aged(store: Store, source_id: str) -> None:
"""Make a role look unlisted for longer than `STALE_DAYS`."""
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(timespec="seconds")
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = ?", (old, source_id))
def test_an_incomplete_read_closes_what_no_read_has_listed_for_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""A board past its ceiling or page cap is never read complete; a role
it has not listed for three weeks still closes (ADR 1.3)."""
_answer(monkeypatch, ([_row("1"), _row("2"), _row("3")], True))
_pull(acme)
_aged(acme, "workable:acme:3")
_answer(monkeypatch, ([_row("1")], False))
result = _pull(acme)
# 2 was listed two pulls ago and stays; 3 has been unlisted for 22 days.
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
assert (result["incomplete"], result["closed"]) == (1, 1)
def test_a_filtered_read_closes_a_spared_role_once_it_is_stale(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""A filtered read spares a role acted on from its list; after three
weeks unlisted the role closes by age like any other."""
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
uid = {r.source_id: r.uid for r in acme.roles()}
agent = acme.save_agent("Hunt")
acme.set_vote(agent.id, uid["workable:acme:2"], 1)
_answer(monkeypatch, ([_row("1")], True), filtered=True)
assert _pull(acme)["closed"] == 0
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
_aged(acme, "workable:acme:2")
assert _pull(acme)["closed"] == 1
assert _open(acme) == {"workable:acme:1"}
def test_a_failing_board_keeps_its_roles_before_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
+12 -2
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, adzuna # noqa: E402
from plugins.readers import Ask, _runner, adzuna # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
@@ -39,8 +39,9 @@ def test_it_declares_the_key_and_states_the_robots_finding() -> None:
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
first, predicted = _rows() # javascript: refused
first, predicted, refused = _rows()
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
assert refused["url"].startswith("javascript:") and not _runner.valid(refused)
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
assert first["teaser"] == "Build our backend services in Python & Go."
assert "description" not in first
@@ -89,3 +90,12 @@ def test_saved_params_need_a_known_country() -> None:
"q": "x", "country": "gb"}
with pytest.raises(ValueError, match="country"):
adzuna.clean_params({"q": "x", "country": "dk"})
def test_the_runner_drops_the_ad_without_a_web_link_and_counts_it() -> None:
"""The reader hands the `javascript:` ad over; the runner drops it, and
the read is complete, since all three result ids were read."""
listing = _runner.read_listing(adzuna, "", lambda url, **kw: BODY, ASK,
limits=_runner.Limits(page_delay=0))
assert listing.ids == ["4412345678", "4412345679"]
assert (listing.handed, listing.dropped, listing.complete) == (3, 1, True)
+7 -4
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, freehire # noqa: E402
from plugins.readers import Ask, _runner, freehire # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
@@ -119,14 +119,17 @@ def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None:
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None:
def test_a_hit_without_a_title_an_id_or_a_web_link_is_handed_to_the_runner() -> None:
"""The reader hands every hit over; the runner keeps only the whole one."""
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
found = freehire.rows({"data": hits}, "")
assert [row["source_id"] for row in found] == ["d"]
assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == []
assert [row["source_id"] for row in found] == ["a", "", "c", "d"]
kept = [row for row in found if _runner.valid(row)]
assert [row["source_id"] for row in kept] == ["d"]
assert kept[0]["url"] == "https://x.example/d" and kept[0]["places"] == []
def test_a_title_and_a_company_have_their_entities_decoded() -> None:
+3 -2
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, jobbank # noqa: E402
from plugins.readers import Ask, _runner, jobbank # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
@@ -170,7 +170,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
assert len(results) == 100 and nxt == 2
assert len(jobbank.rows(decoded, "")) == 99
rows = jobbank.rows(decoded, "")
assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
+3 -2
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, jobindex # noqa: E402
from plugins.readers import Ask, _runner, jobindex # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
@@ -89,7 +89,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
assert len(results) == 20 and nxt == 2
assert len(jobindex.rows(decoded, "")) == 19
rows = jobindex.rows(decoded, "")
assert len(rows) == 20 and sum(map(_runner.valid, rows)) == 19
def test_an_answer_without_a_channel_raises() -> None:
+6 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, jobnet # noqa: E402
from plugins.readers import Ask, _runner, jobnet # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
@@ -100,9 +100,12 @@ def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None:
assert "applicationDeadline" not in row
def test_an_ad_without_an_id_or_a_title_is_skipped() -> None:
def test_an_ad_without_an_id_or_a_title_is_handed_over_for_the_runner_to_drop() -> None:
ads = json.loads(OTHER)["jobAds"]
assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == []
rows = jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "")
assert [(row["source_id"] == "", row["title"] == "") for row in rows] == [
(False, True), (True, False)]
assert not any(map(_runner.valid, rows))
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
+5 -4
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, platsbanken # noqa: E402
from plugins.readers import Ask, _runner, platsbanken # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
@@ -47,8 +47,9 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
rows = platsbanken.rows(decoded, "")
assert total == 32
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
first, no_town, bare = rows
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400001", "31400002"]
assert not _runner.valid(rows[2]) # javascript:, dropped by the runner
first, no_town, _refused, bare = rows
assert [p["text"] for p in first["places"]] == [
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
assert first["location"] == "Lund"
@@ -58,7 +59,7 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
def test_it_reads_title_company_date_and_the_full_text() -> None:
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
first, _no_town, remote = platsbanken.rows(decoded, "")
first, _no_town, _refused, remote = platsbanken.rows(decoded, "")
assert first["title"] == "Postdoctoral Researcher"
assert first["company"] == "European Spallation Source Eric"
assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
+2 -2
View File
@@ -534,7 +534,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
migrated = Store(path)
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34, 35]
assert migrated.gazetteer().anywhere is not None
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
"WHERE uid = ?", (board,))
@@ -544,7 +544,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
assert area(migrated, search.uid) == ("", "")
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 26")
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32, 33]
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34, 35]
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
+107 -2
View File
@@ -1,7 +1,7 @@
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
subscription resolves to, series and their union, whole and paged listings,
completeness from the declaration, the sweep across pulls, the 10% guard,
the facets kept in state, the detail reads and the `raw` an item carries. No
completeness from the declaration, the rows kept and dropped, the sweep
across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
network: `get` is a dict of URL -> body or exception."""
from __future__ import annotations
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
assert filtered.ids == ["1"] and filtered.filtered
# --------------------------------------------------------- the rows kept
def _handed(i: str) -> dict[str, Any]:
"""A row as a reader hands it over: whole, or, for an id of `-id`,
`-title`, `-link` or `-url`, without that value or with a link that is
no web URL (`javascript:`, or one httpx cannot read)."""
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
return _row(i, **broken.get(i.split(":")[0], {}))
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
assert _runner.valid(_handed("1"))
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
assert not _runner.valid({"source_id": "1", "title": "Role"})
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
caplog: pytest.LogCaptureFixture) -> None:
module = _whole()
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
web = Web({"https://jobs.fake.example/acme/list":
b'["1", "-id", "2", "-title", "-link", "-url"]'})
with caplog.at_level("WARNING", logger=_runner.__name__):
listing = _runner.read_listing(module, "acme", web)
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert (listing.handed, listing.dropped) == (6, 4)
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
caplog: pytest.LogCaptureFixture) -> None:
"""Completeness counts the result ids `page` returns, not the rows kept,
so the dropped rows still count as read; the drop is logged once."""
pages = [["1", "-title:a"], ["2", "-link:b"]]
module = _paged(pages, total=4)
module.rows = lambda ids, slug: [_handed(i) for i in ids]
with caplog.at_level("WARNING", logger=_runner.__name__):
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert (listing.handed, listing.dropped) == (4, 2)
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
def test_the_stamp_keeps_the_rows_dropped() -> None:
state: dict[str, str] = {}
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
assert state["ats:dropped:t"] == "2"
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
assert state["ats:dropped:t"] == ""
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
"""A whole board whose feed hands over `handed` rows, `dropped` of them
without a title."""
module = _whole()
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
handed: int, dropped: int, fails: bool) -> None:
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
a page whose shape changed, which must close nothing."""
module, web = _dropping(handed, dropped)
if not fails:
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
return
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
_runner.read({}, "ats", "t", module, "acme", web)
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
module, web = _dropping(20, 5)
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
listing = _runner.read(state, "ats", "t", module, "acme", web,
limits=_runner.Limits(page_delay=0, stale_days=21))
assert (len(listing.ids), listing.dropped) == (15, 5)
# ------------------------------------------------------ through ATSSource
@pytest.fixture
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
assert state["ats:filtered:fake:acme"] == "1"
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
board.rows = lambda ids, slug: [_handed(i) for i in ids]
board.web.answers = {"https://jobs.fake.example/acme/page/0":
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
board.READS = board.READS.model_copy(update={"complete": "end"})
state: dict[str, str] = {}
assert len(list(_source(state).fetch())) == 19
assert state["ats:dropped:fake:acme"] == "1"
board.web.answers = {"https://jobs.fake.example/acme/page/0":
json.dumps([str(n) for n in range(15)]
+ [f"-id:{n}" for n in range(5)]).encode()}
state = {}
assert list(_source(state).fetch()) == []
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
assert "ats:ok:fake:acme" not in state
# -------------------------------------------------------------------- raw
def test_raw_keeps_every_stated_key_and_the_places() -> None:
+24
View File
@@ -417,6 +417,30 @@ def test_a_region_that_holds_the_agents_places_is_no_evidence_against(
assert oresund not in {r.uid for r in store.roles(places=["country:DK"])}
def test_a_role_whose_text_says_remote_passes_a_gate_on_remote_until_extract_reads_it(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""Berlin is evidence against Denmark, but a text that says remote while
no field does makes the role possibly remote (`remote_text`, prep): it
meets the remote member. Once extract says onsite, it fails again."""
said = role(store, "1", place="Berlin, Germany", remote="")
silent = role(store, "2", place="Berlin, Germany", remote="")
with store.transaction() as conn:
conn.execute("UPDATE roles SET remote_text = 1 WHERE uid = ?", (said,))
agent = store.save_agent("DK or remote", "", [
Gate(field="country", values=["DK"], mode="any"),
Gate(field="remote", values=["remote"], mode="any")])
run(config, store, score.run_score)
scores = store.scores(agent.id)
assert scores[said][1] == ""
assert scores[silent] == (0.0, "none of country DK, remote remote")
with store.transaction() as conn:
conn.execute("UPDATE roles SET remote = 'onsite' WHERE uid = ?", (said,))
run(config, store, score.run_score)
assert store.scores(agent.id)[said] == (0.0, "none of country DK, remote remote")
def test_a_wider_place_beside_a_place_elsewhere_is_no_match(
store: Store, config: Config, fake: FakeEmbedder) -> None:
"""Europe alone is no evidence against Denmark; `London; Europe` is in
+17
View File
@@ -383,6 +383,23 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch)
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count(
monkeypatch: pytest.MonkeyPatch) -> None:
"""The search driver reads through the same listing code as a board: the
runner drops the row the provider handed over without a title, and the
read is still complete, by the distinct result ids."""
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
module = readers.PROVIDERS["fake"]
handed = module.rows
module.rows = lambda decoded, slug: [
row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)]
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
"params": {"q": "x"}}]})
assert [i.source_id for i in source.fetch()] == ["fake:h1"]
assert source.state["search:complete:fake:cph"] == "1"
assert source.state["search:dropped:fake:cph"] == "1"
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
+14 -14
View File
@@ -71,7 +71,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
"application_events", "agents", "scores", "state"} <= tables
assert "idx_roles_cluster" in indexes
assert store.migrations_applied == []
assert LATEST == 33
assert LATEST == 35
# A second open finds it current and applies nothing.
assert Store(path).migrations_applied == []
@@ -169,8 +169,8 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
for table in ("vectors", "role_tags", "agent_companies", "role_votes"):
conn.execute(f"DROP TABLE {table}")
for column in ("musts", "broken"): # v29
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
# The must-have columns v28 and v29 added and v34 drops are not in a
# fresh file either.
conn.execute("ALTER TABLE agents DROP COLUMN like_text") # v30
conn.execute("PRAGMA user_version = 1")
@@ -191,7 +191,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
migrated = Store(path)
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
acme = migrated.company("acme")
assert acme is not None
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
@@ -396,7 +396,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
Prompt(name="Backend", text="python backend")]
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
@@ -420,7 +420,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
migrated = Store(path)
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
acme = migrated.company("acme")
assert acme is not None
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
@@ -449,7 +449,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
migrated = Store(path)
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
with sqlite3.connect(path) as conn:
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
"SELECT uid, place_fold, seniority FROM roles")}
@@ -691,7 +691,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
migrated = Store(path)
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
assert migrated.role_places(stated.uid) == []
@@ -733,7 +733,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
migrated = Store(path)
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
with sqlite3.connect(path) as conn:
found = {sid: rest for sid, *rest in conn.execute(
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
@@ -751,12 +751,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
migrated = Store(path)
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33]
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
role = migrated.role(posting("a").uid)
assert role is not None and role.posting_url == ""
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 25") # the column is there already
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33]
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
@@ -1855,7 +1855,7 @@ def test_v32_resets_the_limits_drops_prefer_gates_and_renames_enrich(tmp_path: P
migrated = Store(path)
assert migrated.migrations_applied == [32, 33]
assert migrated.migrations_applied == [32, 33, 34, 35]
by_name = {a.name: a for a in migrated.agents()}
assert by_name["Wolf"].limits == Limits() == by_name["Bare"].limits
assert [(g.field, g.mode) for g in by_name["Wolf"].gates] == [
@@ -1880,14 +1880,14 @@ def test_v32_runs_twice_and_keeps_limits_set_since(tmp_path: Path) -> None:
store = Store(path)
agent = store.save_agent("Wolf", limits=Limits(cut_fit_top=900))
_v31(path)
assert Store(path).migrations_applied == [32, 33]
assert Store(path).migrations_applied == [32, 33, 34, 35]
assert Store(path).agent(agent.id).limits == Limits() # type: ignore[union-attr]
narrow = Limits(cut_fit_top=900, summarise_top=10)
store.save_agent("Wolf", agent_id=agent.id, limits=narrow)
with sqlite3.connect(path) as conn:
conn.execute("PRAGMA user_version = 31") # the columns are there already
again = Store(path)
assert again.migrations_applied == [32, 33]
assert again.migrations_applied == [32, 33, 34, 35]
assert again.agent(agent.id).limits == narrow # type: ignore[union-attr]
+29 -1
View File
@@ -998,7 +998,13 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
template = page[page.index('<template id="agent-form-template">'):page.index("</template>")]
assert 'id="agent-form"' in template and 'id="a-cancel"' in template
assert 'id="a-name"' in page and 'placeholder="My search"' in page
assert 'id="a-prompts"' in page and "<details" in page and 'id="a-gates"' in page
assert 'id="a-prompts"' in page
# The gates are the four parts and nothing else: no rows of other rules,
# only a note naming a stored rule the parts cannot say.
for gone in ("<details", 'id="a-gates"', 'id="a-rules"', 'id="a-add-rule"', "Other rules"):
assert gone not in template
assert template.index('id="a-where-sum"') < template.index('id="a-kept"') \
< template.index("</fieldset>", template.index('id="a-where-sum"'))
for weight in ("a-fit", "a-recency", "a-affinity", "a-taste"):
assert f'id="{weight}"' in page
assert 'id="a-taste" min="0" step="0.05" value="0.2"' in page
@@ -1170,6 +1176,28 @@ def test_agents_are_created_updated_listed_and_deleted(client: TestClient) -> No
assert client.delete(f"/api/agents/{agent['id']}").status_code == 404
def test_an_agents_fetch_gate_and_a_rule_the_form_cannot_say_round_trip(client: TestClient) -> None:
"""What the form sends for the four parts, plus a stored gate on a field the
form has no part for, is served back as sent, so a load and a save keep it."""
europe = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS",
"LI", "NO", "GB", "CH", "EU"]
gates = [{"field": "country", "values": europe, "mode": "require"},
{"field": "country", "values": ["DK"], "mode": "any"},
{"field": "remote_from", "values": ["country:DK", "anywhere"], "mode": "any"},
{"field": "place", "values": ["Malmö"], "mode": "any"},
{"field": "title", "values": ["account executive"], "mode": "exclude"},
{"field": "seniority", "values": ["senior"], "mode": "require"}]
body = {"name": "Nordics", "prompts": [{"name": "Data", "text": "Head of Data"}],
"gates": gates, "fetch_terms": ["data"]}
agent = client.post("/api/agents", json=body).json()
assert agent["gates"] == gates and agent["fetch_terms"] == ["data"]
served = next(a for a in client.get("/api/agents").json() if a["id"] == agent["id"])
assert served["gates"] == gates
again = client.post("/api/agents", json={**body, "id": agent["id"], "gates": served["gates"]})
assert again.status_code == 200 and again.json()["gates"] == gates
def test_agents_take_and_serve_prompts_and_still_take_an_intent(client: TestClient) -> None:
prompts = [{"name": "AI and data leadership", "text": "VP, Head or Director of AI"},
{"name": "Forward deployed", "text": ""}]