Compare commits
16
Commits
e61185e4b9
...
920457d326
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
920457d326 | ||
|
|
2262cda8f1 | ||
|
|
1913101061 | ||
|
|
849adb197d | ||
|
|
136047023e | ||
|
|
911dfe8584 | ||
|
|
8b875b377b | ||
|
|
c6730062fd | ||
|
|
49929372c7 | ||
|
|
f141cffc3c | ||
|
|
02ea191b53 | ||
|
|
1976f6a353 | ||
|
|
2159c7925f | ||
|
|
0c4f3c9bbe | ||
|
|
38c31c830b | ||
|
|
4a9bcb2b1c |
No files matched your search
+13
-3
@@ -111,6 +111,12 @@ def same_posting(a: Copy, b: Copy) -> bool:
|
||||
return not ta or not tb or bool(ta & tb)
|
||||
|
||||
|
||||
def _same_country(a: Copy, b: Copy) -> bool:
|
||||
"""Whether two rows can be in one country: equal, or either unknown."""
|
||||
ca, cb = _country(a), _country(b)
|
||||
return not ca or not cb or ca == cb
|
||||
|
||||
|
||||
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
||||
"""The sort key that picks the kept copy of a group, smallest first: a
|
||||
board over a search provider, then one with a description, then the
|
||||
@@ -193,12 +199,16 @@ def fold(rows: Iterable[Copy]) -> Folded:
|
||||
|
||||
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
||||
again, in `keep_order`: a row joins the first group whose members are all
|
||||
of its host, `similar` to it and remote as it is or not. Only a row with
|
||||
of its host, in its country (equal, or either unknown, as `same_posting`
|
||||
reads it), `similar` to it and remote as it is or not. Only a row with
|
||||
a link can be folded, as its link is what the kept role keeps. Every row
|
||||
of a merged group points at its first row, the rows that pointed at a
|
||||
folded one included.
|
||||
|
||||
A remote listing and an office listing stay two roles: the kept role
|
||||
One country, as ADR Fetch prep and score 1.4 has it: a job posted once
|
||||
per office in several countries is several roles, one per country, which
|
||||
keeps a different opening with an alike text (a federal sales role in
|
||||
the US and one in Australia) from merging. A remote listing and an office listing stay two roles: the kept role
|
||||
takes a copy's places and not its remote, so the fetch gate, which reads
|
||||
a remote role by its area, would reject the remote listing with an
|
||||
office role elsewhere, and the office one with a remote role elsewhere."""
|
||||
@@ -216,7 +226,7 @@ def fold(rows: Iterable[Copy]) -> Folded:
|
||||
for row in sorted(part, key=keep_order):
|
||||
for group in groups:
|
||||
if all(m.host == row.host and m.remote == row.remote
|
||||
and similar(m.text, row.text) for m in group):
|
||||
and _same_country(m, row) and similar(m.text, row.text) for m in group):
|
||||
group.append(row)
|
||||
break
|
||||
else:
|
||||
|
||||
+25
-70
@@ -7,18 +7,14 @@ to three strengths and gaps. A model's rating is an estimate, so the scale is
|
||||
coarse. The total is `app.models.cv_score`, the three numbers under the
|
||||
agent's `cv_*` weights times 10, so 0 to 100, worked out when a role is served
|
||||
and not stored, so the weights are no part of `input_key`; it is what the
|
||||
feed's CV match sorts by. The model also reports a verdict on each of the
|
||||
agent's must-haves (one per line of its must text, numbered M1..Mn in the
|
||||
prompt); it does not apply them. The rule is code's: a "broken" one makes
|
||||
career count as at most `app.models.BROKEN_CAP` (4), when the role is
|
||||
served. Culture and location are left out: the first is guessed from ad text,
|
||||
the second is what the fetch gate does.
|
||||
feed's CV match sorts by. Culture and location are left out: the first is
|
||||
guessed from ad text, the second is what the fetch gate does.
|
||||
|
||||
The work is pairs, `(agent, role)`: an enabled agent with a CV, with or
|
||||
without a prompt, and an open role in its cut (`scores.cut`, see `app.jobs.score`: the best by
|
||||
fit, the best by CV closeness that those leave out, and the liked and saved)
|
||||
with some text, that has no judgement yet, or one read from another CV,
|
||||
prompts, like, avoid or must text or role text than today's (`input_key`).
|
||||
prompts, languages, like or avoid text or role text than today's (`input_key`).
|
||||
The cut is the only selection: the pull judges it after extract and the
|
||||
rescore that follows, so it reads the cut as recomputed. A pair is named by the composite key `"<agent_id>:<uid>"`,
|
||||
which is what `desk.modeltask` hands round as a uid, so `desk/` is untouched.
|
||||
@@ -51,9 +47,7 @@ from pipeline import NULL, RunContext, Task, job
|
||||
from pipeline.task import Batch
|
||||
from app.config import Config
|
||||
from app.jobs import stages
|
||||
from app.models import (
|
||||
LANGUAGE_LEVELS, MUST_VERDICTS, Agent, CvLanguage, cv_languages, must_lines,
|
||||
)
|
||||
from app.models import LANGUAGE_LEVELS, Agent, CvLanguage, _language_name, cv_languages
|
||||
from app.storage import Store
|
||||
|
||||
#: How much of a posting goes in one item: summarise's 8000, since the
|
||||
@@ -66,22 +60,28 @@ EXCERPT = 8000
|
||||
#: 3: what the person likes (`like_text`) beside what they do not, and the
|
||||
#: career rubric reading both.
|
||||
#: 4: the three numbers 0 to 10 rather than 0 to 100.
|
||||
FORMAT = 4
|
||||
#: 5: the person's languages, read into skills.
|
||||
#: 6: no must-haves: the profile is the CV, languages, likes and dislikes.
|
||||
FORMAT = 6
|
||||
|
||||
#: How many strengths and gaps are kept, and how long each may be.
|
||||
MOST, LONGEST = 3, 300
|
||||
|
||||
SYSTEM = """You judge job postings for one person: how well each fits their CV and what they want.
|
||||
|
||||
Below the rules is the person: their CV, what they want, what they like and
|
||||
what they do not like (the work, the workplace, situations), and their
|
||||
must-haves and deal-breakers when they gave any. Each item is one posting with a key, its title, company, place
|
||||
and text. Answer one judgement per item.
|
||||
Below the rules is the person: their CV, the languages they speak and how
|
||||
well, what they want, what they like and what they do not like (the work,
|
||||
the workplace, situations). Each item is one posting with a key, its title,
|
||||
company, place and text. Answer one judgement per item.
|
||||
|
||||
Score three dimensions, each a whole number from 0 to 10:
|
||||
- skills: 8-10 the core requirements are the person's primary skills; 6-7
|
||||
most requirements match, with 1-2 gaps that are learnable; 4-5 a partial
|
||||
match, significant upskilling needed; 0-3 a fundamental mismatch.
|
||||
match, significant upskilling needed; 0-3 a fundamental mismatch. A
|
||||
language the posting requires counts as a requirement: one the person
|
||||
speaks below the level asked, or not at all when their languages are
|
||||
listed, lowers skills and is a gap. A language the posting does not
|
||||
mention changes nothing.
|
||||
- experience: match on the function and nature of the work performed, not the
|
||||
literal job title, and on its level. 8-10 direct experience in the same
|
||||
domain and role type; 6-7 related, with transferable skills clear; 4-5
|
||||
@@ -93,16 +93,6 @@ Score three dimensions, each a whole number from 0 to 10:
|
||||
whether the tasks and the setting would energise them, not only whether
|
||||
they could do them.
|
||||
|
||||
Their must-haves and deal-breakers are numbered M1, M2 and so on. Give a
|
||||
verdict on every one of them, in `musts`, for each item:
|
||||
- "met": the posting text says it is met.
|
||||
- "broken": only when the posting text clearly contradicts it.
|
||||
- "unclear": the text does not say. Never infer from silence: a posting that
|
||||
is silent on a must-have is "unclear", not "broken".
|
||||
Judge a must-have by its own words, not by how well the role fits otherwise.
|
||||
Answer musts as [] when there are none. The verdicts do not change the three
|
||||
scores, which follow their rubrics; name a broken must-have as a gap.
|
||||
|
||||
Then 1 to 3 strengths and 1 to 3 gaps, each one short sentence grounded in
|
||||
the posting text. State the gaps honestly: a prestigious role that fits
|
||||
poorly scores low.
|
||||
@@ -126,21 +116,8 @@ SCHEMA: dict[str, Any] = {
|
||||
"career": {"type": "integer"},
|
||||
"strengths": {"type": "array", "items": {"type": "string"}},
|
||||
"gaps": {"type": "array", "items": {"type": "string"}},
|
||||
"musts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"verdict": {"type": "string", "enum": list(MUST_VERDICTS)},
|
||||
},
|
||||
"required": ["id", "verdict"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["key", "skills", "experience", "career", "strengths", "gaps",
|
||||
"musts"],
|
||||
"required": ["key", "skills", "experience", "career", "strengths", "gaps"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
@@ -181,10 +158,11 @@ def _text(row: Any) -> str:
|
||||
|
||||
def input_key(agent: Agent, text: str) -> str:
|
||||
"""A hash of everything a judgement reads: the CV, the prompts, the
|
||||
like, avoid and must texts, and the role's text as sent. A judgement stored
|
||||
under another key is read again."""
|
||||
languages, the like and avoid texts, and the role's text as sent. A judgement
|
||||
stored under another key is read again."""
|
||||
blob = json.dumps([FORMAT, agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
||||
agent.like_text, agent.avoid_text, agent.must_text, text], ensure_ascii=False)
|
||||
[[l.language, l.level] for l in agent.cv_languages],
|
||||
agent.like_text, agent.avoid_text, text], ensure_ascii=False)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
@@ -230,6 +208,9 @@ def pending(store: Store) -> int:
|
||||
def system(agent: Agent) -> str:
|
||||
"""The rubric, and the person as this agent describes them."""
|
||||
parts = [SYSTEM, "", "The person.", "", f"<cv>\n{agent.cv_text.strip()}\n</cv>"]
|
||||
if agent.cv_languages:
|
||||
spoken = ", ".join(f"{_language_name(l.language)} {l.level}" for l in agent.cv_languages)
|
||||
parts.append(f"\n<languages title=\"the languages they speak, and how well\">\n{spoken}\n</languages>")
|
||||
wants = "\n".join(f"- {p.name}: {' '.join(p.text.split())}"
|
||||
for p in agent.prompts if p.text.strip())
|
||||
if wants:
|
||||
@@ -240,9 +221,6 @@ def system(agent: Agent) -> str:
|
||||
if agent.avoid_text.strip():
|
||||
parts.append("\n<dislikes title=\"what they do not like\">\n"
|
||||
f"{agent.avoid_text.strip()}\n</dislikes>")
|
||||
if musts := must_lines(agent.must_text):
|
||||
lines = "\n".join(f"M{n}: {line}" for n, line in enumerate(musts, start=1))
|
||||
parts.append(f"\n<must title=\"must-haves and deal-breakers\">\n{lines}\n</must>")
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
@@ -290,25 +268,6 @@ def _lines(value: Any) -> list[str]:
|
||||
return [line for line in found if line][:MOST]
|
||||
|
||||
|
||||
def _musts(agent: Agent | None, value: Any) -> list[dict[str, str]]:
|
||||
"""The verdicts on the agent's must-haves as they are stored: one
|
||||
`{id, text, verdict}` per line of its must text, in order. A must-have
|
||||
the model left out, or gave a verdict outside "met", "broken", "unclear",
|
||||
is "unclear": nothing is broken by silence. An id that is not one of the
|
||||
agent's is ignored, the first answer for an id counts."""
|
||||
said: dict[str, str] = {}
|
||||
for one in value if isinstance(value, list) else []:
|
||||
if isinstance(one, dict):
|
||||
said.setdefault(str(one.get("id") or "").strip().upper(),
|
||||
str(one.get("verdict") or "").strip().lower())
|
||||
found = []
|
||||
for n, line in enumerate(must_lines(agent.must_text if agent else ""), start=1):
|
||||
verdict = said.get(f"M{n}", "")
|
||||
found.append({"id": f"M{n}", "text": line,
|
||||
"verdict": verdict if verdict in MUST_VERDICTS else "unclear"})
|
||||
return found
|
||||
|
||||
|
||||
def apply_batch(store: Store, answer: dict[str, Any],
|
||||
batch: dict[str, Any] | None = None,
|
||||
config: Config | None = None) -> JudgeResult:
|
||||
@@ -317,7 +276,6 @@ def apply_batch(store: Store, answer: dict[str, Any],
|
||||
all three numbers; the pair stays waiting."""
|
||||
keys: dict[str, str] = dict((batch or {}).get("keys") or {})
|
||||
stats = JudgeResult()
|
||||
agents: dict[int, Agent | None] = {}
|
||||
for one in answer.get("judgements") or []:
|
||||
key = str(one.get("key") or "")
|
||||
pair = pair_of(key)
|
||||
@@ -326,11 +284,8 @@ def apply_batch(store: Store, answer: dict[str, Any],
|
||||
stats.rejected.append(key or "?")
|
||||
continue
|
||||
skills, experience, career = (int(m) for m in marks if m is not None)
|
||||
if pair[0] not in agents:
|
||||
agents[pair[0]] = store.agent(pair[0])
|
||||
if not store.set_judgement(pair[0], pair[1], keys.pop(key), skills, experience, career,
|
||||
_lines(one.get("strengths")), _lines(one.get("gaps")),
|
||||
_musts(agents[pair[0]], one.get("musts"))):
|
||||
_lines(one.get("strengths")), _lines(one.get("gaps"))):
|
||||
stats.rejected.append(key)
|
||||
continue
|
||||
stats.judged += 1
|
||||
|
||||
+17
-3
@@ -10,7 +10,10 @@ both; nothing here decides how a board is read, so the words never disagree
|
||||
with the pull as long as they read the keys the pull writes.
|
||||
|
||||
A word with a number carries it after a colon ("without text: 12"), and the
|
||||
filter matches the phrase before the colon (`WORDS`).
|
||||
filter matches the phrase before the colon (`WORDS`). One word is outside
|
||||
`WORDS`: the listings the last read dropped for want of an id, a title or a
|
||||
link ("3 listings dropped: no title or link", `_runner.valid`), shown on the
|
||||
board's row and not offered by the filter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -49,7 +52,7 @@ WORDS: dict[str, str] = {
|
||||
|
||||
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
||||
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
||||
"narrowed", "detailfailed")
|
||||
"narrowed", "detailfailed", "dropped")
|
||||
|
||||
|
||||
class Texts(NamedTuple):
|
||||
@@ -84,7 +87,8 @@ def texts(store: Store) -> dict[int, Texts]:
|
||||
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
||||
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
||||
line: str = "", counts: Texts | None = None) -> list[str]:
|
||||
"""The limits of one board, in the order of `WORDS`, those that apply.
|
||||
"""The limits of one board, in the order of `WORDS`, those that apply, the
|
||||
rows the last read dropped after "read incomplete".
|
||||
|
||||
`state` holds the board's keys (`state_of`), `size` its kept size,
|
||||
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
||||
@@ -105,6 +109,8 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
|
||||
out.append("failing")
|
||||
if state.get(key("ok")) and not state.get(key("complete")):
|
||||
out.append("read incomplete")
|
||||
if dropped := _count(state.get(key("dropped"))):
|
||||
out.append(f"{dropped} listing{'' if dropped == 1 else 's'} dropped: no title or link")
|
||||
if state.get(key("swept")) == "1" or key("sweep") in state:
|
||||
out.append("swept")
|
||||
if state.get(key("filtered")) == "1":
|
||||
@@ -130,6 +136,14 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
|
||||
return out
|
||||
|
||||
|
||||
def _count(value: str | None) -> int:
|
||||
"""A number kept in state, 0 for none or one that does not parse."""
|
||||
try:
|
||||
return max(0, int(value or 0))
|
||||
except ValueError:
|
||||
return 0
|
||||
|
||||
|
||||
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
||||
"""Whether the reach leaves the board unread by place: its reader cannot
|
||||
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
||||
|
||||
+21
-14
@@ -37,7 +37,12 @@ not swept clears them.
|
||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||
the roles matching a word, which says nothing of the roles it did not list,
|
||||
so it closes by list nothing: a role of such a board closes once the board
|
||||
has not listed it for `STALE_DAYS`, the rule searches have.
|
||||
has not listed it for `STALE_DAYS`, the rule searches have. The same age
|
||||
rule is the floor under every board (ADR Fetch prep and score 1.3): an
|
||||
incomplete read (a ceiling, a page cap without a sweep) and a filtered read,
|
||||
which spares the roles acted on and the remote ones it could not ask for,
|
||||
close by age what no read has listed for `STALE_DAYS` (`_close_aged`), so no
|
||||
role stays open for good.
|
||||
|
||||
A search never closes by its list, even one stamped complete because it
|
||||
read as many results as its provider stated: a search is never a company's
|
||||
@@ -365,7 +370,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
incomplete += 1
|
||||
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||
partial.append(sub.target)
|
||||
continue
|
||||
# An incomplete read closes nothing by its list; what no
|
||||
# read has listed for STALE_DAYS still goes.
|
||||
read = _close_aged(store, source, sub.id, json.loads(
|
||||
store.state.get(f"{source}:ids:{sub.target}", "[]")), stale)
|
||||
shut = read
|
||||
closed += shut
|
||||
before = open_before.get(sub.id, 0)
|
||||
@@ -394,8 +402,9 @@ def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||
clears that record and closes what it did not list, all of it unfiltered
|
||||
(`Store.mark_closed`), filtered sparing the roles acted on
|
||||
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
||||
(`remote_unasked`) spares the remote ones as well, and closes those by
|
||||
age instead."""
|
||||
(`remote_unasked`) spares the remote ones as well. What a filtered read
|
||||
spares closes by age (`_close_aged`), as on an incomplete read
|
||||
(`close_boards`)."""
|
||||
state, target = store.state, sub.target
|
||||
if not state.get(f"{source}:complete:{target}", ""):
|
||||
return None
|
||||
@@ -404,8 +413,7 @@ def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||
return _close_aged(store, source, sub.id, seen_ids, stale)
|
||||
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
||||
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
||||
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
||||
if spare_remote else 0)
|
||||
aged = _close_aged(store, source, sub.id, seen_ids, stale) if is_filtered else 0
|
||||
if state.get(f"{source}:swept:{target}", ""):
|
||||
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
||||
spare_remote=spare_remote)
|
||||
@@ -484,20 +492,19 @@ def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription
|
||||
|
||||
|
||||
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
||||
stale: str, *, remote_only: bool = False) -> int:
|
||||
stale: str) -> int:
|
||||
"""Close the open roles of one board that its read did not list and that
|
||||
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
||||
writes on every listing): the rule for what a read cannot speak for. A
|
||||
read narrowed by terms closes all such roles, as a search does; with
|
||||
`remote_only`, a filtered read that could not ask for remote closes only
|
||||
its remote roles so, sparing those acted on as its list does."""
|
||||
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
||||
writes on every listing): the rule for what a read cannot speak for, a
|
||||
read narrowed by terms, an incomplete one, or what a filtered read
|
||||
spares. A role acted on is not spared here: three weeks unlisted is the
|
||||
one evidence a filtered or incomplete read gives."""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
f"""UPDATE roles SET closed_at = ?
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND last_seen < ?
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
||||
(store.now(), source, subscription_id, stale,
|
||||
json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
+39
-8
@@ -283,9 +283,18 @@ class Plan:
|
||||
return self._big(sub)
|
||||
|
||||
def _big(self, sub: Subscription) -> tuple[Reach, str]:
|
||||
"""The reach of `sub` as a board of `SMALL` roles or more."""
|
||||
"""The reach of `sub` as a board of `SMALL` roles or more. A host
|
||||
that cannot filter on remote while the agents want remote roles is
|
||||
asked for the wide reach, the agents' world (ADR Fetch prep and
|
||||
score 1.2): asked for "DK, SE" alone it would never list a remote
|
||||
role filed under Germany, and the gate sorts the rest out. Only
|
||||
where the host takes the world in one read, a list of countries or
|
||||
their ids: on a host that takes one country a request, Europe is
|
||||
the whole board, so the location read stays and remote is not asked."""
|
||||
if company_key(sub.company or sub.label) in self.all_countries:
|
||||
return self.wide, "countries"
|
||||
if self.base.remote and not _filters_remote(sub) and _takes_world(sub, self.wide):
|
||||
return self.wide, "world"
|
||||
return self.base, "agents"
|
||||
|
||||
def narrows_roles(self, sub: Subscription) -> bool:
|
||||
@@ -305,15 +314,34 @@ class Plan:
|
||||
if why == "unknown":
|
||||
found, why = self._big(sub)
|
||||
out = found.params()
|
||||
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
|
||||
if self.terms and why in ("countries", "agents", "world") and self.narrows_roles(sub):
|
||||
out = (out or {}) | {"terms": list(self.terms)}
|
||||
if why == "countries":
|
||||
if why in ("countries", "world"):
|
||||
out["wide"] = True
|
||||
if sub.target not in self.sizes:
|
||||
out = (out or {}) | {"unsized": True, "small": SMALL}
|
||||
return out
|
||||
|
||||
|
||||
def _filters_remote(sub: Subscription) -> bool:
|
||||
"""Whether `sub`'s host declares a remote filter at all. A host that
|
||||
declares one by ids may still lack the facet on a site, which the
|
||||
runner finds out when it asks (`_runner.ask_for`)."""
|
||||
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||
return module is None or module.READS.filters.remote != "none"
|
||||
|
||||
|
||||
def _takes_world(sub: Subscription, wide: Reach) -> bool:
|
||||
"""Whether `sub`'s host can be asked for the wide reach in one read: a
|
||||
country filter that takes a list (`many`, `ids`), or one value a request
|
||||
with the countries within `MAX_SERIES`; never everywhere."""
|
||||
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||
if module is None or wide.everywhere or not wide.countries:
|
||||
return False
|
||||
reach = module.READS.filters.country
|
||||
return reach in ("many", "ids") or (reach == "one" and len(wide.countries) <= _runner.MAX_SERIES)
|
||||
|
||||
|
||||
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
||||
"""Each board's size as kept (`ats:size:<target>`); a board never sized
|
||||
has none."""
|
||||
@@ -381,7 +409,7 @@ def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str
|
||||
#: Why a board is read whole, as the Companies list says it.
|
||||
_WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
|
||||
"override": "set here", "countries": "all locations (everywhere)",
|
||||
"agents": "an agent takes any place"}
|
||||
"world": "the agents' world (everywhere)", "agents": "an agent takes any place"}
|
||||
|
||||
|
||||
def _codes(codes: Iterable[str]) -> list[str]:
|
||||
@@ -486,6 +514,9 @@ def line(found: Plan, sub: Subscription) -> str:
|
||||
if why == "countries":
|
||||
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
||||
return f"all locations ({named}){unasked} · {roles}"
|
||||
if why == "world":
|
||||
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
||||
return f"follows agents' world ({named}){unasked} · {roles}"
|
||||
if why == "override":
|
||||
return f"set here: {_places(ask)}{unasked} · {roles}"
|
||||
return f"follows agents: {_places(ask)}{unasked} · {roles}"
|
||||
@@ -494,14 +525,14 @@ def line(found: Plan, sub: Subscription) -> str:
|
||||
what = "all locations (everywhere)" if why == "countries" else _WHOLE[why]
|
||||
return f"{what} · {roles}"
|
||||
return f"all places, too many countries to ask · {roles}"
|
||||
if why in ("agents", "countries") and given.everywhere:
|
||||
if why in ("agents", "countries", "world") and given.everywhere:
|
||||
return f"{_WHOLE[why]}: read whole"
|
||||
if why in ("agents", "countries", "override") and not module.READS.filters.declared():
|
||||
if why in ("agents", "countries", "world", "override") and not module.READS.filters.declared():
|
||||
return f"{module.LABEL} cannot filter: read whole"
|
||||
reads = module.READS.filters
|
||||
if why in ("agents", "countries") and reads.country == "one" \
|
||||
if why in ("agents", "countries", "world") and reads.country == "one" \
|
||||
and len(given.countries) > _runner.MAX_SERIES:
|
||||
return "too many countries to ask: read whole"
|
||||
if why in ("agents", "countries"):
|
||||
if why in ("agents", "countries", "world"):
|
||||
return f"{module.LABEL} cannot ask for these: read whole"
|
||||
return f"{_WHOLE[why]}: read whole"
|
||||
+9
-62
@@ -1008,79 +1008,30 @@ def language_flag(required: list[RoleLanguage],
|
||||
return "", ""
|
||||
|
||||
|
||||
#: What career counts as at most, on the judge's 0 to 10, when a posting
|
||||
#: breaks one of the agent's must-haves: the foot of the rubric's 4-5 band,
|
||||
#: a decent job that does not build toward the person's goals. The model
|
||||
#: reports whether each must-have is met; this cap is applied in code, when a
|
||||
#: role is served (`cv_score`, `Store.judgement`, `store._cv_sql`).
|
||||
BROKEN_CAP = 4
|
||||
|
||||
#: A must-have's verdict, as the judge answers it.
|
||||
MUST_VERDICTS = ("met", "broken", "unclear")
|
||||
|
||||
_BULLET = re.compile(r"^\s*(?:[-*\u2022]+|\(?\d+[.)])\s+")
|
||||
|
||||
|
||||
def must_lines(text: str) -> list[str]:
|
||||
"""The agent's must-haves text read as one must-have per line: a leading
|
||||
bullet (`-`, `*`, `•`) or number (`1.`, `2)`) is dropped, so are empty
|
||||
lines."""
|
||||
found = []
|
||||
for line in text.splitlines():
|
||||
line = " ".join(_BULLET.sub("", line, count=1).split())
|
||||
if line:
|
||||
found.append(line)
|
||||
return found
|
||||
|
||||
|
||||
def capped_career(career: int, broken: int = 0) -> int:
|
||||
"""`career` as it counts: at most `BROKEN_CAP` when any must-have is
|
||||
broken."""
|
||||
return min(career, BROKEN_CAP) if broken else career
|
||||
|
||||
|
||||
def cv_score(skills: int, experience: int, career: int,
|
||||
weights: Weights | None = None, broken: int = 0) -> float | None:
|
||||
weights: Weights | None = None) -> float | None:
|
||||
"""The one number of a CV match, 0 to 100 for sorting and display: the
|
||||
judge's three numbers, each 0 to 10, averaged by the agent's
|
||||
`cv_skills`, `cv_experience` and `cv_career` weights (30, 25 and 30 by
|
||||
default), career as `capped_career` has it, times 10. None when all
|
||||
three weights are 0."""
|
||||
default), times 10. None when all three weights are 0."""
|
||||
w = weights or Weights()
|
||||
total = w.cv_skills + w.cv_experience + w.cv_career
|
||||
if not total:
|
||||
return None
|
||||
career = capped_career(career, broken)
|
||||
return round(10 * (w.cv_skills * skills + w.cv_experience * experience
|
||||
+ w.cv_career * career) / total, 1)
|
||||
|
||||
|
||||
class Must(Model):
|
||||
"""One must-have of an agent as the judge read it against a role: its
|
||||
`id` (`M1`, by line), the line's `text` and the `verdict`."""
|
||||
|
||||
id: str
|
||||
text: str = ""
|
||||
verdict: Literal["met", "broken", "unclear"] = "unclear"
|
||||
|
||||
|
||||
class Judgement(Model):
|
||||
"""What `judge` read of one role for one agent (`role_judgements`): the
|
||||
three dimensions 0 to 10 as the model gave them, `cv_score`
|
||||
(`cv_score`, 0 to 100) and 1 to 3 strengths and gaps grounded in the
|
||||
posting.
|
||||
`musts` is the verdict on each must-have and `broken` how many are
|
||||
broken; any makes `career_capped` the lower of `career` and
|
||||
`BROKEN_CAP`. `cv_score` and `career_capped` are not read from the
|
||||
table: `Store.judgement` sets them from the agent's weights; `cv_score`
|
||||
is None when all three are 0."""
|
||||
posting. `cv_score` is not read from the table: `Store.judgement` sets
|
||||
it from the agent's weights; it is None when all three are 0."""
|
||||
|
||||
skills: int
|
||||
experience: int
|
||||
career: int
|
||||
career_capped: int | None = None
|
||||
broken: int = 0
|
||||
musts: Annotated[list[Must], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
||||
cv_score: float | None = None
|
||||
strengths: JsonStrings = []
|
||||
gaps: JsonStrings = []
|
||||
@@ -1574,14 +1525,11 @@ class Agent(Model):
|
||||
are the union over the enabled agents, and an agent with none means no
|
||||
board is narrowed by words. They decide what is fetched, not what ranks.
|
||||
|
||||
`cv_text`, `cv_languages`, `like_text`, `avoid_text` and `must_text` are
|
||||
what `judge` reads a role against: the CV, the languages it states with
|
||||
a level, what the person likes and does not like (the work, the
|
||||
workplace, situations; both feed the career score), and their
|
||||
must-haves and deal-breakers (set through the API only, not on the
|
||||
form; soft, read by the model only; a hard rule is a gate). An agent without
|
||||
a CV is not judged, and one without languages flags none
|
||||
(`language_flag`)."""
|
||||
`cv_text`, `cv_languages`, `like_text` and `avoid_text` are what `judge`
|
||||
reads a role against: the CV, the languages it states with a level, and
|
||||
what the person likes and does not like (the work, the workplace,
|
||||
situations; both feed the career score). An agent without a CV is not
|
||||
judged, and one without languages flags none (`language_flag`)."""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
@@ -1594,7 +1542,6 @@ class Agent(Model):
|
||||
cv_languages: CvLanguages = []
|
||||
like_text: str = ""
|
||||
avoid_text: str = ""
|
||||
must_text: str = ""
|
||||
enabled: bool = True
|
||||
created_at: str
|
||||
|
||||
|
||||
+6
-2
@@ -37,8 +37,9 @@ from app.storage import Store
|
||||
#: Raised when the rules in this package change the way a stored word maps,
|
||||
#: so every open role is mapped again as it is after a table changes. 3: the
|
||||
#: remote area, and remote as the model's before the host's word. 4: the
|
||||
#: deadline, the host's before the model's.
|
||||
VERSION = 4
|
||||
#: deadline, the host's before the model's. 5: `remote_text`, the title or
|
||||
#: text saying remote while no field does.
|
||||
VERSION = 5
|
||||
|
||||
#: The state key the last run's `fingerprint` is kept under.
|
||||
STATE = "prep:tables"
|
||||
@@ -61,6 +62,7 @@ def fingerprint(tables: Tables = TABLES) -> str:
|
||||
"""A hash of every host table, the shared remote words, the gazetteer the
|
||||
code describes and `VERSION`."""
|
||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
||||
"remote_text": [words.REMOTE_TEXT, words.REMOTE_TEXT_NOT],
|
||||
"places": repr(places.seed())}, sort_keys=True)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()
|
||||
|
||||
@@ -114,6 +116,7 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
||||
"seniority_title": titled,
|
||||
"employment": words.employment(host, row["employment_host"], tables),
|
||||
"remote": remote,
|
||||
"remote_text": int(words.remote_in_text(row["title"], row["text"] or "")),
|
||||
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
||||
if remote == "remote" else ()),
|
||||
"country": country(gaz, place_ids, row["country_host"]),
|
||||
@@ -125,6 +128,7 @@ def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||
or found["remote"] != row["remote"] \
|
||||
or found["remote_text"] != row["remote_text"] \
|
||||
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
||||
or (found["country"] and found["country"] != row["country"]) \
|
||||
or found["deadline"] != row["deadline"] \
|
||||
|
||||
@@ -58,6 +58,39 @@ _NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\
|
||||
re.IGNORECASE)
|
||||
|
||||
|
||||
#: Phrases in a title or text that say the role is worked remotely, and the
|
||||
#: phrases that take one back. Both are read on the lowercased text with its
|
||||
#: hyphens and runs of spaces squashed to one space. A match marks the role as
|
||||
#: possibly remote (`remote_in_text`); it decides nothing by itself.
|
||||
REMOTE_TEXT: tuple[str, ...] = (
|
||||
"fully remote", "100% remote", "100 % remote", "remote first", "remote work",
|
||||
"remote working", "work remotely", "working remotely", "remote position", "remote role",
|
||||
"remote job", "remote team", "remote friendly", "remote possible", "remote option",
|
||||
"remote ok", "work from home", "working from home", "work from anywhere", "home based",
|
||||
"home office", "homeoffice", "hjemmearbejde", "fjernarbejde", "distansarbete",
|
||||
)
|
||||
REMOTE_TEXT_NOT: tuple[str, ...] = (
|
||||
"no remote", "not remote", "not a remote", "non remote", "not be remote", "cannot be remote",
|
||||
"no home office", "kein homeoffice", "not available remotely", "not open to remote",
|
||||
"remote work is not", "remote working is not", "is not a remote", "remote sensing",
|
||||
"remote control", "remote monitoring",
|
||||
)
|
||||
|
||||
_REMOTE_TEXT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT))
|
||||
_REMOTE_TEXT_NOT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT_NOT))
|
||||
_SQUASHED = re.compile(r"[\s-]+")
|
||||
|
||||
|
||||
def remote_in_text(title: str, text: str) -> bool:
|
||||
"""Whether the title or the text says the role is worked remotely: a
|
||||
phrase of `REMOTE_TEXT` left once those of `REMOTE_TEXT_NOT` are taken
|
||||
out. Cheap evidence, no model: a role marked so passes a location gate
|
||||
on remote until extract has read it (ADR Fetch prep and score 1.1), so a
|
||||
false mark costs a cut slot at most, and a missed one nothing new."""
|
||||
folded = _SQUASHED.sub(" ", f"{title or ''} {text or ''}".lower())
|
||||
return bool(_REMOTE_TEXT.search(_REMOTE_TEXT_NOT.sub(" ", folded)))
|
||||
|
||||
|
||||
def _line_word(text: str) -> str:
|
||||
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
||||
if found := remote_of(text):
|
||||
|
||||
+55
-87
@@ -9,13 +9,15 @@
|
||||
* ADR's order: the world (the scope select), the location (a short list of
|
||||
* lines a role passes by matching any one of: countries, towns, remote), the
|
||||
* role exclusion (the titles to leave out) and the role fetch (the fetch
|
||||
* terms), over a row per remaining rule under Advanced; a gate the four parts
|
||||
* cannot say lands in a row, so a load and a save lose nothing. A gate is a
|
||||
* field, values and a mode (must, one of, not), with no weight. After it come
|
||||
* the analytics filter (the best N by prompt closeness and the best M by CV
|
||||
* closeness, each with a minimum level), the want feed score weights (prompt
|
||||
* closeness, newness, company like, job like), the CV match weights and the
|
||||
* size of summarise. Edit turns a card
|
||||
* terms), and nothing else: seniority, pay and the like are filters on Roles.
|
||||
* A gate is a field, values and a mode (must, one of, not), with no weight.
|
||||
* A stored gate the four parts cannot say, which only the config or the API
|
||||
* writes, is named in a note under the fetch gate and sent back as it came,
|
||||
* so a load and a save lose nothing. After the fetch gate come the analytics
|
||||
* filter (the best N by prompt closeness and the best M by CV closeness, each
|
||||
* with a minimum level), the want feed score weights (prompt closeness,
|
||||
* newness, company like, job like), the CV match weights and the size of
|
||||
* summarise. Edit turns a card
|
||||
* into the form from the page's `<template>`, in place, and New agent puts a
|
||||
* blank card on top in the same state. One card edits at a time, so the
|
||||
* form's ids stay unique. Votes, liked companies and scores are the agent's,
|
||||
@@ -30,10 +32,7 @@
|
||||
* text (typed, or read from a .md/.txt file by the browser), the languages it
|
||||
* states as rows of a code and a level, and the two soft texts, what the
|
||||
* person likes and what they do not like (the work, the workplace,
|
||||
* situations), which the judge reads for the career score. The must-have text
|
||||
* is no longer on the form: a save leaves `must_text` out, so the stored one
|
||||
* is kept (it is set through the API, and still caps career at 4 in code
|
||||
* when the judge reports one broken).
|
||||
* situations), which the judge reads for the career score.
|
||||
* Every save sends the CV, languages and the two texts, and the card's switch
|
||||
* re-posts the row as the server gave it, so neither drops them.
|
||||
*
|
||||
@@ -64,9 +63,8 @@
|
||||
const NEW = "new";
|
||||
// Europe as the starter agents spell it (static/remote.js).
|
||||
const EUROPE = rh.EUROPE;
|
||||
const FIELDS = ["place", "country", "remote", "remote_from", "seniority", "function", "salary_min",
|
||||
"currency", "permit", "languages", "company", "title"];
|
||||
const MODES = [["require", "must"], ["any", "one of"], ["exclude", "not"]];
|
||||
// A gate's mode in the words of the note on the gates the form keeps.
|
||||
const MODES = { require: "must", any: "one of", exclude: "not" };
|
||||
|
||||
/** @type {object[]} every agent, as the server lists them */
|
||||
let agents = [];
|
||||
@@ -266,8 +264,8 @@
|
||||
}
|
||||
|
||||
// The region and place in words, one sentence led by the region, then the lines a
|
||||
// role may match, with the one-of rows among them since they are the same
|
||||
// group: "Within Europe: keeps roles anywhere in Denmark, in Malmö, Lund or
|
||||
// role may match, with any other one-of gates among them since they are the
|
||||
// same group: "Within Europe: keeps roles anywhere in Denmark, in Malmö, Lund or
|
||||
// Helsingborg, or remote, and leaves out the rest." The lines add up, which
|
||||
// the first wording ("Roles in ...") left the owner to guess.
|
||||
function whereSentence(where, others) {
|
||||
@@ -384,7 +382,7 @@
|
||||
// ---------------------------------------------------------------- the form
|
||||
|
||||
// Tell the form something changed that no field typed: a chip or a row
|
||||
// added or removed. It marks the form dirty and redraws the gates view.
|
||||
// added or removed. It marks the form dirty and redraws the gates' sentence.
|
||||
function changed(node) {
|
||||
node.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
}
|
||||
@@ -559,9 +557,9 @@
|
||||
return box;
|
||||
}
|
||||
|
||||
// A language as a row, after the rule rows: a code, a level, Remove. The
|
||||
// code is two letters, which the server turns into the language; its name
|
||||
// is the row's title where the browser knows it.
|
||||
// A language as a row: a code, a level, Remove. The code is two letters,
|
||||
// which the server turns into the language; its name is the row's title
|
||||
// where the browser knows it.
|
||||
function languageRow(language, remove) {
|
||||
const row = el("div", "rule-row");
|
||||
const code = el("input");
|
||||
@@ -587,32 +585,32 @@
|
||||
return { node: row, read: () => ({ language: code.value.trim().toLowerCase(), level: level.value }) };
|
||||
}
|
||||
|
||||
// A gate as a row: field, mode, values and Remove. Values untouched since the load are sent as they came, so a
|
||||
// value with a comma in it survives a save.
|
||||
function ruleRow(gate, remove) {
|
||||
const row = el("div", "rule-row");
|
||||
const field = select(FIELDS.map((f) => [f, f.replace("_", " ")]), gate.field, "Field");
|
||||
const mode = select(MODES, gate.mode, "Mode");
|
||||
const values = el("input");
|
||||
values.type = "text";
|
||||
values.required = true;
|
||||
values.spellcheck = false;
|
||||
values.placeholder = "senior, staff";
|
||||
values.value = gate.values.join(", ");
|
||||
values.setAttribute("aria-label", "Values, separated by commas");
|
||||
const loaded = values.value;
|
||||
const drop = linky("Remove");
|
||||
drop.addEventListener("click", () => remove(row));
|
||||
row.append(field, mode, values, drop);
|
||||
// The gates the fetch gate's four parts say, in the order the starter
|
||||
// agents have them: the world, the location lines and the titles left out,
|
||||
// then `kept`, the stored gates the parts cannot say, as they came. `parts`
|
||||
// holds what the form reads: `scope` (europe, countries, everywhere) and
|
||||
// its `scoped` countries, the `within` countries, the `towns`, `remote`
|
||||
// (none, scope, from) and its `from` areas, and the `titles`.
|
||||
function formGates(parts, kept = []) {
|
||||
const out = [];
|
||||
const { scope, scoped, within, towns: lines, remote, from, titles } = parts;
|
||||
if (scope === "europe") out.push({ field: "country", values: rh.europeWith(within), mode: "require" });
|
||||
if (scope === "countries" && scoped.length) out.push({ field: "country", values: scoped, mode: "require" });
|
||||
if (within.length) out.push({ field: "country", values: within, mode: "any" });
|
||||
if (remote === "scope") out.push({ field: "remote", values: ["remote"], mode: "any" });
|
||||
if (remote === "from" && from.length) out.push({ field: "remote_from", values: from, mode: "any" });
|
||||
if (lines.length) out.push({ field: "place", values: lines, mode: "any" });
|
||||
if (titles.length) out.push({ field: "title", values: titles, mode: "exclude" });
|
||||
return out.concat(kept);
|
||||
}
|
||||
|
||||
function read() {
|
||||
const parts = values.value === loaded
|
||||
? gate.values
|
||||
: values.value.split(",").map((v) => v.trim()).filter(Boolean)
|
||||
.map((v) => (field.value === "salary_min" ? Number(v) : v));
|
||||
return { field: field.value, values: parts, mode: mode.value };
|
||||
}
|
||||
return { node: row, read };
|
||||
// The note on the gates the form keeps: "The form cannot edit these rules,
|
||||
// and Save keeps them as stored: must: with seniority senior; not: at Acme."
|
||||
function keptNote(kept) {
|
||||
if (!kept.length) return "";
|
||||
const said = kept.map((g) => `${MODES[g.mode] || g.mode}: ${phrase(g)}`).join("; ");
|
||||
return `The form cannot edit ${kept.length > 1 ? "these rules" : "this rule"}, and Save keeps `
|
||||
+ `${kept.length > 1 ? "them" : "it"} as stored: ${said}.`;
|
||||
}
|
||||
|
||||
function form(row) {
|
||||
@@ -713,51 +711,18 @@
|
||||
remoteFrom.node.hidden = remoteMode.value !== "from";
|
||||
};
|
||||
offer();
|
||||
$("details.advanced", f).open = rest.length > 0;
|
||||
const kept = $("#a-kept", f);
|
||||
kept.textContent = keptNote(rest);
|
||||
kept.hidden = !rest.length;
|
||||
|
||||
const rules = [];
|
||||
const dropRule = (node) => {
|
||||
rules.splice(rules.findIndex((r) => r.node === node), 1);
|
||||
node.remove();
|
||||
changed(f);
|
||||
};
|
||||
const addRule = (gate) => {
|
||||
const entry = ruleRow(gate, dropRule);
|
||||
rules.push(entry);
|
||||
$("#a-rules", f).appendChild(entry.node);
|
||||
return entry;
|
||||
};
|
||||
rest.forEach(addRule);
|
||||
$("#a-add-rule", f).addEventListener("click", () => {
|
||||
$("select", addRule({ field: "seniority", values: [], mode: "require" }).node).focus();
|
||||
changed(f);
|
||||
});
|
||||
|
||||
// The gates the fetch gate's three parts say, in the order the starter
|
||||
// agents have them: the region, the place lines, and the titles left out.
|
||||
function whereGates() {
|
||||
const out = [];
|
||||
const within = countries.values();
|
||||
const towns = places.values();
|
||||
const from = remoteFrom.values();
|
||||
if (scope.value === "europe") out.push({ field: "country", values: rh.europeWith(within), mode: "require" });
|
||||
if (scope.value === "countries" && scoped.values().length) {
|
||||
out.push({ field: "country", values: scoped.values(), mode: "require" });
|
||||
}
|
||||
if (within.length) out.push({ field: "country", values: within, mode: "any" });
|
||||
if (remoteMode.value === "scope") out.push({ field: "remote", values: ["remote"], mode: "any" });
|
||||
if (remoteMode.value === "from" && from.length) out.push({ field: "remote_from", values: from, mode: "any" });
|
||||
if (towns.length) out.push({ field: "place", values: towns, mode: "any" });
|
||||
const left = titles.values();
|
||||
if (left.length) out.push({ field: "title", values: left, mode: "exclude" });
|
||||
return out;
|
||||
}
|
||||
// The gates Save sends: the fetch gate's, then the rows. The sentence
|
||||
// under it says them all, as the card will, rows included.
|
||||
const gates = () => whereGates().concat(rules.map((r) => r.read()));
|
||||
// The gates Save sends: the four parts', then the kept ones. The sentence
|
||||
// under them says them all, as the card will, the kept ones included.
|
||||
const gates = () => formGates({
|
||||
scope: scope.value, scoped: scoped.values(), within: countries.values(), towns: places.values(),
|
||||
remote: remoteMode.value, from: remoteFrom.values(), titles: titles.values(),
|
||||
}, rest);
|
||||
const view = () => {
|
||||
offer();
|
||||
$("#a-gates", f).value = JSON.stringify(gates(), null, 2);
|
||||
$("#a-where-sum", f).textContent = rulesLine(gates()) || "Passes every role.";
|
||||
};
|
||||
view();
|
||||
@@ -981,6 +946,9 @@
|
||||
}
|
||||
}
|
||||
|
||||
// For tests/js/agents.test.mjs: what Save sends and how it is said.
|
||||
window.agentGates = { formGates, keptNote, rulesLine };
|
||||
|
||||
document.addEventListener("DOMContentLoaded", async () => {
|
||||
if (!$("#agents-list")) return;
|
||||
$("#a-new").addEventListener("click", () => open(null));
|
||||
|
||||
+3
-17
@@ -978,14 +978,6 @@ body[data-reader="open"] .company-strip { display: none; }
|
||||
.reader-cv .cv-total { margin: 0 0 .45rem; font-variant-numeric: tabular-nums; }
|
||||
.reader-cv .cv-total b { font-size: 1.25rem; }
|
||||
.reader-cv .cv-total span { display: inline-block; margin-left: .4rem; color: var(--dim); font-size: .88rem; }
|
||||
/* The must-haves, one line each with its verdict's mark. */
|
||||
.cv-musts { list-style: none; margin: 0 0 .55rem; padding: 0; font-size: .92rem; line-height: 1.4; }
|
||||
.cv-musts li { display: flex; gap: .45rem; }
|
||||
.cv-musts li + li { margin-top: .15rem; }
|
||||
.cv-musts .mark { flex: none; width: 1rem; text-align: center; font-weight: 700; }
|
||||
.cv-musts .must-met .mark { color: var(--good); }
|
||||
.cv-musts .must-broken .mark { color: var(--error); }
|
||||
.cv-musts .must-unclear .mark { color: var(--faint); }
|
||||
.cv-lists { display: grid; grid-template-columns: 1fr 1fr; gap: .5rem 1.2rem; }
|
||||
.cv-lists ul { margin: 0; padding-left: 1.1rem; font-size: .92rem; line-height: 1.45; }
|
||||
.cv-lists li + li { margin-top: .2rem; }
|
||||
@@ -1308,8 +1300,6 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
background: var(--bg); border: 1px solid var(--line);
|
||||
border-radius: .3rem; padding: .35rem .5rem; width: 100%;
|
||||
}
|
||||
/* Prose in a textarea is read in the page face; JSON is in the mono one. */
|
||||
.agent-form textarea.code { font-family: var(--mono); font-size: .85rem; }
|
||||
.agent-form .hint { color: var(--dim); font-size: .8rem; overflow-wrap: anywhere; }
|
||||
.weights { display: flex; gap: .75rem; flex-wrap: wrap; border: 1px solid var(--line); border-radius: .3rem; }
|
||||
/* A label that wraps to two lines pushes its box down; the boxes stay in line. */
|
||||
@@ -1324,14 +1314,13 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
}
|
||||
/* The fieldsets that hold rows: a column, the Add button under the rows. */
|
||||
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
|
||||
.agent-form fieldset.terms,
|
||||
.agent-form fieldset.rules {
|
||||
.agent-form fieldset.terms {
|
||||
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
|
||||
border: 1px solid var(--line); border-radius: .3rem;
|
||||
}
|
||||
.agent-form fieldset > button { align-self: flex-start; }
|
||||
/* A prompt or a rule as a row of a list, after `.crow`: the name and Remove
|
||||
on line one, the text spanning the row under them. A rule row wraps like
|
||||
/* A prompt or a language as a row of a list, after `.crow`: the name and Remove
|
||||
on line one, the text spanning the row under them. A language row wraps like
|
||||
`.crow .board-edit`, so a phone takes it in two lines rather than scrolling. */
|
||||
.prompt-row {
|
||||
display: grid; grid-template-columns: 1fr auto; align-items: center;
|
||||
@@ -1361,9 +1350,6 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
.agent-form .chips input[type="text"] { flex: 1 1 6rem; width: auto; min-width: 0; }
|
||||
.agent-form .chips select { flex: 0 1 12rem; min-width: 0; max-width: 100%; }
|
||||
.chips .tag span { color: var(--faint); }
|
||||
.agent-form details summary { cursor: pointer; color: var(--dim); font-size: .9rem; }
|
||||
.agent-form details[open] summary { margin-bottom: .5rem; }
|
||||
.agent-form .advanced-body { display: flex; flex-direction: column; gap: .6rem; }
|
||||
|
||||
/* ================================================================ */
|
||||
/* companies and sources — the Sources page's sections, */
|
||||
|
||||
+12
-10
@@ -1,7 +1,8 @@
|
||||
/* Where a remote role may be worked from, as the pages say it: the words of
|
||||
* the badge, the picks of the agent's "Remote, open to someone in" chips, and
|
||||
* how an agent's gates share out between the fetch gate's blocks and Other rules. No
|
||||
* DOM in it, so `tests/js/remote.test.mjs` can walk it without a browser.
|
||||
* how an agent's gates share out between the fetch gate's blocks and the rest
|
||||
* the form keeps as stored. No DOM in it, so `tests/js/remote.test.mjs` can
|
||||
* walk it without a browser.
|
||||
*
|
||||
* The area is a place token in three spellings, one per layer: `remote:<place
|
||||
* token>` on the Roles rail and in the address, `<place token>` in a
|
||||
@@ -61,18 +62,19 @@
|
||||
}
|
||||
|
||||
/** Share out an agent's gates between the fetch gate's blocks (region,
|
||||
place, titles to leave out) and Other rules. Each line takes the first
|
||||
gate it can say whole; the rest are rows. A gate has a field, values and
|
||||
a mode (require, any, exclude) and no weight; a gate with a mode the
|
||||
form does not know, such as an old prefer, stays a row.
|
||||
place, titles to leave out) and the rest, which the form cannot edit
|
||||
and keeps as stored. Each line takes the first gate it can say whole.
|
||||
A gate has a field, values and a mode (require, any, exclude) and no
|
||||
weight; a gate with a mode the form does not know, such as an old
|
||||
prefer, is in the rest.
|
||||
|
||||
The scope line is the first plain require gate on country: `europe`
|
||||
when it is exactly Europe and the countries line (`europeWith`),
|
||||
`countries` with its codes as chips otherwise, `everywhere` when there
|
||||
is none. Any other require gate on country is a row. The remote line is
|
||||
`scope` for the one-of on `remote: [remote]` (remote, within the
|
||||
scope), `from` for a one-of on `remote_from`, else `none`; when both
|
||||
are there `remote_from` wins and the plain one stays a row. */
|
||||
is none. Any other require gate on country is in the rest. The remote
|
||||
line is `scope` for the one-of on `remote: [remote]` (remote, within
|
||||
the scope), `from` for a one-of on `remote_from`, else `none`; when
|
||||
both are there `remote_from` wins and the plain one is in the rest. */
|
||||
function distribute(gates) {
|
||||
const where = { scope: null, countries: null, places: null, remote: { mode: "none", values: [] }, titles: null };
|
||||
const rest = [];
|
||||
|
||||
+4
-18
@@ -1744,34 +1744,20 @@
|
||||
try { return languageNames?.of(code) || code; } catch (err) { return code; }
|
||||
}
|
||||
|
||||
const MUST_MARKS = { met: ["✓", "met"], broken: ["✗", "broken"], unclear: ["?", "unclear"] };
|
||||
|
||||
/* The agent's `judge` reading against its CV: the total and its three
|
||||
numbers, each of 10, career as the server counted it (capped at 4 when a must-have
|
||||
is broken, with what the model said), the must-haves with a verdict
|
||||
each, then the strengths and gaps it named. A box as the summary is,
|
||||
since it is the same kind of thing: a model's reading, not the posting. */
|
||||
numbers, each of 10, then the strengths and gaps it named. A box as the
|
||||
summary is, since it is the same kind of thing: a model's reading, not
|
||||
the posting. */
|
||||
function cvMatch(r) {
|
||||
const j = r.judgement;
|
||||
if (!j) return "";
|
||||
const list = (items, none) => (items?.length
|
||||
? `<ul>${items.map((x) => `<li>${esc(x)}</li>`).join("")}</ul>`
|
||||
: `<p class="muted small">${none}</p>`);
|
||||
const counted = j.career_capped ?? j.career;
|
||||
const career = counted < j.career
|
||||
? `career ${counted}/10 (capped: breaks a must-have; model said ${j.career}/10)`
|
||||
: `career ${j.career}/10`;
|
||||
const musts = Array.isArray(j.musts) && j.musts.length
|
||||
? `<ul class="cv-musts">${j.musts.map((m) => {
|
||||
const [mark, word] = MUST_MARKS[m.verdict] || MUST_MARKS.unclear;
|
||||
return `<li class="must-${word}"><span class="mark" role="img"
|
||||
aria-label="${word}" title="${word}">${mark}</span><span>${esc(m.text)}</span></li>`;
|
||||
}).join("")}</ul>` : "";
|
||||
return `<section class="reader-summary reader-cv">
|
||||
<h2 class="reader-label">CV match</h2>
|
||||
<p class="cv-total"><b>${Math.round(j.cv_score)}</b> of 100
|
||||
<span>skills ${j.skills}/10 · experience ${j.experience}/10 · ${career}</span></p>
|
||||
${musts}
|
||||
<span>skills ${j.skills}/10 · experience ${j.experience}/10 · career ${j.career}/10</span></p>
|
||||
<div class="cv-lists">
|
||||
<div><h3 class="reader-label">Strengths</h3>${list(j.strengths, "None named.")}</div>
|
||||
<div><h3 class="reader-label">Gaps</h3>${list(j.gaps, "None named.")}</div>
|
||||
|
||||
+43
-16
@@ -55,6 +55,10 @@ CREATE TABLE IF NOT EXISTS roles (
|
||||
country TEXT NOT NULL DEFAULT '',
|
||||
remote TEXT NOT NULL DEFAULT ''
|
||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||
-- 1 when the title or text says the role is worked remotely while no
|
||||
-- field does (prep, app.prep.words.remote_in_text): the location gate
|
||||
-- lets such a role through until extract has read it.
|
||||
remote_text INTEGER NOT NULL DEFAULT 0,
|
||||
salary_min INTEGER,
|
||||
salary_max INTEGER,
|
||||
currency TEXT NOT NULL DEFAULT '',
|
||||
@@ -328,9 +332,8 @@ END;
|
||||
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
|
||||
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
|
||||
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
|
||||
-- level}), like_text (what they like: the work, the workplace, situations),
|
||||
-- avoid_text (what they do not like) and must_text (must-haves and
|
||||
-- deal-breakers, set through the API only) are what judge reads a role
|
||||
-- level}), like_text (what they like: the work, the workplace, situations)
|
||||
-- and avoid_text (what they do not like) are what judge reads a role
|
||||
-- against; an agent with no CV is not judged.
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
id INTEGER PRIMARY KEY,
|
||||
@@ -344,7 +347,6 @@ CREATE TABLE IF NOT EXISTS agents (
|
||||
cv_text TEXT NOT NULL DEFAULT '',
|
||||
cv_languages TEXT NOT NULL DEFAULT '[]',
|
||||
avoid_text TEXT NOT NULL DEFAULT '',
|
||||
must_text TEXT NOT NULL DEFAULT '',
|
||||
like_text TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
@@ -435,11 +437,8 @@ CREATE TABLE IF NOT EXISTS agent_cv_vectors (
|
||||
-- 100 until migration 33), cv_score their weighted total times 10 at the
|
||||
-- default weights (not read: app.models.cv_score is worked out when a role
|
||||
-- is served), strengths and gaps JSON lists. input_key
|
||||
-- is a hash of the CV, prompts, avoid text and the role's text it read: a
|
||||
-- different one is read again. musts is a JSON list of {id, text, verdict},
|
||||
-- the model's verdict on each of the agent's must-haves (one per line of its
|
||||
-- must text), broken how many are "broken": career counts as at most 4 then
|
||||
-- (app.models.BROKEN_CAP), applied when the role is served. tries counts
|
||||
-- is a hash of the CV, prompts, languages, like and avoid texts and the
|
||||
-- role's text it read: a different one is read again. tries counts
|
||||
-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A
|
||||
-- row with judged_at NULL holds only tries.
|
||||
CREATE TABLE IF NOT EXISTS role_judgements (
|
||||
@@ -451,8 +450,6 @@ CREATE TABLE IF NOT EXISTS role_judgements (
|
||||
cv_score REAL,
|
||||
strengths TEXT NOT NULL DEFAULT '[]',
|
||||
gaps TEXT NOT NULL DEFAULT '[]',
|
||||
musts TEXT NOT NULL DEFAULT '[]',
|
||||
broken INTEGER NOT NULL DEFAULT 0,
|
||||
input_key TEXT NOT NULL DEFAULT '',
|
||||
tries INTEGER NOT NULL DEFAULT 0,
|
||||
judged_at TEXT,
|
||||
@@ -763,6 +760,12 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
||||
# A Python step: JSON limits, and a count for the log.
|
||||
(33, "the judge on 0 to 10: every judgement cleared to be made again, and summarise_top "
|
||||
"50 becomes 30", lambda conn: _judge_out_of_ten(conn)),
|
||||
# A Python step: it drops the columns only where they are there. The
|
||||
# judge reads every role again (judge.FORMAT).
|
||||
(34, "agents.must_text, role_judgements.musts and broken dropped: the profile is the "
|
||||
"CV, languages, likes and dislikes", lambda conn: _drop_musts(conn)),
|
||||
(35, "roles.remote_text: the text says remote while no field does, read by prep for the "
|
||||
"location gate", lambda conn: _add_remote_text(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -1012,6 +1015,15 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
||||
|
||||
|
||||
def _add_remote_text(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 35's step: `roles.remote_text`, 0 until the next prep,
|
||||
which maps every open role again since its rules changed. Runs twice
|
||||
without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||
if "remote_text" not in columns:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN remote_text INTEGER NOT NULL DEFAULT 0")
|
||||
|
||||
|
||||
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 27's step: `roles.remote_model` empty and
|
||||
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
||||
@@ -1203,12 +1215,14 @@ def _judge_out_of_ten(conn: sqlite3.Connection) -> None:
|
||||
old number is shown on the new scale. The rows stay, as rows of no
|
||||
judgement, and the judge reads every pair again. An agent whose
|
||||
`summarise_top` is 50, migration 32's default, takes 30, the new one.
|
||||
Runs twice without harm."""
|
||||
Runs twice without harm, also after migration 34 dropped the verdicts."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
||||
verdicts = ", musts = '[]', broken = 0" if {"musts", "broken"} <= columns else ""
|
||||
cleared = conn.execute(
|
||||
"""UPDATE role_judgements
|
||||
SET skills = NULL, experience = NULL, career = NULL, cv_score = NULL,
|
||||
strengths = '[]', gaps = '[]', musts = '[]', broken = 0,
|
||||
input_key = '', tries = 0, judged_at = NULL""").rowcount
|
||||
f"""UPDATE role_judgements
|
||||
SET skills = NULL, experience = NULL, career = NULL, cv_score = NULL,
|
||||
strengths = '[]', gaps = '[]'{verdicts},
|
||||
input_key = '', tries = 0, judged_at = NULL""").rowcount
|
||||
lowered = conn.execute(
|
||||
"""UPDATE agents SET limits = json_set(limits, '$.summarise_top', 30)
|
||||
WHERE json_valid(limits) AND json_type(limits) = 'object'
|
||||
@@ -1217,6 +1231,19 @@ def _judge_out_of_ten(conn: sqlite3.Connection) -> None:
|
||||
"summarise_top 50 set to 30", cleared, lowered)
|
||||
|
||||
|
||||
def _drop_musts(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 34's step: the must-haves are gone, the agent's text
|
||||
(`agents.must_text`) and the judge's verdicts on it
|
||||
(`role_judgements.musts`, `broken`), and with them the cap on career.
|
||||
Runs twice without harm."""
|
||||
for table, columns in (("agents", ("must_text",)),
|
||||
("role_judgements", ("musts", "broken"))):
|
||||
present = {r[1] for r in conn.execute(f"PRAGMA table_info({table})")}
|
||||
for column in columns:
|
||||
if column in present:
|
||||
conn.execute(f"ALTER TABLE {table} DROP COLUMN {column}")
|
||||
|
||||
|
||||
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 21's step. The column; the tag for every company that is a
|
||||
favourite of any agent, since favourite no longer widens what a pull
|
||||
|
||||
+32
-39
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -26,7 +26,7 @@ from app.models import (
|
||||
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
|
||||
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
|
||||
BROKEN_CAP, capped_career, language_flag, role_languages,
|
||||
language_flag, role_languages,
|
||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
@@ -140,7 +140,7 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
|
||||
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
||||
|
||||
#: The agent's `judge` readings beside every role, the same way: `j.skills`,
|
||||
#: `j.experience`, `j.career` and `j.broken` are NULL for a role not judged and with no
|
||||
#: `j.experience` and `j.career` are NULL for a role not judged and with no
|
||||
#: agent.
|
||||
_WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? "
|
||||
"AND j.judged_at IS NOT NULL")
|
||||
@@ -150,19 +150,15 @@ _WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agen
|
||||
def _cv_sql(weights: Weights | None) -> str:
|
||||
"""`cv_score` as SQL for one agent's weights, 0 to 100 from the judge's
|
||||
three numbers 0 to 10, unrounded (`app.models.cv_score` rounds it): NULL
|
||||
for a role not judged, and for all three weights 0. Career
|
||||
counts as at most `BROKEN_CAP` when the judgement holds a broken
|
||||
must-have (`j.broken`), as `app.models.capped_career` has it. The
|
||||
numbers are the model's validated, finite floats, so they are written in
|
||||
rather than bound, the select's placeholders being then no concern."""
|
||||
for a role not judged, and for all three weights 0. The weights are
|
||||
validated, finite floats, so they are written in rather than bound, the
|
||||
select's placeholders being then no concern."""
|
||||
w = weights or Weights()
|
||||
total = w.cv_skills + w.cv_experience + w.cv_career
|
||||
if not total:
|
||||
return "NULL"
|
||||
career = (f"(CASE WHEN j.broken > 0 AND j.career > {BROKEN_CAP} THEN {BROKEN_CAP} "
|
||||
"ELSE j.career END)")
|
||||
return (f"(10.0 * ({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience "
|
||||
f"+ {w.cv_career!r} * {career}) / {total!r})")
|
||||
f"+ {w.cv_career!r} * j.career) / {total!r})")
|
||||
|
||||
|
||||
def _by_cv(expression: str) -> str:
|
||||
@@ -359,6 +355,12 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
||||
return ("r.languages IN ('', '[]')",
|
||||
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
||||
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
||||
if gate.field == "remote" and "remote" in values:
|
||||
# A role whose text says remote while no field does is possibly
|
||||
# remote (`remote_text`, prep): it meets a gate on remote until
|
||||
# extract has read it, and is not empty for it.
|
||||
return (f"({column} = '' AND r.remote_text = 0)",
|
||||
f"(LOWER({column}) IN ({marks}) OR ({column} = '' AND r.remote_text = 1))", values)
|
||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||
|
||||
|
||||
@@ -1629,8 +1631,9 @@ class Store:
|
||||
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
||||
seniority_host, employment_host, remote_host, country_host,
|
||||
seniority_model, seniority_stated, seniority_title, seniority,
|
||||
employment, remote, country, remote_model, remote_where_model,
|
||||
deadline_host, deadline_model, deadline,
|
||||
employment, remote, remote_text, country, remote_model,
|
||||
remote_where_model, deadline_host, deadline_model, deadline,
|
||||
SUBSTR(COALESCE(NULLIF(description, ''), teaser), 1, 6000) AS text,
|
||||
json_extract(raw, '$.places') AS raw_places,
|
||||
(SELECT json_group_array(place_id) FROM (
|
||||
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
||||
@@ -1656,8 +1659,8 @@ class Store:
|
||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
|
||||
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
||||
`seniority_title`, `employment`, `remote`, `remote_text`, `remote_area`,
|
||||
`country` and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
||||
the title's. `remote` is written as prep composed it (the model's,
|
||||
else the host's word, else the place line) and `remote_area`, place
|
||||
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
||||
@@ -1673,6 +1676,7 @@ class Store:
|
||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||
employment = :employment,
|
||||
remote = :remote,
|
||||
remote_text = :remote_text,
|
||||
country = COALESCE(NULLIF(:country, ''), country),
|
||||
deadline = :deadline,
|
||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||
@@ -2021,33 +2025,28 @@ class Store:
|
||||
|
||||
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
|
||||
experience: int, career: int, strengths: Sequence[str],
|
||||
gaps: Sequence[str], musts: Sequence[Mapping[str, str]] = ()) -> bool:
|
||||
gaps: Sequence[str]) -> bool:
|
||||
"""Store `judge`'s reading of one role for one agent, replacing the
|
||||
one before: the three numbers as the model gave them, the verdict on
|
||||
each of the agent's must-haves (`musts`, `{id, text, verdict}`) and how
|
||||
many are broken, `cv_score` (`app.models.cv_score`, at the default
|
||||
weights and with the cap: the table's column is not read, see
|
||||
`judgement`) and its tries back at 0. False for an unknown role or
|
||||
agent."""
|
||||
broken = sum(1 for m in musts if m.get("verdict") == "broken")
|
||||
one before: the three numbers as the model gave them, `cv_score`
|
||||
(`app.models.cv_score`, at the default weights: the table's column is
|
||||
not read, see `judgement`) and its tries back at 0. False for an
|
||||
unknown role or agent."""
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
conn.execute(
|
||||
"""INSERT INTO role_judgements (uid, agent_id, skills, experience, career,
|
||||
cv_score, strengths, gaps, musts, broken,
|
||||
cv_score, strengths, gaps,
|
||||
input_key, tries, judged_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,0,?)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,0,?)
|
||||
ON CONFLICT(uid, agent_id) DO UPDATE SET
|
||||
skills=excluded.skills, experience=excluded.experience,
|
||||
career=excluded.career, cv_score=excluded.cv_score,
|
||||
strengths=excluded.strengths, gaps=excluded.gaps,
|
||||
musts=excluded.musts, broken=excluded.broken,
|
||||
input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""",
|
||||
(uid, agent_id, skills, experience, career,
|
||||
cv_score(skills, experience, career, broken=broken),
|
||||
cv_score(skills, experience, career),
|
||||
json.dumps(list(strengths), ensure_ascii=False),
|
||||
json.dumps(list(gaps), ensure_ascii=False),
|
||||
json.dumps([dict(m) for m in musts], ensure_ascii=False), broken,
|
||||
input_key, self.now()))
|
||||
except sqlite3.IntegrityError:
|
||||
return False
|
||||
@@ -2075,8 +2074,7 @@ class Store:
|
||||
def judgement(self, agent_id: int | None, uid: str) -> Judgement | None:
|
||||
"""The agent's judgement of one role, None when it has none, its
|
||||
`cv_score` the three numbers under the agent's weights as they are
|
||||
now, not as stored, and `career_capped` the model's career as the
|
||||
broken must-haves leave it (`BROKEN_CAP`)."""
|
||||
now, not as stored."""
|
||||
if agent_id is None:
|
||||
return None
|
||||
row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? "
|
||||
@@ -2085,9 +2083,8 @@ class Store:
|
||||
return None
|
||||
found = Judgement.from_row(row)
|
||||
agent = self.agent(agent_id)
|
||||
found.career_capped = capped_career(found.career, found.broken)
|
||||
found.cv_score = cv_score(found.skills, found.experience, found.career,
|
||||
agent.weights if agent else None, found.broken)
|
||||
agent.weights if agent else None)
|
||||
return found
|
||||
|
||||
def set_article_text(self, uid: str, text: str) -> Role | None:
|
||||
@@ -2209,14 +2206,13 @@ class Store:
|
||||
cv_text: str | None = None,
|
||||
cv_languages: Sequence[CvLanguage] | None = None,
|
||||
avoid_text: str | None = None,
|
||||
must_text: str | None = None,
|
||||
like_text: str | None = None) -> Agent:
|
||||
"""Create an agent, or with `agent_id` replace that one's fields and
|
||||
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
|
||||
named after the agent. Without `fetch_terms` an update keeps the
|
||||
stored ones and a new agent has none, so a caller that does not know
|
||||
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
|
||||
`like_text`, `avoid_text` and `must_text` are kept the same way. A prompt whose text is
|
||||
`like_text` and `avoid_text` are kept the same way. A prompt whose text is
|
||||
unchanged keeps its cached vector, whatever its name or place, so an
|
||||
edit re-embeds only the prompts it changed. A save gives every role
|
||||
`judge` gave up on for the agent its tries back, since what it is
|
||||
@@ -2237,7 +2233,6 @@ class Store:
|
||||
(limits or Limits()).model_dump_json(), int(enabled), terms,
|
||||
None if cv_text is None else cv_text.strip(), spoken,
|
||||
None if avoid_text is None else avoid_text.strip(),
|
||||
None if must_text is None else must_text.strip(),
|
||||
None if like_text is None else like_text.strip())
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
@@ -2245,10 +2240,9 @@ class Store:
|
||||
agent_id = conn.execute(
|
||||
"""INSERT INTO agents (name, gates, weights, limits, enabled,
|
||||
fetch_terms, cv_text, cv_languages, avoid_text,
|
||||
must_text, like_text, created_at)
|
||||
like_text, created_at)
|
||||
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
|
||||
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),
|
||||
COALESCE(?, ''),?)""",
|
||||
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),?)""",
|
||||
(*fields, self.now())).lastrowid
|
||||
elif not conn.execute(
|
||||
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
|
||||
@@ -2256,7 +2250,6 @@ class Store:
|
||||
cv_text=COALESCE(?, cv_text),
|
||||
cv_languages=COALESCE(?, cv_languages),
|
||||
avoid_text=COALESCE(?, avoid_text),
|
||||
must_text=COALESCE(?, must_text),
|
||||
like_text=COALESCE(?, like_text)
|
||||
WHERE id=?""",
|
||||
(*fields, agent_id)).rowcount:
|
||||
|
||||
@@ -74,6 +74,7 @@
|
||||
</div>
|
||||
<span class="hint">Fetch terms, separated by a comma or Enter: <code>machine learning, data science</code> is two terms, and <code>machine learning</code> is searched as one. A host that takes a query sends only roles with one of the terms anywhere in the ad, but only for boards of 300 or more roles; smaller boards and other hosts are read whole. The terms are ignored after the fetch, since prompt closeness is the finer filter. Empty means no narrowing.</span>
|
||||
<p id="a-where-sum" class="where-sum" role="status"></p>
|
||||
<p id="a-kept" class="hint" hidden></p>
|
||||
</fieldset>
|
||||
<fieldset class="terms cv">
|
||||
<legend>Your CV</legend>
|
||||
@@ -94,7 +95,7 @@
|
||||
<textarea id="a-like-text" rows="4" placeholder="Building a team from scratch, close work with the product side, small companies, a say in the roadmap"></textarea></label>
|
||||
<label>What I don't like
|
||||
<textarea id="a-avoid-text" rows="4" placeholder="Constant on-call, sales targets, large consultancies, open-plan offices"></textarea></label>
|
||||
<span class="hint">The work, the workplace, situations; read by the judge for the career rating. Rules that must hold go under the fetch gate or Other rules instead.</span>
|
||||
<span class="hint">The work, the workplace, situations; read by the judge for the career rating. Rules that must hold go under the fetch gate instead.</span>
|
||||
</fieldset>
|
||||
<fieldset class="weights limits">
|
||||
<legend>Analytics filter</legend>
|
||||
@@ -124,20 +125,6 @@
|
||||
<label>Summarise the top <input type="number" id="a-summarise-top" min="0" step="5" value="30"> of each feed order</label>
|
||||
</fieldset>
|
||||
<span class="hint">Summarise writes a short summary and tags for the top N of What I'm looking for and the top N of What fits me, at most twice N roles, and for every role you liked or saved. It is no part of any score.</span>
|
||||
<details class="advanced">
|
||||
<summary>Advanced</summary>
|
||||
<div class="advanced-body">
|
||||
<fieldset class="rules">
|
||||
<legend>Other rules</legend>
|
||||
<div id="a-rules" class="rule-rows"></div>
|
||||
<button type="button" id="a-add-rule" class="secondary">Add rule</button>
|
||||
<span class="hint">Must: a role must meet it. One of: a role must meet one of these or a Place line. Not: a role that meets it is left out. Values are separated by commas; salary is a number.</span>
|
||||
</fieldset>
|
||||
<label>Rules as sent
|
||||
<textarea id="a-gates" class="code" rows="6" spellcheck="false" readonly>[]</textarea>
|
||||
<span class="hint">Read only: what Save sends as the agent's gates, from the fetch gate and Other rules.</span></label>
|
||||
</div>
|
||||
</details>
|
||||
<div class="agent-actions">
|
||||
<button type="submit">Save and score</button>
|
||||
<button type="button" id="a-cancel" class="secondary">Cancel</button>
|
||||
|
||||
+2
-4
@@ -1251,7 +1251,7 @@ class AgentBody(BaseModel):
|
||||
it is stored as one prompt named after the agent. `fetch_terms` narrow
|
||||
what big boards send (`app/jobs/reach.py`); `[]` narrows nothing, and
|
||||
leaving the key out keeps an existing agent's. `cv_text`,
|
||||
`cv_languages`, `like_text`, `avoid_text` and `must_text` are what `judge` reads a
|
||||
`cv_languages`, `like_text` and `avoid_text` are what `judge` reads a
|
||||
role against (`app/jobs/judge.py`), kept the same way when left out or
|
||||
null."""
|
||||
|
||||
@@ -1267,7 +1267,6 @@ class AgentBody(BaseModel):
|
||||
cv_languages: CvLanguages | None = None
|
||||
like_text: str | None = None
|
||||
avoid_text: str | None = None
|
||||
must_text: str | None = None
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
@@ -1307,8 +1306,7 @@ def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent:
|
||||
body.enabled, agent_id=body.id, limits=body.limits,
|
||||
prompts=body.prompts, fetch_terms=body.fetch_terms,
|
||||
cv_text=body.cv_text, cv_languages=body.cv_languages,
|
||||
like_text=body.like_text, avoid_text=body.avoid_text,
|
||||
must_text=body.must_text)
|
||||
like_text=body.like_text, avoid_text=body.avoid_text)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(409, str(exc)) from exc
|
||||
except KeyError:
|
||||
|
||||
+10
-2
@@ -16,7 +16,7 @@ What was decided for roledesk and the reason for each. The target design is the
|
||||
|
||||
**A group is rows that all match each other, never a chain.** A row with no place matches Belgium and the Netherlands, and joining on one match merged both on the live data. Rows are taken sharpest first (a town, then only a country, then neither) and join a group only when they match every member.
|
||||
|
||||
**One posting per place is folded too, on every host.** A host that lists one job once per office gives rows of the same company and title whose towns never meet; `dupes.fold` merges those of one host when their texts share at least `SAME_TEXT` (30%) of their words, and the kept role keeps each copy's place and link. It applies to every host, not only to readers that declare `places="copies"`, because most such rows came from hosts that do not declare it. A remote listing and an office listing stay two roles: the kept role takes a copy's places and not its remote, and the fetch gate reads a remote role by its area, so a remote listing folded behind an office elsewhere was rejected with it, and an office listing behind a remote role elsewhere likewise.
|
||||
**One posting per place is folded too, on every host.** A host that lists one job once per office gives rows of the same company and title whose towns never meet; `dupes.fold` merges those of one host when their texts share at least `SAME_TEXT` (30%) of their words, and the kept role keeps each copy's place and link. It applies to every host, not only to readers that declare `places="copies"`, because most such rows came from hosts that do not declare it. It keeps to one country, as ADR 1.4 has it, since the same rule across countries merged different openings whose texts happened to be alike (a federal sales role in the US behind one in Australia); a job posted once per country is then one role per country, which costs a few rows for a pan-European posting and hides nothing. A remote listing and an office listing stay two roles: the kept role takes a copy's places and not its remote, and the fetch gate reads a remote role by its area, so a remote listing folded behind an office elsewhere was rejected with it, and an office listing behind a remote role elsewhere likewise.
|
||||
|
||||
**A copy is stored as `roles.dup_of` and rebuilt from scratch after every pull.** Every reader of open roles adds `dup_of IS NULL`, which keeps `idx_roles_posted` usable where a window function over the filtered rows would scan them all. Where a group changed, the kept role takes the earliest `read_at` and each agent's latest vote from its copies; an unchanged group is left alone, so marking a role unread again sticks. The per-board counts on Sources still count copies, since they report what a board holds.
|
||||
|
||||
@@ -70,6 +70,12 @@ What was decided for roledesk and the reason for each. The target design is the
|
||||
|
||||
**A read that stalls is incomplete, and a sweep's cursor may be a pair.** A reader that sees the host send a page again hands it over empty with a cursor, and the runner ends the read incomplete; a cursor of None there read as the last page, and on a host that states no total as a complete read that closed what the pages had not listed. With a stated total a page of ids already read is not a stall, since a resumed sweep starts on one when the list grew at the top. A cursor is kept across pulls when it survives JSON apart from a tuple coming back as a list, so Workday's and SuccessFactors' sweeps resume.
|
||||
|
||||
**A host that cannot filter on remote is asked for the agents' world where it takes the world in one read** (ADR Fetch prep and score 1.2). Asked for "DK, SE" alone such a host never listed a remote role filed under Germany, and 18 boards fetched no new remote role outside those two countries. Phenom and the Workday tenants without a remote facet take a list of countries, so Europe is one request and the gate sorts the rest out. SmartRecruiters, SuccessFactors and Eightfold take one country a request, where Europe is the whole board (Bosch 49 pages, a SuccessFactors site 10 roles a page), so there the location read stays and the row says remote was not asked.
|
||||
|
||||
**A role whose text says remote while no field does is possibly remote, read in prep with no model** (ADR 1.1 and 1.5). A role outside the agent's locations whose remote is unstated could never "turn out to be remote": extract reads only the cut, and the cut needs the gate. 5,016 open roles were rejected that way, 805 of them saying "remote" in their text. A phrase table ("fully remote", "work from home", the Danish and Swedish words) less its denials ("no remote", "remote sensing") marks `remote_text`; the gate's remote member takes the mark as a match while `remote` is empty, and once extract has answered, prep composes `remote` from the model and the mark no longer counts. A false mark costs at most a cut slot.
|
||||
|
||||
**Age is the floor under every board's closing** (ADR Fetch prep and score 1.3). An incomplete read (a ceiling, a page cap without a sweep) closed nothing ever, and a filtered read spared the roles acted on from its list and from age alike, so such roles stayed open for good. Now what no read has listed for 21 days closes under an incomplete read, and what a filtered read spares closes by age too; a role acted on is spared from the list close, which is guesswork, not from three weeks of absence. The alternative was to strike the sentence from the ADR, which would have left "may stay open for good" in the design.
|
||||
|
||||
**Switching a board or search off closes its open roles.** Only a pull that reads a target closes its roles, so a target no longer read would keep them open for good. Switched back on, the next pull reopens what it still lists. A batch in flight when the switch happens reopens them, so the settle closes every switched-off target once more.
|
||||
|
||||
**A destructive switch asks first; a reopening one queues a pull.** Switching a board or search off, replacing a board and detaching one each close roles in a tap, so the page asks first when there are roles to lose, naming the count. Switching a target on, following or setting a board, setting All locations or All roles on and saving a search each queue `pull` at once instead of waiting up to six hours for the schedule. The pull is queued in the API route, not in the page's script, so a script calling the endpoint gets what a click gets; `claim_run`'s refusal is not an error, since the run in flight or the next one reads the change.
|
||||
@@ -182,6 +188,8 @@ What was decided for roledesk and the reason for each. The target design is the
|
||||
|
||||
**`extract_batch`, `judge_batch` and `summarise_batch` run isolated, with a timeout.** `context.check()` runs only between the calls `modeltask.one` makes, not inside one, so a call that hangs was unkillable short of restarting the server; a process boundary lets a Kill, or the timeout, end it. One call is bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for judge and summarise): a batch still running after that is stuck, not slow. The cost is that an isolated job's context has no working `stop`, `task` or `fan_out` (`context_for` in `pipeline/child.py`), so a refusal or an account-level failure inside a batch cannot spare the task's queued siblings.
|
||||
|
||||
**The profile is the CV, the languages, the prompts, the likes and the dislikes; must-haves are gone** (ADR 4.2). The must-have text with a verdict per line and a career cap at 4 was the harder form of what likes and dislikes now say softly; it left the form on 2026-10-03 and its machinery followed on 2026-10-06 (migration 34). The judge reads the languages with their levels, since a role that asks for fluent Danish against conversational Danish is a skills gap the CV text alone did not show; a change to the languages re-judges the cut like a change to the CV.
|
||||
|
||||
**Judge reads a role against a written profile, after the `/rank` of github.com/MadsLorentzen/ai-job-search.** Prompt closeness compares topics; it cannot tell a step down, years of experience or a deal-breaker, which a model reading both texts can. The profile is the CV, its languages with a level, and two soft texts, "What I like" (`like_text`) and "What I don't like" (`avoid_text`), each covering the work, the workplace and situations with no split into tasks versus environment, since the prompts already say what the owner wants. A deal-breaker is a hard rule and belongs in the fetch gate, so the form has no must-haves box; `must_text` is set only through the API. Culture is left out because it is guessed from ad text, and location because the fetch gate does it.
|
||||
|
||||
**The model reports facts and code applies the rules.** For each must-have line the model answers "met", "broken" only when the posting clearly contradicts it, or "unclear" when the text does not say, never inferred from silence; a must-have it leaves out counts as unclear. The cap is code's: a broken must-have makes career count as at most `BROKEN_CAP` (4) when a role is served (`app/models.py`), so the stored career stays the model's and the reader shows both. The CV match weights are applied when a role is served too, so changing them needs no model call. The verdicts store each line's text, so the reader still shows what a judgement was made against after the text is edited. The `input_key` carries `judge.FORMAT` (4) and the texts, so a change to the prompt's contract makes every judgement again.
|
||||
@@ -236,7 +244,7 @@ What was decided for roledesk and the reason for each. The target design is the
|
||||
|
||||
**Search agents are cards, edited in place.** One person may run several agents: their own hunt and one that watches roles for someone else. An agent is defined by what it looks for, so its prompts are the card's body and the weights a small line under it. Edit turns that card into the form and New puts a blank one on top, because a form at the bottom of the page was out of sight of the card it changed. Job boards and Watchlist companies are edited the same way, one open at a time.
|
||||
|
||||
**A gate the form cannot say lands in a row under Advanced, so a load and a save lose nothing.** The form asks the fetch gate in its four parts by name rather than as the stored `require` and `any` sets with ISO codes, which the owner could not read; any other gate is a row under Other rules. Spellings of a town that differ by accents only share one chip, and both are stored.
|
||||
**The form's gates are the four parts of the fetch gate, and nothing else** (ADR Fetch prep and score 5.2 and Settings). Free rows of field, mode and values over twelve fields sat under Advanced, first to keep a rule an older form had saved; a row on seniority or pay was a permanent cut on fields half the roles leave empty, which the rail's filters do with a click and undo. The rows are gone; a stored gate the form cannot say is named under the fetch gate and kept as it is on save, so nothing is lost silently.
|
||||
|
||||
**Viewing as on Roles is a select, also with one agent.** The owner looked for the select to switch agent and found it gone after the second agent was switched off. It lists the enabled agents, since an agent that is off has no fresh scores.
|
||||
|
||||
|
||||
+6
-36
@@ -1,48 +1,18 @@
|
||||
# Open
|
||||
|
||||
What is known and not yet fixed, checked against the code on 2026-10-05, the day of the architecture review. What was decided is in `decisions.md`, the target design in `documentation/`, the unbuilt readers in `plan-hosts-wave-2.md`. Counts are of the live database on that day.
|
||||
What is known and not yet fixed, checked against the code on 2026-10-06, after the architecture review of 2026-10-05. What was decided is in `decisions.md`, the target design in `documentation/`, the unbuilt readers in `plan-hosts-wave-2.md`. Counts are of the live database on that day.
|
||||
|
||||
## For the owner to decide
|
||||
|
||||
Points where the code and `ADR Fetch prep and score` disagree, or where the ADR is silent and roles are lost.
|
||||
|
||||
- **Remote roles on hosts that cannot filter on remote.** On 18 boards (SmartRecruiters, Phenom, SuccessFactors, Eightfold, and Workday tenants without a remote facet) the read for "DK, SE, remote" goes out as "DK, SE". The row says "remote not asked" and the remote roles already held are kept, but a new remote role elsewhere in the world gate is never fetched. ADR 1.2 ("a host that cannot filter on a part sends the board whole for that part") reads as reading such boards whole; Bosch is then about 50 requests a pull.
|
||||
- **A role outside the agent's locations with remote unstated is never read by extract.** ADR 4.1 says a role can "turn out to be remote", but extract reads only the cut and the cut requires the gate. 5,016 open roles are rejected for location with remote unknown, and 805 of them say "remote" in their text. A word scan in prep costs no tokens and would let those through to extract.
|
||||
- **Copies across countries.** ADR 1.4 requires the same country and overlapping towns. `dupes.fold` also merges one host's listings of one title across towns and countries when 30% of their words are shared: 2,125 open copies, about 1,550 in another country than the kept role, some of them different openings (Datadog's US federal role behind the Canberra one). The gate reads the kept role through its copies' places, so no office listing is hidden. Remote and office listings are kept apart since 2026-10-05; 176 remote listings still sit behind a remote kept role in another country, whose area alone the world gate reads.
|
||||
- **Judge and languages.** ADR 4.2 names the agent's languages as an input. The judge does not read them (they only set the flag on a row), and editing them re-judges nothing. The ADR's list also omits the prompts and the must-haves, which the judge reads and which re-judge.
|
||||
- **Must-haves.** A broken must-have caps career at 4, but `must_text` is set only through the API: it is not on the form and not in the ADR's Settings.
|
||||
- **Show gated and Advanced rules.** ADR 5 says no order shows a role the gate rejects and that seniority and pay are filters. Show gated lists the rejected roles with the reason, on purpose; an Advanced row can gate on any of twelve fields.
|
||||
- **Closing by age on boards.** ADR 1.3 closes a role no read has listed for 21 days. That holds for searches and for boards narrowed by terms. A board read that stays incomplete (a ceiling, the page cap without a sweep) closes nothing ever, and on a filtered board a role with any vote or an application never closes. Since closing is not valued, striking the rule from the ADR is the cheaper way to agree.
|
||||
- **Remote on hosts that take one country a request.** SmartRecruiters, SuccessFactors and Eightfold (14 boards, about 15,000 roles) are still asked for DK, SE alone when the agents want remote, since Europe there is the whole board. Reading them whole is the only way to their remote roles elsewhere.
|
||||
- **Remote listings across countries.** A job posted remote once per country is now one role per country (the fold keeps to one country), which lists a pan-European remote posting several times.
|
||||
- **A search cut at its page cap leaves no record.** In the last pull 4 of 26 searches stopped at a cap (Jobbank and freehire at 1,000 rows, two Jobindex searches at 200). Nothing on a page says so, and Jobbank's feed is not sorted by date, so new ads can lie past the cap.
|
||||
- **A row without a title, link or id is dropped by its reader, unrecorded,** while its id still counts toward completeness. `ADR Plugins` says both "never drops rows" and "title and url are required"; the runner could validate, log and mark the read incomplete.
|
||||
- **The world gate is not sent to a host when an agent sets no location.** Every big board is then read whole. Nothing is lost.
|
||||
|
||||
## What the ADRs do not say
|
||||
## What the ADRs still do not say
|
||||
|
||||
What someone rebuilding from `ADR Fetch prep and score` would have to guess, by section.
|
||||
`ADR Fetch prep and score` was brought up to the code on 2026-10-06; left for the owner's read as a whole.
|
||||
|
||||
- **Prep has no section**, though the title names it: a host's words mapped through a table per host, a place text resolved against a gazetteer of towns, regions, countries and continents with regions across borders, remote and the remote area read from the place words, the seniority ladder, and that a text nothing maps is listed and passes the gate.
|
||||
- **1.1 Fetch gate.** Every role is stored whatever the gates say, and each agent applies its gate at scoring. An unknown location is no resolved place and no country, which includes an unresolved text and a wider region alone. A role in several places needs one inside. The world gate reads a remote role by its remote area. Exclusion matches whole words of the title.
|
||||
- **1.2 Reading a board.** The union is per agent the location gate, or the world gate when no location is set, then over the agents. What happens when a host filters countries but not remote. A board's own saved params win over the agents'. With no enabled agent every board is read whole. A search reads 5 pages by default and 10 at most.
|
||||
- **1.3 Closing.** "Filtered read" is not defined. Switching a board or search off closes its roles; a closed role listed again reopens.
|
||||
- **1.4 Copies.** The second rule, one host's listings of one title merged when their texts are alike. Groups never chain. Votes and read state move to the kept copy. How titles and company names are normalised.
|
||||
- **2.1 Closeness.** The embedding model (`intfloat/multilingual-e5-small` with its query and passage prefixes) and what is embedded (title and the first 2,000 characters of the text, else the teaser). The CV is cut into chunks of at most 1,200 characters. A level is a linear clip between two cosines measured per model (prompt 0.75 to 0.88, CV 0.77 to 0.90, job like -0.14 to 0.19). With several prompts each is scaled and the best counts. The bonus divides by all four weights also when one has left the sum.
|
||||
- **2.2 Filter.** The bullets carry the "and" and the "or" by layout only. A liked or saved role must still pass the gate. A like is this agent's; a save under any agent counts.
|
||||
- **3 Fetch text.** Only hosts with a role page are read, Jobindex never in a pull, and the caps (200 a board, 1,000 a run, retry after 7 days) are in `ADR Plugins` only.
|
||||
- **4 Model analytics.** The model of each call and the batch sizes (20, 5, 10). Extract reads the first 2,500 and last 1,500 characters and also fills pay, currency, function and work permit; it gives up after 3 unanswered tries. The judge reads up to 8,000 characters, rates career against the prompts, likes and dislikes, and the CV match is 10 times the weighted mean, 0 to 100. Summarise writes about 60 words and 3 to 8 tags, once. A stage that is off, or has no model to ask, is skipped without a notice; only Settings › Sources › Runs says so. A cut role without text is neither judged nor summarised.
|
||||
- **5 Feed.** Open roles and kept copies only. Opening or voting marks a role read. The list of filters, 60 rows a page, that ★ also likes, and what a role without an embedding scores. Newest is a third order, where the glossary and the flow say two.
|
||||
- **Settings.** The agent's On/Off switch and the pull's schedule (every 6 h).
|
||||
|
||||
Wording to settle: 2.1 "over the sum of the want feed score weights" against 5.1.3 "its weight leaves the sum"; 4 "once extract has completed the ingestion" against 4.1 "a role that enters is extracted on the next pull"; 4.1's field list against "raw and processed fields for e.g. salary"; "model analytics score" is never defined (the number is the CV match); 1.3 against `ADR Plugins` 3.2, which lets a search declare `stated`, and 2.1, which says an incomplete read closes nothing; the title is "Architecture" while the other ADR calls it by its file name.
|
||||
|
||||
`ADR Plugins`, read lightly since it is not final:
|
||||
|
||||
- **The API block already differs from the code:** `empty` is "trusted" or "suspect", not None; `Page` is a plain tuple; `Reads` has an unused `title="provisional"`; a provider must define `allowed`; a reader may import `desk`.
|
||||
- **Rules the code needed and the ADR lacks:** a cursor must survive JSON, where a tuple returns as a list; a reader that sees a page come again hands it over empty with a cursor; a wrong-shaped answer is an error, never an empty list.
|
||||
- **Rules the code does not keep:** "at most 8 series" is enforced only with terms or more than 8 countries; the 10% guard starts at 1 role, not 10, for filtered, narrowed and suspect reads; a 429 after page one starts a sweep; a module that breaks the contract stops the import at the first problem, and with it the app.
|
||||
- **"Never maps a word"** is broken by `Place.remote`, which makes every reader decide whether "hybrid" is office, and by the national providers that state their country (Jobindex DK, Platsbanken SE).
|
||||
- **"Nothing else changes"** holds for a host. A provider with a new param or a key touches the Sources form, its script, the CLI, the API body and `sources.search_params`.
|
||||
- Cases 4 and 5 are handled after the plugin, so "upstream" reads the wrong way round. The constants (0.5 s, 100 pages, 96 MiB, 21 and 7 days) will go stale faster than the rules.
|
||||
`ADR Plugins` was rewritten on 2026-10-06 and names a target the code has not reached: a broken plugin left out instead of stopping the app; providers declaring their params with label, kind and choices, so form, CLI and API follow from the module; detail state kept for a search's roles as for a board's; a national provider declaring its implied country; page markers for hosts on a company's own domain; a contract test that names no reader.
|
||||
|
||||
## Structure
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Architecture
|
||||
# ADR Fetch prep and score
|
||||
|
||||
The target design of roledesk's ranking: how a role gets from a board to the Roles feed for one search agent, and the key decisions behind it. What gets built is tested against this document.
|
||||
|
||||
- **Fetch gate**: the agent's hard rules, a yes or no per role, in four parts: world, location, role exclusion and role fetch.
|
||||
- **Input analytics score** and **input analytics filter**: what decides the **cut**, the roles that pass the analytics filter and get fetch text and the `model analytics score`.
|
||||
- **Feed scores**: the two orders of the feed, the want feed score and the match feed score.
|
||||
- **Input analytics score** and **input analytics filter**: what decides the **cut**, the roles that pass the analytics filter and get fetch text and the `model analytics score`, which is the `CV match` (4.2.1).
|
||||
- **Feed scores**: the two scores that order the feed, the want feed score and the match feed score; Newest is the third order.
|
||||
- **Fetch text**: the second HTTP read of a role's own page, without a model.
|
||||
- **Extract**, **judge**, **summarise**: the model calls (Haiku).
|
||||
- **Affinity**: an agent's setting for a company: hidden, neutral, liked or favourite.
|
||||
@@ -17,6 +17,7 @@ The target design of roledesk's ranking: how a role gets from a board to the Rol
|
||||
world · location · role exclusion · role fetch, with All locations and All roles as overrides
|
||||
company boards: whole under 300 roles, else filtered at the host where it can
|
||||
search providers: their own saved query
|
||||
then prep, no tokens: host words, places, remote, country; the copies folded
|
||||
│
|
||||
▼
|
||||
2 Input analytics score, no tokens ─ prompt closeness, CV closeness, affinity → the cut
|
||||
@@ -28,7 +29,7 @@ The target design of roledesk's ranking: how a role gets from a board to the Rol
|
||||
4 Model analytics score ─ extract the cut → fetch gate and cut again → judge the cut
|
||||
│
|
||||
▼
|
||||
5 Feed ─ every role that passes the fetch gate, in two orders;
|
||||
5 Feed ─ every role that passes the fetch gate, in three orders;
|
||||
filters and saved filters narrow it
|
||||
│
|
||||
▼
|
||||
@@ -51,7 +52,7 @@ The stricter location filter. A role passes the `location fetch gate` when it ma
|
||||
|
||||
#### 1.1.3 Role exclusion gate
|
||||
|
||||
A list of titles to leave out: phrases that reject a role by its title. No host can filter by exclusion, so this gate is applied to every role after fetch.
|
||||
A list of titles to leave out: phrases that reject a role by its title, matched on whole words. No host can filter by exclusion, so this gate is applied to every role after fetch.
|
||||
|
||||
#### 1.1.4 Role fetch gate
|
||||
|
||||
@@ -64,26 +65,34 @@ The fetch terms: words a big board's host must match before it sends a role. The
|
||||
- A search provider is read by its own saved query, which replaces the `role fetch gate` (1.1.4).
|
||||
- A hidden company never passes.
|
||||
|
||||
The gate rejects on evidence only. A role with an unknown location passes until extract has read it.
|
||||
The gate rejects on evidence only. A role with an unknown location passes until extract has read it. A location is unknown when no place of the role resolved and no country is stated, an unresolved place text included. A place that holds the agent's places (Øresund or Europe for Denmark) is no evidence against the role, unless the role also names a place elsewhere. A role in several places passes when one is inside. A hybrid or onsite role at an unknown location passes. A role whose text says remote while its fields do not (1.5) passes the location gate until extract has read it. The world gate reads a remote role by its remote area.
|
||||
|
||||
### 1.2 Reading a board
|
||||
|
||||
- A company board with fewer than 300 roles is read whole.
|
||||
- A company board with 300 or more roles is asked for the fetch gate at the host: the world and location gates, and the fetch terms, where the host can filter on them. A host that cannot filter on a part sends the board whole for that part.
|
||||
- A board's size is measured once, when it is added or rechecked, not on every read.
|
||||
- One fetch serves all enabled agents. The host is asked for the union of their world and location gates and the union of their fetch terms; one enabled agent without fetch terms means no terms. Each agent then applies its own gate to what arrived.
|
||||
- One fetch serves all enabled agents. The host is asked for, per agent, its location gate, or its world gate when it sets no location, united over the agents, and for the union of their fetch terms; one enabled agent without fetch terms means no terms. Each agent then applies its own gate to what arrived.
|
||||
- A host that filters on countries but not on remote is asked for the world gate's countries when an agent wants remote roles, so a remote role listed under any country in the world gate still arrives; the read says that remote was not asked. This holds where the host takes the world in one read; on a host that takes one country a request, the world is the whole board, so the location gate is asked and remote roles elsewhere are not fetched.
|
||||
- A board's own saved params win over the agents' gates on the dimensions they set. With no enabled agent every board is read whole.
|
||||
|
||||
### 1.3 Closing
|
||||
|
||||
- A role closes when a complete read of its board no longer lists it.
|
||||
- A read that is not complete closes nothing: one narrowed by fetch terms, a search provider's, a partial or failed read. Roles on such boards close by age. This is accepted because ranking relies on recency, not on closing.
|
||||
- A read that is not complete closes nothing: one narrowed by fetch terms, a search provider's, a partial or failed read. Roles on such boards close by age. This is accepted because ranking relies on recency, not on closing. A read filtered by location is complete for what it asked: it closes the roles inside its filter that it no longer lists.
|
||||
- Switching a board or search off closes its roles. A closed role that a read lists again reopens. A search is read 5 pages deep, 10 at most.
|
||||
- A role that no read has listed for 21 days closes. A board that has failed on every read for 21 days closes its roles.
|
||||
- A liked or saved role is never closed by a filtered read.
|
||||
|
||||
### 1.4 Copies
|
||||
|
||||
- One posting listed several times (by a search provider and by the company's board, or by one board under several ids) is one role. Two listings are one posting when they share company, title and country, and their towns overlap; a listing without a town matches any town.
|
||||
- The kept copy is the company board's over a search provider's, then the one with text, then the earliest posted. Only the kept copy is scored, read by a model and shown; it lists the others.
|
||||
- The kept copy is the company board's over a search provider's, then the one with text, then the earliest posted. Only the kept copy is scored, read by a model and shown; it lists the others. Votes and read state move to the kept copy.
|
||||
- A host that posts one job once per office gives listings of one title whose towns never meet. Those of one host are one role when their texts share 30% of their words, within one country, remote and office kept apart; the kept copy carries every office's place.
|
||||
|
||||
### 1.5 Prep
|
||||
|
||||
After fetch, without a model: a host's words for seniority, employment and remote map through a table per host; each place text resolves against a gazetteer of towns, regions, countries and continents, where a region may lie in two countries; remote and the remote area are read from the place words; the country comes from the resolved places; a role whose title or text says remote ("fully remote", "work from home", not "no remote") is marked as possibly remote for the gate (1.1). A text nothing maps is listed under Unmapped for the owner to add, and until then the role's location counts as unknown.
|
||||
|
||||
## 2 Input analytics score and filter
|
||||
|
||||
@@ -94,17 +103,18 @@ Each role is embedded and scored semantically:
|
||||
- **Prompt closeness** is its level against the best of the agent's prompts.
|
||||
- **CV closeness** is its level against the closest part of the CV.
|
||||
|
||||
Both are calibrated levels from 0 to 1. A company's affinity adds to both levels the share it has in the `want feed score`: the company like weight times the company like's rise above neutral (5.1.1), over the sum of the want feed score weights. On the start weights a liked company adds about 0.056, a favourite about 0.111. The bonus counts before the count and the minimum apply.
|
||||
Both are calibrated levels from 0 to 1. A company's affinity adds to both levels the share it has in the `want feed score`: the company like weight times the company like's rise above neutral (5.1.1), over the sum of all four want feed score weights, also when one of them has left the sum. On the start weights a liked company adds about 0.056, a favourite about 0.111. The bonus counts before the count and the minimum apply.
|
||||
|
||||
The embedding is `intfloat/multilingual-e5-small`, with its query and passage prefixes, over the title and the first 2,000 characters of the text, else the teaser. The CV is cut into chunks of at most 1,200 characters and the closest chunk counts. A level is a linear clip of the cosine between two bounds measured per model: prompt closeness 0.75 to 0.88, CV closeness 0.77 to 0.90, job like -0.14 to 0.19. With several prompts each is scaled on its own and the best counts.
|
||||
|
||||
#### 2.2 Input analytics filter
|
||||
|
||||
The `closeness`es are the main input for the `input analytics filter`, which determines which roles get `fetch text` and the `model analytics score`. A role passes the `input analytics filter` when it:
|
||||
|
||||
- passes the `fetch gate` (1.1.1 to 1.1.3)
|
||||
|
||||
- passes the `fetch gate` (1.1.1 to 1.1.3), and
|
||||
- is among the best N by `prompt closeness`, each at least X (500 and 0.6 to start), or
|
||||
- is among the best M by `CV closeness` that the first list leaves out, each at least Y (150 and 0.6 to start), or
|
||||
- is liked or saved not matter the `closeness`.
|
||||
- is liked by this agent or saved under any agent, whatever its `closeness`.
|
||||
|
||||
The two lists are kept apart because they answer different questions (what I want, what I am), and a merged number lets the stronger side crowd the other out. For a broad agent the count stops each list; for a narrow agent the minimum stops it, so roles that do not fit are not read just to fill the count.
|
||||
|
||||
@@ -114,43 +124,42 @@ An agent without a CV has no CV list and no judge. An agent without prompts has
|
||||
|
||||
## 3 Fetch text
|
||||
|
||||
Some job boards have one API for the listings and another for the job specs. `fetch text` reads the role's own page for every role in the cut that came without text. Once the full text is fetched, the role is embedded again and its levels recomputed. A role that enters the cut after this recompute gets its text on the next pull.
|
||||
Some job boards have one API for the listings and another for the job specs. `fetch text` reads the role's own page for every role in the cut that came without text, on hosts that have a role page, at most 200 a board and 1,000 a run, a failed read retried after 7 days; Jobindex is never read in a pull, since each read counts as a click there. Once the full text is fetched, the role is embedded again, its levels recomputed, and extract reads it again if it had read the title alone. A role that enters the cut after this recompute gets its text on the next pull.
|
||||
|
||||
## 4 Model analytics
|
||||
|
||||
All roles in the cut go through the detailed analysis and metadata extraction. Extract and judge are two calls: `extract` needs no CV, it is faster, and it is the final step of the ingestion, so it decides what finally passes the `analytics filter`.
|
||||
All roles in the cut go through the detailed analysis and metadata extraction. Extract and judge are two calls: `extract` needs no CV, it is faster, and it is the final step of the ingestion, so it decides what finally passes the `analytics filter`. All three calls go to Haiku (`claude-haiku-4-5`), extract 20 roles a call, judge 5, summarise 10. A stage that is switched off, or has no model to ask, is skipped, and the Runs page says so. A cut role without text is neither judged nor summarised.
|
||||
|
||||
### 4.1 Extract
|
||||
|
||||
Often information is only in the free text, e.g. that a role in a specific location is actually remote. So every role has raw and processed fields for e.g. salary, location, remote and seniority. Extract reads what is hidden in the free text. A field the board states wins over what extract reads, except seniority and remote, where extract wins.
|
||||
|
||||
**Extract** reads the cut first. It fills location, remote area, seniority, languages and deadline.
|
||||
**Extract** reads the cut first, the first 2,500 and last 1,500 characters of each role. It fills location, remote and remote area, seniority, languages, deadline, pay and currency, function and work permit. A role the model leaves unanswered three times is given up.
|
||||
|
||||
The fetch gate and the cut are then applied again with what extract filled. A role can drop out, such as one that turns out to be outside the agent's locations, or enter, such as one that turns out to be remote. A role that enters is extracted on the next pull.
|
||||
The fetch gate and the cut are then applied again with what extract filled. A role can drop out, such as one that turns out to be outside the agent's locations, or enter, such as one that turns out to be remote. A role that enters is judged in this pull and extracted on the next.
|
||||
|
||||
### 4.2 Judge
|
||||
|
||||
Once extract has completed the ingestion, `judge` reads each role in the cut against the agent's CV, languages, likes and dislikes. It outputs:
|
||||
Once extract has completed the ingestion, `judge` reads each role in the cut, up to 8,000 characters of it, against the agent's CV, languages, prompts, likes and dislikes. It outputs:
|
||||
|
||||
- the `CV match`
|
||||
- one to three strengths and gaps
|
||||
- a verdict on each must-have
|
||||
|
||||
Each role is extracted once. A judgement is redone only when its input changes: a change to the CV, a prompt, or the like or dislike text re-judges the whole cut (about an hour for 650 roles); a changed posting re-judges that role only.
|
||||
Each role is judged once. A judgement is redone only when its input changes: a change to the CV, the languages, a prompt, or the like or dislike text re-judges the whole cut (about an hour for 650 roles); a changed posting re-judges that role only.
|
||||
|
||||
#### 4.2.1 CV match
|
||||
|
||||
Judge rates how closely the job spec matches the experience in the CV on three dimensions:
|
||||
|
||||
- skills,
|
||||
- experience,
|
||||
- career,
|
||||
- skills, where a language the posting requires above the level spoken counts as a missing skill,
|
||||
- experience,
|
||||
- career, rated against the prompts, likes and dislikes rather than the CV,
|
||||
|
||||
each from 0 to 10. A model's rating is an estimate, not a measurement, so a coarse scale serves better than 0 to 100. The **CV match** is the three numbers weighted as given in the agent settings (30/25/30 to start). A must-have the role breaks caps career at 4. Roles with the same CV match are ordered by CV closeness.
|
||||
each from 0 to 10. A model's rating is an estimate, not a measurement, so a coarse scale serves better than 0 to 100. The **CV match** is the three numbers weighted as given in the agent settings (30/25/30 to start), times 10, so 0 to 100. Roles with the same CV match are ordered by CV closeness.
|
||||
|
||||
## 5 Feed
|
||||
|
||||
The feed shows every role that passes the `world fetch gate`, the `location fetch gate` and the `role exclusion gate` (1.1.1 to 1.1.3), loaded as you scroll, in every order, newest first included. No order shows a role the fetch gate rejects. A hidden company's roles are never shown.
|
||||
The feed shows every open role, kept copies only, that passes the `world fetch gate`, the `location fetch gate` and the `role exclusion gate` (1.1.1 to 1.1.3), loaded as you scroll, 60 at a time, in every order, newest first included. No order shows a role the fetch gate rejects. A hidden company's roles are never shown. Opening a role or voting on it marks it read; Show read, on by default, hides the read ones when switched off. Saving a role (★) also likes it.
|
||||
|
||||
### 5.1 Feed scores
|
||||
|
||||
@@ -161,6 +170,7 @@ The `want feed score` consists of `prompt closeness`, newness, company likes and
|
||||
- `newness` counts from the posting date, the earlier of when the app first saw the role and the date the board states (only for boards whose dates are reliable), and halves every 14 days.
|
||||
- `company like` is 1/3 for a neutral company, 2/3 for a liked one and 1 for a favourite.
|
||||
- `job like` is described in 5.1.3.
|
||||
- A role without an embedding yet is scored without `prompt closeness` and `job like`; their weights leave the sum.
|
||||
|
||||
#### 5.1.2 Match feed score, "What fits me"
|
||||
|
||||
@@ -185,16 +195,17 @@ The user likes or passes on roles in the feed, per agent. Each role has an embed
|
||||
|
||||
- The Roles page says that the feed is already filtered by the agent's fetch gate, location included.
|
||||
- The company view says what is filtered: which parts of the fetch gate apply to this company, for example "every role in Europe, not filtered by country" for an All locations company. Looking up a company never silently hides roles elsewhere.
|
||||
- Show gated lists the roles the fetch gate rejects, each with the reason, so what is kept out can be checked. It is the one way a rejected role is shown.
|
||||
|
||||
## 6 After the ranking
|
||||
|
||||
Summarise is not part of any score. It runs after the feed scores, on the top N of the `want feed score` and the top N of the `match feed score` (N set per agent, 30 to start, so at most 60 roles), and on every role liked or saved.
|
||||
Summarise is not part of any score. It runs after the feed scores, on the top N of the `want feed score` and the top N of the `match feed score` (N set per agent, 30 to start, so at most 60 roles), and on every role liked or saved. It writes about 60 words and 3 to 8 tags, once per role, and needs the role's text.
|
||||
|
||||
## Settings
|
||||
|
||||
Per agent, on the form:
|
||||
|
||||
- name and prompts
|
||||
- name, an On/Off switch, and prompts
|
||||
- the fetch gate: world (the outer countries), location (countries, towns, remote), role exclusion (titles to leave out), role fetch (fetch terms)
|
||||
- CV, languages, likes and dislikes
|
||||
- the analytics filter: count and minimum by prompt closeness, count and minimum by CV closeness
|
||||
@@ -202,3 +213,5 @@ Per agent, on the form:
|
||||
- summarise size
|
||||
|
||||
Per agent, per company: affinity (hidden, neutral, liked, favourite). Per company, for every agent: All locations and All roles.
|
||||
|
||||
The pull runs every 6 hours.
|
||||
@@ -5,104 +5,112 @@ What a plugin has to deliver so that the flow in `ADR Fetch prep and score` gets
|
||||
## Terms
|
||||
|
||||
- **Source**: a site that lists roles.
|
||||
- **Plugin** (reader): the code for one kind of source. There are two kinds:
|
||||
- **Plugin**: one module for one kind of source. There are two kinds:
|
||||
- a **host**, which reads a company's own list on an ATS (Workday, Greenhouse, ...). That list is a **board**, attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
|
||||
- a **provider**, which runs a **search**, a saved query on a site that lists roles across companies (Jobindex, Jobnet, ...).
|
||||
- **Runner**: the shared code that calls a plugin. It asks, pages, retries, guards and stamps, so no plugin repeats that.
|
||||
- **Runner**: the shared code that calls a plugin. It asks, pages, retries, guards, validates and stamps, so no plugin repeats that. It has one driver per kind, for boards and for searches, which differ only in what they store.
|
||||
- **Row**: one role as a plugin hands it over, in the host's own words.
|
||||
- **Declaration**: what a plugin states it can do (`READS`). The runner acts on the declaration, never on guesses.
|
||||
- **Read**: one pull's requests to one target, and what they yielded. A read is **complete** when it listed every role the target has, else **incomplete**; the runner stamps which.
|
||||
|
||||
## 1 What a plugin delivers
|
||||
|
||||
A plugin turns one source into rows. It:
|
||||
|
||||
- hands over every role the source lists for what it was asked, as one row per role the host lists;
|
||||
- hands over every role the source lists for what it was asked, one row per listing, as the host lists them;
|
||||
- asks the host only for the filters it was handed, by building them into the host's query;
|
||||
- keeps the host's own words for places, remote, seniority, employment and country;
|
||||
- declares truthfully what the source can do: how the list arrives, where the text is, how completeness is known, what it can filter and which fields it states.
|
||||
- declares truthfully what the source can do: how the list arrives, where the text is, how completeness is known, what it can filter and which fields it states;
|
||||
- raises on anything it cannot read: an error page, an answer of another shape, a page that is not the page it asked for. An empty list is a valid answer only when the host says the list is empty.
|
||||
|
||||
A plugin never:
|
||||
|
||||
- filters, drops or reorders rows itself;
|
||||
- filters, drops or reorders rows; a listing it cannot make a row of (no id, no title or no link) it hands over as far as it can, and the runner drops and counts it (3.8);
|
||||
- maps a word to the app's vocabulary;
|
||||
- merges, splits or multiplies rows;
|
||||
- decides closing, retries or failure;
|
||||
- reads robots.txt at run time.
|
||||
- reads robots.txt at run time;
|
||||
- logs in, runs a page's script, or uses a key taken from a site's script.
|
||||
|
||||
## 2 The five cases
|
||||
|
||||
A source can present roles in five ways that need handling. Three are the plugin's job and two are handled upstream.
|
||||
A source can present roles in five ways that need handling. Three are the plugin's job and two are the app's, after the plugin.
|
||||
|
||||
| # | Case | Who handles it |
|
||||
|---|---|---|
|
||||
| 1 | The list arrives in pages | Plugin, with the runner |
|
||||
| 2 | The list has no text; each role's text is on its own page | Plugin, with the fetch text stage |
|
||||
| 3 | One role states several places | Plugin |
|
||||
| 4 | One job is listed several times | Upstream |
|
||||
| 5 | Facts such as remote are only in the free text | Upstream |
|
||||
| 4 | One job is listed several times | The app |
|
||||
| 5 | Facts such as remote are only in the free text | The app |
|
||||
|
||||
### 2.1 Pagination (plugin, with the runner)
|
||||
|
||||
The plugin says how to ask for one page and what the page holds. The runner does the rest.
|
||||
|
||||
- The plugin declares `listing="paged"`, the page size and the host's **ceiling**, the offset past which the host answers nothing whatever is asked (Workday 2,000).
|
||||
- `page` asks for one page from a cursor and returns the page's rows, the raw results with their ids, the total the host states if any, and the next cursor, or none after the last page.
|
||||
- The runner reads the pages one after another, 0.5 s apart, at most 100 pages a pull. A board above 100 pages is **swept**: each pull continues from the saved cursor.
|
||||
- **Completeness**: a read is complete when the distinct ids read equal the stated total (`complete="stated"`), or when it reaches an empty page (`complete="end"`). A read that reaches the ceiling is incomplete, is not swept and closes nothing; only a filter brings such a board under the ceiling.
|
||||
- `page` asks for one page from a cursor and returns the page's rows, the raw results with their ids, the total the host states if any, and the next cursor, or none after the last page. The first call gets cursor none. A cursor is any JSON value; a tuple comes back as a list when the runner hands it over again (a sweep), and a plugin reads it by position.
|
||||
- The runner reads the pages one after another, 0.5 s apart, at most 100 pages a pull. A board above 100 pages is **swept**: each pull continues from the saved cursor, series and ids, and the sweep is finished when the read reaches the end.
|
||||
- **Completeness**: a read is complete when the distinct ids read equal the stated total (`complete="stated"`), or when the plugin hands over the last page with no cursor (`complete="end"`: an empty page, or a short one where the host has no empty page). A read that reaches the ceiling is incomplete, is not swept and closes nothing; only a filter brings such a board under the ceiling.
|
||||
- **Stalls**: a page that brings no new id while the plugin hands over a cursor is a stall when the host states no total, or when the page came empty: the host ignores the cursor, or sent a page again. A plugin that sees a page come again (the same first id as the page before) hands it over with no results and a cursor that is not none, never none, since none would read as the last page. A stalled read is incomplete. With a stated total, a page of ids already read is not a stall, since a resumed sweep starts on one when roles were added at the top since.
|
||||
- A role of a swept board closes only when two finished sweeps in a row missed it, since roles closing between pulls shift later roles back past the cursor.
|
||||
- A 429 after the first page ends the read. What was read is kept, and the read is incomplete.
|
||||
- A 429 after the first page ends the read. What was read is kept, the read is incomplete, and a board continues from there in the next pull as a sweep.
|
||||
- A plugin whose list comes in one answer declares `listing="whole"` and `complete="whole"`; a whole answer that parses is complete. One whose list comes in one answer that states a total it does not fill raises.
|
||||
|
||||
### 2.2 List without text (plugin, with the fetch text stage)
|
||||
|
||||
Some hosts list title, place and date only, and state the text on each role's own page.
|
||||
|
||||
- The plugin declares `text="detail"` and supplies `detail`, which reads one role's page and returns the fields it found. A list that carries a short excerpt hands it over as `teaser` and declares `teaser=True`.
|
||||
- The plugin does not decide which roles are read. The fetch text stage calls `detail` after scoring, only for roles in the cut (`ADR Fetch prep and score`, section 3), at most 200 per board and 1,000 per run.
|
||||
- The plugin does not decide which roles are read. The fetch text stage calls `detail` after scoring, only for roles in the cut (`ADR Fetch prep and score`, section 3), at most 200 per board and 1,000 per run, 0.5 s apart.
|
||||
- `detail` may fill only description, location, places, teaser, remote, country, employment, pay figures, currency and the posting date. Title, link, department and the pay line come from the list, which states them on every pull.
|
||||
- A role read in detail is sent by later pulls without the list's place and remote, so what its own page said stays.
|
||||
- A 404 or 410 counts as read. Another failure is retried on a later run. A read that gave no text is retried after 7 days. A 429 ends the reads of that board for the run.
|
||||
- A role read in detail is sent by later pulls without the list's place and remote, so what its own page said stays. This holds for a search's roles as for a board's.
|
||||
- A 404 or 410 counts as read. Another failure is retried on a later run, after the roles never tried. A read that gave no text is retried after 7 days. A 429 ends the reads of that target for the run.
|
||||
- Completeness, and so closing, is decided by the list alone.
|
||||
|
||||
### 2.3 One role, several places (plugin)
|
||||
|
||||
- A row carries `places`: every place the host states for the role, in the host's order and the host's words, each marked remote, office or unsaid.
|
||||
- A row carries `places`: every place the host states for the role, in the host's order and the host's words, each marked remote, office or unsaid as the host marks that place. The host's own word for how the role is worked ("Hybrid", "Remote", "On-site") goes on the row as `remote`, unmapped.
|
||||
- A plugin never makes one row per place, never drops a place and never joins places into one text.
|
||||
- A host that states no place declares `places="none"` and hands over an empty list.
|
||||
- Resolving a place text to a town, region or country is the app's job.
|
||||
|
||||
### 2.4 One job listed several times (upstream)
|
||||
### 2.4 One job listed several times (the app)
|
||||
|
||||
One job can come as several rows: one per office, each with its own id; several requisition ids for the same job; or a search provider and the company's board listing the same role.
|
||||
|
||||
- The plugin hands every listing over as its own row, with its own id and link. A host that publishes one posting per place declares `places="copies"`.
|
||||
- Folding them into one role, and choosing the copy that is kept, happens upstream (`ADR Fetch prep and score`, section 1.4).
|
||||
- Folding them into one role, and choosing the copy that is kept, happens in the app (`ADR Fetch prep and score`, section 1.4).
|
||||
- Two listings under one id are one row; a host that lists one id once per place hands over one row with all the places.
|
||||
|
||||
### 2.5 Facts in the free text (upstream)
|
||||
### 2.5 Facts in the free text (the app)
|
||||
|
||||
- The plugin passes on only what the host states in a field: remote, country, seniority, employment, pay and date, in the host's words, and declares each as `stated` or `none`.
|
||||
- Mapping the host's words to the app's vocabulary happens upstream, through a table per host.
|
||||
- Facts found only in the prose, e.g. a Copenhagen role that is actually remote, are read upstream: from the place words first, then by extract (`ADR Fetch prep and score`, section 4.1).
|
||||
- The plugin passes on only what the host states in a field: remote, country, seniority, employment, pay and date, in the host's words, and declares each as `stated` or `none`. A provider that covers one country states that country on every row and declares it `stated`.
|
||||
- Mapping the host's words to the app's vocabulary happens in the app, through a table per host.
|
||||
- Facts found only in the prose, e.g. a Copenhagen role that is actually remote, are read in the app: from the place words first, then by extract (`ADR Fetch prep and score`, section 4.1).
|
||||
|
||||
## 3 Further rules
|
||||
|
||||
### 3.1 Filtering at the host
|
||||
|
||||
- The plugin declares per dimension (country, place, remote, query) whether the host filters it: `none`, `one` (one value a request), `many` (a list in one request) or `ids` (the host's own ids, looked up with `facets`).
|
||||
- The runner hands the plugin an `Ask` that holds only declared dimensions. With `one`, it runs one series of requests per value and unions the rows by id. A read costs at most 8 series. Above that the runner narrows less: a read for the location gate keeps the places and drops the fetch terms; a read of an All locations company keeps the terms and drops the place part; more than 8 terms alone are dropped.
|
||||
- "In these countries, or remote" cannot be asked in one request, because two facets intersect. The runner makes remote a series of its own, so a plugin never gets remote and a country in the same `Ask`.
|
||||
- The runner hands the plugin an `Ask` that holds only declared dimensions. With `one`, it runs one series of requests per value and unions the rows by id. A read costs at most 8 series. Above that the runner narrows less: more than 8 countries on a `one` host means a read without countries; when terms times the location series pass 8, a read for the location gate keeps the places and drops the terms, and a read of an All locations company keeps the terms and drops the place part; more than 8 terms alone are dropped.
|
||||
- "In these countries, or remote" cannot be asked in one request, because two facets intersect. The runner makes remote a series of its own, so a plugin never gets remote and a country in the same `Ask`. A host that cannot filter on remote is asked for the countries alone, and the read is stamped as not having asked for remote, which the app shows and spares the remote roles it holds from closing.
|
||||
- A host with `query="one"` also takes the fetch terms, one term a series, handed over as `query`.
|
||||
- A host's facet ids are looked up at most once a day per board.
|
||||
- Which boards are filtered at all, and by what, is decided upstream (`ADR Fetch prep and score`, section 1.2).
|
||||
- A host's facet ids are looked up at most once a day per board. A value the host's facets do not know is dropped; a read for which nothing remains asks nothing, lists nothing and closes nothing.
|
||||
- A board's own saved params win over the agents' reach on the dimensions they set.
|
||||
- Which boards are filtered at all, and by what, is decided in the app (`ADR Fetch prep and score`, section 1.2).
|
||||
|
||||
### 3.2 Completeness and closing
|
||||
|
||||
The plugin declares how completeness is known (`stated`, `end`, `whole` for a one-answer list, `never`). The runner stamps each read as complete or not. Closing follows `ADR Fetch prep and score`, section 1.3. A search is never a company's whole list, so it declares `never` or `stated`.
|
||||
The plugin declares how completeness is known (`stated`, `end`, `whole` for a one-answer list, `never`). The runner stamps each read as complete or not, as filtered (a location filter), as narrowed (fetch terms or a provider's query), and with the ids it listed. Closing follows `ADR Fetch prep and score`, section 1.3: a search never closes a role by its list, whatever it declares, so a provider's `stated` only says how far a read got.
|
||||
|
||||
### 3.3 Failures
|
||||
|
||||
- A plugin lets every error through: HTTP errors, timeouts, parse errors. The runner logs, records and decides.
|
||||
- A read that lists under 10% of what the same read listed last time, once that was at least 10 roles, counts as failed. This catches a host that answers an empty list for a mistyped board; such a host declares `empty="suspect"`.
|
||||
- A plugin lets every error through: HTTP errors, timeouts, parse errors, an answer of another shape. The runner logs, records and decides. An error on a later page fails the whole read; what a 429 read is kept (2.1).
|
||||
- A read that lists under 10% of what the same read listed last time counts as failed. For a whole read of a board this holds once the last count was at least 10 roles; a filtered or narrowed read, and a read of a host that answers an empty list for an unknown slug (`empty="suspect"`), is held from a count of 1. "The same read" is one with the same filters: a changed filter starts the count again.
|
||||
- A board that fails on every read for 21 days closes its roles, and a suspect answer that has stood that long is accepted.
|
||||
- A failing target is named on its company's row and in the pull's notice, with the reason.
|
||||
|
||||
### 3.4 Fetching
|
||||
|
||||
@@ -114,13 +122,15 @@ Every request goes through the runner's `get`:
|
||||
- 10 s to connect, 30 s between bytes, 60 s for the whole request;
|
||||
- at most 96 MiB a response. A plugin may name a smaller fallback feed, read once when the main one is over the cap.
|
||||
|
||||
The figures in this document are the defaults as they stand; the rules are what counts.
|
||||
|
||||
### 3.5 Searches
|
||||
|
||||
A provider has saved params instead of a board. It declares which filter dimension each param fills, cleans a param it would not accept with a reason a person can read, and receives the saved params in `Ask.params`. A search row carries `company`, the employer's name. A provider has no slug patterns, probes or careers page, and takes no fetch terms: its query replaces them. A provider that needs an API key declares it, and the runner hands it the key.
|
||||
A provider has saved params instead of a board. It declares each param: the filter dimension it fills, its label, its kind (text, a choice from a table, a list of country codes) and the table of choices where there is one, so the form, the command line and the API offer a provider's params from the declaration alone. It cleans a param it would not accept with a reason a person can read, and receives the saved params in `Ask.params`. A search row carries `company`, the employer's name. A provider has no slug patterns, probes or careers page, and takes no fetch terms: its query replaces them. A provider that needs an API key declares it, and the runner hands it the key; a search reads 5 pages by default and 10 at most.
|
||||
|
||||
### 3.6 Discovery
|
||||
|
||||
A host can be found from a careers page link: its slug patterns are regexes over a URL, with the slug in group 1, or several groups joined by `/`. A host may offer a cheap probe URL that answers 2xx only for a real board, and a check of the probe's body where 2xx is not enough. The slug is lowercased unless the host's paths are case-sensitive.
|
||||
A host can be found from a careers page link: its slug patterns are regexes over a URL, with the slug in group 1, or several groups joined by `/`. A host may offer a cheap probe URL that answers 2xx only for a real board, and a check of the probe's body where 2xx is not enough. The slug is lowercased unless the host's paths are case-sensitive. A host whose boards live on a company's own domain names the page markers that identify it, and the path discovery confirms with.
|
||||
|
||||
### 3.7 robots.txt
|
||||
|
||||
@@ -128,18 +138,22 @@ Whoever builds a plugin reads the robots.txt of every host it requests, for the
|
||||
|
||||
### 3.8 Rows
|
||||
|
||||
- `source_id` is the host's id, unique on the board or search. `title` and `url` (http or https) are required.
|
||||
- `source_id` is the host's id, unique on the board or search. `title` and `url` (http or https) are required. The runner, not the plugin, drops a row that lacks one of them; the read's record counts the rows dropped, and a board whose dropped rows pass 10% of its list counts as failed, since a changed page shape looks like that.
|
||||
- `description` and `teaser` are plain text, never HTML.
|
||||
- `posted` is an aware datetime or none. `posted="trusted"` declares it the posting date; `listed` declares a date the host moves, e.g. on an edit.
|
||||
- `posted` is an aware datetime or none. `posted="trusted"` declares it the posting date; `listed` declares a date the host moves, e.g. on an edit. A plugin whose only date comes from the role's page or from a field the host moves declares `listed`.
|
||||
- Pay figures are yearly integers with an ISO 4217 currency. A monthly figure is multiplied by 12 where the host states the period. An hourly, daily or weekly figure is kept as a pay line only.
|
||||
- `country` is ISO 3166 alpha-2, and only where the host states it.
|
||||
- `country` is ISO 3166 alpha-2, and only where the host states it (or the provider covers that one country, 2.5).
|
||||
- Any other key the host states is kept as raw, so a new field needs no change in the runner.
|
||||
|
||||
### 3.9 What the runner keeps per target
|
||||
|
||||
So that a plugin keeps nothing: the last read's marker, whether it was complete, filtered, narrowed and asked for remote, the ids it listed, the sweep in progress, the roles read and failed in detail, the facet ids and when they were read, the board's size and when it was measured, the last count the guard compares with, and the failure record with its first date and reason. All of it is shown or summarised on the board's row.
|
||||
|
||||
---
|
||||
|
||||
## API
|
||||
|
||||
A plugin is one module, named after its source, importing only the shared plugin helpers, the standard library, `httpx` and `pydantic`. It registers by existing: a module that breaks the contract fails at import, naming every problem.
|
||||
A plugin is one module, named after its source, importing only the shared plugin helpers, the standard library, `httpx`, `pydantic` and the shared model base. It registers by existing: a module that breaks the contract is refused at import, every problem named; the other plugins load, and the Sources page names the refused one.
|
||||
|
||||
### Module attributes
|
||||
|
||||
@@ -149,7 +163,11 @@ LABEL = "Workday" # shown in the source select
|
||||
PATTERNS = (r"...",) # host only: discovery regexes over a URL, IGNORECASE
|
||||
RANK = 100 # optional: order in the select and in probes, lower first
|
||||
KEEP_CASE = False # optional: True keeps a discovered slug's case
|
||||
PARAMS = {"q": "query"} # provider only: saved param -> filter dimension it fills
|
||||
MARKERS = ("...",) # optional, host on a company's own domain: page markers
|
||||
PARAMS = { # provider only: saved param -> what it is
|
||||
"q": Param(fills="query", label="Query", kind="text"),
|
||||
"region": Param(fills="place", label="Region", kind="choice", choices=REGIONS),
|
||||
}
|
||||
|
||||
READS = Reads(
|
||||
scope="board", # "board" | "search"
|
||||
@@ -167,7 +185,7 @@ READS = Reads(
|
||||
# pay "figures" | "line" | "none"; posted "trusted" | "listed" | "none";
|
||||
# the rest "stated" | "none"
|
||||
key=None, # Key(name, fields, env) when an API key is needed
|
||||
empty=None, # "suspect" when an unknown slug answers an empty list
|
||||
empty="trusted", # "suspect" when an unknown slug answers an empty list
|
||||
robots="read: ... (checked 2026-09-30)", # required, with the date
|
||||
)
|
||||
```
|
||||
@@ -175,10 +193,12 @@ READS = Reads(
|
||||
### Functions
|
||||
|
||||
```python
|
||||
# Every plugin
|
||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a request may reach
|
||||
|
||||
# Every host
|
||||
def slug_ok(slug: str) -> bool # full match; refuses anything that adds a path,
|
||||
# query, fragment, port or user to a URL
|
||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a request may reach
|
||||
def careers_url(slug: str) -> str # the page a person opens
|
||||
|
||||
# listing="whole"
|
||||
@@ -213,18 +233,20 @@ def probe_slugs(guess: str) -> tuple[str, ...] # slugs a guessed company name
|
||||
```python
|
||||
get(url, *, method="GET", payload=None, headers=None) -> bytes
|
||||
# POST sends payload as JSON. Raises httpx.HTTPStatusError on non-2xx,
|
||||
# httpx.HTTPError on timeout or connection failure, ValueError on a refused
|
||||
# host, an oversized body or a broken redirect. A plugin lets these through.
|
||||
# httpx.HTTPError on timeout or connection failure, and the runner's own
|
||||
# errors on a refused host, an oversized body or a broken redirect. A
|
||||
# plugin lets these through.
|
||||
|
||||
Ask(countries=(), places=(), remote=None, remote_ids=(), query=None, terms=(), params={})
|
||||
Ask(countries=(), places=(), remote=None, remote_ids=(), query="", terms=(), params={})
|
||||
# only declared dimensions are set; with "one" a series carries one value;
|
||||
# with "ids" countries and places hold host ids; never remote together with
|
||||
# a country or place; terms are handed over one a series as query, so a
|
||||
# plugin reads query only; params holds a search's saved params and a key under "key"
|
||||
|
||||
Page(decoded, results, total, cursor)
|
||||
Page = (decoded, results, total, cursor)
|
||||
# decoded: handed to rows; results: raw dicts with an "id", counted by
|
||||
# distinct id; total: stated by the host, or None; cursor: next page, or None
|
||||
# distinct id; total: stated by the host, or None; cursor: next page, or
|
||||
# None after the last; any JSON value, a tuple returning as a list
|
||||
|
||||
Place = {"text": str, "remote": bool | None}
|
||||
# the host's words; True remote, False office, None unsaid
|
||||
@@ -237,20 +259,24 @@ Row = {
|
||||
"country": str, "seniority": str, "employment": str,
|
||||
"company": str, # search rows only
|
||||
}
|
||||
|
||||
Param(fills, label, kind, choices=None)
|
||||
# fills: "query" | "country" | "place" | "remote" | "pages" | "" (the host's own);
|
||||
# kind: "text" | "choice" | "countries" | "number"; choices: {value: label}
|
||||
```
|
||||
|
||||
### Helpers the plugins share
|
||||
|
||||
- `place(text, remote=None)` makes a `Place`, remote when the text says remote; `add_place(found, entry)` appends one unless it is empty or already there.
|
||||
- `html_text(html)` turns HTML into plain text; `parse_dt(value)` makes an aware datetime; `yearly_amount(...)` makes a yearly figure; `is_web_url(url)` checks a link.
|
||||
- `html_text(html)` turns HTML into plain text, one line per paragraph, with list items and headings marked; `parse_dt(value)` makes an aware datetime; `yearly_amount(...)` makes a yearly figure and `pay_line(...)` a pay line; `is_web_url(url)` checks a link.
|
||||
- `path_slug_ok(slug)` and `label_slug_ok(slug)` check a slug that is one URL path segment or one DNS label; `only(host)` is the usual `allowed`.
|
||||
- A table of the country names hosts write.
|
||||
- A table of the country names hosts write, both ways (`code_of`, `name_of`).
|
||||
|
||||
### Adding a plugin
|
||||
|
||||
1. Write the module with its declaration and functions.
|
||||
2. Add a table of the host's words for seniority, employment and remote, when the host states them in its own words.
|
||||
3. Add fixtures from real answers, and tests. A contract test checks each declared field against the fixtures: a field declared `stated` must show on some row, a field declared `none` on none.
|
||||
3. Add fixtures from real answers, and tests. A contract test, shared by every plugin and naming none, checks each declared field against the fixtures: a field declared `stated` must show on some row, a field declared `none` on none; required row keys, plain text, aware dates, ISO codes and integer pay; that results carry ids and cursors survive JSON; that a declared filter reaches the request.
|
||||
4. Record the robots.txt decision in the declaration.
|
||||
|
||||
Nothing else changes: the list of plugins, the source select, discovery and the filters a board is read with all follow from the module.
|
||||
Nothing else changes: the list of plugins, the source select, discovery, a provider's form, the filters a board is read with and what its row says all follow from the module.
|
||||
+14
-6
@@ -17,8 +17,9 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
||||
its slug is. Every board is read independently; one that fails is logged,
|
||||
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
||||
whose read lists under a tenth of what it listed last time (`_runner.read`'s
|
||||
guard), until it has failed that way for the pull's `stale_days`. A board's
|
||||
whose read lists under a tenth of what it listed last time, or drops more
|
||||
than a tenth of the rows its reader handed over (`_runner.read`'s guards),
|
||||
until it has failed that way for the pull's `stale_days`. A board's
|
||||
filter is the `reach` option the pull hands it (the agents', see
|
||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||
@@ -343,8 +344,10 @@ SIZE_PAGES = 100
|
||||
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
||||
"""How many roles board `target` lists, unfiltered and without terms: a
|
||||
whole host's listing, the total a `complete="stated"` host gives on page
|
||||
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
|
||||
as `_fetch_rows` does, so a board that does not answer is refused."""
|
||||
1, else the listing read to the end, at most `SIZE_PAGES` pages. A read
|
||||
counts the rows the runner dropped as well (`_listed`), as a stated total
|
||||
does. Raises as `_fetch_rows` does, so a board that does not answer is
|
||||
refused."""
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
if client is None:
|
||||
@@ -358,7 +361,12 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
||||
if isinstance(total, int):
|
||||
return total
|
||||
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
|
||||
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
||||
return _listed(_runner.read_listing(module, slug, get, limits=limits))
|
||||
|
||||
|
||||
def _listed(listing: _runner.Listing) -> int:
|
||||
"""The listings a read found: the ids it kept and the rows it dropped."""
|
||||
return len(listing.ids) + listing.dropped
|
||||
|
||||
|
||||
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
||||
@@ -462,7 +470,7 @@ class ATSSource(Source):
|
||||
[compound(i) for i in listing.ids])
|
||||
self.state.pop(failing, None)
|
||||
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
||||
keep_size(self.state, target, len(listing.ids), reset=False)
|
||||
keep_size(self.state, target, _listed(listing), reset=False)
|
||||
if detail_sets is not None:
|
||||
# A complete listing drops the ids it no longer lists.
|
||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||
|
||||
@@ -179,7 +179,7 @@ ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
|
||||
|
||||
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
|
||||
own id for the role, unique on the board), `title`, `url` (http or https,
|
||||
checked with `_shared.is_web_url`) and `places`; `location` (the host's
|
||||
which the runner checks with `_shared.is_web_url`) and `places`; `location` (the host's
|
||||
one-line place, kept while storage reads it), `posted` (an aware datetime
|
||||
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
|
||||
see `_shared.html_text`), `remote` (True, False, a word or None) and
|
||||
@@ -187,7 +187,12 @@ see `_shared.html_text`), `remote` (True, False, a word or None) and
|
||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
||||
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
||||
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
|
||||
every stated key in `raw`, so a new field needs no change there.
|
||||
every stated key in `raw`, so a new field needs no change there. A reader
|
||||
never drops a listing: one it cannot make a whole row of (no id, no title or
|
||||
no link) it hands over with what it has, an empty string for what is
|
||||
missing, and the runner drops it and counts it (`_runner.valid`). A reader
|
||||
leaves out only what is not a listing of the board: an entry that is not an
|
||||
object, a second copy of an id, a role of another board.
|
||||
|
||||
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
||||
remote=...)]`, one entry per place the host states, in the host's order, in
|
||||
|
||||
@@ -205,6 +205,8 @@ class Row(TypedDict, total=False):
|
||||
`company` is the employer's name on a search row, which lists many
|
||||
(`plugins/search.py` stores it); a board row carries none, the board is
|
||||
the company. Any other stated key is kept in `raw` by the runner.
|
||||
`source_id`, `title` and `url` come empty from a listing that lacks
|
||||
them: the runner drops such a row and counts it (`_runner.valid`).
|
||||
"""
|
||||
|
||||
source_id: Required[str]
|
||||
|
||||
+98
-20
@@ -4,10 +4,12 @@ the reader declares in `READS`.
|
||||
A reader says how its host answers; this module reads it: the guarded fetch
|
||||
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
||||
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
||||
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
||||
the sweep of an oversized paged board across pulls, the 10% guard on every
|
||||
read that is not a sweep (`read`), the per-role detail reads
|
||||
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||
series per `one`-filter value unioned by id (`read_listing`), the rows it
|
||||
keeps (`valid`: a row without an id, a title or a link is dropped and
|
||||
counted here, never by the reader), completeness, the sweep of an oversized
|
||||
paged board across pulls, the two 10% guards (`read`: on what a read that
|
||||
is not a sweep listed, and on what any read dropped), the per-role detail
|
||||
reads (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
||||
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
||||
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
||||
@@ -30,6 +32,8 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
||||
drifts when roles close between pulls, so `app.jobs.pull` closes
|
||||
a role of such a board only when two finished sweeps in a row
|
||||
missed it (it keeps the ids the last one missed in `missing`)
|
||||
dropped the rows the last stamped read dropped (`Listing.dropped`), as a
|
||||
number, "" for none; `app.jobs.limits` names it on the board's row
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -52,7 +56,7 @@ import httpx
|
||||
from desk.sources import USER_AGENT
|
||||
|
||||
from ._contract import Ask, Key, Place, Reads
|
||||
from ._shared import Get, Row
|
||||
from ._shared import Get, Row, is_web_url
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -96,7 +100,8 @@ SUSPECT_SHARE = 0.1
|
||||
#: The last count from which the guard holds every read. A read that is
|
||||
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
||||
#: a count of 1, as it was before every read was; a whole read of a board
|
||||
#: that listed a handful of roles can drop to none by itself.
|
||||
#: that listed a handful of roles can drop to none by itself. It is also the
|
||||
#: rows a read must hand over before the share it dropped can fail it.
|
||||
SUSPECT_MIN = 10
|
||||
|
||||
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
||||
@@ -113,8 +118,8 @@ class TooLarge(ValueError):
|
||||
|
||||
|
||||
class Suspect(Exception):
|
||||
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
|
||||
(see `read`)."""
|
||||
"""A read that listed under `SUSPECT_SHARE` of what it listed last time,
|
||||
or dropped more than that share of its rows (see `read`)."""
|
||||
|
||||
|
||||
class MissingKey(ValueError):
|
||||
@@ -147,7 +152,9 @@ class Listing:
|
||||
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||
roles back past the cursor, so `ids` can miss a role that is still open.
|
||||
`remote_unasked` says a filtered read was meant to ask for remote roles
|
||||
as well and could not, so its `ids` say nothing of the remote roles."""
|
||||
as well and could not, so its `ids` say nothing of the remote roles.
|
||||
`handed` counts the rows the reader handed over this pull, `dropped`
|
||||
those of them `valid` refused, which are in neither `rows` nor `ids`."""
|
||||
|
||||
rows: list[Row]
|
||||
complete: bool
|
||||
@@ -158,6 +165,8 @@ class Listing:
|
||||
narrowed: bool = False
|
||||
swept: bool = False
|
||||
remote_unasked: bool = False
|
||||
handed: int = 0
|
||||
dropped: int = 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------- modules
|
||||
@@ -524,8 +533,42 @@ def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
|
||||
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||
|
||||
|
||||
def valid(row: Mapping[str, Any]) -> bool:
|
||||
"""Whether the runner keeps a row: a non-empty `source_id`, a non-empty
|
||||
`title` and a web URL (`is_web_url`). A reader hands over every listing
|
||||
as far as it can make it out; a row without one of the three cannot be
|
||||
stored or opened, and the runner drops it and counts it."""
|
||||
try:
|
||||
link = is_web_url(str(row.get("url") or ""))
|
||||
except (httpx.InvalidURL, ValueError): # a string httpx cannot read as a URL
|
||||
link = False
|
||||
return (link and bool(str(row.get("source_id") or "").strip())
|
||||
and bool(str(row.get("title") or "").strip()))
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _Tally:
|
||||
"""The rows a read was handed and dropped, across its series and pages."""
|
||||
|
||||
handed: int = 0
|
||||
dropped: int = 0
|
||||
|
||||
|
||||
def _keep(found: Iterable[Row], rows: dict[str, Row], tally: _Tally) -> None:
|
||||
"""The rows a reader's `parse` or `rows` returned, into `rows` by
|
||||
`source_id`, the first of an id kept; those `valid` refuses are counted
|
||||
in `tally` and left out. The one place a row is checked, for the whole
|
||||
and the paged listing alike."""
|
||||
for row in found:
|
||||
tally.handed += 1
|
||||
if not valid(row):
|
||||
tally.dropped += 1
|
||||
continue
|
||||
rows.setdefault(row["source_id"], row)
|
||||
|
||||
|
||||
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||
limits: Limits) -> dict[str, Row]:
|
||||
limits: Limits, tally: _Tally) -> dict[str, Row]:
|
||||
"""A whole listing, one request a series, unioned by `source_id`. A body
|
||||
over the cap gets one retry at the reader's `fallback_url`."""
|
||||
rows: dict[str, Row] = {}
|
||||
@@ -541,8 +584,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
||||
module.NAME, slug)
|
||||
body = get(module.fallback_url(slug))
|
||||
for row in module.parse(body, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
_keep(module.parse(body, slug), rows, tally)
|
||||
return rows
|
||||
|
||||
|
||||
@@ -580,7 +622,29 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
only: a search's places are what the search is, as its query is.
|
||||
`remote` says the reach or the params wanted remote roles; when no series
|
||||
asked for them (a reader without a remote filter, or one whose facets
|
||||
state none beside places) the listing is `remote_unasked`."""
|
||||
state none beside places) the listing is `remote_unasked`.
|
||||
|
||||
Every row `rows` or `parse` returns goes through `valid`: one without a
|
||||
`source_id`, a title or a web URL is dropped here, counted in
|
||||
`Listing.dropped` and logged once for the read. Completeness counts the
|
||||
distinct result ids `page` returns beside its rows, not the rows kept, so
|
||||
a dropped row still counts as read; a whole listing is complete whenever
|
||||
it parses, whatever it dropped."""
|
||||
tally = _Tally()
|
||||
listing = _listing(module, slug, get, ask, limits=limits, sweep=sweep,
|
||||
can_sweep=can_sweep, facets=facets, remote=remote, tally=tally)
|
||||
listing.handed, listing.dropped = tally.handed, tally.dropped
|
||||
if tally.dropped:
|
||||
log.warning("readers: %s:%s: dropped %d of %d listings: no id, title or link",
|
||||
module.NAME, slug, tally.dropped, tally.handed)
|
||||
return listing
|
||||
|
||||
|
||||
def _listing(module: ModuleType, slug: str, get: Get, ask: Ask | None, *,
|
||||
limits: Limits | None, sweep: Mapping[str, Any] | None, can_sweep: bool,
|
||||
facets: Mapping[str, Mapping[str, str]] | None, remote: bool,
|
||||
tally: _Tally) -> Listing:
|
||||
"""`read_listing` before its count of dropped rows, which `tally` keeps."""
|
||||
limits = limits or Limits()
|
||||
ask = ask or Ask()
|
||||
reads: Reads = module.READS
|
||||
@@ -595,7 +659,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
"nothing asked, not closing", module.NAME, slug)
|
||||
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
||||
if not hasattr(module, "page"):
|
||||
whole_rows = _whole(module, slug, get, asks, limits)
|
||||
whole_rows = _whole(module, slug, get, asks, limits, tally)
|
||||
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
||||
key, by_words, remote_unasked=unasked)
|
||||
|
||||
@@ -632,8 +696,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
pages += 1
|
||||
before = len(seen)
|
||||
seen |= _result_ids(results)
|
||||
for row in module.rows(data, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
_keep(module.rows(data, slug), rows, tally)
|
||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||
ended = True
|
||||
break
|
||||
@@ -747,13 +810,26 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
|
||||
that really shrank is accepted once the target has been failing for
|
||||
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
||||
stamp then resets the count; a Recheck resets it at once
|
||||
(`plugins.ats.keep_size`)."""
|
||||
(`plugins.ats.keep_size`).
|
||||
|
||||
A read, a sweep's included, that dropped more than `SUSPECT_SHARE` of
|
||||
the rows its reader handed over (`valid`), once it was handed at least
|
||||
`SUSPECT_MIN`, raises `Suspect` the same way: a page whose shape changed
|
||||
parses to rows without a title or a link, and the roles it hid would
|
||||
close. It gives way after `limits.stale_days` of failing as well."""
|
||||
limits = limits or Limits()
|
||||
ask = ask or Ask()
|
||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
||||
remote=remote)
|
||||
if listing.handed >= SUSPECT_MIN and listing.dropped > SUSPECT_SHARE * listing.handed:
|
||||
why = (f"dropped {listing.dropped} of {listing.handed} listings, over "
|
||||
f"{SUSPECT_SHARE:.0%}: no id, title or link")
|
||||
if not _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
|
||||
raise Suspect(why)
|
||||
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
|
||||
module.NAME, slug, why, limits.stale_days)
|
||||
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
||||
last = _load(state, f"{prefix}:count:{target}")
|
||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||
@@ -788,9 +864,10 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||
against), whether it finished a sweep from earlier pulls (`swept`),
|
||||
whether a filtered read left remote unasked (`remote_unasked`), the
|
||||
sweep to continue or none, and the count for the 10% guard once a read
|
||||
is not a sweep. A read that asked nothing keeps the count it had."""
|
||||
whether a filtered read left remote unasked (`remote_unasked`), the rows
|
||||
it dropped (`dropped`, "" for none), the sweep to continue or none, and
|
||||
the count for the 10% guard once a read is not a sweep. A read that
|
||||
asked nothing keeps the count it had."""
|
||||
state[f"{prefix}:ok:{target}"] = run
|
||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||
@@ -798,6 +875,7 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
||||
state[f"{prefix}:dropped:{target}"] = str(listing.dropped) if listing.dropped else ""
|
||||
if listing.sweep is not None:
|
||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||
return
|
||||
|
||||
@@ -24,7 +24,7 @@ from urllib.parse import urlencode
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Key, Place, Reads
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "adzuna"
|
||||
LABEL = "Adzuna"
|
||||
@@ -111,8 +111,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
for job in decoded["results"]:
|
||||
title = _text(job.get("title") or "")
|
||||
url = (job.get("redirect_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
||||
town = ((job.get("location") or {}).get("display_name") or "").strip()
|
||||
places: list[Place] = [{"text": town, "remote": None}] if town else []
|
||||
|
||||
@@ -14,7 +14,7 @@ from typing import Any
|
||||
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -126,8 +126,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
for j in data.get("jobs") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
||||
compensation = j.get("compensation") or {}
|
||||
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
|
||||
|
||||
@@ -106,8 +106,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
for j in listed:
|
||||
title = (j.get("jobOpeningName") or "").strip()
|
||||
source_id = str(j.get("id") or "")
|
||||
if not title or not source_id:
|
||||
continue
|
||||
places = _places(j)
|
||||
row: Row = {
|
||||
"source_id": source_id,
|
||||
|
||||
@@ -132,8 +132,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
for p in postings:
|
||||
title = (p.get("title") or "").strip()
|
||||
post_id = str(p.get("id") or "")
|
||||
if not title or not post_id:
|
||||
continue
|
||||
job = p.get("job") or {}
|
||||
places = [name for loc in job.get("jobLocations") or []
|
||||
if (name := ((loc.get("location") or {}).get("name") or "").strip())]
|
||||
|
||||
@@ -47,7 +47,7 @@ from urllib.parse import quote
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads, Row
|
||||
from ._countries import name_of
|
||||
from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place
|
||||
from ._shared import Get, Page, add_place, html_text, only, parse_dt, place
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||
@@ -151,8 +151,6 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
source_id = str(p.get("id") or "")
|
||||
path = str(p.get("positionUrl") or "").strip()
|
||||
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
|
||||
if not title or not source_id or not is_web_url(url):
|
||||
continue
|
||||
places = _places(p.get("locations") or p.get("standardizedLocations"))
|
||||
out.append({
|
||||
"source_id": source_id,
|
||||
|
||||
@@ -37,7 +37,7 @@ from urllib.parse import urlencode, urlsplit, urlunsplit
|
||||
from desk.sources import USER_AGENT
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line,
|
||||
from ._shared import (Get, Page, Row, html_text, num, only, parse_dt, pay_line,
|
||||
yearly_amount)
|
||||
|
||||
NAME = "freehire"
|
||||
@@ -170,8 +170,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
title = " ".join(html.unescape(str(hit.get("title") or "")).split())
|
||||
source_id = str(hit.get("public_slug") or "").strip()
|
||||
url = _url(str(hit.get("url") or ""))
|
||||
if not title or not source_id or not is_web_url(url):
|
||||
continue
|
||||
mode = str(hit.get("work_mode") or "").strip()
|
||||
places = _places(hit, mode)
|
||||
countries = [str(c) for c in hit.get("countries") or [] if c]
|
||||
|
||||
@@ -116,8 +116,6 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
for p in postings:
|
||||
title = (p.get("title") or "").strip()
|
||||
ext_id = str(p.get("extId") or "")
|
||||
if not title or not ext_id:
|
||||
continue
|
||||
locations = p.get("locations") or []
|
||||
places: list[Place] = []
|
||||
for loc in locations:
|
||||
|
||||
@@ -13,7 +13,7 @@ from typing import Any
|
||||
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
|
||||
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place
|
||||
|
||||
NAME = "greenhouse"
|
||||
LABEL = "Greenhouse"
|
||||
@@ -149,8 +149,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
for j in data.get("jobs") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("absolute_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
departments = j.get("departments") or []
|
||||
ranges = j.get("pay_input_ranges") or []
|
||||
salary_min, salary_max, currency = _salary(ranges)
|
||||
|
||||
@@ -90,8 +90,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
for ad in data.get("jobAdDetails") or []:
|
||||
title = (ad.get("title") or "").strip()
|
||||
ad_id = str(ad.get("id") or "").strip()
|
||||
if not title or not ad_id:
|
||||
continue
|
||||
site = str(ad.get("site") or "").strip()
|
||||
country = str(ad.get("country") or "").strip()
|
||||
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
|
||||
|
||||
@@ -30,7 +30,7 @@ from typing import Any
|
||||
|
||||
|
||||
from ._contract import Ask, Fields, Reads, Row
|
||||
from ._shared import Get, Page, html_text, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, html_text, only, parse_dt
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=100, text="listing", complete="end",
|
||||
@@ -106,8 +106,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
for item in items:
|
||||
title = (item.findtext("title") or "").strip()
|
||||
url = (item.findtext("link") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
out.append({
|
||||
"source_id": (item.findtext("guid") or url).strip(),
|
||||
"title": title,
|
||||
|
||||
@@ -127,8 +127,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
j = job.get("data") or {}
|
||||
title = (j.get("title") or "").strip()
|
||||
role = str(j.get("slug") or j.get("req_id") or "")
|
||||
if not title or not role:
|
||||
continue
|
||||
categories = j.get("categories") or []
|
||||
text = "\n".join(p for p in (html_text(j.get("description")),
|
||||
html_text(j.get("qualifications"))) if p)
|
||||
|
||||
@@ -36,7 +36,7 @@ from typing import Any
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt, place
|
||||
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt, place
|
||||
|
||||
NAME = "jobbank"
|
||||
LABEL = "Akademikernes Jobbank (Denmark)"
|
||||
@@ -132,8 +132,8 @@ def _query(ask: Ask, number: int) -> tuple[str, str]:
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of a search; the cursor is the page number. The next page
|
||||
follows a full page (counted before any item is dropped, so a dropped item
|
||||
does not pass for the end) while the ceiling is not reached."""
|
||||
follows a full page (counted before the runner drops an item, so a
|
||||
dropped item does not pass for the end) while the ceiling is not reached."""
|
||||
number = int(cursor or 1)
|
||||
url, remote = _query(ask, number)
|
||||
try:
|
||||
@@ -184,8 +184,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
out: list[Row] = []
|
||||
for item in decoded["items"]:
|
||||
link, title = item["link"], item["title"]
|
||||
if not title or not item["id"] or not is_web_url(link):
|
||||
continue
|
||||
line = _LINE.fullmatch(item["description"])
|
||||
kind = line.group("kind").strip() if line else ""
|
||||
company, regions, cut = _regions(line.group("rest")) if line else ("", [], False)
|
||||
|
||||
@@ -117,9 +117,9 @@ def _query(ask: Ask, number: int) -> str:
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of a search, newest first; the cursor is the page number.
|
||||
The next page follows a full page (counted before any item is dropped, so
|
||||
a dropped item does not pass for the end) while the 200-result ceiling is
|
||||
not reached."""
|
||||
The next page follows a full page (counted before the runner drops an
|
||||
item, so a dropped item does not pass for the end) while the 200-result
|
||||
ceiling is not reached."""
|
||||
number = int(cursor or 1)
|
||||
root = ET.fromstring(get(_query(ask, number)))
|
||||
channel = root.find("channel")
|
||||
@@ -143,8 +143,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
out: list[Row] = []
|
||||
for item in decoded["items"]:
|
||||
link, full_title = item["link"], item["title"]
|
||||
if not full_title or not is_web_url(link):
|
||||
continue
|
||||
description = item["description"]
|
||||
h4 = _H4.search(description)
|
||||
title = _text(h4.group(1)) if h4 else full_title
|
||||
|
||||
@@ -138,8 +138,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
for ad in decoded.get("jobAds") or []:
|
||||
ident = str(ad.get("jobAdId") or "").strip()
|
||||
title = _text(ad.get("title"))
|
||||
if not ident or not title:
|
||||
continue
|
||||
link = str(ad.get("jobAdUrl") or "").strip()
|
||||
town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
|
||||
places: list[Place] = [place(town)] if town else []
|
||||
|
||||
@@ -176,8 +176,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
title = (j.get("title") or "").strip()
|
||||
job_id = str(j.get("id") or "").strip()
|
||||
path = str(j.get("url") or "").strip()
|
||||
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
|
||||
continue
|
||||
function = str(j.get("function") or "").strip()
|
||||
places = [place(name) for name in dict.fromkeys(
|
||||
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
|
||||
@@ -186,7 +184,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"title": title,
|
||||
"places": places,
|
||||
"location": "; ".join(pl["text"] for pl in places),
|
||||
"url": f"https://{_EMP}{path}",
|
||||
"url": f"https://{_EMP}{path}" if _ROLE_PATH.fullmatch(path) else "",
|
||||
"posted": _published(j.get("published_date")),
|
||||
"department": "" if function == "All functions" else function,
|
||||
"description": "",
|
||||
|
||||
@@ -22,7 +22,7 @@ from typing import Any
|
||||
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads, Row
|
||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -123,8 +123,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
for j in data:
|
||||
title = (j.get("text") or "").strip()
|
||||
url = (j.get("hostedUrl") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
categories = j.get("categories") or {}
|
||||
description = j.get("description") or j.get("descriptionPlain") or ""
|
||||
salary_range = j.get("salaryRange") or {}
|
||||
|
||||
@@ -198,9 +198,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
for role in search.get("requisitionList") or []:
|
||||
role_id = str(role.get("Id") or "").strip()
|
||||
title = _text(role.get("Title"))
|
||||
# The id goes into a URL, and a listing is external content.
|
||||
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id):
|
||||
continue
|
||||
# The id goes into a URL, and a listing is external content: an id
|
||||
# of other characters gets no link, and the runner drops the row.
|
||||
safe = bool(re.fullmatch(r"[A-Za-z0-9_-]+", role_id))
|
||||
remote = _text(role.get("WorkplaceTypeCode")) or None
|
||||
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
|
||||
places: list[Place] = []
|
||||
@@ -214,7 +214,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
"title": title,
|
||||
"places": places,
|
||||
"location": "; ".join(p["text"] for p in places),
|
||||
"url": f"{_base(slug)}/job/{role_id}",
|
||||
"url": f"{_base(slug)}/job/{role_id}" if safe else "",
|
||||
"posted": parse_dt(role.get("PostedDate")),
|
||||
"department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction"))
|
||||
or _text(role.get("Department")) or _text(role.get("Organization"))),
|
||||
|
||||
@@ -104,8 +104,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
for position in root.findall("position"):
|
||||
job_id = (position.findtext("id") or "").strip()
|
||||
title = (position.findtext("name") or "").strip()
|
||||
if not job_id or not title:
|
||||
continue
|
||||
description = " ".join(
|
||||
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
||||
salary_info = position.find("salaryInformation")
|
||||
|
||||
@@ -168,8 +168,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
for job in search.get("data", {}).get("jobs") or []:
|
||||
key = _key(job)
|
||||
title = _text(job.get("title"))
|
||||
if not title or not key:
|
||||
continue
|
||||
places: list[Place] = []
|
||||
for text in (job.get("location"), *(job.get("multi_location") or [])):
|
||||
if isinstance(text, str):
|
||||
|
||||
@@ -22,7 +22,7 @@ from typing import Any
|
||||
|
||||
|
||||
from ._contract import Ask, Fields, Reads
|
||||
from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
|
||||
from ._shared import HEADING, Row, html_text, label_slug_ok, num, only, pay_line, yearly_amount
|
||||
|
||||
NAME = "pinpoint"
|
||||
LABEL = "Pinpoint"
|
||||
@@ -121,8 +121,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("url") or "").strip()
|
||||
source_id = str(j.get("id") or "")
|
||||
if not title or not source_id or not is_web_url(url):
|
||||
continue
|
||||
job = j.get("job") or {}
|
||||
low, high, currency, salary_text = _pay(j)
|
||||
workplace = str(j.get("workplace_type") or "").strip()
|
||||
|
||||
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, html_text, only, parse_dt
|
||||
|
||||
NAME = "platsbanken"
|
||||
LABEL = "Platsbanken"
|
||||
@@ -108,8 +108,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
||||
for hit in decoded.get("hits") or []:
|
||||
title = _text(hit.get("headline") or "")
|
||||
url = (hit.get("webpage_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
|
||||
places = [place for address in addresses if isinstance(address, dict)
|
||||
if (place := _place(address))]
|
||||
|
||||
@@ -13,7 +13,7 @@ from typing import Any
|
||||
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
NAME = "recruitee"
|
||||
LABEL = "Recruitee"
|
||||
@@ -105,8 +105,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
for j in data.get("offers") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("careers_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
||||
salary_min, salary_max, currency = yearly_amount(
|
||||
salary.get("min"), salary.get("max"), salary.get("currency"),
|
||||
|
||||
+14
-10
@@ -82,27 +82,31 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
||||
raise ValueError("not a Rippling board: the answer is not a list")
|
||||
roles: dict[str, Row] = {}
|
||||
places: dict[str, list[Place]] = {}
|
||||
for entry in data:
|
||||
for index, entry in enumerate(data):
|
||||
job_id = str(entry.get("uuid") or "").strip()
|
||||
title = (entry.get("name") or "").strip()
|
||||
if not title or not _UUID.fullmatch(job_id):
|
||||
continue
|
||||
# The id goes into a URL: one of other characters gets no link, and
|
||||
# the runner drops the row. An entry without an id is a role of its
|
||||
# own, not merged with the others that lack one.
|
||||
key = job_id or f"#{index}"
|
||||
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
|
||||
places.setdefault(job_id, [])
|
||||
places.setdefault(key, [])
|
||||
if place:
|
||||
add_place(places[job_id], _place(place))
|
||||
roles.setdefault(job_id, {
|
||||
add_place(places[key], _place(place))
|
||||
roles.setdefault(key, {
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"url": f"https://{_SITE}/{slug}/jobs/{job_id}",
|
||||
"url": f"https://{_SITE}/{slug}/jobs/{job_id}" if _UUID.fullmatch(job_id) else "",
|
||||
"posted": None,
|
||||
"department": str(((entry.get("department") or {}).get("label")) or "").strip(),
|
||||
"description": "",
|
||||
"salary_text": "", # the board's list states no pay
|
||||
})
|
||||
for job_id, row in roles.items():
|
||||
row["places"] = places[job_id]
|
||||
row["location"] = "; ".join(p["text"] for p in places[job_id])
|
||||
if title and not roles[key]["title"]:
|
||||
roles[key]["title"] = title # an entry of the role that has one
|
||||
for key, row in roles.items():
|
||||
row["places"] = places[key]
|
||||
row["location"] = "; ".join(p["text"] for p in places[key])
|
||||
return list(roles.values())
|
||||
|
||||
|
||||
|
||||
@@ -115,8 +115,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
for j in data.get("content") or []:
|
||||
title = (j.get("name") or "").strip()
|
||||
job_id = str(j.get("id") or "")
|
||||
if not title or not job_id:
|
||||
continue
|
||||
location = j.get("location") or {}
|
||||
loc_text = (location.get("fullLocation")
|
||||
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
||||
|
||||
@@ -63,7 +63,7 @@ from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._countries import NAMES
|
||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
|
||||
|
||||
NAME = "successfactors"
|
||||
LABEL = "SuccessFactors"
|
||||
@@ -247,8 +247,9 @@ def _tile_fields(chunk: str) -> tuple[str, str, str]:
|
||||
def _entries(text: str) -> list[dict[str, Any]]:
|
||||
"""The roles of one search page, in the order shown, whichever layout it is
|
||||
in and before the prefix filter: `id`, `href` (as written, relative),
|
||||
`title`, `place`, `date` and `department`. A role is listed once, and a row
|
||||
without a link to a role is not one."""
|
||||
`title`, `place`, `date` and `department`. A role is listed once. The id
|
||||
and the title are in the role's link, so a row without one is handed over
|
||||
with an empty `id`, `href` and `title`, for the runner to drop and count."""
|
||||
table = _ROW.split(text)[1:]
|
||||
chunks = table or _TILE.split(text)[1:]
|
||||
out: list[dict[str, Any]] = []
|
||||
@@ -256,10 +257,12 @@ def _entries(text: str) -> list[dict[str, Any]]:
|
||||
for chunk in chunks:
|
||||
if table:
|
||||
chunk = chunk.split("</tr>", 1)[0]
|
||||
if not (link := _LINK.search(chunk)):
|
||||
continue
|
||||
href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3))
|
||||
if job_id in seen:
|
||||
if link := _LINK.search(chunk):
|
||||
href, job_id, title = (html.unescape(link.group(1)), link.group(2),
|
||||
_text(link.group(3)))
|
||||
else:
|
||||
href = job_id = title = ""
|
||||
if job_id and job_id in seen:
|
||||
continue
|
||||
seen.add(job_id)
|
||||
place, date, department = (_table_fields if table else _tile_fields)(chunk)
|
||||
@@ -283,12 +286,14 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
if not _SEARCH_PAGE.search(text):
|
||||
raise ValueError("not a SuccessFactors search page")
|
||||
found = _entries(text)
|
||||
ids = [entry["id"] for entry in found if entry["id"]]
|
||||
total = _total(text)
|
||||
if found and found[0]["id"] == previous:
|
||||
first = ids[0] if ids else None
|
||||
if first is not None and first == previous:
|
||||
return text, [], total, (start, previous)
|
||||
more = bool(found) and (total is None or start + len(found) < total)
|
||||
return (text, [{"id": entry["id"]} for entry in found], total,
|
||||
(start + len(found), found[0]["id"]) if more else None)
|
||||
return (text, [{"id": job_id} for job_id in ids], total,
|
||||
(start + len(found), first) if more else None)
|
||||
|
||||
|
||||
def rows(decoded: str, slug: str) -> list[Row]:
|
||||
@@ -296,17 +301,21 @@ def rows(decoded: str, slug: str) -> list[Row]:
|
||||
slug's prefix when it has one; the id in it is the requisition's `source_id`.
|
||||
The table and the tile layout give a title, a location, a date and, where
|
||||
the site has the column, a department; nothing else about a role is on
|
||||
the list, so its description and pay are empty."""
|
||||
the list, so its description and pay are empty.
|
||||
|
||||
A role linked on another host, or outside the prefix, is another board's
|
||||
(another site, or another brand of a shared site), and is left out: it is
|
||||
no role of this company. A row with no link or no title is handed over,
|
||||
and the runner drops it."""
|
||||
host, prefix = _parts(slug)
|
||||
base = f"https://{host}/"
|
||||
out = []
|
||||
for entry in _entries(decoded):
|
||||
url = urljoin(base, entry["href"])
|
||||
if urlsplit(url).hostname != host or not is_web_url(url):
|
||||
url = urljoin(base, entry["href"]) if entry["href"] else ""
|
||||
if url and urlsplit(url).hostname != host:
|
||||
continue
|
||||
if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"):
|
||||
continue
|
||||
if not entry["title"]:
|
||||
if url and prefix and not urlsplit(url).path.lower().startswith(
|
||||
f"{prefix.lower()}/job/"):
|
||||
continue
|
||||
out.append({
|
||||
"source_id": entry["id"],
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing import Any
|
||||
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
|
||||
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
|
||||
|
||||
NAME = "teamtailor"
|
||||
LABEL = "Teamtailor"
|
||||
@@ -115,8 +115,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
for item in items:
|
||||
title = (item.findtext("title") or "").strip()
|
||||
url = (item.findtext("link") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
status = (item.findtext("remoteStatus") or "").strip()
|
||||
remote = {"fully": True, "hybrid": False, "onsite": False}.get(status)
|
||||
places: list[Place] = []
|
||||
|
||||
@@ -102,8 +102,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
for j in data.get("results") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
shortcode = (j.get("shortcode") or "").strip()
|
||||
if not title or not shortcode:
|
||||
continue
|
||||
workplace = str(j.get("workplace") or "").strip()
|
||||
remote = bool(j.get("remote")) or workplace == "remote"
|
||||
places = _places(j, remote)
|
||||
|
||||
@@ -124,8 +124,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
|
||||
postings = data.get("jobPostings") or []
|
||||
# Every posting is counted, a path-less one (LEGO lists one that has only
|
||||
# `bulletFields`) by its position, so the parser dropping it does not
|
||||
# leave the board short of its total.
|
||||
# `bulletFields`) by its position, so the runner dropping its row does
|
||||
# not leave the board short of its total.
|
||||
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
|
||||
following = offset + len(postings)
|
||||
more = bool(postings) and (total is None or following < total)
|
||||
@@ -274,13 +274,13 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
||||
`bulletFields`, which lead with the requisition id, and on some tenants
|
||||
`remoteType`. No description, no date and no country: `detail` has
|
||||
those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the
|
||||
path where not; a posting with no title or no path is skipped."""
|
||||
path where not. A posting with no path has neither an id nor a link, and
|
||||
the runner drops it."""
|
||||
out = []
|
||||
for p in decoded.get("jobPostings") or []:
|
||||
title = (p.get("title") or "").strip()
|
||||
path = (p.get("externalPath") or "").strip()
|
||||
if not title or not _ROLE_PATH.fullmatch(path):
|
||||
continue
|
||||
path = path if _ROLE_PATH.fullmatch(path) else ""
|
||||
remote = _remote(p.get("remoteType"))
|
||||
place = _place(p.get("locationsText"), remote)
|
||||
out.append({
|
||||
@@ -288,7 +288,7 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
||||
"title": title,
|
||||
"places": [place] if place else [],
|
||||
"location": place["text"] if place else "",
|
||||
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}", # type: ignore[index]
|
||||
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}" if path else "", # type: ignore[index]
|
||||
"posted": _posted(p.get("postedOn") or ""),
|
||||
"department": "",
|
||||
"description": "",
|
||||
|
||||
+2
-1
@@ -19,7 +19,8 @@ roles close once no search has seen them for `STALE_DAYS` (see
|
||||
`app/jobs/pull.py`).
|
||||
|
||||
In a pull a search is read by `_runner.read` under the prefix `search`, as a
|
||||
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
|
||||
board is under `ats`: the runner drops a row without an id, a title or a
|
||||
link, the 10% guards hold it, `_runner.stamp` records it,
|
||||
and a search that fails or is suspect is kept as failing under
|
||||
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
|
||||
the settle reads as it reads a failing board's.
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// static/agents.js without a browser: the gates Save sends from the fetch
|
||||
// gate's four parts, and the stored gates the parts cannot say, which the
|
||||
// form names in a note and sends back as they came.
|
||||
//
|
||||
// node --test tests/js/*.test.mjs
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import vm from "node:vm";
|
||||
|
||||
// Only what agents.js touches before DOMContentLoaded.
|
||||
const window = { api: async () => [], count: (n, noun) => `${n} ${noun}` };
|
||||
const document = { addEventListener() {} };
|
||||
Object.assign(globalThis, { window, document });
|
||||
for (const file of ["remote.js", "agents.js"]) {
|
||||
vm.runInThisContext(readFileSync(new URL(`../../app/static/${file}`, import.meta.url), "utf8"));
|
||||
}
|
||||
const rh = window.remoteHelpers;
|
||||
const { formGates, keptNote, rulesLine } = window.agentGates;
|
||||
|
||||
/** The four parts as the form shows them after loading `gates`, and what it keeps. */
|
||||
function load(gates) {
|
||||
const { where, rest } = rh.distribute(gates);
|
||||
const parts = {
|
||||
scope: where.scope.kind, scoped: where.scope.values,
|
||||
within: where.countries ? where.countries.values : [],
|
||||
towns: where.places ? where.places.values : [],
|
||||
remote: where.remote.mode, from: where.remote.values,
|
||||
titles: where.titles ? where.titles.values : [],
|
||||
};
|
||||
return { parts, rest };
|
||||
}
|
||||
|
||||
const europe = { field: "country", values: rh.europeWith(["DK"]), mode: "require" };
|
||||
const dk = { field: "country", values: ["DK"], mode: "any" };
|
||||
const remote = { field: "remote", values: ["remote"], mode: "any" };
|
||||
const towns = { field: "place", values: ["Malmö", "Lund"], mode: "any" };
|
||||
const titles = { field: "title", values: ["account executive", "sales"], mode: "exclude" };
|
||||
const level = { field: "seniority", values: ["senior", "staff"], mode: "require" };
|
||||
const acme = { field: "company", values: ["acme"], mode: "exclude" };
|
||||
|
||||
test("a save sends the four parts back as they were loaded", () => {
|
||||
const agents = [
|
||||
[europe, dk, remote, towns, titles],
|
||||
[{ field: "country", values: ["DK", "SE"], mode: "require" },
|
||||
{ field: "remote_from", values: ["country:DK", "country:EU"], mode: "any" }, titles],
|
||||
[dk],
|
||||
[],
|
||||
];
|
||||
for (const gates of agents) {
|
||||
const { parts, rest } = load(gates);
|
||||
assert.deepEqual(rest, []);
|
||||
assert.deepEqual(formGates(parts, rest), gates);
|
||||
}
|
||||
});
|
||||
|
||||
test("each part sends its own gate, and an empty part none", () => {
|
||||
const parts = { scope: "everywhere", scoped: [], within: [], towns: [], remote: "none", from: [], titles: [] };
|
||||
assert.deepEqual(formGates(parts), []);
|
||||
// A remote line open to no area yet, or chosen countries with none picked, say nothing.
|
||||
assert.deepEqual(formGates({ ...parts, remote: "from", scope: "countries" }), []);
|
||||
assert.deepEqual(formGates({ ...parts, scope: "europe", within: ["US"] }), [
|
||||
{ field: "country", values: rh.europeWith(["US"]), mode: "require" },
|
||||
{ field: "country", values: ["US"], mode: "any" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("a stored gate the parts cannot say is kept across a save, named in the note and on the card", () => {
|
||||
const gates = [europe, dk, remote, titles, level, acme];
|
||||
const { parts, rest } = load(gates);
|
||||
assert.deepEqual(rest, [level, acme]);
|
||||
assert.deepEqual(formGates(parts, rest), gates);
|
||||
assert.equal(keptNote(rest),
|
||||
"The form cannot edit these rules, and Save keeps them as stored: "
|
||||
+ "must: with seniority senior, staff; not: at acme.");
|
||||
const words = rulesLine(gates);
|
||||
assert.match(words, /Only roles with seniority senior, staff\./);
|
||||
assert.match(words, /Leaves out roles at acme\./);
|
||||
});
|
||||
|
||||
test("an agent with nothing to keep has no note", () => {
|
||||
assert.equal(keptNote([]), "");
|
||||
assert.equal(keptNote([level]), "The form cannot edit this rule, and Save keeps it as stored: must: with seniority senior, staff.");
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
// static/remote.js without a browser: the badge, the suggestions the agent's
|
||||
// remote line offers, and how gates share out between Where and Other rules.
|
||||
// remote line offers, and how gates share out between the fetch gate and the
|
||||
// rest the form keeps as stored.
|
||||
//
|
||||
// node --test tests/js/*.test.mjs
|
||||
|
||||
@@ -89,14 +90,14 @@ test("any other require gate on country is the scope with its codes as chips", (
|
||||
assert.deepEqual(rest, []);
|
||||
});
|
||||
|
||||
test("a second require gate on country stays a row", () => {
|
||||
test("a second require gate on country is kept as stored", () => {
|
||||
const second = must("NO");
|
||||
const { where, rest } = rh.distribute([europe, second]);
|
||||
assert.equal(where.scope.kind, "europe");
|
||||
assert.deepEqual(rest, [second]);
|
||||
});
|
||||
|
||||
test("the titles to leave out are the exclude gate on title, and any other exclude is a row", () => {
|
||||
test("the titles to leave out are the exclude gate on title, and any other exclude is kept as stored", () => {
|
||||
const titles = { field: "title", values: ["account executive"], mode: "exclude" };
|
||||
const level = { field: "seniority", values: ["junior"], mode: "exclude" };
|
||||
const { where, rest } = rh.distribute([titles, level]);
|
||||
@@ -104,7 +105,7 @@ test("the titles to leave out are the exclude gate on title, and any other exclu
|
||||
assert.deepEqual(rest, [level]);
|
||||
});
|
||||
|
||||
test("a gate with a mode the form does not know stays a row and takes no Where line", () => {
|
||||
test("a gate with a mode the form does not know is kept as stored and takes no Where line", () => {
|
||||
const prefer = { field: "remote", values: ["hybrid"], mode: "prefer" };
|
||||
const { where, rest } = rh.distribute([prefer]);
|
||||
assert.deepEqual(where.remote, { mode: "none", values: [] });
|
||||
@@ -112,7 +113,7 @@ test("a gate with a mode the form does not know stays a row and takes no Where l
|
||||
assert.deepEqual(rest, [prefer]);
|
||||
});
|
||||
|
||||
test("a require gate on another field is a row and leaves the scope everywhere", () => {
|
||||
test("a require gate on another field is kept as stored and leaves the scope everywhere", () => {
|
||||
const level = { field: "seniority", values: ["senior"], mode: "require" };
|
||||
const { where, rest } = rh.distribute([level]);
|
||||
assert.equal(where.scope.kind, "everywhere");
|
||||
@@ -129,7 +130,7 @@ test("remote, three modes: the plain gate is within the scope", () => {
|
||||
assert.deepEqual(named.rest, []);
|
||||
});
|
||||
|
||||
test("a remote_from gate wins over the plain one, which stays a row", () => {
|
||||
test("a remote_from gate wins over the plain one, which is kept as stored", () => {
|
||||
for (const gates of [[older, from("country:DK")], [from("country:DK"), older]]) {
|
||||
const { where, rest } = rh.distribute(gates);
|
||||
assert.deepEqual(where.remote, { mode: "from", values: ["country:DK"] });
|
||||
@@ -137,7 +138,7 @@ test("a remote_from gate wins over the plain one, which stays a row", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("a second remote_from gate or a must stays a row", () => {
|
||||
test("a second remote_from gate or a must is kept as stored", () => {
|
||||
const second = from("country:SE");
|
||||
const required = { field: "remote_from", values: ["country:DK"], mode: "require" };
|
||||
const { where, rest } = rh.distribute([from("country:DK"), second, required]);
|
||||
|
||||
+15
-8
@@ -24,18 +24,24 @@ from plugins.readers import _runner, smartrecruiters # noqa: E402
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
|
||||
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||
def _parsed(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
||||
body = path.read_bytes()
|
||||
return ats._PARSERS[host](body, slug)
|
||||
return ats._PARSERS[host](path.read_bytes(), slug)
|
||||
|
||||
|
||||
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||
"""The rows of a host's fixture the runner keeps (`_runner.valid`)."""
|
||||
return [row for row in _parsed(host, slug) if _runner.valid(row)]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ hosts
|
||||
|
||||
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
||||
# The javascript: and data: rows are handed over and the runner drops
|
||||
# them: a feed is external content, never trusted as a URL.
|
||||
assert [r["source_id"] for r in _parsed("greenhouse")] == [
|
||||
"1001", "1002", "1003", "1004", "1005", "1006"]
|
||||
rows = _rows("greenhouse")
|
||||
# The javascript: and data: rows are refused, same as the four real ones
|
||||
# are kept: a feed is external content, never trusted as a URL.
|
||||
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
||||
first = rows[0]
|
||||
assert first["title"] == "Senior Backend Engineer" # trimmed
|
||||
@@ -269,10 +275,11 @@ TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||
# the javascript: link is refused like on the other hosts
|
||||
# the javascript: link is handed over and refused by the runner
|
||||
assert [r["source_id"] for r in rows] == [
|
||||
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
|
||||
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"]
|
||||
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"]
|
||||
assert [_runner.valid(r) for r in rows] == [True, True, True, False]
|
||||
first = rows[0]
|
||||
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
||||
assert first["department"] == "Engineering"
|
||||
@@ -294,7 +301,7 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
|
||||
|
||||
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
|
||||
assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them
|
||||
assert rows[1]["title"] == "Läkare" # trimmed
|
||||
assert rows[1]["department"] == "Software Development"
|
||||
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
||||
|
||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, bamboohr # noqa: E402
|
||||
from plugins.readers import Ask, _runner, bamboohr # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -84,11 +84,16 @@ def test_a_page_that_is_not_a_listing_raises_and_an_empty_board_is_no_rows() ->
|
||||
bamboohr.parse(body, "configit")
|
||||
|
||||
|
||||
def test_a_role_without_a_title_or_an_id_is_left_out() -> None:
|
||||
def test_a_role_without_a_title_or_an_id_reaches_the_runner_and_is_dropped_there() -> None:
|
||||
body = json.loads(_read("bamboohr.json"))
|
||||
body["result"][0]["jobOpeningName"] = " "
|
||||
body["result"][1]["id"] = None
|
||||
assert bamboohr.parse(json.dumps(body).encode(), "configit") == []
|
||||
raw = json.dumps(body).encode()
|
||||
rows = bamboohr.parse(raw, "configit")
|
||||
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||
listing = _runner.read_listing(bamboohr, "configit", lambda url, **kw: raw)
|
||||
assert (listing.rows, listing.ids, listing.complete) == ([], [], True)
|
||||
assert (listing.handed, listing.dropped) == (2, 2)
|
||||
|
||||
|
||||
def test_detail_reads_the_text_the_date_and_the_country() -> None:
|
||||
|
||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, deel # noqa: E402
|
||||
from plugins.readers import Ask, _runner, deel # noqa: E402
|
||||
|
||||
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
|
||||
URL = "https://jobs.deel.com/klarna"
|
||||
@@ -89,10 +89,12 @@ def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board
|
||||
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
|
||||
|
||||
|
||||
def test_a_posting_without_a_title_or_id_is_skipped() -> None:
|
||||
def test_a_posting_without_a_title_or_id_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
|
||||
{"id": "c", "title": "Cook", "job": {}}]}
|
||||
assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"]
|
||||
rows = deel.parse(_page_with(body), "acme")
|
||||
assert [r["source_id"] for r in rows] == ["a", "", "c"]
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------ the fetch
|
||||
|
||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, eightfold # noqa: E402
|
||||
from plugins.readers import Ask, _runner, eightfold # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
SLUG = "jobs.ericsson.com/ericsson.com"
|
||||
@@ -93,10 +93,12 @@ def test_the_work_location_option_is_the_hosts_word(
|
||||
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
|
||||
|
||||
|
||||
def test_a_position_without_a_name_an_id_or_a_path_is_left_out() -> None:
|
||||
def test_a_position_without_a_name_an_id_or_a_path_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
|
||||
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
|
||||
assert eightfold.rows(bad, SLUG) == []
|
||||
rows = eightfold.rows(bad, SLUG)
|
||||
assert len(rows) == 4 and not any(map(_runner.valid, rows))
|
||||
assert [r["url"] for r in rows[2:]] == ["", ""] # no link built off the host
|
||||
|
||||
|
||||
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, gem # noqa: E402
|
||||
from plugins.readers import Ask, _runner, gem # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
API = "https://jobs.gem.com/api/public/graphql"
|
||||
@@ -79,8 +79,9 @@ def test_remote_is_the_location_type_then_the_locations(
|
||||
assert gem.rows([posting], "acme")[0]["remote"] == remote
|
||||
|
||||
|
||||
def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None:
|
||||
assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == []
|
||||
def test_a_posting_without_a_title_or_an_ext_id_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
rows = gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme")
|
||||
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||
|
||||
|
||||
def test_the_board_is_one_post_without_origin_or_referer() -> None:
|
||||
|
||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, hibob # noqa: E402
|
||||
from plugins.readers import Ask, _runner, hibob # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "hibob.json").read_bytes()
|
||||
@@ -87,10 +87,12 @@ def test_the_place_leaves_the_country_off_where_the_site_names_it() -> None:
|
||||
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
|
||||
|
||||
|
||||
def test_an_ad_without_a_title_or_id_is_skipped_and_an_empty_board_is_empty() -> None:
|
||||
def test_an_ad_without_a_title_or_id_is_handed_over_and_an_empty_board_is_empty() -> None:
|
||||
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
|
||||
{"id": "c", "title": "Cook"}]}).encode()
|
||||
assert [r["source_id"] for r in hibob.parse(body, "acme")] == ["c"]
|
||||
rows = hibob.parse(body, "acme")
|
||||
assert [r["source_id"] for r in rows] == ["a", "", "c"]
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
|
||||
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, hrmanager # noqa: E402
|
||||
from plugins.readers import Ask, _runner, hrmanager # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
||||
@@ -52,12 +52,14 @@ def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_f
|
||||
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
|
||||
|
||||
|
||||
def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None:
|
||||
def test_an_item_without_a_title_or_a_web_link_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
|
||||
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
|
||||
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
|
||||
b"</channel></rss>")
|
||||
assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"]
|
||||
rows = hrmanager.parse(body, "x")
|
||||
assert [r["source_id"] for r in rows] == ["1", "2", "3"]
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["3"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------- paging
|
||||
|
||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, jibe # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jibe # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
SITE = "jobs.booking.com"
|
||||
@@ -72,9 +72,10 @@ def test_the_location_falls_back_to_city_and_country() -> None:
|
||||
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
|
||||
|
||||
|
||||
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
|
||||
def test_a_role_without_a_title_or_a_slug_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
|
||||
assert jibe.rows({"jobs": jobs}, SITE) == []
|
||||
rows = jibe.rows({"jobs": jobs}, SITE)
|
||||
assert len(rows) == 3 and not any(map(_runner.valid, rows))
|
||||
|
||||
|
||||
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, jobylon # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jobylon # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
||||
@@ -65,12 +65,14 @@ def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
|
||||
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
|
||||
|
||||
|
||||
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None:
|
||||
def test_a_role_whose_link_is_not_a_role_page_gets_no_link_and_is_dropped() -> None:
|
||||
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
|
||||
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
|
||||
b"{id: '3', title: 'C', url: '/companies/3/'},"
|
||||
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
|
||||
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"]
|
||||
rows = jobylon.parse(body, "9")
|
||||
assert [r["url"] for r in rows[:3]] == ["", "", ""]
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["4"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import oracle # noqa: E402
|
||||
from plugins.readers import _runner, oracle # noqa: E402
|
||||
from plugins.readers._shared import is_web_url # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -84,14 +84,15 @@ def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
|
||||
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
|
||||
|
||||
|
||||
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_left_out() -> None:
|
||||
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_dropped_by_the_runner() -> None:
|
||||
data = _listing()
|
||||
roles = data["items"][0]["requisitionList"]
|
||||
roles[0]["Title"] = " "
|
||||
roles[1]["Id"] = "1/../../x"
|
||||
roles[2]["Id"] = ""
|
||||
assert [r["source_id"] for r in oracle.parse(json.dumps(data).encode(), SLUG)] == \
|
||||
["40975", "31680"]
|
||||
rows = oracle.parse(json.dumps(data).encode(), SLUG)
|
||||
assert rows[1]["url"] == "" # the id builds no link
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["40975", "31680"]
|
||||
|
||||
|
||||
def test_a_listing_without_the_expand_has_no_roles() -> None:
|
||||
|
||||
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import phenom # noqa: E402
|
||||
from plugins.readers import _runner, phenom # noqa: E402
|
||||
from plugins.readers._shared import is_web_url # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -101,13 +101,15 @@ def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None
|
||||
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
|
||||
|
||||
|
||||
def test_a_role_without_title_or_id_is_left_out() -> None:
|
||||
def test_a_role_without_title_or_id_is_dropped_by_the_runner() -> None:
|
||||
data = _answer()
|
||||
jobs = data["refineSearch"]["data"]["jobs"]
|
||||
jobs[0]["title"] = ""
|
||||
for key in ("jobSeqNo", "jobId", "reqId"):
|
||||
jobs[1].pop(key, None)
|
||||
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
|
||||
rows = phenom.parse(json.dumps(data).encode(), SLUG)
|
||||
assert len(rows) == len(IDS)
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == IDS[2:]
|
||||
|
||||
|
||||
# ---------------------------------------------------------- the request
|
||||
|
||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import pinpoint # noqa: E402
|
||||
from plugins.readers import _runner, pinpoint # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -86,11 +86,12 @@ def test_an_unknown_pay_frequency_keeps_the_text_and_leaves_the_columns() -> Non
|
||||
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
|
||||
|
||||
|
||||
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_left_out() -> None:
|
||||
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_dropped_by_the_runner() -> None:
|
||||
body = json.loads(_body())
|
||||
body["data"][0]["title"] = ""
|
||||
body["data"][1]["url"] = "javascript:alert(1)"
|
||||
assert pinpoint.parse(json.dumps(body).encode(), "xeneta") == []
|
||||
rows = pinpoint.parse(json.dumps(body).encode(), "xeneta")
|
||||
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||
|
||||
|
||||
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
|
||||
|
||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import rippling # noqa: E402
|
||||
from plugins.readers import _runner, rippling # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "rippling.json").read_bytes()
|
||||
@@ -61,11 +61,14 @@ def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None
|
||||
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
|
||||
|
||||
|
||||
def test_an_entry_without_a_title_or_a_plain_id_is_skipped() -> None:
|
||||
def test_an_entry_without_a_title_or_a_plain_id_is_dropped_by_the_runner() -> None:
|
||||
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
|
||||
{"uuid": "../../x", "name": "C"},
|
||||
{"uuid": "abcdefgh-2", "name": "D"}]).encode()
|
||||
assert [r["source_id"] for r in rippling.parse(body, "acme")] == ["abcdefgh-2"]
|
||||
rows = rippling.parse(body, "acme")
|
||||
assert [r["source_id"] for r in rows] == ["abcdefgh-1", "", "../../x", "abcdefgh-2"]
|
||||
assert rows[2]["url"] == "" # the id builds no link
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["abcdefgh-2"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
@@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins.readers import Ask, teamtailor # noqa: E402
|
||||
from plugins.readers import Ask, _runner, teamtailor # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -40,7 +40,8 @@ def test_page_takes_the_ask_and_reads_the_same_url() -> None:
|
||||
|
||||
|
||||
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
|
||||
first, second, third = _rows()
|
||||
first, second, third, refused = _rows()
|
||||
assert not _runner.valid(refused) # javascript:, for the runner
|
||||
# "Sverige" (a city that is a country) once, Stockholm's twin once.
|
||||
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
|
||||
{"text": "Stockholm, Sweden", "remote": False}]
|
||||
|
||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import workday # noqa: E402
|
||||
from plugins.readers import _runner, workday # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
SLUG = "zendesk.wd1/zendesk"
|
||||
@@ -96,7 +96,7 @@ def test_posted_is_read_only_where_the_text_is_exact(text: str, expected: dateti
|
||||
assert workday._posted(text) == expected
|
||||
|
||||
|
||||
def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
|
||||
def test_a_posting_without_a_title_or_a_path_reaches_the_runner_and_is_dropped() -> None:
|
||||
page = {"jobPostings": [
|
||||
{"bulletFields": ["0000036369"]}, # LEGO's, no title
|
||||
{"title": "No path", "bulletFields": ["R1"]},
|
||||
@@ -105,7 +105,16 @@ def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
|
||||
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
|
||||
"postedOn": "Posted Today", "bulletFields": ["R4"]},
|
||||
]}
|
||||
assert [r["title"] for r in workday.rows(page, SLUG)] == ["Whole"]
|
||||
rows = workday.rows(page, SLUG)
|
||||
assert len(rows) == 5
|
||||
assert [(r["source_id"], r["url"]) for r in rows[:2]] == [("", ""), ("", "")]
|
||||
assert [r["title"] for r in rows if _runner.valid(r)] == ["Whole"]
|
||||
# Through the runner: four dropped and counted, and the read complete,
|
||||
# since every posting counts by its position towards the stated total.
|
||||
body = json.dumps(page | {"total": 5}).encode()
|
||||
listing = _runner.read_listing(workday, SLUG, lambda url, **kw: body)
|
||||
assert [r["title"] for r in listing.rows] == ["Whole"]
|
||||
assert (listing.handed, listing.dropped, listing.complete) == (5, 4, True)
|
||||
|
||||
|
||||
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
|
||||
|
||||
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
||||
"VALUES (?, ?, '', '')", (key, name))
|
||||
conn.execute("PRAGMA user_version = 19")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(reopened.companies()) == 3
|
||||
|
||||
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
||||
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
|
||||
conn.execute("PRAGMA user_version = 20")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
# The tag of 21 is All countries and All roles since 24.
|
||||
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
|
||||
|
||||
+2
-34
@@ -91,37 +91,6 @@ def test_best_cv_match_puts_the_judged_first_by_cv_score(
|
||||
"order": "best"}).status_code == 422
|
||||
|
||||
|
||||
def test_a_broken_must_have_caps_career_in_the_order_and_the_reader(
|
||||
client: TestClient, store: Store) -> None:
|
||||
owner, uid = setup(store)
|
||||
# c: 9 on all three, but one must-have broken: career counts as 4.
|
||||
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
|
||||
{"id": "M2", "text": "Remote", "verdict": "broken"}]
|
||||
store.set_judgement(owner.id, uid["c"], "k", 9, 9, 9, [], [], musts)
|
||||
capped = round(10 * (30 * 9 + 25 * 9 + 30 * 4) / 85, 1)
|
||||
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
|
||||
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["c"], capped), (uid["b"], 60.0)]
|
||||
# With career weighted up the capped value drops under b's 60.
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=owner.cv_text,
|
||||
weights=Weights(cv_skills=0, cv_experience=0, cv_career=1))
|
||||
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
|
||||
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["b"], 60.0), (uid["c"], 40.0)]
|
||||
one = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()
|
||||
j = one["judgement"]
|
||||
assert (j["career"], j["career_capped"], j["broken"]) == (9, 4, 1)
|
||||
assert j["musts"] == musts and one["cv_score"] == j["cv_score"] == 40.0
|
||||
|
||||
|
||||
def test_a_met_or_unclear_must_have_leaves_career_as_the_model_gave_it(
|
||||
client: TestClient, store: Store) -> None:
|
||||
owner, uid = setup(store)
|
||||
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
|
||||
{"id": "M2", "text": "Remote", "verdict": "unclear"}]
|
||||
store.set_judgement(owner.id, uid["c"], "k", 9, 9, 9, [], [], musts)
|
||||
j = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()["judgement"]
|
||||
assert (j["career"], j["career_capped"], j["broken"], j["cv_score"]) == (9, 9, 0, 90.0)
|
||||
|
||||
|
||||
def test_the_agents_weights_make_the_cv_score_at_read_time(
|
||||
client: TestClient, store: Store) -> None:
|
||||
owner, uid = setup(store)
|
||||
@@ -218,15 +187,14 @@ def test_an_agent_without_languages_flags_nothing(client: TestClient, store: Sto
|
||||
|
||||
def test_the_cv_fields_round_trip_and_are_kept_when_left_out(client: TestClient) -> None:
|
||||
body = {"name": "Data", "cv_text": "VP Data.", "like_text": "Teams.", "avoid_text": "Reporting.",
|
||||
"must_text": "Permanent.",
|
||||
"cv_languages": [{"language": "da", "level": "native"},
|
||||
{"language": "EN", "level": "fluent"}]}
|
||||
made = client.post("/api/agents", json=body).json()
|
||||
assert made["cv_languages"] == [{"language": "da", "level": "native"},
|
||||
{"language": "en", "level": "fluent"}]
|
||||
kept = client.post("/api/agents", json={"id": made["id"], "name": "Data"}).json()
|
||||
assert (kept["cv_text"], kept["like_text"], kept["avoid_text"], kept["must_text"]) == \
|
||||
("VP Data.", "Teams.", "Reporting.", "Permanent.")
|
||||
assert (kept["cv_text"], kept["like_text"], kept["avoid_text"]) == \
|
||||
("VP Data.", "Teams.", "Reporting.")
|
||||
assert len(kept["cv_languages"]) == 2
|
||||
nulls = client.post("/api/agents", json={"id": made["id"], "name": "Data",
|
||||
"cv_text": None, "cv_languages": None}).json()
|
||||
|
||||
+46
-42
@@ -16,7 +16,7 @@ from pydantic import ValidationError
|
||||
from app import prep
|
||||
from app.config import Config
|
||||
from app.jobs import extract
|
||||
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag, must_lines
|
||||
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag
|
||||
from app.storage import Store
|
||||
from desk import modeltask
|
||||
from desk.models import Item
|
||||
@@ -44,24 +44,21 @@ def test_an_agent_keeps_its_cv_fields_when_a_save_leaves_them_out(store: Store)
|
||||
cv_languages=[CvLanguage(language="DA", level="native"),
|
||||
CvLanguage(language="en", level="fluent"),
|
||||
CvLanguage(language="da", level="basic")],
|
||||
like_text=" Building teams. ", avoid_text="Pure reporting.",
|
||||
must_text="Permanent contract.")
|
||||
like_text=" Building teams. ", avoid_text="Pure reporting.")
|
||||
assert made.cv_text == "VP Data."
|
||||
assert [(lang.language, lang.level) for lang in made.cv_languages] == \
|
||||
[("da", "native"), ("en", "fluent")]
|
||||
assert (made.like_text, made.avoid_text, made.must_text) == \
|
||||
("Building teams.", "Pure reporting.", "Permanent contract.")
|
||||
assert (made.like_text, made.avoid_text) == ("Building teams.", "Pure reporting.")
|
||||
|
||||
kept = store.save_agent("Data", "data leadership", agent_id=made.id)
|
||||
assert (kept.cv_text, kept.like_text, kept.avoid_text, kept.must_text) == \
|
||||
("VP Data.", "Building teams.", "Pure reporting.", "Permanent contract.")
|
||||
assert (kept.cv_text, kept.like_text, kept.avoid_text) == \
|
||||
("VP Data.", "Building teams.", "Pure reporting.")
|
||||
assert len(kept.cv_languages) == 2
|
||||
|
||||
cleared = store.save_agent("Data", "data leadership", agent_id=made.id, cv_text="",
|
||||
cv_languages=[], like_text="", avoid_text="",
|
||||
must_text="")
|
||||
assert (cleared.cv_text, cleared.cv_languages, cleared.like_text, cleared.avoid_text,
|
||||
cleared.must_text) == ("", [], "", "", "")
|
||||
cv_languages=[], like_text="", avoid_text="")
|
||||
assert (cleared.cv_text, cleared.cv_languages, cleared.like_text, cleared.avoid_text) == \
|
||||
("", [], "", "")
|
||||
|
||||
|
||||
def test_a_cv_language_is_a_code_and_a_level_of_the_ladder() -> None:
|
||||
@@ -92,25 +89,24 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
|
||||
conn.execute("DROP TABLE role_judgements")
|
||||
for column in ("languages_required", "deadline_host", "deadline_model", "deadline"):
|
||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||
for column in ("cv_text", "cv_languages", "avoid_text", "must_text"):
|
||||
for column in ("cv_text", "cv_languages", "avoid_text"):
|
||||
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
|
||||
conn.execute("PRAGMA user_version = 27")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [28, 29, 30, 31, 32, 33, 34, 35]
|
||||
rows = {r["source_id"]: r for r in migrated._rows(
|
||||
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
|
||||
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
|
||||
assert rows["a"]["languages_required"] is None and rows["b"]["deadline_host"] == ""
|
||||
[data] = migrated.agents()
|
||||
assert (data.cv_text, data.cv_languages, data.avoid_text, data.must_text) == \
|
||||
("", [], "", "")
|
||||
assert (data.cv_text, data.cv_languages, data.avoid_text) == ("", [], "")
|
||||
prep.run(migrated, remap=True, dupes=False)
|
||||
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 27")
|
||||
assert Store(path).migrations_applied == [28, 29, 30, 31, 32, 33]
|
||||
assert Store(path).migrations_applied == [28, 29, 30, 31, 32, 33, 34, 35]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- deadline
|
||||
@@ -231,7 +227,6 @@ def test_the_language_flag() -> None:
|
||||
def test_the_cv_score_weighs_skills_and_career_above_experience() -> None:
|
||||
assert cv_score(10, 10, 10) == 100.0
|
||||
assert cv_score(8, 6, 4) == round(10 * (240 + 150 + 120) / 85, 1)
|
||||
assert cv_score(8, 6, 9, broken=1) == cv_score(8, 6, 4)
|
||||
|
||||
|
||||
def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
|
||||
@@ -244,20 +239,36 @@ def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
|
||||
assert [lang.language for lang in store.role(item.uid).languages_required] == ["da"]
|
||||
|
||||
|
||||
def test_v29_adds_the_musts_and_broken_columns_and_runs_twice(tmp_path: Path) -> None:
|
||||
path = tmp_path / "v28.db"
|
||||
Store(path)
|
||||
def test_v34_drops_the_must_haves_and_runs_twice(tmp_path: Path) -> None:
|
||||
"""A file from before migration 34 holds the agent's must text and the
|
||||
judge's verdicts on it; the columns go, the rest of the agent and the
|
||||
judgement stay."""
|
||||
path = tmp_path / "v33.db"
|
||||
store = Store(path)
|
||||
item = posting("1")
|
||||
upsert(store, item)
|
||||
agent = store.save_agent("Data", "data leadership", cv_text="CV.", like_text="Teams.")
|
||||
assert store.set_judgement(agent.id, item.uid, "key", 8, 7, 9, ["Good."], ["Bad."])
|
||||
with sqlite3.connect(path) as conn:
|
||||
for column in ("musts", "broken"):
|
||||
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
|
||||
conn.execute("PRAGMA user_version = 28")
|
||||
conn.execute("ALTER TABLE agents ADD COLUMN must_text TEXT NOT NULL DEFAULT ''")
|
||||
conn.execute("ALTER TABLE role_judgements ADD COLUMN musts TEXT NOT NULL DEFAULT '[]'")
|
||||
conn.execute("ALTER TABLE role_judgements ADD COLUMN broken INTEGER NOT NULL DEFAULT 0")
|
||||
conn.execute("UPDATE agents SET must_text = 'Remote.'")
|
||||
conn.execute("UPDATE role_judgements SET musts = ?, broken = 1",
|
||||
(json.dumps([{"id": "M1", "text": "Remote.", "verdict": "broken"}]),))
|
||||
conn.execute("PRAGMA user_version = 33")
|
||||
|
||||
assert Store(path).migrations_applied == [29, 30, 31, 32, 33]
|
||||
with sqlite3.connect(path) as conn:
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
||||
assert {"musts", "broken"} <= columns
|
||||
conn.execute("PRAGMA user_version = 28")
|
||||
assert Store(path).migrations_applied == [29, 30, 31, 32, 33]
|
||||
for _ in range(2): # runs twice without harm
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [34, 35]
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert "must_text" not in {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
|
||||
assert not {"musts", "broken"} & {
|
||||
r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
||||
conn.execute("PRAGMA user_version = 33")
|
||||
assert migrated.agent(agent.id).like_text == "Teams." # type: ignore[union-attr]
|
||||
found = migrated.judgement(agent.id, item.uid)
|
||||
assert found is not None and (found.career, found.cv_score) == (9, cv_score(8, 7, 9))
|
||||
|
||||
|
||||
def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
|
||||
@@ -269,12 +280,12 @@ def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [30, 31, 32, 33, 34, 35]
|
||||
[data] = migrated.agents()
|
||||
assert (data.like_text, data.avoid_text) == ("", "Reporting.")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 29")
|
||||
assert Store(path).migrations_applied == [30, 31, 32, 33]
|
||||
assert Store(path).migrations_applied == [30, 31, 32, 33, 34, 35]
|
||||
|
||||
|
||||
def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
||||
@@ -290,8 +301,7 @@ def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
||||
old = store.save_agent("Old", "data leadership", cv_text="CV.")
|
||||
own = store.save_agent("Own", "data leadership")
|
||||
bare = store.save_agent("Bare", "data leadership")
|
||||
musts = [{"id": "M1", "text": "Remote", "verdict": "broken"}]
|
||||
assert store.set_judgement(old.id, item.uid, "key", 80, 70, 90, ["Good."], ["Bad."], musts)
|
||||
assert store.set_judgement(old.id, item.uid, "key", 80, 70, 90, ["Good."], ["Bad."])
|
||||
store.judge_missed([(own.id, item.uid)])
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("UPDATE agents SET limits = json_set(limits, '$.summarise_top', 50) "
|
||||
@@ -303,22 +313,16 @@ def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
||||
|
||||
for _ in range(2): # runs twice without harm
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [33]
|
||||
assert migrated.migrations_applied == [33, 34, 35]
|
||||
assert migrated.judgement(old.id, item.uid) is None
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT skills, experience, career, cv_score, strengths, gaps, musts, broken, "
|
||||
"SELECT skills, experience, career, cv_score, strengths, gaps, "
|
||||
"input_key, tries, judged_at FROM role_judgements").fetchall()
|
||||
assert rows == [(None, None, None, None, "[]", "[]", "[]", 0, "", 0, None)] * 2
|
||||
assert rows == [(None, None, None, None, "[]", "[]", "", 0, None)] * 2
|
||||
conn.execute("PRAGMA user_version = 32")
|
||||
assert {a.name: a.limits.summarise_top for a in migrated.agents()} == {
|
||||
"Old": 30, "Own": 12, "Bare": 30}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT limits FROM agents WHERE id = ?",
|
||||
(bare.id,)).fetchone() == ("{}",)
|
||||
|
||||
|
||||
def test_the_musts_text_is_one_must_have_per_line() -> None:
|
||||
assert must_lines("- A\n* B c\n\u2022 D\n1. E\n2) F\n \nG\n-H") == [
|
||||
"A", "B c", "D", "E", "F", "G", "-H"]
|
||||
assert must_lines("") == []
|
||||
@@ -282,7 +282,7 @@ def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Pat
|
||||
conn.execute("DROP TABLE agent_cv_vectors")
|
||||
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
|
||||
conn.execute("PRAGMA user_version = 30")
|
||||
assert Store(path).migrations_applied == [31, 32, 33]
|
||||
assert Store(path).migrations_applied == [31, 32, 33, 34, 35]
|
||||
step = {v: s for v, _, s in MIGRATIONS}[31]
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert callable(step)
|
||||
|
||||
+14
-2
@@ -239,14 +239,26 @@ def test_folding_needs_the_same_title_a_link_and_texts_that_can_be_one_job() ->
|
||||
assert fold([pc("1", "Berlin", a), pc("2", "Munich", "")]).placed == {"2"}
|
||||
|
||||
|
||||
def test_the_per_place_fold_keeps_to_one_country() -> None:
|
||||
"""A federal sales role posted in Canberra and in Washington has alike
|
||||
text and is two openings; Berlin and Munich are one job in two offices.
|
||||
A row with no country joins either."""
|
||||
text = "Sell to federal agencies, run the public sector pipeline and partners."
|
||||
got = fold([replace(pc("au", "Canberra", text), country="AU"),
|
||||
replace(pc("us", "Washington", text), country="US"),
|
||||
pc("b", "Berlin", text), pc("m", "Munich", text),
|
||||
replace(pc("n", "", text), country="")])
|
||||
assert got.dup_of == {"au": None, "us": None, "b": None, "m": "b", "n": "au"}
|
||||
|
||||
|
||||
def test_a_remote_listing_is_not_folded_into_an_office_one() -> None:
|
||||
"""The kept role takes a copy's places, not its remote, so the gate would
|
||||
read "Remote, Germany" as an office role in Vienna and reject it for an
|
||||
agent that takes remote roles. Two remote listings still fold."""
|
||||
text = "We build the payment platform for our bank customers in Kotlin."
|
||||
office = replace(pc("v", "Vienna", text), country="AT")
|
||||
remote = replace(pc("r", "Remote, Germany", text), remote=True)
|
||||
other = replace(pc("s", "Remote, Spain", text), country="ES", remote=True)
|
||||
remote = replace(pc("r", "Remote, Berlin", text), remote=True)
|
||||
other = replace(pc("s", "Remote, Hamburg", text), remote=True)
|
||||
got = fold([office, remote, other])
|
||||
assert got.dup_of == {"v": None, "r": None, "s": "r"}
|
||||
assert got.placed == {"s"}
|
||||
|
||||
+22
-19
@@ -103,10 +103,11 @@ def test_berlin_and_munich_of_one_host_fold_into_one_role_with_two_places(
|
||||
assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0)
|
||||
|
||||
|
||||
def test_a_place_copy_in_europe_meets_a_europe_gate_for_its_kept_role(store: Store) -> None:
|
||||
"""OpenAI posts one role in San Francisco and in Dublin. The kept role
|
||||
holds both places, so a Europe gate and the Place filter find it through
|
||||
Dublin, though `roles.place` and `roles.country` are San Francisco's."""
|
||||
def test_a_posting_per_office_in_two_countries_is_two_roles(store: Store) -> None:
|
||||
"""OpenAI posts one role in San Francisco and in Dublin. The fold keeps
|
||||
to one country (ADR 1.4), so each is a role of its own: a Europe gate
|
||||
takes Dublin and rejects San Francisco, and the Place filter finds
|
||||
Dublin alone."""
|
||||
from app import prep
|
||||
from app.models import Gate
|
||||
sf = role("sf", company="OpenAI", place="San Francisco", country="US", board="ashby")
|
||||
@@ -114,10 +115,11 @@ def test_a_place_copy_in_europe_meets_a_europe_gate_for_its_kept_role(store: Sto
|
||||
for item in (sf, dublin):
|
||||
store.upsert_role(item)
|
||||
prep.run(store)
|
||||
kept = kept_of(store, sf, dublin)
|
||||
assert dup_of(store) == {sf.uid: None, dublin.uid: None}
|
||||
agent = store.save_agent("A", gates=[Gate(field="country", values=["FR", "IE", "EU"])])
|
||||
assert {c.uid: c.gates for c in store.score_candidates(agent.id, agent.gates)} == {kept: [1]}
|
||||
assert {r.uid for r in store.roles(places=["country:IE"], by_score=False)} == {kept}
|
||||
assert {c.uid: c.gates for c in store.score_candidates(agent.id, agent.gates)} == {
|
||||
sf.uid: [0], dublin.uid: [1]}
|
||||
assert {r.uid for r in store.roles(places=["country:IE"], by_score=False)} == {dublin.uid}
|
||||
|
||||
|
||||
def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None:
|
||||
@@ -279,7 +281,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||
@@ -305,30 +307,31 @@ def test_switching_a_board_off_shows_its_roles_copies_at_once(store: Store) -> N
|
||||
|
||||
|
||||
def test_a_role_in_several_cities_leads_with_the_one_nearest_the_agent(store: Store) -> None:
|
||||
"""Mistral posts one role per city. For an agent kept to Europe the kept
|
||||
Montréal role leads with Paris, in the row and the reader, and opens the
|
||||
Paris copy; without an agent the host's order stands."""
|
||||
"""A Danish company posts one role per city. For an agent whose towns
|
||||
hold Copenhagen the kept Aarhus role leads with Copenhagen, in the row
|
||||
and the reader, and opens the Copenhagen copy; without an agent the
|
||||
host's order stands."""
|
||||
from app import prep
|
||||
from app.models import Gate
|
||||
items = [role(c, company="Mistral AI", place=p, board="ashby") for c, p in (
|
||||
("mtl", "Montréal"), ("nyc", "New York"), ("par", "Paris"))]
|
||||
items = [role(c, company="Danske", place=p, board="ashby") for c, p in (
|
||||
("aar", "Aarhus"), ("bil", "Billund"), ("cph", "Copenhagen"))]
|
||||
for item in items:
|
||||
store.upsert_role(item)
|
||||
prep.run(store)
|
||||
kept = kept_of(store, *items)
|
||||
agent = store.save_agent("A", gates=[
|
||||
Gate(field="country", values=["FR", "DK", "EU"], mode="require"),
|
||||
Gate(field="country", values=["DK"], mode="any")])
|
||||
Gate(field="place", values=["copenhagen"], mode="any")])
|
||||
|
||||
got = store.role(kept, agent.id)
|
||||
assert got is not None
|
||||
own = next(p.text for p in got.places if not p.url)
|
||||
assert own != "Paris"
|
||||
assert (got.lead_place, got.lead_url) == ("Paris", "https://jobs.example/par")
|
||||
assert got.places[0].text == "Paris"
|
||||
assert own != "Copenhagen"
|
||||
assert (got.lead_place, got.lead_url) == ("Copenhagen", "https://jobs.example/cph")
|
||||
assert got.places[0].text == "Copenhagen"
|
||||
(row,) = store.roles_page(agent=agent.id, by_score=False)
|
||||
assert (row.place, row.lead_place, row.lead_url) == ("Paris +2", "Paris", got.lead_url)
|
||||
assert row.place_texts[0] == "Paris" and len(row.place_texts) == 3
|
||||
assert (row.place, row.lead_place, row.lead_url) == ("Copenhagen +2", "Copenhagen", got.lead_url)
|
||||
assert row.place_texts[0] == "Copenhagen" and len(row.place_texts) == 3
|
||||
|
||||
plain = store.role(kept)
|
||||
assert plain is not None
|
||||
|
||||
@@ -520,7 +520,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
|
||||
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
|
||||
conn.execute("PRAGMA user_version = 22")
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
assert migrated.agents()[0].fetch_terms == []
|
||||
|
||||
|
||||
|
||||
+17
-53
@@ -14,7 +14,7 @@ import pytest
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs import judge, stages
|
||||
from app.models import CvLanguage, Prompt
|
||||
from app.models import CvLanguage, Prompt, Weights
|
||||
from app.storage import Store
|
||||
from desk import modeltask
|
||||
from desk.models import Item
|
||||
@@ -76,8 +76,7 @@ def config(tmp_path: Path) -> Config:
|
||||
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
|
||||
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
|
||||
cv_text=cv, like_text="Building teams.",
|
||||
avoid_text="Pure reporting.",
|
||||
must_text="A permanent contract.", **kwargs)
|
||||
avoid_text="Pure reporting.", **kwargs)
|
||||
|
||||
|
||||
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
|
||||
@@ -109,59 +108,18 @@ def test_a_run_writes_the_judgement_and_its_total(
|
||||
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
|
||||
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
|
||||
[system] = model.systems
|
||||
assert CV in system and "Pure reporting." in system and "A permanent contract." in system
|
||||
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
|
||||
assert CV in system and "Pure reporting." in system
|
||||
assert "- AI lead: Head of AI" in system
|
||||
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
|
||||
in system
|
||||
assert '<dislikes title="what they do not like">\nPure reporting.' in system
|
||||
assert "<avoid" not in system
|
||||
|
||||
|
||||
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
owner = hunter(store)
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
|
||||
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
|
||||
[uid] = ranked(store, owner, "1")
|
||||
|
||||
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [{**one, "career": 8, "musts": [
|
||||
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
|
||||
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
|
||||
for one in judged(items)]
|
||||
|
||||
model = stub(monkeypatch, reply)
|
||||
modeltask.run(judge.TASK, config, store)
|
||||
|
||||
[system] = model.systems
|
||||
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
|
||||
found = store.judgement(owner.id, uid)
|
||||
assert found is not None
|
||||
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
|
||||
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
|
||||
("M3", "Danish team", "unclear")]
|
||||
assert (found.career, found.career_capped, found.broken) == (8, 4, 1)
|
||||
assert found.cv_score == round(10 * (30 * 8 + 25 * 7 + 30 * 4) / 85, 1)
|
||||
|
||||
|
||||
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
|
||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
owner = hunter(store)
|
||||
[uid] = ranked(store, owner, "1")
|
||||
stub(monkeypatch) # answers without any `musts`
|
||||
modeltask.run(judge.TASK, config, store)
|
||||
found = store.judgement(owner.id, uid)
|
||||
assert found is not None
|
||||
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
|
||||
assert (found.career, found.career_capped, found.broken) == (9, 9, 0)
|
||||
|
||||
|
||||
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
|
||||
def test_the_schema_asks_for_three_numbers_strengths_and_gaps() -> None:
|
||||
item = judge.SCHEMA["properties"]["judgements"]["items"]
|
||||
assert "musts" in item["required"]
|
||||
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
|
||||
assert verdict["enum"] == ["met", "broken", "unclear"]
|
||||
assert "career 0-3" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
|
||||
assert item["required"] == ["key", "skills", "experience", "career", "strengths", "gaps"]
|
||||
assert "must-have" not in judge.SYSTEM
|
||||
assert "0 to 10" in judge.SYSTEM and "80-100" not in judge.SYSTEM
|
||||
|
||||
|
||||
@@ -190,7 +148,7 @@ def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) ->
|
||||
|
||||
def hashlib_old_key(agent: Any, text: str) -> str:
|
||||
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
||||
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
|
||||
agent.avoid_text, text], ensure_ascii=False)
|
||||
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
@@ -259,12 +217,18 @@ def test_a_judgement_is_read_again_only_when_what_it_read_changed(
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
|
||||
|
||||
# The languages are not part of what it read; the must text is.
|
||||
# The languages and the dislikes are part of what it read; the weights
|
||||
# are not.
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
cv_languages=[CvLanguage(language="da", level="native")])
|
||||
weights=Weights(cv_skills=50))
|
||||
assert judge.pending(store) == 0
|
||||
spoken = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
cv_languages=[CvLanguage(language="da", level="native")])
|
||||
assert judge.pending(store) == 1
|
||||
assert '<languages title="the languages they speak, and how well">\nDanish native' \
|
||||
in judge.system(spoken)
|
||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||
must_text="Remote two days a week.")
|
||||
avoid_text="Long commutes.")
|
||||
assert judge.pending(store) == 1
|
||||
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
|
||||
|
||||
|
||||
@@ -116,6 +116,16 @@ def test_roles_without_text_and_text_reads_that_failed() -> None:
|
||||
assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("dropped", "found"), [
|
||||
("3", ["3 listings dropped: no title or link"]),
|
||||
("1", ["1 listing dropped: no title or link"]),
|
||||
("", []), ("x", [])])
|
||||
def test_the_listings_the_last_read_dropped_are_named(dropped: str, found: list[str]) -> None:
|
||||
"""The rows the runner dropped (`_runner.valid`), after "read incomplete"."""
|
||||
assert words(fake(), state(dropped=dropped)) == found
|
||||
assert words(fake(), state(dropped=dropped, complete=""))[1:] == found
|
||||
|
||||
|
||||
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
|
||||
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
|
||||
assert words(fake(), state(), reach.SMALL) == []
|
||||
@@ -199,6 +209,17 @@ def test_companies_carry_their_limits_and_the_boards_own_open_roles(
|
||||
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
|
||||
|
||||
|
||||
def test_a_board_that_dropped_listings_says_so_on_its_row(
|
||||
client: TestClient, store: Store) -> None:
|
||||
store.add_subscription("ats", "lever:acme", company="Acme", board="lever")
|
||||
store.add_company("Acme", key="acme", ats="lever")
|
||||
store.state["ats:ok:lever:acme"] = "run"
|
||||
store.state["ats:complete:lever:acme"] = "1"
|
||||
store.state["ats:dropped:lever:acme"] = "3"
|
||||
[row] = client.get("/api/companies").json()
|
||||
assert row["limits"][0] == "3 listings dropped: no title or link"
|
||||
|
||||
|
||||
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
|
||||
store.add_company("Novo", key="novo", ats="taleo")
|
||||
[row] = client.get("/api/companies").json()
|
||||
|
||||
@@ -51,8 +51,8 @@ def _read(store: Store, target: str) -> tuple[Ask, str]:
|
||||
|
||||
|
||||
@pytest.mark.parametrize("countries, roles, line", [
|
||||
(False, False, "follows agents: DK, SE · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
||||
(False, True, "follows agents: DK, SE · remote not asked on Phenom · all roles"),
|
||||
(False, False, "follows agents' world (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
||||
(False, True, "follows agents' world (Europe) · remote not asked on Phenom · all roles"),
|
||||
(True, False, "all locations (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
||||
(True, True, "all locations (Europe) · remote not asked on Phenom · all roles"),
|
||||
], ids=["neither", "all roles alone", "all locations alone", "both"])
|
||||
@@ -62,11 +62,9 @@ def test_the_four_combinations(store: Store, countries: bool, roles: bool, line:
|
||||
store.set_all_roles("philips", roles)
|
||||
ask, said = _read(store, PHILIPS)
|
||||
assert said == line
|
||||
# Phenom asks for countries, many in one request, and not for remote.
|
||||
if countries:
|
||||
assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30
|
||||
else:
|
||||
assert ask.countries == ("DK", "SE")
|
||||
# Phenom asks for countries, many in one request, and not for remote: so
|
||||
# the agents want remote, it is asked for their world either way.
|
||||
assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30
|
||||
# Terms go to every big board, and never to a company set to All roles.
|
||||
assert ask.terms == (() if roles else ("data", "engineer", "product"))
|
||||
|
||||
@@ -122,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
|
||||
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
|
||||
conn.execute("PRAGMA user_version = 23")
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
|
||||
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
|
||||
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
|
||||
|
||||
+26
-2
@@ -224,7 +224,31 @@ def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
|
||||
("senior", "lead", "onsite")
|
||||
|
||||
|
||||
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
||||
def test_remote_in_the_text_is_marked_and_a_denial_is_not() -> None:
|
||||
yes = ("Fully-remote within the EU, we meet twice a year.",
|
||||
"Work from home or from our Berlin office, as you like.",
|
||||
"Dette er en stilling med mulighed for fjernarbejde.")
|
||||
no = ("No remote work: this role is on site in Munich.",
|
||||
"Remote sensing analyst for our satellite team.",
|
||||
"A great office in Copenhagen with free lunch.")
|
||||
assert all(words.remote_in_text("Engineer", text) for text in yes)
|
||||
assert not any(words.remote_in_text("Engineer", text) for text in no)
|
||||
assert words.remote_in_text("Backend Developer, remote-first team", "")
|
||||
|
||||
|
||||
def test_prep_marks_a_role_whose_text_says_remote(store: Store) -> None:
|
||||
said = item("1", title="Engineer", place="Berlin")
|
||||
said.body = "Fully remote within Germany, with a Berlin office."
|
||||
silent = item("2", title="Engineer", place="Berlin")
|
||||
silent.body = "On site in Berlin."
|
||||
store.upsert_role(said)
|
||||
store.upsert_role(silent)
|
||||
prep.run(store, tables=TABLES)
|
||||
assert (row(store, "1")["remote"], row(store, "1")["remote_text"]) == ("", 1)
|
||||
assert (row(store, "2")["remote"], row(store, "2")["remote_text"]) == ("", 0)
|
||||
|
||||
|
||||
|
||||
store.add_company("Ørsted")
|
||||
store.upsert_role(item("1", company="ORSTED A/S"))
|
||||
assert row(store, "1")["company_key"] != "ørsted"
|
||||
@@ -316,7 +340,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
one = row(migrated, "1")
|
||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||
|
||||
@@ -250,6 +250,44 @@ def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
|
||||
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
|
||||
|
||||
|
||||
def _aged(store: Store, source_id: str) -> None:
|
||||
"""Make a role look unlisted for longer than `STALE_DAYS`."""
|
||||
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(timespec="seconds")
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = ?", (old, source_id))
|
||||
|
||||
|
||||
def test_an_incomplete_read_closes_what_no_read_has_listed_for_stale_days(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A board past its ceiling or page cap is never read complete; a role
|
||||
it has not listed for three weeks still closes (ADR 1.3)."""
|
||||
_answer(monkeypatch, ([_row("1"), _row("2"), _row("3")], True))
|
||||
_pull(acme)
|
||||
_aged(acme, "workable:acme:3")
|
||||
_answer(monkeypatch, ([_row("1")], False))
|
||||
result = _pull(acme)
|
||||
# 2 was listed two pulls ago and stays; 3 has been unlisted for 22 days.
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||
assert (result["incomplete"], result["closed"]) == (1, 1)
|
||||
|
||||
|
||||
def test_a_filtered_read_closes_a_spared_role_once_it_is_stale(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A filtered read spares a role acted on from its list; after three
|
||||
weeks unlisted the role closes by age like any other."""
|
||||
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
||||
_pull(acme)
|
||||
uid = {r.source_id: r.uid for r in acme.roles()}
|
||||
agent = acme.save_agent("Hunt")
|
||||
acme.set_vote(agent.id, uid["workable:acme:2"], 1)
|
||||
_answer(monkeypatch, ([_row("1")], True), filtered=True)
|
||||
assert _pull(acme)["closed"] == 0
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||
_aged(acme, "workable:acme:2")
|
||||
assert _pull(acme)["closed"] == 1
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
|
||||
|
||||
def test_a_failing_board_keeps_its_roles_before_stale_days(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, adzuna # noqa: E402
|
||||
from plugins.readers import Ask, _runner, adzuna # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
|
||||
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
|
||||
@@ -39,8 +39,9 @@ def test_it_declares_the_key_and_states_the_robots_finding() -> None:
|
||||
|
||||
|
||||
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
|
||||
first, predicted = _rows() # javascript: refused
|
||||
first, predicted, refused = _rows()
|
||||
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
|
||||
assert refused["url"].startswith("javascript:") and not _runner.valid(refused)
|
||||
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
||||
assert first["teaser"] == "Build our backend services in Python & Go."
|
||||
assert "description" not in first
|
||||
@@ -89,3 +90,12 @@ def test_saved_params_need_a_known_country() -> None:
|
||||
"q": "x", "country": "gb"}
|
||||
with pytest.raises(ValueError, match="country"):
|
||||
adzuna.clean_params({"q": "x", "country": "dk"})
|
||||
|
||||
|
||||
def test_the_runner_drops_the_ad_without_a_web_link_and_counts_it() -> None:
|
||||
"""The reader hands the `javascript:` ad over; the runner drops it, and
|
||||
the read is complete, since all three result ids were read."""
|
||||
listing = _runner.read_listing(adzuna, "", lambda url, **kw: BODY, ASK,
|
||||
limits=_runner.Limits(page_delay=0))
|
||||
assert listing.ids == ["4412345678", "4412345679"]
|
||||
assert (listing.handed, listing.dropped, listing.complete) == (3, 1, True)
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, freehire # noqa: E402
|
||||
from plugins.readers import Ask, _runner, freehire # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
|
||||
|
||||
@@ -119,14 +119,17 @@ def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None:
|
||||
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
|
||||
|
||||
|
||||
def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None:
|
||||
def test_a_hit_without_a_title_an_id_or_a_web_link_is_handed_to_the_runner() -> None:
|
||||
"""The reader hands every hit over; the runner keeps only the whole one."""
|
||||
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
|
||||
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
|
||||
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
|
||||
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
|
||||
found = freehire.rows({"data": hits}, "")
|
||||
assert [row["source_id"] for row in found] == ["d"]
|
||||
assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == []
|
||||
assert [row["source_id"] for row in found] == ["a", "", "c", "d"]
|
||||
kept = [row for row in found if _runner.valid(row)]
|
||||
assert [row["source_id"] for row in kept] == ["d"]
|
||||
assert kept[0]["url"] == "https://x.example/d" and kept[0]["places"] == []
|
||||
|
||||
|
||||
def test_a_title_and_a_company_have_their_entities_decoded() -> None:
|
||||
|
||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, jobbank # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jobbank # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
|
||||
@@ -170,7 +170,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
||||
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
|
||||
assert len(results) == 100 and nxt == 2
|
||||
assert len(jobbank.rows(decoded, "")) == 99
|
||||
rows = jobbank.rows(decoded, "")
|
||||
assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
|
||||
|
||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, jobindex # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jobindex # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
|
||||
|
||||
@@ -89,7 +89,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
||||
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
|
||||
assert len(results) == 20 and nxt == 2
|
||||
assert len(jobindex.rows(decoded, "")) == 19
|
||||
rows = jobindex.rows(decoded, "")
|
||||
assert len(rows) == 20 and sum(map(_runner.valid, rows)) == 19
|
||||
|
||||
|
||||
def test_an_answer_without_a_channel_raises() -> None:
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, jobnet # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jobnet # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
|
||||
@@ -100,9 +100,12 @@ def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None:
|
||||
assert "applicationDeadline" not in row
|
||||
|
||||
|
||||
def test_an_ad_without_an_id_or_a_title_is_skipped() -> None:
|
||||
def test_an_ad_without_an_id_or_a_title_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
ads = json.loads(OTHER)["jobAds"]
|
||||
assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == []
|
||||
rows = jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "")
|
||||
assert [(row["source_id"] == "", row["title"] == "") for row in rows] == [
|
||||
(False, True), (True, False)]
|
||||
assert not any(map(_runner.valid, rows))
|
||||
|
||||
|
||||
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, platsbanken # noqa: E402
|
||||
from plugins.readers import Ask, _runner, platsbanken # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
|
||||
|
||||
@@ -47,8 +47,9 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
|
||||
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||
rows = platsbanken.rows(decoded, "")
|
||||
assert total == 32
|
||||
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
||||
first, no_town, bare = rows
|
||||
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400001", "31400002"]
|
||||
assert not _runner.valid(rows[2]) # javascript:, dropped by the runner
|
||||
first, no_town, _refused, bare = rows
|
||||
assert [p["text"] for p in first["places"]] == [
|
||||
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
|
||||
assert first["location"] == "Lund"
|
||||
@@ -58,7 +59,7 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
|
||||
|
||||
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
||||
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||
first, _no_town, remote = platsbanken.rows(decoded, "")
|
||||
first, _no_town, _refused, remote = platsbanken.rows(decoded, "")
|
||||
assert first["title"] == "Postdoctoral Researcher"
|
||||
assert first["company"] == "European Spallation Source Eric"
|
||||
assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
|
||||
|
||||
@@ -534,7 +534,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
assert migrated.gazetteer().anywhere is not None
|
||||
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
|
||||
"WHERE uid = ?", (board,))
|
||||
@@ -544,7 +544,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
|
||||
assert area(migrated, search.uid) == ("", "")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 26")
|
||||
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32, 33]
|
||||
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
|
||||
|
||||
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
|
||||
|
||||
+107
-2
@@ -1,7 +1,7 @@
|
||||
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
||||
subscription resolves to, series and their union, whole and paged listings,
|
||||
completeness from the declaration, the sweep across pulls, the 10% guard,
|
||||
the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||
completeness from the declaration, the rows kept and dropped, the sweep
|
||||
across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||
network: `get` is a dict of URL -> body or exception."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
||||
assert filtered.ids == ["1"] and filtered.filtered
|
||||
|
||||
|
||||
# --------------------------------------------------------- the rows kept
|
||||
|
||||
def _handed(i: str) -> dict[str, Any]:
|
||||
"""A row as a reader hands it over: whole, or, for an id of `-id`,
|
||||
`-title`, `-link` or `-url`, without that value or with a link that is
|
||||
no web URL (`javascript:`, or one httpx cannot read)."""
|
||||
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
|
||||
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
|
||||
return _row(i, **broken.get(i.split(":")[0], {}))
|
||||
|
||||
|
||||
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
|
||||
assert _runner.valid(_handed("1"))
|
||||
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
|
||||
assert not _runner.valid({"source_id": "1", "title": "Role"})
|
||||
|
||||
|
||||
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
module = _whole()
|
||||
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||
web = Web({"https://jobs.fake.example/acme/list":
|
||||
b'["1", "-id", "2", "-title", "-link", "-url"]'})
|
||||
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||
listing = _runner.read_listing(module, "acme", web)
|
||||
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
assert (listing.handed, listing.dropped) == (6, 4)
|
||||
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
|
||||
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
|
||||
|
||||
|
||||
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""Completeness counts the result ids `page` returns, not the rows kept,
|
||||
so the dropped rows still count as read; the drop is logged once."""
|
||||
pages = [["1", "-title:a"], ["2", "-link:b"]]
|
||||
module = _paged(pages, total=4)
|
||||
module.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
assert (listing.handed, listing.dropped) == (4, 2)
|
||||
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
|
||||
|
||||
|
||||
def test_the_stamp_keeps_the_rows_dropped() -> None:
|
||||
state: dict[str, str] = {}
|
||||
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
|
||||
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
|
||||
assert state["ats:dropped:t"] == "2"
|
||||
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
|
||||
assert state["ats:dropped:t"] == ""
|
||||
|
||||
|
||||
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
|
||||
"""A whole board whose feed hands over `handed` rows, `dropped` of them
|
||||
without a title."""
|
||||
module = _whole()
|
||||
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
|
||||
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
|
||||
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
|
||||
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
|
||||
handed: int, dropped: int, fails: bool) -> None:
|
||||
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
|
||||
a page whose shape changed, which must close nothing."""
|
||||
module, web = _dropping(handed, dropped)
|
||||
if not fails:
|
||||
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
|
||||
return
|
||||
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
|
||||
_runner.read({}, "ats", "t", module, "acme", web)
|
||||
|
||||
|
||||
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
|
||||
module, web = _dropping(20, 5)
|
||||
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
|
||||
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
|
||||
listing = _runner.read(state, "ats", "t", module, "acme", web,
|
||||
limits=_runner.Limits(page_delay=0, stale_days=21))
|
||||
assert (len(listing.ids), listing.dropped) == (15, 5)
|
||||
|
||||
|
||||
# ------------------------------------------------------ through ATSSource
|
||||
|
||||
@pytest.fixture
|
||||
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
|
||||
assert state["ats:filtered:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
|
||||
board.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
|
||||
board.READS = board.READS.model_copy(update={"complete": "end"})
|
||||
state: dict[str, str] = {}
|
||||
assert len(list(_source(state).fetch())) == 19
|
||||
assert state["ats:dropped:fake:acme"] == "1"
|
||||
|
||||
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||
json.dumps([str(n) for n in range(15)]
|
||||
+ [f"-id:{n}" for n in range(5)]).encode()}
|
||||
state = {}
|
||||
assert list(_source(state).fetch()) == []
|
||||
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- raw
|
||||
|
||||
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
||||
|
||||
@@ -417,6 +417,30 @@ def test_a_region_that_holds_the_agents_places_is_no_evidence_against(
|
||||
assert oresund not in {r.uid for r in store.roles(places=["country:DK"])}
|
||||
|
||||
|
||||
def test_a_role_whose_text_says_remote_passes_a_gate_on_remote_until_extract_reads_it(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""Berlin is evidence against Denmark, but a text that says remote while
|
||||
no field does makes the role possibly remote (`remote_text`, prep): it
|
||||
meets the remote member. Once extract says onsite, it fails again."""
|
||||
said = role(store, "1", place="Berlin, Germany", remote="")
|
||||
silent = role(store, "2", place="Berlin, Germany", remote="")
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET remote_text = 1 WHERE uid = ?", (said,))
|
||||
agent = store.save_agent("DK or remote", "", [
|
||||
Gate(field="country", values=["DK"], mode="any"),
|
||||
Gate(field="remote", values=["remote"], mode="any")])
|
||||
|
||||
run(config, store, score.run_score)
|
||||
scores = store.scores(agent.id)
|
||||
assert scores[said][1] == ""
|
||||
assert scores[silent] == (0.0, "none of country DK, remote remote")
|
||||
|
||||
with store.transaction() as conn:
|
||||
conn.execute("UPDATE roles SET remote = 'onsite' WHERE uid = ?", (said,))
|
||||
run(config, store, score.run_score)
|
||||
assert store.scores(agent.id)[said] == (0.0, "none of country DK, remote remote")
|
||||
|
||||
|
||||
def test_a_wider_place_beside_a_place_elsewhere_is_no_match(
|
||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||
"""Europe alone is no evidence against Denmark; `London; Europe` is in
|
||||
|
||||
@@ -383,6 +383,23 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch)
|
||||
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
||||
|
||||
|
||||
def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The search driver reads through the same listing code as a board: the
|
||||
runner drops the row the provider handed over without a title, and the
|
||||
read is still complete, by the distinct result ids."""
|
||||
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
||||
module = readers.PROVIDERS["fake"]
|
||||
handed = module.rows
|
||||
module.rows = lambda decoded, slug: [
|
||||
row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)]
|
||||
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
|
||||
"params": {"q": "x"}}]})
|
||||
assert [i.source_id for i in source.fetch()] == ["fake:h1"]
|
||||
assert source.state["search:complete:fake:cph"] == "1"
|
||||
assert source.state["search:dropped:fake:cph"] == "1"
|
||||
|
||||
|
||||
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||
|
||||
+14
-14
@@ -71,7 +71,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 33
|
||||
assert LATEST == 35
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -169,8 +169,8 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
|
||||
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
||||
for table in ("vectors", "role_tags", "agent_companies", "role_votes"):
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
for column in ("musts", "broken"): # v29
|
||||
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
|
||||
# The must-have columns v28 and v29 added and v34 drops are not in a
|
||||
# fresh file either.
|
||||
conn.execute("ALTER TABLE agents DROP COLUMN like_text") # v30
|
||||
conn.execute("PRAGMA user_version = 1")
|
||||
|
||||
@@ -191,7 +191,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||
@@ -396,7 +396,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||
Prompt(name="Backend", text="python backend")]
|
||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||
@@ -420,7 +420,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||
@@ -449,7 +449,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||
"SELECT uid, place_fold, seniority FROM roles")}
|
||||
@@ -691,7 +691,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||
assert migrated.role_places(stated.uid) == []
|
||||
@@ -733,7 +733,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
with sqlite3.connect(path) as conn:
|
||||
found = {sid: rest for sid, *rest in conn.execute(
|
||||
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
|
||||
@@ -751,12 +751,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
role = migrated.role(posting("a").uid)
|
||||
assert role is not None and role.posting_url == ""
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 25") # the column is there already
|
||||
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33]
|
||||
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||
|
||||
|
||||
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
|
||||
@@ -1855,7 +1855,7 @@ def test_v32_resets_the_limits_drops_prefer_gates_and_renames_enrich(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [32, 33]
|
||||
assert migrated.migrations_applied == [32, 33, 34, 35]
|
||||
by_name = {a.name: a for a in migrated.agents()}
|
||||
assert by_name["Wolf"].limits == Limits() == by_name["Bare"].limits
|
||||
assert [(g.field, g.mode) for g in by_name["Wolf"].gates] == [
|
||||
@@ -1880,14 +1880,14 @@ def test_v32_runs_twice_and_keeps_limits_set_since(tmp_path: Path) -> None:
|
||||
store = Store(path)
|
||||
agent = store.save_agent("Wolf", limits=Limits(cut_fit_top=900))
|
||||
_v31(path)
|
||||
assert Store(path).migrations_applied == [32, 33]
|
||||
assert Store(path).migrations_applied == [32, 33, 34, 35]
|
||||
assert Store(path).agent(agent.id).limits == Limits() # type: ignore[union-attr]
|
||||
narrow = Limits(cut_fit_top=900, summarise_top=10)
|
||||
store.save_agent("Wolf", agent_id=agent.id, limits=narrow)
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("PRAGMA user_version = 31") # the columns are there already
|
||||
again = Store(path)
|
||||
assert again.migrations_applied == [32, 33]
|
||||
assert again.migrations_applied == [32, 33, 34, 35]
|
||||
assert again.agent(agent.id).limits == narrow # type: ignore[union-attr]
|
||||
|
||||
|
||||
|
||||
+29
-1
@@ -998,7 +998,13 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
|
||||
template = page[page.index('<template id="agent-form-template">'):page.index("</template>")]
|
||||
assert 'id="agent-form"' in template and 'id="a-cancel"' in template
|
||||
assert 'id="a-name"' in page and 'placeholder="My search"' in page
|
||||
assert 'id="a-prompts"' in page and "<details" in page and 'id="a-gates"' in page
|
||||
assert 'id="a-prompts"' in page
|
||||
# The gates are the four parts and nothing else: no rows of other rules,
|
||||
# only a note naming a stored rule the parts cannot say.
|
||||
for gone in ("<details", 'id="a-gates"', 'id="a-rules"', 'id="a-add-rule"', "Other rules"):
|
||||
assert gone not in template
|
||||
assert template.index('id="a-where-sum"') < template.index('id="a-kept"') \
|
||||
< template.index("</fieldset>", template.index('id="a-where-sum"'))
|
||||
for weight in ("a-fit", "a-recency", "a-affinity", "a-taste"):
|
||||
assert f'id="{weight}"' in page
|
||||
assert 'id="a-taste" min="0" step="0.05" value="0.2"' in page
|
||||
@@ -1170,6 +1176,28 @@ def test_agents_are_created_updated_listed_and_deleted(client: TestClient) -> No
|
||||
assert client.delete(f"/api/agents/{agent['id']}").status_code == 404
|
||||
|
||||
|
||||
def test_an_agents_fetch_gate_and_a_rule_the_form_cannot_say_round_trip(client: TestClient) -> None:
|
||||
"""What the form sends for the four parts, plus a stored gate on a field the
|
||||
form has no part for, is served back as sent, so a load and a save keep it."""
|
||||
europe = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
|
||||
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS",
|
||||
"LI", "NO", "GB", "CH", "EU"]
|
||||
gates = [{"field": "country", "values": europe, "mode": "require"},
|
||||
{"field": "country", "values": ["DK"], "mode": "any"},
|
||||
{"field": "remote_from", "values": ["country:DK", "anywhere"], "mode": "any"},
|
||||
{"field": "place", "values": ["Malmö"], "mode": "any"},
|
||||
{"field": "title", "values": ["account executive"], "mode": "exclude"},
|
||||
{"field": "seniority", "values": ["senior"], "mode": "require"}]
|
||||
body = {"name": "Nordics", "prompts": [{"name": "Data", "text": "Head of Data"}],
|
||||
"gates": gates, "fetch_terms": ["data"]}
|
||||
agent = client.post("/api/agents", json=body).json()
|
||||
assert agent["gates"] == gates and agent["fetch_terms"] == ["data"]
|
||||
served = next(a for a in client.get("/api/agents").json() if a["id"] == agent["id"])
|
||||
assert served["gates"] == gates
|
||||
again = client.post("/api/agents", json={**body, "id": agent["id"], "gates": served["gates"]})
|
||||
assert again.status_code == 200 and again.json()["gates"] == gates
|
||||
|
||||
|
||||
def test_agents_take_and_serve_prompts_and_still_take_an_intent(client: TestClient) -> None:
|
||||
prompts = [{"name": "AI and data leadership", "text": "VP, Head or Director of AI"},
|
||||
{"name": "Forward deployed", "text": ""}]
|
||||
|
||||
Reference in new issue
Block a user