Compare commits
16
Commits
e61185e4b9
...
920457d326
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
920457d326 | ||
|
|
2262cda8f1 | ||
|
|
1913101061 | ||
|
|
849adb197d | ||
|
|
136047023e | ||
|
|
911dfe8584 | ||
|
|
8b875b377b | ||
|
|
c6730062fd | ||
|
|
49929372c7 | ||
|
|
f141cffc3c | ||
|
|
02ea191b53 | ||
|
|
1976f6a353 | ||
|
|
2159c7925f | ||
|
|
0c4f3c9bbe | ||
|
|
38c31c830b | ||
|
|
4a9bcb2b1c |
No files matched your search
+13
-3
@@ -111,6 +111,12 @@ def same_posting(a: Copy, b: Copy) -> bool:
|
|||||||
return not ta or not tb or bool(ta & tb)
|
return not ta or not tb or bool(ta & tb)
|
||||||
|
|
||||||
|
|
||||||
|
def _same_country(a: Copy, b: Copy) -> bool:
|
||||||
|
"""Whether two rows can be in one country: equal, or either unknown."""
|
||||||
|
ca, cb = _country(a), _country(b)
|
||||||
|
return not ca or not cb or ca == cb
|
||||||
|
|
||||||
|
|
||||||
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
||||||
"""The sort key that picks the kept copy of a group, smallest first: a
|
"""The sort key that picks the kept copy of a group, smallest first: a
|
||||||
board over a search provider, then one with a description, then the
|
board over a search provider, then one with a description, then the
|
||||||
@@ -193,12 +199,16 @@ def fold(rows: Iterable[Copy]) -> Folded:
|
|||||||
|
|
||||||
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
||||||
again, in `keep_order`: a row joins the first group whose members are all
|
again, in `keep_order`: a row joins the first group whose members are all
|
||||||
of its host, `similar` to it and remote as it is or not. Only a row with
|
of its host, in its country (equal, or either unknown, as `same_posting`
|
||||||
|
reads it), `similar` to it and remote as it is or not. Only a row with
|
||||||
a link can be folded, as its link is what the kept role keeps. Every row
|
a link can be folded, as its link is what the kept role keeps. Every row
|
||||||
of a merged group points at its first row, the rows that pointed at a
|
of a merged group points at its first row, the rows that pointed at a
|
||||||
folded one included.
|
folded one included.
|
||||||
|
|
||||||
A remote listing and an office listing stay two roles: the kept role
|
One country, as ADR Fetch prep and score 1.4 has it: a job posted once
|
||||||
|
per office in several countries is several roles, one per country, which
|
||||||
|
keeps a different opening with an alike text (a federal sales role in
|
||||||
|
the US and one in Australia) from merging. A remote listing and an office listing stay two roles: the kept role
|
||||||
takes a copy's places and not its remote, so the fetch gate, which reads
|
takes a copy's places and not its remote, so the fetch gate, which reads
|
||||||
a remote role by its area, would reject the remote listing with an
|
a remote role by its area, would reject the remote listing with an
|
||||||
office role elsewhere, and the office one with a remote role elsewhere."""
|
office role elsewhere, and the office one with a remote role elsewhere."""
|
||||||
@@ -216,7 +226,7 @@ def fold(rows: Iterable[Copy]) -> Folded:
|
|||||||
for row in sorted(part, key=keep_order):
|
for row in sorted(part, key=keep_order):
|
||||||
for group in groups:
|
for group in groups:
|
||||||
if all(m.host == row.host and m.remote == row.remote
|
if all(m.host == row.host and m.remote == row.remote
|
||||||
and similar(m.text, row.text) for m in group):
|
and _same_country(m, row) and similar(m.text, row.text) for m in group):
|
||||||
group.append(row)
|
group.append(row)
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
|
|||||||
+25
-70
@@ -7,18 +7,14 @@ to three strengths and gaps. A model's rating is an estimate, so the scale is
|
|||||||
coarse. The total is `app.models.cv_score`, the three numbers under the
|
coarse. The total is `app.models.cv_score`, the three numbers under the
|
||||||
agent's `cv_*` weights times 10, so 0 to 100, worked out when a role is served
|
agent's `cv_*` weights times 10, so 0 to 100, worked out when a role is served
|
||||||
and not stored, so the weights are no part of `input_key`; it is what the
|
and not stored, so the weights are no part of `input_key`; it is what the
|
||||||
feed's CV match sorts by. The model also reports a verdict on each of the
|
feed's CV match sorts by. Culture and location are left out: the first is
|
||||||
agent's must-haves (one per line of its must text, numbered M1..Mn in the
|
guessed from ad text, the second is what the fetch gate does.
|
||||||
prompt); it does not apply them. The rule is code's: a "broken" one makes
|
|
||||||
career count as at most `app.models.BROKEN_CAP` (4), when the role is
|
|
||||||
served. Culture and location are left out: the first is guessed from ad text,
|
|
||||||
the second is what the fetch gate does.
|
|
||||||
|
|
||||||
The work is pairs, `(agent, role)`: an enabled agent with a CV, with or
|
The work is pairs, `(agent, role)`: an enabled agent with a CV, with or
|
||||||
without a prompt, and an open role in its cut (`scores.cut`, see `app.jobs.score`: the best by
|
without a prompt, and an open role in its cut (`scores.cut`, see `app.jobs.score`: the best by
|
||||||
fit, the best by CV closeness that those leave out, and the liked and saved)
|
fit, the best by CV closeness that those leave out, and the liked and saved)
|
||||||
with some text, that has no judgement yet, or one read from another CV,
|
with some text, that has no judgement yet, or one read from another CV,
|
||||||
prompts, like, avoid or must text or role text than today's (`input_key`).
|
prompts, languages, like or avoid text or role text than today's (`input_key`).
|
||||||
The cut is the only selection: the pull judges it after extract and the
|
The cut is the only selection: the pull judges it after extract and the
|
||||||
rescore that follows, so it reads the cut as recomputed. A pair is named by the composite key `"<agent_id>:<uid>"`,
|
rescore that follows, so it reads the cut as recomputed. A pair is named by the composite key `"<agent_id>:<uid>"`,
|
||||||
which is what `desk.modeltask` hands round as a uid, so `desk/` is untouched.
|
which is what `desk.modeltask` hands round as a uid, so `desk/` is untouched.
|
||||||
@@ -51,9 +47,7 @@ from pipeline import NULL, RunContext, Task, job
|
|||||||
from pipeline.task import Batch
|
from pipeline.task import Batch
|
||||||
from app.config import Config
|
from app.config import Config
|
||||||
from app.jobs import stages
|
from app.jobs import stages
|
||||||
from app.models import (
|
from app.models import LANGUAGE_LEVELS, Agent, CvLanguage, _language_name, cv_languages
|
||||||
LANGUAGE_LEVELS, MUST_VERDICTS, Agent, CvLanguage, cv_languages, must_lines,
|
|
||||||
)
|
|
||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
|
|
||||||
#: How much of a posting goes in one item: summarise's 8000, since the
|
#: How much of a posting goes in one item: summarise's 8000, since the
|
||||||
@@ -66,22 +60,28 @@ EXCERPT = 8000
|
|||||||
#: 3: what the person likes (`like_text`) beside what they do not, and the
|
#: 3: what the person likes (`like_text`) beside what they do not, and the
|
||||||
#: career rubric reading both.
|
#: career rubric reading both.
|
||||||
#: 4: the three numbers 0 to 10 rather than 0 to 100.
|
#: 4: the three numbers 0 to 10 rather than 0 to 100.
|
||||||
FORMAT = 4
|
#: 5: the person's languages, read into skills.
|
||||||
|
#: 6: no must-haves: the profile is the CV, languages, likes and dislikes.
|
||||||
|
FORMAT = 6
|
||||||
|
|
||||||
#: How many strengths and gaps are kept, and how long each may be.
|
#: How many strengths and gaps are kept, and how long each may be.
|
||||||
MOST, LONGEST = 3, 300
|
MOST, LONGEST = 3, 300
|
||||||
|
|
||||||
SYSTEM = """You judge job postings for one person: how well each fits their CV and what they want.
|
SYSTEM = """You judge job postings for one person: how well each fits their CV and what they want.
|
||||||
|
|
||||||
Below the rules is the person: their CV, what they want, what they like and
|
Below the rules is the person: their CV, the languages they speak and how
|
||||||
what they do not like (the work, the workplace, situations), and their
|
well, what they want, what they like and what they do not like (the work,
|
||||||
must-haves and deal-breakers when they gave any. Each item is one posting with a key, its title, company, place
|
the workplace, situations). Each item is one posting with a key, its title,
|
||||||
and text. Answer one judgement per item.
|
company, place and text. Answer one judgement per item.
|
||||||
|
|
||||||
Score three dimensions, each a whole number from 0 to 10:
|
Score three dimensions, each a whole number from 0 to 10:
|
||||||
- skills: 8-10 the core requirements are the person's primary skills; 6-7
|
- skills: 8-10 the core requirements are the person's primary skills; 6-7
|
||||||
most requirements match, with 1-2 gaps that are learnable; 4-5 a partial
|
most requirements match, with 1-2 gaps that are learnable; 4-5 a partial
|
||||||
match, significant upskilling needed; 0-3 a fundamental mismatch.
|
match, significant upskilling needed; 0-3 a fundamental mismatch. A
|
||||||
|
language the posting requires counts as a requirement: one the person
|
||||||
|
speaks below the level asked, or not at all when their languages are
|
||||||
|
listed, lowers skills and is a gap. A language the posting does not
|
||||||
|
mention changes nothing.
|
||||||
- experience: match on the function and nature of the work performed, not the
|
- experience: match on the function and nature of the work performed, not the
|
||||||
literal job title, and on its level. 8-10 direct experience in the same
|
literal job title, and on its level. 8-10 direct experience in the same
|
||||||
domain and role type; 6-7 related, with transferable skills clear; 4-5
|
domain and role type; 6-7 related, with transferable skills clear; 4-5
|
||||||
@@ -93,16 +93,6 @@ Score three dimensions, each a whole number from 0 to 10:
|
|||||||
whether the tasks and the setting would energise them, not only whether
|
whether the tasks and the setting would energise them, not only whether
|
||||||
they could do them.
|
they could do them.
|
||||||
|
|
||||||
Their must-haves and deal-breakers are numbered M1, M2 and so on. Give a
|
|
||||||
verdict on every one of them, in `musts`, for each item:
|
|
||||||
- "met": the posting text says it is met.
|
|
||||||
- "broken": only when the posting text clearly contradicts it.
|
|
||||||
- "unclear": the text does not say. Never infer from silence: a posting that
|
|
||||||
is silent on a must-have is "unclear", not "broken".
|
|
||||||
Judge a must-have by its own words, not by how well the role fits otherwise.
|
|
||||||
Answer musts as [] when there are none. The verdicts do not change the three
|
|
||||||
scores, which follow their rubrics; name a broken must-have as a gap.
|
|
||||||
|
|
||||||
Then 1 to 3 strengths and 1 to 3 gaps, each one short sentence grounded in
|
Then 1 to 3 strengths and 1 to 3 gaps, each one short sentence grounded in
|
||||||
the posting text. State the gaps honestly: a prestigious role that fits
|
the posting text. State the gaps honestly: a prestigious role that fits
|
||||||
poorly scores low.
|
poorly scores low.
|
||||||
@@ -126,21 +116,8 @@ SCHEMA: dict[str, Any] = {
|
|||||||
"career": {"type": "integer"},
|
"career": {"type": "integer"},
|
||||||
"strengths": {"type": "array", "items": {"type": "string"}},
|
"strengths": {"type": "array", "items": {"type": "string"}},
|
||||||
"gaps": {"type": "array", "items": {"type": "string"}},
|
"gaps": {"type": "array", "items": {"type": "string"}},
|
||||||
"musts": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"id": {"type": "string"},
|
|
||||||
"verdict": {"type": "string", "enum": list(MUST_VERDICTS)},
|
|
||||||
},
|
|
||||||
"required": ["id", "verdict"],
|
|
||||||
"additionalProperties": False,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"required": ["key", "skills", "experience", "career", "strengths", "gaps",
|
"required": ["key", "skills", "experience", "career", "strengths", "gaps"],
|
||||||
"musts"],
|
|
||||||
"additionalProperties": False,
|
"additionalProperties": False,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -181,10 +158,11 @@ def _text(row: Any) -> str:
|
|||||||
|
|
||||||
def input_key(agent: Agent, text: str) -> str:
|
def input_key(agent: Agent, text: str) -> str:
|
||||||
"""A hash of everything a judgement reads: the CV, the prompts, the
|
"""A hash of everything a judgement reads: the CV, the prompts, the
|
||||||
like, avoid and must texts, and the role's text as sent. A judgement stored
|
languages, the like and avoid texts, and the role's text as sent. A judgement
|
||||||
under another key is read again."""
|
stored under another key is read again."""
|
||||||
blob = json.dumps([FORMAT, agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
blob = json.dumps([FORMAT, agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
||||||
agent.like_text, agent.avoid_text, agent.must_text, text], ensure_ascii=False)
|
[[l.language, l.level] for l in agent.cv_languages],
|
||||||
|
agent.like_text, agent.avoid_text, text], ensure_ascii=False)
|
||||||
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
@@ -230,6 +208,9 @@ def pending(store: Store) -> int:
|
|||||||
def system(agent: Agent) -> str:
|
def system(agent: Agent) -> str:
|
||||||
"""The rubric, and the person as this agent describes them."""
|
"""The rubric, and the person as this agent describes them."""
|
||||||
parts = [SYSTEM, "", "The person.", "", f"<cv>\n{agent.cv_text.strip()}\n</cv>"]
|
parts = [SYSTEM, "", "The person.", "", f"<cv>\n{agent.cv_text.strip()}\n</cv>"]
|
||||||
|
if agent.cv_languages:
|
||||||
|
spoken = ", ".join(f"{_language_name(l.language)} {l.level}" for l in agent.cv_languages)
|
||||||
|
parts.append(f"\n<languages title=\"the languages they speak, and how well\">\n{spoken}\n</languages>")
|
||||||
wants = "\n".join(f"- {p.name}: {' '.join(p.text.split())}"
|
wants = "\n".join(f"- {p.name}: {' '.join(p.text.split())}"
|
||||||
for p in agent.prompts if p.text.strip())
|
for p in agent.prompts if p.text.strip())
|
||||||
if wants:
|
if wants:
|
||||||
@@ -240,9 +221,6 @@ def system(agent: Agent) -> str:
|
|||||||
if agent.avoid_text.strip():
|
if agent.avoid_text.strip():
|
||||||
parts.append("\n<dislikes title=\"what they do not like\">\n"
|
parts.append("\n<dislikes title=\"what they do not like\">\n"
|
||||||
f"{agent.avoid_text.strip()}\n</dislikes>")
|
f"{agent.avoid_text.strip()}\n</dislikes>")
|
||||||
if musts := must_lines(agent.must_text):
|
|
||||||
lines = "\n".join(f"M{n}: {line}" for n, line in enumerate(musts, start=1))
|
|
||||||
parts.append(f"\n<must title=\"must-haves and deal-breakers\">\n{lines}\n</must>")
|
|
||||||
return "\n".join(parts)
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
@@ -290,25 +268,6 @@ def _lines(value: Any) -> list[str]:
|
|||||||
return [line for line in found if line][:MOST]
|
return [line for line in found if line][:MOST]
|
||||||
|
|
||||||
|
|
||||||
def _musts(agent: Agent | None, value: Any) -> list[dict[str, str]]:
|
|
||||||
"""The verdicts on the agent's must-haves as they are stored: one
|
|
||||||
`{id, text, verdict}` per line of its must text, in order. A must-have
|
|
||||||
the model left out, or gave a verdict outside "met", "broken", "unclear",
|
|
||||||
is "unclear": nothing is broken by silence. An id that is not one of the
|
|
||||||
agent's is ignored, the first answer for an id counts."""
|
|
||||||
said: dict[str, str] = {}
|
|
||||||
for one in value if isinstance(value, list) else []:
|
|
||||||
if isinstance(one, dict):
|
|
||||||
said.setdefault(str(one.get("id") or "").strip().upper(),
|
|
||||||
str(one.get("verdict") or "").strip().lower())
|
|
||||||
found = []
|
|
||||||
for n, line in enumerate(must_lines(agent.must_text if agent else ""), start=1):
|
|
||||||
verdict = said.get(f"M{n}", "")
|
|
||||||
found.append({"id": f"M{n}", "text": line,
|
|
||||||
"verdict": verdict if verdict in MUST_VERDICTS else "unclear"})
|
|
||||||
return found
|
|
||||||
|
|
||||||
|
|
||||||
def apply_batch(store: Store, answer: dict[str, Any],
|
def apply_batch(store: Store, answer: dict[str, Any],
|
||||||
batch: dict[str, Any] | None = None,
|
batch: dict[str, Any] | None = None,
|
||||||
config: Config | None = None) -> JudgeResult:
|
config: Config | None = None) -> JudgeResult:
|
||||||
@@ -317,7 +276,6 @@ def apply_batch(store: Store, answer: dict[str, Any],
|
|||||||
all three numbers; the pair stays waiting."""
|
all three numbers; the pair stays waiting."""
|
||||||
keys: dict[str, str] = dict((batch or {}).get("keys") or {})
|
keys: dict[str, str] = dict((batch or {}).get("keys") or {})
|
||||||
stats = JudgeResult()
|
stats = JudgeResult()
|
||||||
agents: dict[int, Agent | None] = {}
|
|
||||||
for one in answer.get("judgements") or []:
|
for one in answer.get("judgements") or []:
|
||||||
key = str(one.get("key") or "")
|
key = str(one.get("key") or "")
|
||||||
pair = pair_of(key)
|
pair = pair_of(key)
|
||||||
@@ -326,11 +284,8 @@ def apply_batch(store: Store, answer: dict[str, Any],
|
|||||||
stats.rejected.append(key or "?")
|
stats.rejected.append(key or "?")
|
||||||
continue
|
continue
|
||||||
skills, experience, career = (int(m) for m in marks if m is not None)
|
skills, experience, career = (int(m) for m in marks if m is not None)
|
||||||
if pair[0] not in agents:
|
|
||||||
agents[pair[0]] = store.agent(pair[0])
|
|
||||||
if not store.set_judgement(pair[0], pair[1], keys.pop(key), skills, experience, career,
|
if not store.set_judgement(pair[0], pair[1], keys.pop(key), skills, experience, career,
|
||||||
_lines(one.get("strengths")), _lines(one.get("gaps")),
|
_lines(one.get("strengths")), _lines(one.get("gaps"))):
|
||||||
_musts(agents[pair[0]], one.get("musts"))):
|
|
||||||
stats.rejected.append(key)
|
stats.rejected.append(key)
|
||||||
continue
|
continue
|
||||||
stats.judged += 1
|
stats.judged += 1
|
||||||
|
|||||||
+17
-3
@@ -10,7 +10,10 @@ both; nothing here decides how a board is read, so the words never disagree
|
|||||||
with the pull as long as they read the keys the pull writes.
|
with the pull as long as they read the keys the pull writes.
|
||||||
|
|
||||||
A word with a number carries it after a colon ("without text: 12"), and the
|
A word with a number carries it after a colon ("without text: 12"), and the
|
||||||
filter matches the phrase before the colon (`WORDS`).
|
filter matches the phrase before the colon (`WORDS`). One word is outside
|
||||||
|
`WORDS`: the listings the last read dropped for want of an id, a title or a
|
||||||
|
link ("3 listings dropped: no title or link", `_runner.valid`), shown on the
|
||||||
|
board's row and not offered by the filter.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -49,7 +52,7 @@ WORDS: dict[str, str] = {
|
|||||||
|
|
||||||
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
||||||
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
||||||
"narrowed", "detailfailed")
|
"narrowed", "detailfailed", "dropped")
|
||||||
|
|
||||||
|
|
||||||
class Texts(NamedTuple):
|
class Texts(NamedTuple):
|
||||||
@@ -84,7 +87,8 @@ def texts(store: Store) -> dict[int, Texts]:
|
|||||||
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
||||||
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
||||||
line: str = "", counts: Texts | None = None) -> list[str]:
|
line: str = "", counts: Texts | None = None) -> list[str]:
|
||||||
"""The limits of one board, in the order of `WORDS`, those that apply.
|
"""The limits of one board, in the order of `WORDS`, those that apply, the
|
||||||
|
rows the last read dropped after "read incomplete".
|
||||||
|
|
||||||
`state` holds the board's keys (`state_of`), `size` its kept size,
|
`state` holds the board's keys (`state_of`), `size` its kept size,
|
||||||
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
||||||
@@ -105,6 +109,8 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
|
|||||||
out.append("failing")
|
out.append("failing")
|
||||||
if state.get(key("ok")) and not state.get(key("complete")):
|
if state.get(key("ok")) and not state.get(key("complete")):
|
||||||
out.append("read incomplete")
|
out.append("read incomplete")
|
||||||
|
if dropped := _count(state.get(key("dropped"))):
|
||||||
|
out.append(f"{dropped} listing{'' if dropped == 1 else 's'} dropped: no title or link")
|
||||||
if state.get(key("swept")) == "1" or key("sweep") in state:
|
if state.get(key("swept")) == "1" or key("sweep") in state:
|
||||||
out.append("swept")
|
out.append("swept")
|
||||||
if state.get(key("filtered")) == "1":
|
if state.get(key("filtered")) == "1":
|
||||||
@@ -130,6 +136,14 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _count(value: str | None) -> int:
|
||||||
|
"""A number kept in state, 0 for none or one that does not parse."""
|
||||||
|
try:
|
||||||
|
return max(0, int(value or 0))
|
||||||
|
except ValueError:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
||||||
"""Whether the reach leaves the board unread by place: its reader cannot
|
"""Whether the reach leaves the board unread by place: its reader cannot
|
||||||
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
||||||
|
|||||||
+21
-14
@@ -37,7 +37,12 @@ not swept clears them.
|
|||||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||||
the roles matching a word, which says nothing of the roles it did not list,
|
the roles matching a word, which says nothing of the roles it did not list,
|
||||||
so it closes by list nothing: a role of such a board closes once the board
|
so it closes by list nothing: a role of such a board closes once the board
|
||||||
has not listed it for `STALE_DAYS`, the rule searches have.
|
has not listed it for `STALE_DAYS`, the rule searches have. The same age
|
||||||
|
rule is the floor under every board (ADR Fetch prep and score 1.3): an
|
||||||
|
incomplete read (a ceiling, a page cap without a sweep) and a filtered read,
|
||||||
|
which spares the roles acted on and the remote ones it could not ask for,
|
||||||
|
close by age what no read has listed for `STALE_DAYS` (`_close_aged`), so no
|
||||||
|
role stays open for good.
|
||||||
|
|
||||||
A search never closes by its list, even one stamped complete because it
|
A search never closes by its list, even one stamped complete because it
|
||||||
read as many results as its provider stated: a search is never a company's
|
read as many results as its provider stated: a search is never a company's
|
||||||
@@ -365,7 +370,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
incomplete += 1
|
incomplete += 1
|
||||||
if not store.state.get(f"ats:sweep:{sub.target}"):
|
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||||
partial.append(sub.target)
|
partial.append(sub.target)
|
||||||
continue
|
# An incomplete read closes nothing by its list; what no
|
||||||
|
# read has listed for STALE_DAYS still goes.
|
||||||
|
read = _close_aged(store, source, sub.id, json.loads(
|
||||||
|
store.state.get(f"{source}:ids:{sub.target}", "[]")), stale)
|
||||||
shut = read
|
shut = read
|
||||||
closed += shut
|
closed += shut
|
||||||
before = open_before.get(sub.id, 0)
|
before = open_before.get(sub.id, 0)
|
||||||
@@ -394,8 +402,9 @@ def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
|||||||
clears that record and closes what it did not list, all of it unfiltered
|
clears that record and closes what it did not list, all of it unfiltered
|
||||||
(`Store.mark_closed`), filtered sparing the roles acted on
|
(`Store.mark_closed`), filtered sparing the roles acted on
|
||||||
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
||||||
(`remote_unasked`) spares the remote ones as well, and closes those by
|
(`remote_unasked`) spares the remote ones as well. What a filtered read
|
||||||
age instead."""
|
spares closes by age (`_close_aged`), as on an incomplete read
|
||||||
|
(`close_boards`)."""
|
||||||
state, target = store.state, sub.target
|
state, target = store.state, sub.target
|
||||||
if not state.get(f"{source}:complete:{target}", ""):
|
if not state.get(f"{source}:complete:{target}", ""):
|
||||||
return None
|
return None
|
||||||
@@ -404,8 +413,7 @@ def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
|||||||
return _close_aged(store, source, sub.id, seen_ids, stale)
|
return _close_aged(store, source, sub.id, seen_ids, stale)
|
||||||
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
||||||
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
||||||
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
aged = _close_aged(store, source, sub.id, seen_ids, stale) if is_filtered else 0
|
||||||
if spare_remote else 0)
|
|
||||||
if state.get(f"{source}:swept:{target}", ""):
|
if state.get(f"{source}:swept:{target}", ""):
|
||||||
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
||||||
spare_remote=spare_remote)
|
spare_remote=spare_remote)
|
||||||
@@ -484,20 +492,19 @@ def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription
|
|||||||
|
|
||||||
|
|
||||||
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
||||||
stale: str, *, remote_only: bool = False) -> int:
|
stale: str) -> int:
|
||||||
"""Close the open roles of one board that its read did not list and that
|
"""Close the open roles of one board that its read did not list and that
|
||||||
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
||||||
writes on every listing): the rule for what a read cannot speak for. A
|
writes on every listing): the rule for what a read cannot speak for, a
|
||||||
read narrowed by terms closes all such roles, as a search does; with
|
read narrowed by terms, an incomplete one, or what a filtered read
|
||||||
`remote_only`, a filtered read that could not ask for remote closes only
|
spares. A role acted on is not spared here: three weeks unlisted is the
|
||||||
its remote roles so, sparing those acted on as its list does."""
|
one evidence a filtered or incomplete read gives."""
|
||||||
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
return conn.execute(
|
return conn.execute(
|
||||||
f"""UPDATE roles SET closed_at = ?
|
"""UPDATE roles SET closed_at = ?
|
||||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
AND last_seen < ?
|
AND last_seen < ?
|
||||||
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
||||||
(store.now(), source, subscription_id, stale,
|
(store.now(), source, subscription_id, stale,
|
||||||
json.dumps(list(seen_ids)))).rowcount
|
json.dumps(list(seen_ids)))).rowcount
|
||||||
|
|
||||||
|
|||||||
+39
-8
@@ -283,9 +283,18 @@ class Plan:
|
|||||||
return self._big(sub)
|
return self._big(sub)
|
||||||
|
|
||||||
def _big(self, sub: Subscription) -> tuple[Reach, str]:
|
def _big(self, sub: Subscription) -> tuple[Reach, str]:
|
||||||
"""The reach of `sub` as a board of `SMALL` roles or more."""
|
"""The reach of `sub` as a board of `SMALL` roles or more. A host
|
||||||
|
that cannot filter on remote while the agents want remote roles is
|
||||||
|
asked for the wide reach, the agents' world (ADR Fetch prep and
|
||||||
|
score 1.2): asked for "DK, SE" alone it would never list a remote
|
||||||
|
role filed under Germany, and the gate sorts the rest out. Only
|
||||||
|
where the host takes the world in one read, a list of countries or
|
||||||
|
their ids: on a host that takes one country a request, Europe is
|
||||||
|
the whole board, so the location read stays and remote is not asked."""
|
||||||
if company_key(sub.company or sub.label) in self.all_countries:
|
if company_key(sub.company or sub.label) in self.all_countries:
|
||||||
return self.wide, "countries"
|
return self.wide, "countries"
|
||||||
|
if self.base.remote and not _filters_remote(sub) and _takes_world(sub, self.wide):
|
||||||
|
return self.wide, "world"
|
||||||
return self.base, "agents"
|
return self.base, "agents"
|
||||||
|
|
||||||
def narrows_roles(self, sub: Subscription) -> bool:
|
def narrows_roles(self, sub: Subscription) -> bool:
|
||||||
@@ -305,15 +314,34 @@ class Plan:
|
|||||||
if why == "unknown":
|
if why == "unknown":
|
||||||
found, why = self._big(sub)
|
found, why = self._big(sub)
|
||||||
out = found.params()
|
out = found.params()
|
||||||
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
|
if self.terms and why in ("countries", "agents", "world") and self.narrows_roles(sub):
|
||||||
out = (out or {}) | {"terms": list(self.terms)}
|
out = (out or {}) | {"terms": list(self.terms)}
|
||||||
if why == "countries":
|
if why in ("countries", "world"):
|
||||||
out["wide"] = True
|
out["wide"] = True
|
||||||
if sub.target not in self.sizes:
|
if sub.target not in self.sizes:
|
||||||
out = (out or {}) | {"unsized": True, "small": SMALL}
|
out = (out or {}) | {"unsized": True, "small": SMALL}
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _filters_remote(sub: Subscription) -> bool:
|
||||||
|
"""Whether `sub`'s host declares a remote filter at all. A host that
|
||||||
|
declares one by ids may still lack the facet on a site, which the
|
||||||
|
runner finds out when it asks (`_runner.ask_for`)."""
|
||||||
|
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||||
|
return module is None or module.READS.filters.remote != "none"
|
||||||
|
|
||||||
|
|
||||||
|
def _takes_world(sub: Subscription, wide: Reach) -> bool:
|
||||||
|
"""Whether `sub`'s host can be asked for the wide reach in one read: a
|
||||||
|
country filter that takes a list (`many`, `ids`), or one value a request
|
||||||
|
with the countries within `MAX_SERIES`; never everywhere."""
|
||||||
|
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||||
|
if module is None or wide.everywhere or not wide.countries:
|
||||||
|
return False
|
||||||
|
reach = module.READS.filters.country
|
||||||
|
return reach in ("many", "ids") or (reach == "one" and len(wide.countries) <= _runner.MAX_SERIES)
|
||||||
|
|
||||||
|
|
||||||
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
||||||
"""Each board's size as kept (`ats:size:<target>`); a board never sized
|
"""Each board's size as kept (`ats:size:<target>`); a board never sized
|
||||||
has none."""
|
has none."""
|
||||||
@@ -381,7 +409,7 @@ def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str
|
|||||||
#: Why a board is read whole, as the Companies list says it.
|
#: Why a board is read whole, as the Companies list says it.
|
||||||
_WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
|
_WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
|
||||||
"override": "set here", "countries": "all locations (everywhere)",
|
"override": "set here", "countries": "all locations (everywhere)",
|
||||||
"agents": "an agent takes any place"}
|
"world": "the agents' world (everywhere)", "agents": "an agent takes any place"}
|
||||||
|
|
||||||
|
|
||||||
def _codes(codes: Iterable[str]) -> list[str]:
|
def _codes(codes: Iterable[str]) -> list[str]:
|
||||||
@@ -486,6 +514,9 @@ def line(found: Plan, sub: Subscription) -> str:
|
|||||||
if why == "countries":
|
if why == "countries":
|
||||||
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
||||||
return f"all locations ({named}){unasked} · {roles}"
|
return f"all locations ({named}){unasked} · {roles}"
|
||||||
|
if why == "world":
|
||||||
|
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
||||||
|
return f"follows agents' world ({named}){unasked} · {roles}"
|
||||||
if why == "override":
|
if why == "override":
|
||||||
return f"set here: {_places(ask)}{unasked} · {roles}"
|
return f"set here: {_places(ask)}{unasked} · {roles}"
|
||||||
return f"follows agents: {_places(ask)}{unasked} · {roles}"
|
return f"follows agents: {_places(ask)}{unasked} · {roles}"
|
||||||
@@ -494,14 +525,14 @@ def line(found: Plan, sub: Subscription) -> str:
|
|||||||
what = "all locations (everywhere)" if why == "countries" else _WHOLE[why]
|
what = "all locations (everywhere)" if why == "countries" else _WHOLE[why]
|
||||||
return f"{what} · {roles}"
|
return f"{what} · {roles}"
|
||||||
return f"all places, too many countries to ask · {roles}"
|
return f"all places, too many countries to ask · {roles}"
|
||||||
if why in ("agents", "countries") and given.everywhere:
|
if why in ("agents", "countries", "world") and given.everywhere:
|
||||||
return f"{_WHOLE[why]}: read whole"
|
return f"{_WHOLE[why]}: read whole"
|
||||||
if why in ("agents", "countries", "override") and not module.READS.filters.declared():
|
if why in ("agents", "countries", "world", "override") and not module.READS.filters.declared():
|
||||||
return f"{module.LABEL} cannot filter: read whole"
|
return f"{module.LABEL} cannot filter: read whole"
|
||||||
reads = module.READS.filters
|
reads = module.READS.filters
|
||||||
if why in ("agents", "countries") and reads.country == "one" \
|
if why in ("agents", "countries", "world") and reads.country == "one" \
|
||||||
and len(given.countries) > _runner.MAX_SERIES:
|
and len(given.countries) > _runner.MAX_SERIES:
|
||||||
return "too many countries to ask: read whole"
|
return "too many countries to ask: read whole"
|
||||||
if why in ("agents", "countries"):
|
if why in ("agents", "countries", "world"):
|
||||||
return f"{module.LABEL} cannot ask for these: read whole"
|
return f"{module.LABEL} cannot ask for these: read whole"
|
||||||
return f"{_WHOLE[why]}: read whole"
|
return f"{_WHOLE[why]}: read whole"
|
||||||
+9
-62
@@ -1008,79 +1008,30 @@ def language_flag(required: list[RoleLanguage],
|
|||||||
return "", ""
|
return "", ""
|
||||||
|
|
||||||
|
|
||||||
#: What career counts as at most, on the judge's 0 to 10, when a posting
|
|
||||||
#: breaks one of the agent's must-haves: the foot of the rubric's 4-5 band,
|
|
||||||
#: a decent job that does not build toward the person's goals. The model
|
|
||||||
#: reports whether each must-have is met; this cap is applied in code, when a
|
|
||||||
#: role is served (`cv_score`, `Store.judgement`, `store._cv_sql`).
|
|
||||||
BROKEN_CAP = 4
|
|
||||||
|
|
||||||
#: A must-have's verdict, as the judge answers it.
|
|
||||||
MUST_VERDICTS = ("met", "broken", "unclear")
|
|
||||||
|
|
||||||
_BULLET = re.compile(r"^\s*(?:[-*\u2022]+|\(?\d+[.)])\s+")
|
|
||||||
|
|
||||||
|
|
||||||
def must_lines(text: str) -> list[str]:
|
|
||||||
"""The agent's must-haves text read as one must-have per line: a leading
|
|
||||||
bullet (`-`, `*`, `•`) or number (`1.`, `2)`) is dropped, so are empty
|
|
||||||
lines."""
|
|
||||||
found = []
|
|
||||||
for line in text.splitlines():
|
|
||||||
line = " ".join(_BULLET.sub("", line, count=1).split())
|
|
||||||
if line:
|
|
||||||
found.append(line)
|
|
||||||
return found
|
|
||||||
|
|
||||||
|
|
||||||
def capped_career(career: int, broken: int = 0) -> int:
|
|
||||||
"""`career` as it counts: at most `BROKEN_CAP` when any must-have is
|
|
||||||
broken."""
|
|
||||||
return min(career, BROKEN_CAP) if broken else career
|
|
||||||
|
|
||||||
|
|
||||||
def cv_score(skills: int, experience: int, career: int,
|
def cv_score(skills: int, experience: int, career: int,
|
||||||
weights: Weights | None = None, broken: int = 0) -> float | None:
|
weights: Weights | None = None) -> float | None:
|
||||||
"""The one number of a CV match, 0 to 100 for sorting and display: the
|
"""The one number of a CV match, 0 to 100 for sorting and display: the
|
||||||
judge's three numbers, each 0 to 10, averaged by the agent's
|
judge's three numbers, each 0 to 10, averaged by the agent's
|
||||||
`cv_skills`, `cv_experience` and `cv_career` weights (30, 25 and 30 by
|
`cv_skills`, `cv_experience` and `cv_career` weights (30, 25 and 30 by
|
||||||
default), career as `capped_career` has it, times 10. None when all
|
default), times 10. None when all three weights are 0."""
|
||||||
three weights are 0."""
|
|
||||||
w = weights or Weights()
|
w = weights or Weights()
|
||||||
total = w.cv_skills + w.cv_experience + w.cv_career
|
total = w.cv_skills + w.cv_experience + w.cv_career
|
||||||
if not total:
|
if not total:
|
||||||
return None
|
return None
|
||||||
career = capped_career(career, broken)
|
|
||||||
return round(10 * (w.cv_skills * skills + w.cv_experience * experience
|
return round(10 * (w.cv_skills * skills + w.cv_experience * experience
|
||||||
+ w.cv_career * career) / total, 1)
|
+ w.cv_career * career) / total, 1)
|
||||||
|
|
||||||
|
|
||||||
class Must(Model):
|
|
||||||
"""One must-have of an agent as the judge read it against a role: its
|
|
||||||
`id` (`M1`, by line), the line's `text` and the `verdict`."""
|
|
||||||
|
|
||||||
id: str
|
|
||||||
text: str = ""
|
|
||||||
verdict: Literal["met", "broken", "unclear"] = "unclear"
|
|
||||||
|
|
||||||
|
|
||||||
class Judgement(Model):
|
class Judgement(Model):
|
||||||
"""What `judge` read of one role for one agent (`role_judgements`): the
|
"""What `judge` read of one role for one agent (`role_judgements`): the
|
||||||
three dimensions 0 to 10 as the model gave them, `cv_score`
|
three dimensions 0 to 10 as the model gave them, `cv_score`
|
||||||
(`cv_score`, 0 to 100) and 1 to 3 strengths and gaps grounded in the
|
(`cv_score`, 0 to 100) and 1 to 3 strengths and gaps grounded in the
|
||||||
posting.
|
posting. `cv_score` is not read from the table: `Store.judgement` sets
|
||||||
`musts` is the verdict on each must-have and `broken` how many are
|
it from the agent's weights; it is None when all three are 0."""
|
||||||
broken; any makes `career_capped` the lower of `career` and
|
|
||||||
`BROKEN_CAP`. `cv_score` and `career_capped` are not read from the
|
|
||||||
table: `Store.judgement` sets them from the agent's weights; `cv_score`
|
|
||||||
is None when all three are 0."""
|
|
||||||
|
|
||||||
skills: int
|
skills: int
|
||||||
experience: int
|
experience: int
|
||||||
career: int
|
career: int
|
||||||
career_capped: int | None = None
|
|
||||||
broken: int = 0
|
|
||||||
musts: Annotated[list[Must], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
|
||||||
cv_score: float | None = None
|
cv_score: float | None = None
|
||||||
strengths: JsonStrings = []
|
strengths: JsonStrings = []
|
||||||
gaps: JsonStrings = []
|
gaps: JsonStrings = []
|
||||||
@@ -1574,14 +1525,11 @@ class Agent(Model):
|
|||||||
are the union over the enabled agents, and an agent with none means no
|
are the union over the enabled agents, and an agent with none means no
|
||||||
board is narrowed by words. They decide what is fetched, not what ranks.
|
board is narrowed by words. They decide what is fetched, not what ranks.
|
||||||
|
|
||||||
`cv_text`, `cv_languages`, `like_text`, `avoid_text` and `must_text` are
|
`cv_text`, `cv_languages`, `like_text` and `avoid_text` are what `judge`
|
||||||
what `judge` reads a role against: the CV, the languages it states with
|
reads a role against: the CV, the languages it states with a level, and
|
||||||
a level, what the person likes and does not like (the work, the
|
what the person likes and does not like (the work, the workplace,
|
||||||
workplace, situations; both feed the career score), and their
|
situations; both feed the career score). An agent without a CV is not
|
||||||
must-haves and deal-breakers (set through the API only, not on the
|
judged, and one without languages flags none (`language_flag`)."""
|
||||||
form; soft, read by the model only; a hard rule is a gate). An agent without
|
|
||||||
a CV is not judged, and one without languages flags none
|
|
||||||
(`language_flag`)."""
|
|
||||||
|
|
||||||
id: int
|
id: int
|
||||||
name: str
|
name: str
|
||||||
@@ -1594,7 +1542,6 @@ class Agent(Model):
|
|||||||
cv_languages: CvLanguages = []
|
cv_languages: CvLanguages = []
|
||||||
like_text: str = ""
|
like_text: str = ""
|
||||||
avoid_text: str = ""
|
avoid_text: str = ""
|
||||||
must_text: str = ""
|
|
||||||
enabled: bool = True
|
enabled: bool = True
|
||||||
created_at: str
|
created_at: str
|
||||||
|
|
||||||
|
|||||||
+6
-2
@@ -37,8 +37,9 @@ from app.storage import Store
|
|||||||
#: Raised when the rules in this package change the way a stored word maps,
|
#: Raised when the rules in this package change the way a stored word maps,
|
||||||
#: so every open role is mapped again as it is after a table changes. 3: the
|
#: so every open role is mapped again as it is after a table changes. 3: the
|
||||||
#: remote area, and remote as the model's before the host's word. 4: the
|
#: remote area, and remote as the model's before the host's word. 4: the
|
||||||
#: deadline, the host's before the model's.
|
#: deadline, the host's before the model's. 5: `remote_text`, the title or
|
||||||
VERSION = 4
|
#: text saying remote while no field does.
|
||||||
|
VERSION = 5
|
||||||
|
|
||||||
#: The state key the last run's `fingerprint` is kept under.
|
#: The state key the last run's `fingerprint` is kept under.
|
||||||
STATE = "prep:tables"
|
STATE = "prep:tables"
|
||||||
@@ -61,6 +62,7 @@ def fingerprint(tables: Tables = TABLES) -> str:
|
|||||||
"""A hash of every host table, the shared remote words, the gazetteer the
|
"""A hash of every host table, the shared remote words, the gazetteer the
|
||||||
code describes and `VERSION`."""
|
code describes and `VERSION`."""
|
||||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
||||||
|
"remote_text": [words.REMOTE_TEXT, words.REMOTE_TEXT_NOT],
|
||||||
"places": repr(places.seed())}, sort_keys=True)
|
"places": repr(places.seed())}, sort_keys=True)
|
||||||
return hashlib.sha1(blob.encode()).hexdigest()
|
return hashlib.sha1(blob.encode()).hexdigest()
|
||||||
|
|
||||||
@@ -114,6 +116,7 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
|||||||
"seniority_title": titled,
|
"seniority_title": titled,
|
||||||
"employment": words.employment(host, row["employment_host"], tables),
|
"employment": words.employment(host, row["employment_host"], tables),
|
||||||
"remote": remote,
|
"remote": remote,
|
||||||
|
"remote_text": int(words.remote_in_text(row["title"], row["text"] or "")),
|
||||||
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
||||||
if remote == "remote" else ()),
|
if remote == "remote" else ()),
|
||||||
"country": country(gaz, place_ids, row["country_host"]),
|
"country": country(gaz, place_ids, row["country_host"]),
|
||||||
@@ -125,6 +128,7 @@ def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
|||||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||||
or found["remote"] != row["remote"] \
|
or found["remote"] != row["remote"] \
|
||||||
|
or found["remote_text"] != row["remote_text"] \
|
||||||
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
||||||
or (found["country"] and found["country"] != row["country"]) \
|
or (found["country"] and found["country"] != row["country"]) \
|
||||||
or found["deadline"] != row["deadline"] \
|
or found["deadline"] != row["deadline"] \
|
||||||
|
|||||||
@@ -58,6 +58,39 @@ _NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\
|
|||||||
re.IGNORECASE)
|
re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
#: Phrases in a title or text that say the role is worked remotely, and the
|
||||||
|
#: phrases that take one back. Both are read on the lowercased text with its
|
||||||
|
#: hyphens and runs of spaces squashed to one space. A match marks the role as
|
||||||
|
#: possibly remote (`remote_in_text`); it decides nothing by itself.
|
||||||
|
REMOTE_TEXT: tuple[str, ...] = (
|
||||||
|
"fully remote", "100% remote", "100 % remote", "remote first", "remote work",
|
||||||
|
"remote working", "work remotely", "working remotely", "remote position", "remote role",
|
||||||
|
"remote job", "remote team", "remote friendly", "remote possible", "remote option",
|
||||||
|
"remote ok", "work from home", "working from home", "work from anywhere", "home based",
|
||||||
|
"home office", "homeoffice", "hjemmearbejde", "fjernarbejde", "distansarbete",
|
||||||
|
)
|
||||||
|
REMOTE_TEXT_NOT: tuple[str, ...] = (
|
||||||
|
"no remote", "not remote", "not a remote", "non remote", "not be remote", "cannot be remote",
|
||||||
|
"no home office", "kein homeoffice", "not available remotely", "not open to remote",
|
||||||
|
"remote work is not", "remote working is not", "is not a remote", "remote sensing",
|
||||||
|
"remote control", "remote monitoring",
|
||||||
|
)
|
||||||
|
|
||||||
|
_REMOTE_TEXT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT))
|
||||||
|
_REMOTE_TEXT_NOT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT_NOT))
|
||||||
|
_SQUASHED = re.compile(r"[\s-]+")
|
||||||
|
|
||||||
|
|
||||||
|
def remote_in_text(title: str, text: str) -> bool:
|
||||||
|
"""Whether the title or the text says the role is worked remotely: a
|
||||||
|
phrase of `REMOTE_TEXT` left once those of `REMOTE_TEXT_NOT` are taken
|
||||||
|
out. Cheap evidence, no model: a role marked so passes a location gate
|
||||||
|
on remote until extract has read it (ADR Fetch prep and score 1.1), so a
|
||||||
|
false mark costs a cut slot at most, and a missed one nothing new."""
|
||||||
|
folded = _SQUASHED.sub(" ", f"{title or ''} {text or ''}".lower())
|
||||||
|
return bool(_REMOTE_TEXT.search(_REMOTE_TEXT_NOT.sub(" ", folded)))
|
||||||
|
|
||||||
|
|
||||||
def _line_word(text: str) -> str:
|
def _line_word(text: str) -> str:
|
||||||
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
||||||
if found := remote_of(text):
|
if found := remote_of(text):
|
||||||
|
|||||||
+55
-87
@@ -9,13 +9,15 @@
|
|||||||
* ADR's order: the world (the scope select), the location (a short list of
|
* ADR's order: the world (the scope select), the location (a short list of
|
||||||
* lines a role passes by matching any one of: countries, towns, remote), the
|
* lines a role passes by matching any one of: countries, towns, remote), the
|
||||||
* role exclusion (the titles to leave out) and the role fetch (the fetch
|
* role exclusion (the titles to leave out) and the role fetch (the fetch
|
||||||
* terms), over a row per remaining rule under Advanced; a gate the four parts
|
* terms), and nothing else: seniority, pay and the like are filters on Roles.
|
||||||
* cannot say lands in a row, so a load and a save lose nothing. A gate is a
|
* A gate is a field, values and a mode (must, one of, not), with no weight.
|
||||||
* field, values and a mode (must, one of, not), with no weight. After it come
|
* A stored gate the four parts cannot say, which only the config or the API
|
||||||
* the analytics filter (the best N by prompt closeness and the best M by CV
|
* writes, is named in a note under the fetch gate and sent back as it came,
|
||||||
* closeness, each with a minimum level), the want feed score weights (prompt
|
* so a load and a save lose nothing. After the fetch gate come the analytics
|
||||||
* closeness, newness, company like, job like), the CV match weights and the
|
* filter (the best N by prompt closeness and the best M by CV closeness, each
|
||||||
* size of summarise. Edit turns a card
|
* with a minimum level), the want feed score weights (prompt closeness,
|
||||||
|
* newness, company like, job like), the CV match weights and the size of
|
||||||
|
* summarise. Edit turns a card
|
||||||
* into the form from the page's `<template>`, in place, and New agent puts a
|
* into the form from the page's `<template>`, in place, and New agent puts a
|
||||||
* blank card on top in the same state. One card edits at a time, so the
|
* blank card on top in the same state. One card edits at a time, so the
|
||||||
* form's ids stay unique. Votes, liked companies and scores are the agent's,
|
* form's ids stay unique. Votes, liked companies and scores are the agent's,
|
||||||
@@ -30,10 +32,7 @@
|
|||||||
* text (typed, or read from a .md/.txt file by the browser), the languages it
|
* text (typed, or read from a .md/.txt file by the browser), the languages it
|
||||||
* states as rows of a code and a level, and the two soft texts, what the
|
* states as rows of a code and a level, and the two soft texts, what the
|
||||||
* person likes and what they do not like (the work, the workplace,
|
* person likes and what they do not like (the work, the workplace,
|
||||||
* situations), which the judge reads for the career score. The must-have text
|
* situations), which the judge reads for the career score.
|
||||||
* is no longer on the form: a save leaves `must_text` out, so the stored one
|
|
||||||
* is kept (it is set through the API, and still caps career at 4 in code
|
|
||||||
* when the judge reports one broken).
|
|
||||||
* Every save sends the CV, languages and the two texts, and the card's switch
|
* Every save sends the CV, languages and the two texts, and the card's switch
|
||||||
* re-posts the row as the server gave it, so neither drops them.
|
* re-posts the row as the server gave it, so neither drops them.
|
||||||
*
|
*
|
||||||
@@ -64,9 +63,8 @@
|
|||||||
const NEW = "new";
|
const NEW = "new";
|
||||||
// Europe as the starter agents spell it (static/remote.js).
|
// Europe as the starter agents spell it (static/remote.js).
|
||||||
const EUROPE = rh.EUROPE;
|
const EUROPE = rh.EUROPE;
|
||||||
const FIELDS = ["place", "country", "remote", "remote_from", "seniority", "function", "salary_min",
|
// A gate's mode in the words of the note on the gates the form keeps.
|
||||||
"currency", "permit", "languages", "company", "title"];
|
const MODES = { require: "must", any: "one of", exclude: "not" };
|
||||||
const MODES = [["require", "must"], ["any", "one of"], ["exclude", "not"]];
|
|
||||||
|
|
||||||
/** @type {object[]} every agent, as the server lists them */
|
/** @type {object[]} every agent, as the server lists them */
|
||||||
let agents = [];
|
let agents = [];
|
||||||
@@ -266,8 +264,8 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The region and place in words, one sentence led by the region, then the lines a
|
// The region and place in words, one sentence led by the region, then the lines a
|
||||||
// role may match, with the one-of rows among them since they are the same
|
// role may match, with any other one-of gates among them since they are the
|
||||||
// group: "Within Europe: keeps roles anywhere in Denmark, in Malmö, Lund or
|
// same group: "Within Europe: keeps roles anywhere in Denmark, in Malmö, Lund or
|
||||||
// Helsingborg, or remote, and leaves out the rest." The lines add up, which
|
// Helsingborg, or remote, and leaves out the rest." The lines add up, which
|
||||||
// the first wording ("Roles in ...") left the owner to guess.
|
// the first wording ("Roles in ...") left the owner to guess.
|
||||||
function whereSentence(where, others) {
|
function whereSentence(where, others) {
|
||||||
@@ -384,7 +382,7 @@
|
|||||||
// ---------------------------------------------------------------- the form
|
// ---------------------------------------------------------------- the form
|
||||||
|
|
||||||
// Tell the form something changed that no field typed: a chip or a row
|
// Tell the form something changed that no field typed: a chip or a row
|
||||||
// added or removed. It marks the form dirty and redraws the gates view.
|
// added or removed. It marks the form dirty and redraws the gates' sentence.
|
||||||
function changed(node) {
|
function changed(node) {
|
||||||
node.dispatchEvent(new Event("input", { bubbles: true }));
|
node.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
}
|
}
|
||||||
@@ -559,9 +557,9 @@
|
|||||||
return box;
|
return box;
|
||||||
}
|
}
|
||||||
|
|
||||||
// A language as a row, after the rule rows: a code, a level, Remove. The
|
// A language as a row: a code, a level, Remove. The code is two letters,
|
||||||
// code is two letters, which the server turns into the language; its name
|
// which the server turns into the language; its name is the row's title
|
||||||
// is the row's title where the browser knows it.
|
// where the browser knows it.
|
||||||
function languageRow(language, remove) {
|
function languageRow(language, remove) {
|
||||||
const row = el("div", "rule-row");
|
const row = el("div", "rule-row");
|
||||||
const code = el("input");
|
const code = el("input");
|
||||||
@@ -587,32 +585,32 @@
|
|||||||
return { node: row, read: () => ({ language: code.value.trim().toLowerCase(), level: level.value }) };
|
return { node: row, read: () => ({ language: code.value.trim().toLowerCase(), level: level.value }) };
|
||||||
}
|
}
|
||||||
|
|
||||||
// A gate as a row: field, mode, values and Remove. Values untouched since the load are sent as they came, so a
|
// The gates the fetch gate's four parts say, in the order the starter
|
||||||
// value with a comma in it survives a save.
|
// agents have them: the world, the location lines and the titles left out,
|
||||||
function ruleRow(gate, remove) {
|
// then `kept`, the stored gates the parts cannot say, as they came. `parts`
|
||||||
const row = el("div", "rule-row");
|
// holds what the form reads: `scope` (europe, countries, everywhere) and
|
||||||
const field = select(FIELDS.map((f) => [f, f.replace("_", " ")]), gate.field, "Field");
|
// its `scoped` countries, the `within` countries, the `towns`, `remote`
|
||||||
const mode = select(MODES, gate.mode, "Mode");
|
// (none, scope, from) and its `from` areas, and the `titles`.
|
||||||
const values = el("input");
|
function formGates(parts, kept = []) {
|
||||||
values.type = "text";
|
const out = [];
|
||||||
values.required = true;
|
const { scope, scoped, within, towns: lines, remote, from, titles } = parts;
|
||||||
values.spellcheck = false;
|
if (scope === "europe") out.push({ field: "country", values: rh.europeWith(within), mode: "require" });
|
||||||
values.placeholder = "senior, staff";
|
if (scope === "countries" && scoped.length) out.push({ field: "country", values: scoped, mode: "require" });
|
||||||
values.value = gate.values.join(", ");
|
if (within.length) out.push({ field: "country", values: within, mode: "any" });
|
||||||
values.setAttribute("aria-label", "Values, separated by commas");
|
if (remote === "scope") out.push({ field: "remote", values: ["remote"], mode: "any" });
|
||||||
const loaded = values.value;
|
if (remote === "from" && from.length) out.push({ field: "remote_from", values: from, mode: "any" });
|
||||||
const drop = linky("Remove");
|
if (lines.length) out.push({ field: "place", values: lines, mode: "any" });
|
||||||
drop.addEventListener("click", () => remove(row));
|
if (titles.length) out.push({ field: "title", values: titles, mode: "exclude" });
|
||||||
row.append(field, mode, values, drop);
|
return out.concat(kept);
|
||||||
|
}
|
||||||
|
|
||||||
function read() {
|
// The note on the gates the form keeps: "The form cannot edit these rules,
|
||||||
const parts = values.value === loaded
|
// and Save keeps them as stored: must: with seniority senior; not: at Acme."
|
||||||
? gate.values
|
function keptNote(kept) {
|
||||||
: values.value.split(",").map((v) => v.trim()).filter(Boolean)
|
if (!kept.length) return "";
|
||||||
.map((v) => (field.value === "salary_min" ? Number(v) : v));
|
const said = kept.map((g) => `${MODES[g.mode] || g.mode}: ${phrase(g)}`).join("; ");
|
||||||
return { field: field.value, values: parts, mode: mode.value };
|
return `The form cannot edit ${kept.length > 1 ? "these rules" : "this rule"}, and Save keeps `
|
||||||
}
|
+ `${kept.length > 1 ? "them" : "it"} as stored: ${said}.`;
|
||||||
return { node: row, read };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function form(row) {
|
function form(row) {
|
||||||
@@ -713,51 +711,18 @@
|
|||||||
remoteFrom.node.hidden = remoteMode.value !== "from";
|
remoteFrom.node.hidden = remoteMode.value !== "from";
|
||||||
};
|
};
|
||||||
offer();
|
offer();
|
||||||
$("details.advanced", f).open = rest.length > 0;
|
const kept = $("#a-kept", f);
|
||||||
|
kept.textContent = keptNote(rest);
|
||||||
|
kept.hidden = !rest.length;
|
||||||
|
|
||||||
const rules = [];
|
// The gates Save sends: the four parts', then the kept ones. The sentence
|
||||||
const dropRule = (node) => {
|
// under them says them all, as the card will, the kept ones included.
|
||||||
rules.splice(rules.findIndex((r) => r.node === node), 1);
|
const gates = () => formGates({
|
||||||
node.remove();
|
scope: scope.value, scoped: scoped.values(), within: countries.values(), towns: places.values(),
|
||||||
changed(f);
|
remote: remoteMode.value, from: remoteFrom.values(), titles: titles.values(),
|
||||||
};
|
}, rest);
|
||||||
const addRule = (gate) => {
|
|
||||||
const entry = ruleRow(gate, dropRule);
|
|
||||||
rules.push(entry);
|
|
||||||
$("#a-rules", f).appendChild(entry.node);
|
|
||||||
return entry;
|
|
||||||
};
|
|
||||||
rest.forEach(addRule);
|
|
||||||
$("#a-add-rule", f).addEventListener("click", () => {
|
|
||||||
$("select", addRule({ field: "seniority", values: [], mode: "require" }).node).focus();
|
|
||||||
changed(f);
|
|
||||||
});
|
|
||||||
|
|
||||||
// The gates the fetch gate's three parts say, in the order the starter
|
|
||||||
// agents have them: the region, the place lines, and the titles left out.
|
|
||||||
function whereGates() {
|
|
||||||
const out = [];
|
|
||||||
const within = countries.values();
|
|
||||||
const towns = places.values();
|
|
||||||
const from = remoteFrom.values();
|
|
||||||
if (scope.value === "europe") out.push({ field: "country", values: rh.europeWith(within), mode: "require" });
|
|
||||||
if (scope.value === "countries" && scoped.values().length) {
|
|
||||||
out.push({ field: "country", values: scoped.values(), mode: "require" });
|
|
||||||
}
|
|
||||||
if (within.length) out.push({ field: "country", values: within, mode: "any" });
|
|
||||||
if (remoteMode.value === "scope") out.push({ field: "remote", values: ["remote"], mode: "any" });
|
|
||||||
if (remoteMode.value === "from" && from.length) out.push({ field: "remote_from", values: from, mode: "any" });
|
|
||||||
if (towns.length) out.push({ field: "place", values: towns, mode: "any" });
|
|
||||||
const left = titles.values();
|
|
||||||
if (left.length) out.push({ field: "title", values: left, mode: "exclude" });
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
// The gates Save sends: the fetch gate's, then the rows. The sentence
|
|
||||||
// under it says them all, as the card will, rows included.
|
|
||||||
const gates = () => whereGates().concat(rules.map((r) => r.read()));
|
|
||||||
const view = () => {
|
const view = () => {
|
||||||
offer();
|
offer();
|
||||||
$("#a-gates", f).value = JSON.stringify(gates(), null, 2);
|
|
||||||
$("#a-where-sum", f).textContent = rulesLine(gates()) || "Passes every role.";
|
$("#a-where-sum", f).textContent = rulesLine(gates()) || "Passes every role.";
|
||||||
};
|
};
|
||||||
view();
|
view();
|
||||||
@@ -981,6 +946,9 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// For tests/js/agents.test.mjs: what Save sends and how it is said.
|
||||||
|
window.agentGates = { formGates, keptNote, rulesLine };
|
||||||
|
|
||||||
document.addEventListener("DOMContentLoaded", async () => {
|
document.addEventListener("DOMContentLoaded", async () => {
|
||||||
if (!$("#agents-list")) return;
|
if (!$("#agents-list")) return;
|
||||||
$("#a-new").addEventListener("click", () => open(null));
|
$("#a-new").addEventListener("click", () => open(null));
|
||||||
|
|||||||
+3
-17
@@ -978,14 +978,6 @@ body[data-reader="open"] .company-strip { display: none; }
|
|||||||
.reader-cv .cv-total { margin: 0 0 .45rem; font-variant-numeric: tabular-nums; }
|
.reader-cv .cv-total { margin: 0 0 .45rem; font-variant-numeric: tabular-nums; }
|
||||||
.reader-cv .cv-total b { font-size: 1.25rem; }
|
.reader-cv .cv-total b { font-size: 1.25rem; }
|
||||||
.reader-cv .cv-total span { display: inline-block; margin-left: .4rem; color: var(--dim); font-size: .88rem; }
|
.reader-cv .cv-total span { display: inline-block; margin-left: .4rem; color: var(--dim); font-size: .88rem; }
|
||||||
/* The must-haves, one line each with its verdict's mark. */
|
|
||||||
.cv-musts { list-style: none; margin: 0 0 .55rem; padding: 0; font-size: .92rem; line-height: 1.4; }
|
|
||||||
.cv-musts li { display: flex; gap: .45rem; }
|
|
||||||
.cv-musts li + li { margin-top: .15rem; }
|
|
||||||
.cv-musts .mark { flex: none; width: 1rem; text-align: center; font-weight: 700; }
|
|
||||||
.cv-musts .must-met .mark { color: var(--good); }
|
|
||||||
.cv-musts .must-broken .mark { color: var(--error); }
|
|
||||||
.cv-musts .must-unclear .mark { color: var(--faint); }
|
|
||||||
.cv-lists { display: grid; grid-template-columns: 1fr 1fr; gap: .5rem 1.2rem; }
|
.cv-lists { display: grid; grid-template-columns: 1fr 1fr; gap: .5rem 1.2rem; }
|
||||||
.cv-lists ul { margin: 0; padding-left: 1.1rem; font-size: .92rem; line-height: 1.45; }
|
.cv-lists ul { margin: 0; padding-left: 1.1rem; font-size: .92rem; line-height: 1.45; }
|
||||||
.cv-lists li + li { margin-top: .2rem; }
|
.cv-lists li + li { margin-top: .2rem; }
|
||||||
@@ -1308,8 +1300,6 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
|||||||
background: var(--bg); border: 1px solid var(--line);
|
background: var(--bg); border: 1px solid var(--line);
|
||||||
border-radius: .3rem; padding: .35rem .5rem; width: 100%;
|
border-radius: .3rem; padding: .35rem .5rem; width: 100%;
|
||||||
}
|
}
|
||||||
/* Prose in a textarea is read in the page face; JSON is in the mono one. */
|
|
||||||
.agent-form textarea.code { font-family: var(--mono); font-size: .85rem; }
|
|
||||||
.agent-form .hint { color: var(--dim); font-size: .8rem; overflow-wrap: anywhere; }
|
.agent-form .hint { color: var(--dim); font-size: .8rem; overflow-wrap: anywhere; }
|
||||||
.weights { display: flex; gap: .75rem; flex-wrap: wrap; border: 1px solid var(--line); border-radius: .3rem; }
|
.weights { display: flex; gap: .75rem; flex-wrap: wrap; border: 1px solid var(--line); border-radius: .3rem; }
|
||||||
/* A label that wraps to two lines pushes its box down; the boxes stay in line. */
|
/* A label that wraps to two lines pushes its box down; the boxes stay in line. */
|
||||||
@@ -1324,14 +1314,13 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
|||||||
}
|
}
|
||||||
/* The fieldsets that hold rows: a column, the Add button under the rows. */
|
/* The fieldsets that hold rows: a column, the Add button under the rows. */
|
||||||
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
|
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
|
||||||
.agent-form fieldset.terms,
|
.agent-form fieldset.terms {
|
||||||
.agent-form fieldset.rules {
|
|
||||||
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
|
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
|
||||||
border: 1px solid var(--line); border-radius: .3rem;
|
border: 1px solid var(--line); border-radius: .3rem;
|
||||||
}
|
}
|
||||||
.agent-form fieldset > button { align-self: flex-start; }
|
.agent-form fieldset > button { align-self: flex-start; }
|
||||||
/* A prompt or a rule as a row of a list, after `.crow`: the name and Remove
|
/* A prompt or a language as a row of a list, after `.crow`: the name and Remove
|
||||||
on line one, the text spanning the row under them. A rule row wraps like
|
on line one, the text spanning the row under them. A language row wraps like
|
||||||
`.crow .board-edit`, so a phone takes it in two lines rather than scrolling. */
|
`.crow .board-edit`, so a phone takes it in two lines rather than scrolling. */
|
||||||
.prompt-row {
|
.prompt-row {
|
||||||
display: grid; grid-template-columns: 1fr auto; align-items: center;
|
display: grid; grid-template-columns: 1fr auto; align-items: center;
|
||||||
@@ -1361,9 +1350,6 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
|||||||
.agent-form .chips input[type="text"] { flex: 1 1 6rem; width: auto; min-width: 0; }
|
.agent-form .chips input[type="text"] { flex: 1 1 6rem; width: auto; min-width: 0; }
|
||||||
.agent-form .chips select { flex: 0 1 12rem; min-width: 0; max-width: 100%; }
|
.agent-form .chips select { flex: 0 1 12rem; min-width: 0; max-width: 100%; }
|
||||||
.chips .tag span { color: var(--faint); }
|
.chips .tag span { color: var(--faint); }
|
||||||
.agent-form details summary { cursor: pointer; color: var(--dim); font-size: .9rem; }
|
|
||||||
.agent-form details[open] summary { margin-bottom: .5rem; }
|
|
||||||
.agent-form .advanced-body { display: flex; flex-direction: column; gap: .6rem; }
|
|
||||||
|
|
||||||
/* ================================================================ */
|
/* ================================================================ */
|
||||||
/* companies and sources — the Sources page's sections, */
|
/* companies and sources — the Sources page's sections, */
|
||||||
|
|||||||
+12
-10
@@ -1,7 +1,8 @@
|
|||||||
/* Where a remote role may be worked from, as the pages say it: the words of
|
/* Where a remote role may be worked from, as the pages say it: the words of
|
||||||
* the badge, the picks of the agent's "Remote, open to someone in" chips, and
|
* the badge, the picks of the agent's "Remote, open to someone in" chips, and
|
||||||
* how an agent's gates share out between the fetch gate's blocks and Other rules. No
|
* how an agent's gates share out between the fetch gate's blocks and the rest
|
||||||
* DOM in it, so `tests/js/remote.test.mjs` can walk it without a browser.
|
* the form keeps as stored. No DOM in it, so `tests/js/remote.test.mjs` can
|
||||||
|
* walk it without a browser.
|
||||||
*
|
*
|
||||||
* The area is a place token in three spellings, one per layer: `remote:<place
|
* The area is a place token in three spellings, one per layer: `remote:<place
|
||||||
* token>` on the Roles rail and in the address, `<place token>` in a
|
* token>` on the Roles rail and in the address, `<place token>` in a
|
||||||
@@ -61,18 +62,19 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Share out an agent's gates between the fetch gate's blocks (region,
|
/** Share out an agent's gates between the fetch gate's blocks (region,
|
||||||
place, titles to leave out) and Other rules. Each line takes the first
|
place, titles to leave out) and the rest, which the form cannot edit
|
||||||
gate it can say whole; the rest are rows. A gate has a field, values and
|
and keeps as stored. Each line takes the first gate it can say whole.
|
||||||
a mode (require, any, exclude) and no weight; a gate with a mode the
|
A gate has a field, values and a mode (require, any, exclude) and no
|
||||||
form does not know, such as an old prefer, stays a row.
|
weight; a gate with a mode the form does not know, such as an old
|
||||||
|
prefer, is in the rest.
|
||||||
|
|
||||||
The scope line is the first plain require gate on country: `europe`
|
The scope line is the first plain require gate on country: `europe`
|
||||||
when it is exactly Europe and the countries line (`europeWith`),
|
when it is exactly Europe and the countries line (`europeWith`),
|
||||||
`countries` with its codes as chips otherwise, `everywhere` when there
|
`countries` with its codes as chips otherwise, `everywhere` when there
|
||||||
is none. Any other require gate on country is a row. The remote line is
|
is none. Any other require gate on country is in the rest. The remote
|
||||||
`scope` for the one-of on `remote: [remote]` (remote, within the
|
line is `scope` for the one-of on `remote: [remote]` (remote, within
|
||||||
scope), `from` for a one-of on `remote_from`, else `none`; when both
|
the scope), `from` for a one-of on `remote_from`, else `none`; when
|
||||||
are there `remote_from` wins and the plain one stays a row. */
|
both are there `remote_from` wins and the plain one is in the rest. */
|
||||||
function distribute(gates) {
|
function distribute(gates) {
|
||||||
const where = { scope: null, countries: null, places: null, remote: { mode: "none", values: [] }, titles: null };
|
const where = { scope: null, countries: null, places: null, remote: { mode: "none", values: [] }, titles: null };
|
||||||
const rest = [];
|
const rest = [];
|
||||||
|
|||||||
+4
-18
@@ -1744,34 +1744,20 @@
|
|||||||
try { return languageNames?.of(code) || code; } catch (err) { return code; }
|
try { return languageNames?.of(code) || code; } catch (err) { return code; }
|
||||||
}
|
}
|
||||||
|
|
||||||
const MUST_MARKS = { met: ["✓", "met"], broken: ["✗", "broken"], unclear: ["?", "unclear"] };
|
|
||||||
|
|
||||||
/* The agent's `judge` reading against its CV: the total and its three
|
/* The agent's `judge` reading against its CV: the total and its three
|
||||||
numbers, each of 10, career as the server counted it (capped at 4 when a must-have
|
numbers, each of 10, then the strengths and gaps it named. A box as the
|
||||||
is broken, with what the model said), the must-haves with a verdict
|
summary is, since it is the same kind of thing: a model's reading, not
|
||||||
each, then the strengths and gaps it named. A box as the summary is,
|
the posting. */
|
||||||
since it is the same kind of thing: a model's reading, not the posting. */
|
|
||||||
function cvMatch(r) {
|
function cvMatch(r) {
|
||||||
const j = r.judgement;
|
const j = r.judgement;
|
||||||
if (!j) return "";
|
if (!j) return "";
|
||||||
const list = (items, none) => (items?.length
|
const list = (items, none) => (items?.length
|
||||||
? `<ul>${items.map((x) => `<li>${esc(x)}</li>`).join("")}</ul>`
|
? `<ul>${items.map((x) => `<li>${esc(x)}</li>`).join("")}</ul>`
|
||||||
: `<p class="muted small">${none}</p>`);
|
: `<p class="muted small">${none}</p>`);
|
||||||
const counted = j.career_capped ?? j.career;
|
|
||||||
const career = counted < j.career
|
|
||||||
? `career ${counted}/10 (capped: breaks a must-have; model said ${j.career}/10)`
|
|
||||||
: `career ${j.career}/10`;
|
|
||||||
const musts = Array.isArray(j.musts) && j.musts.length
|
|
||||||
? `<ul class="cv-musts">${j.musts.map((m) => {
|
|
||||||
const [mark, word] = MUST_MARKS[m.verdict] || MUST_MARKS.unclear;
|
|
||||||
return `<li class="must-${word}"><span class="mark" role="img"
|
|
||||||
aria-label="${word}" title="${word}">${mark}</span><span>${esc(m.text)}</span></li>`;
|
|
||||||
}).join("")}</ul>` : "";
|
|
||||||
return `<section class="reader-summary reader-cv">
|
return `<section class="reader-summary reader-cv">
|
||||||
<h2 class="reader-label">CV match</h2>
|
<h2 class="reader-label">CV match</h2>
|
||||||
<p class="cv-total"><b>${Math.round(j.cv_score)}</b> of 100
|
<p class="cv-total"><b>${Math.round(j.cv_score)}</b> of 100
|
||||||
<span>skills ${j.skills}/10 · experience ${j.experience}/10 · ${career}</span></p>
|
<span>skills ${j.skills}/10 · experience ${j.experience}/10 · career ${j.career}/10</span></p>
|
||||||
${musts}
|
|
||||||
<div class="cv-lists">
|
<div class="cv-lists">
|
||||||
<div><h3 class="reader-label">Strengths</h3>${list(j.strengths, "None named.")}</div>
|
<div><h3 class="reader-label">Strengths</h3>${list(j.strengths, "None named.")}</div>
|
||||||
<div><h3 class="reader-label">Gaps</h3>${list(j.gaps, "None named.")}</div>
|
<div><h3 class="reader-label">Gaps</h3>${list(j.gaps, "None named.")}</div>
|
||||||
|
|||||||
+43
-16
@@ -55,6 +55,10 @@ CREATE TABLE IF NOT EXISTS roles (
|
|||||||
country TEXT NOT NULL DEFAULT '',
|
country TEXT NOT NULL DEFAULT '',
|
||||||
remote TEXT NOT NULL DEFAULT ''
|
remote TEXT NOT NULL DEFAULT ''
|
||||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||||
|
-- 1 when the title or text says the role is worked remotely while no
|
||||||
|
-- field does (prep, app.prep.words.remote_in_text): the location gate
|
||||||
|
-- lets such a role through until extract has read it.
|
||||||
|
remote_text INTEGER NOT NULL DEFAULT 0,
|
||||||
salary_min INTEGER,
|
salary_min INTEGER,
|
||||||
salary_max INTEGER,
|
salary_max INTEGER,
|
||||||
currency TEXT NOT NULL DEFAULT '',
|
currency TEXT NOT NULL DEFAULT '',
|
||||||
@@ -328,9 +332,8 @@ END;
|
|||||||
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
|
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
|
||||||
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
|
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
|
||||||
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
|
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
|
||||||
-- level}), like_text (what they like: the work, the workplace, situations),
|
-- level}), like_text (what they like: the work, the workplace, situations)
|
||||||
-- avoid_text (what they do not like) and must_text (must-haves and
|
-- and avoid_text (what they do not like) are what judge reads a role
|
||||||
-- deal-breakers, set through the API only) are what judge reads a role
|
|
||||||
-- against; an agent with no CV is not judged.
|
-- against; an agent with no CV is not judged.
|
||||||
CREATE TABLE IF NOT EXISTS agents (
|
CREATE TABLE IF NOT EXISTS agents (
|
||||||
id INTEGER PRIMARY KEY,
|
id INTEGER PRIMARY KEY,
|
||||||
@@ -344,7 +347,6 @@ CREATE TABLE IF NOT EXISTS agents (
|
|||||||
cv_text TEXT NOT NULL DEFAULT '',
|
cv_text TEXT NOT NULL DEFAULT '',
|
||||||
cv_languages TEXT NOT NULL DEFAULT '[]',
|
cv_languages TEXT NOT NULL DEFAULT '[]',
|
||||||
avoid_text TEXT NOT NULL DEFAULT '',
|
avoid_text TEXT NOT NULL DEFAULT '',
|
||||||
must_text TEXT NOT NULL DEFAULT '',
|
|
||||||
like_text TEXT NOT NULL DEFAULT ''
|
like_text TEXT NOT NULL DEFAULT ''
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -435,11 +437,8 @@ CREATE TABLE IF NOT EXISTS agent_cv_vectors (
|
|||||||
-- 100 until migration 33), cv_score their weighted total times 10 at the
|
-- 100 until migration 33), cv_score their weighted total times 10 at the
|
||||||
-- default weights (not read: app.models.cv_score is worked out when a role
|
-- default weights (not read: app.models.cv_score is worked out when a role
|
||||||
-- is served), strengths and gaps JSON lists. input_key
|
-- is served), strengths and gaps JSON lists. input_key
|
||||||
-- is a hash of the CV, prompts, avoid text and the role's text it read: a
|
-- is a hash of the CV, prompts, languages, like and avoid texts and the
|
||||||
-- different one is read again. musts is a JSON list of {id, text, verdict},
|
-- role's text it read: a different one is read again. tries counts
|
||||||
-- the model's verdict on each of the agent's must-haves (one per line of its
|
|
||||||
-- must text), broken how many are "broken": career counts as at most 4 then
|
|
||||||
-- (app.models.BROKEN_CAP), applied when the role is served. tries counts
|
|
||||||
-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A
|
-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A
|
||||||
-- row with judged_at NULL holds only tries.
|
-- row with judged_at NULL holds only tries.
|
||||||
CREATE TABLE IF NOT EXISTS role_judgements (
|
CREATE TABLE IF NOT EXISTS role_judgements (
|
||||||
@@ -451,8 +450,6 @@ CREATE TABLE IF NOT EXISTS role_judgements (
|
|||||||
cv_score REAL,
|
cv_score REAL,
|
||||||
strengths TEXT NOT NULL DEFAULT '[]',
|
strengths TEXT NOT NULL DEFAULT '[]',
|
||||||
gaps TEXT NOT NULL DEFAULT '[]',
|
gaps TEXT NOT NULL DEFAULT '[]',
|
||||||
musts TEXT NOT NULL DEFAULT '[]',
|
|
||||||
broken INTEGER NOT NULL DEFAULT 0,
|
|
||||||
input_key TEXT NOT NULL DEFAULT '',
|
input_key TEXT NOT NULL DEFAULT '',
|
||||||
tries INTEGER NOT NULL DEFAULT 0,
|
tries INTEGER NOT NULL DEFAULT 0,
|
||||||
judged_at TEXT,
|
judged_at TEXT,
|
||||||
@@ -763,6 +760,12 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
|||||||
# A Python step: JSON limits, and a count for the log.
|
# A Python step: JSON limits, and a count for the log.
|
||||||
(33, "the judge on 0 to 10: every judgement cleared to be made again, and summarise_top "
|
(33, "the judge on 0 to 10: every judgement cleared to be made again, and summarise_top "
|
||||||
"50 becomes 30", lambda conn: _judge_out_of_ten(conn)),
|
"50 becomes 30", lambda conn: _judge_out_of_ten(conn)),
|
||||||
|
# A Python step: it drops the columns only where they are there. The
|
||||||
|
# judge reads every role again (judge.FORMAT).
|
||||||
|
(34, "agents.must_text, role_judgements.musts and broken dropped: the profile is the "
|
||||||
|
"CV, languages, likes and dislikes", lambda conn: _drop_musts(conn)),
|
||||||
|
(35, "roles.remote_text: the text says remote while no field does, read by prep for the "
|
||||||
|
"location gate", lambda conn: _add_remote_text(conn)),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: What a file created from `SCHEMA` today is already at.
|
#: What a file created from `SCHEMA` today is already at.
|
||||||
@@ -1012,6 +1015,15 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
|
|||||||
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
||||||
|
|
||||||
|
|
||||||
|
def _add_remote_text(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Migration 35's step: `roles.remote_text`, 0 until the next prep,
|
||||||
|
which maps every open role again since its rules changed. Runs twice
|
||||||
|
without harm."""
|
||||||
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||||
|
if "remote_text" not in columns:
|
||||||
|
conn.execute("ALTER TABLE roles ADD COLUMN remote_text INTEGER NOT NULL DEFAULT 0")
|
||||||
|
|
||||||
|
|
||||||
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
||||||
"""Migration 27's step: `roles.remote_model` empty and
|
"""Migration 27's step: `roles.remote_model` empty and
|
||||||
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
||||||
@@ -1203,12 +1215,14 @@ def _judge_out_of_ten(conn: sqlite3.Connection) -> None:
|
|||||||
old number is shown on the new scale. The rows stay, as rows of no
|
old number is shown on the new scale. The rows stay, as rows of no
|
||||||
judgement, and the judge reads every pair again. An agent whose
|
judgement, and the judge reads every pair again. An agent whose
|
||||||
`summarise_top` is 50, migration 32's default, takes 30, the new one.
|
`summarise_top` is 50, migration 32's default, takes 30, the new one.
|
||||||
Runs twice without harm."""
|
Runs twice without harm, also after migration 34 dropped the verdicts."""
|
||||||
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
||||||
|
verdicts = ", musts = '[]', broken = 0" if {"musts", "broken"} <= columns else ""
|
||||||
cleared = conn.execute(
|
cleared = conn.execute(
|
||||||
"""UPDATE role_judgements
|
f"""UPDATE role_judgements
|
||||||
SET skills = NULL, experience = NULL, career = NULL, cv_score = NULL,
|
SET skills = NULL, experience = NULL, career = NULL, cv_score = NULL,
|
||||||
strengths = '[]', gaps = '[]', musts = '[]', broken = 0,
|
strengths = '[]', gaps = '[]'{verdicts},
|
||||||
input_key = '', tries = 0, judged_at = NULL""").rowcount
|
input_key = '', tries = 0, judged_at = NULL""").rowcount
|
||||||
lowered = conn.execute(
|
lowered = conn.execute(
|
||||||
"""UPDATE agents SET limits = json_set(limits, '$.summarise_top', 30)
|
"""UPDATE agents SET limits = json_set(limits, '$.summarise_top', 30)
|
||||||
WHERE json_valid(limits) AND json_type(limits) = 'object'
|
WHERE json_valid(limits) AND json_type(limits) = 'object'
|
||||||
@@ -1217,6 +1231,19 @@ def _judge_out_of_ten(conn: sqlite3.Connection) -> None:
|
|||||||
"summarise_top 50 set to 30", cleared, lowered)
|
"summarise_top 50 set to 30", cleared, lowered)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_musts(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Migration 34's step: the must-haves are gone, the agent's text
|
||||||
|
(`agents.must_text`) and the judge's verdicts on it
|
||||||
|
(`role_judgements.musts`, `broken`), and with them the cap on career.
|
||||||
|
Runs twice without harm."""
|
||||||
|
for table, columns in (("agents", ("must_text",)),
|
||||||
|
("role_judgements", ("musts", "broken"))):
|
||||||
|
present = {r[1] for r in conn.execute(f"PRAGMA table_info({table})")}
|
||||||
|
for column in columns:
|
||||||
|
if column in present:
|
||||||
|
conn.execute(f"ALTER TABLE {table} DROP COLUMN {column}")
|
||||||
|
|
||||||
|
|
||||||
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||||
"""Migration 21's step. The column; the tag for every company that is a
|
"""Migration 21's step. The column; the tag for every company that is a
|
||||||
favourite of any agent, since favourite no longer widens what a pull
|
favourite of any agent, since favourite no longer widens what a pull
|
||||||
|
|||||||
+32
-39
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence
|
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
|
||||||
from contextlib import closing, contextmanager
|
from contextlib import closing, contextmanager
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -26,7 +26,7 @@ from app.models import (
|
|||||||
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||||
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
|
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
|
||||||
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
|
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
|
||||||
BROKEN_CAP, capped_career, language_flag, role_languages,
|
language_flag, role_languages,
|
||||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||||
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
|
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
|
||||||
)
|
)
|
||||||
@@ -140,7 +140,7 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
|
|||||||
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
||||||
|
|
||||||
#: The agent's `judge` readings beside every role, the same way: `j.skills`,
|
#: The agent's `judge` readings beside every role, the same way: `j.skills`,
|
||||||
#: `j.experience`, `j.career` and `j.broken` are NULL for a role not judged and with no
|
#: `j.experience` and `j.career` are NULL for a role not judged and with no
|
||||||
#: agent.
|
#: agent.
|
||||||
_WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? "
|
_WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? "
|
||||||
"AND j.judged_at IS NOT NULL")
|
"AND j.judged_at IS NOT NULL")
|
||||||
@@ -150,19 +150,15 @@ _WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agen
|
|||||||
def _cv_sql(weights: Weights | None) -> str:
|
def _cv_sql(weights: Weights | None) -> str:
|
||||||
"""`cv_score` as SQL for one agent's weights, 0 to 100 from the judge's
|
"""`cv_score` as SQL for one agent's weights, 0 to 100 from the judge's
|
||||||
three numbers 0 to 10, unrounded (`app.models.cv_score` rounds it): NULL
|
three numbers 0 to 10, unrounded (`app.models.cv_score` rounds it): NULL
|
||||||
for a role not judged, and for all three weights 0. Career
|
for a role not judged, and for all three weights 0. The weights are
|
||||||
counts as at most `BROKEN_CAP` when the judgement holds a broken
|
validated, finite floats, so they are written in rather than bound, the
|
||||||
must-have (`j.broken`), as `app.models.capped_career` has it. The
|
select's placeholders being then no concern."""
|
||||||
numbers are the model's validated, finite floats, so they are written in
|
|
||||||
rather than bound, the select's placeholders being then no concern."""
|
|
||||||
w = weights or Weights()
|
w = weights or Weights()
|
||||||
total = w.cv_skills + w.cv_experience + w.cv_career
|
total = w.cv_skills + w.cv_experience + w.cv_career
|
||||||
if not total:
|
if not total:
|
||||||
return "NULL"
|
return "NULL"
|
||||||
career = (f"(CASE WHEN j.broken > 0 AND j.career > {BROKEN_CAP} THEN {BROKEN_CAP} "
|
|
||||||
"ELSE j.career END)")
|
|
||||||
return (f"(10.0 * ({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience "
|
return (f"(10.0 * ({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience "
|
||||||
f"+ {w.cv_career!r} * {career}) / {total!r})")
|
f"+ {w.cv_career!r} * j.career) / {total!r})")
|
||||||
|
|
||||||
|
|
||||||
def _by_cv(expression: str) -> str:
|
def _by_cv(expression: str) -> str:
|
||||||
@@ -359,6 +355,12 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
|||||||
return ("r.languages IN ('', '[]')",
|
return ("r.languages IN ('', '[]')",
|
||||||
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
||||||
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
||||||
|
if gate.field == "remote" and "remote" in values:
|
||||||
|
# A role whose text says remote while no field does is possibly
|
||||||
|
# remote (`remote_text`, prep): it meets a gate on remote until
|
||||||
|
# extract has read it, and is not empty for it.
|
||||||
|
return (f"({column} = '' AND r.remote_text = 0)",
|
||||||
|
f"(LOWER({column}) IN ({marks}) OR ({column} = '' AND r.remote_text = 1))", values)
|
||||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||||
|
|
||||||
|
|
||||||
@@ -1629,8 +1631,9 @@ class Store:
|
|||||||
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
||||||
seniority_host, employment_host, remote_host, country_host,
|
seniority_host, employment_host, remote_host, country_host,
|
||||||
seniority_model, seniority_stated, seniority_title, seniority,
|
seniority_model, seniority_stated, seniority_title, seniority,
|
||||||
employment, remote, country, remote_model, remote_where_model,
|
employment, remote, remote_text, country, remote_model,
|
||||||
deadline_host, deadline_model, deadline,
|
remote_where_model, deadline_host, deadline_model, deadline,
|
||||||
|
SUBSTR(COALESCE(NULLIF(description, ''), teaser), 1, 6000) AS text,
|
||||||
json_extract(raw, '$.places') AS raw_places,
|
json_extract(raw, '$.places') AS raw_places,
|
||||||
(SELECT json_group_array(place_id) FROM (
|
(SELECT json_group_array(place_id) FROM (
|
||||||
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
||||||
@@ -1656,8 +1659,8 @@ class Store:
|
|||||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||||
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
|
`seniority_title`, `employment`, `remote`, `remote_text`, `remote_area`,
|
||||||
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
`country` and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
||||||
the title's. `remote` is written as prep composed it (the model's,
|
the title's. `remote` is written as prep composed it (the model's,
|
||||||
else the host's word, else the place line) and `remote_area`, place
|
else the host's word, else the place line) and `remote_area`, place
|
||||||
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
||||||
@@ -1673,6 +1676,7 @@ class Store:
|
|||||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||||
employment = :employment,
|
employment = :employment,
|
||||||
remote = :remote,
|
remote = :remote,
|
||||||
|
remote_text = :remote_text,
|
||||||
country = COALESCE(NULLIF(:country, ''), country),
|
country = COALESCE(NULLIF(:country, ''), country),
|
||||||
deadline = :deadline,
|
deadline = :deadline,
|
||||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||||
@@ -2021,33 +2025,28 @@ class Store:
|
|||||||
|
|
||||||
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
|
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
|
||||||
experience: int, career: int, strengths: Sequence[str],
|
experience: int, career: int, strengths: Sequence[str],
|
||||||
gaps: Sequence[str], musts: Sequence[Mapping[str, str]] = ()) -> bool:
|
gaps: Sequence[str]) -> bool:
|
||||||
"""Store `judge`'s reading of one role for one agent, replacing the
|
"""Store `judge`'s reading of one role for one agent, replacing the
|
||||||
one before: the three numbers as the model gave them, the verdict on
|
one before: the three numbers as the model gave them, `cv_score`
|
||||||
each of the agent's must-haves (`musts`, `{id, text, verdict}`) and how
|
(`app.models.cv_score`, at the default weights: the table's column is
|
||||||
many are broken, `cv_score` (`app.models.cv_score`, at the default
|
not read, see `judgement`) and its tries back at 0. False for an
|
||||||
weights and with the cap: the table's column is not read, see
|
unknown role or agent."""
|
||||||
`judgement`) and its tries back at 0. False for an unknown role or
|
|
||||||
agent."""
|
|
||||||
broken = sum(1 for m in musts if m.get("verdict") == "broken")
|
|
||||||
try:
|
try:
|
||||||
with self.transaction() as conn:
|
with self.transaction() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""INSERT INTO role_judgements (uid, agent_id, skills, experience, career,
|
"""INSERT INTO role_judgements (uid, agent_id, skills, experience, career,
|
||||||
cv_score, strengths, gaps, musts, broken,
|
cv_score, strengths, gaps,
|
||||||
input_key, tries, judged_at)
|
input_key, tries, judged_at)
|
||||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,0,?)
|
VALUES (?,?,?,?,?,?,?,?,?,0,?)
|
||||||
ON CONFLICT(uid, agent_id) DO UPDATE SET
|
ON CONFLICT(uid, agent_id) DO UPDATE SET
|
||||||
skills=excluded.skills, experience=excluded.experience,
|
skills=excluded.skills, experience=excluded.experience,
|
||||||
career=excluded.career, cv_score=excluded.cv_score,
|
career=excluded.career, cv_score=excluded.cv_score,
|
||||||
strengths=excluded.strengths, gaps=excluded.gaps,
|
strengths=excluded.strengths, gaps=excluded.gaps,
|
||||||
musts=excluded.musts, broken=excluded.broken,
|
|
||||||
input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""",
|
input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""",
|
||||||
(uid, agent_id, skills, experience, career,
|
(uid, agent_id, skills, experience, career,
|
||||||
cv_score(skills, experience, career, broken=broken),
|
cv_score(skills, experience, career),
|
||||||
json.dumps(list(strengths), ensure_ascii=False),
|
json.dumps(list(strengths), ensure_ascii=False),
|
||||||
json.dumps(list(gaps), ensure_ascii=False),
|
json.dumps(list(gaps), ensure_ascii=False),
|
||||||
json.dumps([dict(m) for m in musts], ensure_ascii=False), broken,
|
|
||||||
input_key, self.now()))
|
input_key, self.now()))
|
||||||
except sqlite3.IntegrityError:
|
except sqlite3.IntegrityError:
|
||||||
return False
|
return False
|
||||||
@@ -2075,8 +2074,7 @@ class Store:
|
|||||||
def judgement(self, agent_id: int | None, uid: str) -> Judgement | None:
|
def judgement(self, agent_id: int | None, uid: str) -> Judgement | None:
|
||||||
"""The agent's judgement of one role, None when it has none, its
|
"""The agent's judgement of one role, None when it has none, its
|
||||||
`cv_score` the three numbers under the agent's weights as they are
|
`cv_score` the three numbers under the agent's weights as they are
|
||||||
now, not as stored, and `career_capped` the model's career as the
|
now, not as stored."""
|
||||||
broken must-haves leave it (`BROKEN_CAP`)."""
|
|
||||||
if agent_id is None:
|
if agent_id is None:
|
||||||
return None
|
return None
|
||||||
row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? "
|
row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? "
|
||||||
@@ -2085,9 +2083,8 @@ class Store:
|
|||||||
return None
|
return None
|
||||||
found = Judgement.from_row(row)
|
found = Judgement.from_row(row)
|
||||||
agent = self.agent(agent_id)
|
agent = self.agent(agent_id)
|
||||||
found.career_capped = capped_career(found.career, found.broken)
|
|
||||||
found.cv_score = cv_score(found.skills, found.experience, found.career,
|
found.cv_score = cv_score(found.skills, found.experience, found.career,
|
||||||
agent.weights if agent else None, found.broken)
|
agent.weights if agent else None)
|
||||||
return found
|
return found
|
||||||
|
|
||||||
def set_article_text(self, uid: str, text: str) -> Role | None:
|
def set_article_text(self, uid: str, text: str) -> Role | None:
|
||||||
@@ -2209,14 +2206,13 @@ class Store:
|
|||||||
cv_text: str | None = None,
|
cv_text: str | None = None,
|
||||||
cv_languages: Sequence[CvLanguage] | None = None,
|
cv_languages: Sequence[CvLanguage] | None = None,
|
||||||
avoid_text: str | None = None,
|
avoid_text: str | None = None,
|
||||||
must_text: str | None = None,
|
|
||||||
like_text: str | None = None) -> Agent:
|
like_text: str | None = None) -> Agent:
|
||||||
"""Create an agent, or with `agent_id` replace that one's fields and
|
"""Create an agent, or with `agent_id` replace that one's fields and
|
||||||
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
|
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
|
||||||
named after the agent. Without `fetch_terms` an update keeps the
|
named after the agent. Without `fetch_terms` an update keeps the
|
||||||
stored ones and a new agent has none, so a caller that does not know
|
stored ones and a new agent has none, so a caller that does not know
|
||||||
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
|
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
|
||||||
`like_text`, `avoid_text` and `must_text` are kept the same way. A prompt whose text is
|
`like_text` and `avoid_text` are kept the same way. A prompt whose text is
|
||||||
unchanged keeps its cached vector, whatever its name or place, so an
|
unchanged keeps its cached vector, whatever its name or place, so an
|
||||||
edit re-embeds only the prompts it changed. A save gives every role
|
edit re-embeds only the prompts it changed. A save gives every role
|
||||||
`judge` gave up on for the agent its tries back, since what it is
|
`judge` gave up on for the agent its tries back, since what it is
|
||||||
@@ -2237,7 +2233,6 @@ class Store:
|
|||||||
(limits or Limits()).model_dump_json(), int(enabled), terms,
|
(limits or Limits()).model_dump_json(), int(enabled), terms,
|
||||||
None if cv_text is None else cv_text.strip(), spoken,
|
None if cv_text is None else cv_text.strip(), spoken,
|
||||||
None if avoid_text is None else avoid_text.strip(),
|
None if avoid_text is None else avoid_text.strip(),
|
||||||
None if must_text is None else must_text.strip(),
|
|
||||||
None if like_text is None else like_text.strip())
|
None if like_text is None else like_text.strip())
|
||||||
try:
|
try:
|
||||||
with self.transaction() as conn:
|
with self.transaction() as conn:
|
||||||
@@ -2245,10 +2240,9 @@ class Store:
|
|||||||
agent_id = conn.execute(
|
agent_id = conn.execute(
|
||||||
"""INSERT INTO agents (name, gates, weights, limits, enabled,
|
"""INSERT INTO agents (name, gates, weights, limits, enabled,
|
||||||
fetch_terms, cv_text, cv_languages, avoid_text,
|
fetch_terms, cv_text, cv_languages, avoid_text,
|
||||||
must_text, like_text, created_at)
|
like_text, created_at)
|
||||||
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
|
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
|
||||||
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),
|
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),?)""",
|
||||||
COALESCE(?, ''),?)""",
|
|
||||||
(*fields, self.now())).lastrowid
|
(*fields, self.now())).lastrowid
|
||||||
elif not conn.execute(
|
elif not conn.execute(
|
||||||
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
|
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
|
||||||
@@ -2256,7 +2250,6 @@ class Store:
|
|||||||
cv_text=COALESCE(?, cv_text),
|
cv_text=COALESCE(?, cv_text),
|
||||||
cv_languages=COALESCE(?, cv_languages),
|
cv_languages=COALESCE(?, cv_languages),
|
||||||
avoid_text=COALESCE(?, avoid_text),
|
avoid_text=COALESCE(?, avoid_text),
|
||||||
must_text=COALESCE(?, must_text),
|
|
||||||
like_text=COALESCE(?, like_text)
|
like_text=COALESCE(?, like_text)
|
||||||
WHERE id=?""",
|
WHERE id=?""",
|
||||||
(*fields, agent_id)).rowcount:
|
(*fields, agent_id)).rowcount:
|
||||||
|
|||||||
@@ -74,6 +74,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<span class="hint">Fetch terms, separated by a comma or Enter: <code>machine learning, data science</code> is two terms, and <code>machine learning</code> is searched as one. A host that takes a query sends only roles with one of the terms anywhere in the ad, but only for boards of 300 or more roles; smaller boards and other hosts are read whole. The terms are ignored after the fetch, since prompt closeness is the finer filter. Empty means no narrowing.</span>
|
<span class="hint">Fetch terms, separated by a comma or Enter: <code>machine learning, data science</code> is two terms, and <code>machine learning</code> is searched as one. A host that takes a query sends only roles with one of the terms anywhere in the ad, but only for boards of 300 or more roles; smaller boards and other hosts are read whole. The terms are ignored after the fetch, since prompt closeness is the finer filter. Empty means no narrowing.</span>
|
||||||
<p id="a-where-sum" class="where-sum" role="status"></p>
|
<p id="a-where-sum" class="where-sum" role="status"></p>
|
||||||
|
<p id="a-kept" class="hint" hidden></p>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
<fieldset class="terms cv">
|
<fieldset class="terms cv">
|
||||||
<legend>Your CV</legend>
|
<legend>Your CV</legend>
|
||||||
@@ -94,7 +95,7 @@
|
|||||||
<textarea id="a-like-text" rows="4" placeholder="Building a team from scratch, close work with the product side, small companies, a say in the roadmap"></textarea></label>
|
<textarea id="a-like-text" rows="4" placeholder="Building a team from scratch, close work with the product side, small companies, a say in the roadmap"></textarea></label>
|
||||||
<label>What I don't like
|
<label>What I don't like
|
||||||
<textarea id="a-avoid-text" rows="4" placeholder="Constant on-call, sales targets, large consultancies, open-plan offices"></textarea></label>
|
<textarea id="a-avoid-text" rows="4" placeholder="Constant on-call, sales targets, large consultancies, open-plan offices"></textarea></label>
|
||||||
<span class="hint">The work, the workplace, situations; read by the judge for the career rating. Rules that must hold go under the fetch gate or Other rules instead.</span>
|
<span class="hint">The work, the workplace, situations; read by the judge for the career rating. Rules that must hold go under the fetch gate instead.</span>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
<fieldset class="weights limits">
|
<fieldset class="weights limits">
|
||||||
<legend>Analytics filter</legend>
|
<legend>Analytics filter</legend>
|
||||||
@@ -124,20 +125,6 @@
|
|||||||
<label>Summarise the top <input type="number" id="a-summarise-top" min="0" step="5" value="30"> of each feed order</label>
|
<label>Summarise the top <input type="number" id="a-summarise-top" min="0" step="5" value="30"> of each feed order</label>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
<span class="hint">Summarise writes a short summary and tags for the top N of What I'm looking for and the top N of What fits me, at most twice N roles, and for every role you liked or saved. It is no part of any score.</span>
|
<span class="hint">Summarise writes a short summary and tags for the top N of What I'm looking for and the top N of What fits me, at most twice N roles, and for every role you liked or saved. It is no part of any score.</span>
|
||||||
<details class="advanced">
|
|
||||||
<summary>Advanced</summary>
|
|
||||||
<div class="advanced-body">
|
|
||||||
<fieldset class="rules">
|
|
||||||
<legend>Other rules</legend>
|
|
||||||
<div id="a-rules" class="rule-rows"></div>
|
|
||||||
<button type="button" id="a-add-rule" class="secondary">Add rule</button>
|
|
||||||
<span class="hint">Must: a role must meet it. One of: a role must meet one of these or a Place line. Not: a role that meets it is left out. Values are separated by commas; salary is a number.</span>
|
|
||||||
</fieldset>
|
|
||||||
<label>Rules as sent
|
|
||||||
<textarea id="a-gates" class="code" rows="6" spellcheck="false" readonly>[]</textarea>
|
|
||||||
<span class="hint">Read only: what Save sends as the agent's gates, from the fetch gate and Other rules.</span></label>
|
|
||||||
</div>
|
|
||||||
</details>
|
|
||||||
<div class="agent-actions">
|
<div class="agent-actions">
|
||||||
<button type="submit">Save and score</button>
|
<button type="submit">Save and score</button>
|
||||||
<button type="button" id="a-cancel" class="secondary">Cancel</button>
|
<button type="button" id="a-cancel" class="secondary">Cancel</button>
|
||||||
|
|||||||
+2
-4
@@ -1251,7 +1251,7 @@ class AgentBody(BaseModel):
|
|||||||
it is stored as one prompt named after the agent. `fetch_terms` narrow
|
it is stored as one prompt named after the agent. `fetch_terms` narrow
|
||||||
what big boards send (`app/jobs/reach.py`); `[]` narrows nothing, and
|
what big boards send (`app/jobs/reach.py`); `[]` narrows nothing, and
|
||||||
leaving the key out keeps an existing agent's. `cv_text`,
|
leaving the key out keeps an existing agent's. `cv_text`,
|
||||||
`cv_languages`, `like_text`, `avoid_text` and `must_text` are what `judge` reads a
|
`cv_languages`, `like_text` and `avoid_text` are what `judge` reads a
|
||||||
role against (`app/jobs/judge.py`), kept the same way when left out or
|
role against (`app/jobs/judge.py`), kept the same way when left out or
|
||||||
null."""
|
null."""
|
||||||
|
|
||||||
@@ -1267,7 +1267,6 @@ class AgentBody(BaseModel):
|
|||||||
cv_languages: CvLanguages | None = None
|
cv_languages: CvLanguages | None = None
|
||||||
like_text: str | None = None
|
like_text: str | None = None
|
||||||
avoid_text: str | None = None
|
avoid_text: str | None = None
|
||||||
must_text: str | None = None
|
|
||||||
enabled: bool = True
|
enabled: bool = True
|
||||||
|
|
||||||
|
|
||||||
@@ -1307,8 +1306,7 @@ def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent:
|
|||||||
body.enabled, agent_id=body.id, limits=body.limits,
|
body.enabled, agent_id=body.id, limits=body.limits,
|
||||||
prompts=body.prompts, fetch_terms=body.fetch_terms,
|
prompts=body.prompts, fetch_terms=body.fetch_terms,
|
||||||
cv_text=body.cv_text, cv_languages=body.cv_languages,
|
cv_text=body.cv_text, cv_languages=body.cv_languages,
|
||||||
like_text=body.like_text, avoid_text=body.avoid_text,
|
like_text=body.like_text, avoid_text=body.avoid_text)
|
||||||
must_text=body.must_text)
|
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(409, str(exc)) from exc
|
raise HTTPException(409, str(exc)) from exc
|
||||||
except KeyError:
|
except KeyError:
|
||||||
|
|||||||
+10
-2
@@ -16,7 +16,7 @@ What was decided for roledesk and the reason for each. The target design is the
|
|||||||
|
|
||||||
**A group is rows that all match each other, never a chain.** A row with no place matches Belgium and the Netherlands, and joining on one match merged both on the live data. Rows are taken sharpest first (a town, then only a country, then neither) and join a group only when they match every member.
|
**A group is rows that all match each other, never a chain.** A row with no place matches Belgium and the Netherlands, and joining on one match merged both on the live data. Rows are taken sharpest first (a town, then only a country, then neither) and join a group only when they match every member.
|
||||||
|
|
||||||
**One posting per place is folded too, on every host.** A host that lists one job once per office gives rows of the same company and title whose towns never meet; `dupes.fold` merges those of one host when their texts share at least `SAME_TEXT` (30%) of their words, and the kept role keeps each copy's place and link. It applies to every host, not only to readers that declare `places="copies"`, because most such rows came from hosts that do not declare it. A remote listing and an office listing stay two roles: the kept role takes a copy's places and not its remote, and the fetch gate reads a remote role by its area, so a remote listing folded behind an office elsewhere was rejected with it, and an office listing behind a remote role elsewhere likewise.
|
**One posting per place is folded too, on every host.** A host that lists one job once per office gives rows of the same company and title whose towns never meet; `dupes.fold` merges those of one host when their texts share at least `SAME_TEXT` (30%) of their words, and the kept role keeps each copy's place and link. It applies to every host, not only to readers that declare `places="copies"`, because most such rows came from hosts that do not declare it. It keeps to one country, as ADR 1.4 has it, since the same rule across countries merged different openings whose texts happened to be alike (a federal sales role in the US behind one in Australia); a job posted once per country is then one role per country, which costs a few rows for a pan-European posting and hides nothing. A remote listing and an office listing stay two roles: the kept role takes a copy's places and not its remote, and the fetch gate reads a remote role by its area, so a remote listing folded behind an office elsewhere was rejected with it, and an office listing behind a remote role elsewhere likewise.
|
||||||
|
|
||||||
**A copy is stored as `roles.dup_of` and rebuilt from scratch after every pull.** Every reader of open roles adds `dup_of IS NULL`, which keeps `idx_roles_posted` usable where a window function over the filtered rows would scan them all. Where a group changed, the kept role takes the earliest `read_at` and each agent's latest vote from its copies; an unchanged group is left alone, so marking a role unread again sticks. The per-board counts on Sources still count copies, since they report what a board holds.
|
**A copy is stored as `roles.dup_of` and rebuilt from scratch after every pull.** Every reader of open roles adds `dup_of IS NULL`, which keeps `idx_roles_posted` usable where a window function over the filtered rows would scan them all. Where a group changed, the kept role takes the earliest `read_at` and each agent's latest vote from its copies; an unchanged group is left alone, so marking a role unread again sticks. The per-board counts on Sources still count copies, since they report what a board holds.
|
||||||
|
|
||||||
@@ -70,6 +70,12 @@ What was decided for roledesk and the reason for each. The target design is the
|
|||||||
|
|
||||||
**A read that stalls is incomplete, and a sweep's cursor may be a pair.** A reader that sees the host send a page again hands it over empty with a cursor, and the runner ends the read incomplete; a cursor of None there read as the last page, and on a host that states no total as a complete read that closed what the pages had not listed. With a stated total a page of ids already read is not a stall, since a resumed sweep starts on one when the list grew at the top. A cursor is kept across pulls when it survives JSON apart from a tuple coming back as a list, so Workday's and SuccessFactors' sweeps resume.
|
**A read that stalls is incomplete, and a sweep's cursor may be a pair.** A reader that sees the host send a page again hands it over empty with a cursor, and the runner ends the read incomplete; a cursor of None there read as the last page, and on a host that states no total as a complete read that closed what the pages had not listed. With a stated total a page of ids already read is not a stall, since a resumed sweep starts on one when the list grew at the top. A cursor is kept across pulls when it survives JSON apart from a tuple coming back as a list, so Workday's and SuccessFactors' sweeps resume.
|
||||||
|
|
||||||
|
**A host that cannot filter on remote is asked for the agents' world where it takes the world in one read** (ADR Fetch prep and score 1.2). Asked for "DK, SE" alone such a host never listed a remote role filed under Germany, and 18 boards fetched no new remote role outside those two countries. Phenom and the Workday tenants without a remote facet take a list of countries, so Europe is one request and the gate sorts the rest out. SmartRecruiters, SuccessFactors and Eightfold take one country a request, where Europe is the whole board (Bosch 49 pages, a SuccessFactors site 10 roles a page), so there the location read stays and the row says remote was not asked.
|
||||||
|
|
||||||
|
**A role whose text says remote while no field does is possibly remote, read in prep with no model** (ADR 1.1 and 1.5). A role outside the agent's locations whose remote is unstated could never "turn out to be remote": extract reads only the cut, and the cut needs the gate. 5,016 open roles were rejected that way, 805 of them saying "remote" in their text. A phrase table ("fully remote", "work from home", the Danish and Swedish words) less its denials ("no remote", "remote sensing") marks `remote_text`; the gate's remote member takes the mark as a match while `remote` is empty, and once extract has answered, prep composes `remote` from the model and the mark no longer counts. A false mark costs at most a cut slot.
|
||||||
|
|
||||||
|
**Age is the floor under every board's closing** (ADR Fetch prep and score 1.3). An incomplete read (a ceiling, a page cap without a sweep) closed nothing ever, and a filtered read spared the roles acted on from its list and from age alike, so such roles stayed open for good. Now what no read has listed for 21 days closes under an incomplete read, and what a filtered read spares closes by age too; a role acted on is spared from the list close, which is guesswork, not from three weeks of absence. The alternative was to strike the sentence from the ADR, which would have left "may stay open for good" in the design.
|
||||||
|
|
||||||
**Switching a board or search off closes its open roles.** Only a pull that reads a target closes its roles, so a target no longer read would keep them open for good. Switched back on, the next pull reopens what it still lists. A batch in flight when the switch happens reopens them, so the settle closes every switched-off target once more.
|
**Switching a board or search off closes its open roles.** Only a pull that reads a target closes its roles, so a target no longer read would keep them open for good. Switched back on, the next pull reopens what it still lists. A batch in flight when the switch happens reopens them, so the settle closes every switched-off target once more.
|
||||||
|
|
||||||
**A destructive switch asks first; a reopening one queues a pull.** Switching a board or search off, replacing a board and detaching one each close roles in a tap, so the page asks first when there are roles to lose, naming the count. Switching a target on, following or setting a board, setting All locations or All roles on and saving a search each queue `pull` at once instead of waiting up to six hours for the schedule. The pull is queued in the API route, not in the page's script, so a script calling the endpoint gets what a click gets; `claim_run`'s refusal is not an error, since the run in flight or the next one reads the change.
|
**A destructive switch asks first; a reopening one queues a pull.** Switching a board or search off, replacing a board and detaching one each close roles in a tap, so the page asks first when there are roles to lose, naming the count. Switching a target on, following or setting a board, setting All locations or All roles on and saving a search each queue `pull` at once instead of waiting up to six hours for the schedule. The pull is queued in the API route, not in the page's script, so a script calling the endpoint gets what a click gets; `claim_run`'s refusal is not an error, since the run in flight or the next one reads the change.
|
||||||
@@ -182,6 +188,8 @@ What was decided for roledesk and the reason for each. The target design is the
|
|||||||
|
|
||||||
**`extract_batch`, `judge_batch` and `summarise_batch` run isolated, with a timeout.** `context.check()` runs only between the calls `modeltask.one` makes, not inside one, so a call that hangs was unkillable short of restarting the server; a process boundary lets a Kill, or the timeout, end it. One call is bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for judge and summarise): a batch still running after that is stuck, not slow. The cost is that an isolated job's context has no working `stop`, `task` or `fan_out` (`context_for` in `pipeline/child.py`), so a refusal or an account-level failure inside a batch cannot spare the task's queued siblings.
|
**`extract_batch`, `judge_batch` and `summarise_batch` run isolated, with a timeout.** `context.check()` runs only between the calls `modeltask.one` makes, not inside one, so a call that hangs was unkillable short of restarting the server; a process boundary lets a Kill, or the timeout, end it. One call is bounded at `agent.timeout`, so the deadline is a few calls' worth (30 minutes for extract, 15 for judge and summarise): a batch still running after that is stuck, not slow. The cost is that an isolated job's context has no working `stop`, `task` or `fan_out` (`context_for` in `pipeline/child.py`), so a refusal or an account-level failure inside a batch cannot spare the task's queued siblings.
|
||||||
|
|
||||||
|
**The profile is the CV, the languages, the prompts, the likes and the dislikes; must-haves are gone** (ADR 4.2). The must-have text with a verdict per line and a career cap at 4 was the harder form of what likes and dislikes now say softly; it left the form on 2026-10-03 and its machinery followed on 2026-10-06 (migration 34). The judge reads the languages with their levels, since a role that asks for fluent Danish against conversational Danish is a skills gap the CV text alone did not show; a change to the languages re-judges the cut like a change to the CV.
|
||||||
|
|
||||||
**Judge reads a role against a written profile, after the `/rank` of github.com/MadsLorentzen/ai-job-search.** Prompt closeness compares topics; it cannot tell a step down, years of experience or a deal-breaker, which a model reading both texts can. The profile is the CV, its languages with a level, and two soft texts, "What I like" (`like_text`) and "What I don't like" (`avoid_text`), each covering the work, the workplace and situations with no split into tasks versus environment, since the prompts already say what the owner wants. A deal-breaker is a hard rule and belongs in the fetch gate, so the form has no must-haves box; `must_text` is set only through the API. Culture is left out because it is guessed from ad text, and location because the fetch gate does it.
|
**Judge reads a role against a written profile, after the `/rank` of github.com/MadsLorentzen/ai-job-search.** Prompt closeness compares topics; it cannot tell a step down, years of experience or a deal-breaker, which a model reading both texts can. The profile is the CV, its languages with a level, and two soft texts, "What I like" (`like_text`) and "What I don't like" (`avoid_text`), each covering the work, the workplace and situations with no split into tasks versus environment, since the prompts already say what the owner wants. A deal-breaker is a hard rule and belongs in the fetch gate, so the form has no must-haves box; `must_text` is set only through the API. Culture is left out because it is guessed from ad text, and location because the fetch gate does it.
|
||||||
|
|
||||||
**The model reports facts and code applies the rules.** For each must-have line the model answers "met", "broken" only when the posting clearly contradicts it, or "unclear" when the text does not say, never inferred from silence; a must-have it leaves out counts as unclear. The cap is code's: a broken must-have makes career count as at most `BROKEN_CAP` (4) when a role is served (`app/models.py`), so the stored career stays the model's and the reader shows both. The CV match weights are applied when a role is served too, so changing them needs no model call. The verdicts store each line's text, so the reader still shows what a judgement was made against after the text is edited. The `input_key` carries `judge.FORMAT` (4) and the texts, so a change to the prompt's contract makes every judgement again.
|
**The model reports facts and code applies the rules.** For each must-have line the model answers "met", "broken" only when the posting clearly contradicts it, or "unclear" when the text does not say, never inferred from silence; a must-have it leaves out counts as unclear. The cap is code's: a broken must-have makes career count as at most `BROKEN_CAP` (4) when a role is served (`app/models.py`), so the stored career stays the model's and the reader shows both. The CV match weights are applied when a role is served too, so changing them needs no model call. The verdicts store each line's text, so the reader still shows what a judgement was made against after the text is edited. The `input_key` carries `judge.FORMAT` (4) and the texts, so a change to the prompt's contract makes every judgement again.
|
||||||
@@ -236,7 +244,7 @@ What was decided for roledesk and the reason for each. The target design is the
|
|||||||
|
|
||||||
**Search agents are cards, edited in place.** One person may run several agents: their own hunt and one that watches roles for someone else. An agent is defined by what it looks for, so its prompts are the card's body and the weights a small line under it. Edit turns that card into the form and New puts a blank one on top, because a form at the bottom of the page was out of sight of the card it changed. Job boards and Watchlist companies are edited the same way, one open at a time.
|
**Search agents are cards, edited in place.** One person may run several agents: their own hunt and one that watches roles for someone else. An agent is defined by what it looks for, so its prompts are the card's body and the weights a small line under it. Edit turns that card into the form and New puts a blank one on top, because a form at the bottom of the page was out of sight of the card it changed. Job boards and Watchlist companies are edited the same way, one open at a time.
|
||||||
|
|
||||||
**A gate the form cannot say lands in a row under Advanced, so a load and a save lose nothing.** The form asks the fetch gate in its four parts by name rather than as the stored `require` and `any` sets with ISO codes, which the owner could not read; any other gate is a row under Other rules. Spellings of a town that differ by accents only share one chip, and both are stored.
|
**The form's gates are the four parts of the fetch gate, and nothing else** (ADR Fetch prep and score 5.2 and Settings). Free rows of field, mode and values over twelve fields sat under Advanced, first to keep a rule an older form had saved; a row on seniority or pay was a permanent cut on fields half the roles leave empty, which the rail's filters do with a click and undo. The rows are gone; a stored gate the form cannot say is named under the fetch gate and kept as it is on save, so nothing is lost silently.
|
||||||
|
|
||||||
**Viewing as on Roles is a select, also with one agent.** The owner looked for the select to switch agent and found it gone after the second agent was switched off. It lists the enabled agents, since an agent that is off has no fresh scores.
|
**Viewing as on Roles is a select, also with one agent.** The owner looked for the select to switch agent and found it gone after the second agent was switched off. It lists the enabled agents, since an agent that is off has no fresh scores.
|
||||||
|
|
||||||
|
|||||||
+6
-36
@@ -1,48 +1,18 @@
|
|||||||
# Open
|
# Open
|
||||||
|
|
||||||
What is known and not yet fixed, checked against the code on 2026-10-05, the day of the architecture review. What was decided is in `decisions.md`, the target design in `documentation/`, the unbuilt readers in `plan-hosts-wave-2.md`. Counts are of the live database on that day.
|
What is known and not yet fixed, checked against the code on 2026-10-06, after the architecture review of 2026-10-05. What was decided is in `decisions.md`, the target design in `documentation/`, the unbuilt readers in `plan-hosts-wave-2.md`. Counts are of the live database on that day.
|
||||||
|
|
||||||
## For the owner to decide
|
## For the owner to decide
|
||||||
|
|
||||||
Points where the code and `ADR Fetch prep and score` disagree, or where the ADR is silent and roles are lost.
|
- **Remote on hosts that take one country a request.** SmartRecruiters, SuccessFactors and Eightfold (14 boards, about 15,000 roles) are still asked for DK, SE alone when the agents want remote, since Europe there is the whole board. Reading them whole is the only way to their remote roles elsewhere.
|
||||||
|
- **Remote listings across countries.** A job posted remote once per country is now one role per country (the fold keeps to one country), which lists a pan-European remote posting several times.
|
||||||
- **Remote roles on hosts that cannot filter on remote.** On 18 boards (SmartRecruiters, Phenom, SuccessFactors, Eightfold, and Workday tenants without a remote facet) the read for "DK, SE, remote" goes out as "DK, SE". The row says "remote not asked" and the remote roles already held are kept, but a new remote role elsewhere in the world gate is never fetched. ADR 1.2 ("a host that cannot filter on a part sends the board whole for that part") reads as reading such boards whole; Bosch is then about 50 requests a pull.
|
|
||||||
- **A role outside the agent's locations with remote unstated is never read by extract.** ADR 4.1 says a role can "turn out to be remote", but extract reads only the cut and the cut requires the gate. 5,016 open roles are rejected for location with remote unknown, and 805 of them say "remote" in their text. A word scan in prep costs no tokens and would let those through to extract.
|
|
||||||
- **Copies across countries.** ADR 1.4 requires the same country and overlapping towns. `dupes.fold` also merges one host's listings of one title across towns and countries when 30% of their words are shared: 2,125 open copies, about 1,550 in another country than the kept role, some of them different openings (Datadog's US federal role behind the Canberra one). The gate reads the kept role through its copies' places, so no office listing is hidden. Remote and office listings are kept apart since 2026-10-05; 176 remote listings still sit behind a remote kept role in another country, whose area alone the world gate reads.
|
|
||||||
- **Judge and languages.** ADR 4.2 names the agent's languages as an input. The judge does not read them (they only set the flag on a row), and editing them re-judges nothing. The ADR's list also omits the prompts and the must-haves, which the judge reads and which re-judge.
|
|
||||||
- **Must-haves.** A broken must-have caps career at 4, but `must_text` is set only through the API: it is not on the form and not in the ADR's Settings.
|
|
||||||
- **Show gated and Advanced rules.** ADR 5 says no order shows a role the gate rejects and that seniority and pay are filters. Show gated lists the rejected roles with the reason, on purpose; an Advanced row can gate on any of twelve fields.
|
|
||||||
- **Closing by age on boards.** ADR 1.3 closes a role no read has listed for 21 days. That holds for searches and for boards narrowed by terms. A board read that stays incomplete (a ceiling, the page cap without a sweep) closes nothing ever, and on a filtered board a role with any vote or an application never closes. Since closing is not valued, striking the rule from the ADR is the cheaper way to agree.
|
|
||||||
- **A search cut at its page cap leaves no record.** In the last pull 4 of 26 searches stopped at a cap (Jobbank and freehire at 1,000 rows, two Jobindex searches at 200). Nothing on a page says so, and Jobbank's feed is not sorted by date, so new ads can lie past the cap.
|
- **A search cut at its page cap leaves no record.** In the last pull 4 of 26 searches stopped at a cap (Jobbank and freehire at 1,000 rows, two Jobindex searches at 200). Nothing on a page says so, and Jobbank's feed is not sorted by date, so new ads can lie past the cap.
|
||||||
- **A row without a title, link or id is dropped by its reader, unrecorded,** while its id still counts toward completeness. `ADR Plugins` says both "never drops rows" and "title and url are required"; the runner could validate, log and mark the read incomplete.
|
|
||||||
- **The world gate is not sent to a host when an agent sets no location.** Every big board is then read whole. Nothing is lost.
|
|
||||||
|
|
||||||
## What the ADRs do not say
|
## What the ADRs still do not say
|
||||||
|
|
||||||
What someone rebuilding from `ADR Fetch prep and score` would have to guess, by section.
|
`ADR Fetch prep and score` was brought up to the code on 2026-10-06; left for the owner's read as a whole.
|
||||||
|
|
||||||
- **Prep has no section**, though the title names it: a host's words mapped through a table per host, a place text resolved against a gazetteer of towns, regions, countries and continents with regions across borders, remote and the remote area read from the place words, the seniority ladder, and that a text nothing maps is listed and passes the gate.
|
`ADR Plugins` was rewritten on 2026-10-06 and names a target the code has not reached: a broken plugin left out instead of stopping the app; providers declaring their params with label, kind and choices, so form, CLI and API follow from the module; detail state kept for a search's roles as for a board's; a national provider declaring its implied country; page markers for hosts on a company's own domain; a contract test that names no reader.
|
||||||
- **1.1 Fetch gate.** Every role is stored whatever the gates say, and each agent applies its gate at scoring. An unknown location is no resolved place and no country, which includes an unresolved text and a wider region alone. A role in several places needs one inside. The world gate reads a remote role by its remote area. Exclusion matches whole words of the title.
|
|
||||||
- **1.2 Reading a board.** The union is per agent the location gate, or the world gate when no location is set, then over the agents. What happens when a host filters countries but not remote. A board's own saved params win over the agents'. With no enabled agent every board is read whole. A search reads 5 pages by default and 10 at most.
|
|
||||||
- **1.3 Closing.** "Filtered read" is not defined. Switching a board or search off closes its roles; a closed role listed again reopens.
|
|
||||||
- **1.4 Copies.** The second rule, one host's listings of one title merged when their texts are alike. Groups never chain. Votes and read state move to the kept copy. How titles and company names are normalised.
|
|
||||||
- **2.1 Closeness.** The embedding model (`intfloat/multilingual-e5-small` with its query and passage prefixes) and what is embedded (title and the first 2,000 characters of the text, else the teaser). The CV is cut into chunks of at most 1,200 characters. A level is a linear clip between two cosines measured per model (prompt 0.75 to 0.88, CV 0.77 to 0.90, job like -0.14 to 0.19). With several prompts each is scaled and the best counts. The bonus divides by all four weights also when one has left the sum.
|
|
||||||
- **2.2 Filter.** The bullets carry the "and" and the "or" by layout only. A liked or saved role must still pass the gate. A like is this agent's; a save under any agent counts.
|
|
||||||
- **3 Fetch text.** Only hosts with a role page are read, Jobindex never in a pull, and the caps (200 a board, 1,000 a run, retry after 7 days) are in `ADR Plugins` only.
|
|
||||||
- **4 Model analytics.** The model of each call and the batch sizes (20, 5, 10). Extract reads the first 2,500 and last 1,500 characters and also fills pay, currency, function and work permit; it gives up after 3 unanswered tries. The judge reads up to 8,000 characters, rates career against the prompts, likes and dislikes, and the CV match is 10 times the weighted mean, 0 to 100. Summarise writes about 60 words and 3 to 8 tags, once. A stage that is off, or has no model to ask, is skipped without a notice; only Settings › Sources › Runs says so. A cut role without text is neither judged nor summarised.
|
|
||||||
- **5 Feed.** Open roles and kept copies only. Opening or voting marks a role read. The list of filters, 60 rows a page, that ★ also likes, and what a role without an embedding scores. Newest is a third order, where the glossary and the flow say two.
|
|
||||||
- **Settings.** The agent's On/Off switch and the pull's schedule (every 6 h).
|
|
||||||
|
|
||||||
Wording to settle: 2.1 "over the sum of the want feed score weights" against 5.1.3 "its weight leaves the sum"; 4 "once extract has completed the ingestion" against 4.1 "a role that enters is extracted on the next pull"; 4.1's field list against "raw and processed fields for e.g. salary"; "model analytics score" is never defined (the number is the CV match); 1.3 against `ADR Plugins` 3.2, which lets a search declare `stated`, and 2.1, which says an incomplete read closes nothing; the title is "Architecture" while the other ADR calls it by its file name.
|
|
||||||
|
|
||||||
`ADR Plugins`, read lightly since it is not final:
|
|
||||||
|
|
||||||
- **The API block already differs from the code:** `empty` is "trusted" or "suspect", not None; `Page` is a plain tuple; `Reads` has an unused `title="provisional"`; a provider must define `allowed`; a reader may import `desk`.
|
|
||||||
- **Rules the code needed and the ADR lacks:** a cursor must survive JSON, where a tuple returns as a list; a reader that sees a page come again hands it over empty with a cursor; a wrong-shaped answer is an error, never an empty list.
|
|
||||||
- **Rules the code does not keep:** "at most 8 series" is enforced only with terms or more than 8 countries; the 10% guard starts at 1 role, not 10, for filtered, narrowed and suspect reads; a 429 after page one starts a sweep; a module that breaks the contract stops the import at the first problem, and with it the app.
|
|
||||||
- **"Never maps a word"** is broken by `Place.remote`, which makes every reader decide whether "hybrid" is office, and by the national providers that state their country (Jobindex DK, Platsbanken SE).
|
|
||||||
- **"Nothing else changes"** holds for a host. A provider with a new param or a key touches the Sources form, its script, the CLI, the API body and `sources.search_params`.
|
|
||||||
- Cases 4 and 5 are handled after the plugin, so "upstream" reads the wrong way round. The constants (0.5 s, 100 pages, 96 MiB, 21 and 7 days) will go stale faster than the rules.
|
|
||||||
|
|
||||||
## Structure
|
## Structure
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# Architecture
|
# ADR Fetch prep and score
|
||||||
|
|
||||||
The target design of roledesk's ranking: how a role gets from a board to the Roles feed for one search agent, and the key decisions behind it. What gets built is tested against this document.
|
The target design of roledesk's ranking: how a role gets from a board to the Roles feed for one search agent, and the key decisions behind it. What gets built is tested against this document.
|
||||||
|
|
||||||
- **Fetch gate**: the agent's hard rules, a yes or no per role, in four parts: world, location, role exclusion and role fetch.
|
- **Fetch gate**: the agent's hard rules, a yes or no per role, in four parts: world, location, role exclusion and role fetch.
|
||||||
- **Input analytics score** and **input analytics filter**: what decides the **cut**, the roles that pass the analytics filter and get fetch text and the `model analytics score`.
|
- **Input analytics score** and **input analytics filter**: what decides the **cut**, the roles that pass the analytics filter and get fetch text and the `model analytics score`, which is the `CV match` (4.2.1).
|
||||||
- **Feed scores**: the two orders of the feed, the want feed score and the match feed score.
|
- **Feed scores**: the two scores that order the feed, the want feed score and the match feed score; Newest is the third order.
|
||||||
- **Fetch text**: the second HTTP read of a role's own page, without a model.
|
- **Fetch text**: the second HTTP read of a role's own page, without a model.
|
||||||
- **Extract**, **judge**, **summarise**: the model calls (Haiku).
|
- **Extract**, **judge**, **summarise**: the model calls (Haiku).
|
||||||
- **Affinity**: an agent's setting for a company: hidden, neutral, liked or favourite.
|
- **Affinity**: an agent's setting for a company: hidden, neutral, liked or favourite.
|
||||||
@@ -17,6 +17,7 @@ The target design of roledesk's ranking: how a role gets from a board to the Rol
|
|||||||
world · location · role exclusion · role fetch, with All locations and All roles as overrides
|
world · location · role exclusion · role fetch, with All locations and All roles as overrides
|
||||||
company boards: whole under 300 roles, else filtered at the host where it can
|
company boards: whole under 300 roles, else filtered at the host where it can
|
||||||
search providers: their own saved query
|
search providers: their own saved query
|
||||||
|
then prep, no tokens: host words, places, remote, country; the copies folded
|
||||||
│
|
│
|
||||||
▼
|
▼
|
||||||
2 Input analytics score, no tokens ─ prompt closeness, CV closeness, affinity → the cut
|
2 Input analytics score, no tokens ─ prompt closeness, CV closeness, affinity → the cut
|
||||||
@@ -28,7 +29,7 @@ The target design of roledesk's ranking: how a role gets from a board to the Rol
|
|||||||
4 Model analytics score ─ extract the cut → fetch gate and cut again → judge the cut
|
4 Model analytics score ─ extract the cut → fetch gate and cut again → judge the cut
|
||||||
│
|
│
|
||||||
▼
|
▼
|
||||||
5 Feed ─ every role that passes the fetch gate, in two orders;
|
5 Feed ─ every role that passes the fetch gate, in three orders;
|
||||||
filters and saved filters narrow it
|
filters and saved filters narrow it
|
||||||
│
|
│
|
||||||
▼
|
▼
|
||||||
@@ -51,7 +52,7 @@ The stricter location filter. A role passes the `location fetch gate` when it ma
|
|||||||
|
|
||||||
#### 1.1.3 Role exclusion gate
|
#### 1.1.3 Role exclusion gate
|
||||||
|
|
||||||
A list of titles to leave out: phrases that reject a role by its title. No host can filter by exclusion, so this gate is applied to every role after fetch.
|
A list of titles to leave out: phrases that reject a role by its title, matched on whole words. No host can filter by exclusion, so this gate is applied to every role after fetch.
|
||||||
|
|
||||||
#### 1.1.4 Role fetch gate
|
#### 1.1.4 Role fetch gate
|
||||||
|
|
||||||
@@ -64,26 +65,34 @@ The fetch terms: words a big board's host must match before it sends a role. The
|
|||||||
- A search provider is read by its own saved query, which replaces the `role fetch gate` (1.1.4).
|
- A search provider is read by its own saved query, which replaces the `role fetch gate` (1.1.4).
|
||||||
- A hidden company never passes.
|
- A hidden company never passes.
|
||||||
|
|
||||||
The gate rejects on evidence only. A role with an unknown location passes until extract has read it.
|
The gate rejects on evidence only. A role with an unknown location passes until extract has read it. A location is unknown when no place of the role resolved and no country is stated, an unresolved place text included. A place that holds the agent's places (Øresund or Europe for Denmark) is no evidence against the role, unless the role also names a place elsewhere. A role in several places passes when one is inside. A hybrid or onsite role at an unknown location passes. A role whose text says remote while its fields do not (1.5) passes the location gate until extract has read it. The world gate reads a remote role by its remote area.
|
||||||
|
|
||||||
### 1.2 Reading a board
|
### 1.2 Reading a board
|
||||||
|
|
||||||
- A company board with fewer than 300 roles is read whole.
|
- A company board with fewer than 300 roles is read whole.
|
||||||
- A company board with 300 or more roles is asked for the fetch gate at the host: the world and location gates, and the fetch terms, where the host can filter on them. A host that cannot filter on a part sends the board whole for that part.
|
- A company board with 300 or more roles is asked for the fetch gate at the host: the world and location gates, and the fetch terms, where the host can filter on them. A host that cannot filter on a part sends the board whole for that part.
|
||||||
- A board's size is measured once, when it is added or rechecked, not on every read.
|
- A board's size is measured once, when it is added or rechecked, not on every read.
|
||||||
- One fetch serves all enabled agents. The host is asked for the union of their world and location gates and the union of their fetch terms; one enabled agent without fetch terms means no terms. Each agent then applies its own gate to what arrived.
|
- One fetch serves all enabled agents. The host is asked for, per agent, its location gate, or its world gate when it sets no location, united over the agents, and for the union of their fetch terms; one enabled agent without fetch terms means no terms. Each agent then applies its own gate to what arrived.
|
||||||
|
- A host that filters on countries but not on remote is asked for the world gate's countries when an agent wants remote roles, so a remote role listed under any country in the world gate still arrives; the read says that remote was not asked. This holds where the host takes the world in one read; on a host that takes one country a request, the world is the whole board, so the location gate is asked and remote roles elsewhere are not fetched.
|
||||||
|
- A board's own saved params win over the agents' gates on the dimensions they set. With no enabled agent every board is read whole.
|
||||||
|
|
||||||
### 1.3 Closing
|
### 1.3 Closing
|
||||||
|
|
||||||
- A role closes when a complete read of its board no longer lists it.
|
- A role closes when a complete read of its board no longer lists it.
|
||||||
- A read that is not complete closes nothing: one narrowed by fetch terms, a search provider's, a partial or failed read. Roles on such boards close by age. This is accepted because ranking relies on recency, not on closing.
|
- A read that is not complete closes nothing: one narrowed by fetch terms, a search provider's, a partial or failed read. Roles on such boards close by age. This is accepted because ranking relies on recency, not on closing. A read filtered by location is complete for what it asked: it closes the roles inside its filter that it no longer lists.
|
||||||
|
- Switching a board or search off closes its roles. A closed role that a read lists again reopens. A search is read 5 pages deep, 10 at most.
|
||||||
- A role that no read has listed for 21 days closes. A board that has failed on every read for 21 days closes its roles.
|
- A role that no read has listed for 21 days closes. A board that has failed on every read for 21 days closes its roles.
|
||||||
- A liked or saved role is never closed by a filtered read.
|
- A liked or saved role is never closed by a filtered read.
|
||||||
|
|
||||||
### 1.4 Copies
|
### 1.4 Copies
|
||||||
|
|
||||||
- One posting listed several times (by a search provider and by the company's board, or by one board under several ids) is one role. Two listings are one posting when they share company, title and country, and their towns overlap; a listing without a town matches any town.
|
- One posting listed several times (by a search provider and by the company's board, or by one board under several ids) is one role. Two listings are one posting when they share company, title and country, and their towns overlap; a listing without a town matches any town.
|
||||||
- The kept copy is the company board's over a search provider's, then the one with text, then the earliest posted. Only the kept copy is scored, read by a model and shown; it lists the others.
|
- The kept copy is the company board's over a search provider's, then the one with text, then the earliest posted. Only the kept copy is scored, read by a model and shown; it lists the others. Votes and read state move to the kept copy.
|
||||||
|
- A host that posts one job once per office gives listings of one title whose towns never meet. Those of one host are one role when their texts share 30% of their words, within one country, remote and office kept apart; the kept copy carries every office's place.
|
||||||
|
|
||||||
|
### 1.5 Prep
|
||||||
|
|
||||||
|
After fetch, without a model: a host's words for seniority, employment and remote map through a table per host; each place text resolves against a gazetteer of towns, regions, countries and continents, where a region may lie in two countries; remote and the remote area are read from the place words; the country comes from the resolved places; a role whose title or text says remote ("fully remote", "work from home", not "no remote") is marked as possibly remote for the gate (1.1). A text nothing maps is listed under Unmapped for the owner to add, and until then the role's location counts as unknown.
|
||||||
|
|
||||||
## 2 Input analytics score and filter
|
## 2 Input analytics score and filter
|
||||||
|
|
||||||
@@ -94,17 +103,18 @@ Each role is embedded and scored semantically:
|
|||||||
- **Prompt closeness** is its level against the best of the agent's prompts.
|
- **Prompt closeness** is its level against the best of the agent's prompts.
|
||||||
- **CV closeness** is its level against the closest part of the CV.
|
- **CV closeness** is its level against the closest part of the CV.
|
||||||
|
|
||||||
Both are calibrated levels from 0 to 1. A company's affinity adds to both levels the share it has in the `want feed score`: the company like weight times the company like's rise above neutral (5.1.1), over the sum of the want feed score weights. On the start weights a liked company adds about 0.056, a favourite about 0.111. The bonus counts before the count and the minimum apply.
|
Both are calibrated levels from 0 to 1. A company's affinity adds to both levels the share it has in the `want feed score`: the company like weight times the company like's rise above neutral (5.1.1), over the sum of all four want feed score weights, also when one of them has left the sum. On the start weights a liked company adds about 0.056, a favourite about 0.111. The bonus counts before the count and the minimum apply.
|
||||||
|
|
||||||
|
The embedding is `intfloat/multilingual-e5-small`, with its query and passage prefixes, over the title and the first 2,000 characters of the text, else the teaser. The CV is cut into chunks of at most 1,200 characters and the closest chunk counts. A level is a linear clip of the cosine between two bounds measured per model: prompt closeness 0.75 to 0.88, CV closeness 0.77 to 0.90, job like -0.14 to 0.19. With several prompts each is scaled on its own and the best counts.
|
||||||
|
|
||||||
#### 2.2 Input analytics filter
|
#### 2.2 Input analytics filter
|
||||||
|
|
||||||
The `closeness`es are the main input for the `input analytics filter`, which determines which roles get `fetch text` and the `model analytics score`. A role passes the `input analytics filter` when it:
|
The `closeness`es are the main input for the `input analytics filter`, which determines which roles get `fetch text` and the `model analytics score`. A role passes the `input analytics filter` when it:
|
||||||
|
|
||||||
- passes the `fetch gate` (1.1.1 to 1.1.3)
|
- passes the `fetch gate` (1.1.1 to 1.1.3), and
|
||||||
|
|
||||||
- is among the best N by `prompt closeness`, each at least X (500 and 0.6 to start), or
|
- is among the best N by `prompt closeness`, each at least X (500 and 0.6 to start), or
|
||||||
- is among the best M by `CV closeness` that the first list leaves out, each at least Y (150 and 0.6 to start), or
|
- is among the best M by `CV closeness` that the first list leaves out, each at least Y (150 and 0.6 to start), or
|
||||||
- is liked or saved not matter the `closeness`.
|
- is liked by this agent or saved under any agent, whatever its `closeness`.
|
||||||
|
|
||||||
The two lists are kept apart because they answer different questions (what I want, what I am), and a merged number lets the stronger side crowd the other out. For a broad agent the count stops each list; for a narrow agent the minimum stops it, so roles that do not fit are not read just to fill the count.
|
The two lists are kept apart because they answer different questions (what I want, what I am), and a merged number lets the stronger side crowd the other out. For a broad agent the count stops each list; for a narrow agent the minimum stops it, so roles that do not fit are not read just to fill the count.
|
||||||
|
|
||||||
@@ -114,43 +124,42 @@ An agent without a CV has no CV list and no judge. An agent without prompts has
|
|||||||
|
|
||||||
## 3 Fetch text
|
## 3 Fetch text
|
||||||
|
|
||||||
Some job boards have one API for the listings and another for the job specs. `fetch text` reads the role's own page for every role in the cut that came without text. Once the full text is fetched, the role is embedded again and its levels recomputed. A role that enters the cut after this recompute gets its text on the next pull.
|
Some job boards have one API for the listings and another for the job specs. `fetch text` reads the role's own page for every role in the cut that came without text, on hosts that have a role page, at most 200 a board and 1,000 a run, a failed read retried after 7 days; Jobindex is never read in a pull, since each read counts as a click there. Once the full text is fetched, the role is embedded again, its levels recomputed, and extract reads it again if it had read the title alone. A role that enters the cut after this recompute gets its text on the next pull.
|
||||||
|
|
||||||
## 4 Model analytics
|
## 4 Model analytics
|
||||||
|
|
||||||
All roles in the cut go through the detailed analysis and metadata extraction. Extract and judge are two calls: `extract` needs no CV, it is faster, and it is the final step of the ingestion, so it decides what finally passes the `analytics filter`.
|
All roles in the cut go through the detailed analysis and metadata extraction. Extract and judge are two calls: `extract` needs no CV, it is faster, and it is the final step of the ingestion, so it decides what finally passes the `analytics filter`. All three calls go to Haiku (`claude-haiku-4-5`), extract 20 roles a call, judge 5, summarise 10. A stage that is switched off, or has no model to ask, is skipped, and the Runs page says so. A cut role without text is neither judged nor summarised.
|
||||||
|
|
||||||
### 4.1 Extract
|
### 4.1 Extract
|
||||||
|
|
||||||
Often information is only in the free text, e.g. that a role in a specific location is actually remote. So every role has raw and processed fields for e.g. salary, location, remote and seniority. Extract reads what is hidden in the free text. A field the board states wins over what extract reads, except seniority and remote, where extract wins.
|
Often information is only in the free text, e.g. that a role in a specific location is actually remote. So every role has raw and processed fields for e.g. salary, location, remote and seniority. Extract reads what is hidden in the free text. A field the board states wins over what extract reads, except seniority and remote, where extract wins.
|
||||||
|
|
||||||
**Extract** reads the cut first. It fills location, remote area, seniority, languages and deadline.
|
**Extract** reads the cut first, the first 2,500 and last 1,500 characters of each role. It fills location, remote and remote area, seniority, languages, deadline, pay and currency, function and work permit. A role the model leaves unanswered three times is given up.
|
||||||
|
|
||||||
The fetch gate and the cut are then applied again with what extract filled. A role can drop out, such as one that turns out to be outside the agent's locations, or enter, such as one that turns out to be remote. A role that enters is extracted on the next pull.
|
The fetch gate and the cut are then applied again with what extract filled. A role can drop out, such as one that turns out to be outside the agent's locations, or enter, such as one that turns out to be remote. A role that enters is judged in this pull and extracted on the next.
|
||||||
|
|
||||||
### 4.2 Judge
|
### 4.2 Judge
|
||||||
|
|
||||||
Once extract has completed the ingestion, `judge` reads each role in the cut against the agent's CV, languages, likes and dislikes. It outputs:
|
Once extract has completed the ingestion, `judge` reads each role in the cut, up to 8,000 characters of it, against the agent's CV, languages, prompts, likes and dislikes. It outputs:
|
||||||
|
|
||||||
- the `CV match`
|
- the `CV match`
|
||||||
- one to three strengths and gaps
|
- one to three strengths and gaps
|
||||||
- a verdict on each must-have
|
|
||||||
|
|
||||||
Each role is extracted once. A judgement is redone only when its input changes: a change to the CV, a prompt, or the like or dislike text re-judges the whole cut (about an hour for 650 roles); a changed posting re-judges that role only.
|
Each role is judged once. A judgement is redone only when its input changes: a change to the CV, the languages, a prompt, or the like or dislike text re-judges the whole cut (about an hour for 650 roles); a changed posting re-judges that role only.
|
||||||
|
|
||||||
#### 4.2.1 CV match
|
#### 4.2.1 CV match
|
||||||
|
|
||||||
Judge rates how closely the job spec matches the experience in the CV on three dimensions:
|
Judge rates how closely the job spec matches the experience in the CV on three dimensions:
|
||||||
|
|
||||||
- skills,
|
- skills, where a language the posting requires above the level spoken counts as a missing skill,
|
||||||
- experience,
|
- experience,
|
||||||
- career,
|
- career, rated against the prompts, likes and dislikes rather than the CV,
|
||||||
|
|
||||||
each from 0 to 10. A model's rating is an estimate, not a measurement, so a coarse scale serves better than 0 to 100. The **CV match** is the three numbers weighted as given in the agent settings (30/25/30 to start). A must-have the role breaks caps career at 4. Roles with the same CV match are ordered by CV closeness.
|
each from 0 to 10. A model's rating is an estimate, not a measurement, so a coarse scale serves better than 0 to 100. The **CV match** is the three numbers weighted as given in the agent settings (30/25/30 to start), times 10, so 0 to 100. Roles with the same CV match are ordered by CV closeness.
|
||||||
|
|
||||||
## 5 Feed
|
## 5 Feed
|
||||||
|
|
||||||
The feed shows every role that passes the `world fetch gate`, the `location fetch gate` and the `role exclusion gate` (1.1.1 to 1.1.3), loaded as you scroll, in every order, newest first included. No order shows a role the fetch gate rejects. A hidden company's roles are never shown.
|
The feed shows every open role, kept copies only, that passes the `world fetch gate`, the `location fetch gate` and the `role exclusion gate` (1.1.1 to 1.1.3), loaded as you scroll, 60 at a time, in every order, newest first included. No order shows a role the fetch gate rejects. A hidden company's roles are never shown. Opening a role or voting on it marks it read; Show read, on by default, hides the read ones when switched off. Saving a role (★) also likes it.
|
||||||
|
|
||||||
### 5.1 Feed scores
|
### 5.1 Feed scores
|
||||||
|
|
||||||
@@ -161,6 +170,7 @@ The `want feed score` consists of `prompt closeness`, newness, company likes and
|
|||||||
- `newness` counts from the posting date, the earlier of when the app first saw the role and the date the board states (only for boards whose dates are reliable), and halves every 14 days.
|
- `newness` counts from the posting date, the earlier of when the app first saw the role and the date the board states (only for boards whose dates are reliable), and halves every 14 days.
|
||||||
- `company like` is 1/3 for a neutral company, 2/3 for a liked one and 1 for a favourite.
|
- `company like` is 1/3 for a neutral company, 2/3 for a liked one and 1 for a favourite.
|
||||||
- `job like` is described in 5.1.3.
|
- `job like` is described in 5.1.3.
|
||||||
|
- A role without an embedding yet is scored without `prompt closeness` and `job like`; their weights leave the sum.
|
||||||
|
|
||||||
#### 5.1.2 Match feed score, "What fits me"
|
#### 5.1.2 Match feed score, "What fits me"
|
||||||
|
|
||||||
@@ -185,16 +195,17 @@ The user likes or passes on roles in the feed, per agent. Each role has an embed
|
|||||||
|
|
||||||
- The Roles page says that the feed is already filtered by the agent's fetch gate, location included.
|
- The Roles page says that the feed is already filtered by the agent's fetch gate, location included.
|
||||||
- The company view says what is filtered: which parts of the fetch gate apply to this company, for example "every role in Europe, not filtered by country" for an All locations company. Looking up a company never silently hides roles elsewhere.
|
- The company view says what is filtered: which parts of the fetch gate apply to this company, for example "every role in Europe, not filtered by country" for an All locations company. Looking up a company never silently hides roles elsewhere.
|
||||||
|
- Show gated lists the roles the fetch gate rejects, each with the reason, so what is kept out can be checked. It is the one way a rejected role is shown.
|
||||||
|
|
||||||
## 6 After the ranking
|
## 6 After the ranking
|
||||||
|
|
||||||
Summarise is not part of any score. It runs after the feed scores, on the top N of the `want feed score` and the top N of the `match feed score` (N set per agent, 30 to start, so at most 60 roles), and on every role liked or saved.
|
Summarise is not part of any score. It runs after the feed scores, on the top N of the `want feed score` and the top N of the `match feed score` (N set per agent, 30 to start, so at most 60 roles), and on every role liked or saved. It writes about 60 words and 3 to 8 tags, once per role, and needs the role's text.
|
||||||
|
|
||||||
## Settings
|
## Settings
|
||||||
|
|
||||||
Per agent, on the form:
|
Per agent, on the form:
|
||||||
|
|
||||||
- name and prompts
|
- name, an On/Off switch, and prompts
|
||||||
- the fetch gate: world (the outer countries), location (countries, towns, remote), role exclusion (titles to leave out), role fetch (fetch terms)
|
- the fetch gate: world (the outer countries), location (countries, towns, remote), role exclusion (titles to leave out), role fetch (fetch terms)
|
||||||
- CV, languages, likes and dislikes
|
- CV, languages, likes and dislikes
|
||||||
- the analytics filter: count and minimum by prompt closeness, count and minimum by CV closeness
|
- the analytics filter: count and minimum by prompt closeness, count and minimum by CV closeness
|
||||||
@@ -202,3 +213,5 @@ Per agent, on the form:
|
|||||||
- summarise size
|
- summarise size
|
||||||
|
|
||||||
Per agent, per company: affinity (hidden, neutral, liked, favourite). Per company, for every agent: All locations and All roles.
|
Per agent, per company: affinity (hidden, neutral, liked, favourite). Per company, for every agent: All locations and All roles.
|
||||||
|
|
||||||
|
The pull runs every 6 hours.
|
||||||
@@ -5,104 +5,112 @@ What a plugin has to deliver so that the flow in `ADR Fetch prep and score` gets
|
|||||||
## Terms
|
## Terms
|
||||||
|
|
||||||
- **Source**: a site that lists roles.
|
- **Source**: a site that lists roles.
|
||||||
- **Plugin** (reader): the code for one kind of source. There are two kinds:
|
- **Plugin**: one module for one kind of source. There are two kinds:
|
||||||
- a **host**, which reads a company's own list on an ATS (Workday, Greenhouse, ...). That list is a **board**, attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
|
- a **host**, which reads a company's own list on an ATS (Workday, Greenhouse, ...). That list is a **board**, attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
|
||||||
- a **provider**, which runs a **search**, a saved query on a site that lists roles across companies (Jobindex, Jobnet, ...).
|
- a **provider**, which runs a **search**, a saved query on a site that lists roles across companies (Jobindex, Jobnet, ...).
|
||||||
- **Runner**: the shared code that calls a plugin. It asks, pages, retries, guards and stamps, so no plugin repeats that.
|
- **Runner**: the shared code that calls a plugin. It asks, pages, retries, guards, validates and stamps, so no plugin repeats that. It has one driver per kind, for boards and for searches, which differ only in what they store.
|
||||||
- **Row**: one role as a plugin hands it over, in the host's own words.
|
- **Row**: one role as a plugin hands it over, in the host's own words.
|
||||||
- **Declaration**: what a plugin states it can do (`READS`). The runner acts on the declaration, never on guesses.
|
- **Declaration**: what a plugin states it can do (`READS`). The runner acts on the declaration, never on guesses.
|
||||||
|
- **Read**: one pull's requests to one target, and what they yielded. A read is **complete** when it listed every role the target has, else **incomplete**; the runner stamps which.
|
||||||
|
|
||||||
## 1 What a plugin delivers
|
## 1 What a plugin delivers
|
||||||
|
|
||||||
A plugin turns one source into rows. It:
|
A plugin turns one source into rows. It:
|
||||||
|
|
||||||
- hands over every role the source lists for what it was asked, as one row per role the host lists;
|
- hands over every role the source lists for what it was asked, one row per listing, as the host lists them;
|
||||||
- asks the host only for the filters it was handed, by building them into the host's query;
|
- asks the host only for the filters it was handed, by building them into the host's query;
|
||||||
- keeps the host's own words for places, remote, seniority, employment and country;
|
- keeps the host's own words for places, remote, seniority, employment and country;
|
||||||
- declares truthfully what the source can do: how the list arrives, where the text is, how completeness is known, what it can filter and which fields it states.
|
- declares truthfully what the source can do: how the list arrives, where the text is, how completeness is known, what it can filter and which fields it states;
|
||||||
|
- raises on anything it cannot read: an error page, an answer of another shape, a page that is not the page it asked for. An empty list is a valid answer only when the host says the list is empty.
|
||||||
|
|
||||||
A plugin never:
|
A plugin never:
|
||||||
|
|
||||||
- filters, drops or reorders rows itself;
|
- filters, drops or reorders rows; a listing it cannot make a row of (no id, no title or no link) it hands over as far as it can, and the runner drops and counts it (3.8);
|
||||||
- maps a word to the app's vocabulary;
|
- maps a word to the app's vocabulary;
|
||||||
- merges, splits or multiplies rows;
|
- merges, splits or multiplies rows;
|
||||||
- decides closing, retries or failure;
|
- decides closing, retries or failure;
|
||||||
- reads robots.txt at run time.
|
- reads robots.txt at run time;
|
||||||
|
- logs in, runs a page's script, or uses a key taken from a site's script.
|
||||||
|
|
||||||
## 2 The five cases
|
## 2 The five cases
|
||||||
|
|
||||||
A source can present roles in five ways that need handling. Three are the plugin's job and two are handled upstream.
|
A source can present roles in five ways that need handling. Three are the plugin's job and two are the app's, after the plugin.
|
||||||
|
|
||||||
| # | Case | Who handles it |
|
| # | Case | Who handles it |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| 1 | The list arrives in pages | Plugin, with the runner |
|
| 1 | The list arrives in pages | Plugin, with the runner |
|
||||||
| 2 | The list has no text; each role's text is on its own page | Plugin, with the fetch text stage |
|
| 2 | The list has no text; each role's text is on its own page | Plugin, with the fetch text stage |
|
||||||
| 3 | One role states several places | Plugin |
|
| 3 | One role states several places | Plugin |
|
||||||
| 4 | One job is listed several times | Upstream |
|
| 4 | One job is listed several times | The app |
|
||||||
| 5 | Facts such as remote are only in the free text | Upstream |
|
| 5 | Facts such as remote are only in the free text | The app |
|
||||||
|
|
||||||
### 2.1 Pagination (plugin, with the runner)
|
### 2.1 Pagination (plugin, with the runner)
|
||||||
|
|
||||||
The plugin says how to ask for one page and what the page holds. The runner does the rest.
|
The plugin says how to ask for one page and what the page holds. The runner does the rest.
|
||||||
|
|
||||||
- The plugin declares `listing="paged"`, the page size and the host's **ceiling**, the offset past which the host answers nothing whatever is asked (Workday 2,000).
|
- The plugin declares `listing="paged"`, the page size and the host's **ceiling**, the offset past which the host answers nothing whatever is asked (Workday 2,000).
|
||||||
- `page` asks for one page from a cursor and returns the page's rows, the raw results with their ids, the total the host states if any, and the next cursor, or none after the last page.
|
- `page` asks for one page from a cursor and returns the page's rows, the raw results with their ids, the total the host states if any, and the next cursor, or none after the last page. The first call gets cursor none. A cursor is any JSON value; a tuple comes back as a list when the runner hands it over again (a sweep), and a plugin reads it by position.
|
||||||
- The runner reads the pages one after another, 0.5 s apart, at most 100 pages a pull. A board above 100 pages is **swept**: each pull continues from the saved cursor.
|
- The runner reads the pages one after another, 0.5 s apart, at most 100 pages a pull. A board above 100 pages is **swept**: each pull continues from the saved cursor, series and ids, and the sweep is finished when the read reaches the end.
|
||||||
- **Completeness**: a read is complete when the distinct ids read equal the stated total (`complete="stated"`), or when it reaches an empty page (`complete="end"`). A read that reaches the ceiling is incomplete, is not swept and closes nothing; only a filter brings such a board under the ceiling.
|
- **Completeness**: a read is complete when the distinct ids read equal the stated total (`complete="stated"`), or when the plugin hands over the last page with no cursor (`complete="end"`: an empty page, or a short one where the host has no empty page). A read that reaches the ceiling is incomplete, is not swept and closes nothing; only a filter brings such a board under the ceiling.
|
||||||
|
- **Stalls**: a page that brings no new id while the plugin hands over a cursor is a stall when the host states no total, or when the page came empty: the host ignores the cursor, or sent a page again. A plugin that sees a page come again (the same first id as the page before) hands it over with no results and a cursor that is not none, never none, since none would read as the last page. A stalled read is incomplete. With a stated total, a page of ids already read is not a stall, since a resumed sweep starts on one when roles were added at the top since.
|
||||||
- A role of a swept board closes only when two finished sweeps in a row missed it, since roles closing between pulls shift later roles back past the cursor.
|
- A role of a swept board closes only when two finished sweeps in a row missed it, since roles closing between pulls shift later roles back past the cursor.
|
||||||
- A 429 after the first page ends the read. What was read is kept, and the read is incomplete.
|
- A 429 after the first page ends the read. What was read is kept, the read is incomplete, and a board continues from there in the next pull as a sweep.
|
||||||
|
- A plugin whose list comes in one answer declares `listing="whole"` and `complete="whole"`; a whole answer that parses is complete. One whose list comes in one answer that states a total it does not fill raises.
|
||||||
|
|
||||||
### 2.2 List without text (plugin, with the fetch text stage)
|
### 2.2 List without text (plugin, with the fetch text stage)
|
||||||
|
|
||||||
Some hosts list title, place and date only, and state the text on each role's own page.
|
Some hosts list title, place and date only, and state the text on each role's own page.
|
||||||
|
|
||||||
- The plugin declares `text="detail"` and supplies `detail`, which reads one role's page and returns the fields it found. A list that carries a short excerpt hands it over as `teaser` and declares `teaser=True`.
|
- The plugin declares `text="detail"` and supplies `detail`, which reads one role's page and returns the fields it found. A list that carries a short excerpt hands it over as `teaser` and declares `teaser=True`.
|
||||||
- The plugin does not decide which roles are read. The fetch text stage calls `detail` after scoring, only for roles in the cut (`ADR Fetch prep and score`, section 3), at most 200 per board and 1,000 per run.
|
- The plugin does not decide which roles are read. The fetch text stage calls `detail` after scoring, only for roles in the cut (`ADR Fetch prep and score`, section 3), at most 200 per board and 1,000 per run, 0.5 s apart.
|
||||||
- `detail` may fill only description, location, places, teaser, remote, country, employment, pay figures, currency and the posting date. Title, link, department and the pay line come from the list, which states them on every pull.
|
- `detail` may fill only description, location, places, teaser, remote, country, employment, pay figures, currency and the posting date. Title, link, department and the pay line come from the list, which states them on every pull.
|
||||||
- A role read in detail is sent by later pulls without the list's place and remote, so what its own page said stays.
|
- A role read in detail is sent by later pulls without the list's place and remote, so what its own page said stays. This holds for a search's roles as for a board's.
|
||||||
- A 404 or 410 counts as read. Another failure is retried on a later run. A read that gave no text is retried after 7 days. A 429 ends the reads of that board for the run.
|
- A 404 or 410 counts as read. Another failure is retried on a later run, after the roles never tried. A read that gave no text is retried after 7 days. A 429 ends the reads of that target for the run.
|
||||||
- Completeness, and so closing, is decided by the list alone.
|
- Completeness, and so closing, is decided by the list alone.
|
||||||
|
|
||||||
### 2.3 One role, several places (plugin)
|
### 2.3 One role, several places (plugin)
|
||||||
|
|
||||||
- A row carries `places`: every place the host states for the role, in the host's order and the host's words, each marked remote, office or unsaid.
|
- A row carries `places`: every place the host states for the role, in the host's order and the host's words, each marked remote, office or unsaid as the host marks that place. The host's own word for how the role is worked ("Hybrid", "Remote", "On-site") goes on the row as `remote`, unmapped.
|
||||||
- A plugin never makes one row per place, never drops a place and never joins places into one text.
|
- A plugin never makes one row per place, never drops a place and never joins places into one text.
|
||||||
- A host that states no place declares `places="none"` and hands over an empty list.
|
- A host that states no place declares `places="none"` and hands over an empty list.
|
||||||
- Resolving a place text to a town, region or country is the app's job.
|
- Resolving a place text to a town, region or country is the app's job.
|
||||||
|
|
||||||
### 2.4 One job listed several times (upstream)
|
### 2.4 One job listed several times (the app)
|
||||||
|
|
||||||
One job can come as several rows: one per office, each with its own id; several requisition ids for the same job; or a search provider and the company's board listing the same role.
|
One job can come as several rows: one per office, each with its own id; several requisition ids for the same job; or a search provider and the company's board listing the same role.
|
||||||
|
|
||||||
- The plugin hands every listing over as its own row, with its own id and link. A host that publishes one posting per place declares `places="copies"`.
|
- The plugin hands every listing over as its own row, with its own id and link. A host that publishes one posting per place declares `places="copies"`.
|
||||||
- Folding them into one role, and choosing the copy that is kept, happens upstream (`ADR Fetch prep and score`, section 1.4).
|
- Folding them into one role, and choosing the copy that is kept, happens in the app (`ADR Fetch prep and score`, section 1.4).
|
||||||
|
- Two listings under one id are one row; a host that lists one id once per place hands over one row with all the places.
|
||||||
|
|
||||||
### 2.5 Facts in the free text (upstream)
|
### 2.5 Facts in the free text (the app)
|
||||||
|
|
||||||
- The plugin passes on only what the host states in a field: remote, country, seniority, employment, pay and date, in the host's words, and declares each as `stated` or `none`.
|
- The plugin passes on only what the host states in a field: remote, country, seniority, employment, pay and date, in the host's words, and declares each as `stated` or `none`. A provider that covers one country states that country on every row and declares it `stated`.
|
||||||
- Mapping the host's words to the app's vocabulary happens upstream, through a table per host.
|
- Mapping the host's words to the app's vocabulary happens in the app, through a table per host.
|
||||||
- Facts found only in the prose, e.g. a Copenhagen role that is actually remote, are read upstream: from the place words first, then by extract (`ADR Fetch prep and score`, section 4.1).
|
- Facts found only in the prose, e.g. a Copenhagen role that is actually remote, are read in the app: from the place words first, then by extract (`ADR Fetch prep and score`, section 4.1).
|
||||||
|
|
||||||
## 3 Further rules
|
## 3 Further rules
|
||||||
|
|
||||||
### 3.1 Filtering at the host
|
### 3.1 Filtering at the host
|
||||||
|
|
||||||
- The plugin declares per dimension (country, place, remote, query) whether the host filters it: `none`, `one` (one value a request), `many` (a list in one request) or `ids` (the host's own ids, looked up with `facets`).
|
- The plugin declares per dimension (country, place, remote, query) whether the host filters it: `none`, `one` (one value a request), `many` (a list in one request) or `ids` (the host's own ids, looked up with `facets`).
|
||||||
- The runner hands the plugin an `Ask` that holds only declared dimensions. With `one`, it runs one series of requests per value and unions the rows by id. A read costs at most 8 series. Above that the runner narrows less: a read for the location gate keeps the places and drops the fetch terms; a read of an All locations company keeps the terms and drops the place part; more than 8 terms alone are dropped.
|
- The runner hands the plugin an `Ask` that holds only declared dimensions. With `one`, it runs one series of requests per value and unions the rows by id. A read costs at most 8 series. Above that the runner narrows less: more than 8 countries on a `one` host means a read without countries; when terms times the location series pass 8, a read for the location gate keeps the places and drops the terms, and a read of an All locations company keeps the terms and drops the place part; more than 8 terms alone are dropped.
|
||||||
- "In these countries, or remote" cannot be asked in one request, because two facets intersect. The runner makes remote a series of its own, so a plugin never gets remote and a country in the same `Ask`.
|
- "In these countries, or remote" cannot be asked in one request, because two facets intersect. The runner makes remote a series of its own, so a plugin never gets remote and a country in the same `Ask`. A host that cannot filter on remote is asked for the countries alone, and the read is stamped as not having asked for remote, which the app shows and spares the remote roles it holds from closing.
|
||||||
- A host with `query="one"` also takes the fetch terms, one term a series, handed over as `query`.
|
- A host with `query="one"` also takes the fetch terms, one term a series, handed over as `query`.
|
||||||
- A host's facet ids are looked up at most once a day per board.
|
- A host's facet ids are looked up at most once a day per board. A value the host's facets do not know is dropped; a read for which nothing remains asks nothing, lists nothing and closes nothing.
|
||||||
- Which boards are filtered at all, and by what, is decided upstream (`ADR Fetch prep and score`, section 1.2).
|
- A board's own saved params win over the agents' reach on the dimensions they set.
|
||||||
|
- Which boards are filtered at all, and by what, is decided in the app (`ADR Fetch prep and score`, section 1.2).
|
||||||
|
|
||||||
### 3.2 Completeness and closing
|
### 3.2 Completeness and closing
|
||||||
|
|
||||||
The plugin declares how completeness is known (`stated`, `end`, `whole` for a one-answer list, `never`). The runner stamps each read as complete or not. Closing follows `ADR Fetch prep and score`, section 1.3. A search is never a company's whole list, so it declares `never` or `stated`.
|
The plugin declares how completeness is known (`stated`, `end`, `whole` for a one-answer list, `never`). The runner stamps each read as complete or not, as filtered (a location filter), as narrowed (fetch terms or a provider's query), and with the ids it listed. Closing follows `ADR Fetch prep and score`, section 1.3: a search never closes a role by its list, whatever it declares, so a provider's `stated` only says how far a read got.
|
||||||
|
|
||||||
### 3.3 Failures
|
### 3.3 Failures
|
||||||
|
|
||||||
- A plugin lets every error through: HTTP errors, timeouts, parse errors. The runner logs, records and decides.
|
- A plugin lets every error through: HTTP errors, timeouts, parse errors, an answer of another shape. The runner logs, records and decides. An error on a later page fails the whole read; what a 429 read is kept (2.1).
|
||||||
- A read that lists under 10% of what the same read listed last time, once that was at least 10 roles, counts as failed. This catches a host that answers an empty list for a mistyped board; such a host declares `empty="suspect"`.
|
- A read that lists under 10% of what the same read listed last time counts as failed. For a whole read of a board this holds once the last count was at least 10 roles; a filtered or narrowed read, and a read of a host that answers an empty list for an unknown slug (`empty="suspect"`), is held from a count of 1. "The same read" is one with the same filters: a changed filter starts the count again.
|
||||||
- A board that fails on every read for 21 days closes its roles, and a suspect answer that has stood that long is accepted.
|
- A board that fails on every read for 21 days closes its roles, and a suspect answer that has stood that long is accepted.
|
||||||
|
- A failing target is named on its company's row and in the pull's notice, with the reason.
|
||||||
|
|
||||||
### 3.4 Fetching
|
### 3.4 Fetching
|
||||||
|
|
||||||
@@ -114,13 +122,15 @@ Every request goes through the runner's `get`:
|
|||||||
- 10 s to connect, 30 s between bytes, 60 s for the whole request;
|
- 10 s to connect, 30 s between bytes, 60 s for the whole request;
|
||||||
- at most 96 MiB a response. A plugin may name a smaller fallback feed, read once when the main one is over the cap.
|
- at most 96 MiB a response. A plugin may name a smaller fallback feed, read once when the main one is over the cap.
|
||||||
|
|
||||||
|
The figures in this document are the defaults as they stand; the rules are what counts.
|
||||||
|
|
||||||
### 3.5 Searches
|
### 3.5 Searches
|
||||||
|
|
||||||
A provider has saved params instead of a board. It declares which filter dimension each param fills, cleans a param it would not accept with a reason a person can read, and receives the saved params in `Ask.params`. A search row carries `company`, the employer's name. A provider has no slug patterns, probes or careers page, and takes no fetch terms: its query replaces them. A provider that needs an API key declares it, and the runner hands it the key.
|
A provider has saved params instead of a board. It declares each param: the filter dimension it fills, its label, its kind (text, a choice from a table, a list of country codes) and the table of choices where there is one, so the form, the command line and the API offer a provider's params from the declaration alone. It cleans a param it would not accept with a reason a person can read, and receives the saved params in `Ask.params`. A search row carries `company`, the employer's name. A provider has no slug patterns, probes or careers page, and takes no fetch terms: its query replaces them. A provider that needs an API key declares it, and the runner hands it the key; a search reads 5 pages by default and 10 at most.
|
||||||
|
|
||||||
### 3.6 Discovery
|
### 3.6 Discovery
|
||||||
|
|
||||||
A host can be found from a careers page link: its slug patterns are regexes over a URL, with the slug in group 1, or several groups joined by `/`. A host may offer a cheap probe URL that answers 2xx only for a real board, and a check of the probe's body where 2xx is not enough. The slug is lowercased unless the host's paths are case-sensitive.
|
A host can be found from a careers page link: its slug patterns are regexes over a URL, with the slug in group 1, or several groups joined by `/`. A host may offer a cheap probe URL that answers 2xx only for a real board, and a check of the probe's body where 2xx is not enough. The slug is lowercased unless the host's paths are case-sensitive. A host whose boards live on a company's own domain names the page markers that identify it, and the path discovery confirms with.
|
||||||
|
|
||||||
### 3.7 robots.txt
|
### 3.7 robots.txt
|
||||||
|
|
||||||
@@ -128,18 +138,22 @@ Whoever builds a plugin reads the robots.txt of every host it requests, for the
|
|||||||
|
|
||||||
### 3.8 Rows
|
### 3.8 Rows
|
||||||
|
|
||||||
- `source_id` is the host's id, unique on the board or search. `title` and `url` (http or https) are required.
|
- `source_id` is the host's id, unique on the board or search. `title` and `url` (http or https) are required. The runner, not the plugin, drops a row that lacks one of them; the read's record counts the rows dropped, and a board whose dropped rows pass 10% of its list counts as failed, since a changed page shape looks like that.
|
||||||
- `description` and `teaser` are plain text, never HTML.
|
- `description` and `teaser` are plain text, never HTML.
|
||||||
- `posted` is an aware datetime or none. `posted="trusted"` declares it the posting date; `listed` declares a date the host moves, e.g. on an edit.
|
- `posted` is an aware datetime or none. `posted="trusted"` declares it the posting date; `listed` declares a date the host moves, e.g. on an edit. A plugin whose only date comes from the role's page or from a field the host moves declares `listed`.
|
||||||
- Pay figures are yearly integers with an ISO 4217 currency. A monthly figure is multiplied by 12 where the host states the period. An hourly, daily or weekly figure is kept as a pay line only.
|
- Pay figures are yearly integers with an ISO 4217 currency. A monthly figure is multiplied by 12 where the host states the period. An hourly, daily or weekly figure is kept as a pay line only.
|
||||||
- `country` is ISO 3166 alpha-2, and only where the host states it.
|
- `country` is ISO 3166 alpha-2, and only where the host states it (or the provider covers that one country, 2.5).
|
||||||
- Any other key the host states is kept as raw, so a new field needs no change in the runner.
|
- Any other key the host states is kept as raw, so a new field needs no change in the runner.
|
||||||
|
|
||||||
|
### 3.9 What the runner keeps per target
|
||||||
|
|
||||||
|
So that a plugin keeps nothing: the last read's marker, whether it was complete, filtered, narrowed and asked for remote, the ids it listed, the sweep in progress, the roles read and failed in detail, the facet ids and when they were read, the board's size and when it was measured, the last count the guard compares with, and the failure record with its first date and reason. All of it is shown or summarised on the board's row.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## API
|
## API
|
||||||
|
|
||||||
A plugin is one module, named after its source, importing only the shared plugin helpers, the standard library, `httpx` and `pydantic`. It registers by existing: a module that breaks the contract fails at import, naming every problem.
|
A plugin is one module, named after its source, importing only the shared plugin helpers, the standard library, `httpx`, `pydantic` and the shared model base. It registers by existing: a module that breaks the contract is refused at import, every problem named; the other plugins load, and the Sources page names the refused one.
|
||||||
|
|
||||||
### Module attributes
|
### Module attributes
|
||||||
|
|
||||||
@@ -149,7 +163,11 @@ LABEL = "Workday" # shown in the source select
|
|||||||
PATTERNS = (r"...",) # host only: discovery regexes over a URL, IGNORECASE
|
PATTERNS = (r"...",) # host only: discovery regexes over a URL, IGNORECASE
|
||||||
RANK = 100 # optional: order in the select and in probes, lower first
|
RANK = 100 # optional: order in the select and in probes, lower first
|
||||||
KEEP_CASE = False # optional: True keeps a discovered slug's case
|
KEEP_CASE = False # optional: True keeps a discovered slug's case
|
||||||
PARAMS = {"q": "query"} # provider only: saved param -> filter dimension it fills
|
MARKERS = ("...",) # optional, host on a company's own domain: page markers
|
||||||
|
PARAMS = { # provider only: saved param -> what it is
|
||||||
|
"q": Param(fills="query", label="Query", kind="text"),
|
||||||
|
"region": Param(fills="place", label="Region", kind="choice", choices=REGIONS),
|
||||||
|
}
|
||||||
|
|
||||||
READS = Reads(
|
READS = Reads(
|
||||||
scope="board", # "board" | "search"
|
scope="board", # "board" | "search"
|
||||||
@@ -167,7 +185,7 @@ READS = Reads(
|
|||||||
# pay "figures" | "line" | "none"; posted "trusted" | "listed" | "none";
|
# pay "figures" | "line" | "none"; posted "trusted" | "listed" | "none";
|
||||||
# the rest "stated" | "none"
|
# the rest "stated" | "none"
|
||||||
key=None, # Key(name, fields, env) when an API key is needed
|
key=None, # Key(name, fields, env) when an API key is needed
|
||||||
empty=None, # "suspect" when an unknown slug answers an empty list
|
empty="trusted", # "suspect" when an unknown slug answers an empty list
|
||||||
robots="read: ... (checked 2026-09-30)", # required, with the date
|
robots="read: ... (checked 2026-09-30)", # required, with the date
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
@@ -175,10 +193,12 @@ READS = Reads(
|
|||||||
### Functions
|
### Functions
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
# Every plugin
|
||||||
|
def allowed(slug: str) -> Callable[[str], bool] # the host names a request may reach
|
||||||
|
|
||||||
# Every host
|
# Every host
|
||||||
def slug_ok(slug: str) -> bool # full match; refuses anything that adds a path,
|
def slug_ok(slug: str) -> bool # full match; refuses anything that adds a path,
|
||||||
# query, fragment, port or user to a URL
|
# query, fragment, port or user to a URL
|
||||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a request may reach
|
|
||||||
def careers_url(slug: str) -> str # the page a person opens
|
def careers_url(slug: str) -> str # the page a person opens
|
||||||
|
|
||||||
# listing="whole"
|
# listing="whole"
|
||||||
@@ -213,18 +233,20 @@ def probe_slugs(guess: str) -> tuple[str, ...] # slugs a guessed company name
|
|||||||
```python
|
```python
|
||||||
get(url, *, method="GET", payload=None, headers=None) -> bytes
|
get(url, *, method="GET", payload=None, headers=None) -> bytes
|
||||||
# POST sends payload as JSON. Raises httpx.HTTPStatusError on non-2xx,
|
# POST sends payload as JSON. Raises httpx.HTTPStatusError on non-2xx,
|
||||||
# httpx.HTTPError on timeout or connection failure, ValueError on a refused
|
# httpx.HTTPError on timeout or connection failure, and the runner's own
|
||||||
# host, an oversized body or a broken redirect. A plugin lets these through.
|
# errors on a refused host, an oversized body or a broken redirect. A
|
||||||
|
# plugin lets these through.
|
||||||
|
|
||||||
Ask(countries=(), places=(), remote=None, remote_ids=(), query=None, terms=(), params={})
|
Ask(countries=(), places=(), remote=None, remote_ids=(), query="", terms=(), params={})
|
||||||
# only declared dimensions are set; with "one" a series carries one value;
|
# only declared dimensions are set; with "one" a series carries one value;
|
||||||
# with "ids" countries and places hold host ids; never remote together with
|
# with "ids" countries and places hold host ids; never remote together with
|
||||||
# a country or place; terms are handed over one a series as query, so a
|
# a country or place; terms are handed over one a series as query, so a
|
||||||
# plugin reads query only; params holds a search's saved params and a key under "key"
|
# plugin reads query only; params holds a search's saved params and a key under "key"
|
||||||
|
|
||||||
Page(decoded, results, total, cursor)
|
Page = (decoded, results, total, cursor)
|
||||||
# decoded: handed to rows; results: raw dicts with an "id", counted by
|
# decoded: handed to rows; results: raw dicts with an "id", counted by
|
||||||
# distinct id; total: stated by the host, or None; cursor: next page, or None
|
# distinct id; total: stated by the host, or None; cursor: next page, or
|
||||||
|
# None after the last; any JSON value, a tuple returning as a list
|
||||||
|
|
||||||
Place = {"text": str, "remote": bool | None}
|
Place = {"text": str, "remote": bool | None}
|
||||||
# the host's words; True remote, False office, None unsaid
|
# the host's words; True remote, False office, None unsaid
|
||||||
@@ -237,20 +259,24 @@ Row = {
|
|||||||
"country": str, "seniority": str, "employment": str,
|
"country": str, "seniority": str, "employment": str,
|
||||||
"company": str, # search rows only
|
"company": str, # search rows only
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Param(fills, label, kind, choices=None)
|
||||||
|
# fills: "query" | "country" | "place" | "remote" | "pages" | "" (the host's own);
|
||||||
|
# kind: "text" | "choice" | "countries" | "number"; choices: {value: label}
|
||||||
```
|
```
|
||||||
|
|
||||||
### Helpers the plugins share
|
### Helpers the plugins share
|
||||||
|
|
||||||
- `place(text, remote=None)` makes a `Place`, remote when the text says remote; `add_place(found, entry)` appends one unless it is empty or already there.
|
- `place(text, remote=None)` makes a `Place`, remote when the text says remote; `add_place(found, entry)` appends one unless it is empty or already there.
|
||||||
- `html_text(html)` turns HTML into plain text; `parse_dt(value)` makes an aware datetime; `yearly_amount(...)` makes a yearly figure; `is_web_url(url)` checks a link.
|
- `html_text(html)` turns HTML into plain text, one line per paragraph, with list items and headings marked; `parse_dt(value)` makes an aware datetime; `yearly_amount(...)` makes a yearly figure and `pay_line(...)` a pay line; `is_web_url(url)` checks a link.
|
||||||
- `path_slug_ok(slug)` and `label_slug_ok(slug)` check a slug that is one URL path segment or one DNS label; `only(host)` is the usual `allowed`.
|
- `path_slug_ok(slug)` and `label_slug_ok(slug)` check a slug that is one URL path segment or one DNS label; `only(host)` is the usual `allowed`.
|
||||||
- A table of the country names hosts write.
|
- A table of the country names hosts write, both ways (`code_of`, `name_of`).
|
||||||
|
|
||||||
### Adding a plugin
|
### Adding a plugin
|
||||||
|
|
||||||
1. Write the module with its declaration and functions.
|
1. Write the module with its declaration and functions.
|
||||||
2. Add a table of the host's words for seniority, employment and remote, when the host states them in its own words.
|
2. Add a table of the host's words for seniority, employment and remote, when the host states them in its own words.
|
||||||
3. Add fixtures from real answers, and tests. A contract test checks each declared field against the fixtures: a field declared `stated` must show on some row, a field declared `none` on none.
|
3. Add fixtures from real answers, and tests. A contract test, shared by every plugin and naming none, checks each declared field against the fixtures: a field declared `stated` must show on some row, a field declared `none` on none; required row keys, plain text, aware dates, ISO codes and integer pay; that results carry ids and cursors survive JSON; that a declared filter reaches the request.
|
||||||
4. Record the robots.txt decision in the declaration.
|
4. Record the robots.txt decision in the declaration.
|
||||||
|
|
||||||
Nothing else changes: the list of plugins, the source select, discovery and the filters a board is read with all follow from the module.
|
Nothing else changes: the list of plugins, the source select, discovery, a provider's form, the filters a board is read with and what its row says all follow from the module.
|
||||||
+14
-6
@@ -17,8 +17,9 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|||||||
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
||||||
its slug is. Every board is read independently; one that fails is logged,
|
its slug is. Every board is read independently; one that fails is logged,
|
||||||
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
||||||
whose read lists under a tenth of what it listed last time (`_runner.read`'s
|
whose read lists under a tenth of what it listed last time, or drops more
|
||||||
guard), until it has failed that way for the pull's `stale_days`. A board's
|
than a tenth of the rows its reader handed over (`_runner.read`'s guards),
|
||||||
|
until it has failed that way for the pull's `stale_days`. A board's
|
||||||
filter is the `reach` option the pull hands it (the agents', see
|
filter is the `reach` option the pull hands it (the agents', see
|
||||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||||
@@ -343,8 +344,10 @@ SIZE_PAGES = 100
|
|||||||
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
||||||
"""How many roles board `target` lists, unfiltered and without terms: a
|
"""How many roles board `target` lists, unfiltered and without terms: a
|
||||||
whole host's listing, the total a `complete="stated"` host gives on page
|
whole host's listing, the total a `complete="stated"` host gives on page
|
||||||
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
|
1, else the listing read to the end, at most `SIZE_PAGES` pages. A read
|
||||||
as `_fetch_rows` does, so a board that does not answer is refused."""
|
counts the rows the runner dropped as well (`_listed`), as a stated total
|
||||||
|
does. Raises as `_fetch_rows` does, so a board that does not answer is
|
||||||
|
refused."""
|
||||||
if not is_known_target(target):
|
if not is_known_target(target):
|
||||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||||
if client is None:
|
if client is None:
|
||||||
@@ -358,7 +361,12 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
|||||||
if isinstance(total, int):
|
if isinstance(total, int):
|
||||||
return total
|
return total
|
||||||
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
|
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
|
||||||
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
return _listed(_runner.read_listing(module, slug, get, limits=limits))
|
||||||
|
|
||||||
|
|
||||||
|
def _listed(listing: _runner.Listing) -> int:
|
||||||
|
"""The listings a read found: the ids it kept and the rows it dropped."""
|
||||||
|
return len(listing.ids) + listing.dropped
|
||||||
|
|
||||||
|
|
||||||
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
||||||
@@ -462,7 +470,7 @@ class ATSSource(Source):
|
|||||||
[compound(i) for i in listing.ids])
|
[compound(i) for i in listing.ids])
|
||||||
self.state.pop(failing, None)
|
self.state.pop(failing, None)
|
||||||
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
||||||
keep_size(self.state, target, len(listing.ids), reset=False)
|
keep_size(self.state, target, _listed(listing), reset=False)
|
||||||
if detail_sets is not None:
|
if detail_sets is not None:
|
||||||
# A complete listing drops the ids it no longer lists.
|
# A complete listing drops the ids it no longer lists.
|
||||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
|
|||||||
|
|
||||||
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
|
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
|
||||||
own id for the role, unique on the board), `title`, `url` (http or https,
|
own id for the role, unique on the board), `title`, `url` (http or https,
|
||||||
checked with `_shared.is_web_url`) and `places`; `location` (the host's
|
which the runner checks with `_shared.is_web_url`) and `places`; `location` (the host's
|
||||||
one-line place, kept while storage reads it), `posted` (an aware datetime
|
one-line place, kept while storage reads it), `posted` (an aware datetime
|
||||||
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
|
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
|
||||||
see `_shared.html_text`), `remote` (True, False, a word or None) and
|
see `_shared.html_text`), `remote` (True, False, a word or None) and
|
||||||
@@ -187,7 +187,12 @@ see `_shared.html_text`), `remote` (True, False, a word or None) and
|
|||||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
||||||
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
||||||
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
|
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
|
||||||
every stated key in `raw`, so a new field needs no change there.
|
every stated key in `raw`, so a new field needs no change there. A reader
|
||||||
|
never drops a listing: one it cannot make a whole row of (no id, no title or
|
||||||
|
no link) it hands over with what it has, an empty string for what is
|
||||||
|
missing, and the runner drops it and counts it (`_runner.valid`). A reader
|
||||||
|
leaves out only what is not a listing of the board: an entry that is not an
|
||||||
|
object, a second copy of an id, a role of another board.
|
||||||
|
|
||||||
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
||||||
remote=...)]`, one entry per place the host states, in the host's order, in
|
remote=...)]`, one entry per place the host states, in the host's order, in
|
||||||
|
|||||||
@@ -205,6 +205,8 @@ class Row(TypedDict, total=False):
|
|||||||
`company` is the employer's name on a search row, which lists many
|
`company` is the employer's name on a search row, which lists many
|
||||||
(`plugins/search.py` stores it); a board row carries none, the board is
|
(`plugins/search.py` stores it); a board row carries none, the board is
|
||||||
the company. Any other stated key is kept in `raw` by the runner.
|
the company. Any other stated key is kept in `raw` by the runner.
|
||||||
|
`source_id`, `title` and `url` come empty from a listing that lacks
|
||||||
|
them: the runner drops such a row and counts it (`_runner.valid`).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
source_id: Required[str]
|
source_id: Required[str]
|
||||||
|
|||||||
+98
-20
@@ -4,10 +4,12 @@ the reader declares in `READS`.
|
|||||||
A reader says how its host answers; this module reads it: the guarded fetch
|
A reader says how its host answers; this module reads it: the guarded fetch
|
||||||
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
||||||
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
||||||
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
series per `one`-filter value unioned by id (`read_listing`), the rows it
|
||||||
the sweep of an oversized paged board across pulls, the 10% guard on every
|
keeps (`valid`: a row without an id, a title or a link is dropped and
|
||||||
read that is not a sweep (`read`), the per-role detail reads
|
counted here, never by the reader), completeness, the sweep of an oversized
|
||||||
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
paged board across pulls, the two 10% guards (`read`: on what a read that
|
||||||
|
is not a sweep listed, and on what any read dropped), the per-role detail
|
||||||
|
reads (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||||
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
||||||
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
||||||
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
||||||
@@ -30,6 +32,8 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
|||||||
drifts when roles close between pulls, so `app.jobs.pull` closes
|
drifts when roles close between pulls, so `app.jobs.pull` closes
|
||||||
a role of such a board only when two finished sweeps in a row
|
a role of such a board only when two finished sweeps in a row
|
||||||
missed it (it keeps the ids the last one missed in `missing`)
|
missed it (it keeps the ids the last one missed in `missing`)
|
||||||
|
dropped the rows the last stamped read dropped (`Listing.dropped`), as a
|
||||||
|
number, "" for none; `app.jobs.limits` names it on the board's row
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -52,7 +56,7 @@ import httpx
|
|||||||
from desk.sources import USER_AGENT
|
from desk.sources import USER_AGENT
|
||||||
|
|
||||||
from ._contract import Ask, Key, Place, Reads
|
from ._contract import Ask, Key, Place, Reads
|
||||||
from ._shared import Get, Row
|
from ._shared import Get, Row, is_web_url
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -96,7 +100,8 @@ SUSPECT_SHARE = 0.1
|
|||||||
#: The last count from which the guard holds every read. A read that is
|
#: The last count from which the guard holds every read. A read that is
|
||||||
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
||||||
#: a count of 1, as it was before every read was; a whole read of a board
|
#: a count of 1, as it was before every read was; a whole read of a board
|
||||||
#: that listed a handful of roles can drop to none by itself.
|
#: that listed a handful of roles can drop to none by itself. It is also the
|
||||||
|
#: rows a read must hand over before the share it dropped can fail it.
|
||||||
SUSPECT_MIN = 10
|
SUSPECT_MIN = 10
|
||||||
|
|
||||||
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
||||||
@@ -113,8 +118,8 @@ class TooLarge(ValueError):
|
|||||||
|
|
||||||
|
|
||||||
class Suspect(Exception):
|
class Suspect(Exception):
|
||||||
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
|
"""A read that listed under `SUSPECT_SHARE` of what it listed last time,
|
||||||
(see `read`)."""
|
or dropped more than that share of its rows (see `read`)."""
|
||||||
|
|
||||||
|
|
||||||
class MissingKey(ValueError):
|
class MissingKey(ValueError):
|
||||||
@@ -147,7 +152,9 @@ class Listing:
|
|||||||
and roles closing meanwhile may have shifted an offset-paged host's later
|
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||||
roles back past the cursor, so `ids` can miss a role that is still open.
|
roles back past the cursor, so `ids` can miss a role that is still open.
|
||||||
`remote_unasked` says a filtered read was meant to ask for remote roles
|
`remote_unasked` says a filtered read was meant to ask for remote roles
|
||||||
as well and could not, so its `ids` say nothing of the remote roles."""
|
as well and could not, so its `ids` say nothing of the remote roles.
|
||||||
|
`handed` counts the rows the reader handed over this pull, `dropped`
|
||||||
|
those of them `valid` refused, which are in neither `rows` nor `ids`."""
|
||||||
|
|
||||||
rows: list[Row]
|
rows: list[Row]
|
||||||
complete: bool
|
complete: bool
|
||||||
@@ -158,6 +165,8 @@ class Listing:
|
|||||||
narrowed: bool = False
|
narrowed: bool = False
|
||||||
swept: bool = False
|
swept: bool = False
|
||||||
remote_unasked: bool = False
|
remote_unasked: bool = False
|
||||||
|
handed: int = 0
|
||||||
|
dropped: int = 0
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- modules
|
# --------------------------------------------------------------- modules
|
||||||
@@ -524,8 +533,42 @@ def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
|
|||||||
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||||
|
|
||||||
|
|
||||||
|
def valid(row: Mapping[str, Any]) -> bool:
|
||||||
|
"""Whether the runner keeps a row: a non-empty `source_id`, a non-empty
|
||||||
|
`title` and a web URL (`is_web_url`). A reader hands over every listing
|
||||||
|
as far as it can make it out; a row without one of the three cannot be
|
||||||
|
stored or opened, and the runner drops it and counts it."""
|
||||||
|
try:
|
||||||
|
link = is_web_url(str(row.get("url") or ""))
|
||||||
|
except (httpx.InvalidURL, ValueError): # a string httpx cannot read as a URL
|
||||||
|
link = False
|
||||||
|
return (link and bool(str(row.get("source_id") or "").strip())
|
||||||
|
and bool(str(row.get("title") or "").strip()))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class _Tally:
|
||||||
|
"""The rows a read was handed and dropped, across its series and pages."""
|
||||||
|
|
||||||
|
handed: int = 0
|
||||||
|
dropped: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def _keep(found: Iterable[Row], rows: dict[str, Row], tally: _Tally) -> None:
|
||||||
|
"""The rows a reader's `parse` or `rows` returned, into `rows` by
|
||||||
|
`source_id`, the first of an id kept; those `valid` refuses are counted
|
||||||
|
in `tally` and left out. The one place a row is checked, for the whole
|
||||||
|
and the paged listing alike."""
|
||||||
|
for row in found:
|
||||||
|
tally.handed += 1
|
||||||
|
if not valid(row):
|
||||||
|
tally.dropped += 1
|
||||||
|
continue
|
||||||
|
rows.setdefault(row["source_id"], row)
|
||||||
|
|
||||||
|
|
||||||
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||||
limits: Limits) -> dict[str, Row]:
|
limits: Limits, tally: _Tally) -> dict[str, Row]:
|
||||||
"""A whole listing, one request a series, unioned by `source_id`. A body
|
"""A whole listing, one request a series, unioned by `source_id`. A body
|
||||||
over the cap gets one retry at the reader's `fallback_url`."""
|
over the cap gets one retry at the reader's `fallback_url`."""
|
||||||
rows: dict[str, Row] = {}
|
rows: dict[str, Row] = {}
|
||||||
@@ -541,8 +584,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
|||||||
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
||||||
module.NAME, slug)
|
module.NAME, slug)
|
||||||
body = get(module.fallback_url(slug))
|
body = get(module.fallback_url(slug))
|
||||||
for row in module.parse(body, slug):
|
_keep(module.parse(body, slug), rows, tally)
|
||||||
rows.setdefault(row["source_id"], row)
|
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@@ -580,7 +622,29 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
only: a search's places are what the search is, as its query is.
|
only: a search's places are what the search is, as its query is.
|
||||||
`remote` says the reach or the params wanted remote roles; when no series
|
`remote` says the reach or the params wanted remote roles; when no series
|
||||||
asked for them (a reader without a remote filter, or one whose facets
|
asked for them (a reader without a remote filter, or one whose facets
|
||||||
state none beside places) the listing is `remote_unasked`."""
|
state none beside places) the listing is `remote_unasked`.
|
||||||
|
|
||||||
|
Every row `rows` or `parse` returns goes through `valid`: one without a
|
||||||
|
`source_id`, a title or a web URL is dropped here, counted in
|
||||||
|
`Listing.dropped` and logged once for the read. Completeness counts the
|
||||||
|
distinct result ids `page` returns beside its rows, not the rows kept, so
|
||||||
|
a dropped row still counts as read; a whole listing is complete whenever
|
||||||
|
it parses, whatever it dropped."""
|
||||||
|
tally = _Tally()
|
||||||
|
listing = _listing(module, slug, get, ask, limits=limits, sweep=sweep,
|
||||||
|
can_sweep=can_sweep, facets=facets, remote=remote, tally=tally)
|
||||||
|
listing.handed, listing.dropped = tally.handed, tally.dropped
|
||||||
|
if tally.dropped:
|
||||||
|
log.warning("readers: %s:%s: dropped %d of %d listings: no id, title or link",
|
||||||
|
module.NAME, slug, tally.dropped, tally.handed)
|
||||||
|
return listing
|
||||||
|
|
||||||
|
|
||||||
|
def _listing(module: ModuleType, slug: str, get: Get, ask: Ask | None, *,
|
||||||
|
limits: Limits | None, sweep: Mapping[str, Any] | None, can_sweep: bool,
|
||||||
|
facets: Mapping[str, Mapping[str, str]] | None, remote: bool,
|
||||||
|
tally: _Tally) -> Listing:
|
||||||
|
"""`read_listing` before its count of dropped rows, which `tally` keeps."""
|
||||||
limits = limits or Limits()
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
reads: Reads = module.READS
|
reads: Reads = module.READS
|
||||||
@@ -595,7 +659,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
"nothing asked, not closing", module.NAME, slug)
|
"nothing asked, not closing", module.NAME, slug)
|
||||||
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
||||||
if not hasattr(module, "page"):
|
if not hasattr(module, "page"):
|
||||||
whole_rows = _whole(module, slug, get, asks, limits)
|
whole_rows = _whole(module, slug, get, asks, limits, tally)
|
||||||
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
||||||
key, by_words, remote_unasked=unasked)
|
key, by_words, remote_unasked=unasked)
|
||||||
|
|
||||||
@@ -632,8 +696,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
pages += 1
|
pages += 1
|
||||||
before = len(seen)
|
before = len(seen)
|
||||||
seen |= _result_ids(results)
|
seen |= _result_ids(results)
|
||||||
for row in module.rows(data, slug):
|
_keep(module.rows(data, slug), rows, tally)
|
||||||
rows.setdefault(row["source_id"], row)
|
|
||||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||||
ended = True
|
ended = True
|
||||||
break
|
break
|
||||||
@@ -747,13 +810,26 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
|
|||||||
that really shrank is accepted once the target has been failing for
|
that really shrank is accepted once the target has been failing for
|
||||||
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
||||||
stamp then resets the count; a Recheck resets it at once
|
stamp then resets the count; a Recheck resets it at once
|
||||||
(`plugins.ats.keep_size`)."""
|
(`plugins.ats.keep_size`).
|
||||||
|
|
||||||
|
A read, a sweep's included, that dropped more than `SUSPECT_SHARE` of
|
||||||
|
the rows its reader handed over (`valid`), once it was handed at least
|
||||||
|
`SUSPECT_MIN`, raises `Suspect` the same way: a page whose shape changed
|
||||||
|
parses to rows without a title or a link, and the roles it hid would
|
||||||
|
close. It gives way after `limits.stale_days` of failing as well."""
|
||||||
limits = limits or Limits()
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||||
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
||||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
||||||
remote=remote)
|
remote=remote)
|
||||||
|
if listing.handed >= SUSPECT_MIN and listing.dropped > SUSPECT_SHARE * listing.handed:
|
||||||
|
why = (f"dropped {listing.dropped} of {listing.handed} listings, over "
|
||||||
|
f"{SUSPECT_SHARE:.0%}: no id, title or link")
|
||||||
|
if not _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
|
||||||
|
raise Suspect(why)
|
||||||
|
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
|
||||||
|
module.NAME, slug, why, limits.stale_days)
|
||||||
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
||||||
last = _load(state, f"{prefix}:count:{target}")
|
last = _load(state, f"{prefix}:count:{target}")
|
||||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||||
@@ -788,9 +864,10 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
|||||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||||
against), whether it finished a sweep from earlier pulls (`swept`),
|
against), whether it finished a sweep from earlier pulls (`swept`),
|
||||||
whether a filtered read left remote unasked (`remote_unasked`), the
|
whether a filtered read left remote unasked (`remote_unasked`), the rows
|
||||||
sweep to continue or none, and the count for the 10% guard once a read
|
it dropped (`dropped`, "" for none), the sweep to continue or none, and
|
||||||
is not a sweep. A read that asked nothing keeps the count it had."""
|
the count for the 10% guard once a read is not a sweep. A read that
|
||||||
|
asked nothing keeps the count it had."""
|
||||||
state[f"{prefix}:ok:{target}"] = run
|
state[f"{prefix}:ok:{target}"] = run
|
||||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||||
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||||
@@ -798,6 +875,7 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
|||||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||||
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
||||||
|
state[f"{prefix}:dropped:{target}"] = str(listing.dropped) if listing.dropped else ""
|
||||||
if listing.sweep is not None:
|
if listing.sweep is not None:
|
||||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from urllib.parse import urlencode
|
|||||||
from desk.sources import strip_html
|
from desk.sources import strip_html
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Key, Place, Reads
|
from ._contract import Ask, Fields, Filters, Key, Place, Reads
|
||||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
from ._shared import Get, Page, Row, only, parse_dt
|
||||||
|
|
||||||
NAME = "adzuna"
|
NAME = "adzuna"
|
||||||
LABEL = "Adzuna"
|
LABEL = "Adzuna"
|
||||||
@@ -111,8 +111,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
for job in decoded["results"]:
|
for job in decoded["results"]:
|
||||||
title = _text(job.get("title") or "")
|
title = _text(job.get("title") or "")
|
||||||
url = (job.get("redirect_url") or "").strip()
|
url = (job.get("redirect_url") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
||||||
town = ((job.get("location") or {}).get("display_name") or "").strip()
|
town = ((job.get("location") or {}).get("display_name") or "").strip()
|
||||||
places: list[Place] = [{"text": town, "remote": None}] if town else []
|
places: list[Place] = [{"text": town, "remote": None}] if town else []
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads, Row
|
from ._contract import Ask, Fields, Place, Reads, Row
|
||||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -126,8 +126,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in data.get("jobs") or []:
|
for j in data.get("jobs") or []:
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
||||||
compensation = j.get("compensation") or {}
|
compensation = j.get("compensation") or {}
|
||||||
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
|
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
|
||||||
|
|||||||
@@ -106,8 +106,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in listed:
|
for j in listed:
|
||||||
title = (j.get("jobOpeningName") or "").strip()
|
title = (j.get("jobOpeningName") or "").strip()
|
||||||
source_id = str(j.get("id") or "")
|
source_id = str(j.get("id") or "")
|
||||||
if not title or not source_id:
|
|
||||||
continue
|
|
||||||
places = _places(j)
|
places = _places(j)
|
||||||
row: Row = {
|
row: Row = {
|
||||||
"source_id": source_id,
|
"source_id": source_id,
|
||||||
|
|||||||
@@ -132,8 +132,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for p in postings:
|
for p in postings:
|
||||||
title = (p.get("title") or "").strip()
|
title = (p.get("title") or "").strip()
|
||||||
post_id = str(p.get("id") or "")
|
post_id = str(p.get("id") or "")
|
||||||
if not title or not post_id:
|
|
||||||
continue
|
|
||||||
job = p.get("job") or {}
|
job = p.get("job") or {}
|
||||||
places = [name for loc in job.get("jobLocations") or []
|
places = [name for loc in job.get("jobLocations") or []
|
||||||
if (name := ((loc.get("location") or {}).get("name") or "").strip())]
|
if (name := ((loc.get("location") or {}).get("name") or "").strip())]
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ from urllib.parse import quote
|
|||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads, Row
|
from ._contract import Ask, Fields, Filters, Place, Reads, Row
|
||||||
from ._countries import name_of
|
from ._countries import name_of
|
||||||
from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place
|
from ._shared import Get, Page, add_place, html_text, only, parse_dt, place
|
||||||
|
|
||||||
READS = Reads(
|
READS = Reads(
|
||||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||||
@@ -151,8 +151,6 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
|
|||||||
source_id = str(p.get("id") or "")
|
source_id = str(p.get("id") or "")
|
||||||
path = str(p.get("positionUrl") or "").strip()
|
path = str(p.get("positionUrl") or "").strip()
|
||||||
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
|
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
|
||||||
if not title or not source_id or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
places = _places(p.get("locations") or p.get("standardizedLocations"))
|
places = _places(p.get("locations") or p.get("standardizedLocations"))
|
||||||
out.append({
|
out.append({
|
||||||
"source_id": source_id,
|
"source_id": source_id,
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ from urllib.parse import urlencode, urlsplit, urlunsplit
|
|||||||
from desk.sources import USER_AGENT
|
from desk.sources import USER_AGENT
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||||
from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line,
|
from ._shared import (Get, Page, Row, html_text, num, only, parse_dt, pay_line,
|
||||||
yearly_amount)
|
yearly_amount)
|
||||||
|
|
||||||
NAME = "freehire"
|
NAME = "freehire"
|
||||||
@@ -170,8 +170,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
title = " ".join(html.unescape(str(hit.get("title") or "")).split())
|
title = " ".join(html.unescape(str(hit.get("title") or "")).split())
|
||||||
source_id = str(hit.get("public_slug") or "").strip()
|
source_id = str(hit.get("public_slug") or "").strip()
|
||||||
url = _url(str(hit.get("url") or ""))
|
url = _url(str(hit.get("url") or ""))
|
||||||
if not title or not source_id or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
mode = str(hit.get("work_mode") or "").strip()
|
mode = str(hit.get("work_mode") or "").strip()
|
||||||
places = _places(hit, mode)
|
places = _places(hit, mode)
|
||||||
countries = [str(c) for c in hit.get("countries") or [] if c]
|
countries = [str(c) for c in hit.get("countries") or [] if c]
|
||||||
|
|||||||
@@ -116,8 +116,6 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
|||||||
for p in postings:
|
for p in postings:
|
||||||
title = (p.get("title") or "").strip()
|
title = (p.get("title") or "").strip()
|
||||||
ext_id = str(p.get("extId") or "")
|
ext_id = str(p.get("extId") or "")
|
||||||
if not title or not ext_id:
|
|
||||||
continue
|
|
||||||
locations = p.get("locations") or []
|
locations = p.get("locations") or []
|
||||||
places: list[Place] = []
|
places: list[Place] = []
|
||||||
for loc in locations:
|
for loc in locations:
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads, Row
|
from ._contract import Ask, Fields, Place, Reads, Row
|
||||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
|
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place
|
||||||
|
|
||||||
NAME = "greenhouse"
|
NAME = "greenhouse"
|
||||||
LABEL = "Greenhouse"
|
LABEL = "Greenhouse"
|
||||||
@@ -149,8 +149,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in data.get("jobs") or []:
|
for j in data.get("jobs") or []:
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
url = (j.get("absolute_url") or "").strip()
|
url = (j.get("absolute_url") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
departments = j.get("departments") or []
|
departments = j.get("departments") or []
|
||||||
ranges = j.get("pay_input_ranges") or []
|
ranges = j.get("pay_input_ranges") or []
|
||||||
salary_min, salary_max, currency = _salary(ranges)
|
salary_min, salary_max, currency = _salary(ranges)
|
||||||
|
|||||||
@@ -90,8 +90,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for ad in data.get("jobAdDetails") or []:
|
for ad in data.get("jobAdDetails") or []:
|
||||||
title = (ad.get("title") or "").strip()
|
title = (ad.get("title") or "").strip()
|
||||||
ad_id = str(ad.get("id") or "").strip()
|
ad_id = str(ad.get("id") or "").strip()
|
||||||
if not title or not ad_id:
|
|
||||||
continue
|
|
||||||
site = str(ad.get("site") or "").strip()
|
site = str(ad.get("site") or "").strip()
|
||||||
country = str(ad.get("country") or "").strip()
|
country = str(ad.get("country") or "").strip()
|
||||||
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
|
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Reads, Row
|
from ._contract import Ask, Fields, Reads, Row
|
||||||
from ._shared import Get, Page, html_text, is_web_url, only, parse_dt
|
from ._shared import Get, Page, html_text, only, parse_dt
|
||||||
|
|
||||||
READS = Reads(
|
READS = Reads(
|
||||||
scope="board", listing="paged", page_size=100, text="listing", complete="end",
|
scope="board", listing="paged", page_size=100, text="listing", complete="end",
|
||||||
@@ -106,8 +106,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
|||||||
for item in items:
|
for item in items:
|
||||||
title = (item.findtext("title") or "").strip()
|
title = (item.findtext("title") or "").strip()
|
||||||
url = (item.findtext("link") or "").strip()
|
url = (item.findtext("link") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
out.append({
|
out.append({
|
||||||
"source_id": (item.findtext("guid") or url).strip(),
|
"source_id": (item.findtext("guid") or url).strip(),
|
||||||
"title": title,
|
"title": title,
|
||||||
|
|||||||
@@ -127,8 +127,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
j = job.get("data") or {}
|
j = job.get("data") or {}
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
role = str(j.get("slug") or j.get("req_id") or "")
|
role = str(j.get("slug") or j.get("req_id") or "")
|
||||||
if not title or not role:
|
|
||||||
continue
|
|
||||||
categories = j.get("categories") or []
|
categories = j.get("categories") or []
|
||||||
text = "\n".join(p for p in (html_text(j.get("description")),
|
text = "\n".join(p for p in (html_text(j.get("description")),
|
||||||
html_text(j.get("qualifications"))) if p)
|
html_text(j.get("qualifications"))) if p)
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from typing import Any
|
|||||||
from urllib.parse import urlencode, urlsplit
|
from urllib.parse import urlencode, urlsplit
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt, place
|
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt, place
|
||||||
|
|
||||||
NAME = "jobbank"
|
NAME = "jobbank"
|
||||||
LABEL = "Akademikernes Jobbank (Denmark)"
|
LABEL = "Akademikernes Jobbank (Denmark)"
|
||||||
@@ -132,8 +132,8 @@ def _query(ask: Ask, number: int) -> tuple[str, str]:
|
|||||||
|
|
||||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||||
"""One page of a search; the cursor is the page number. The next page
|
"""One page of a search; the cursor is the page number. The next page
|
||||||
follows a full page (counted before any item is dropped, so a dropped item
|
follows a full page (counted before the runner drops an item, so a
|
||||||
does not pass for the end) while the ceiling is not reached."""
|
dropped item does not pass for the end) while the ceiling is not reached."""
|
||||||
number = int(cursor or 1)
|
number = int(cursor or 1)
|
||||||
url, remote = _query(ask, number)
|
url, remote = _query(ask, number)
|
||||||
try:
|
try:
|
||||||
@@ -184,8 +184,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
out: list[Row] = []
|
out: list[Row] = []
|
||||||
for item in decoded["items"]:
|
for item in decoded["items"]:
|
||||||
link, title = item["link"], item["title"]
|
link, title = item["link"], item["title"]
|
||||||
if not title or not item["id"] or not is_web_url(link):
|
|
||||||
continue
|
|
||||||
line = _LINE.fullmatch(item["description"])
|
line = _LINE.fullmatch(item["description"])
|
||||||
kind = line.group("kind").strip() if line else ""
|
kind = line.group("kind").strip() if line else ""
|
||||||
company, regions, cut = _regions(line.group("rest")) if line else ("", [], False)
|
company, regions, cut = _regions(line.group("rest")) if line else ("", [], False)
|
||||||
|
|||||||
@@ -117,9 +117,9 @@ def _query(ask: Ask, number: int) -> str:
|
|||||||
|
|
||||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||||
"""One page of a search, newest first; the cursor is the page number.
|
"""One page of a search, newest first; the cursor is the page number.
|
||||||
The next page follows a full page (counted before any item is dropped, so
|
The next page follows a full page (counted before the runner drops an
|
||||||
a dropped item does not pass for the end) while the 200-result ceiling is
|
item, so a dropped item does not pass for the end) while the 200-result
|
||||||
not reached."""
|
ceiling is not reached."""
|
||||||
number = int(cursor or 1)
|
number = int(cursor or 1)
|
||||||
root = ET.fromstring(get(_query(ask, number)))
|
root = ET.fromstring(get(_query(ask, number)))
|
||||||
channel = root.find("channel")
|
channel = root.find("channel")
|
||||||
@@ -143,8 +143,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
out: list[Row] = []
|
out: list[Row] = []
|
||||||
for item in decoded["items"]:
|
for item in decoded["items"]:
|
||||||
link, full_title = item["link"], item["title"]
|
link, full_title = item["link"], item["title"]
|
||||||
if not full_title or not is_web_url(link):
|
|
||||||
continue
|
|
||||||
description = item["description"]
|
description = item["description"]
|
||||||
h4 = _H4.search(description)
|
h4 = _H4.search(description)
|
||||||
title = _text(h4.group(1)) if h4 else full_title
|
title = _text(h4.group(1)) if h4 else full_title
|
||||||
|
|||||||
@@ -138,8 +138,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
for ad in decoded.get("jobAds") or []:
|
for ad in decoded.get("jobAds") or []:
|
||||||
ident = str(ad.get("jobAdId") or "").strip()
|
ident = str(ad.get("jobAdId") or "").strip()
|
||||||
title = _text(ad.get("title"))
|
title = _text(ad.get("title"))
|
||||||
if not ident or not title:
|
|
||||||
continue
|
|
||||||
link = str(ad.get("jobAdUrl") or "").strip()
|
link = str(ad.get("jobAdUrl") or "").strip()
|
||||||
town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
|
town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
|
||||||
places: list[Place] = [place(town)] if town else []
|
places: list[Place] = [place(town)] if town else []
|
||||||
|
|||||||
@@ -176,8 +176,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
job_id = str(j.get("id") or "").strip()
|
job_id = str(j.get("id") or "").strip()
|
||||||
path = str(j.get("url") or "").strip()
|
path = str(j.get("url") or "").strip()
|
||||||
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
|
|
||||||
continue
|
|
||||||
function = str(j.get("function") or "").strip()
|
function = str(j.get("function") or "").strip()
|
||||||
places = [place(name) for name in dict.fromkeys(
|
places = [place(name) for name in dict.fromkeys(
|
||||||
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
|
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
|
||||||
@@ -186,7 +184,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
"title": title,
|
"title": title,
|
||||||
"places": places,
|
"places": places,
|
||||||
"location": "; ".join(pl["text"] for pl in places),
|
"location": "; ".join(pl["text"] for pl in places),
|
||||||
"url": f"https://{_EMP}{path}",
|
"url": f"https://{_EMP}{path}" if _ROLE_PATH.fullmatch(path) else "",
|
||||||
"posted": _published(j.get("published_date")),
|
"posted": _published(j.get("published_date")),
|
||||||
"department": "" if function == "All functions" else function,
|
"department": "" if function == "All functions" else function,
|
||||||
"description": "",
|
"description": "",
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads, Row
|
from ._contract import Ask, Fields, Place, Reads, Row
|
||||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -123,8 +123,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in data:
|
for j in data:
|
||||||
title = (j.get("text") or "").strip()
|
title = (j.get("text") or "").strip()
|
||||||
url = (j.get("hostedUrl") or "").strip()
|
url = (j.get("hostedUrl") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
categories = j.get("categories") or {}
|
categories = j.get("categories") or {}
|
||||||
description = j.get("description") or j.get("descriptionPlain") or ""
|
description = j.get("description") or j.get("descriptionPlain") or ""
|
||||||
salary_range = j.get("salaryRange") or {}
|
salary_range = j.get("salaryRange") or {}
|
||||||
|
|||||||
@@ -198,9 +198,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for role in search.get("requisitionList") or []:
|
for role in search.get("requisitionList") or []:
|
||||||
role_id = str(role.get("Id") or "").strip()
|
role_id = str(role.get("Id") or "").strip()
|
||||||
title = _text(role.get("Title"))
|
title = _text(role.get("Title"))
|
||||||
# The id goes into a URL, and a listing is external content.
|
# The id goes into a URL, and a listing is external content: an id
|
||||||
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id):
|
# of other characters gets no link, and the runner drops the row.
|
||||||
continue
|
safe = bool(re.fullmatch(r"[A-Za-z0-9_-]+", role_id))
|
||||||
remote = _text(role.get("WorkplaceTypeCode")) or None
|
remote = _text(role.get("WorkplaceTypeCode")) or None
|
||||||
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
|
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
|
||||||
places: list[Place] = []
|
places: list[Place] = []
|
||||||
@@ -214,7 +214,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
"title": title,
|
"title": title,
|
||||||
"places": places,
|
"places": places,
|
||||||
"location": "; ".join(p["text"] for p in places),
|
"location": "; ".join(p["text"] for p in places),
|
||||||
"url": f"{_base(slug)}/job/{role_id}",
|
"url": f"{_base(slug)}/job/{role_id}" if safe else "",
|
||||||
"posted": parse_dt(role.get("PostedDate")),
|
"posted": parse_dt(role.get("PostedDate")),
|
||||||
"department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction"))
|
"department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction"))
|
||||||
or _text(role.get("Department")) or _text(role.get("Organization"))),
|
or _text(role.get("Department")) or _text(role.get("Organization"))),
|
||||||
|
|||||||
@@ -104,8 +104,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for position in root.findall("position"):
|
for position in root.findall("position"):
|
||||||
job_id = (position.findtext("id") or "").strip()
|
job_id = (position.findtext("id") or "").strip()
|
||||||
title = (position.findtext("name") or "").strip()
|
title = (position.findtext("name") or "").strip()
|
||||||
if not job_id or not title:
|
|
||||||
continue
|
|
||||||
description = " ".join(
|
description = " ".join(
|
||||||
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
||||||
salary_info = position.find("salaryInformation")
|
salary_info = position.find("salaryInformation")
|
||||||
|
|||||||
@@ -168,8 +168,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for job in search.get("data", {}).get("jobs") or []:
|
for job in search.get("data", {}).get("jobs") or []:
|
||||||
key = _key(job)
|
key = _key(job)
|
||||||
title = _text(job.get("title"))
|
title = _text(job.get("title"))
|
||||||
if not title or not key:
|
|
||||||
continue
|
|
||||||
places: list[Place] = []
|
places: list[Place] = []
|
||||||
for text in (job.get("location"), *(job.get("multi_location") or [])):
|
for text in (job.get("location"), *(job.get("multi_location") or [])):
|
||||||
if isinstance(text, str):
|
if isinstance(text, str):
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Reads
|
from ._contract import Ask, Fields, Reads
|
||||||
from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
|
from ._shared import HEADING, Row, html_text, label_slug_ok, num, only, pay_line, yearly_amount
|
||||||
|
|
||||||
NAME = "pinpoint"
|
NAME = "pinpoint"
|
||||||
LABEL = "Pinpoint"
|
LABEL = "Pinpoint"
|
||||||
@@ -121,8 +121,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
url = (j.get("url") or "").strip()
|
url = (j.get("url") or "").strip()
|
||||||
source_id = str(j.get("id") or "")
|
source_id = str(j.get("id") or "")
|
||||||
if not title or not source_id or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
job = j.get("job") or {}
|
job = j.get("job") or {}
|
||||||
low, high, currency, salary_text = _pay(j)
|
low, high, currency, salary_text = _pay(j)
|
||||||
workplace = str(j.get("workplace_type") or "").strip()
|
workplace = str(j.get("workplace_type") or "").strip()
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
|
|||||||
from desk.sources import strip_html
|
from desk.sources import strip_html
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||||
from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt
|
from ._shared import Get, Page, Row, html_text, only, parse_dt
|
||||||
|
|
||||||
NAME = "platsbanken"
|
NAME = "platsbanken"
|
||||||
LABEL = "Platsbanken"
|
LABEL = "Platsbanken"
|
||||||
@@ -108,8 +108,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
for hit in decoded.get("hits") or []:
|
for hit in decoded.get("hits") or []:
|
||||||
title = _text(hit.get("headline") or "")
|
title = _text(hit.get("headline") or "")
|
||||||
url = (hit.get("webpage_url") or "").strip()
|
url = (hit.get("webpage_url") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
|
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
|
||||||
places = [place for address in addresses if isinstance(address, dict)
|
places = [place for address in addresses if isinstance(address, dict)
|
||||||
if (place := _place(address))]
|
if (place := _place(address))]
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads
|
from ._contract import Ask, Fields, Place, Reads
|
||||||
from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||||
|
|
||||||
NAME = "recruitee"
|
NAME = "recruitee"
|
||||||
LABEL = "Recruitee"
|
LABEL = "Recruitee"
|
||||||
@@ -105,8 +105,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in data.get("offers") or []:
|
for j in data.get("offers") or []:
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
url = (j.get("careers_url") or "").strip()
|
url = (j.get("careers_url") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
||||||
salary_min, salary_max, currency = yearly_amount(
|
salary_min, salary_max, currency = yearly_amount(
|
||||||
salary.get("min"), salary.get("max"), salary.get("currency"),
|
salary.get("min"), salary.get("max"), salary.get("currency"),
|
||||||
|
|||||||
+14
-10
@@ -82,27 +82,31 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
raise ValueError("not a Rippling board: the answer is not a list")
|
raise ValueError("not a Rippling board: the answer is not a list")
|
||||||
roles: dict[str, Row] = {}
|
roles: dict[str, Row] = {}
|
||||||
places: dict[str, list[Place]] = {}
|
places: dict[str, list[Place]] = {}
|
||||||
for entry in data:
|
for index, entry in enumerate(data):
|
||||||
job_id = str(entry.get("uuid") or "").strip()
|
job_id = str(entry.get("uuid") or "").strip()
|
||||||
title = (entry.get("name") or "").strip()
|
title = (entry.get("name") or "").strip()
|
||||||
if not title or not _UUID.fullmatch(job_id):
|
# The id goes into a URL: one of other characters gets no link, and
|
||||||
continue
|
# the runner drops the row. An entry without an id is a role of its
|
||||||
|
# own, not merged with the others that lack one.
|
||||||
|
key = job_id or f"#{index}"
|
||||||
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
|
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
|
||||||
places.setdefault(job_id, [])
|
places.setdefault(key, [])
|
||||||
if place:
|
if place:
|
||||||
add_place(places[job_id], _place(place))
|
add_place(places[key], _place(place))
|
||||||
roles.setdefault(job_id, {
|
roles.setdefault(key, {
|
||||||
"source_id": job_id,
|
"source_id": job_id,
|
||||||
"title": title,
|
"title": title,
|
||||||
"url": f"https://{_SITE}/{slug}/jobs/{job_id}",
|
"url": f"https://{_SITE}/{slug}/jobs/{job_id}" if _UUID.fullmatch(job_id) else "",
|
||||||
"posted": None,
|
"posted": None,
|
||||||
"department": str(((entry.get("department") or {}).get("label")) or "").strip(),
|
"department": str(((entry.get("department") or {}).get("label")) or "").strip(),
|
||||||
"description": "",
|
"description": "",
|
||||||
"salary_text": "", # the board's list states no pay
|
"salary_text": "", # the board's list states no pay
|
||||||
})
|
})
|
||||||
for job_id, row in roles.items():
|
if title and not roles[key]["title"]:
|
||||||
row["places"] = places[job_id]
|
roles[key]["title"] = title # an entry of the role that has one
|
||||||
row["location"] = "; ".join(p["text"] for p in places[job_id])
|
for key, row in roles.items():
|
||||||
|
row["places"] = places[key]
|
||||||
|
row["location"] = "; ".join(p["text"] for p in places[key])
|
||||||
return list(roles.values())
|
return list(roles.values())
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -115,8 +115,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for j in data.get("content") or []:
|
for j in data.get("content") or []:
|
||||||
title = (j.get("name") or "").strip()
|
title = (j.get("name") or "").strip()
|
||||||
job_id = str(j.get("id") or "")
|
job_id = str(j.get("id") or "")
|
||||||
if not title or not job_id:
|
|
||||||
continue
|
|
||||||
location = j.get("location") or {}
|
location = j.get("location") or {}
|
||||||
loc_text = (location.get("fullLocation")
|
loc_text = (location.get("fullLocation")
|
||||||
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ from desk.sources import strip_html
|
|||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||||
from ._countries import NAMES
|
from ._countries import NAMES
|
||||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
|
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
|
||||||
|
|
||||||
NAME = "successfactors"
|
NAME = "successfactors"
|
||||||
LABEL = "SuccessFactors"
|
LABEL = "SuccessFactors"
|
||||||
@@ -247,8 +247,9 @@ def _tile_fields(chunk: str) -> tuple[str, str, str]:
|
|||||||
def _entries(text: str) -> list[dict[str, Any]]:
|
def _entries(text: str) -> list[dict[str, Any]]:
|
||||||
"""The roles of one search page, in the order shown, whichever layout it is
|
"""The roles of one search page, in the order shown, whichever layout it is
|
||||||
in and before the prefix filter: `id`, `href` (as written, relative),
|
in and before the prefix filter: `id`, `href` (as written, relative),
|
||||||
`title`, `place`, `date` and `department`. A role is listed once, and a row
|
`title`, `place`, `date` and `department`. A role is listed once. The id
|
||||||
without a link to a role is not one."""
|
and the title are in the role's link, so a row without one is handed over
|
||||||
|
with an empty `id`, `href` and `title`, for the runner to drop and count."""
|
||||||
table = _ROW.split(text)[1:]
|
table = _ROW.split(text)[1:]
|
||||||
chunks = table or _TILE.split(text)[1:]
|
chunks = table or _TILE.split(text)[1:]
|
||||||
out: list[dict[str, Any]] = []
|
out: list[dict[str, Any]] = []
|
||||||
@@ -256,10 +257,12 @@ def _entries(text: str) -> list[dict[str, Any]]:
|
|||||||
for chunk in chunks:
|
for chunk in chunks:
|
||||||
if table:
|
if table:
|
||||||
chunk = chunk.split("</tr>", 1)[0]
|
chunk = chunk.split("</tr>", 1)[0]
|
||||||
if not (link := _LINK.search(chunk)):
|
if link := _LINK.search(chunk):
|
||||||
continue
|
href, job_id, title = (html.unescape(link.group(1)), link.group(2),
|
||||||
href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3))
|
_text(link.group(3)))
|
||||||
if job_id in seen:
|
else:
|
||||||
|
href = job_id = title = ""
|
||||||
|
if job_id and job_id in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(job_id)
|
seen.add(job_id)
|
||||||
place, date, department = (_table_fields if table else _tile_fields)(chunk)
|
place, date, department = (_table_fields if table else _tile_fields)(chunk)
|
||||||
@@ -283,12 +286,14 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
|||||||
if not _SEARCH_PAGE.search(text):
|
if not _SEARCH_PAGE.search(text):
|
||||||
raise ValueError("not a SuccessFactors search page")
|
raise ValueError("not a SuccessFactors search page")
|
||||||
found = _entries(text)
|
found = _entries(text)
|
||||||
|
ids = [entry["id"] for entry in found if entry["id"]]
|
||||||
total = _total(text)
|
total = _total(text)
|
||||||
if found and found[0]["id"] == previous:
|
first = ids[0] if ids else None
|
||||||
|
if first is not None and first == previous:
|
||||||
return text, [], total, (start, previous)
|
return text, [], total, (start, previous)
|
||||||
more = bool(found) and (total is None or start + len(found) < total)
|
more = bool(found) and (total is None or start + len(found) < total)
|
||||||
return (text, [{"id": entry["id"]} for entry in found], total,
|
return (text, [{"id": job_id} for job_id in ids], total,
|
||||||
(start + len(found), found[0]["id"]) if more else None)
|
(start + len(found), first) if more else None)
|
||||||
|
|
||||||
|
|
||||||
def rows(decoded: str, slug: str) -> list[Row]:
|
def rows(decoded: str, slug: str) -> list[Row]:
|
||||||
@@ -296,17 +301,21 @@ def rows(decoded: str, slug: str) -> list[Row]:
|
|||||||
slug's prefix when it has one; the id in it is the requisition's `source_id`.
|
slug's prefix when it has one; the id in it is the requisition's `source_id`.
|
||||||
The table and the tile layout give a title, a location, a date and, where
|
The table and the tile layout give a title, a location, a date and, where
|
||||||
the site has the column, a department; nothing else about a role is on
|
the site has the column, a department; nothing else about a role is on
|
||||||
the list, so its description and pay are empty."""
|
the list, so its description and pay are empty.
|
||||||
|
|
||||||
|
A role linked on another host, or outside the prefix, is another board's
|
||||||
|
(another site, or another brand of a shared site), and is left out: it is
|
||||||
|
no role of this company. A row with no link or no title is handed over,
|
||||||
|
and the runner drops it."""
|
||||||
host, prefix = _parts(slug)
|
host, prefix = _parts(slug)
|
||||||
base = f"https://{host}/"
|
base = f"https://{host}/"
|
||||||
out = []
|
out = []
|
||||||
for entry in _entries(decoded):
|
for entry in _entries(decoded):
|
||||||
url = urljoin(base, entry["href"])
|
url = urljoin(base, entry["href"]) if entry["href"] else ""
|
||||||
if urlsplit(url).hostname != host or not is_web_url(url):
|
if url and urlsplit(url).hostname != host:
|
||||||
continue
|
continue
|
||||||
if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"):
|
if url and prefix and not urlsplit(url).path.lower().startswith(
|
||||||
continue
|
f"{prefix.lower()}/job/"):
|
||||||
if not entry["title"]:
|
|
||||||
continue
|
continue
|
||||||
out.append({
|
out.append({
|
||||||
"source_id": entry["id"],
|
"source_id": entry["id"],
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads
|
from ._contract import Ask, Fields, Place, Reads
|
||||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
|
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
|
||||||
|
|
||||||
NAME = "teamtailor"
|
NAME = "teamtailor"
|
||||||
LABEL = "Teamtailor"
|
LABEL = "Teamtailor"
|
||||||
@@ -115,8 +115,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
|||||||
for item in items:
|
for item in items:
|
||||||
title = (item.findtext("title") or "").strip()
|
title = (item.findtext("title") or "").strip()
|
||||||
url = (item.findtext("link") or "").strip()
|
url = (item.findtext("link") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
status = (item.findtext("remoteStatus") or "").strip()
|
status = (item.findtext("remoteStatus") or "").strip()
|
||||||
remote = {"fully": True, "hybrid": False, "onsite": False}.get(status)
|
remote = {"fully": True, "hybrid": False, "onsite": False}.get(status)
|
||||||
places: list[Place] = []
|
places: list[Place] = []
|
||||||
|
|||||||
@@ -102,8 +102,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for j in data.get("results") or []:
|
for j in data.get("results") or []:
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
shortcode = (j.get("shortcode") or "").strip()
|
shortcode = (j.get("shortcode") or "").strip()
|
||||||
if not title or not shortcode:
|
|
||||||
continue
|
|
||||||
workplace = str(j.get("workplace") or "").strip()
|
workplace = str(j.get("workplace") or "").strip()
|
||||||
remote = bool(j.get("remote")) or workplace == "remote"
|
remote = bool(j.get("remote")) or workplace == "remote"
|
||||||
places = _places(j, remote)
|
places = _places(j, remote)
|
||||||
|
|||||||
@@ -124,8 +124,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
|||||||
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
|
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
|
||||||
postings = data.get("jobPostings") or []
|
postings = data.get("jobPostings") or []
|
||||||
# Every posting is counted, a path-less one (LEGO lists one that has only
|
# Every posting is counted, a path-less one (LEGO lists one that has only
|
||||||
# `bulletFields`) by its position, so the parser dropping it does not
|
# `bulletFields`) by its position, so the runner dropping its row does
|
||||||
# leave the board short of its total.
|
# not leave the board short of its total.
|
||||||
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
|
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
|
||||||
following = offset + len(postings)
|
following = offset + len(postings)
|
||||||
more = bool(postings) and (total is None or following < total)
|
more = bool(postings) and (total is None or following < total)
|
||||||
@@ -274,13 +274,13 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
`bulletFields`, which lead with the requisition id, and on some tenants
|
`bulletFields`, which lead with the requisition id, and on some tenants
|
||||||
`remoteType`. No description, no date and no country: `detail` has
|
`remoteType`. No description, no date and no country: `detail` has
|
||||||
those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the
|
those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the
|
||||||
path where not; a posting with no title or no path is skipped."""
|
path where not. A posting with no path has neither an id nor a link, and
|
||||||
|
the runner drops it."""
|
||||||
out = []
|
out = []
|
||||||
for p in decoded.get("jobPostings") or []:
|
for p in decoded.get("jobPostings") or []:
|
||||||
title = (p.get("title") or "").strip()
|
title = (p.get("title") or "").strip()
|
||||||
path = (p.get("externalPath") or "").strip()
|
path = (p.get("externalPath") or "").strip()
|
||||||
if not title or not _ROLE_PATH.fullmatch(path):
|
path = path if _ROLE_PATH.fullmatch(path) else ""
|
||||||
continue
|
|
||||||
remote = _remote(p.get("remoteType"))
|
remote = _remote(p.get("remoteType"))
|
||||||
place = _place(p.get("locationsText"), remote)
|
place = _place(p.get("locationsText"), remote)
|
||||||
out.append({
|
out.append({
|
||||||
@@ -288,7 +288,7 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
"title": title,
|
"title": title,
|
||||||
"places": [place] if place else [],
|
"places": [place] if place else [],
|
||||||
"location": place["text"] if place else "",
|
"location": place["text"] if place else "",
|
||||||
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}", # type: ignore[index]
|
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}" if path else "", # type: ignore[index]
|
||||||
"posted": _posted(p.get("postedOn") or ""),
|
"posted": _posted(p.get("postedOn") or ""),
|
||||||
"department": "",
|
"department": "",
|
||||||
"description": "",
|
"description": "",
|
||||||
|
|||||||
+2
-1
@@ -19,7 +19,8 @@ roles close once no search has seen them for `STALE_DAYS` (see
|
|||||||
`app/jobs/pull.py`).
|
`app/jobs/pull.py`).
|
||||||
|
|
||||||
In a pull a search is read by `_runner.read` under the prefix `search`, as a
|
In a pull a search is read by `_runner.read` under the prefix `search`, as a
|
||||||
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
|
board is under `ats`: the runner drops a row without an id, a title or a
|
||||||
|
link, the 10% guards hold it, `_runner.stamp` records it,
|
||||||
and a search that fails or is suspect is kept as failing under
|
and a search that fails or is suspect is kept as failing under
|
||||||
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
|
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
|
||||||
the settle reads as it reads a failing board's.
|
the settle reads as it reads a failing board's.
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
// static/agents.js without a browser: the gates Save sends from the fetch
|
||||||
|
// gate's four parts, and the stored gates the parts cannot say, which the
|
||||||
|
// form names in a note and sends back as they came.
|
||||||
|
//
|
||||||
|
// node --test tests/js/*.test.mjs
|
||||||
|
|
||||||
|
import { test } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import vm from "node:vm";
|
||||||
|
|
||||||
|
// Only what agents.js touches before DOMContentLoaded.
|
||||||
|
const window = { api: async () => [], count: (n, noun) => `${n} ${noun}` };
|
||||||
|
const document = { addEventListener() {} };
|
||||||
|
Object.assign(globalThis, { window, document });
|
||||||
|
for (const file of ["remote.js", "agents.js"]) {
|
||||||
|
vm.runInThisContext(readFileSync(new URL(`../../app/static/${file}`, import.meta.url), "utf8"));
|
||||||
|
}
|
||||||
|
const rh = window.remoteHelpers;
|
||||||
|
const { formGates, keptNote, rulesLine } = window.agentGates;
|
||||||
|
|
||||||
|
/** The four parts as the form shows them after loading `gates`, and what it keeps. */
|
||||||
|
function load(gates) {
|
||||||
|
const { where, rest } = rh.distribute(gates);
|
||||||
|
const parts = {
|
||||||
|
scope: where.scope.kind, scoped: where.scope.values,
|
||||||
|
within: where.countries ? where.countries.values : [],
|
||||||
|
towns: where.places ? where.places.values : [],
|
||||||
|
remote: where.remote.mode, from: where.remote.values,
|
||||||
|
titles: where.titles ? where.titles.values : [],
|
||||||
|
};
|
||||||
|
return { parts, rest };
|
||||||
|
}
|
||||||
|
|
||||||
|
const europe = { field: "country", values: rh.europeWith(["DK"]), mode: "require" };
|
||||||
|
const dk = { field: "country", values: ["DK"], mode: "any" };
|
||||||
|
const remote = { field: "remote", values: ["remote"], mode: "any" };
|
||||||
|
const towns = { field: "place", values: ["Malmö", "Lund"], mode: "any" };
|
||||||
|
const titles = { field: "title", values: ["account executive", "sales"], mode: "exclude" };
|
||||||
|
const level = { field: "seniority", values: ["senior", "staff"], mode: "require" };
|
||||||
|
const acme = { field: "company", values: ["acme"], mode: "exclude" };
|
||||||
|
|
||||||
|
test("a save sends the four parts back as they were loaded", () => {
|
||||||
|
const agents = [
|
||||||
|
[europe, dk, remote, towns, titles],
|
||||||
|
[{ field: "country", values: ["DK", "SE"], mode: "require" },
|
||||||
|
{ field: "remote_from", values: ["country:DK", "country:EU"], mode: "any" }, titles],
|
||||||
|
[dk],
|
||||||
|
[],
|
||||||
|
];
|
||||||
|
for (const gates of agents) {
|
||||||
|
const { parts, rest } = load(gates);
|
||||||
|
assert.deepEqual(rest, []);
|
||||||
|
assert.deepEqual(formGates(parts, rest), gates);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("each part sends its own gate, and an empty part none", () => {
|
||||||
|
const parts = { scope: "everywhere", scoped: [], within: [], towns: [], remote: "none", from: [], titles: [] };
|
||||||
|
assert.deepEqual(formGates(parts), []);
|
||||||
|
// A remote line open to no area yet, or chosen countries with none picked, say nothing.
|
||||||
|
assert.deepEqual(formGates({ ...parts, remote: "from", scope: "countries" }), []);
|
||||||
|
assert.deepEqual(formGates({ ...parts, scope: "europe", within: ["US"] }), [
|
||||||
|
{ field: "country", values: rh.europeWith(["US"]), mode: "require" },
|
||||||
|
{ field: "country", values: ["US"], mode: "any" },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a stored gate the parts cannot say is kept across a save, named in the note and on the card", () => {
|
||||||
|
const gates = [europe, dk, remote, titles, level, acme];
|
||||||
|
const { parts, rest } = load(gates);
|
||||||
|
assert.deepEqual(rest, [level, acme]);
|
||||||
|
assert.deepEqual(formGates(parts, rest), gates);
|
||||||
|
assert.equal(keptNote(rest),
|
||||||
|
"The form cannot edit these rules, and Save keeps them as stored: "
|
||||||
|
+ "must: with seniority senior, staff; not: at acme.");
|
||||||
|
const words = rulesLine(gates);
|
||||||
|
assert.match(words, /Only roles with seniority senior, staff\./);
|
||||||
|
assert.match(words, /Leaves out roles at acme\./);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an agent with nothing to keep has no note", () => {
|
||||||
|
assert.equal(keptNote([]), "");
|
||||||
|
assert.equal(keptNote([level]), "The form cannot edit this rule, and Save keeps it as stored: must: with seniority senior, staff.");
|
||||||
|
});
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
// static/remote.js without a browser: the badge, the suggestions the agent's
|
// static/remote.js without a browser: the badge, the suggestions the agent's
|
||||||
// remote line offers, and how gates share out between Where and Other rules.
|
// remote line offers, and how gates share out between the fetch gate and the
|
||||||
|
// rest the form keeps as stored.
|
||||||
//
|
//
|
||||||
// node --test tests/js/*.test.mjs
|
// node --test tests/js/*.test.mjs
|
||||||
|
|
||||||
@@ -89,14 +90,14 @@ test("any other require gate on country is the scope with its codes as chips", (
|
|||||||
assert.deepEqual(rest, []);
|
assert.deepEqual(rest, []);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("a second require gate on country stays a row", () => {
|
test("a second require gate on country is kept as stored", () => {
|
||||||
const second = must("NO");
|
const second = must("NO");
|
||||||
const { where, rest } = rh.distribute([europe, second]);
|
const { where, rest } = rh.distribute([europe, second]);
|
||||||
assert.equal(where.scope.kind, "europe");
|
assert.equal(where.scope.kind, "europe");
|
||||||
assert.deepEqual(rest, [second]);
|
assert.deepEqual(rest, [second]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("the titles to leave out are the exclude gate on title, and any other exclude is a row", () => {
|
test("the titles to leave out are the exclude gate on title, and any other exclude is kept as stored", () => {
|
||||||
const titles = { field: "title", values: ["account executive"], mode: "exclude" };
|
const titles = { field: "title", values: ["account executive"], mode: "exclude" };
|
||||||
const level = { field: "seniority", values: ["junior"], mode: "exclude" };
|
const level = { field: "seniority", values: ["junior"], mode: "exclude" };
|
||||||
const { where, rest } = rh.distribute([titles, level]);
|
const { where, rest } = rh.distribute([titles, level]);
|
||||||
@@ -104,7 +105,7 @@ test("the titles to leave out are the exclude gate on title, and any other exclu
|
|||||||
assert.deepEqual(rest, [level]);
|
assert.deepEqual(rest, [level]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("a gate with a mode the form does not know stays a row and takes no Where line", () => {
|
test("a gate with a mode the form does not know is kept as stored and takes no Where line", () => {
|
||||||
const prefer = { field: "remote", values: ["hybrid"], mode: "prefer" };
|
const prefer = { field: "remote", values: ["hybrid"], mode: "prefer" };
|
||||||
const { where, rest } = rh.distribute([prefer]);
|
const { where, rest } = rh.distribute([prefer]);
|
||||||
assert.deepEqual(where.remote, { mode: "none", values: [] });
|
assert.deepEqual(where.remote, { mode: "none", values: [] });
|
||||||
@@ -112,7 +113,7 @@ test("a gate with a mode the form does not know stays a row and takes no Where l
|
|||||||
assert.deepEqual(rest, [prefer]);
|
assert.deepEqual(rest, [prefer]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("a require gate on another field is a row and leaves the scope everywhere", () => {
|
test("a require gate on another field is kept as stored and leaves the scope everywhere", () => {
|
||||||
const level = { field: "seniority", values: ["senior"], mode: "require" };
|
const level = { field: "seniority", values: ["senior"], mode: "require" };
|
||||||
const { where, rest } = rh.distribute([level]);
|
const { where, rest } = rh.distribute([level]);
|
||||||
assert.equal(where.scope.kind, "everywhere");
|
assert.equal(where.scope.kind, "everywhere");
|
||||||
@@ -129,7 +130,7 @@ test("remote, three modes: the plain gate is within the scope", () => {
|
|||||||
assert.deepEqual(named.rest, []);
|
assert.deepEqual(named.rest, []);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("a remote_from gate wins over the plain one, which stays a row", () => {
|
test("a remote_from gate wins over the plain one, which is kept as stored", () => {
|
||||||
for (const gates of [[older, from("country:DK")], [from("country:DK"), older]]) {
|
for (const gates of [[older, from("country:DK")], [from("country:DK"), older]]) {
|
||||||
const { where, rest } = rh.distribute(gates);
|
const { where, rest } = rh.distribute(gates);
|
||||||
assert.deepEqual(where.remote, { mode: "from", values: ["country:DK"] });
|
assert.deepEqual(where.remote, { mode: "from", values: ["country:DK"] });
|
||||||
@@ -137,7 +138,7 @@ test("a remote_from gate wins over the plain one, which stays a row", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test("a second remote_from gate or a must stays a row", () => {
|
test("a second remote_from gate or a must is kept as stored", () => {
|
||||||
const second = from("country:SE");
|
const second = from("country:SE");
|
||||||
const required = { field: "remote_from", values: ["country:DK"], mode: "require" };
|
const required = { field: "remote_from", values: ["country:DK"], mode: "require" };
|
||||||
const { where, rest } = rh.distribute([from("country:DK"), second, required]);
|
const { where, rest } = rh.distribute([from("country:DK"), second, required]);
|
||||||
|
|||||||
+15
-8
@@ -24,18 +24,24 @@ from plugins.readers import _runner, smartrecruiters # noqa: E402
|
|||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
|
|
||||||
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
def _parsed(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||||
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
||||||
body = path.read_bytes()
|
return ats._PARSERS[host](path.read_bytes(), slug)
|
||||||
return ats._PARSERS[host](body, slug)
|
|
||||||
|
|
||||||
|
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||||
|
"""The rows of a host's fixture the runner keeps (`_runner.valid`)."""
|
||||||
|
return [row for row in _parsed(host, slug) if _runner.valid(row)]
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------ hosts
|
# ------------------------------------------------------------------ hosts
|
||||||
|
|
||||||
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
||||||
|
# The javascript: and data: rows are handed over and the runner drops
|
||||||
|
# them: a feed is external content, never trusted as a URL.
|
||||||
|
assert [r["source_id"] for r in _parsed("greenhouse")] == [
|
||||||
|
"1001", "1002", "1003", "1004", "1005", "1006"]
|
||||||
rows = _rows("greenhouse")
|
rows = _rows("greenhouse")
|
||||||
# The javascript: and data: rows are refused, same as the four real ones
|
|
||||||
# are kept: a feed is external content, never trusted as a URL.
|
|
||||||
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
||||||
first = rows[0]
|
first = rows[0]
|
||||||
assert first["title"] == "Senior Backend Engineer" # trimmed
|
assert first["title"] == "Senior Backend Engineer" # trimmed
|
||||||
@@ -269,10 +275,11 @@ TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
|
|||||||
|
|
||||||
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
||||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||||
# the javascript: link is refused like on the other hosts
|
# the javascript: link is handed over and refused by the runner
|
||||||
assert [r["source_id"] for r in rows] == [
|
assert [r["source_id"] for r in rows] == [
|
||||||
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
|
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
|
||||||
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"]
|
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"]
|
||||||
|
assert [_runner.valid(r) for r in rows] == [True, True, True, False]
|
||||||
first = rows[0]
|
first = rows[0]
|
||||||
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
||||||
assert first["department"] == "Engineering"
|
assert first["department"] == "Engineering"
|
||||||
@@ -294,7 +301,7 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
|
|||||||
|
|
||||||
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
||||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||||
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
|
assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them
|
||||||
assert rows[1]["title"] == "Läkare" # trimmed
|
assert rows[1]["title"] == "Läkare" # trimmed
|
||||||
assert rows[1]["department"] == "Software Development"
|
assert rows[1]["department"] == "Software Development"
|
||||||
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, bamboohr # noqa: E402
|
from plugins.readers import Ask, _runner, bamboohr # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -84,11 +84,16 @@ def test_a_page_that_is_not_a_listing_raises_and_an_empty_board_is_no_rows() ->
|
|||||||
bamboohr.parse(body, "configit")
|
bamboohr.parse(body, "configit")
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_without_a_title_or_an_id_is_left_out() -> None:
|
def test_a_role_without_a_title_or_an_id_reaches_the_runner_and_is_dropped_there() -> None:
|
||||||
body = json.loads(_read("bamboohr.json"))
|
body = json.loads(_read("bamboohr.json"))
|
||||||
body["result"][0]["jobOpeningName"] = " "
|
body["result"][0]["jobOpeningName"] = " "
|
||||||
body["result"][1]["id"] = None
|
body["result"][1]["id"] = None
|
||||||
assert bamboohr.parse(json.dumps(body).encode(), "configit") == []
|
raw = json.dumps(body).encode()
|
||||||
|
rows = bamboohr.parse(raw, "configit")
|
||||||
|
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||||
|
listing = _runner.read_listing(bamboohr, "configit", lambda url, **kw: raw)
|
||||||
|
assert (listing.rows, listing.ids, listing.complete) == ([], [], True)
|
||||||
|
assert (listing.handed, listing.dropped) == (2, 2)
|
||||||
|
|
||||||
|
|
||||||
def test_detail_reads_the_text_the_date_and_the_country() -> None:
|
def test_detail_reads_the_text_the_date_and_the_country() -> None:
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, deel # noqa: E402
|
from plugins.readers import Ask, _runner, deel # noqa: E402
|
||||||
|
|
||||||
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
|
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
|
||||||
URL = "https://jobs.deel.com/klarna"
|
URL = "https://jobs.deel.com/klarna"
|
||||||
@@ -89,10 +89,12 @@ def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board
|
|||||||
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
|
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
|
||||||
|
|
||||||
|
|
||||||
def test_a_posting_without_a_title_or_id_is_skipped() -> None:
|
def test_a_posting_without_a_title_or_id_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
|
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
|
||||||
{"id": "c", "title": "Cook", "job": {}}]}
|
{"id": "c", "title": "Cook", "job": {}}]}
|
||||||
assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"]
|
rows = deel.parse(_page_with(body), "acme")
|
||||||
|
assert [r["source_id"] for r in rows] == ["a", "", "c"]
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ the fetch
|
# ------------------------------------------------------------ the fetch
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, eightfold # noqa: E402
|
from plugins.readers import Ask, _runner, eightfold # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SLUG = "jobs.ericsson.com/ericsson.com"
|
SLUG = "jobs.ericsson.com/ericsson.com"
|
||||||
@@ -93,10 +93,12 @@ def test_the_work_location_option_is_the_hosts_word(
|
|||||||
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
|
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
|
||||||
|
|
||||||
|
|
||||||
def test_a_position_without_a_name_an_id_or_a_path_is_left_out() -> None:
|
def test_a_position_without_a_name_an_id_or_a_path_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
|
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
|
||||||
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
|
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
|
||||||
assert eightfold.rows(bad, SLUG) == []
|
rows = eightfold.rows(bad, SLUG)
|
||||||
|
assert len(rows) == 4 and not any(map(_runner.valid, rows))
|
||||||
|
assert [r["url"] for r in rows[2:]] == ["", ""] # no link built off the host
|
||||||
|
|
||||||
|
|
||||||
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
|
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, gem # noqa: E402
|
from plugins.readers import Ask, _runner, gem # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
API = "https://jobs.gem.com/api/public/graphql"
|
API = "https://jobs.gem.com/api/public/graphql"
|
||||||
@@ -79,8 +79,9 @@ def test_remote_is_the_location_type_then_the_locations(
|
|||||||
assert gem.rows([posting], "acme")[0]["remote"] == remote
|
assert gem.rows([posting], "acme")[0]["remote"] == remote
|
||||||
|
|
||||||
|
|
||||||
def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None:
|
def test_a_posting_without_a_title_or_an_ext_id_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == []
|
rows = gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme")
|
||||||
|
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_the_board_is_one_post_without_origin_or_referer() -> None:
|
def test_the_board_is_one_post_without_origin_or_referer() -> None:
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, hibob # noqa: E402
|
from plugins.readers import Ask, _runner, hibob # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "hibob.json").read_bytes()
|
BOARD = (FIXTURES / "hibob.json").read_bytes()
|
||||||
@@ -87,10 +87,12 @@ def test_the_place_leaves_the_country_off_where_the_site_names_it() -> None:
|
|||||||
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
|
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
|
||||||
|
|
||||||
|
|
||||||
def test_an_ad_without_a_title_or_id_is_skipped_and_an_empty_board_is_empty() -> None:
|
def test_an_ad_without_a_title_or_id_is_handed_over_and_an_empty_board_is_empty() -> None:
|
||||||
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
|
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
|
||||||
{"id": "c", "title": "Cook"}]}).encode()
|
{"id": "c", "title": "Cook"}]}).encode()
|
||||||
assert [r["source_id"] for r in hibob.parse(body, "acme")] == ["c"]
|
rows = hibob.parse(body, "acme")
|
||||||
|
assert [r["source_id"] for r in rows] == ["a", "", "c"]
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
|
||||||
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
|
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, hrmanager # noqa: E402
|
from plugins.readers import Ask, _runner, hrmanager # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
||||||
@@ -52,12 +52,14 @@ def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_f
|
|||||||
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
|
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
|
||||||
|
|
||||||
|
|
||||||
def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None:
|
def test_an_item_without_a_title_or_a_web_link_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
|
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
|
||||||
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
|
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
|
||||||
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
|
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
|
||||||
b"</channel></rss>")
|
b"</channel></rss>")
|
||||||
assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"]
|
rows = hrmanager.parse(body, "x")
|
||||||
|
assert [r["source_id"] for r in rows] == ["1", "2", "3"]
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["3"]
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- paging
|
# ------------------------------------------------------------- paging
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, jibe # noqa: E402
|
from plugins.readers import Ask, _runner, jibe # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SITE = "jobs.booking.com"
|
SITE = "jobs.booking.com"
|
||||||
@@ -72,9 +72,10 @@ def test_the_location_falls_back_to_city_and_country() -> None:
|
|||||||
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
|
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
|
def test_a_role_without_a_title_or_a_slug_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
|
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
|
||||||
assert jibe.rows({"jobs": jobs}, SITE) == []
|
rows = jibe.rows({"jobs": jobs}, SITE)
|
||||||
|
assert len(rows) == 3 and not any(map(_runner.valid, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, jobylon # noqa: E402
|
from plugins.readers import Ask, _runner, jobylon # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
||||||
@@ -65,12 +65,14 @@ def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
|
|||||||
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
|
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None:
|
def test_a_role_whose_link_is_not_a_role_page_gets_no_link_and_is_dropped() -> None:
|
||||||
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
|
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
|
||||||
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
|
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
|
||||||
b"{id: '3', title: 'C', url: '/companies/3/'},"
|
b"{id: '3', title: 'C', url: '/companies/3/'},"
|
||||||
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
|
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
|
||||||
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"]
|
rows = jobylon.parse(body, "9")
|
||||||
|
assert [r["url"] for r in rows[:3]] == ["", "", ""]
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["4"]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- detail
|
# ---------------------------------------------------------------- detail
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.readers import oracle # noqa: E402
|
from plugins.readers import _runner, oracle # noqa: E402
|
||||||
from plugins.readers._shared import is_web_url # noqa: E402
|
from plugins.readers._shared import is_web_url # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
@@ -84,14 +84,15 @@ def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
|
|||||||
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
|
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_left_out() -> None:
|
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_dropped_by_the_runner() -> None:
|
||||||
data = _listing()
|
data = _listing()
|
||||||
roles = data["items"][0]["requisitionList"]
|
roles = data["items"][0]["requisitionList"]
|
||||||
roles[0]["Title"] = " "
|
roles[0]["Title"] = " "
|
||||||
roles[1]["Id"] = "1/../../x"
|
roles[1]["Id"] = "1/../../x"
|
||||||
roles[2]["Id"] = ""
|
roles[2]["Id"] = ""
|
||||||
assert [r["source_id"] for r in oracle.parse(json.dumps(data).encode(), SLUG)] == \
|
rows = oracle.parse(json.dumps(data).encode(), SLUG)
|
||||||
["40975", "31680"]
|
assert rows[1]["url"] == "" # the id builds no link
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["40975", "31680"]
|
||||||
|
|
||||||
|
|
||||||
def test_a_listing_without_the_expand_has_no_roles() -> None:
|
def test_a_listing_without_the_expand_has_no_roles() -> None:
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.readers import phenom # noqa: E402
|
from plugins.readers import _runner, phenom # noqa: E402
|
||||||
from plugins.readers._shared import is_web_url # noqa: E402
|
from plugins.readers._shared import is_web_url # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
@@ -101,13 +101,15 @@ def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None
|
|||||||
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
|
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_without_title_or_id_is_left_out() -> None:
|
def test_a_role_without_title_or_id_is_dropped_by_the_runner() -> None:
|
||||||
data = _answer()
|
data = _answer()
|
||||||
jobs = data["refineSearch"]["data"]["jobs"]
|
jobs = data["refineSearch"]["data"]["jobs"]
|
||||||
jobs[0]["title"] = ""
|
jobs[0]["title"] = ""
|
||||||
for key in ("jobSeqNo", "jobId", "reqId"):
|
for key in ("jobSeqNo", "jobId", "reqId"):
|
||||||
jobs[1].pop(key, None)
|
jobs[1].pop(key, None)
|
||||||
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
|
rows = phenom.parse(json.dumps(data).encode(), SLUG)
|
||||||
|
assert len(rows) == len(IDS)
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == IDS[2:]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------- the request
|
# ---------------------------------------------------------- the request
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import pinpoint # noqa: E402
|
from plugins.readers import _runner, pinpoint # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -86,11 +86,12 @@ def test_an_unknown_pay_frequency_keeps_the_text_and_leaves_the_columns() -> Non
|
|||||||
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
|
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
|
||||||
|
|
||||||
|
|
||||||
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_left_out() -> None:
|
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_dropped_by_the_runner() -> None:
|
||||||
body = json.loads(_body())
|
body = json.loads(_body())
|
||||||
body["data"][0]["title"] = ""
|
body["data"][0]["title"] = ""
|
||||||
body["data"][1]["url"] = "javascript:alert(1)"
|
body["data"][1]["url"] = "javascript:alert(1)"
|
||||||
assert pinpoint.parse(json.dumps(body).encode(), "xeneta") == []
|
rows = pinpoint.parse(json.dumps(body).encode(), "xeneta")
|
||||||
|
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
|
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import rippling # noqa: E402
|
from plugins.readers import _runner, rippling # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "rippling.json").read_bytes()
|
BOARD = (FIXTURES / "rippling.json").read_bytes()
|
||||||
@@ -61,11 +61,14 @@ def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None
|
|||||||
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
|
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
|
||||||
|
|
||||||
|
|
||||||
def test_an_entry_without_a_title_or_a_plain_id_is_skipped() -> None:
|
def test_an_entry_without_a_title_or_a_plain_id_is_dropped_by_the_runner() -> None:
|
||||||
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
|
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
|
||||||
{"uuid": "../../x", "name": "C"},
|
{"uuid": "../../x", "name": "C"},
|
||||||
{"uuid": "abcdefgh-2", "name": "D"}]).encode()
|
{"uuid": "abcdefgh-2", "name": "D"}]).encode()
|
||||||
assert [r["source_id"] for r in rippling.parse(body, "acme")] == ["abcdefgh-2"]
|
rows = rippling.parse(body, "acme")
|
||||||
|
assert [r["source_id"] for r in rows] == ["abcdefgh-1", "", "../../x", "abcdefgh-2"]
|
||||||
|
assert rows[2]["url"] == "" # the id builds no link
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["abcdefgh-2"]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- detail
|
# ---------------------------------------------------------------- detail
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
|||||||
if str(ROOT) not in sys.path:
|
if str(ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins.readers import Ask, teamtailor # noqa: E402
|
from plugins.readers import Ask, _runner, teamtailor # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -40,7 +40,8 @@ def test_page_takes_the_ask_and_reads_the_same_url() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
|
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
|
||||||
first, second, third = _rows()
|
first, second, third, refused = _rows()
|
||||||
|
assert not _runner.valid(refused) # javascript:, for the runner
|
||||||
# "Sverige" (a city that is a country) once, Stockholm's twin once.
|
# "Sverige" (a city that is a country) once, Stockholm's twin once.
|
||||||
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
|
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
|
||||||
{"text": "Stockholm, Sweden", "remote": False}]
|
{"text": "Stockholm, Sweden", "remote": False}]
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.readers import workday # noqa: E402
|
from plugins.readers import _runner, workday # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SLUG = "zendesk.wd1/zendesk"
|
SLUG = "zendesk.wd1/zendesk"
|
||||||
@@ -96,7 +96,7 @@ def test_posted_is_read_only_where_the_text_is_exact(text: str, expected: dateti
|
|||||||
assert workday._posted(text) == expected
|
assert workday._posted(text) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
|
def test_a_posting_without_a_title_or_a_path_reaches_the_runner_and_is_dropped() -> None:
|
||||||
page = {"jobPostings": [
|
page = {"jobPostings": [
|
||||||
{"bulletFields": ["0000036369"]}, # LEGO's, no title
|
{"bulletFields": ["0000036369"]}, # LEGO's, no title
|
||||||
{"title": "No path", "bulletFields": ["R1"]},
|
{"title": "No path", "bulletFields": ["R1"]},
|
||||||
@@ -105,7 +105,16 @@ def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
|
|||||||
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
|
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
|
||||||
"postedOn": "Posted Today", "bulletFields": ["R4"]},
|
"postedOn": "Posted Today", "bulletFields": ["R4"]},
|
||||||
]}
|
]}
|
||||||
assert [r["title"] for r in workday.rows(page, SLUG)] == ["Whole"]
|
rows = workday.rows(page, SLUG)
|
||||||
|
assert len(rows) == 5
|
||||||
|
assert [(r["source_id"], r["url"]) for r in rows[:2]] == [("", ""), ("", "")]
|
||||||
|
assert [r["title"] for r in rows if _runner.valid(r)] == ["Whole"]
|
||||||
|
# Through the runner: four dropped and counted, and the read complete,
|
||||||
|
# since every posting counts by its position towards the stated total.
|
||||||
|
body = json.dumps(page | {"total": 5}).encode()
|
||||||
|
listing = _runner.read_listing(workday, SLUG, lambda url, **kw: body)
|
||||||
|
assert [r["title"] for r in listing.rows] == ["Whole"]
|
||||||
|
assert (listing.handed, listing.dropped, listing.complete) == (5, 4, True)
|
||||||
|
|
||||||
|
|
||||||
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
|
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
|||||||
"VALUES (?, ?, '', '')", (key, name))
|
"VALUES (?, ?, '', '')", (key, name))
|
||||||
conn.execute("PRAGMA user_version = 19")
|
conn.execute("PRAGMA user_version = 19")
|
||||||
reopened = Store(path)
|
reopened = Store(path)
|
||||||
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert reopened.migrations_applied == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||||
assert len(reopened.companies()) == 3
|
assert len(reopened.companies()) == 3
|
||||||
|
|
||||||
@@ -186,7 +186,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
|||||||
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
|
conn.execute("ALTER TABLE companies DROP COLUMN all_countries")
|
||||||
conn.execute("PRAGMA user_version = 20")
|
conn.execute("PRAGMA user_version = 20")
|
||||||
reopened = Store(path)
|
reopened = Store(path)
|
||||||
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert reopened.migrations_applied == [21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
# The tag of 21 is All countries and All roles since 24.
|
# The tag of 21 is All countries and All roles since 24.
|
||||||
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
tagged = {"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||||
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
|
assert {c.key: c.all_roles for c in reopened.companies()} == tagged
|
||||||
|
|||||||
+2
-34
@@ -91,37 +91,6 @@ def test_best_cv_match_puts_the_judged_first_by_cv_score(
|
|||||||
"order": "best"}).status_code == 422
|
"order": "best"}).status_code == 422
|
||||||
|
|
||||||
|
|
||||||
def test_a_broken_must_have_caps_career_in_the_order_and_the_reader(
|
|
||||||
client: TestClient, store: Store) -> None:
|
|
||||||
owner, uid = setup(store)
|
|
||||||
# c: 9 on all three, but one must-have broken: career counts as 4.
|
|
||||||
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
|
|
||||||
{"id": "M2", "text": "Remote", "verdict": "broken"}]
|
|
||||||
store.set_judgement(owner.id, uid["c"], "k", 9, 9, 9, [], [], musts)
|
|
||||||
capped = round(10 * (30 * 9 + 25 * 9 + 30 * 4) / 85, 1)
|
|
||||||
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
|
|
||||||
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["c"], capped), (uid["b"], 60.0)]
|
|
||||||
# With career weighted up the capped value drops under b's 60.
|
|
||||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=owner.cv_text,
|
|
||||||
weights=Weights(cv_skills=0, cv_experience=0, cv_career=1))
|
|
||||||
rows = client.get("/api/roles", params={"agent": owner.id, "order": "cv"}).json()["items"]
|
|
||||||
assert [(r["uid"], r["cv_score"]) for r in rows[:2]] == [(uid["b"], 60.0), (uid["c"], 40.0)]
|
|
||||||
one = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()
|
|
||||||
j = one["judgement"]
|
|
||||||
assert (j["career"], j["career_capped"], j["broken"]) == (9, 4, 1)
|
|
||||||
assert j["musts"] == musts and one["cv_score"] == j["cv_score"] == 40.0
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_met_or_unclear_must_have_leaves_career_as_the_model_gave_it(
|
|
||||||
client: TestClient, store: Store) -> None:
|
|
||||||
owner, uid = setup(store)
|
|
||||||
musts = [{"id": "M1", "text": "Permanent", "verdict": "met"},
|
|
||||||
{"id": "M2", "text": "Remote", "verdict": "unclear"}]
|
|
||||||
store.set_judgement(owner.id, uid["c"], "k", 9, 9, 9, [], [], musts)
|
|
||||||
j = client.get(f"/api/roles/{uid['c']}", params={"agent": owner.id}).json()["judgement"]
|
|
||||||
assert (j["career"], j["career_capped"], j["broken"], j["cv_score"]) == (9, 9, 0, 90.0)
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_agents_weights_make_the_cv_score_at_read_time(
|
def test_the_agents_weights_make_the_cv_score_at_read_time(
|
||||||
client: TestClient, store: Store) -> None:
|
client: TestClient, store: Store) -> None:
|
||||||
owner, uid = setup(store)
|
owner, uid = setup(store)
|
||||||
@@ -218,15 +187,14 @@ def test_an_agent_without_languages_flags_nothing(client: TestClient, store: Sto
|
|||||||
|
|
||||||
def test_the_cv_fields_round_trip_and_are_kept_when_left_out(client: TestClient) -> None:
|
def test_the_cv_fields_round_trip_and_are_kept_when_left_out(client: TestClient) -> None:
|
||||||
body = {"name": "Data", "cv_text": "VP Data.", "like_text": "Teams.", "avoid_text": "Reporting.",
|
body = {"name": "Data", "cv_text": "VP Data.", "like_text": "Teams.", "avoid_text": "Reporting.",
|
||||||
"must_text": "Permanent.",
|
|
||||||
"cv_languages": [{"language": "da", "level": "native"},
|
"cv_languages": [{"language": "da", "level": "native"},
|
||||||
{"language": "EN", "level": "fluent"}]}
|
{"language": "EN", "level": "fluent"}]}
|
||||||
made = client.post("/api/agents", json=body).json()
|
made = client.post("/api/agents", json=body).json()
|
||||||
assert made["cv_languages"] == [{"language": "da", "level": "native"},
|
assert made["cv_languages"] == [{"language": "da", "level": "native"},
|
||||||
{"language": "en", "level": "fluent"}]
|
{"language": "en", "level": "fluent"}]
|
||||||
kept = client.post("/api/agents", json={"id": made["id"], "name": "Data"}).json()
|
kept = client.post("/api/agents", json={"id": made["id"], "name": "Data"}).json()
|
||||||
assert (kept["cv_text"], kept["like_text"], kept["avoid_text"], kept["must_text"]) == \
|
assert (kept["cv_text"], kept["like_text"], kept["avoid_text"]) == \
|
||||||
("VP Data.", "Teams.", "Reporting.", "Permanent.")
|
("VP Data.", "Teams.", "Reporting.")
|
||||||
assert len(kept["cv_languages"]) == 2
|
assert len(kept["cv_languages"]) == 2
|
||||||
nulls = client.post("/api/agents", json={"id": made["id"], "name": "Data",
|
nulls = client.post("/api/agents", json={"id": made["id"], "name": "Data",
|
||||||
"cv_text": None, "cv_languages": None}).json()
|
"cv_text": None, "cv_languages": None}).json()
|
||||||
|
|||||||
+46
-42
@@ -16,7 +16,7 @@ from pydantic import ValidationError
|
|||||||
from app import prep
|
from app import prep
|
||||||
from app.config import Config
|
from app.config import Config
|
||||||
from app.jobs import extract
|
from app.jobs import extract
|
||||||
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag, must_lines
|
from app.models import CvLanguage, RoleLanguage, cv_score, iso_day, language_flag
|
||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
from desk import modeltask
|
from desk import modeltask
|
||||||
from desk.models import Item
|
from desk.models import Item
|
||||||
@@ -44,24 +44,21 @@ def test_an_agent_keeps_its_cv_fields_when_a_save_leaves_them_out(store: Store)
|
|||||||
cv_languages=[CvLanguage(language="DA", level="native"),
|
cv_languages=[CvLanguage(language="DA", level="native"),
|
||||||
CvLanguage(language="en", level="fluent"),
|
CvLanguage(language="en", level="fluent"),
|
||||||
CvLanguage(language="da", level="basic")],
|
CvLanguage(language="da", level="basic")],
|
||||||
like_text=" Building teams. ", avoid_text="Pure reporting.",
|
like_text=" Building teams. ", avoid_text="Pure reporting.")
|
||||||
must_text="Permanent contract.")
|
|
||||||
assert made.cv_text == "VP Data."
|
assert made.cv_text == "VP Data."
|
||||||
assert [(lang.language, lang.level) for lang in made.cv_languages] == \
|
assert [(lang.language, lang.level) for lang in made.cv_languages] == \
|
||||||
[("da", "native"), ("en", "fluent")]
|
[("da", "native"), ("en", "fluent")]
|
||||||
assert (made.like_text, made.avoid_text, made.must_text) == \
|
assert (made.like_text, made.avoid_text) == ("Building teams.", "Pure reporting.")
|
||||||
("Building teams.", "Pure reporting.", "Permanent contract.")
|
|
||||||
|
|
||||||
kept = store.save_agent("Data", "data leadership", agent_id=made.id)
|
kept = store.save_agent("Data", "data leadership", agent_id=made.id)
|
||||||
assert (kept.cv_text, kept.like_text, kept.avoid_text, kept.must_text) == \
|
assert (kept.cv_text, kept.like_text, kept.avoid_text) == \
|
||||||
("VP Data.", "Building teams.", "Pure reporting.", "Permanent contract.")
|
("VP Data.", "Building teams.", "Pure reporting.")
|
||||||
assert len(kept.cv_languages) == 2
|
assert len(kept.cv_languages) == 2
|
||||||
|
|
||||||
cleared = store.save_agent("Data", "data leadership", agent_id=made.id, cv_text="",
|
cleared = store.save_agent("Data", "data leadership", agent_id=made.id, cv_text="",
|
||||||
cv_languages=[], like_text="", avoid_text="",
|
cv_languages=[], like_text="", avoid_text="")
|
||||||
must_text="")
|
assert (cleared.cv_text, cleared.cv_languages, cleared.like_text, cleared.avoid_text) == \
|
||||||
assert (cleared.cv_text, cleared.cv_languages, cleared.like_text, cleared.avoid_text,
|
("", [], "", "")
|
||||||
cleared.must_text) == ("", [], "", "", "")
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_cv_language_is_a_code_and_a_level_of_the_ladder() -> None:
|
def test_a_cv_language_is_a_code_and_a_level_of_the_ladder() -> None:
|
||||||
@@ -92,25 +89,24 @@ def test_v28_adds_the_cv_columns_and_the_host_deadline_and_runs_twice(tmp_path:
|
|||||||
conn.execute("DROP TABLE role_judgements")
|
conn.execute("DROP TABLE role_judgements")
|
||||||
for column in ("languages_required", "deadline_host", "deadline_model", "deadline"):
|
for column in ("languages_required", "deadline_host", "deadline_model", "deadline"):
|
||||||
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
conn.execute(f"ALTER TABLE roles DROP COLUMN {column}")
|
||||||
for column in ("cv_text", "cv_languages", "avoid_text", "must_text"):
|
for column in ("cv_text", "cv_languages", "avoid_text"):
|
||||||
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
|
conn.execute(f"ALTER TABLE agents DROP COLUMN {column}")
|
||||||
conn.execute("PRAGMA user_version = 27")
|
conn.execute("PRAGMA user_version = 27")
|
||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
rows = {r["source_id"]: r for r in migrated._rows(
|
rows = {r["source_id"]: r for r in migrated._rows(
|
||||||
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
|
"SELECT source_id, languages_required, deadline_host, deadline FROM roles")}
|
||||||
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
|
assert (rows["a"]["deadline_host"], rows["a"]["deadline"]) == ("2026-10-18", None)
|
||||||
assert rows["a"]["languages_required"] is None and rows["b"]["deadline_host"] == ""
|
assert rows["a"]["languages_required"] is None and rows["b"]["deadline_host"] == ""
|
||||||
[data] = migrated.agents()
|
[data] = migrated.agents()
|
||||||
assert (data.cv_text, data.cv_languages, data.avoid_text, data.must_text) == \
|
assert (data.cv_text, data.cv_languages, data.avoid_text) == ("", [], "")
|
||||||
("", [], "", "")
|
|
||||||
prep.run(migrated, remap=True, dupes=False)
|
prep.run(migrated, remap=True, dupes=False)
|
||||||
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
|
assert migrated.role(posting("a").uid).deadline == "2026-10-18"
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 27")
|
conn.execute("PRAGMA user_version = 27")
|
||||||
assert Store(path).migrations_applied == [28, 29, 30, 31, 32, 33]
|
assert Store(path).migrations_applied == [28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- deadline
|
# ---------------------------------------------------------------- deadline
|
||||||
@@ -231,7 +227,6 @@ def test_the_language_flag() -> None:
|
|||||||
def test_the_cv_score_weighs_skills_and_career_above_experience() -> None:
|
def test_the_cv_score_weighs_skills_and_career_above_experience() -> None:
|
||||||
assert cv_score(10, 10, 10) == 100.0
|
assert cv_score(10, 10, 10) == 100.0
|
||||||
assert cv_score(8, 6, 4) == round(10 * (240 + 150 + 120) / 85, 1)
|
assert cv_score(8, 6, 4) == round(10 * (240 + 150 + 120) / 85, 1)
|
||||||
assert cv_score(8, 6, 9, broken=1) == cv_score(8, 6, 4)
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
|
def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
|
||||||
@@ -244,20 +239,36 @@ def test_a_stored_language_list_survives_a_bad_entry(store: Store) -> None:
|
|||||||
assert [lang.language for lang in store.role(item.uid).languages_required] == ["da"]
|
assert [lang.language for lang in store.role(item.uid).languages_required] == ["da"]
|
||||||
|
|
||||||
|
|
||||||
def test_v29_adds_the_musts_and_broken_columns_and_runs_twice(tmp_path: Path) -> None:
|
def test_v34_drops_the_must_haves_and_runs_twice(tmp_path: Path) -> None:
|
||||||
path = tmp_path / "v28.db"
|
"""A file from before migration 34 holds the agent's must text and the
|
||||||
Store(path)
|
judge's verdicts on it; the columns go, the rest of the agent and the
|
||||||
|
judgement stay."""
|
||||||
|
path = tmp_path / "v33.db"
|
||||||
|
store = Store(path)
|
||||||
|
item = posting("1")
|
||||||
|
upsert(store, item)
|
||||||
|
agent = store.save_agent("Data", "data leadership", cv_text="CV.", like_text="Teams.")
|
||||||
|
assert store.set_judgement(agent.id, item.uid, "key", 8, 7, 9, ["Good."], ["Bad."])
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
for column in ("musts", "broken"):
|
conn.execute("ALTER TABLE agents ADD COLUMN must_text TEXT NOT NULL DEFAULT ''")
|
||||||
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
|
conn.execute("ALTER TABLE role_judgements ADD COLUMN musts TEXT NOT NULL DEFAULT '[]'")
|
||||||
conn.execute("PRAGMA user_version = 28")
|
conn.execute("ALTER TABLE role_judgements ADD COLUMN broken INTEGER NOT NULL DEFAULT 0")
|
||||||
|
conn.execute("UPDATE agents SET must_text = 'Remote.'")
|
||||||
|
conn.execute("UPDATE role_judgements SET musts = ?, broken = 1",
|
||||||
|
(json.dumps([{"id": "M1", "text": "Remote.", "verdict": "broken"}]),))
|
||||||
|
conn.execute("PRAGMA user_version = 33")
|
||||||
|
|
||||||
assert Store(path).migrations_applied == [29, 30, 31, 32, 33]
|
for _ in range(2): # runs twice without harm
|
||||||
with sqlite3.connect(path) as conn:
|
migrated = Store(path)
|
||||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
assert migrated.migrations_applied == [34, 35]
|
||||||
assert {"musts", "broken"} <= columns
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 28")
|
assert "must_text" not in {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
|
||||||
assert Store(path).migrations_applied == [29, 30, 31, 32, 33]
|
assert not {"musts", "broken"} & {
|
||||||
|
r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
||||||
|
conn.execute("PRAGMA user_version = 33")
|
||||||
|
assert migrated.agent(agent.id).like_text == "Teams." # type: ignore[union-attr]
|
||||||
|
found = migrated.judgement(agent.id, item.uid)
|
||||||
|
assert found is not None and (found.career, found.cv_score) == (9, cv_score(8, 7, 9))
|
||||||
|
|
||||||
|
|
||||||
def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
|
def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
|
||||||
@@ -269,12 +280,12 @@ def test_v30_adds_like_text_and_runs_twice(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [30, 31, 32, 33]
|
assert migrated.migrations_applied == [30, 31, 32, 33, 34, 35]
|
||||||
[data] = migrated.agents()
|
[data] = migrated.agents()
|
||||||
assert (data.like_text, data.avoid_text) == ("", "Reporting.")
|
assert (data.like_text, data.avoid_text) == ("", "Reporting.")
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 29")
|
conn.execute("PRAGMA user_version = 29")
|
||||||
assert Store(path).migrations_applied == [30, 31, 32, 33]
|
assert Store(path).migrations_applied == [30, 31, 32, 33, 34, 35]
|
||||||
|
|
||||||
|
|
||||||
def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
||||||
@@ -290,8 +301,7 @@ def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
|||||||
old = store.save_agent("Old", "data leadership", cv_text="CV.")
|
old = store.save_agent("Old", "data leadership", cv_text="CV.")
|
||||||
own = store.save_agent("Own", "data leadership")
|
own = store.save_agent("Own", "data leadership")
|
||||||
bare = store.save_agent("Bare", "data leadership")
|
bare = store.save_agent("Bare", "data leadership")
|
||||||
musts = [{"id": "M1", "text": "Remote", "verdict": "broken"}]
|
assert store.set_judgement(old.id, item.uid, "key", 80, 70, 90, ["Good."], ["Bad."])
|
||||||
assert store.set_judgement(old.id, item.uid, "key", 80, 70, 90, ["Good."], ["Bad."], musts)
|
|
||||||
store.judge_missed([(own.id, item.uid)])
|
store.judge_missed([(own.id, item.uid)])
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("UPDATE agents SET limits = json_set(limits, '$.summarise_top', 50) "
|
conn.execute("UPDATE agents SET limits = json_set(limits, '$.summarise_top', 50) "
|
||||||
@@ -303,22 +313,16 @@ def test_v33_clears_every_judgement_and_lowers_the_old_summarise_default(
|
|||||||
|
|
||||||
for _ in range(2): # runs twice without harm
|
for _ in range(2): # runs twice without harm
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
assert migrated.migrations_applied == [33]
|
assert migrated.migrations_applied == [33, 34, 35]
|
||||||
assert migrated.judgement(old.id, item.uid) is None
|
assert migrated.judgement(old.id, item.uid) is None
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT skills, experience, career, cv_score, strengths, gaps, musts, broken, "
|
"SELECT skills, experience, career, cv_score, strengths, gaps, "
|
||||||
"input_key, tries, judged_at FROM role_judgements").fetchall()
|
"input_key, tries, judged_at FROM role_judgements").fetchall()
|
||||||
assert rows == [(None, None, None, None, "[]", "[]", "[]", 0, "", 0, None)] * 2
|
assert rows == [(None, None, None, None, "[]", "[]", "", 0, None)] * 2
|
||||||
conn.execute("PRAGMA user_version = 32")
|
conn.execute("PRAGMA user_version = 32")
|
||||||
assert {a.name: a.limits.summarise_top for a in migrated.agents()} == {
|
assert {a.name: a.limits.summarise_top for a in migrated.agents()} == {
|
||||||
"Old": 30, "Own": 12, "Bare": 30}
|
"Old": 30, "Own": 12, "Bare": 30}
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
assert conn.execute("SELECT limits FROM agents WHERE id = ?",
|
assert conn.execute("SELECT limits FROM agents WHERE id = ?",
|
||||||
(bare.id,)).fetchone() == ("{}",)
|
(bare.id,)).fetchone() == ("{}",)
|
||||||
|
|
||||||
|
|
||||||
def test_the_musts_text_is_one_must_have_per_line() -> None:
|
|
||||||
assert must_lines("- A\n* B c\n\u2022 D\n1. E\n2) F\n \nG\n-H") == [
|
|
||||||
"A", "B c", "D", "E", "F", "G", "-H"]
|
|
||||||
assert must_lines("") == []
|
|
||||||
@@ -282,7 +282,7 @@ def test_migration_31_adds_the_column_and_the_table_and_runs_twice(tmp_path: Pat
|
|||||||
conn.execute("DROP TABLE agent_cv_vectors")
|
conn.execute("DROP TABLE agent_cv_vectors")
|
||||||
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
|
conn.execute("ALTER TABLE scores DROP COLUMN cv_fit")
|
||||||
conn.execute("PRAGMA user_version = 30")
|
conn.execute("PRAGMA user_version = 30")
|
||||||
assert Store(path).migrations_applied == [31, 32, 33]
|
assert Store(path).migrations_applied == [31, 32, 33, 34, 35]
|
||||||
step = {v: s for v, _, s in MIGRATIONS}[31]
|
step = {v: s for v, _, s in MIGRATIONS}[31]
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
assert callable(step)
|
assert callable(step)
|
||||||
|
|||||||
+14
-2
@@ -239,14 +239,26 @@ def test_folding_needs_the_same_title_a_link_and_texts_that_can_be_one_job() ->
|
|||||||
assert fold([pc("1", "Berlin", a), pc("2", "Munich", "")]).placed == {"2"}
|
assert fold([pc("1", "Berlin", a), pc("2", "Munich", "")]).placed == {"2"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_per_place_fold_keeps_to_one_country() -> None:
|
||||||
|
"""A federal sales role posted in Canberra and in Washington has alike
|
||||||
|
text and is two openings; Berlin and Munich are one job in two offices.
|
||||||
|
A row with no country joins either."""
|
||||||
|
text = "Sell to federal agencies, run the public sector pipeline and partners."
|
||||||
|
got = fold([replace(pc("au", "Canberra", text), country="AU"),
|
||||||
|
replace(pc("us", "Washington", text), country="US"),
|
||||||
|
pc("b", "Berlin", text), pc("m", "Munich", text),
|
||||||
|
replace(pc("n", "", text), country="")])
|
||||||
|
assert got.dup_of == {"au": None, "us": None, "b": None, "m": "b", "n": "au"}
|
||||||
|
|
||||||
|
|
||||||
def test_a_remote_listing_is_not_folded_into_an_office_one() -> None:
|
def test_a_remote_listing_is_not_folded_into_an_office_one() -> None:
|
||||||
"""The kept role takes a copy's places, not its remote, so the gate would
|
"""The kept role takes a copy's places, not its remote, so the gate would
|
||||||
read "Remote, Germany" as an office role in Vienna and reject it for an
|
read "Remote, Germany" as an office role in Vienna and reject it for an
|
||||||
agent that takes remote roles. Two remote listings still fold."""
|
agent that takes remote roles. Two remote listings still fold."""
|
||||||
text = "We build the payment platform for our bank customers in Kotlin."
|
text = "We build the payment platform for our bank customers in Kotlin."
|
||||||
office = replace(pc("v", "Vienna", text), country="AT")
|
office = replace(pc("v", "Vienna", text), country="AT")
|
||||||
remote = replace(pc("r", "Remote, Germany", text), remote=True)
|
remote = replace(pc("r", "Remote, Berlin", text), remote=True)
|
||||||
other = replace(pc("s", "Remote, Spain", text), country="ES", remote=True)
|
other = replace(pc("s", "Remote, Hamburg", text), remote=True)
|
||||||
got = fold([office, remote, other])
|
got = fold([office, remote, other])
|
||||||
assert got.dup_of == {"v": None, "r": None, "s": "r"}
|
assert got.dup_of == {"v": None, "r": None, "s": "r"}
|
||||||
assert got.placed == {"s"}
|
assert got.placed == {"s"}
|
||||||
|
|||||||
+22
-19
@@ -103,10 +103,11 @@ def test_berlin_and_munich_of_one_host_fold_into_one_role_with_two_places(
|
|||||||
assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0)
|
assert (row.uid, row.place, row.copies) == (kept, f"{got.places[0].text} +1", 0)
|
||||||
|
|
||||||
|
|
||||||
def test_a_place_copy_in_europe_meets_a_europe_gate_for_its_kept_role(store: Store) -> None:
|
def test_a_posting_per_office_in_two_countries_is_two_roles(store: Store) -> None:
|
||||||
"""OpenAI posts one role in San Francisco and in Dublin. The kept role
|
"""OpenAI posts one role in San Francisco and in Dublin. The fold keeps
|
||||||
holds both places, so a Europe gate and the Place filter find it through
|
to one country (ADR 1.4), so each is a role of its own: a Europe gate
|
||||||
Dublin, though `roles.place` and `roles.country` are San Francisco's."""
|
takes Dublin and rejects San Francisco, and the Place filter finds
|
||||||
|
Dublin alone."""
|
||||||
from app import prep
|
from app import prep
|
||||||
from app.models import Gate
|
from app.models import Gate
|
||||||
sf = role("sf", company="OpenAI", place="San Francisco", country="US", board="ashby")
|
sf = role("sf", company="OpenAI", place="San Francisco", country="US", board="ashby")
|
||||||
@@ -114,10 +115,11 @@ def test_a_place_copy_in_europe_meets_a_europe_gate_for_its_kept_role(store: Sto
|
|||||||
for item in (sf, dublin):
|
for item in (sf, dublin):
|
||||||
store.upsert_role(item)
|
store.upsert_role(item)
|
||||||
prep.run(store)
|
prep.run(store)
|
||||||
kept = kept_of(store, sf, dublin)
|
assert dup_of(store) == {sf.uid: None, dublin.uid: None}
|
||||||
agent = store.save_agent("A", gates=[Gate(field="country", values=["FR", "IE", "EU"])])
|
agent = store.save_agent("A", gates=[Gate(field="country", values=["FR", "IE", "EU"])])
|
||||||
assert {c.uid: c.gates for c in store.score_candidates(agent.id, agent.gates)} == {kept: [1]}
|
assert {c.uid: c.gates for c in store.score_candidates(agent.id, agent.gates)} == {
|
||||||
assert {r.uid for r in store.roles(places=["country:IE"], by_score=False)} == {kept}
|
sf.uid: [0], dublin.uid: [1]}
|
||||||
|
assert {r.uid for r in store.roles(places=["country:IE"], by_score=False)} == {dublin.uid}
|
||||||
|
|
||||||
|
|
||||||
def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None:
|
def test_places_of_two_hosts_or_two_different_texts_stay_apart(store: Store) -> None:
|
||||||
@@ -279,7 +281,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||||
@@ -305,30 +307,31 @@ def test_switching_a_board_off_shows_its_roles_copies_at_once(store: Store) -> N
|
|||||||
|
|
||||||
|
|
||||||
def test_a_role_in_several_cities_leads_with_the_one_nearest_the_agent(store: Store) -> None:
|
def test_a_role_in_several_cities_leads_with_the_one_nearest_the_agent(store: Store) -> None:
|
||||||
"""Mistral posts one role per city. For an agent kept to Europe the kept
|
"""A Danish company posts one role per city. For an agent whose towns
|
||||||
Montréal role leads with Paris, in the row and the reader, and opens the
|
hold Copenhagen the kept Aarhus role leads with Copenhagen, in the row
|
||||||
Paris copy; without an agent the host's order stands."""
|
and the reader, and opens the Copenhagen copy; without an agent the
|
||||||
|
host's order stands."""
|
||||||
from app import prep
|
from app import prep
|
||||||
from app.models import Gate
|
from app.models import Gate
|
||||||
items = [role(c, company="Mistral AI", place=p, board="ashby") for c, p in (
|
items = [role(c, company="Danske", place=p, board="ashby") for c, p in (
|
||||||
("mtl", "Montréal"), ("nyc", "New York"), ("par", "Paris"))]
|
("aar", "Aarhus"), ("bil", "Billund"), ("cph", "Copenhagen"))]
|
||||||
for item in items:
|
for item in items:
|
||||||
store.upsert_role(item)
|
store.upsert_role(item)
|
||||||
prep.run(store)
|
prep.run(store)
|
||||||
kept = kept_of(store, *items)
|
kept = kept_of(store, *items)
|
||||||
agent = store.save_agent("A", gates=[
|
agent = store.save_agent("A", gates=[
|
||||||
Gate(field="country", values=["FR", "DK", "EU"], mode="require"),
|
Gate(field="country", values=["FR", "DK", "EU"], mode="require"),
|
||||||
Gate(field="country", values=["DK"], mode="any")])
|
Gate(field="place", values=["copenhagen"], mode="any")])
|
||||||
|
|
||||||
got = store.role(kept, agent.id)
|
got = store.role(kept, agent.id)
|
||||||
assert got is not None
|
assert got is not None
|
||||||
own = next(p.text for p in got.places if not p.url)
|
own = next(p.text for p in got.places if not p.url)
|
||||||
assert own != "Paris"
|
assert own != "Copenhagen"
|
||||||
assert (got.lead_place, got.lead_url) == ("Paris", "https://jobs.example/par")
|
assert (got.lead_place, got.lead_url) == ("Copenhagen", "https://jobs.example/cph")
|
||||||
assert got.places[0].text == "Paris"
|
assert got.places[0].text == "Copenhagen"
|
||||||
(row,) = store.roles_page(agent=agent.id, by_score=False)
|
(row,) = store.roles_page(agent=agent.id, by_score=False)
|
||||||
assert (row.place, row.lead_place, row.lead_url) == ("Paris +2", "Paris", got.lead_url)
|
assert (row.place, row.lead_place, row.lead_url) == ("Copenhagen +2", "Copenhagen", got.lead_url)
|
||||||
assert row.place_texts[0] == "Paris" and len(row.place_texts) == 3
|
assert row.place_texts[0] == "Copenhagen" and len(row.place_texts) == 3
|
||||||
|
|
||||||
plain = store.role(kept)
|
plain = store.role(kept)
|
||||||
assert plain is not None
|
assert plain is not None
|
||||||
|
|||||||
@@ -520,7 +520,7 @@ def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
|
|||||||
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
|
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
|
||||||
conn.execute("PRAGMA user_version = 22")
|
conn.execute("PRAGMA user_version = 22")
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert migrated.agents()[0].fetch_terms == []
|
assert migrated.agents()[0].fetch_terms == []
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+17
-53
@@ -14,7 +14,7 @@ import pytest
|
|||||||
|
|
||||||
from app.config import Config
|
from app.config import Config
|
||||||
from app.jobs import judge, stages
|
from app.jobs import judge, stages
|
||||||
from app.models import CvLanguage, Prompt
|
from app.models import CvLanguage, Prompt, Weights
|
||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
from desk import modeltask
|
from desk import modeltask
|
||||||
from desk.models import Item
|
from desk.models import Item
|
||||||
@@ -76,8 +76,7 @@ def config(tmp_path: Path) -> Config:
|
|||||||
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
|
def hunter(store: Store, name: str = "Data", cv: str = CV, **kwargs: Any) -> Any:
|
||||||
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
|
return store.save_agent(name, prompts=[Prompt(name="AI lead", text="Head of AI")],
|
||||||
cv_text=cv, like_text="Building teams.",
|
cv_text=cv, like_text="Building teams.",
|
||||||
avoid_text="Pure reporting.",
|
avoid_text="Pure reporting.", **kwargs)
|
||||||
must_text="A permanent contract.", **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
|
def ranked(store: Store, owner: Any, *ids: str) -> list[str]:
|
||||||
@@ -109,59 +108,18 @@ def test_a_run_writes_the_judgement_and_its_total(
|
|||||||
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
|
assert item == {"key": f"{owner.id}:{uid}", "title": "Head of AI 1", "company": "Acme",
|
||||||
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
|
"place": "Copenhagen", "text": "Head of AI. Lead the data science team."}
|
||||||
[system] = model.systems
|
[system] = model.systems
|
||||||
assert CV in system and "Pure reporting." in system and "A permanent contract." in system
|
assert CV in system and "Pure reporting." in system
|
||||||
assert "- AI lead: Head of AI" in system and "deal-breaker" in system
|
assert "- AI lead: Head of AI" in system
|
||||||
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
|
assert '<likes title="what they like: work, workplace, situations">\nBuilding teams.' \
|
||||||
in system
|
in system
|
||||||
assert '<dislikes title="what they do not like">\nPure reporting.' in system
|
assert '<dislikes title="what they do not like">\nPure reporting.' in system
|
||||||
assert "<avoid" not in system
|
assert "<avoid" not in system
|
||||||
|
|
||||||
|
|
||||||
def test_the_musts_are_numbered_in_the_profile_and_stored_with_their_text(
|
def test_the_schema_asks_for_three_numbers_strengths_and_gaps() -> None:
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
owner = hunter(store)
|
|
||||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id, cv_text=CV,
|
|
||||||
must_text="- Permanent contract\n\n * Remote two days\n3) Danish team\n")
|
|
||||||
[uid] = ranked(store, owner, "1")
|
|
||||||
|
|
||||||
def reply(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
||||||
return [{**one, "career": 8, "musts": [
|
|
||||||
{"id": "M1", "verdict": "met"}, {"id": "M2", "verdict": "broken"},
|
|
||||||
{"id": "M9", "verdict": "broken"}, {"id": "M3", "verdict": "maybe"}]}
|
|
||||||
for one in judged(items)]
|
|
||||||
|
|
||||||
model = stub(monkeypatch, reply)
|
|
||||||
modeltask.run(judge.TASK, config, store)
|
|
||||||
|
|
||||||
[system] = model.systems
|
|
||||||
assert "M1: Permanent contract\nM2: Remote two days\nM3: Danish team" in system
|
|
||||||
found = store.judgement(owner.id, uid)
|
|
||||||
assert found is not None
|
|
||||||
assert [(m.id, m.text, m.verdict) for m in found.musts] == [
|
|
||||||
("M1", "Permanent contract", "met"), ("M2", "Remote two days", "broken"),
|
|
||||||
("M3", "Danish team", "unclear")]
|
|
||||||
assert (found.career, found.career_capped, found.broken) == (8, 4, 1)
|
|
||||||
assert found.cv_score == round(10 * (30 * 8 + 25 * 7 + 30 * 4) / 85, 1)
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_must_the_model_left_out_is_unclear_and_nothing_is_broken_by_silence(
|
|
||||||
store: Store, config: Config, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
owner = hunter(store)
|
|
||||||
[uid] = ranked(store, owner, "1")
|
|
||||||
stub(monkeypatch) # answers without any `musts`
|
|
||||||
modeltask.run(judge.TASK, config, store)
|
|
||||||
found = store.judgement(owner.id, uid)
|
|
||||||
assert found is not None
|
|
||||||
assert [(m.id, m.verdict) for m in found.musts] == [("M1", "unclear")]
|
|
||||||
assert (found.career, found.career_capped, found.broken) == (9, 9, 0)
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_schema_requires_the_verdicts_and_the_prompt_leaves_the_cap_to_code() -> None:
|
|
||||||
item = judge.SCHEMA["properties"]["judgements"]["items"]
|
item = judge.SCHEMA["properties"]["judgements"]["items"]
|
||||||
assert "musts" in item["required"]
|
assert item["required"] == ["key", "skills", "experience", "career", "strengths", "gaps"]
|
||||||
verdict = item["properties"]["musts"]["items"]["properties"]["verdict"]
|
assert "must-have" not in judge.SYSTEM
|
||||||
assert verdict["enum"] == ["met", "broken", "unclear"]
|
|
||||||
assert "career 0-3" not in judge.SYSTEM and "clearly contradicts" in judge.SYSTEM
|
|
||||||
assert "0 to 10" in judge.SYSTEM and "80-100" not in judge.SYSTEM
|
assert "0 to 10" in judge.SYSTEM and "80-100" not in judge.SYSTEM
|
||||||
|
|
||||||
|
|
||||||
@@ -190,7 +148,7 @@ def test_what_the_person_likes_is_part_of_what_a_judgement_read(store: Store) ->
|
|||||||
|
|
||||||
def hashlib_old_key(agent: Any, text: str) -> str:
|
def hashlib_old_key(agent: Any, text: str) -> str:
|
||||||
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
blob = json.dumps([agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
||||||
agent.avoid_text, agent.must_text, text], ensure_ascii=False)
|
agent.avoid_text, text], ensure_ascii=False)
|
||||||
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
@@ -259,12 +217,18 @@ def test_a_judgement_is_read_again_only_when_what_it_read_changed(
|
|||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
|
conn.execute("UPDATE roles SET closed_at = NULL WHERE uid = ?", (uid,))
|
||||||
|
|
||||||
# The languages are not part of what it read; the must text is.
|
# The languages and the dislikes are part of what it read; the weights
|
||||||
|
# are not.
|
||||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||||
cv_languages=[CvLanguage(language="da", level="native")])
|
weights=Weights(cv_skills=50))
|
||||||
assert judge.pending(store) == 0
|
assert judge.pending(store) == 0
|
||||||
|
spoken = store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||||
|
cv_languages=[CvLanguage(language="da", level="native")])
|
||||||
|
assert judge.pending(store) == 1
|
||||||
|
assert '<languages title="the languages they speak, and how well">\nDanish native' \
|
||||||
|
in judge.system(spoken)
|
||||||
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
store.save_agent("Data", prompts=owner.prompts, agent_id=owner.id,
|
||||||
must_text="Remote two days a week.")
|
avoid_text="Long commutes.")
|
||||||
assert judge.pending(store) == 1
|
assert judge.pending(store) == 1
|
||||||
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
|
assert store.judgement(owner.id, uid) is not None, "kept until the new one lands"
|
||||||
|
|
||||||
|
|||||||
@@ -116,6 +116,16 @@ def test_roles_without_text_and_text_reads_that_failed() -> None:
|
|||||||
assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
|
assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("dropped", "found"), [
|
||||||
|
("3", ["3 listings dropped: no title or link"]),
|
||||||
|
("1", ["1 listing dropped: no title or link"]),
|
||||||
|
("", []), ("x", [])])
|
||||||
|
def test_the_listings_the_last_read_dropped_are_named(dropped: str, found: list[str]) -> None:
|
||||||
|
"""The rows the runner dropped (`_runner.valid`), after "read incomplete"."""
|
||||||
|
assert words(fake(), state(dropped=dropped)) == found
|
||||||
|
assert words(fake(), state(dropped=dropped, complete=""))[1:] == found
|
||||||
|
|
||||||
|
|
||||||
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
|
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
|
||||||
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
|
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
|
||||||
assert words(fake(), state(), reach.SMALL) == []
|
assert words(fake(), state(), reach.SMALL) == []
|
||||||
@@ -199,6 +209,17 @@ def test_companies_carry_their_limits_and_the_boards_own_open_roles(
|
|||||||
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
|
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_that_dropped_listings_says_so_on_its_row(
|
||||||
|
client: TestClient, store: Store) -> None:
|
||||||
|
store.add_subscription("ats", "lever:acme", company="Acme", board="lever")
|
||||||
|
store.add_company("Acme", key="acme", ats="lever")
|
||||||
|
store.state["ats:ok:lever:acme"] = "run"
|
||||||
|
store.state["ats:complete:lever:acme"] = "1"
|
||||||
|
store.state["ats:dropped:lever:acme"] = "3"
|
||||||
|
[row] = client.get("/api/companies").json()
|
||||||
|
assert row["limits"][0] == "3 listings dropped: no title or link"
|
||||||
|
|
||||||
|
|
||||||
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
|
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
|
||||||
store.add_company("Novo", key="novo", ats="taleo")
|
store.add_company("Novo", key="novo", ats="taleo")
|
||||||
[row] = client.get("/api/companies").json()
|
[row] = client.get("/api/companies").json()
|
||||||
|
|||||||
@@ -51,8 +51,8 @@ def _read(store: Store, target: str) -> tuple[Ask, str]:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("countries, roles, line", [
|
@pytest.mark.parametrize("countries, roles, line", [
|
||||||
(False, False, "follows agents: DK, SE · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
(False, False, "follows agents' world (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
||||||
(False, True, "follows agents: DK, SE · remote not asked on Phenom · all roles"),
|
(False, True, "follows agents' world (Europe) · remote not asked on Phenom · all roles"),
|
||||||
(True, False, "all locations (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
(True, False, "all locations (Europe) · remote not asked on Phenom · follows agents' terms: data, engineer, product"),
|
||||||
(True, True, "all locations (Europe) · remote not asked on Phenom · all roles"),
|
(True, True, "all locations (Europe) · remote not asked on Phenom · all roles"),
|
||||||
], ids=["neither", "all roles alone", "all locations alone", "both"])
|
], ids=["neither", "all roles alone", "all locations alone", "both"])
|
||||||
@@ -62,11 +62,9 @@ def test_the_four_combinations(store: Store, countries: bool, roles: bool, line:
|
|||||||
store.set_all_roles("philips", roles)
|
store.set_all_roles("philips", roles)
|
||||||
ask, said = _read(store, PHILIPS)
|
ask, said = _read(store, PHILIPS)
|
||||||
assert said == line
|
assert said == line
|
||||||
# Phenom asks for countries, many in one request, and not for remote.
|
# Phenom asks for countries, many in one request, and not for remote: so
|
||||||
if countries:
|
# the agents want remote, it is asked for their world either way.
|
||||||
assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30
|
assert {"DE", "FR", "DK", "SE"} <= set(ask.countries) and len(ask.countries) > 30
|
||||||
else:
|
|
||||||
assert ask.countries == ("DK", "SE")
|
|
||||||
# Terms go to every big board, and never to a company set to All roles.
|
# Terms go to every big board, and never to a company set to All roles.
|
||||||
assert ask.terms == (() if roles else ("data", "engineer", "product"))
|
assert ask.terms == (() if roles else ("data", "engineer", "product"))
|
||||||
|
|
||||||
@@ -122,7 +120,7 @@ def test_migration_24_splits_the_tag_and_moves_terms_off(tmp_path: Path) -> None
|
|||||||
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
|
conn.execute("UPDATE companies SET all_roles=1 WHERE key='mistral-ai'")
|
||||||
conn.execute("PRAGMA user_version = 23")
|
conn.execute("PRAGMA user_version = 23")
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
|
flags = {c.key: (c.all_countries, c.all_roles) for c in migrated.companies()}
|
||||||
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
|
assert flags == {"mistral-ai": (1, 1), "acme": (0, 1), "plain": (0, 0)}
|
||||||
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
|
params = {s.target: s.params for s in migrated.subscriptions(source="ats")}
|
||||||
|
|||||||
+26
-2
@@ -224,7 +224,31 @@ def test_the_model_level_and_remote_win_over_the_host_and_the_place_line(
|
|||||||
("senior", "lead", "onsite")
|
("senior", "lead", "onsite")
|
||||||
|
|
||||||
|
|
||||||
def test_the_company_resolves_through_its_aliases(store: Store) -> None:
|
def test_remote_in_the_text_is_marked_and_a_denial_is_not() -> None:
|
||||||
|
yes = ("Fully-remote within the EU, we meet twice a year.",
|
||||||
|
"Work from home or from our Berlin office, as you like.",
|
||||||
|
"Dette er en stilling med mulighed for fjernarbejde.")
|
||||||
|
no = ("No remote work: this role is on site in Munich.",
|
||||||
|
"Remote sensing analyst for our satellite team.",
|
||||||
|
"A great office in Copenhagen with free lunch.")
|
||||||
|
assert all(words.remote_in_text("Engineer", text) for text in yes)
|
||||||
|
assert not any(words.remote_in_text("Engineer", text) for text in no)
|
||||||
|
assert words.remote_in_text("Backend Developer, remote-first team", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_prep_marks_a_role_whose_text_says_remote(store: Store) -> None:
|
||||||
|
said = item("1", title="Engineer", place="Berlin")
|
||||||
|
said.body = "Fully remote within Germany, with a Berlin office."
|
||||||
|
silent = item("2", title="Engineer", place="Berlin")
|
||||||
|
silent.body = "On site in Berlin."
|
||||||
|
store.upsert_role(said)
|
||||||
|
store.upsert_role(silent)
|
||||||
|
prep.run(store, tables=TABLES)
|
||||||
|
assert (row(store, "1")["remote"], row(store, "1")["remote_text"]) == ("", 1)
|
||||||
|
assert (row(store, "2")["remote"], row(store, "2")["remote_text"]) == ("", 0)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
store.add_company("Ørsted")
|
store.add_company("Ørsted")
|
||||||
store.upsert_role(item("1", company="ORSTED A/S"))
|
store.upsert_role(item("1", company="ORSTED A/S"))
|
||||||
assert row(store, "1")["company_key"] != "ørsted"
|
assert row(store, "1")["company_key"] != "ørsted"
|
||||||
@@ -316,7 +340,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
one = row(migrated, "1")
|
one = row(migrated, "1")
|
||||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||||
|
|||||||
@@ -250,6 +250,44 @@ def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
|
|||||||
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
|
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
|
||||||
|
|
||||||
|
|
||||||
|
def _aged(store: Store, source_id: str) -> None:
|
||||||
|
"""Make a role look unlisted for longer than `STALE_DAYS`."""
|
||||||
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(timespec="seconds")
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = ?", (old, source_id))
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_incomplete_read_closes_what_no_read_has_listed_for_stale_days(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""A board past its ceiling or page cap is never read complete; a role
|
||||||
|
it has not listed for three weeks still closes (ADR 1.3)."""
|
||||||
|
_answer(monkeypatch, ([_row("1"), _row("2"), _row("3")], True))
|
||||||
|
_pull(acme)
|
||||||
|
_aged(acme, "workable:acme:3")
|
||||||
|
_answer(monkeypatch, ([_row("1")], False))
|
||||||
|
result = _pull(acme)
|
||||||
|
# 2 was listed two pulls ago and stays; 3 has been unlisted for 22 days.
|
||||||
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||||
|
assert (result["incomplete"], result["closed"]) == (1, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filtered_read_closes_a_spared_role_once_it_is_stale(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""A filtered read spares a role acted on from its list; after three
|
||||||
|
weeks unlisted the role closes by age like any other."""
|
||||||
|
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
||||||
|
_pull(acme)
|
||||||
|
uid = {r.source_id: r.uid for r in acme.roles()}
|
||||||
|
agent = acme.save_agent("Hunt")
|
||||||
|
acme.set_vote(agent.id, uid["workable:acme:2"], 1)
|
||||||
|
_answer(monkeypatch, ([_row("1")], True), filtered=True)
|
||||||
|
assert _pull(acme)["closed"] == 0
|
||||||
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||||
|
_aged(acme, "workable:acme:2")
|
||||||
|
assert _pull(acme)["closed"] == 1
|
||||||
|
assert _open(acme) == {"workable:acme:1"}
|
||||||
|
|
||||||
|
|
||||||
def test_a_failing_board_keeps_its_roles_before_stale_days(
|
def test_a_failing_board_keeps_its_roles_before_stale_days(
|
||||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
_answer(monkeypatch, ([_row("1")], True))
|
_answer(monkeypatch, ([_row("1")], True))
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, adzuna # noqa: E402
|
from plugins.readers import Ask, _runner, adzuna # noqa: E402
|
||||||
|
|
||||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
|
||||||
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
|
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
|
||||||
@@ -39,8 +39,9 @@ def test_it_declares_the_key_and_states_the_robots_finding() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
|
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
|
||||||
first, predicted = _rows() # javascript: refused
|
first, predicted, refused = _rows()
|
||||||
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
|
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
|
||||||
|
assert refused["url"].startswith("javascript:") and not _runner.valid(refused)
|
||||||
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
||||||
assert first["teaser"] == "Build our backend services in Python & Go."
|
assert first["teaser"] == "Build our backend services in Python & Go."
|
||||||
assert "description" not in first
|
assert "description" not in first
|
||||||
@@ -89,3 +90,12 @@ def test_saved_params_need_a_known_country() -> None:
|
|||||||
"q": "x", "country": "gb"}
|
"q": "x", "country": "gb"}
|
||||||
with pytest.raises(ValueError, match="country"):
|
with pytest.raises(ValueError, match="country"):
|
||||||
adzuna.clean_params({"q": "x", "country": "dk"})
|
adzuna.clean_params({"q": "x", "country": "dk"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_runner_drops_the_ad_without_a_web_link_and_counts_it() -> None:
|
||||||
|
"""The reader hands the `javascript:` ad over; the runner drops it, and
|
||||||
|
the read is complete, since all three result ids were read."""
|
||||||
|
listing = _runner.read_listing(adzuna, "", lambda url, **kw: BODY, ASK,
|
||||||
|
limits=_runner.Limits(page_delay=0))
|
||||||
|
assert listing.ids == ["4412345678", "4412345679"]
|
||||||
|
assert (listing.handed, listing.dropped, listing.complete) == (3, 1, True)
|
||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, freehire # noqa: E402
|
from plugins.readers import Ask, _runner, freehire # noqa: E402
|
||||||
|
|
||||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
|
||||||
|
|
||||||
@@ -119,14 +119,17 @@ def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None:
|
|||||||
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
|
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
|
||||||
|
|
||||||
|
|
||||||
def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None:
|
def test_a_hit_without_a_title_an_id_or_a_web_link_is_handed_to_the_runner() -> None:
|
||||||
|
"""The reader hands every hit over; the runner keeps only the whole one."""
|
||||||
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
|
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
|
||||||
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
|
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
|
||||||
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
|
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
|
||||||
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
|
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
|
||||||
found = freehire.rows({"data": hits}, "")
|
found = freehire.rows({"data": hits}, "")
|
||||||
assert [row["source_id"] for row in found] == ["d"]
|
assert [row["source_id"] for row in found] == ["a", "", "c", "d"]
|
||||||
assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == []
|
kept = [row for row in found if _runner.valid(row)]
|
||||||
|
assert [row["source_id"] for row in kept] == ["d"]
|
||||||
|
assert kept[0]["url"] == "https://x.example/d" and kept[0]["places"] == []
|
||||||
|
|
||||||
|
|
||||||
def test_a_title_and_a_company_have_their_entities_decoded() -> None:
|
def test_a_title_and_a_company_have_their_entities_decoded() -> None:
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, jobbank # noqa: E402
|
from plugins.readers import Ask, _runner, jobbank # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||||
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
|
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
|
||||||
@@ -170,7 +170,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
|||||||
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||||
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
|
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
|
||||||
assert len(results) == 100 and nxt == 2
|
assert len(results) == 100 and nxt == 2
|
||||||
assert len(jobbank.rows(decoded, "")) == 99
|
rows = jobbank.rows(decoded, "")
|
||||||
|
assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
|
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, jobindex # noqa: E402
|
from plugins.readers import Ask, _runner, jobindex # noqa: E402
|
||||||
|
|
||||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
|
||||||
|
|
||||||
@@ -89,7 +89,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
|||||||
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||||
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
|
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
|
||||||
assert len(results) == 20 and nxt == 2
|
assert len(results) == 20 and nxt == 2
|
||||||
assert len(jobindex.rows(decoded, "")) == 19
|
rows = jobindex.rows(decoded, "")
|
||||||
|
assert len(rows) == 20 and sum(map(_runner.valid, rows)) == 19
|
||||||
|
|
||||||
|
|
||||||
def test_an_answer_without_a_channel_raises() -> None:
|
def test_an_answer_without_a_channel_raises() -> None:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, jobnet # noqa: E402
|
from plugins.readers import Ask, _runner, jobnet # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||||
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
|
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
|
||||||
@@ -100,9 +100,12 @@ def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None:
|
|||||||
assert "applicationDeadline" not in row
|
assert "applicationDeadline" not in row
|
||||||
|
|
||||||
|
|
||||||
def test_an_ad_without_an_id_or_a_title_is_skipped() -> None:
|
def test_an_ad_without_an_id_or_a_title_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
ads = json.loads(OTHER)["jobAds"]
|
ads = json.loads(OTHER)["jobAds"]
|
||||||
assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == []
|
rows = jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "")
|
||||||
|
assert [(row["source_id"] == "", row["title"] == "") for row in rows] == [
|
||||||
|
(False, True), (True, False)]
|
||||||
|
assert not any(map(_runner.valid, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
|
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, platsbanken # noqa: E402
|
from plugins.readers import Ask, _runner, platsbanken # noqa: E402
|
||||||
|
|
||||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
|
||||||
|
|
||||||
@@ -47,8 +47,9 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
|
|||||||
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||||
rows = platsbanken.rows(decoded, "")
|
rows = platsbanken.rows(decoded, "")
|
||||||
assert total == 32
|
assert total == 32
|
||||||
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400001", "31400002"]
|
||||||
first, no_town, bare = rows
|
assert not _runner.valid(rows[2]) # javascript:, dropped by the runner
|
||||||
|
first, no_town, _refused, bare = rows
|
||||||
assert [p["text"] for p in first["places"]] == [
|
assert [p["text"] for p in first["places"]] == [
|
||||||
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
|
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
|
||||||
assert first["location"] == "Lund"
|
assert first["location"] == "Lund"
|
||||||
@@ -58,7 +59,7 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
|
|||||||
|
|
||||||
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
||||||
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||||
first, _no_town, remote = platsbanken.rows(decoded, "")
|
first, _no_town, _refused, remote = platsbanken.rows(decoded, "")
|
||||||
assert first["title"] == "Postdoctoral Researcher"
|
assert first["title"] == "Postdoctoral Researcher"
|
||||||
assert first["company"] == "European Spallation Source Eric"
|
assert first["company"] == "European Spallation Source Eric"
|
||||||
assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
|
assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
|
||||||
|
|||||||
@@ -534,7 +534,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert migrated.gazetteer().anywhere is not None
|
assert migrated.gazetteer().anywhere is not None
|
||||||
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
|
row = migrated._row("SELECT remote_host, remote_model, remote_where_model FROM roles "
|
||||||
"WHERE uid = ?", (board,))
|
"WHERE uid = ?", (board,))
|
||||||
@@ -544,7 +544,7 @@ def test_migration_27_keeps_an_older_readers_remote_as_the_hosts_word(tmp_path:
|
|||||||
assert area(migrated, search.uid) == ("", "")
|
assert area(migrated, search.uid) == ("", "")
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 26")
|
conn.execute("PRAGMA user_version = 26")
|
||||||
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32, 33]
|
assert Store(path).migrations_applied == [27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
|
def test_a_role_waits_for_extract_until_it_was_asked_where(store: Store) -> None:
|
||||||
|
|||||||
+107
-2
@@ -1,7 +1,7 @@
|
|||||||
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
||||||
subscription resolves to, series and their union, whole and paged listings,
|
subscription resolves to, series and their union, whole and paged listings,
|
||||||
completeness from the declaration, the sweep across pulls, the 10% guard,
|
completeness from the declaration, the rows kept and dropped, the sweep
|
||||||
the facets kept in state, the detail reads and the `raw` an item carries. No
|
across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||||
network: `get` is a dict of URL -> body or exception."""
|
network: `get` is a dict of URL -> body or exception."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
|||||||
assert filtered.ids == ["1"] and filtered.filtered
|
assert filtered.ids == ["1"] and filtered.filtered
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------- the rows kept
|
||||||
|
|
||||||
|
def _handed(i: str) -> dict[str, Any]:
|
||||||
|
"""A row as a reader hands it over: whole, or, for an id of `-id`,
|
||||||
|
`-title`, `-link` or `-url`, without that value or with a link that is
|
||||||
|
no web URL (`javascript:`, or one httpx cannot read)."""
|
||||||
|
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
|
||||||
|
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
|
||||||
|
return _row(i, **broken.get(i.split(":")[0], {}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
|
||||||
|
assert _runner.valid(_handed("1"))
|
||||||
|
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
|
||||||
|
assert not _runner.valid({"source_id": "1", "title": "Role"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
|
||||||
|
caplog: pytest.LogCaptureFixture) -> None:
|
||||||
|
module = _whole()
|
||||||
|
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||||
|
web = Web({"https://jobs.fake.example/acme/list":
|
||||||
|
b'["1", "-id", "2", "-title", "-link", "-url"]'})
|
||||||
|
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||||
|
listing = _runner.read_listing(module, "acme", web)
|
||||||
|
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
|
||||||
|
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||||
|
assert (listing.handed, listing.dropped) == (6, 4)
|
||||||
|
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
|
||||||
|
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
|
||||||
|
caplog: pytest.LogCaptureFixture) -> None:
|
||||||
|
"""Completeness counts the result ids `page` returns, not the rows kept,
|
||||||
|
so the dropped rows still count as read; the drop is logged once."""
|
||||||
|
pages = [["1", "-title:a"], ["2", "-link:b"]]
|
||||||
|
module = _paged(pages, total=4)
|
||||||
|
module.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||||
|
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||||
|
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||||
|
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||||
|
assert (listing.handed, listing.dropped) == (4, 2)
|
||||||
|
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_stamp_keeps_the_rows_dropped() -> None:
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
|
||||||
|
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
|
||||||
|
assert state["ats:dropped:t"] == "2"
|
||||||
|
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
|
||||||
|
assert state["ats:dropped:t"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
|
||||||
|
"""A whole board whose feed hands over `handed` rows, `dropped` of them
|
||||||
|
without a title."""
|
||||||
|
module = _whole()
|
||||||
|
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||||
|
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
|
||||||
|
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
|
||||||
|
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
|
||||||
|
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
|
||||||
|
handed: int, dropped: int, fails: bool) -> None:
|
||||||
|
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
|
||||||
|
a page whose shape changed, which must close nothing."""
|
||||||
|
module, web = _dropping(handed, dropped)
|
||||||
|
if not fails:
|
||||||
|
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
|
||||||
|
return
|
||||||
|
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
|
||||||
|
_runner.read({}, "ats", "t", module, "acme", web)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
|
||||||
|
module, web = _dropping(20, 5)
|
||||||
|
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
|
||||||
|
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
|
||||||
|
listing = _runner.read(state, "ats", "t", module, "acme", web,
|
||||||
|
limits=_runner.Limits(page_delay=0, stale_days=21))
|
||||||
|
assert (len(listing.ids), listing.dropped) == (15, 5)
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------ through ATSSource
|
# ------------------------------------------------------ through ATSSource
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
|
|||||||
assert state["ats:filtered:fake:acme"] == "1"
|
assert state["ats:filtered:fake:acme"] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
|
||||||
|
board.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||||
|
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||||
|
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
|
||||||
|
board.READS = board.READS.model_copy(update={"complete": "end"})
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
assert len(list(_source(state).fetch())) == 19
|
||||||
|
assert state["ats:dropped:fake:acme"] == "1"
|
||||||
|
|
||||||
|
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||||
|
json.dumps([str(n) for n in range(15)]
|
||||||
|
+ [f"-id:{n}" for n in range(5)]).encode()}
|
||||||
|
state = {}
|
||||||
|
assert list(_source(state).fetch()) == []
|
||||||
|
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
|
||||||
|
assert "ats:ok:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
# -------------------------------------------------------------------- raw
|
# -------------------------------------------------------------------- raw
|
||||||
|
|
||||||
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
||||||
|
|||||||
@@ -417,6 +417,30 @@ def test_a_region_that_holds_the_agents_places_is_no_evidence_against(
|
|||||||
assert oresund not in {r.uid for r in store.roles(places=["country:DK"])}
|
assert oresund not in {r.uid for r in store.roles(places=["country:DK"])}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_role_whose_text_says_remote_passes_a_gate_on_remote_until_extract_reads_it(
|
||||||
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||||
|
"""Berlin is evidence against Denmark, but a text that says remote while
|
||||||
|
no field does makes the role possibly remote (`remote_text`, prep): it
|
||||||
|
meets the remote member. Once extract says onsite, it fails again."""
|
||||||
|
said = role(store, "1", place="Berlin, Germany", remote="")
|
||||||
|
silent = role(store, "2", place="Berlin, Germany", remote="")
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET remote_text = 1 WHERE uid = ?", (said,))
|
||||||
|
agent = store.save_agent("DK or remote", "", [
|
||||||
|
Gate(field="country", values=["DK"], mode="any"),
|
||||||
|
Gate(field="remote", values=["remote"], mode="any")])
|
||||||
|
|
||||||
|
run(config, store, score.run_score)
|
||||||
|
scores = store.scores(agent.id)
|
||||||
|
assert scores[said][1] == ""
|
||||||
|
assert scores[silent] == (0.0, "none of country DK, remote remote")
|
||||||
|
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET remote = 'onsite' WHERE uid = ?", (said,))
|
||||||
|
run(config, store, score.run_score)
|
||||||
|
assert store.scores(agent.id)[said] == (0.0, "none of country DK, remote remote")
|
||||||
|
|
||||||
|
|
||||||
def test_a_wider_place_beside_a_place_elsewhere_is_no_match(
|
def test_a_wider_place_beside_a_place_elsewhere_is_no_match(
|
||||||
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
store: Store, config: Config, fake: FakeEmbedder) -> None:
|
||||||
"""Europe alone is no evidence against Denmark; `London; Europe` is in
|
"""Europe alone is no evidence against Denmark; `London; Europe` is in
|
||||||
|
|||||||
@@ -383,6 +383,23 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch)
|
|||||||
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count(
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""The search driver reads through the same listing code as a board: the
|
||||||
|
runner drops the row the provider handed over without a title, and the
|
||||||
|
read is still complete, by the distinct result ids."""
|
||||||
|
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
||||||
|
module = readers.PROVIDERS["fake"]
|
||||||
|
handed = module.rows
|
||||||
|
module.rows = lambda decoded, slug: [
|
||||||
|
row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)]
|
||||||
|
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
|
||||||
|
"params": {"q": "x"}}]})
|
||||||
|
assert [i.source_id for i in source.fetch()] == ["fake:h1"]
|
||||||
|
assert source.state["search:complete:fake:cph"] == "1"
|
||||||
|
assert source.state["search:dropped:fake:cph"] == "1"
|
||||||
|
|
||||||
|
|
||||||
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
||||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||||
|
|||||||
+14
-14
@@ -71,7 +71,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
|||||||
"application_events", "agents", "scores", "state"} <= tables
|
"application_events", "agents", "scores", "state"} <= tables
|
||||||
assert "idx_roles_cluster" in indexes
|
assert "idx_roles_cluster" in indexes
|
||||||
assert store.migrations_applied == []
|
assert store.migrations_applied == []
|
||||||
assert LATEST == 33
|
assert LATEST == 35
|
||||||
# A second open finds it current and applies nothing.
|
# A second open finds it current and applies nothing.
|
||||||
assert Store(path).migrations_applied == []
|
assert Store(path).migrations_applied == []
|
||||||
|
|
||||||
@@ -169,8 +169,8 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
|
|||||||
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
||||||
for table in ("vectors", "role_tags", "agent_companies", "role_votes"):
|
for table in ("vectors", "role_tags", "agent_companies", "role_votes"):
|
||||||
conn.execute(f"DROP TABLE {table}")
|
conn.execute(f"DROP TABLE {table}")
|
||||||
for column in ("musts", "broken"): # v29
|
# The must-have columns v28 and v29 added and v34 drops are not in a
|
||||||
conn.execute(f"ALTER TABLE role_judgements DROP COLUMN {column}")
|
# fresh file either.
|
||||||
conn.execute("ALTER TABLE agents DROP COLUMN like_text") # v30
|
conn.execute("ALTER TABLE agents DROP COLUMN like_text") # v30
|
||||||
conn.execute("PRAGMA user_version = 1")
|
conn.execute("PRAGMA user_version = 1")
|
||||||
|
|
||||||
@@ -191,7 +191,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
acme = migrated.company("acme")
|
acme = migrated.company("acme")
|
||||||
assert acme is not None
|
assert acme is not None
|
||||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||||
@@ -396,7 +396,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||||
Prompt(name="Backend", text="python backend")]
|
Prompt(name="Backend", text="python backend")]
|
||||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||||
@@ -420,7 +420,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
acme = migrated.company("acme")
|
acme = migrated.company("acme")
|
||||||
assert acme is not None
|
assert acme is not None
|
||||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||||
@@ -449,7 +449,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||||
"SELECT uid, place_fold, seniority FROM roles")}
|
"SELECT uid, place_fold, seniority FROM roles")}
|
||||||
@@ -691,7 +691,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||||
assert migrated.role_places(stated.uid) == []
|
assert migrated.role_places(stated.uid) == []
|
||||||
@@ -733,7 +733,7 @@ def test_v25_clears_the_model_levels_the_ladder_splits_and_sets_the_level(
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
found = {sid: rest for sid, *rest in conn.execute(
|
found = {sid: rest for sid, *rest in conn.execute(
|
||||||
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
|
"SELECT source_id, seniority_model, seniority_stated, seniority_title, seniority "
|
||||||
@@ -751,12 +751,12 @@ def test_v26_adds_posting_url_empty_and_runs_twice_without_harm(tmp_path: Path)
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33]
|
assert migrated.migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
role = migrated.role(posting("a").uid)
|
role = migrated.role(posting("a").uid)
|
||||||
assert role is not None and role.posting_url == ""
|
assert role is not None and role.posting_url == ""
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 25") # the column is there already
|
conn.execute("PRAGMA user_version = 25") # the column is there already
|
||||||
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33]
|
assert Store(path).migrations_applied == [26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
|
||||||
|
|
||||||
|
|
||||||
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
|
def test_a_pull_keeps_the_posting_url(store: Store) -> None:
|
||||||
@@ -1855,7 +1855,7 @@ def test_v32_resets_the_limits_drops_prefer_gates_and_renames_enrich(tmp_path: P
|
|||||||
|
|
||||||
migrated = Store(path)
|
migrated = Store(path)
|
||||||
|
|
||||||
assert migrated.migrations_applied == [32, 33]
|
assert migrated.migrations_applied == [32, 33, 34, 35]
|
||||||
by_name = {a.name: a for a in migrated.agents()}
|
by_name = {a.name: a for a in migrated.agents()}
|
||||||
assert by_name["Wolf"].limits == Limits() == by_name["Bare"].limits
|
assert by_name["Wolf"].limits == Limits() == by_name["Bare"].limits
|
||||||
assert [(g.field, g.mode) for g in by_name["Wolf"].gates] == [
|
assert [(g.field, g.mode) for g in by_name["Wolf"].gates] == [
|
||||||
@@ -1880,14 +1880,14 @@ def test_v32_runs_twice_and_keeps_limits_set_since(tmp_path: Path) -> None:
|
|||||||
store = Store(path)
|
store = Store(path)
|
||||||
agent = store.save_agent("Wolf", limits=Limits(cut_fit_top=900))
|
agent = store.save_agent("Wolf", limits=Limits(cut_fit_top=900))
|
||||||
_v31(path)
|
_v31(path)
|
||||||
assert Store(path).migrations_applied == [32, 33]
|
assert Store(path).migrations_applied == [32, 33, 34, 35]
|
||||||
assert Store(path).agent(agent.id).limits == Limits() # type: ignore[union-attr]
|
assert Store(path).agent(agent.id).limits == Limits() # type: ignore[union-attr]
|
||||||
narrow = Limits(cut_fit_top=900, summarise_top=10)
|
narrow = Limits(cut_fit_top=900, summarise_top=10)
|
||||||
store.save_agent("Wolf", agent_id=agent.id, limits=narrow)
|
store.save_agent("Wolf", agent_id=agent.id, limits=narrow)
|
||||||
with sqlite3.connect(path) as conn:
|
with sqlite3.connect(path) as conn:
|
||||||
conn.execute("PRAGMA user_version = 31") # the columns are there already
|
conn.execute("PRAGMA user_version = 31") # the columns are there already
|
||||||
again = Store(path)
|
again = Store(path)
|
||||||
assert again.migrations_applied == [32, 33]
|
assert again.migrations_applied == [32, 33, 34, 35]
|
||||||
assert again.agent(agent.id).limits == narrow # type: ignore[union-attr]
|
assert again.agent(agent.id).limits == narrow # type: ignore[union-attr]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+29
-1
@@ -998,7 +998,13 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
|
|||||||
template = page[page.index('<template id="agent-form-template">'):page.index("</template>")]
|
template = page[page.index('<template id="agent-form-template">'):page.index("</template>")]
|
||||||
assert 'id="agent-form"' in template and 'id="a-cancel"' in template
|
assert 'id="agent-form"' in template and 'id="a-cancel"' in template
|
||||||
assert 'id="a-name"' in page and 'placeholder="My search"' in page
|
assert 'id="a-name"' in page and 'placeholder="My search"' in page
|
||||||
assert 'id="a-prompts"' in page and "<details" in page and 'id="a-gates"' in page
|
assert 'id="a-prompts"' in page
|
||||||
|
# The gates are the four parts and nothing else: no rows of other rules,
|
||||||
|
# only a note naming a stored rule the parts cannot say.
|
||||||
|
for gone in ("<details", 'id="a-gates"', 'id="a-rules"', 'id="a-add-rule"', "Other rules"):
|
||||||
|
assert gone not in template
|
||||||
|
assert template.index('id="a-where-sum"') < template.index('id="a-kept"') \
|
||||||
|
< template.index("</fieldset>", template.index('id="a-where-sum"'))
|
||||||
for weight in ("a-fit", "a-recency", "a-affinity", "a-taste"):
|
for weight in ("a-fit", "a-recency", "a-affinity", "a-taste"):
|
||||||
assert f'id="{weight}"' in page
|
assert f'id="{weight}"' in page
|
||||||
assert 'id="a-taste" min="0" step="0.05" value="0.2"' in page
|
assert 'id="a-taste" min="0" step="0.05" value="0.2"' in page
|
||||||
@@ -1170,6 +1176,28 @@ def test_agents_are_created_updated_listed_and_deleted(client: TestClient) -> No
|
|||||||
assert client.delete(f"/api/agents/{agent['id']}").status_code == 404
|
assert client.delete(f"/api/agents/{agent['id']}").status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_agents_fetch_gate_and_a_rule_the_form_cannot_say_round_trip(client: TestClient) -> None:
|
||||||
|
"""What the form sends for the four parts, plus a stored gate on a field the
|
||||||
|
form has no part for, is served back as sent, so a load and a save keep it."""
|
||||||
|
europe = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
|
||||||
|
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS",
|
||||||
|
"LI", "NO", "GB", "CH", "EU"]
|
||||||
|
gates = [{"field": "country", "values": europe, "mode": "require"},
|
||||||
|
{"field": "country", "values": ["DK"], "mode": "any"},
|
||||||
|
{"field": "remote_from", "values": ["country:DK", "anywhere"], "mode": "any"},
|
||||||
|
{"field": "place", "values": ["Malmö"], "mode": "any"},
|
||||||
|
{"field": "title", "values": ["account executive"], "mode": "exclude"},
|
||||||
|
{"field": "seniority", "values": ["senior"], "mode": "require"}]
|
||||||
|
body = {"name": "Nordics", "prompts": [{"name": "Data", "text": "Head of Data"}],
|
||||||
|
"gates": gates, "fetch_terms": ["data"]}
|
||||||
|
agent = client.post("/api/agents", json=body).json()
|
||||||
|
assert agent["gates"] == gates and agent["fetch_terms"] == ["data"]
|
||||||
|
served = next(a for a in client.get("/api/agents").json() if a["id"] == agent["id"])
|
||||||
|
assert served["gates"] == gates
|
||||||
|
again = client.post("/api/agents", json={**body, "id": agent["id"], "gates": served["gates"]})
|
||||||
|
assert again.status_code == 200 and again.json()["gates"] == gates
|
||||||
|
|
||||||
|
|
||||||
def test_agents_take_and_serve_prompts_and_still_take_an_intent(client: TestClient) -> None:
|
def test_agents_take_and_serve_prompts_and_still_take_an_intent(client: TestClient) -> None:
|
||||||
prompts = [{"name": "AI and data leadership", "text": "VP, Head or Director of AI"},
|
prompts = [{"name": "AI and data leadership", "text": "VP, Head or Director of AI"},
|
||||||
{"name": "Forward deployed", "text": ""}]
|
{"name": "Forward deployed", "text": ""}]
|
||||||
|
|||||||
Reference in new issue
Block a user