Merge branch 'wp-f2-terms'
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
1538c776d6
28 files changed
+839
-113
No files matched your search
@@ -24,6 +24,11 @@ filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
||||
under a tenth of what the same read listed last time is recorded by the
|
||||
runner as a failure, not as a list to close against.
|
||||
|
||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||
the roles matching a word, which says nothing of the roles it did not list,
|
||||
so it closes nothing; only an unfiltered or a place-filtered complete read
|
||||
closes.
|
||||
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
||||
@@ -274,6 +279,8 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
continue
|
||||
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
if store.state.get(f"{source}:filtered:{sub.target}", ""):
|
||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
||||
|
||||
+59
-10
@@ -27,6 +27,24 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
|
||||
A reach of more countries than a `one`-filter host should be asked one by
|
||||
one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`,
|
||||
which knows the reader (`MAX_SERIES`), so such a board is read whole.
|
||||
|
||||
**Fetch terms** (`Agent.fetch_terms`) narrow a wide read by words, so a big
|
||||
board sends the roles that match one of them rather than every role:
|
||||
|
||||
- The terms of a pull are the union over the enabled agents. An enabled
|
||||
agent with none wants every role, so then there are none.
|
||||
- They go only with the wide reach: a company tagged All roles (`wide` in
|
||||
the reach), and a board whose reach `ask_for` drops as too many countries
|
||||
for a `one` host. The narrow reach (DK, SE) already leaves at most about
|
||||
150 roles on any big board, and small, unknown and overridden boards get
|
||||
no reach at all, so none of those take terms.
|
||||
- A board's params switch them off with `{"terms": false}`.
|
||||
- Only a reader that declares `query="one"` takes them, one series a term;
|
||||
`ask_for` has the rule for terms times the reach's series.
|
||||
- A read with terms is never the whole board: it sets no `ats:size`, and
|
||||
the pull closes nothing against it (`app/jobs/pull.py`).
|
||||
- The search providers take none. Every saved search has a query of its
|
||||
own, which terms would replace; a search is a query already.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -162,16 +180,33 @@ def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
|
||||
return Reach(remote=True) if remote and kept is None else EVERYWHERE
|
||||
|
||||
|
||||
def fetch_terms(agents: Iterable[Agent]) -> tuple[str, ...]:
|
||||
"""The fetch terms of a pull: the union of the agents' terms, one of each
|
||||
regardless of case, sorted; none when an agent has none, or with no
|
||||
agent."""
|
||||
found: dict[str, str] = {}
|
||||
listed = False
|
||||
for agent in agents:
|
||||
listed = True
|
||||
terms = [t.strip() for t in agent.fetch_terms if t.strip()]
|
||||
if not terms:
|
||||
return ()
|
||||
for term in terms:
|
||||
found.setdefault(term.casefold(), term)
|
||||
return tuple(found[k] for k in sorted(found)) if listed else ()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Plan:
|
||||
"""The reach of one pull: `base` for every board, `wide` for the boards
|
||||
of the companies in `tagged` (tagged All roles), and each board's size
|
||||
by target."""
|
||||
of the companies in `tagged` (tagged All roles), each board's size by
|
||||
target, and the fetch `terms` of the wide reads."""
|
||||
|
||||
base: Reach
|
||||
wide: Reach = EVERYWHERE
|
||||
sizes: dict[str, int] = field(default_factory=dict)
|
||||
tagged: frozenset[str] = frozenset()
|
||||
terms: tuple[str, ...] = ()
|
||||
|
||||
def why(self, sub: Subscription) -> tuple[Reach, str]:
|
||||
"""The reach `sub` is read with, and why: `override`, `small`,
|
||||
@@ -189,8 +224,17 @@ class Plan:
|
||||
return self.base, "agents"
|
||||
|
||||
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
|
||||
"""The `reach` a board's batch hands `ask_for`; None for none."""
|
||||
return self.why(sub)[0].params()
|
||||
"""The `reach` a board's batch hands `ask_for`; None for none. With
|
||||
fetch terms, a tagged board's carries them and `wide`, and a board on
|
||||
the agents' reach carries them for `ask_for` to use if it drops the
|
||||
countries as too many (see the module docstring)."""
|
||||
found, why = self.why(sub)
|
||||
out = found.params()
|
||||
if self.terms and why in ("tag", "agents") and (sub.params or {}).get("terms") is not False:
|
||||
out = (out or {}) | {"terms": list(self.terms)}
|
||||
if why == "tag":
|
||||
out["wide"] = True
|
||||
return out
|
||||
|
||||
|
||||
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
||||
@@ -204,7 +248,8 @@ def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
||||
kept = state.get(f"{SIZE}{sub.target}", "")
|
||||
if kept.isdigit():
|
||||
out[sub.target] = int(kept)
|
||||
elif state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}"):
|
||||
elif (state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}")
|
||||
and not state.get(f"ats:narrowed:{sub.target}")):
|
||||
out[sub.target] = sum(counts.get(sub.id, {}).values())
|
||||
return out
|
||||
|
||||
@@ -225,7 +270,7 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
|
||||
tagged = frozenset(c.key for c in store.companies() if c.all_roles)
|
||||
# A tag never reads less than the board would untagged: the base's
|
||||
# remote, which a `require` of Europe does not name, is kept.
|
||||
return Plan(base, wide | base, sizes(store, subs), tagged)
|
||||
return Plan(base, wide | base, sizes(store, subs), tagged, fetch_terms(agents))
|
||||
|
||||
|
||||
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
|
||||
@@ -248,19 +293,23 @@ _SAID = {"override": "set here", "tag": "company tag"}
|
||||
def line(found: Plan, sub: Subscription) -> str:
|
||||
"""What a pull asks `sub`'s board for, once its reader has narrowed the
|
||||
reach, in a few words: "DK, SE, remote · agents", "33 countries, remote ·
|
||||
company tag", "all roles · under 300 roles", "all roles · Greenhouse
|
||||
cannot filter"."""
|
||||
company tag", "57 countries · company tag · 3 terms", "all places ·
|
||||
company tag · 3 terms", "all roles · under 300 roles", "all roles ·
|
||||
Greenhouse cannot filter"."""
|
||||
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||
if module is None:
|
||||
return ""
|
||||
given, why = found.why(sub)
|
||||
ask = _runner.ask_for(module, sub.params, given.params())
|
||||
ask = _runner.ask_for(module, sub.params, found.for_subscription(sub))
|
||||
terms = f" · {len(ask.terms)} term{'s' if len(ask.terms) != 1 else ''}" if ask.terms else ""
|
||||
if _runner.filtered(ask):
|
||||
words = sorted(ask.countries) if len(ask.countries) <= 4 \
|
||||
else [f"{len(ask.countries)} countries"]
|
||||
if ask.remote:
|
||||
words.append("remote")
|
||||
return f"{', '.join(words)} · {_SAID.get(why, why)}"
|
||||
return f"{', '.join(words)} · {_SAID.get(why, why)}{terms}"
|
||||
if terms:
|
||||
return f"all places · {_SAID.get(why, why)}{terms}"
|
||||
if why in ("agents", "tag") and given.everywhere:
|
||||
return f"all roles · {'company tag' if why == 'tag' else _WHOLE[why]}"
|
||||
if why in ("agents", "tag") and not module.READS.filters.declared():
|
||||
|
||||
+25
-1
@@ -1141,10 +1141,33 @@ def check_prompts(prompts: list[Prompt]) -> list[Prompt]:
|
||||
Prompts = Annotated[list[Prompt], AfterValidator(check_prompts)]
|
||||
|
||||
|
||||
def fetch_terms(value: Any) -> list[str]:
|
||||
"""Fetch terms as stored and sent: a JSON list or a list, each stripped,
|
||||
empty ones dropped, one of each regardless of case, in the order given."""
|
||||
items = _json(value, "[]")
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for item in items if isinstance(items, list) else []:
|
||||
term = " ".join(str(item).split())
|
||||
if term and term.casefold() not in seen:
|
||||
seen.add(term.casefold())
|
||||
out.append(term)
|
||||
return out
|
||||
|
||||
|
||||
#: An agent's fetch terms, normalised by `fetch_terms`.
|
||||
FetchTerms = Annotated[list[str], BeforeValidator(fetch_terms)]
|
||||
|
||||
|
||||
class Agent(Model):
|
||||
"""A row of `agents`: a named search with its own prompts, gates, weights
|
||||
and limits. The prompts are rows of `agent_prompts`, read alongside; their
|
||||
cached vectors are columns this does not declare."""
|
||||
cached vectors are columns this does not declare.
|
||||
|
||||
`fetch_terms` are words a role must match for a big board read with the
|
||||
wide reach to send it (`app/jobs/reach.py`): the pull's terms are the
|
||||
union over the enabled agents, and an agent with none means no board is
|
||||
narrowed by words. They decide what is fetched, not what ranks."""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
@@ -1152,6 +1175,7 @@ class Agent(Model):
|
||||
gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
||||
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
|
||||
limits: Annotated[Limits, BeforeValidator(lambda v: _json(v, "{}"))] = Limits()
|
||||
fetch_terms: FetchTerms = []
|
||||
enabled: bool = True
|
||||
created_at: str
|
||||
|
||||
|
||||
+27
-10
@@ -14,6 +14,10 @@
|
||||
* form's ids stay unique. Votes, liked companies and scores are the agent's,
|
||||
* so a delete takes them with it, and the confirm says so.
|
||||
*
|
||||
* The fetch terms are chips of free text, as the towns are: words that
|
||||
* narrow what big boards read with the wide reach send (app/jobs/reach.py).
|
||||
* The card says them in a line of their own, or that boards send every role.
|
||||
*
|
||||
* Every request goes through `api()` from app.js, so a failure is one toast
|
||||
* carrying the server's own sentence.
|
||||
*/
|
||||
@@ -283,6 +287,7 @@
|
||||
if (rules) div.appendChild(el("p", "agent-rules", rules));
|
||||
div.appendChild(el("p", "agent-weights", weightsLine(row)));
|
||||
div.appendChild(limitsLine(row));
|
||||
div.appendChild(termsLine(row));
|
||||
|
||||
const actions = el("div", "agent-card-actions");
|
||||
const edit = el("button", "secondary", "Edit");
|
||||
@@ -374,15 +379,15 @@
|
||||
return { node: chips.node, values: () => [...chips.list] };
|
||||
}
|
||||
|
||||
// Towns as chips of free text, added on Enter or a comma, and one chip for
|
||||
// the spellings of a town. What is typed but not added yet counts, so Save
|
||||
// does not drop it.
|
||||
function townInput(values) {
|
||||
// Words as chips of free text, added on Enter or a comma, and one chip for
|
||||
// the spellings that fold alike. What is typed but not added yet counts, so
|
||||
// Save does not drop it. Towns and fetch terms both.
|
||||
function wordInput(values, prompt) {
|
||||
const input = el("input");
|
||||
input.type = "text";
|
||||
input.spellcheck = false;
|
||||
input.placeholder = "Add a town";
|
||||
input.setAttribute("aria-label", "Add a town");
|
||||
input.placeholder = prompt;
|
||||
input.setAttribute("aria-label", prompt);
|
||||
const split = (text) => text.split(",").map((v) => v.trim()).filter(Boolean);
|
||||
const chips = chipList(values, input, { group: fold });
|
||||
const take = () => {
|
||||
@@ -402,6 +407,14 @@
|
||||
};
|
||||
}
|
||||
|
||||
// The fetch terms on a card: the words, or that big boards send every role.
|
||||
function termsLine(row) {
|
||||
const terms = row.fetch_terms || [];
|
||||
return el("p", "agent-terms", terms.length
|
||||
? `Big boards read across Europe send only roles matching ${orList(terms.map((t) => `“${t}”`))}.`
|
||||
: "Big boards send every role: no fetch terms.");
|
||||
}
|
||||
|
||||
// A prompt as a row: its name and Remove on a line, its text under them.
|
||||
function promptRow(prompt, remove) {
|
||||
const row = el("div", "prompt-row");
|
||||
@@ -547,7 +560,9 @@
|
||||
|
||||
const { where, rest } = distribute(row ? row.gates : []);
|
||||
const countries = countryInput(where.countries ? where.countries.values : []);
|
||||
const places = townInput(where.places ? where.places.values : []);
|
||||
const places = wordInput(where.places ? where.places.values : [], "Add a town");
|
||||
const terms = wordInput(row ? row.fetch_terms || [] : [], "Add a word");
|
||||
$("#a-terms", f).replaceWith(terms.node);
|
||||
$("#a-countries", f).replaceWith(countries.node);
|
||||
$("#a-towns", f).replaceWith(places.node);
|
||||
const remote = $("#a-remote", f);
|
||||
@@ -623,7 +638,7 @@
|
||||
$("#a-delete", f).hidden = !row;
|
||||
f.addEventListener("input", () => { f.dataset.dirty = "true"; view(); });
|
||||
f.addEventListener("change", view);
|
||||
f.addEventListener("submit", (event) => save(event, row, () => prompts.map((p) => p.read()), gates));
|
||||
f.addEventListener("submit", (event) => save(event, row, () => prompts.map((p) => p.read()), gates, terms.values));
|
||||
$("#a-cancel", f).addEventListener("click", () => { close(); render(); });
|
||||
$("#a-delete", f).addEventListener("click", () => row && removeAgent(row));
|
||||
div.appendChild(f);
|
||||
@@ -669,9 +684,10 @@
|
||||
: "No search agents yet. New agent describes one.";
|
||||
}
|
||||
|
||||
// `readPrompts` and `readGates` are the form's rows as the API takes them.
|
||||
// `readPrompts`, `readGates` and `readTerms` are the form's rows and chips
|
||||
// as the API takes them.
|
||||
// A prompt with neither name nor text is an empty row, left out.
|
||||
async function save(event, row, readPrompts, readGates) {
|
||||
async function save(event, row, readPrompts, readGates, readTerms) {
|
||||
event.preventDefault();
|
||||
const f = event.target;
|
||||
const prompts = readPrompts().filter((p) => p.name || p.text.trim());
|
||||
@@ -696,6 +712,7 @@
|
||||
summarise_top: Number($("#a-summarise-top", f).value),
|
||||
summarise_min: Number($("#a-summarise-min", f).value),
|
||||
},
|
||||
fetch_terms: readTerms(),
|
||||
// Saving an agent means it is wanted, so a new one starts on; the
|
||||
// card's switch is where an existing one is switched.
|
||||
enabled: row ? row.enabled : true,
|
||||
|
||||
+3
-2
@@ -1149,7 +1149,7 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
.agent-prompt-name { margin: .7rem 0 0; font-size: .8rem; font-weight: 600; color: var(--dim); }
|
||||
.agent-prompt-name + .agent-intent { margin-top: .2rem; }
|
||||
.agent-rules { margin: 0 0 .3rem; font-size: .9rem; color: var(--dim); overflow-wrap: anywhere; }
|
||||
.agent-weights, .agent-limits { margin: 0; font-size: .8rem; color: var(--faint); font-variant-numeric: tabular-nums; }
|
||||
.agent-weights, .agent-limits, .agent-terms { margin: 0; font-size: .8rem; color: var(--faint); font-variant-numeric: tabular-nums; }
|
||||
.agent-card-actions { display: flex; align-items: center; flex-wrap: wrap; gap: .5rem 1rem; margin-top: .8rem; }
|
||||
.agent-card-actions a { font-size: .9rem; font-weight: 600; text-decoration: none; }
|
||||
@media (hover: hover) {
|
||||
@@ -1223,7 +1223,8 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
border-radius: .3rem; padding: .35rem .5rem;
|
||||
}
|
||||
/* The fieldsets that hold rows: a column, the Add button under the rows. */
|
||||
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.rules {
|
||||
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.terms,
|
||||
.agent-form fieldset.rules {
|
||||
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
|
||||
border: 1px solid var(--line); border-radius: .3rem;
|
||||
}
|
||||
|
||||
+17
-2
@@ -295,7 +295,9 @@ END;
|
||||
-- A search agent: a named saved search with its own prompts, gates and
|
||||
-- weights. gates is a list of {field, values, mode: require|prefer, weight}.
|
||||
-- limits is how deep into this agent's ranking extract and summarise read,
|
||||
-- as JSON: a key it lacks means the `Limits` default.
|
||||
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
|
||||
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
|
||||
-- means no narrowing.
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
@@ -303,7 +305,8 @@ CREATE TABLE IF NOT EXISTS agents (
|
||||
weights TEXT NOT NULL DEFAULT '{}',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL,
|
||||
limits TEXT NOT NULL DEFAULT '{}'
|
||||
limits TEXT NOT NULL DEFAULT '{}',
|
||||
fetch_terms TEXT NOT NULL DEFAULT '[]'
|
||||
);
|
||||
|
||||
-- What an agent looks for, as several named texts in order; a role's fit is
|
||||
@@ -598,6 +601,9 @@ ALTER TABLE companies ADD COLUMN hand_reason TEXT NOT NULL DEFAULT '';
|
||||
# A Python step: the host words are read out of raw's JSON.
|
||||
(22, "roles.*_host, roles.employment, roles.prep_due and unmapped: the host's own "
|
||||
"words are stored and app.prep maps them", lambda conn: _add_host_words(conn)),
|
||||
# A Python step: it adds the column only where it is missing.
|
||||
(23, "agents.fetch_terms: words that narrow what big boards read with the wide reach "
|
||||
"send", lambda conn: _add_fetch_terms(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -830,6 +836,15 @@ def _add_company_aliases(conn: sqlite3.Connection) -> None:
|
||||
log.info("migration 20: %d companies folded, %d folds shared", len(rows), len(clashes))
|
||||
|
||||
|
||||
def _add_fetch_terms(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 23's step: `agents.fetch_terms`, empty for every agent, so
|
||||
no board is narrowed by words until an agent is given some. Runs twice
|
||||
without harm."""
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(agents)")}
|
||||
if "fetch_terms" not in columns:
|
||||
conn.execute("ALTER TABLE agents ADD COLUMN fetch_terms TEXT NOT NULL DEFAULT '[]'")
|
||||
|
||||
|
||||
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 21's step. The column; the tag for every company that is a
|
||||
favourite of any agent, since favourite no longer widens what a pull
|
||||
|
||||
+13
-6
@@ -26,7 +26,7 @@ from app.models import (
|
||||
ApplicationEvent, Candidate, Company, Gate, Limits, Prompt, Role, RoleCopy, RolePlace,
|
||||
RoleRow, ScoredRole,
|
||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||
host_word, posted_at, posted_host, remote_of, title_key,
|
||||
fetch_terms as normal_terms, host_word, posted_at, posted_host, remote_of, title_key,
|
||||
)
|
||||
from app.storage import gazetteer, schema
|
||||
|
||||
@@ -1712,10 +1712,13 @@ class Store:
|
||||
def save_agent(self, name: str, intent: str = "", gates: Sequence[Gate] = (),
|
||||
weights: Weights | None = None, enabled: bool = True,
|
||||
agent_id: int | None = None, limits: Limits | None = None,
|
||||
prompts: Sequence[Prompt] | None = None) -> Agent:
|
||||
prompts: Sequence[Prompt] | None = None,
|
||||
fetch_terms: Sequence[str] | None = None) -> Agent:
|
||||
"""Create an agent, or with `agent_id` replace that one's fields and
|
||||
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
|
||||
named after the agent. A prompt whose text is unchanged keeps its
|
||||
named after the agent. Without `fetch_terms` an update keeps the
|
||||
stored ones and a new agent has none, so a caller that does not know
|
||||
them (the CLI's prompt edit) loses none. A prompt whose text is unchanged keeps its
|
||||
cached vector, whatever its name or place, so an edit re-embeds only
|
||||
the prompts it changed. Raises ValueError on an empty or taken name,
|
||||
or two prompts of one name, KeyError on an unknown id."""
|
||||
@@ -1725,18 +1728,22 @@ class Store:
|
||||
if prompts is None:
|
||||
prompts = [Prompt(name=name, text=intent)] if intent.strip() else []
|
||||
prompts = check_prompts(list(prompts))
|
||||
terms = None if fetch_terms is None else json.dumps(
|
||||
normal_terms(list(fetch_terms)), ensure_ascii=False)
|
||||
fields = (name, json.dumps([g.model_dump() for g in gates]),
|
||||
(weights or Weights()).model_dump_json(),
|
||||
(limits or Limits()).model_dump_json(), int(enabled))
|
||||
(limits or Limits()).model_dump_json(), int(enabled), terms)
|
||||
try:
|
||||
with self.transaction() as conn:
|
||||
if agent_id is None:
|
||||
agent_id = conn.execute(
|
||||
"""INSERT INTO agents (name, gates, weights, limits, enabled,
|
||||
created_at) VALUES (?,?,?,?,?,?)""",
|
||||
fetch_terms, created_at)
|
||||
VALUES (?,?,?,?,?,COALESCE(?, '[]'),?)""",
|
||||
(*fields, self.now())).lastrowid
|
||||
elif not conn.execute(
|
||||
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?
|
||||
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
|
||||
fetch_terms=COALESCE(?, fetch_terms)
|
||||
WHERE id=?""",
|
||||
(*fields, agent_id)).rowcount:
|
||||
raise KeyError(agent_id)
|
||||
|
||||
@@ -49,6 +49,14 @@
|
||||
<span class="hint">This one ranks rather than filters: a hybrid role gains a little.</span>
|
||||
<p id="a-where-sum" class="where-sum" role="status"></p>
|
||||
</fieldset>
|
||||
<fieldset class="terms">
|
||||
<legend>What big boards send</legend>
|
||||
<div class="where-line">
|
||||
<span>Only roles matching</span>
|
||||
<div id="a-terms"></div>
|
||||
</div>
|
||||
<span class="hint">Fetch terms narrow what a big board sends when it is read across Europe: a company tagged All roles, or a host that cannot ask for each country. The board then lists only roles whose text has one of these words, one request per word. Empty means no narrowing, and while any agent that is on has none, no board is narrowed. Boards read for Denmark or Sweden, and boards under 300 roles, are never narrowed. The words decide what is fetched, not how it ranks.</span>
|
||||
</fieldset>
|
||||
<fieldset class="weights">
|
||||
<legend>How much each counts</legend>
|
||||
<label>Fit to the text <input type="number" id="a-fit" min="0" step="0.05" value="0.6"></label>
|
||||
|
||||
+6
-3
@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app import jobs, places
|
||||
from app.models import (
|
||||
AFFINITY, LEVELS, Agent, Application, Company, Gate, Limits, Prompts, Role, RoleRow,
|
||||
AFFINITY, LEVELS, Agent, Application, Company, FetchTerms, Gate, Limits, Prompts, Role, RoleRow,
|
||||
HandReason, ScoredRole, Status, Subscription, Weights, company_key,
|
||||
)
|
||||
from app.storage import Store
|
||||
@@ -1064,7 +1064,9 @@ def list_agents(store: StoreDep) -> list[Agent]:
|
||||
class AgentBody(BaseModel):
|
||||
"""Create when `id` is missing, replace that agent's fields when given.
|
||||
`intent` is the body from before an agent had prompts: without `prompts`
|
||||
it is stored as one prompt named after the agent."""
|
||||
it is stored as one prompt named after the agent. `fetch_terms` narrow
|
||||
what big boards send (`app/jobs/reach.py`); `[]` narrows nothing, and
|
||||
leaving the key out keeps an existing agent's."""
|
||||
|
||||
id: int | None = None
|
||||
name: str = Field(min_length=1)
|
||||
@@ -1073,6 +1075,7 @@ class AgentBody(BaseModel):
|
||||
gates: list[Gate] = []
|
||||
weights: Weights = Weights()
|
||||
limits: Limits = Limits()
|
||||
fetch_terms: FetchTerms | None = None
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
@@ -1083,7 +1086,7 @@ def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent:
|
||||
try:
|
||||
saved = store.save_agent(body.name, body.intent, body.gates, body.weights,
|
||||
body.enabled, agent_id=body.id, limits=body.limits,
|
||||
prompts=body.prompts)
|
||||
prompts=body.prompts, fetch_terms=body.fetch_terms)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(409, str(exc)) from exc
|
||||
except KeyError:
|
||||
|
||||
@@ -62,6 +62,7 @@ Terms apply only to boards of 300 roles or more that are not tagged All roles. T
|
||||
| R2 Raw rows o to z | Sonnet | The same for Oracle to Workday and the three providers | those modules and tests, their table sections | K merged |
|
||||
| P Prep stage | Opus | `app/prep/` stage, host-word columns and the `unmapped` table (migration 22), `upsert_role` storing raw values, gazetteer and alias resolution moved into prep, the cascade, `app prep` CLI, the Sources page list of unmapped words | `app/prep/*` except the table sections, `app/storage/*`, `app/jobs/cascade.py`, `app/jobs/stages.py`, `app/cli.py`, `app/static/sources.js`, their tests | K merged; R1 and R2 merge with it |
|
||||
| F2 Terms and towns | Opus | `fetch_terms` on agents (model, API, agent editor), terms and towns in the reach, runner series for `query` and `place`, reader filters per the survey | `app/jobs/reach.py`, `plugins/readers/_runner.py`, `_contract.py`, the filtering readers, `app/static/agents.js` | S, and R1/R2/P merged |
|
||||
| F2 done 2026-09-30 | Opus | `agents.fetch_terms` (migration 23, API, a chip field in the agent editor); terms in the reach for wide reads only; one series per term on SmartRecruiters `q`, SuccessFactors `q`, Workday `searchText`, Phenom `keywords`, Eightfold `query`, Oracle `keyword`; places dropped first, then terms, over `MAX_SERIES`; a term read (`ats:narrowed`) sets no size and closes nothing; the reach line counts the terms. Towns stay fetched as their country | branch `wp-f2-terms` | |
|
||||
|
||||
The survey (`docs/plan-prep-survey.md`) narrowed F2: the narrow reach already leaves at most 149 roles on any big board, and keywords match the whole text, so terms go only to reads with the wide reach (All roles, or a one-country host over `MAX_SERIES`). `query` is built in this order: SmartRecruiters, SuccessFactors, then Workday, Phenom, Eightfold and Oracle. Category and level filters wait until something maps agent terms to host ids. Towns stay fetched as their country. A term-filtered read never sets `ats:size` and never closes roles as gone.
|
||||
|
||||
|
||||
+10
-7
@@ -69,11 +69,14 @@ Numbers are from the live database after the pull of 2026-09-29. For the "list,
|
||||
| robots.txt | read before the list for the 14 hosts that name a page: Workday, SuccessFactors, Oracle, Phenom, Eightfold, Jibe, Gem, BambooHR, Pinpoint, Deel, Jobylon, Rippling, HiBob, HR Manager; kept for one day per board | A disallow fails the board with "robots.txt disallows it" and counts toward the 21 days. An unreachable file (5xx, 429, timeout) fails the board for this pull and is asked again next pull. A 4xx means no rules, which allows. `Crawl-delay` is not honoured. The eight API hosts (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor) and the search providers read no robots.txt. |
|
||||
| 429 | first page: the board fails; a later page: the read stops, keeps what it read and is incomplete; a per-role read: the per-role reads of that board stop until the next pull | Nothing is closed on a list cut short by a 429. A search that fails is skipped for the pull. |
|
||||
|
||||
How far a board is read: a pull asks each board over 300 roles for the reach of the search agents that are switched on (`app/jobs/reach.py`): the union of their `any` gates on country, town (as its country) and remote, or every role when one agent can take a role from anywhere. A company starred as favourite is asked for its agent's `require` reach, Europe today; a host that asks one country a request reads a favourite whole instead of 40 times. A board's own reach, set on the Companies list, wins. A complete filtered read closes the roles outside the reach, except those with a vote or an application.
|
||||
How far a board is read: a pull asks each board over 300 roles for the reach of the search agents that are switched on (`app/jobs/reach.py`): the union of their `any` gates on country, town (as its country) and remote, or every role when one agent can take a role from anywhere. A company tagged All roles is asked for the agents' wide reach, the union of their `require` reach, Europe today; a host that asks one country a request reads such a board whole instead of 30 times, unless fetch terms narrow it. A board's own reach, set on the Companies list, wins. A complete filtered read closes the roles outside the reach, except those with a vote or an application.
|
||||
|
||||
Fetch terms narrow a wide read by words. Each agent has a list of them (`fetch_terms`, set in the agent editor); the terms of a pull are the union over the agents that are on, and none when one of those agents has none. They go only to a board read with the wide reach: a company tagged All roles, or a board whose reach is more countries than a one-country host is asked for (`MAX_SERIES`, 8). The narrow reach (Denmark, Sweden) leaves at most about 150 roles on any big board, so it takes no terms, and nor does a board under 300 roles, one never read whole, or one with its own reach. `{"terms": false}` in a board's params switches them off. A host that takes a keyword gets one series per term, unioned by id, each combined with the country series (an AND on every host the survey tested). When terms times the country series exceed 8, the place part of the reach goes first, whole, and the terms stay; more than 8 terms read the board unfiltered. Every host matches the whole text, not only the title, so a term like `engineer` cuts less than it looks. A read with terms is never the whole board: it records no size, closes no role it did not list, and the 10% guard holds it only to a read with the same terms and places. The search providers take no terms: each saved search has its own query.
|
||||
|
||||
What no source does:
|
||||
|
||||
- Filter on the host's side for a host other than SmartRecruiters, SuccessFactors, Workday, Oracle, Phenom and Eightfold. Greenhouse, Ashby, Recruitee and the rest are read whole.
|
||||
- Filter by category or level at the source. Workday, Oracle, Jibe and Eightfold state category ids, but nothing maps an agent's terms to them yet (`docs/plan-prep-survey.md`).
|
||||
- Read more than one board per company. DFDS is read from its Danish Teamtailor board only.
|
||||
- Use a key taken from a site's script. Maersk, New Work and Welcome to the Jungle are on the Watchlist for that reason.
|
||||
- Read a page that needs a browser to run its script.
|
||||
@@ -225,7 +228,7 @@ What no source does:
|
||||
| Remote | `location.remote`, true or false (122 remote) |
|
||||
| Pay | none: the list states no pay |
|
||||
| Found by discovery | yes, from a `jobs.`, `careers.` or `api.smartrecruiters.com` link; a guessed name is probed |
|
||||
| Filter at the source | `?country=dk`, one country a request; remote is ignored by the API, so a remote role outside the reach is not read. Bosch DK: 2 of 4,830 (2026-09-30). Which countries is the agents' reach, see below |
|
||||
| Filter at the source | `?country=dk`, one country a request; remote is ignored by the API, so a remote role outside the reach is not read. Bosch DK: 2 of 4,830 (2026-09-30). Which countries is the agents' reach, see below. Fetch terms as `?q=engineer`, one a request, AND with the country; it matches title and text (Bosch: 1,713 of 4,830) |
|
||||
| Limitations | 10,000 roles per board (100 pages). The identifier is read without case, so a lowercased slug works. An unknown identifier answers an empty list, not an error: a mistyped slug reads as a complete board with no roles and would close a company's roles. |
|
||||
|
||||
The reader's "Fetch the posting" button shows for these roles, since their description is empty; it reads one posting page on demand and does not feed `embed` or `extract`. Whether it reads a SmartRecruiters page cleanly was not checked.
|
||||
@@ -310,7 +313,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr
|
||||
| Remote | none |
|
||||
| Pay | none |
|
||||
| Found by discovery | yes, from a link to a role, `/go/` page or search on the site's own host; a link to the SAP back end names the host only; not probed. A site on the company's own domain whose pages link only the back end is set by hand |
|
||||
| Filter at the source | `optionsFacetsDD_country=DK`, one country a request; no remote facet. Vestas DK: 64 of 668 (2026-09-30). Which countries is the agents' reach, see below |
|
||||
| Filter at the source | `optionsFacetsDD_country=DK`, one country a request; no remote facet. Vestas DK: 64 of 668 (2026-09-30). Which countries is the agents' reach, see below. Fetch terms as the form's `q=engineer`, one a request, AND with the country (Vestas DK: 40 of 64) |
|
||||
| Limitations | Only the default locale's roles are listed: a role that exists only in another locale (Carlsberg's French brand, 37 of 105) is not read. Only `/search/` and `/job/` are read, never `/services/` or the `successfactors.eu` back end, so a company with only the back end (Bang & Olufsen) cannot be read. 100 pages per board, so 1,000 to 10,000 roles depending on the page size. A third layout or a bot wall that answers 200 fails the board with "not a SuccessFactors search page". Read from this machine only; not yet from the deployed server. |
|
||||
|
||||
### Workday
|
||||
@@ -327,7 +330,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr
|
||||
| Remote | `remoteType`, where the tenant sends it: remote, hybrid, onsite (585 with a value) |
|
||||
| Pay | none: no pay field; pay appears only inside the text |
|
||||
| Found by discovery | yes, from a `myworkdayjobs.com` link with a site; not probed |
|
||||
| Filter at the source | the tenant's own facets in `appliedFacets`, ids read by `facets` once a day: the facet whose values are countries, and any value that reads as remote. Two facets intersect, so remote is a series of its own. LEGO DK: 97 of 455 (2026-09-30). A tenant with no country facet is read whole. Which countries is the agents' reach, see below |
|
||||
| Filter at the source | the tenant's own facets in `appliedFacets`, ids read by `facets` once a day: the facet whose values are countries, and any value that reads as remote. Two facets intersect, so remote is a series of its own. LEGO DK: 97 of 455 (2026-09-30). A tenant with no country facet is read whole. Which countries is the agents' reach, see below. Fetch terms as `searchText` in the body, one a request, AND with the facets (LEGO: 106 for `engineer`) |
|
||||
| Limitations | 2,000 roles per board (100 pages of 20). A tenant has several sites and each is its own board. robots.txt paths are matched without case, as Workday serves any case. Every tenant is behind Cloudflare and was read from this machine only. |
|
||||
|
||||
### Oracle Recruiting Cloud
|
||||
@@ -344,7 +347,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr
|
||||
| Remote | `WorkplaceTypeCode`: remote, hybrid, onsite (741 with a value) |
|
||||
| Pay | none: the resource states no pay |
|
||||
| Found by discovery | yes, from a candidate site or role link on the pod, or a REST call naming `siteNumber`; not probed |
|
||||
| Filter at the source | `selectedLocationsFacet` with the pod's geography ids, and `selectedWorkplaceTypesFacet=ORA_REMOTE` as a series of its own. `locationsFacet` lists the ten biggest countries only, so a country missing from a full list is read unfiltered: fa-evmr (Nokia) has neither DK nor SE listed and is read whole. Which countries is the agents' reach, see below |
|
||||
| Filter at the source | `selectedLocationsFacet` with the pod's geography ids, and `selectedWorkplaceTypesFacet=ORA_REMOTE` as a series of its own. `locationsFacet` lists the ten biggest countries only, so a country missing from a full list is read unfiltered: fa-evmr (Nokia) has neither DK nor SE listed and is read whole. Which countries is the agents' reach, see below. Fetch terms as `keyword=` in the finder, one a request; it matches the text (Nokia: 468 of 554 for `engineer`) |
|
||||
| Limitations | The host is the tenant's Fusion pod, not the company's domain (`jobs.nokia.com` redirects to it). Links are built for locale `en`. The department fields were empty on every role of the four boards. |
|
||||
|
||||
### Phenom
|
||||
@@ -361,7 +364,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr
|
||||
| Remote | per role only, from the page's `checkRemote` (120 with a value) |
|
||||
| Pay | none |
|
||||
| Found by discovery | yes, from a `/<country>/<lang>/search-results` link; a link to Phenom itself names the host only; not probed |
|
||||
| Filter at the source | `selected_fields.country`, a list of English names in one request; no remote. Philips NL and DE: 111 of 873 (2026-09-30). Which countries is the agents' reach, see below |
|
||||
| Filter at the source | `selected_fields.country`, a list of English names in one request; no remote. Philips NL and DE: 111 of 873 (2026-09-30). Which countries is the agents' reach, see below. Fetch terms as `keywords` in the body, one a request, AND with the countries; it matches the text, not the title |
|
||||
| Limitations | The locale is built as `<lang>_<country>`, the spelling of the three sites read; another site may spell it differently. 10,000 roles per board. A role page without a description fails the read and is retried on a later pull. |
|
||||
|
||||
### Eightfold
|
||||
@@ -378,7 +381,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr
|
||||
| Remote | `workLocationOption`: onsite, hybrid, remote (628 with a value) |
|
||||
| Pay | none |
|
||||
| Found by discovery | yes, from an `eightfold.ai` link that carries `domain=`; a bare `eightfold.ai` link names the host only; not probed |
|
||||
| Filter at the source | `location=<country name>`, one country a request; no remote. Ericsson SE: 154 of 618 (2026-09-30). Which countries is the agents' reach, see below |
|
||||
| Filter at the source | `location=<country name>`, one country a request; no remote. Ericsson SE: 154 of 618 (2026-09-30). Which countries is the agents' reach, see below. Fetch terms as `query=`, one a request, AND with the location; it matches title, text and skills (Ericsson SE: 102 of 154 for `software`) |
|
||||
| Limitations | With 10 per request and 100 pages, a board over 1,000 roles is never read to the end; unless the country filter brings it under the cap: Infineon (1,289, 163 in DK and SE) and Microsoft (about 2,300, 60) become readable that way. Verified on three tenants only. |
|
||||
|
||||
### Gem
|
||||
|
||||
+5
-3
@@ -21,7 +21,9 @@ filter is the `reach` option the pull hands it (the agents', see
|
||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||
(`_runner.ask_for`). An unfiltered read records the board's size under
|
||||
`ats:size:<target>`, which the reach reads to leave small boards whole.
|
||||
`ats:size:<target>`, which the reach reads to leave small boards whole. A
|
||||
read narrowed by the agents' fetch terms is never the whole board: it
|
||||
records no size and prunes no per-role state.
|
||||
|
||||
A pull reads listings only. A host that states a role's text on the role's
|
||||
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
||||
@@ -384,7 +386,7 @@ class ATSSource(Source):
|
||||
rows = [row | {"description": ""}
|
||||
| (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {})
|
||||
for row in rows]
|
||||
if listing.complete:
|
||||
if listing.complete and not listing.narrowed:
|
||||
listed = {compound(i) for i in listing.ids}
|
||||
detail_sets = (detailed & listed, failed & listed)
|
||||
|
||||
@@ -395,7 +397,7 @@ class ATSSource(Source):
|
||||
# as the whole of it.
|
||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||
[compound(i) for i in listing.ids])
|
||||
if not listing.filtered:
|
||||
if not (listing.filtered or listing.narrowed):
|
||||
size = len(listing.ids)
|
||||
if not listing.complete:
|
||||
kept = self.state.get(f"ats:size:{target}", "")
|
||||
|
||||
@@ -68,7 +68,8 @@ class Filters(_Frozen):
|
||||
"""What the host filters at the source, per dimension. `country` takes
|
||||
ISO 3166 alpha-2 codes, `place` place names, `remote` a bool, `query`
|
||||
words. The runner never hands a reader a value on a dimension it
|
||||
declared `none`."""
|
||||
declared `none`. A board host with `query="one"` also takes the agents'
|
||||
fetch terms, one term a series (`Ask.terms`, see `_runner.ask_for`)."""
|
||||
|
||||
country: Filter = "none"
|
||||
place: Filter = "none"
|
||||
@@ -159,13 +160,19 @@ class Ask(_Frozen):
|
||||
series of their own: a reader never gets both `remote=True` and a
|
||||
country or a place. With `remote="ids"`, `remote_ids` holds the host's
|
||||
ids of its remote values (`facets(...)["remote"]`, any key), and
|
||||
`remote` stays True."""
|
||||
`remote` stays True.
|
||||
|
||||
`terms` are the agents' fetch terms a board is narrowed by (see
|
||||
`app/jobs/reach.py`). The runner makes one series per term, each with
|
||||
that term as `query`, so a reader only ever reads `query`. A read with
|
||||
terms is never the whole board: it sets no size and closes nothing."""
|
||||
|
||||
countries: tuple[str, ...] = ()
|
||||
places: tuple[str, ...] = ()
|
||||
remote: bool | None = None
|
||||
remote_ids: tuple[str, ...] = ()
|
||||
query: str = ""
|
||||
terms: tuple[str, ...] = ()
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
|
||||
|
||||
+100
-25
@@ -124,7 +124,9 @@ class Listing:
|
||||
`source_id`s the read listed: this pull's, or the whole sweep's once it
|
||||
reached the end. `complete` says `ids` are the whole list. `sweep` is the
|
||||
state to keep for the next pull, None when no sweep is under way.
|
||||
`filtered` says the read asked the host for a part of the list."""
|
||||
`filtered` says the read asked the host for the roles in some places,
|
||||
`narrowed` for the roles matching words (`narrowed()`): the ids of
|
||||
neither are the whole board."""
|
||||
|
||||
rows: list[Row]
|
||||
complete: bool
|
||||
@@ -132,6 +134,7 @@ class Listing:
|
||||
filtered: bool = False
|
||||
sweep: dict[str, Any] | None = None
|
||||
asks: str = ""
|
||||
narrowed: bool = False
|
||||
|
||||
|
||||
# --------------------------------------------------------------- modules
|
||||
@@ -289,11 +292,39 @@ def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
|
||||
return out
|
||||
|
||||
|
||||
#: The most series a reach may cost on a `one` filter: a reach of more
|
||||
#: countries (the wide reach, Europe) reads such a host unfiltered instead.
|
||||
#: The most series a reach may cost: a reach of more countries on a `one`
|
||||
#: filter (the wide reach, Europe) reads such a host unfiltered instead, and
|
||||
#: fetch terms times the reach's series may not cost more either.
|
||||
MAX_SERIES = 8
|
||||
|
||||
|
||||
def _terms(value: Any) -> tuple[str, ...]:
|
||||
"""Fetch terms as given in a reach: stripped, empty ones dropped, one of
|
||||
each regardless of case, in the order given."""
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for term in _values(value):
|
||||
if term.casefold() not in seen:
|
||||
seen.add(term.casefold())
|
||||
out.append(term)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
|
||||
"""How many series the location part of `wanted` takes on this reader:
|
||||
one per value of a `one` dimension (the product of two), one for a
|
||||
`many` or `ids` dimension, and one more for remote beside places, which
|
||||
the runner asks as a series of its own (see `series`)."""
|
||||
count = 1
|
||||
for dimension in ("country", "place"):
|
||||
values = wanted.get(dimension, ())
|
||||
if values and getattr(reads.filters, dimension) == "one":
|
||||
count *= len(values)
|
||||
if wanted.get("remote") and (wanted.get("country") or wanted.get("place")):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||
reach: Mapping[str, Any] | None = None, *,
|
||||
keys: Mapping[str, Any] | None = None) -> Ask:
|
||||
@@ -306,30 +337,63 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
|
||||
dropped. A `remote` of False asks nothing. A search's params go whole into
|
||||
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
||||
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
||||
A module without `READS` gets `Ask()`."""
|
||||
A module without `READS` gets `Ask()`.
|
||||
|
||||
Fetch terms (`reach["terms"]`) go only to a board reader that declares
|
||||
`query="one"`, and only on a wide read: `reach["wide"]` (a company
|
||||
tagged All roles), or a reach this rule just dropped as too many
|
||||
countries. A board whose params say `"terms": false` or name a query of
|
||||
their own takes none. Terms times the reach's series (`_cost`) may cost
|
||||
at most `MAX_SERIES`: over it, the place part of the reach goes first,
|
||||
whole, since remote without its countries would read only the remote
|
||||
roles; over it with terms alone, the terms go too and the board is read
|
||||
unfiltered."""
|
||||
if not declared(module):
|
||||
return Ask()
|
||||
reads: Reads = module.READS
|
||||
params = dict(subscription_params or {})
|
||||
reached = _dimensions(module, reach or {})
|
||||
given = reach or {}
|
||||
reached = _dimensions(module, given)
|
||||
wide = bool(given.get("wide"))
|
||||
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
||||
reached = {}
|
||||
wide = True
|
||||
wanted = reached | _dimensions(module, params)
|
||||
if wanted.get("remote") is False:
|
||||
del wanted["remote"]
|
||||
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()
|
||||
if name in reads.filters.declared()}
|
||||
declared_now = reads.filters.declared()
|
||||
wanted = {name: value for name, value in wanted.items() if name in declared_now}
|
||||
terms = _terms(given.get("terms"))
|
||||
if (not wide or reads.scope != "board" or reads.filters.query != "one"
|
||||
or params.get("terms") is False or "query" in wanted):
|
||||
terms = ()
|
||||
if terms and len(terms) * _cost(reads, wanted) > MAX_SERIES:
|
||||
wanted = {name: value for name, value in wanted.items()
|
||||
if name not in ("country", "place", "remote")}
|
||||
if len(terms) > MAX_SERIES:
|
||||
terms = ()
|
||||
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()}
|
||||
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
|
||||
if reads.key is not None:
|
||||
extra["key"] = resolve_key(reads.key, keys)
|
||||
return Ask(**fields, params=extra)
|
||||
return Ask(**fields, terms=terms, params=extra)
|
||||
|
||||
|
||||
def filtered(ask: Ask) -> bool:
|
||||
"""Whether `ask` reads a part of a board: a country, a place or remote."""
|
||||
"""Whether `ask` reads a part of a board by place: a country, a place or
|
||||
remote. Terms are `narrowed`."""
|
||||
return bool(ask.countries or ask.places or ask.remote is not None)
|
||||
|
||||
|
||||
def narrowed(module: ModuleType, ask: Ask) -> bool:
|
||||
"""Whether `ask` reads a board by words: fetch terms, or a board's own
|
||||
query. Such a read is never the whole board, nor the board within its
|
||||
places, so it sets no size and closes nothing. A search's query is what
|
||||
the search is, and does not count."""
|
||||
board = not declared(module) or module.READS.scope == "board"
|
||||
return bool(ask.terms) or (board and bool(ask.query))
|
||||
|
||||
|
||||
def series(module: ModuleType, ask: Ask,
|
||||
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
|
||||
"""The series of requests `ask` takes on this reader. Remote with a
|
||||
@@ -340,15 +404,22 @@ def series(module: ModuleType, ask: Ask,
|
||||
values `facets` does not know are dropped, and a part with none left is
|
||||
no series, since the host has nothing of it; a dimension `facets` does
|
||||
not state at all is read unfiltered, and so is remote alone, while the
|
||||
remote part beside places is then left out. `[ask]` for a module
|
||||
without `READS`."""
|
||||
remote part beside places is then left out. Each of those is then asked
|
||||
once per fetch term (`ask.terms`), with the term as `query`: the host
|
||||
intersects a keyword with its facets, as the survey found on every
|
||||
host, so the union by id is "one of the terms, in these places". `[ask]`
|
||||
for a module without `READS`."""
|
||||
if not declared(module):
|
||||
return [ask]
|
||||
if ask.remote and (ask.countries or ask.places):
|
||||
return [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
||||
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
||||
facets, beside=True)]
|
||||
return _series(module, ask, facets)
|
||||
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
||||
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
||||
facets, beside=True)]
|
||||
else:
|
||||
parts = _series(module, ask, facets)
|
||||
if not ask.terms:
|
||||
return parts
|
||||
return [part.model_copy(update={"query": term}) for term in ask.terms for part in parts]
|
||||
|
||||
|
||||
def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None,
|
||||
@@ -455,12 +526,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
ask = ask or Ask()
|
||||
asks = series(module, ask, facets)
|
||||
is_filtered = filtered(ask)
|
||||
by_words = narrowed(module, ask)
|
||||
key = fingerprint(asks)
|
||||
if not asks:
|
||||
return Listing([], True, [], is_filtered, None, key)
|
||||
return Listing([], True, [], is_filtered, None, key, by_words)
|
||||
if not hasattr(module, "page"):
|
||||
rows = _whole(module, slug, get, asks, limits)
|
||||
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key)
|
||||
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words)
|
||||
|
||||
reads: Reads | None = module.READS if declared(module) else None
|
||||
never = reads is not None and reads.complete == "never"
|
||||
@@ -518,15 +590,15 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
|
||||
listed = list(dict.fromkeys(earlier + list(rows)))
|
||||
if not stop:
|
||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key)
|
||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words)
|
||||
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
||||
module.NAME, slug, pages, stop)
|
||||
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
||||
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key)
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words)
|
||||
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
||||
"rows": listed, "whole": whole}
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key)
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words)
|
||||
|
||||
|
||||
def _storable(cursor: Any) -> bool:
|
||||
@@ -577,9 +649,10 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
|
||||
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
|
||||
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
||||
the facets it needs, the sweep it continues, and the 10% guard. A read
|
||||
that is filtered or whose reader declares `empty="suspect"`, and that
|
||||
listed under `SUSPECT_SHARE` of what the same read listed last time,
|
||||
raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
||||
that is filtered, narrowed by terms, or whose reader declares
|
||||
`empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same
|
||||
read listed last time (the same `fingerprint`, so a term read is only
|
||||
held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
||||
the rest once the rows are stored."""
|
||||
ask = ask or Ask()
|
||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||
@@ -588,7 +661,7 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
|
||||
suspect = declared(module) and module.READS.empty == "suspect"
|
||||
last = _load(state, f"{prefix}:count:{target}")
|
||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||
if ((listing.filtered or suspect) and listing.sweep is None and count
|
||||
if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count
|
||||
and len(listing.ids) < SUSPECT_SHARE * count):
|
||||
raise Suspect(f"listed {len(listing.ids)} roles, under "
|
||||
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
||||
@@ -599,12 +672,14 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
listing: Listing, ids: list[str]) -> None:
|
||||
"""Record a read whose rows are stored: `ok` (the run's marker),
|
||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||
`filtered`, the sweep to continue or none, and the count for the 10%
|
||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||
against), the sweep to continue or none, and the count for the 10%
|
||||
guard once a read is whole."""
|
||||
state[f"{prefix}:ok:{target}"] = run
|
||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||
if listing.sweep is not None:
|
||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||
else:
|
||||
|
||||
@@ -29,6 +29,11 @@ name in `_countries` is read unfiltered.
|
||||
|
||||
Discovery names Eightfold from an `eightfold.ai` link, and the host and the
|
||||
domain from one that carries `domain=`.
|
||||
|
||||
A fetch term goes as `query=`, one a request (`Filters(query="one")`);
|
||||
it matches title, text and skills, and is an AND with `location` (survey,
|
||||
2026-09-30: `software` with Sweden answered 102 of Ericsson's 154 Swedish
|
||||
roles).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -48,7 +53,7 @@ from ._shared import Get, Page, add_place, is_web_url, only, parse_dt, place
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||
places="many", filters=Filters(country="one"),
|
||||
places="many", filters=Filters(country="one", query="one"),
|
||||
fields=Fields(posted="listed", remote="stated"),
|
||||
robots="read: robots.txt allows /careers (checked 2026-09-29)",
|
||||
)
|
||||
@@ -97,13 +102,15 @@ def careers_url(slug: str) -> str:
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||
"""One page of `/api/pcsx/search` -> `{"status": 200, "data": {"positions":
|
||||
[...], "count": n}}`, in the one country `ask` names; the next starts at
|
||||
the offset after this page's positions, and an empty page ends the read."""
|
||||
the offset after this page's positions, and an empty page ends the read.
|
||||
`query` goes as `query=`."""
|
||||
host, domain = parts(slug)
|
||||
start = int(cursor or 0)
|
||||
country = name_of(ask.countries[0]) if ask.countries else ""
|
||||
where = f"&location={quote(country)}" if country else ""
|
||||
words = f"&query={quote(ask.query, safe='')}" if ask.query else ""
|
||||
answer = json.loads(get(f"https://{host}/api/pcsx/search?domain={domain}"
|
||||
f"&sort_by=timestamp&start={start}{where}"))
|
||||
f"&sort_by=timestamp&start={start}{where}{words}"))
|
||||
data = answer.get("data") if isinstance(answer, dict) else None
|
||||
positions = data.get("positions") if isinstance(data, dict) else None
|
||||
if not isinstance(positions, list):
|
||||
|
||||
@@ -20,6 +20,10 @@ the pod does not know answers the whole board (review). Remote is
|
||||
`selectedWorkplaceTypesFacet=ORA_REMOTE` (`remote="one"`); two facets are an
|
||||
intersection (Finland and remote answered 0 on 2026-09-30), which is why the
|
||||
runner asks for remote in a series of its own.
|
||||
|
||||
A fetch term goes into the finder as `keyword=engineer`, one a request
|
||||
(`Filters(query="one")`); it matches the text (survey, 2026-09-30: 468 of
|
||||
Nokia's 554).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -54,7 +58,7 @@ PATTERNS = (
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
|
||||
complete="stated", places="many", filters=Filters(country="ids", remote="one"),
|
||||
complete="stated", places="many", filters=Filters(country="ids", remote="one", query="one"),
|
||||
fields=Fields(posted="trusted", remote="stated"),
|
||||
robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites/<site>/jobs "
|
||||
"(checked 2026-09-29)"),
|
||||
@@ -126,9 +130,10 @@ FACET_LIMIT = 10
|
||||
|
||||
def _selected(ask: Ask) -> str:
|
||||
"""The finder's facet parameters for `ask`, each ending in a comma: the
|
||||
location ids (none when one is `*`, a country the pod did not list) and
|
||||
remote."""
|
||||
out = ""
|
||||
keyword, the location ids (none when one is `*`, a country the pod did
|
||||
not list) and remote. The keyword is escaped whole, so a comma or a
|
||||
semicolon in it cannot start another finder parameter."""
|
||||
out = f"keyword={quote(ask.query, safe='')}," if ask.query else ""
|
||||
ids = [i for i in ask.countries if re.fullmatch(r"\d+", i)]
|
||||
if ids and "*" not in ask.countries:
|
||||
out += f"selectedLocationsFacet={'%3B'.join(ids)},"
|
||||
|
||||
@@ -17,6 +17,10 @@ matches. Philips answered 111 of 873 for Netherlands and Germany on
|
||||
2026-09-30. There is no remote field to filter on. A code with no English
|
||||
name in `_countries` is left out of the list, and a list left empty reads the
|
||||
whole board.
|
||||
|
||||
A fetch term goes as `keywords` in the body, one a request
|
||||
(`Filters(query="one")`); it matches the text, not only the title, and is an
|
||||
AND with `selected_fields` (survey, 2026-09-30, Philips).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -46,7 +50,7 @@ PATTERNS = (
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=100, text="detail", teaser=True,
|
||||
complete="stated", places="many", filters=Filters(country="many"),
|
||||
complete="stated", places="many", filters=Filters(country="many", query="one"),
|
||||
fields=Fields(posted="trusted", remote="stated"),
|
||||
robots="read: robots.txt allows /<country>/<lang>/search-results (checked 2026-09-29)",
|
||||
)
|
||||
@@ -87,10 +91,11 @@ def careers_url(slug: str) -> str:
|
||||
return f"{_root(slug)}/search-results"
|
||||
|
||||
|
||||
def _body(slug: str, offset: int, countries: tuple[str, ...] = ()) -> dict[str, Any]:
|
||||
def _body(slug: str, offset: int, countries: tuple[str, ...] = (),
|
||||
keywords: str = "") -> dict[str, Any]:
|
||||
"""What the site's search page sends for a listing of every role, newest
|
||||
first, less the parts that only tell the page what to draw, in the
|
||||
`countries` (ISO codes) given. `lang` is `en_global` for `/global/en` and
|
||||
`countries` (ISO codes) given and matching `keywords` when given. `lang` is `en_global` for `/global/en` and
|
||||
`en_us` for `/us/en`, the site's own spelling on the three sites read."""
|
||||
_host, country, lang = _parts(slug)
|
||||
names = [name for code in countries if (name := name_of(code))]
|
||||
@@ -100,7 +105,7 @@ def _body(slug: str, offset: int, countries: tuple[str, ...] = ()) -> dict[str,
|
||||
"subsearch": "", "from": offset, "jobs": True, "counts": True, "all_fetch": False,
|
||||
"pageType": "landingPage", "size": PAGE_SIZE, "clearAll": False,
|
||||
"jdsource": "facets", "isSliderEnable": False, "siteType": "external",
|
||||
"location": "", "keywords": "", "global": True,
|
||||
"location": "", "keywords": keywords, "global": True,
|
||||
"selected_fields": {"country": names} if names else {},
|
||||
"sort": {"order": "desc", "field": "postedDate"}, "locationData": {},
|
||||
}
|
||||
@@ -118,7 +123,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
host = _parts(slug)[0]
|
||||
data = json.loads(get(f"https://{host}/widgets", method="POST",
|
||||
payload=_body(slug, offset, ask.countries)))
|
||||
payload=_body(slug, offset, ask.countries, ask.query)))
|
||||
search = data.get("refineSearch")
|
||||
if not isinstance(search, dict) or search.get("status", 200) != 200:
|
||||
raise ValueError("not a Phenom search answer: no refineSearch")
|
||||
|
||||
@@ -8,6 +8,11 @@ A country is filtered on the server, `?country=dk`, one a request
|
||||
(`Filters(country="one")`): Bosch's DE, SE and DK summed to what the app held
|
||||
(review, 2026-09-29), and DK answered 2 of 4,830 on 2026-09-30. The API
|
||||
ignores a remote parameter, so remote is not filtered.
|
||||
|
||||
A fetch term goes as `?q=engineer`, one a request (`Filters(query="one")`):
|
||||
it matches title and text, and with `country=` it is an AND (survey,
|
||||
2026-09-30: `q=engineer` answered 1,713 of Bosch's 4,830, with `country=dk`
|
||||
none). How several words in one `q` combine is untested.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,6 +21,7 @@ import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
@@ -30,7 +36,7 @@ PATTERNS = (
|
||||
#: An unknown identifier answers an empty list, which is `empty="suspect"`.
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=100, text="none", complete="stated",
|
||||
places="one", empty="suspect", filters=Filters(country="one"),
|
||||
places="one", empty="suspect", filters=Filters(country="one", query="one"),
|
||||
fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated",
|
||||
employment="stated"),
|
||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
||||
@@ -66,12 +72,13 @@ def probe_ok(body: bytes) -> bool:
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
"""One page, in the one country `ask` names, which the API takes in lower
|
||||
case."""
|
||||
case, and matching its `query` (`q=`)."""
|
||||
offset = int(cursor or 0)
|
||||
country = ask.countries[0].lower() if ask.countries else ""
|
||||
where = f"&country={country}" if re.fullmatch(r"[a-z]{2}", country) else ""
|
||||
words = f"&q={quote(ask.query, safe='')}" if ask.query else ""
|
||||
data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings"
|
||||
f"?offset={offset}&limit={PAGE_SIZE}{where}"))
|
||||
f"?offset={offset}&limit={PAGE_SIZE}{where}{words}"))
|
||||
content = data.get("content") or []
|
||||
return data, content, data.get("totalFound"), offset + len(content) if content else None
|
||||
|
||||
|
||||
@@ -44,6 +44,10 @@ places with their country from it; a role at several places repeats the
|
||||
`jobLocation` block, once per place. Novo Nordisk's pages have none: the
|
||||
description is still in `itemprop="description"`, and the listing's date and
|
||||
location stand.
|
||||
|
||||
A fetch term goes as the search form's `q=engineer`, one a request
|
||||
(`Filters(query="one")`); with `optionsFacetsDD_country=DK` it is an AND
|
||||
(survey, 2026-09-30: 40 of Vestas' 64 Danish roles).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -53,7 +57,7 @@ import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
from urllib.parse import quote, urljoin, urlsplit
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
@@ -85,7 +89,7 @@ KEEP_CASE = True
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", text="detail", complete="stated", places="many",
|
||||
filters=Filters(country="one"),
|
||||
filters=Filters(country="one", query="one"),
|
||||
fields=Fields(posted="trusted"),
|
||||
robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)",
|
||||
)
|
||||
@@ -155,12 +159,13 @@ def careers_url(slug: str) -> str:
|
||||
return f"https://{host}{prefix}/search/"
|
||||
|
||||
|
||||
def _search_url(slug: str, start: int, country: str = "") -> str:
|
||||
def _search_url(slug: str, start: int, country: str = "", query: str = "") -> str:
|
||||
"""The search page from `start`, in one country when `country` (an ISO
|
||||
code) is given."""
|
||||
code) is given, matching `query` (the form's `q`) when given."""
|
||||
host, prefix = _parts(slug)
|
||||
facet = f"&optionsFacetsDD_country={country}" if re.fullmatch(r"[A-Z]{2}", country) else ""
|
||||
return f"https://{host}{prefix}/search/?q=&locationsearch={facet}&startrow={start}"
|
||||
return (f"https://{host}{prefix}/search/?q={quote(query, safe='')}&locationsearch="
|
||||
f"{facet}&startrow={start}")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ listing
|
||||
@@ -264,13 +269,13 @@ def _entries(text: str) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
"""One search page, in the one country `ask` names. The cursor is `(start row, first role of the page
|
||||
"""One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page
|
||||
before)`: the next start row is what this page returned, and a page that
|
||||
starts with the same role as the last one is the site ignoring `startrow`,
|
||||
which ends the read (incomplete) instead of looping."""
|
||||
start, previous = cursor or (0, None)
|
||||
country = ask.countries[0] if ask.countries else ""
|
||||
text = get(_search_url(slug, start, country)).decode("utf-8", "replace")
|
||||
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
|
||||
if not _SEARCH_PAGE.search(text):
|
||||
raise ValueError("not a SuccessFactors search page")
|
||||
found = _entries(text)
|
||||
|
||||
@@ -23,6 +23,10 @@ series of its own. Each tenant names its facets its own way (LEGO's
|
||||
one whose values are countries. A host id is `<facetParameter>=<id>`. LEGO
|
||||
answered 97 of 455 for Denmark on 2026-09-30; a role at several places is
|
||||
kept when one of them is in the country.
|
||||
|
||||
A fetch term goes as `searchText` in the body, one a request
|
||||
(`Filters(query="one")`); it is an AND with `appliedFacets` (survey,
|
||||
2026-09-30: LEGO's `engineer` answered 106, 45 of them in Denmark).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -46,7 +50,7 @@ KEEP_CASE = True
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
|
||||
complete="stated", places="many", filters=Filters(country="ids", remote="ids"),
|
||||
complete="stated", places="many", filters=Filters(country="ids", remote="ids", query="one"),
|
||||
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
||||
robots="read: robots.txt allows /<Site>/ (checked 2026-09-29)",
|
||||
)
|
||||
@@ -114,7 +118,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||
data = json.loads(get(
|
||||
f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST",
|
||||
payload={"appliedFacets": _applied(ask), "limit": PAGE_SIZE, "offset": offset,
|
||||
"searchText": ""}))
|
||||
"searchText": ask.query}))
|
||||
if cursor is None:
|
||||
stated = data.get("total")
|
||||
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
|
||||
|
||||
@@ -208,6 +208,7 @@ def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request(
|
||||
reads = eightfold.READS
|
||||
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
|
||||
"paged", 10, "detail", "stated", "many")
|
||||
assert reads.filters.declared() == {"country": "one"} and reads.fields.posted == "listed"
|
||||
assert reads.filters.declared() == {"country": "one", "query": "one"}
|
||||
assert reads.fields.posted == "listed"
|
||||
assert reads.robots.startswith("read: robots.txt allows /careers")
|
||||
assert not hasattr(eightfold, "robots_page")
|
||||
@@ -148,7 +148,7 @@ def test_migration_20_seeds_an_old_file_and_leaves_collisions(tmp_path: Path) ->
|
||||
"VALUES (?, ?, '', '')", (key, name))
|
||||
conn.execute("PRAGMA user_version = 19")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [20, 21, 22]
|
||||
assert reopened.migrations_applied == [20, 21, 22, 23]
|
||||
assert reopened.duplicate_companies() == [("orsted", ["orsted", "ørsted"])]
|
||||
assert len(reopened.companies()) == 3
|
||||
|
||||
@@ -185,7 +185,7 @@ def test_migration_21_tags_the_favourites_and_the_boards_that_read_all_roles(tmp
|
||||
conn.execute("ALTER TABLE companies DROP COLUMN all_roles")
|
||||
conn.execute("PRAGMA user_version = 20")
|
||||
reopened = Store(path)
|
||||
assert reopened.migrations_applied == [21, 22]
|
||||
assert reopened.migrations_applied == [21, 22, 23]
|
||||
assert {c.key: c.all_roles for c in reopened.companies()} == {
|
||||
"mistral-ai": 1, "acme": 0, "black-forest-labs": 1, "liked": 0}
|
||||
params = {s.target: s.params for s in reopened.subscriptions(source="ats")}
|
||||
|
||||
@@ -246,7 +246,7 @@ def test_a_v17_file_gains_dup_of_with_its_first_rebuild(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22]
|
||||
assert migrated.migrations_applied == [18, 19, 20, 21, 22, 23]
|
||||
assert dup_of(migrated) == {first.uid: None, second.uid: first.uid}
|
||||
with sqlite3.connect(path) as conn:
|
||||
assert conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'idx_roles_dup_of'"
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
"""Fetch terms (work package F2 of `docs/plan-prep.md`): an agent's words that
|
||||
narrow what a big board read with the wide reach sends. The terms of a pull,
|
||||
where they apply, how the runner makes series of them, the request each host
|
||||
sends (the parameters `docs/plan-prep-survey.md` verified), the guards on a
|
||||
term read, and the agent editor's API. No network."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
from urllib.parse import parse_qs, unquote, urlsplit
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from pipeline import Tally # noqa: E402
|
||||
from app.config import Config, ServerConfig # noqa: E402
|
||||
from app.jobs import pull, reach # noqa: E402
|
||||
from app.main import create_app # noqa: E402
|
||||
from app.models import Agent, Gate # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import ( # noqa: E402
|
||||
Ask, _runner, adzuna, eightfold, greenhouse, jobindex, oracle, phenom, platsbanken,
|
||||
smartrecruiters, successfactors, workday,
|
||||
)
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
EUROPE = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
|
||||
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS",
|
||||
"LI", "NO", "GB", "CH", "EU"]
|
||||
|
||||
TERMS = ["engineer", "data"]
|
||||
|
||||
|
||||
class FakeConfig:
|
||||
sources: ClassVar[dict[str, object]] = {}
|
||||
searches: ClassVar[list[object]] = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "roledesk.db")
|
||||
|
||||
|
||||
def _wolf(store: Store, terms: list[str] | None = None, name: str = "Wolf") -> int:
|
||||
"""Agent 2 of 2026-09-30: Europe required; Denmark, Sweden, remote or a
|
||||
town in Skåne or Denmark."""
|
||||
gates = [Gate(field="country", values=EUROPE, mode="require"),
|
||||
Gate(field="country", values=["DK", "SE"], mode="any"),
|
||||
Gate(field="remote", values=["remote"], mode="any"),
|
||||
Gate(field="place", values=["Malmö", "copenhagen"], mode="any")]
|
||||
return store.save_agent(name, gates=gates, fetch_terms=terms).id
|
||||
|
||||
|
||||
def _board(store: Store, target: str, company: str, size: int | None = 1000,
|
||||
params: dict[str, object] | None = None, tagged: bool = True) -> None:
|
||||
store.add_subscription("ats", target, label=company, company=company,
|
||||
board=target.partition(":")[0], params=params or {})
|
||||
store.add_company(company)
|
||||
if size is not None:
|
||||
store.state[f"ats:size:{target}"] = str(size)
|
||||
if tagged:
|
||||
from app.models import company_key
|
||||
store.set_all_roles(company_key(company), True)
|
||||
|
||||
|
||||
def _sub(store: Store, target: str) -> Any:
|
||||
return next(s for s in store.subscriptions(source="ats") if s.target == target)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- the terms of a pull
|
||||
|
||||
def _agent(terms: list[str]) -> Agent:
|
||||
return Agent(id=1, name="a", created_at="", fetch_terms=terms)
|
||||
|
||||
|
||||
def test_the_terms_are_the_union_over_the_agents_one_of_each() -> None:
|
||||
found = reach.fetch_terms([_agent(["engineer", "Data"]), _agent(["data", "head of"])])
|
||||
assert found == ("Data", "engineer", "head of")
|
||||
|
||||
|
||||
def test_an_agent_without_terms_means_there_are_none() -> None:
|
||||
assert reach.fetch_terms([_agent(TERMS), _agent([])]) == ()
|
||||
assert reach.fetch_terms([]) == ()
|
||||
|
||||
|
||||
def test_only_enabled_agents_count(store: Store) -> None:
|
||||
_wolf(store, TERMS)
|
||||
off = store.save_agent("Off", fetch_terms=[])
|
||||
store.save_agent(off.name, agent_id=off.id, enabled=False)
|
||||
assert reach.plan(store).terms == ("data", "engineer")
|
||||
store.save_agent("On", fetch_terms=[])
|
||||
assert reach.plan(store).terms == ()
|
||||
|
||||
|
||||
# ------------------------------------------------------------ where they apply
|
||||
|
||||
def test_a_tagged_board_carries_the_terms_and_the_wide_mark(store: Store) -> None:
|
||||
_wolf(store, TERMS)
|
||||
_board(store, "phenom:www.careers.philips.com/global/en", "Philips")
|
||||
found = reach.plan(store).for_subscription(
|
||||
_sub(store, "phenom:www.careers.philips.com/global/en"))
|
||||
assert found is not None
|
||||
assert found["terms"] == ["data", "engineer"] and found["wide"] is True
|
||||
assert {"DE", "DK", "FR"} <= set(found["countries"])
|
||||
|
||||
|
||||
def test_the_narrow_reach_takes_no_terms(store: Store) -> None:
|
||||
_wolf(store, TERMS)
|
||||
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
|
||||
plan = reach.plan(store)
|
||||
sub = _sub(store, "smartrecruiters:BoschGroup")
|
||||
ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub))
|
||||
assert ask == Ask(countries=("DK", "SE"))
|
||||
assert reach.line(plan, sub) == "DK, SE · agents"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size, params", [
|
||||
(299, {}), (None, {}), (5000, {"countries": ["DE"]}), (5000, {"terms": False}),
|
||||
], ids=["small", "never read whole", "own reach", "terms off"])
|
||||
def test_small_unknown_overridden_and_switched_off_boards_take_no_terms(
|
||||
store: Store, size: int | None, params: dict[str, object]) -> None:
|
||||
_wolf(store, TERMS)
|
||||
_board(store, "smartrecruiters:BoschGroup", "Bosch", size, params)
|
||||
plan = reach.plan(store)
|
||||
sub = _sub(store, "smartrecruiters:BoschGroup")
|
||||
assert _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub)).terms == ()
|
||||
|
||||
|
||||
def test_the_agents_reach_takes_terms_once_it_is_too_many_countries_to_ask(
|
||||
store: Store) -> None:
|
||||
_wolf(store, TERMS)
|
||||
store.save_agent("Nordics and more", fetch_terms=["product"], gates=[Gate(
|
||||
field="country", values=["DE", "FR", "NL", "BE", "AT", "PL", "NO", "FI"], mode="any")])
|
||||
_board(store, "smartrecruiters:BoschGroup", "Bosch", tagged=False)
|
||||
plan = reach.plan(store)
|
||||
sub = _sub(store, "smartrecruiters:BoschGroup")
|
||||
ask = _runner.ask_for(smartrecruiters, sub.params, plan.for_subscription(sub))
|
||||
assert ask == Ask(terms=("data", "engineer", "product"))
|
||||
assert reach.line(plan, sub) == "all places · agents · 3 terms"
|
||||
|
||||
|
||||
# ------------------------------------------------------ terms times the reach
|
||||
|
||||
def test_terms_and_a_list_of_countries_combine_under_the_cap() -> None:
|
||||
wide = {"countries": EUROPE[:30], "remote": True, "terms": TERMS, "wide": True}
|
||||
ask = _runner.ask_for(phenom, {}, wide)
|
||||
assert ask.countries == tuple(EUROPE[:30]) and ask.terms == ("engineer", "data")
|
||||
asks = _runner.series(phenom, ask)
|
||||
assert [(a.query, len(a.countries)) for a in asks] == [("engineer", 30), ("data", 30)]
|
||||
|
||||
|
||||
def test_terms_and_countries_one_a_request_combine_as_their_product() -> None:
|
||||
wide = {"countries": ["DK", "SE"], "terms": TERMS, "wide": True}
|
||||
ask = _runner.ask_for(smartrecruiters, {}, wide)
|
||||
asks = _runner.series(smartrecruiters, ask)
|
||||
assert [(a.query, a.countries) for a in asks] == [
|
||||
("engineer", ("DK",)), ("engineer", ("SE",)), ("data", ("DK",)), ("data", ("SE",))]
|
||||
|
||||
|
||||
def test_countries_and_remote_are_two_series_a_term() -> None:
|
||||
ask = _runner.ask_for(workday, {}, {"countries": ["DK"], "remote": True, "terms": TERMS,
|
||||
"wide": True})
|
||||
facets = {"country": {"DK": "Location=dk"}, "remote": {"remote": "remoteType=r"}}
|
||||
asks = _runner.series(workday, ask, facets)
|
||||
assert [(a.query, a.countries, a.remote) for a in asks] == [
|
||||
("engineer", ("Location=dk",), None), ("engineer", (), True),
|
||||
("data", ("Location=dk",), None), ("data", (), True)]
|
||||
|
||||
|
||||
def test_over_the_cap_the_places_go_first_and_the_terms_stay() -> None:
|
||||
five = ["engineer", "data", "product", "head of", "architect"]
|
||||
# Five terms times (Europe + the remote series) is ten series: over eight.
|
||||
ask = _runner.ask_for(workday, {}, {"countries": EUROPE, "remote": True, "terms": five,
|
||||
"wide": True})
|
||||
assert ask == Ask(terms=tuple(five))
|
||||
assert [a.query for a in _runner.series(workday, ask)] == five
|
||||
# Three are six series: the places stay.
|
||||
ask = _runner.ask_for(workday, {}, {"countries": EUROPE, "remote": True,
|
||||
"terms": five[:3], "wide": True})
|
||||
assert ask.countries and ask.remote and ask.terms == tuple(five[:3])
|
||||
|
||||
|
||||
def test_more_terms_than_the_cap_read_the_board_unfiltered() -> None:
|
||||
nine = [f"word{n}" for n in range(9)]
|
||||
assert _runner.MAX_SERIES == 8
|
||||
assert _runner.ask_for(phenom, {}, {"countries": ["DK"], "terms": nine, "wide": True}) == Ask()
|
||||
|
||||
|
||||
def test_a_one_host_drops_europe_and_keeps_the_terms() -> None:
|
||||
ask = _runner.ask_for(smartrecruiters, {}, {"countries": EUROPE, "remote": True,
|
||||
"terms": TERMS, "wide": True})
|
||||
assert ask == Ask(terms=("engineer", "data"))
|
||||
|
||||
|
||||
def test_terms_need_the_wide_mark_or_a_dropped_reach() -> None:
|
||||
assert _runner.ask_for(phenom, {}, {"countries": ["DK", "SE"], "terms": TERMS}).terms == ()
|
||||
assert _runner.ask_for(phenom, {}, {"terms": TERMS}).terms == ()
|
||||
assert _runner.ask_for(phenom, {}, {"terms": TERMS, "wide": True}).terms == ("engineer", "data")
|
||||
|
||||
|
||||
def test_hosts_without_a_query_and_the_search_providers_take_no_terms() -> None:
|
||||
wide = {"countries": ["DK"], "terms": TERMS, "wide": True}
|
||||
assert _runner.ask_for(greenhouse, {}, wide) == Ask()
|
||||
for provider in (jobindex, platsbanken):
|
||||
ask = _runner.ask_for(provider, {"q": "python"}, wide)
|
||||
assert ask.terms == () and ask.query == "python"
|
||||
assert adzuna.READS.scope == "search"
|
||||
|
||||
|
||||
def test_a_boards_own_query_wins_over_the_terms() -> None:
|
||||
ask = _runner.ask_for(smartrecruiters, {"q": "sap"}, {"terms": TERMS, "wide": True})
|
||||
assert (ask.query, ask.terms) == ("sap", ())
|
||||
assert _runner.narrowed(smartrecruiters, ask)
|
||||
|
||||
|
||||
# ------------------------------------------------------------ the line
|
||||
|
||||
def test_the_reach_line_counts_the_terms(store: Store) -> None:
|
||||
_wolf(store, ["engineer", "data", "head of"])
|
||||
_board(store, "phenom:www.careers.philips.com/global/en", "Philips")
|
||||
_board(store, "smartrecruiters:BoschGroup", "Bosch")
|
||||
plan = reach.plan(store)
|
||||
philips = _sub(store, "phenom:www.careers.philips.com/global/en")
|
||||
assert reach.line(plan, philips) == \
|
||||
f"{len(plan.wide.countries)} countries · company tag · 3 terms"
|
||||
assert reach.line(plan, _sub(store, "smartrecruiters:BoschGroup")) == \
|
||||
"all places · company tag · 3 terms"
|
||||
|
||||
|
||||
def test_one_term_is_one_term(store: Store) -> None:
|
||||
_wolf(store, ["engineer"])
|
||||
_board(store, "smartrecruiters:BoschGroup", "Bosch")
|
||||
assert reach.line(reach.plan(store), _sub(store, "smartrecruiters:BoschGroup")) == \
|
||||
"all places · company tag · 1 term"
|
||||
|
||||
|
||||
# ------------------------------------------------------ each host's request
|
||||
|
||||
class Recorder:
|
||||
"""A `get` that answers `body` to every request and keeps what was asked."""
|
||||
|
||||
def __init__(self, body: bytes) -> None:
|
||||
self.body = body
|
||||
self.asked: list[tuple[str, dict[str, Any] | None]] = []
|
||||
|
||||
def __call__(self, url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
self.asked.append((url, payload))
|
||||
return self.body
|
||||
|
||||
|
||||
def _query(url: str) -> dict[str, list[str]]:
|
||||
return parse_qs(urlsplit(url).query)
|
||||
|
||||
|
||||
def test_smartrecruiters_sends_the_term_as_q_beside_the_country() -> None:
|
||||
get = Recorder(b'{"content": [], "totalFound": 0}')
|
||||
smartrecruiters.page(get, "BoschGroup", None, Ask(countries=("DK",), query="engineer"))
|
||||
smartrecruiters.page(get, "BoschGroup", None, Ask(query="head of"))
|
||||
smartrecruiters.page(get, "BoschGroup", None, Ask())
|
||||
assert _query(get.asked[0][0]) == {"offset": ["0"], "limit": ["100"], "country": ["dk"],
|
||||
"q": ["engineer"]}
|
||||
assert "q=head%20of" in get.asked[1][0]
|
||||
assert "q" not in _query(get.asked[2][0])
|
||||
|
||||
|
||||
def test_successfactors_sends_the_term_as_the_forms_q() -> None:
|
||||
get = Recorder((FIXTURES / "successfactors-table.html").read_bytes())
|
||||
successfactors.page(get, "careers.vestas.com", None, Ask(countries=("DK",), query="engineer"))
|
||||
successfactors.page(get, "careers.vestas.com", None, Ask())
|
||||
assert get.asked[0][0] == ("https://careers.vestas.com/search/?q=engineer&locationsearch="
|
||||
"&optionsFacetsDD_country=DK&startrow=0")
|
||||
assert get.asked[1][0].startswith("https://careers.vestas.com/search/?q=&locationsearch=")
|
||||
|
||||
|
||||
def test_workday_sends_the_term_as_search_text() -> None:
|
||||
get = Recorder(b'{"total": 0, "jobPostings": []}')
|
||||
workday.page(get, "lego.wd103/LEGO_External", None,
|
||||
Ask(countries=("Location=dk",), query="engineer"))
|
||||
workday.page(get, "lego.wd103/LEGO_External", None, Ask())
|
||||
assert get.asked[0][0] == "https://lego.wd103.myworkdayjobs.com/wday/cxs/lego/LEGO_External/jobs"
|
||||
assert get.asked[0][1] == {"appliedFacets": {"Location": ["dk"]}, "limit": 20, "offset": 0,
|
||||
"searchText": "engineer"}
|
||||
assert get.asked[1][1]["searchText"] == ""
|
||||
|
||||
|
||||
def test_phenom_sends_the_term_as_keywords() -> None:
|
||||
get = Recorder((FIXTURES / "phenom.json").read_bytes())
|
||||
phenom.page(get, "www.careers.philips.com/global/en", None,
|
||||
Ask(countries=("NL",), query="engineer"))
|
||||
phenom.page(get, "www.careers.philips.com/global/en", None, Ask())
|
||||
assert get.asked[0][0] == "https://www.careers.philips.com/widgets"
|
||||
body = get.asked[0][1]
|
||||
assert body["keywords"] == "engineer"
|
||||
assert body["selected_fields"] == {"country": ["Netherlands"]}
|
||||
assert get.asked[1][1]["keywords"] == ""
|
||||
|
||||
|
||||
def test_eightfold_sends_the_term_as_query() -> None:
|
||||
get = Recorder(b'{"status": 200, "data": {"positions": [], "count": 0}}')
|
||||
eightfold.page(get, "jobs.ericsson.com/ericsson.com", None,
|
||||
Ask(countries=("SE",), query="software"))
|
||||
eightfold.page(get, "jobs.ericsson.com/ericsson.com", None, Ask())
|
||||
assert _query(get.asked[0][0]) == {"domain": ["ericsson.com"], "sort_by": ["timestamp"],
|
||||
"start": ["0"], "location": ["Sweden"],
|
||||
"query": ["software"]}
|
||||
assert "query" not in _query(get.asked[1][0])
|
||||
|
||||
|
||||
def test_oracle_sends_the_term_as_the_finders_keyword() -> None:
|
||||
pod = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com/CX_1"
|
||||
get = Recorder(b'{"items": [{"requisitionList": [], "TotalJobsCount": 0}]}')
|
||||
oracle.page(get, pod, None, Ask(countries=("300000000471640",), query="engineer"))
|
||||
oracle.page(get, pod, None, Ask(query="r&d, lab"))
|
||||
finders = [unquote(_query(url)["finder"][0]) for url, _ in get.asked]
|
||||
assert finders[0] == ("findReqs;siteNumber=CX_1,keyword=engineer,selectedLocationsFacet="
|
||||
"300000000471640,limit=200,offset=0")
|
||||
# A comma in a term stays inside the keyword.
|
||||
assert "keyword=r%26d%2C%20lab," in get.asked[1][0]
|
||||
|
||||
|
||||
# ------------------------------------------------------------- the guards
|
||||
|
||||
def test_a_term_read_is_narrowed_and_its_count_is_its_own(tmp_path: Path) -> None:
|
||||
def get(url: str, **kwargs: Any) -> bytes:
|
||||
q = _query(url).get("q", [""])[0]
|
||||
ids = {"": [str(n) for n in range(40)], "engineer": ["1", "2"], "data": ["2", "3"]}[q]
|
||||
return json.dumps({"totalFound": len(ids), "content": [
|
||||
{"id": i, "name": f"Role {i}", "location": {"city": "Berlin", "country": "de"}}
|
||||
for i in ids]}).encode()
|
||||
|
||||
state: dict[str, str] = {}
|
||||
whole = _runner.read(state, "ats", "t", smartrecruiters, "Acme", get)
|
||||
_runner.stamp(state, "ats", "t", "r", whole, whole.ids)
|
||||
assert not whole.narrowed and len(whole.ids) == 40
|
||||
# Three of forty is under a tenth, but a term read is not held to a whole read.
|
||||
termed = _runner.read(state, "ats", "t", smartrecruiters, "Acme", get,
|
||||
Ask(terms=("engineer", "data")))
|
||||
assert termed.narrowed and not termed.filtered and termed.complete
|
||||
assert termed.ids == ["1", "2", "3"]
|
||||
_runner.stamp(state, "ats", "t", "r", termed, termed.ids)
|
||||
assert state["ats:narrowed:t"] == "1"
|
||||
# The same terms again, far fewer: that is a failure, held to its own count.
|
||||
state["ats:count:t"] = json.dumps({"asks": termed.asks, "count": 100})
|
||||
with pytest.raises(_runner.Suspect):
|
||||
_runner.read(state, "ats", "t", smartrecruiters, "Acme", get,
|
||||
Ask(terms=("engineer", "data")))
|
||||
|
||||
|
||||
def _posting(job_id: str) -> dict[str, Any]:
|
||||
return {"id": job_id, "name": f"Engineer {job_id}",
|
||||
"location": {"city": "Berlin", "country": "de"}}
|
||||
|
||||
|
||||
def test_a_term_read_writes_no_size_and_closes_nothing(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_wolf(store)
|
||||
_board(store, "smartrecruiters:bfl", "Black Forest Labs")
|
||||
asked: list[str] = []
|
||||
|
||||
def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
|
||||
asked.append(url)
|
||||
q = _query(url).get("q", [""])[0]
|
||||
ids = {"": ["1", "2", "9"], "engineer": ["1"], "data": ["2"]}[q]
|
||||
return json.dumps({"totalFound": len(ids),
|
||||
"content": [_posting(i) for i in ids]}).encode()
|
||||
|
||||
monkeypatch.setattr(ats, "_get", get)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
|
||||
def run() -> dict[str, Any]:
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
results = [pull.run_batch(FakeConfig(), store, **b) # type: ignore[arg-type]
|
||||
for b in batches]
|
||||
return pull.close_boards(FakeConfig(), store, # type: ignore[arg-type]
|
||||
Tally(results=tuple(results)))
|
||||
|
||||
# No terms: Europe is too many countries for SmartRecruiters, so whole.
|
||||
run()
|
||||
assert [_query(u).get("q") for u in asked] == [None]
|
||||
assert store.state["ats:size:smartrecruiters:bfl"] == "3"
|
||||
store.state["ats:size:smartrecruiters:bfl"] = "1000"
|
||||
open_before = {r.source_id for r in store.roles(limit=50)}
|
||||
assert open_before == {f"smartrecruiters:bfl:{n}" for n in ("1", "2", "9")}
|
||||
|
||||
agent = store.agents()[0]
|
||||
store.save_agent(agent.name, gates=agent.gates, agent_id=agent.id, fetch_terms=TERMS)
|
||||
asked.clear()
|
||||
settled = run()
|
||||
assert sorted(_query(u)["q"][0] for u in asked) == ["data", "engineer"]
|
||||
assert all("country" not in _query(u) for u in asked)
|
||||
assert store.state["ats:narrowed:smartrecruiters:bfl"] == "1"
|
||||
assert store.state["ats:complete:smartrecruiters:bfl"] == "1"
|
||||
# Role 9 is outside the terms, not gone: it stays open, and the size stays.
|
||||
assert {r.source_id for r in store.roles(limit=50)} == open_before
|
||||
assert settled["closed"] == 0
|
||||
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
|
||||
# A board sized by its open roles is never sized by a term read.
|
||||
del store.state["ats:size:smartrecruiters:bfl"]
|
||||
assert reach.sizes(store, store.subscriptions(source="ats")) == {}
|
||||
|
||||
|
||||
# ------------------------------------------------------- storage and the API
|
||||
|
||||
def test_terms_are_stored_normalised_and_kept_by_a_save_without_them(store: Store) -> None:
|
||||
agent = store.save_agent("Wolf", fetch_terms=[" engineer ", "Engineer", "", "head of"])
|
||||
assert agent.fetch_terms == ["engineer", "head of"]
|
||||
kept = store.save_agent("Wolf", agent_id=agent.id, intent="Data leadership.")
|
||||
assert kept.fetch_terms == ["engineer", "head of"]
|
||||
cleared = store.save_agent("Wolf", agent_id=agent.id, fetch_terms=[])
|
||||
assert cleared.fetch_terms == []
|
||||
assert store.save_agent("New").fetch_terms == []
|
||||
|
||||
|
||||
def test_migration_23_gives_every_agent_no_terms(tmp_path: Path) -> None:
|
||||
path = tmp_path / "old.db"
|
||||
Store(path).save_agent("Wolf")
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("ALTER TABLE agents DROP COLUMN fetch_terms")
|
||||
conn.execute("PRAGMA user_version = 22")
|
||||
migrated = Store(path)
|
||||
assert migrated.migrations_applied == [23]
|
||||
assert migrated.agents()[0].fetch_terms == []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path: Path) -> FastAPI:
|
||||
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
||||
database=tmp_path / "roledesk.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app: FastAPI) -> Iterator[TestClient]:
|
||||
with TestClient(app) as c:
|
||||
yield c
|
||||
|
||||
|
||||
def test_the_agent_editor_saves_and_lists_the_terms(client: TestClient) -> None:
|
||||
res = client.post("/api/agents", json={"name": "Wolf",
|
||||
"fetch_terms": ["engineer", " data ", "DATA"]})
|
||||
assert res.status_code == 200 and res.json()["fetch_terms"] == ["engineer", "data"]
|
||||
agent_id = res.json()["id"]
|
||||
[listed] = client.get("/api/agents").json()
|
||||
assert listed["fetch_terms"] == ["engineer", "data"]
|
||||
# The card's switch posts the row back as it came: the terms stay.
|
||||
res = client.post("/api/agents", json=listed | {"enabled": False})
|
||||
assert res.json()["fetch_terms"] == ["engineer", "data"]
|
||||
# A body without the key keeps them; an empty list clears them.
|
||||
res = client.post("/api/agents", json={"id": agent_id, "name": "Wolf"})
|
||||
assert res.json()["fetch_terms"] == ["engineer", "data"]
|
||||
res = client.post("/api/agents", json={"id": agent_id, "name": "Wolf", "fetch_terms": []})
|
||||
assert res.json()["fetch_terms"] == []
|
||||
|
||||
|
||||
def test_the_companies_list_says_the_terms(client: TestClient, app: FastAPI) -> None:
|
||||
store: Store = app.state.store
|
||||
_wolf(store, ["engineer", "data", "head of"])
|
||||
_board(store, "smartrecruiters:BoschGroup", "Bosch")
|
||||
row = next(c for c in client.get("/api/companies").json() if c["key"] == "bosch")
|
||||
assert row["reach"] == "all places · company tag · 3 terms"
|
||||
+1
-1
@@ -299,7 +299,7 @@ def test_migration_22_fills_the_host_words_from_raw(tmp_path: Path) -> None:
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [22]
|
||||
assert migrated.migrations_applied == [22, 23]
|
||||
one = row(migrated, "1")
|
||||
assert (one["seniority_host"], one["employment_host"], one["remote_host"],
|
||||
one["country_host"]) == ("senior", "full_time", "true", "DK")
|
||||
|
||||
@@ -48,12 +48,12 @@ def test_the_six_hosts_declare_what_their_search_filters() -> None:
|
||||
declared = {module.NAME: module.READS.filters.declared() for module in
|
||||
(smartrecruiters, eightfold, workday, successfactors, phenom, oracle)}
|
||||
assert declared == {
|
||||
"smartrecruiters": {"country": "one"},
|
||||
"eightfold": {"country": "one"},
|
||||
"workday": {"country": "ids", "remote": "ids"},
|
||||
"successfactors": {"country": "one"},
|
||||
"phenom": {"country": "many"},
|
||||
"oracle": {"country": "ids", "remote": "one"},
|
||||
"smartrecruiters": {"country": "one", "query": "one"},
|
||||
"eightfold": {"country": "one", "query": "one"},
|
||||
"workday": {"country": "ids", "remote": "ids", "query": "one"},
|
||||
"successfactors": {"country": "one", "query": "one"},
|
||||
"phenom": {"country": "many", "query": "one"},
|
||||
"oracle": {"country": "ids", "remote": "one", "query": "one"},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 22
|
||||
assert LATEST == 23
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -187,7 +187,7 @@ def test_a_v14_file_gains_the_watchlist_columns_and_leaves_companies_unrated(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22]
|
||||
assert migrated.migrations_applied == [15, 16, 17, 18, 19, 20, 21, 22, 23]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.ats, acme.careers_url) == ("teamtailor", "https://a.example")
|
||||
@@ -392,7 +392,7 @@ def test_v14_makes_each_intent_the_agents_one_prompt_with_its_vector(tmp_path: P
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22]
|
||||
assert migrated.migrations_applied == [14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
|
||||
assert migrated.agent(backend.id).prompts == [ # type: ignore[union-attr]
|
||||
Prompt(name="Backend", text="python backend")]
|
||||
assert migrated.agent(blank.id).prompts == [] # type: ignore[union-attr]
|
||||
@@ -416,7 +416,7 @@ def test_v16_gives_a_company_no_reason_and_keeps_its_row(tmp_path: Path) -> None
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22]
|
||||
assert migrated.migrations_applied == [16, 17, 18, 19, 20, 21, 22, 23]
|
||||
acme = migrated.company("acme")
|
||||
assert acme is not None
|
||||
assert (acme.careers_url, acme.notes, acme.hand_reason) == (
|
||||
@@ -445,7 +445,7 @@ def test_v17_folds_every_place_and_levels_the_titles_extract_has_not_read(
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22]
|
||||
assert migrated.migrations_applied == [17, 18, 19, 20, 21, 22, 23]
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = {uid: (fold, level) for uid, fold, level in conn.execute(
|
||||
"SELECT uid, place_fold, seniority FROM roles")}
|
||||
@@ -687,7 +687,7 @@ def test_v19_gives_every_role_its_places_and_splits_its_level(tmp_path: Path) ->
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22]
|
||||
assert migrated.migrations_applied == [19, 20, 21, 22, 23]
|
||||
assert [p.name for p in migrated.role_places(joined.uid)] == ["Copenhagen", "Malmö"]
|
||||
assert [p.name for p in migrated.role_places(detailed.uid)] == ["Aarhus"]
|
||||
assert migrated.role_places(stated.uid) == []
|
||||
|
||||
@@ -127,7 +127,8 @@ def test_the_readers_are_listed_with_what_they_declare(client: TestClient) -> No
|
||||
found = {r["name"]: r for r in client.get("/api/readers").json()}
|
||||
assert found["workday"] == {
|
||||
"name": "workday", "label": "Workday", "scope": "board", "listing": "paged",
|
||||
"text": "detail", "filters": {"country": "ids", "remote": "ids"}, "places": "many",
|
||||
"text": "detail", "filters": {"country": "ids", "remote": "ids", "query": "one"},
|
||||
"places": "many",
|
||||
"robots": found["workday"]["robots"]}
|
||||
assert found["workday"]["robots"].startswith("read:")
|
||||
assert found["greenhouse"]["filters"] == {}
|
||||
|
||||
Reference in new issue
Block a user