Merge board size set once, Recheck, and the Companies line naming what a site does not know (wp-a-size)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
8c66aeb445
11 files changed
+786
-70
No files matched your search
+45
-1
@@ -3,7 +3,7 @@
|
||||
app serve [--host H] [--port P] [--reload]
|
||||
app job pull|extract|embed|score [--payload JSON]
|
||||
app worker [--workers http=8,cpu=1] [--until-idle]
|
||||
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check
|
||||
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
|
||||
app searches add|list|remove
|
||||
app agents list|prompt
|
||||
app stages [extract|summarise on|off]
|
||||
@@ -32,6 +32,7 @@ from typing import TYPE_CHECKING, Any
|
||||
from app import config as config_mod
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models import Subscription
|
||||
from app.storage import Store
|
||||
|
||||
#: The one plugin this app ships lives outside the `app` package, in
|
||||
@@ -118,6 +119,10 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="add or update the company: its board when one is readable, "
|
||||
"else the host it runs on")
|
||||
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
|
||||
size = companies_sub.add_parser(
|
||||
"size", help="size a company's board again, or --missing every board without a size")
|
||||
size.add_argument("key", nargs="?", default="", help="the company key, e.g. infineon")
|
||||
size.add_argument("--missing", action="store_true", help="every board with no size yet")
|
||||
|
||||
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
||||
searches_sub = searches.add_subparsers(dest="action", required=True)
|
||||
@@ -335,6 +340,8 @@ def run_companies(args: argparse.Namespace) -> int:
|
||||
return detach_board(store, args.key)
|
||||
if args.action == "discover":
|
||||
return discover_company(store, args.name, args.domain, add=args.add)
|
||||
if args.action == "size":
|
||||
return size_boards(store, args.key, missing=args.missing)
|
||||
return check_companies(store)
|
||||
|
||||
|
||||
@@ -503,6 +510,43 @@ def detach_board(store: Store, key: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def size_boards(store: Store, key: str, *, missing: bool) -> int:
|
||||
"""`app companies size infineon`: size its board again and keep it, as
|
||||
the Companies row's Recheck does, and print the size and what a pull asks
|
||||
the board for. `--missing` sizes every board shown that has no size yet,
|
||||
a line each, and exits 1 if any did not answer. The next pull reads a
|
||||
board so; nothing is queued from here."""
|
||||
from app import jobs
|
||||
|
||||
if bool(key) == missing:
|
||||
print("companies size: name a company key, or --missing", file=sys.stderr)
|
||||
return 2
|
||||
if missing:
|
||||
subs = [s for s in store.subscriptions(source="ats") if s.show]
|
||||
known = jobs.reach.sizes(store, subs)
|
||||
subs = [s for s in subs if s.target not in known]
|
||||
else:
|
||||
sub = store.company_subscription(key)
|
||||
if sub is None or not sub.show:
|
||||
print(f"companies size: {key!r} has no board", file=sys.stderr)
|
||||
return 1
|
||||
subs = [sub]
|
||||
sized: list[tuple[Subscription, int]] = []
|
||||
for sub in subs:
|
||||
try:
|
||||
sized.append((sub, jobs.sources.size_board(store, sub.target)))
|
||||
except ValueError as exc:
|
||||
print(f"companies size: {exc}", file=sys.stderr)
|
||||
plan = jobs.reach.plan(store, [sub for sub, _size in sized])
|
||||
for sub, size in sized:
|
||||
print(json.dumps({"board": sub.target, "size": size,
|
||||
"reach": jobs.reach.line(plan, sub)}))
|
||||
dead = len(subs) - len(sized)
|
||||
if missing:
|
||||
print(f"sized {len(sized)} of {len(subs)} boards without a size")
|
||||
return 1 if dead else 0
|
||||
|
||||
|
||||
def check_companies(store: Store) -> int:
|
||||
"""`app companies check`: every `ats` board, fetched once. Prints a
|
||||
line per board that did not answer now, answered incompletely, or has
|
||||
|
||||
+113
-19
@@ -21,8 +21,13 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
|
||||
it skips the `any` gates; it does not widen what is fetched.
|
||||
- **Board override**: a board whose subscription `params` name a country, a
|
||||
place or remote gets no reach here; `ask_for` reads the params instead.
|
||||
- **Small boards**: a board that listed under `SMALL` roles when last read
|
||||
whole gets no filter; filtering it saves little and risks closing roles.
|
||||
- **Small boards**: a board of under `SMALL` roles gets no filter;
|
||||
filtering it saves little and risks closing roles. A board's size is set
|
||||
once, when it is added or rechecked (`plugins/ats.py`, `board_size`), and
|
||||
never by a pull. A board with no size yet is handed the reach it would
|
||||
have as a big one, with `unsized` and `small`, and its batch sizes it
|
||||
before it reads it, so a big new board is not read whole first; a board
|
||||
whose reader cannot narrow that reach is sized by its whole read instead.
|
||||
|
||||
A reach of more countries than a `one`-filter host should be asked one by
|
||||
one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`,
|
||||
@@ -43,14 +48,15 @@ board sends the roles that match one of them rather than every role:
|
||||
changes nothing on the narrow reach today, which takes no terms.
|
||||
- Only a reader that declares `query="one"` takes them, one series a term;
|
||||
`ask_for` has the rule for terms times the reach's series.
|
||||
- A read with terms is never the whole board: it sets no `ats:size`, and
|
||||
the pull closes nothing against it (`app/jobs/pull.py`).
|
||||
- A read with terms is never the whole board: the pull closes nothing
|
||||
against it (`app/jobs/pull.py`).
|
||||
- The search providers take none. Every saved search has a query of its
|
||||
own, which terms would replace; a search is a query already.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
@@ -65,9 +71,15 @@ from plugins.readers import _runner
|
||||
#: whole.
|
||||
SMALL = 300
|
||||
|
||||
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
|
||||
#: unfiltered read listed, written by `plugins/ats.py`.
|
||||
#: `state` keys of a board's size, `ats:size:<target>`, the roles it listed
|
||||
#: when it was sized, and of the day it was sized (ISO), both written by
|
||||
#: `plugins/ats.py` (`keep_size`).
|
||||
SIZE = "ats:size:"
|
||||
SIZE_CHECKED = "ats:sizechecked:"
|
||||
|
||||
#: `state` key of the host ids a board's site states (`{"checked",
|
||||
#: "facets"}`), kept by the runner (`_runner.facets_for`).
|
||||
FACETS = "ats:facets:"
|
||||
|
||||
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
|
||||
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote"))
|
||||
@@ -239,7 +251,9 @@ class Plan:
|
||||
"""The reach of one pull: `base` for every board, `wide` for the boards
|
||||
of the companies in `all_countries`, each board's size by target, the
|
||||
fetch `terms` of the wide reads, which skip the companies in
|
||||
`all_roles`, and `wide_names`, what the agents call the wide reach."""
|
||||
`all_roles`, `wide_names`, what the agents call the wide reach, and
|
||||
`facets`, the host ids the runner keeps for a board on a host that
|
||||
filters by them, for the Companies line."""
|
||||
|
||||
base: Reach
|
||||
wide: Reach = EVERYWHERE
|
||||
@@ -248,6 +262,7 @@ class Plan:
|
||||
terms: tuple[str, ...] = ()
|
||||
all_roles: frozenset[str] = frozenset()
|
||||
wide_names: str = ""
|
||||
facets: dict[str, dict[str, dict[str, str]]] = field(default_factory=dict)
|
||||
|
||||
def why(self, sub: Subscription) -> tuple[Reach, str]:
|
||||
"""The reach `sub` is read with, and why: `override`, `small`,
|
||||
@@ -260,6 +275,10 @@ class Plan:
|
||||
return EVERYWHERE, "unknown"
|
||||
if size < SMALL:
|
||||
return EVERYWHERE, "small"
|
||||
return self._big(sub)
|
||||
|
||||
def _big(self, sub: Subscription) -> tuple[Reach, str]:
|
||||
"""The reach of `sub` as a board of `SMALL` roles or more."""
|
||||
if company_key(sub.company or sub.label) in self.all_countries:
|
||||
return self.wide, "countries"
|
||||
return self.base, "agents"
|
||||
@@ -274,30 +293,51 @@ class Plan:
|
||||
fetch terms, and unless its company is set to All roles, a board on
|
||||
the wide reach carries them and `wide`, and a board on the agents'
|
||||
reach carries them for `ask_for` to use if it drops the countries as
|
||||
too many (see the module docstring)."""
|
||||
too many (see the module docstring). A board with no size carries
|
||||
`unsized` and `small`, and the reach of a big board unless its params
|
||||
set their own, for its batch to size it first."""
|
||||
found, why = self.why(sub)
|
||||
if why == "unknown":
|
||||
found, why = self._big(sub)
|
||||
out = found.params()
|
||||
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
|
||||
out = (out or {}) | {"terms": list(self.terms)}
|
||||
if why == "countries":
|
||||
out["wide"] = True
|
||||
if sub.target not in self.sizes:
|
||||
out = (out or {}) | {"unsized": True, "small": SMALL}
|
||||
return out
|
||||
|
||||
|
||||
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
||||
"""Each board's size: the roles its last unfiltered read listed
|
||||
(`ats:size:<target>`), else, for a board read before that was kept and
|
||||
never filtered, its open roles. A board never read has none."""
|
||||
"""Each board's size as kept (`ats:size:<target>`); a board never sized
|
||||
has none."""
|
||||
state = store.state
|
||||
counts = store.open_countries_by_subscription()
|
||||
out: dict[str, int] = {}
|
||||
for sub in subs:
|
||||
kept = state.get(f"{SIZE}{sub.target}", "")
|
||||
if kept.isdigit():
|
||||
out[sub.target] = int(kept)
|
||||
elif (state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}")
|
||||
and not state.get(f"ats:narrowed:{sub.target}")):
|
||||
out[sub.target] = sum(counts.get(sub.id, {}).values())
|
||||
return out
|
||||
|
||||
|
||||
def site_facets(store: Store, subs: Iterable[Subscription]) -> dict[str, dict[str, dict[str, str]]]:
|
||||
"""The host ids the runner keeps for each board on a host whose country
|
||||
or remote filter is `ids` (`ats:facets:<target>`); a board with none kept
|
||||
is left out."""
|
||||
out: dict[str, dict[str, dict[str, str]]] = {}
|
||||
for sub in subs:
|
||||
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||
if module is None or not _runner.declared(module) \
|
||||
or "ids" not in (module.READS.filters.country, module.READS.filters.remote):
|
||||
continue
|
||||
try:
|
||||
kept = json.loads(store.state.get(f"{FACETS}{sub.target}") or "{}")
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
found = kept.get("facets") if isinstance(kept, dict) else None
|
||||
if isinstance(found, dict):
|
||||
out[sub.target] = found
|
||||
return out
|
||||
|
||||
|
||||
@@ -321,7 +361,8 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
|
||||
all_countries=frozenset(c.key for c in rows if c.all_countries),
|
||||
terms=fetch_terms(agents),
|
||||
all_roles=frozenset(c.key for c in rows if c.all_roles),
|
||||
wide_names=wide_names(agents, gaz) if agents else "everywhere")
|
||||
wide_names=wide_names(agents, gaz) if agents else "everywhere",
|
||||
facets=site_facets(store, subs))
|
||||
|
||||
|
||||
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
|
||||
@@ -338,14 +379,58 @@ _WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
|
||||
"agents": "an agent takes any place"}
|
||||
|
||||
|
||||
def _codes(codes: Iterable[str]) -> list[str]:
|
||||
"""Country codes as the line names them: up to four, else a count."""
|
||||
found = sorted(codes)
|
||||
return found if len(found) <= 4 else [f"{len(found)} countries"]
|
||||
|
||||
|
||||
def _places(ask: readers.Ask) -> str:
|
||||
words = sorted(ask.countries) if len(ask.countries) <= 4 \
|
||||
else [f"{len(ask.countries)} countries"]
|
||||
words = _codes(ask.countries)
|
||||
if ask.remote:
|
||||
words.append("remote")
|
||||
return ", ".join(words)
|
||||
|
||||
|
||||
def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] | None,
|
||||
lead: str, roles: str) -> str:
|
||||
"""The line of a board whose site does not know every value `ask` wants
|
||||
on an `ids` filter, from the host ids the runner keeps (`facets`, None
|
||||
before any are kept), as `_runner.series` drops them: what is asked,
|
||||
then what the site does not know ("DK · SE not on this site · no remote
|
||||
option"), after `lead` and before `roles`; or, with nothing left to
|
||||
ask, "site lists none of DK, SE: nothing asked". "" when the site knows
|
||||
every value, or `lead` is "" and something is still asked."""
|
||||
if facets is None:
|
||||
return ""
|
||||
reads = module.READS.filters
|
||||
known, missing = list(ask.countries), []
|
||||
if reads.country == "ids" and ask.countries and "country" in facets:
|
||||
listed = facets.get("country") or {}
|
||||
known = [c for c in ask.countries if c in listed]
|
||||
missing = [c for c in ask.countries if c not in listed]
|
||||
no_remote = bool(ask.remote) and reads.remote == "ids" and not facets.get("remote")
|
||||
if not (missing or no_remote):
|
||||
return ""
|
||||
if no_remote and not (ask.countries or ask.places) and "remote" not in facets:
|
||||
# Remote alone, on a site that states no remote facet, is read whole.
|
||||
return f"all places, no remote option on this site · {roles}" if lead else ""
|
||||
remote = bool(ask.remote) and not no_remote
|
||||
if not (known or ask.places or remote):
|
||||
if not missing:
|
||||
return "site has no remote option: nothing asked"
|
||||
named = _codes(missing) + (["remote"] if no_remote else [])
|
||||
return f"site lists none of {', '.join(named)}: nothing asked"
|
||||
if not lead:
|
||||
return ""
|
||||
words = [", ".join(_codes(known) + list(ask.places) + (["remote"] if remote else []))]
|
||||
if missing:
|
||||
words.append(f"{', '.join(_codes(missing))} not on this site")
|
||||
if no_remote:
|
||||
words.append("no remote option")
|
||||
return f"{lead}: {' · '.join(w for w in words if w)} · {roles}"
|
||||
|
||||
|
||||
def _roles(ask: readers.Ask) -> str:
|
||||
"""The role-type half of the line: the agents' terms the board is
|
||||
asked for, or all roles."""
|
||||
@@ -363,14 +448,23 @@ def line(found: Plan, sub: Subscription) -> str:
|
||||
· all roles", "set here: DE · all roles", "all places, too many countries
|
||||
to ask · follows agents' terms: engineer", or, for a board read whole,
|
||||
why: "under 300 roles: read whole", "Greenhouse cannot filter: read
|
||||
whole"."""
|
||||
whole". A board with no size yet says it is sized on the next pull. On
|
||||
a host that filters by its own ids, the line names what the site does
|
||||
not know once the runner has asked it (`_site`): "follows agents: DK ·
|
||||
SE not on this site · no remote option · all roles", or "site lists
|
||||
none of DK, SE: nothing asked"."""
|
||||
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||
if module is None:
|
||||
return ""
|
||||
given, why = found.why(sub)
|
||||
if why == "unknown":
|
||||
return f"{_WHOLE[why]}: sized on the next pull"
|
||||
ask = _runner.ask_for(module, sub.params, found.for_subscription(sub))
|
||||
roles = _roles(ask)
|
||||
if _runner.filtered(ask):
|
||||
lead = {"agents": "follows agents", "override": "set here"}.get(why, "")
|
||||
if site := _site(module, ask, found.facets.get(sub.target), lead, roles):
|
||||
return site
|
||||
if why == "countries":
|
||||
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
||||
return f"all countries ({named}) · {roles}"
|
||||
|
||||
+29
-10
@@ -46,10 +46,11 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
|
||||
"""Read company `key` from `board` (`host:slug`): attach it to a company
|
||||
that has none, replace the one it has, or create the company from `name`
|
||||
when there is no row. The board is fetched first and refused if its host
|
||||
is unknown or it does not answer; a board listing zero roles is kept, since
|
||||
a company between openings is still one to follow. `check=False` skips the
|
||||
fetch, for the seed, whose boards were read once when it was written.
|
||||
when there is no row. The board is sized first (`board_size`) and refused
|
||||
if its host is unknown or it does not answer; a board listing zero roles
|
||||
is kept, since a company between openings is still one to follow.
|
||||
`check=False` skips the request, for the seed, whose boards were read
|
||||
once when it was written; such a board is sized by its first pull.
|
||||
|
||||
The row's `ats` and `ats_slug` become the board's host and slug, its
|
||||
`careers_url` the board's page where it was empty or the old board's, and
|
||||
@@ -59,7 +60,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
change in one transaction (`Store.set_company_board`), and replacing a
|
||||
board closes the old one's open roles. Returns the subscription, how many
|
||||
roles the board lists (None unchecked) and how many roles closed."""
|
||||
from plugins.ats import careers_url, fetch_board, is_known_target
|
||||
from plugins.ats import board_size, careers_url, is_known_target, keep_size
|
||||
|
||||
board, name = board.strip(), name.strip()
|
||||
key = key.strip() or company_key(name)
|
||||
@@ -72,10 +73,10 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
raise ValueError(f"no company {key!r}; a new company needs a name")
|
||||
if company_key(name) != key:
|
||||
raise ValueError(f"{name!r} is not the company {key!r}")
|
||||
rows: list[Any] | None = None
|
||||
size: int | None = None
|
||||
if check:
|
||||
try:
|
||||
rows, _complete = fetch_board(board)
|
||||
size = board_size(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
host, _, slug = board.partition(":")
|
||||
@@ -95,7 +96,23 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
assert sub is not None
|
||||
if before is not None and before.target != board:
|
||||
_forget_failing(store, before.target)
|
||||
return sub, None if rows is None else len(rows), closed
|
||||
if size is not None:
|
||||
keep_size(store.state, board, size)
|
||||
return sub, size, closed
|
||||
|
||||
|
||||
def size_board(store: Store, board: str) -> int:
|
||||
"""Size `board` again (`plugins.ats.board_size`) and keep it with the
|
||||
day: how a board's size changes once it was added (a Recheck, `app
|
||||
companies size`). Raises ValueError when it does not answer."""
|
||||
from plugins.ats import board_size, keep_size
|
||||
|
||||
try:
|
||||
size = board_size(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
keep_size(store.state, board, size)
|
||||
return size
|
||||
|
||||
|
||||
def detach_board(store: Store, key: str) -> int:
|
||||
@@ -118,9 +135,11 @@ def _forget_failing(store: Store, target: str) -> None:
|
||||
record of `target` failing would otherwise outlive the board, and count
|
||||
in `app companies check` for good. The `ats` plugin's robots.txt verdict
|
||||
and its lists of roles read and failed in detail go with it, since they
|
||||
are about that board alone and would only grow stale."""
|
||||
are about that board alone and would only grow stale, as would its size,
|
||||
which a board attached again gets anew."""
|
||||
store.state.pop(FAILING + target, None)
|
||||
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
|
||||
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:", "ats:size:",
|
||||
"ats:sizechecked:"):
|
||||
store.state.pop(key + target, None)
|
||||
|
||||
|
||||
|
||||
@@ -28,6 +28,11 @@
|
||||
* posted to `/api/sources/{id}/reach`. The wide reach (Europe) and no role
|
||||
* narrowing are the company's two overrides instead, and the board's own
|
||||
* countries win over All countries.
|
||||
*
|
||||
* Beside that line stands the board's size, set when it was added and
|
||||
* changed only by a Recheck ("999 roles · checked 30 Sep"). Recheck, beside
|
||||
* Change, sizes the board again (`/api/sources/{id}/size`) and queues a
|
||||
* pull, since the size decides whether the board is filtered.
|
||||
*/
|
||||
(function () {
|
||||
"use strict";
|
||||
@@ -67,6 +72,41 @@
|
||||
|
||||
// ---------------------------------------------------------------- the list
|
||||
|
||||
/** "30 Sep", or "30 Sep 2025" when it is not this year. A day alone
|
||||
("2026-09-30") is read as local midnight, not UTC's. */
|
||||
function checkedOn(iso) {
|
||||
const text = String(iso || "");
|
||||
const at = new Date(text.length === 10 ? `${text}T00:00` : text);
|
||||
if (Number.isNaN(at.getTime())) return "";
|
||||
const options = { day: "numeric", month: "short" };
|
||||
if (at.getFullYear() !== new Date().getFullYear()) options.year = "numeric";
|
||||
return at.toLocaleDateString("en-GB", options);
|
||||
}
|
||||
|
||||
/** The row's Recheck: size the board again, which may change what it is
|
||||
asked for, and read it with the pull that follows. */
|
||||
function recheckButton(company, rerender) {
|
||||
const button = document.createElement("button");
|
||||
button.type = "button";
|
||||
button.className = "linky";
|
||||
button.textContent = "Recheck";
|
||||
button.title = "Count the board's roles again";
|
||||
button.addEventListener("click", async () => {
|
||||
button.disabled = true;
|
||||
try {
|
||||
const result = await api(`/api/sources/${company.subscription_id}/size`,
|
||||
{ method: "POST" });
|
||||
window.toast.info(`${company.name}: ${count(result.size, "role")}, `
|
||||
+ `reads ${result.reach}. Reading now, or with the pull right after.`);
|
||||
await loadCompanies(agentId());
|
||||
rerender();
|
||||
} catch (err) {
|
||||
button.disabled = false;
|
||||
}
|
||||
});
|
||||
return button;
|
||||
}
|
||||
|
||||
function subLine(company, rerender) {
|
||||
const sub = document.createElement("span");
|
||||
sub.className = "sub";
|
||||
@@ -104,6 +144,10 @@
|
||||
}
|
||||
sub.append(` · ${count(company.open_roles, "open role")}`);
|
||||
if (company.reach) sub.append(` · reads ${company.reach}`);
|
||||
if (company.size != null) {
|
||||
const checked = checkedOn(company.size_checked);
|
||||
sub.append(` · ${count(company.size, "role")}${checked ? ` · checked ${checked}` : ""}`);
|
||||
}
|
||||
if (company.failing_since) {
|
||||
// The board has failed on every pull since then; after 21 days its
|
||||
// roles close.
|
||||
@@ -115,6 +159,10 @@
|
||||
sub.appendChild(failing);
|
||||
}
|
||||
sub.append(" ");
|
||||
if (company.subscription_id != null) {
|
||||
sub.appendChild(recheckButton(company, rerender));
|
||||
sub.append(" ");
|
||||
}
|
||||
const edit = document.createElement("button");
|
||||
edit.type = "button";
|
||||
edit.className = "linky";
|
||||
|
||||
+27
-1
@@ -491,6 +491,10 @@ class CompanyOut(BaseModel):
|
||||
#: what a pull asks its board for (`jobs.reach.line`).
|
||||
reach_params: dict[str, Any] = {}
|
||||
reach: str = ""
|
||||
#: The roles the board listed when it was last sized, and that day
|
||||
#: (ISO); None and "" before it is sized (`jobs.reach.SIZE`).
|
||||
size: int | None = None
|
||||
size_checked: str = ""
|
||||
|
||||
|
||||
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
|
||||
@@ -534,7 +538,10 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]:
|
||||
all_roles=bool(c.all_roles),
|
||||
reach_params={k: v for k, v in (sub.params if sub else {}).items()
|
||||
if k in jobs.reach.OVERRIDE_KEYS},
|
||||
reach=jobs.reach.line(plan, sub) if sub else ""))
|
||||
reach=jobs.reach.line(plan, sub) if sub else "",
|
||||
size=plan.sizes.get(target) if sub else None,
|
||||
size_checked=store.state.get(
|
||||
jobs.reach.SIZE_CHECKED + target, "") if sub else ""))
|
||||
return out
|
||||
|
||||
|
||||
@@ -897,6 +904,25 @@ def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]:
|
||||
"reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)}
|
||||
|
||||
|
||||
@router.post("/sources/{sub_id}/size")
|
||||
def recheck_size(sub_id: int, store: StoreDep, queue: QueueDep) -> dict[str, Any]:
|
||||
"""Size a board again, the one way its size changes once it was added
|
||||
(`jobs.sources.size_board`): its reach may change with it, so a pull is
|
||||
queued right after, as the other routes that change what is read do. 422
|
||||
when the board does not answer; the size it had stays."""
|
||||
sub = store.subscription(sub_id)
|
||||
if sub is None or sub.source != "ats":
|
||||
raise HTTPException(404, f"no such board: {sub_id}")
|
||||
try:
|
||||
size = jobs.sources.size_board(store, sub.target)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
queue.enqueue("pull")
|
||||
checked = store.state.get(jobs.reach.SIZE_CHECKED + sub.target, "")
|
||||
return {"size": size, "size_checked": checked,
|
||||
"reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)}
|
||||
|
||||
|
||||
class ReaderOut(BaseModel):
|
||||
"""What a reader declares (`plugins/readers/_contract.py`), for the
|
||||
Sources page: `filters` holds only the dimensions it filters."""
|
||||
|
||||
+72
-13
@@ -20,10 +20,18 @@ recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
||||
filter is the `reach` option the pull hands it (the agents', see
|
||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||
(`_runner.ask_for`). An unfiltered read records the board's size under
|
||||
`ats:size:<target>`, which the reach reads to leave small boards whole. A
|
||||
read narrowed by the agents' fetch terms is never the whole board: it
|
||||
records no size and prunes no per-role state.
|
||||
(`_runner.ask_for`). A read narrowed by the agents' fetch terms is never
|
||||
the whole board: it prunes no per-role state.
|
||||
|
||||
A board's size (`board_size`) is set once, when the board is added
|
||||
(`app.jobs.sources.set_board`) or rechecked, and kept under
|
||||
`ats:size:<target>` with the day it was checked under
|
||||
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small
|
||||
boards whole. A pull does not rewrite it. A board with no size yet is sized
|
||||
in its batch before it is read: its reach then says `unsized` and the size
|
||||
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
|
||||
whose reader cannot narrow that reach is read whole anyway, and sized by
|
||||
that read once it is complete, rather than by a second request.
|
||||
|
||||
A pull reads listings only. A host that states a role's text on the role's
|
||||
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
||||
@@ -307,12 +315,46 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no state, just "does
|
||||
this board answer, what does it list, and is that all of it". For `app
|
||||
companies add|check`. The listing only, unfiltered, no detail reads.
|
||||
companies check`. The listing only, unfiltered, no detail reads.
|
||||
Raises the same way `_fetch_rows` does."""
|
||||
with httpx.Client() as client:
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
#: Pages read at most to size a board whose host states no total on page 1:
|
||||
#: 5,000 roles on Teamtailor, 10,000 on HR Manager, far past any board seen.
|
||||
SIZE_PAGES = 100
|
||||
|
||||
|
||||
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
||||
"""How many roles board `target` lists, unfiltered and without terms: a
|
||||
whole host's listing, the total a `complete="stated"` host gives on page
|
||||
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
|
||||
as `_fetch_rows` does, so a board that does not answer is refused."""
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
if client is None:
|
||||
with httpx.Client() as own:
|
||||
return board_size(target, client=own)
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
if (_runner.declared(module) and module.READS.listing == "paged"
|
||||
and module.READS.complete == "stated"):
|
||||
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
|
||||
if isinstance(total, int):
|
||||
return total
|
||||
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
|
||||
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
||||
|
||||
|
||||
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
|
||||
"""Keep board `target`'s size and the day it was checked (ISO,
|
||||
`today`, else today in UTC)."""
|
||||
state[f"ats:size:{target}"] = str(size)
|
||||
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
|
||||
|
||||
|
||||
failure_reason = _runner.failure_reason
|
||||
|
||||
|
||||
@@ -358,8 +400,7 @@ class ATSSource(Source):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"),
|
||||
keys=self.options.get("keys"))
|
||||
ask, unsized = self._ask(client, target, module)
|
||||
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
||||
limits=_limits())
|
||||
except Exception as exc:
|
||||
@@ -397,18 +438,36 @@ class ATSSource(Source):
|
||||
# as the whole of it.
|
||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||
[compound(i) for i in listing.ids])
|
||||
if not (listing.filtered or listing.narrowed):
|
||||
size = len(listing.ids)
|
||||
if not listing.complete:
|
||||
kept = self.state.get(f"ats:size:{target}", "")
|
||||
size = max(size, int(kept) if kept.isdigit() else 0)
|
||||
self.state[f"ats:size:{target}"] = str(size)
|
||||
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
||||
keep_size(self.state, target, len(listing.ids))
|
||||
if detail_sets is not None:
|
||||
# A complete listing drops the ids it no longer lists.
|
||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||
if key in self.state:
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
|
||||
def _ask(self, client: httpx.Client, target: str,
|
||||
module: Any) -> tuple[readers.Ask, bool]:
|
||||
"""What `target` is asked for, from the reach its batch handed it and
|
||||
its params, and whether it still needs a size. A board with no size
|
||||
yet (`unsized` in its reach) whose reach would narrow the read is
|
||||
sized first (`board_size`, kept by `keep_size`), and read whole under
|
||||
`small` roles. One whose read is whole anyway is sized by that read,
|
||||
once it is complete."""
|
||||
given = self.options.get("reach")
|
||||
params, keys = self.params_for(target), self.options.get("keys")
|
||||
if not isinstance(given, dict) or not given.get("unsized"):
|
||||
return _runner.ask_for(module, params, given, keys=keys), False
|
||||
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
|
||||
ask = _runner.ask_for(module, params, reach or None, keys=keys)
|
||||
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
|
||||
return ask, True
|
||||
size = board_size(target, client=client)
|
||||
keep_size(self.state, target, size)
|
||||
if size < int(given.get("small") or 0):
|
||||
ask = _runner.ask_for(module, params, None, keys=keys)
|
||||
return ask, False
|
||||
|
||||
|
||||
# ------------------------------------------------------ the per-role reads
|
||||
|
||||
|
||||
@@ -0,0 +1,428 @@
|
||||
"""A board's size, set once: `ats.board_size` per kind of host, `set_board`
|
||||
and the Recheck (route and CLI) that keep it, the pull that sizes a board
|
||||
with none before it reads it, and the Companies line that names what a
|
||||
site's own ids leave out. No network: `ats._get` or the runner's listing is
|
||||
stubbed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from pipeline import Tally # noqa: E402
|
||||
from app.cli import main # noqa: E402
|
||||
from app.config import Config, ServerConfig # noqa: E402
|
||||
from app.jobs import pull, reach, sources # noqa: E402
|
||||
from app.main import create_app # noqa: E402
|
||||
from app.models import Gate # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import Ask, _runner, greenhouse, workday # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOSCH = "smartrecruiters:BoschGroup"
|
||||
TODAY = datetime.now(UTC).date().isoformat()
|
||||
|
||||
|
||||
class FakeConfig:
|
||||
sources: ClassVar[dict[str, object]] = {}
|
||||
searches: ClassVar[list[object]] = []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "t.db")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_pause(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
|
||||
|
||||
def _query(url: str) -> dict[str, list[str]]:
|
||||
return parse_qs(urlsplit(url).query)
|
||||
|
||||
|
||||
def _posting(job_id: str, country: str = "de") -> dict[str, Any]:
|
||||
return {"id": job_id, "name": f"Engineer {job_id}",
|
||||
"location": {"city": "Berlin", "country": country}}
|
||||
|
||||
|
||||
class SmartRecruiters:
|
||||
"""A SmartRecruiters board of `total` roles that answers each request
|
||||
with one page of three and the stated total, or, per country, the roles
|
||||
there. Remembers every URL asked."""
|
||||
|
||||
def __init__(self, total: int | None, *, down: bool = False) -> None:
|
||||
self.total, self.down = total, down
|
||||
self.asked: list[str] = []
|
||||
|
||||
def __call__(self, client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
|
||||
self.asked.append(url)
|
||||
if self.down:
|
||||
raise OSError("connection refused")
|
||||
query = _query(url)
|
||||
if int(query.get("offset", ["0"])[0]):
|
||||
return json.dumps({"content": []}).encode()
|
||||
country = query.get("country", [""])[0]
|
||||
content = [_posting(f"{country}{n}", country or "de") for n in range(3)]
|
||||
answer: dict[str, Any] = {"content": content}
|
||||
if self.total is not None:
|
||||
answer["totalFound"] = 3 if country else self.total
|
||||
return json.dumps(answer).encode()
|
||||
|
||||
|
||||
def _teamtailor(count: int) -> Any:
|
||||
"""A Teamtailor site of `count` roles, served 50 a page by offset."""
|
||||
asked: list[str] = []
|
||||
|
||||
def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
|
||||
asked.append(url)
|
||||
offset = int(_query(url)["offset"][0])
|
||||
items = "".join(
|
||||
f"<item><title>Role {n}</title><link>https://acme.teamtailor.com/jobs/{n}</link>"
|
||||
f"<guid>{n}</guid></item>" for n in range(offset, min(offset + 50, count)))
|
||||
return f"<rss><channel>{items}</channel></rss>".encode()
|
||||
|
||||
get.asked = asked # type: ignore[attr-defined]
|
||||
return get
|
||||
|
||||
|
||||
def _wolf(store: Store, countries: tuple[str, ...] = ("DK", "SE"), remote: bool = True) -> None:
|
||||
gates = [Gate(field="country", values=list(countries), mode="any")]
|
||||
if remote:
|
||||
gates.append(Gate(field="remote", values=["remote"], mode="any"))
|
||||
store.save_agent("Wolf", gates=gates)
|
||||
|
||||
|
||||
def _board(store: Store, target: str, company: str, size: int | None = None,
|
||||
params: dict[str, object] | None = None) -> None:
|
||||
store.add_subscription("ats", target, label=company, company=company,
|
||||
board=target.partition(":")[0], params=params or {})
|
||||
store.add_company(company)
|
||||
if size is not None:
|
||||
store.state[f"ats:size:{target}"] = str(size)
|
||||
|
||||
|
||||
def _sub(store: Store, target: str) -> Any:
|
||||
return next(s for s in store.subscriptions(source="ats") if s.target == target)
|
||||
|
||||
|
||||
def _pull(store: Store) -> list[dict[str, Any]]:
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
results = [pull.run_batch(FakeConfig(), store, **batch)
|
||||
for batch in batches]
|
||||
pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||
return batches
|
||||
|
||||
|
||||
# ------------------------------------------------------------- board_size
|
||||
|
||||
def test_a_host_that_states_a_total_is_sized_by_page_one(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
board = SmartRecruiters(999)
|
||||
monkeypatch.setattr(ats, "_get", board)
|
||||
assert ats.board_size(BOSCH) == 999
|
||||
# One unfiltered request: no country, no terms.
|
||||
[url] = board.asked
|
||||
assert "country" not in _query(url) and "q" not in _query(url)
|
||||
|
||||
|
||||
def test_a_host_that_states_no_total_is_read_to_the_end(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
get = _teamtailor(120)
|
||||
monkeypatch.setattr(ats, "_get", get)
|
||||
assert ats.board_size("teamtailor:acme") == 120
|
||||
assert [_query(u)["offset"][0] for u in get.asked] == ["0", "50", "100", "120"]
|
||||
|
||||
|
||||
def test_a_stated_host_that_leaves_out_its_total_is_read_to_the_end(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ats, "_get", SmartRecruiters(None))
|
||||
assert ats.board_size(BOSCH) == 3
|
||||
|
||||
|
||||
def test_a_whole_host_is_sized_by_its_one_answer(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
body = (FIXTURES / "greenhouse.json").read_bytes()
|
||||
monkeypatch.setattr(ats, "_get", lambda *args, **kwargs: body)
|
||||
assert ats.board_size("greenhouse:acme") == len(greenhouse.parse(body, "acme"))
|
||||
|
||||
|
||||
def test_board_size_refuses_an_unknown_board_and_raises_on_no_answer(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with pytest.raises(ValueError, match="host:slug"):
|
||||
ats.board_size("taleo:acme")
|
||||
monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True))
|
||||
with pytest.raises(OSError):
|
||||
ats.board_size(BOSCH)
|
||||
|
||||
|
||||
# -------------------------------------------------------------- set_board
|
||||
|
||||
def test_set_board_keeps_the_size_and_the_day_without_reading_the_board_whole(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
board = SmartRecruiters(4830)
|
||||
monkeypatch.setattr(ats, "_get", board)
|
||||
sub, found, _closed = sources.set_board(store, "bosch", BOSCH, name="Bosch")
|
||||
assert found == 4830 and len(board.asked) == 1
|
||||
assert store.state[f"ats:size:{BOSCH}"] == "4830"
|
||||
assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY
|
||||
assert reach.plan(store).why(sub)[1] == "agents"
|
||||
|
||||
|
||||
def test_the_seed_keeps_no_size_and_a_detached_board_forgets_it(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True))
|
||||
sources.set_board(store, "bosch", BOSCH, name="Bosch", check=False)
|
||||
assert f"ats:size:{BOSCH}" not in store.state
|
||||
ats.keep_size(store.state, BOSCH, 4830)
|
||||
sources.detach_board(store, "bosch")
|
||||
assert f"ats:size:{BOSCH}" not in store.state
|
||||
assert f"ats:sizechecked:{BOSCH}" not in store.state
|
||||
|
||||
|
||||
# ------------------------------------------------------ the unsized reach
|
||||
|
||||
def test_a_board_with_no_size_is_handed_the_agents_reach_marked_unsized(store: Store) -> None:
|
||||
_wolf(store)
|
||||
_board(store, BOSCH, "Bosch")
|
||||
_board(store, "workday:acme.wd3/Jobs", "Acme", params={"countries": ["DE"]})
|
||||
_board(store, "smartrecruiters:tiny", "Tiny", size=12)
|
||||
plan = reach.plan(store)
|
||||
assert plan.for_subscription(_sub(store, BOSCH)) == {
|
||||
"countries": ["DK", "SE"], "remote": True, "unsized": True, "small": reach.SMALL}
|
||||
# A board with its own reach carries only the marks; a sized one none.
|
||||
assert plan.for_subscription(_sub(store, "workday:acme.wd3/Jobs")) == {
|
||||
"unsized": True, "small": reach.SMALL}
|
||||
assert plan.for_subscription(_sub(store, "smartrecruiters:tiny")) is None
|
||||
assert reach.line(plan, _sub(store, BOSCH)) == "size not known yet: sized on the next pull"
|
||||
|
||||
|
||||
def test_a_big_board_with_no_size_is_sized_then_read_with_the_reach(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_wolf(store)
|
||||
_board(store, BOSCH, "Bosch")
|
||||
board = SmartRecruiters(4830)
|
||||
monkeypatch.setattr(ats, "_get", board)
|
||||
_pull(store)
|
||||
assert store.state[f"ats:size:{BOSCH}"] == "4830"
|
||||
assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY
|
||||
# The first request sizes the board; the read then asks for DK and SE.
|
||||
assert "country" not in _query(board.asked[0])
|
||||
assert sorted(_query(u).get("country", [""])[0] for u in board.asked[1:]) == ["dk", "se"]
|
||||
assert store.state[f"ats:filtered:{BOSCH}"] == "1"
|
||||
# The next pull finds the size and does not ask for it again.
|
||||
board.asked.clear()
|
||||
_pull(store)
|
||||
assert all("country" in _query(u) for u in board.asked)
|
||||
|
||||
|
||||
def test_a_small_board_with_no_size_is_sized_then_read_whole(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_wolf(store)
|
||||
_board(store, BOSCH, "Bosch")
|
||||
board = SmartRecruiters(3)
|
||||
monkeypatch.setattr(ats, "_get", board)
|
||||
_pull(store)
|
||||
assert store.state[f"ats:size:{BOSCH}"] == "3"
|
||||
assert all("country" not in _query(u) for u in board.asked)
|
||||
assert store.state[f"ats:filtered:{BOSCH}"] == ""
|
||||
|
||||
|
||||
def test_a_board_that_does_not_answer_its_sizing_fails_as_any_read(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_wolf(store)
|
||||
_board(store, BOSCH, "Bosch")
|
||||
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True))
|
||||
_pull(store)
|
||||
assert f"ats:size:{BOSCH}" not in store.state
|
||||
assert "connection refused" in json.loads(store.state[f"ats:failing:{BOSCH}"])["reason"]
|
||||
|
||||
|
||||
def test_a_board_read_whole_anyway_is_sized_by_that_read_once_it_is_complete(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_wolf(store)
|
||||
_board(store, "greenhouse:acme", "Acme")
|
||||
rows = [{"source_id": str(n), "title": f"Role {n}", "url": f"https://x.example/{n}",
|
||||
"places": [], "description": ""} for n in range(4)]
|
||||
answer = {"complete": False}
|
||||
asked: list[Ask] = []
|
||||
|
||||
def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object
|
||||
) -> _runner.Listing:
|
||||
asked.append(ask)
|
||||
return _runner.Listing(rows, answer["complete"], [r["source_id"] for r in rows]) # type: ignore[misc]
|
||||
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
monkeypatch.setattr(ats, "board_size", lambda *a, **k: pytest.fail("sized apart"))
|
||||
_pull(store)
|
||||
# Greenhouse cannot filter: one read, and an incomplete one sizes nothing.
|
||||
assert asked == [Ask()]
|
||||
assert "ats:size:greenhouse:acme" not in store.state
|
||||
answer["complete"] = True
|
||||
_pull(store)
|
||||
assert store.state["ats:size:greenhouse:acme"] == "4"
|
||||
assert store.state["ats:sizechecked:greenhouse:acme"] == TODAY
|
||||
|
||||
|
||||
def test_a_pull_leaves_a_kept_size_alone(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_wolf(store)
|
||||
_board(store, "greenhouse:acme", "Acme", size=250)
|
||||
|
||||
def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object
|
||||
) -> _runner.Listing:
|
||||
return _runner.Listing([], True, [])
|
||||
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
_pull(store)
|
||||
assert store.state["ats:size:greenhouse:acme"] == "250"
|
||||
|
||||
|
||||
# ---------------------------------------------------- the line and facets
|
||||
|
||||
WORKDAY = "workday:acme.wd3/Jobs"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("facets, line", [
|
||||
(None, "follows agents: DK, SE, remote · all roles"),
|
||||
({"country": {"DK": "c=dk", "SE": "c=se"}, "remote": {"remote": "r=1"}},
|
||||
"follows agents: DK, SE, remote · all roles"),
|
||||
({"country": {"DK": "c=dk"}},
|
||||
"follows agents: DK · SE not on this site · no remote option · all roles"),
|
||||
({"country": {"DK": "c=dk"}, "remote": {"remote": "r=1"}},
|
||||
"follows agents: DK, remote · SE not on this site · all roles"),
|
||||
({"country": {"DE": "c=de"}, "remote": {"remote": "r=1"}},
|
||||
"follows agents: remote · DK, SE not on this site · all roles"),
|
||||
({"country": {"DE": "c=de"}}, "site lists none of DK, SE, remote: nothing asked"),
|
||||
], ids=["none kept", "all known", "one country, no remote", "one country and remote",
|
||||
"remote only", "none"])
|
||||
def test_the_line_names_what_a_site_does_not_know(
|
||||
store: Store, facets: dict[str, dict[str, str]] | None, line: str) -> None:
|
||||
_wolf(store)
|
||||
_board(store, WORKDAY, "Acme", size=1000)
|
||||
if facets is not None:
|
||||
store.state[f"ats:facets:{WORKDAY}"] = json.dumps(
|
||||
{"checked": "2026-09-30T06:00:00+00:00", "facets": facets})
|
||||
plan = reach.plan(store)
|
||||
sub = _sub(store, WORKDAY)
|
||||
assert reach.line(plan, sub) == line
|
||||
# What the line says is left to ask is what the runner asks.
|
||||
ask = _runner.ask_for(workday, sub.params, plan.for_subscription(sub))
|
||||
assert (_runner.series(workday, ask, facets) == []) == line.endswith("nothing asked")
|
||||
|
||||
|
||||
def test_the_line_says_nothing_is_asked_in_the_plans_words(store: Store) -> None:
|
||||
_wolf(store, remote=False)
|
||||
_board(store, WORKDAY, "Acme", size=1000)
|
||||
store.state[f"ats:facets:{WORKDAY}"] = json.dumps(
|
||||
{"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DE": "c=de"}}})
|
||||
assert reach.line(reach.plan(store), _sub(store, WORKDAY)) == \
|
||||
"site lists none of DK, SE: nothing asked"
|
||||
|
||||
|
||||
def test_a_boards_own_reach_is_named_the_same_and_remote_alone_is_read_whole(
|
||||
store: Store) -> None:
|
||||
_wolf(store)
|
||||
_board(store, WORKDAY, "Acme", size=1000, params={"countries": ["DK", "NO"]})
|
||||
_board(store, "workday:other.wd3/Jobs", "Other", size=1000, params={"remote": True})
|
||||
for target in (WORKDAY, "workday:other.wd3/Jobs"):
|
||||
store.state[f"ats:facets:{target}"] = json.dumps(
|
||||
{"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DK": "c=dk"}}})
|
||||
plan = reach.plan(store)
|
||||
assert reach.line(plan, _sub(store, WORKDAY)) == \
|
||||
"set here: DK · NO not on this site · all roles"
|
||||
assert reach.line(plan, _sub(store, "workday:other.wd3/Jobs")) == \
|
||||
"all places, no remote option on this site · all roles"
|
||||
|
||||
|
||||
# ------------------------------------------------------ Recheck and the CLI
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path: Path) -> FastAPI:
|
||||
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
|
||||
database=tmp_path / "roledesk.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app: FastAPI) -> Iterator[TestClient]:
|
||||
with TestClient(app) as c:
|
||||
yield c
|
||||
|
||||
|
||||
def test_recheck_sizes_the_board_again_and_queues_a_pull(
|
||||
app: FastAPI, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store: Store = app.state.store
|
||||
_wolf(store)
|
||||
_board(store, BOSCH, "Bosch", size=120)
|
||||
store.state[f"ats:sizechecked:{BOSCH}"] = "2026-09-01"
|
||||
sub_id = _sub(store, BOSCH).id
|
||||
[row] = client.get("/api/companies").json()
|
||||
assert (row["size"], row["size_checked"], row["reach"]) == (
|
||||
120, "2026-09-01", "under 300 roles: read whole")
|
||||
|
||||
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830))
|
||||
res = client.post(f"/api/sources/{sub_id}/size")
|
||||
assert res.status_code == 200, res.text
|
||||
assert res.json() == {"size": 4830, "size_checked": TODAY,
|
||||
"reach": "follows agents: DK, SE · all roles"}
|
||||
assert app.state.queue.jobs.last("pull", states=("queued", "running", "done")) is not None
|
||||
[row] = client.get("/api/companies").json()
|
||||
assert (row["size"], row["size_checked"]) == (4830, TODAY)
|
||||
|
||||
# A board that does not answer keeps the size it had.
|
||||
monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True))
|
||||
res = client.post(f"/api/sources/{sub_id}/size")
|
||||
assert res.status_code == 422 and "did not answer" in res.json()["detail"]
|
||||
assert store.state[f"ats:size:{BOSCH}"] == "4830"
|
||||
assert client.post("/api/sources/9999/size").status_code == 404
|
||||
|
||||
|
||||
def _cli(tmp_path: Path, *argv: str) -> int:
|
||||
config = tmp_path / "config.yaml"
|
||||
if not config.exists():
|
||||
config.write_text(f"database: {tmp_path / 't.db'}\n")
|
||||
return main(["-c", str(config), "companies", *argv])
|
||||
|
||||
|
||||
def test_the_cli_sizes_one_board_or_every_board_without_a_size(
|
||||
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str]) -> None:
|
||||
_wolf(store)
|
||||
_board(store, BOSCH, "Bosch", size=12)
|
||||
_board(store, "smartrecruiters:tiny", "Tiny")
|
||||
_board(store, "smartrecruiters:gone", "Gone")
|
||||
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830))
|
||||
assert _cli(tmp_path, "size", "bosch") == 0
|
||||
assert json.loads(capsys.readouterr().out) == {
|
||||
"board": BOSCH, "size": 4830, "reach": "follows agents: DK, SE · all roles"}
|
||||
|
||||
def some(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
|
||||
if slug == "gone":
|
||||
raise OSError("connection refused")
|
||||
return SmartRecruiters(40)(client, url, host, slug)
|
||||
|
||||
monkeypatch.setattr(ats, "_get", some)
|
||||
assert _cli(tmp_path, "size", "--missing") == 1
|
||||
out, err = capsys.readouterr()
|
||||
assert out.splitlines() == [
|
||||
json.dumps({"board": "smartrecruiters:tiny", "size": 40,
|
||||
"reach": "under 300 roles: read whole"}),
|
||||
"sized 1 of 2 boards without a size"]
|
||||
assert "smartrecruiters:gone did not answer" in err
|
||||
assert store.state[f"ats:size:{BOSCH}"] == "4830"
|
||||
assert _cli(tmp_path, "size") == 2
|
||||
assert _cli(tmp_path, "size", "nobody") == 1
|
||||
@@ -127,8 +127,8 @@ def test_the_narrow_reach_takes_no_terms(store: Store) -> None:
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size, params", [
|
||||
(299, {}), (None, {}), (5000, {"countries": ["DE"]}),
|
||||
], ids=["small", "never read whole", "own reach"])
|
||||
(299, {}), (5000, {"countries": ["DE"]}),
|
||||
], ids=["small", "own reach"])
|
||||
def test_small_unknown_overridden_and_switched_off_boards_take_no_terms(
|
||||
store: Store, size: int | None, params: dict[str, object]) -> None:
|
||||
_wolf(store, TERMS)
|
||||
@@ -394,10 +394,10 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
|
||||
Tally(results=tuple(results)))
|
||||
|
||||
# No terms: Europe is too many countries for SmartRecruiters, so whole.
|
||||
# A whole read writes no size either: the board keeps the one it has.
|
||||
run()
|
||||
assert [_query(u).get("q") for u in asked] == [None]
|
||||
assert store.state["ats:size:smartrecruiters:bfl"] == "3"
|
||||
store.state["ats:size:smartrecruiters:bfl"] = "1000"
|
||||
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
|
||||
open_before = {r.source_id for r in store.roles(limit=50)}
|
||||
assert open_before == {f"smartrecruiters:bfl:{n}" for n in ("1", "2", "9")}
|
||||
|
||||
@@ -413,9 +413,6 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
|
||||
assert {r.source_id for r in store.roles(limit=50)} == open_before
|
||||
assert settled["closed"] == 0
|
||||
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
|
||||
# A board sized by its open roles is never sized by a term read.
|
||||
del store.state["ats:size:smartrecruiters:bfl"]
|
||||
assert reach.sizes(store, store.subscriptions(source="ats")) == {}
|
||||
|
||||
|
||||
# ------------------------------------------------------- storage and the API
|
||||
|
||||
+6
-6
@@ -185,14 +185,14 @@ def test_small_unknown_and_overridden_boards_get_no_reach(
|
||||
assert (found.params() is None) == (why != "agents")
|
||||
|
||||
|
||||
def test_a_board_read_before_sizes_were_kept_is_sized_by_its_open_roles(store: Store) -> None:
|
||||
def test_a_board_has_only_the_size_it_was_given(store: Store) -> None:
|
||||
_board(store, "workday:acme.wd3/Jobs", "Acme", size=None)
|
||||
[sub] = store.subscriptions(source="ats")
|
||||
# A board read whole before sizes were kept is not sized by its open roles.
|
||||
store.state["ats:ok:workday:acme.wd3/Jobs"] = "run-0"
|
||||
assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 0}
|
||||
# A filtered board's open roles are not its size.
|
||||
store.state["ats:filtered:workday:acme.wd3/Jobs"] = "1"
|
||||
assert reach.sizes(store, [sub]) == {}
|
||||
store.state["ats:size:workday:acme.wd3/Jobs"] = "12"
|
||||
assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 12}
|
||||
|
||||
|
||||
def test_describe_is_short() -> None:
|
||||
@@ -237,8 +237,8 @@ def test_the_pull_hands_each_board_its_reach_and_the_runner_narrows_it(
|
||||
assert asked["BoschGroup"] == Ask(countries=("DK", "SE"))
|
||||
assert asked["tiny"] == Ask()
|
||||
assert store.state["ats:filtered:smartrecruiters:BoschGroup"] == "1"
|
||||
# The small board, read whole, keeps its size; the filtered one keeps its last.
|
||||
assert store.state["ats:size:smartrecruiters:tiny"] == "1"
|
||||
# A pull rewrites no size: the small board read whole keeps its 12.
|
||||
assert store.state["ats:size:smartrecruiters:tiny"] == "12"
|
||||
assert store.state["ats:size:smartrecruiters:BoschGroup"] == "1000"
|
||||
|
||||
|
||||
|
||||
@@ -29,24 +29,24 @@ def store(tmp_path: Path) -> Store:
|
||||
|
||||
|
||||
class Boards:
|
||||
"""A fake `fetch_board`: `listed` maps a target to how many roles it
|
||||
lists; any other target does not answer. Remembers what was fetched."""
|
||||
"""A fake `board_size`: `listed` maps a target to how many roles it
|
||||
lists; any other target does not answer. Remembers what was sized."""
|
||||
|
||||
def __init__(self, **listed: int) -> None:
|
||||
self.listed = {k.replace("_", ":", 1): v for k, v in listed.items()}
|
||||
self.fetched: list[str] = []
|
||||
|
||||
def __call__(self, target: str) -> tuple[list[dict[str, Any]], bool]:
|
||||
def __call__(self, target: str, **kwargs: Any) -> int:
|
||||
self.fetched.append(target)
|
||||
if target not in self.listed:
|
||||
raise OSError("connection refused")
|
||||
return [{"source_id": str(i)} for i in range(self.listed[target])], True
|
||||
return self.listed[target]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def boards(monkeypatch: pytest.MonkeyPatch) -> Boards:
|
||||
fake = Boards(greenhouse_acme=3, ashby_acme=2, lever_quiet=0, lever_acme=1)
|
||||
monkeypatch.setattr(ats, "fetch_board", fake)
|
||||
monkeypatch.setattr(ats, "board_size", fake)
|
||||
return fake
|
||||
|
||||
|
||||
|
||||
+9
-8
@@ -603,7 +603,8 @@ def test_companies_lists_board_and_open_role_count(
|
||||
"careers_opened_at": "", "failing_since": "", "failing_reason": "",
|
||||
"from_search": False, "hand_reason": "", "by_hand": False,
|
||||
"all_countries": False, "all_roles": False,
|
||||
"reach_params": {}, "reach": "size not known yet: read whole"}
|
||||
"reach_params": {}, "reach": "size not known yet: sized on the next pull",
|
||||
"size": None, "size_checked": ""}
|
||||
[other] = [c for c in res.json() if c["key"] == "other"]
|
||||
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
|
||||
"", None, None, "")
|
||||
@@ -1371,7 +1372,7 @@ def test_a_board_is_followed_only_if_it_answers(
|
||||
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import plugins.ats as ats
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
|
||||
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 1)
|
||||
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
|
||||
assert res.status_code == 201
|
||||
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
|
||||
@@ -1389,10 +1390,10 @@ def test_a_board_is_followed_only_if_it_answers(
|
||||
bad = client.post("/api/boards", json={"board": "acme", "name": "Other"})
|
||||
assert bad.status_code == 422 and "host:slug" in bad.json()["detail"]
|
||||
|
||||
def down(target: str) -> list[dict[str, str]]:
|
||||
def down(target: str, **kw: object) -> int:
|
||||
raise OSError("connection refused")
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", down)
|
||||
monkeypatch.setattr(ats, "board_size", down)
|
||||
dead = client.post("/api/boards", json={"board": "ashby:other", "name": "Other"})
|
||||
assert dead.status_code == 422 and "did not answer" in dead.json()["detail"]
|
||||
assert store.company("other") is None
|
||||
@@ -1402,7 +1403,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
|
||||
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import plugins.ats as ats
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) # between openings
|
||||
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0) # between openings
|
||||
store.add_company("Acme", key="acme", ats="teamtailor", ats_slug="acme")
|
||||
|
||||
res = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"})
|
||||
@@ -1453,7 +1454,7 @@ def test_a_company_is_added_by_hand_with_a_reason_and_a_link(
|
||||
"country": "", "city": "", "watch": None, "careers_opened_at": "",
|
||||
"failing_since": "", "failing_reason": "", "from_search": False,
|
||||
"hand_reason": "added", "by_hand": True, "all_countries": False, "all_roles": False,
|
||||
"reach_params": {}, "reach": ""}
|
||||
"reach_params": {}, "reach": "", "size": None, "size_checked": ""}
|
||||
assert store.company("revolut").hand_reason == "added" # type: ignore[union-attr]
|
||||
|
||||
other = client.post("/api/companies", json={"name": "Meta", "hand_reason": "blocked"})
|
||||
@@ -1522,7 +1523,7 @@ def test_a_company_with_a_board_keeps_its_name_and_takes_the_other_edits(
|
||||
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import plugins.ats as ats
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True))
|
||||
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0)
|
||||
store.add_company("Acme", key="acme")
|
||||
assert client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"}
|
||||
).status_code == 200
|
||||
@@ -1563,7 +1564,7 @@ def test_a_company_is_deleted_unless_it_is_read(
|
||||
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import plugins.ats as ats
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True))
|
||||
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0)
|
||||
agent = store.save_agent("A")
|
||||
store.add_company("Meta", key="meta", hand_reason="blocked")
|
||||
store.set_affinity(agent.id, "meta", 1)
|
||||
|
||||
Reference in new issue
Block a user