Size a board once with board_size, before a pull reads it with no size, and name what a site's ids leave out
set_board sizes a board from page 1's total, or reads to the end on a host that states none, instead of reading it whole up to page 100. A pull no longer rewrites ats:size; a board with none is handed the agents' reach marked unsized and is sized in its batch first, or by its whole read on a host that cannot narrow the reach. reach.sizes drops the open-role fallback. The Companies line names countries a Workday or Oracle site does not list and a missing remote option. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a19c528e3f
commit
a0882e9659
8 files changed
+579
-67
No files matched your search
+113
-19
@@ -21,8 +21,13 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
|
||||
it skips the `any` gates; it does not widen what is fetched.
|
||||
- **Board override**: a board whose subscription `params` name a country, a
|
||||
place or remote gets no reach here; `ask_for` reads the params instead.
|
||||
- **Small boards**: a board that listed under `SMALL` roles when last read
|
||||
whole gets no filter; filtering it saves little and risks closing roles.
|
||||
- **Small boards**: a board of under `SMALL` roles gets no filter;
|
||||
filtering it saves little and risks closing roles. A board's size is set
|
||||
once, when it is added or rechecked (`plugins/ats.py`, `board_size`), and
|
||||
never by a pull. A board with no size yet is handed the reach it would
|
||||
have as a big one, with `unsized` and `small`, and its batch sizes it
|
||||
before it reads it, so a big new board is not read whole first; a board
|
||||
whose reader cannot narrow that reach is sized by its whole read instead.
|
||||
|
||||
A reach of more countries than a `one`-filter host should be asked one by
|
||||
one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`,
|
||||
@@ -43,14 +48,15 @@ board sends the roles that match one of them rather than every role:
|
||||
changes nothing on the narrow reach today, which takes no terms.
|
||||
- Only a reader that declares `query="one"` takes them, one series a term;
|
||||
`ask_for` has the rule for terms times the reach's series.
|
||||
- A read with terms is never the whole board: it sets no `ats:size`, and
|
||||
the pull closes nothing against it (`app/jobs/pull.py`).
|
||||
- A read with terms is never the whole board: the pull closes nothing
|
||||
against it (`app/jobs/pull.py`).
|
||||
- The search providers take none. Every saved search has a query of its
|
||||
own, which terms would replace; a search is a query already.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
@@ -65,9 +71,15 @@ from plugins.readers import _runner
|
||||
#: whole.
|
||||
SMALL = 300
|
||||
|
||||
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
|
||||
#: unfiltered read listed, written by `plugins/ats.py`.
|
||||
#: `state` keys of a board's size, `ats:size:<target>`, the roles it listed
|
||||
#: when it was sized, and of the day it was sized (ISO), both written by
|
||||
#: `plugins/ats.py` (`keep_size`).
|
||||
SIZE = "ats:size:"
|
||||
SIZE_CHECKED = "ats:sizechecked:"
|
||||
|
||||
#: `state` key of the host ids a board's site states (`{"checked",
|
||||
#: "facets"}`), kept by the runner (`_runner.facets_for`).
|
||||
FACETS = "ats:facets:"
|
||||
|
||||
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
|
||||
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote"))
|
||||
@@ -239,7 +251,9 @@ class Plan:
|
||||
"""The reach of one pull: `base` for every board, `wide` for the boards
|
||||
of the companies in `all_countries`, each board's size by target, the
|
||||
fetch `terms` of the wide reads, which skip the companies in
|
||||
`all_roles`, and `wide_names`, what the agents call the wide reach."""
|
||||
`all_roles`, `wide_names`, what the agents call the wide reach, and
|
||||
`facets`, the host ids the runner keeps for a board on a host that
|
||||
filters by them, for the Companies line."""
|
||||
|
||||
base: Reach
|
||||
wide: Reach = EVERYWHERE
|
||||
@@ -248,6 +262,7 @@ class Plan:
|
||||
terms: tuple[str, ...] = ()
|
||||
all_roles: frozenset[str] = frozenset()
|
||||
wide_names: str = ""
|
||||
facets: dict[str, dict[str, dict[str, str]]] = field(default_factory=dict)
|
||||
|
||||
def why(self, sub: Subscription) -> tuple[Reach, str]:
|
||||
"""The reach `sub` is read with, and why: `override`, `small`,
|
||||
@@ -260,6 +275,10 @@ class Plan:
|
||||
return EVERYWHERE, "unknown"
|
||||
if size < SMALL:
|
||||
return EVERYWHERE, "small"
|
||||
return self._big(sub)
|
||||
|
||||
def _big(self, sub: Subscription) -> tuple[Reach, str]:
|
||||
"""The reach of `sub` as a board of `SMALL` roles or more."""
|
||||
if company_key(sub.company or sub.label) in self.all_countries:
|
||||
return self.wide, "countries"
|
||||
return self.base, "agents"
|
||||
@@ -274,30 +293,51 @@ class Plan:
|
||||
fetch terms, and unless its company is set to All roles, a board on
|
||||
the wide reach carries them and `wide`, and a board on the agents'
|
||||
reach carries them for `ask_for` to use if it drops the countries as
|
||||
too many (see the module docstring)."""
|
||||
too many (see the module docstring). A board with no size carries
|
||||
`unsized` and `small`, and the reach of a big board unless its params
|
||||
set their own, for its batch to size it first."""
|
||||
found, why = self.why(sub)
|
||||
if why == "unknown":
|
||||
found, why = self._big(sub)
|
||||
out = found.params()
|
||||
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
|
||||
out = (out or {}) | {"terms": list(self.terms)}
|
||||
if why == "countries":
|
||||
out["wide"] = True
|
||||
if sub.target not in self.sizes:
|
||||
out = (out or {}) | {"unsized": True, "small": SMALL}
|
||||
return out
|
||||
|
||||
|
||||
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
||||
"""Each board's size: the roles its last unfiltered read listed
|
||||
(`ats:size:<target>`), else, for a board read before that was kept and
|
||||
never filtered, its open roles. A board never read has none."""
|
||||
"""Each board's size as kept (`ats:size:<target>`); a board never sized
|
||||
has none."""
|
||||
state = store.state
|
||||
counts = store.open_countries_by_subscription()
|
||||
out: dict[str, int] = {}
|
||||
for sub in subs:
|
||||
kept = state.get(f"{SIZE}{sub.target}", "")
|
||||
if kept.isdigit():
|
||||
out[sub.target] = int(kept)
|
||||
elif (state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}")
|
||||
and not state.get(f"ats:narrowed:{sub.target}")):
|
||||
out[sub.target] = sum(counts.get(sub.id, {}).values())
|
||||
return out
|
||||
|
||||
|
||||
def site_facets(store: Store, subs: Iterable[Subscription]) -> dict[str, dict[str, dict[str, str]]]:
|
||||
"""The host ids the runner keeps for each board on a host whose country
|
||||
or remote filter is `ids` (`ats:facets:<target>`); a board with none kept
|
||||
is left out."""
|
||||
out: dict[str, dict[str, dict[str, str]]] = {}
|
||||
for sub in subs:
|
||||
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||
if module is None or not _runner.declared(module) \
|
||||
or "ids" not in (module.READS.filters.country, module.READS.filters.remote):
|
||||
continue
|
||||
try:
|
||||
kept = json.loads(store.state.get(f"{FACETS}{sub.target}") or "{}")
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
found = kept.get("facets") if isinstance(kept, dict) else None
|
||||
if isinstance(found, dict):
|
||||
out[sub.target] = found
|
||||
return out
|
||||
|
||||
|
||||
@@ -321,7 +361,8 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
|
||||
all_countries=frozenset(c.key for c in rows if c.all_countries),
|
||||
terms=fetch_terms(agents),
|
||||
all_roles=frozenset(c.key for c in rows if c.all_roles),
|
||||
wide_names=wide_names(agents, gaz) if agents else "everywhere")
|
||||
wide_names=wide_names(agents, gaz) if agents else "everywhere",
|
||||
facets=site_facets(store, subs))
|
||||
|
||||
|
||||
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
|
||||
@@ -338,14 +379,58 @@ _WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
|
||||
"agents": "an agent takes any place"}
|
||||
|
||||
|
||||
def _codes(codes: Iterable[str]) -> list[str]:
|
||||
"""Country codes as the line names them: up to four, else a count."""
|
||||
found = sorted(codes)
|
||||
return found if len(found) <= 4 else [f"{len(found)} countries"]
|
||||
|
||||
|
||||
def _places(ask: readers.Ask) -> str:
|
||||
words = sorted(ask.countries) if len(ask.countries) <= 4 \
|
||||
else [f"{len(ask.countries)} countries"]
|
||||
words = _codes(ask.countries)
|
||||
if ask.remote:
|
||||
words.append("remote")
|
||||
return ", ".join(words)
|
||||
|
||||
|
||||
def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] | None,
|
||||
lead: str, roles: str) -> str:
|
||||
"""The line of a board whose site does not know every value `ask` wants
|
||||
on an `ids` filter, from the host ids the runner keeps (`facets`, None
|
||||
before any are kept), as `_runner.series` drops them: what is asked,
|
||||
then what the site does not know ("DK · SE not on this site · no remote
|
||||
option"), after `lead` and before `roles`; or, with nothing left to
|
||||
ask, "site lists none of DK, SE: nothing asked". "" when the site knows
|
||||
every value, or `lead` is "" and something is still asked."""
|
||||
if facets is None:
|
||||
return ""
|
||||
reads = module.READS.filters
|
||||
known, missing = list(ask.countries), []
|
||||
if reads.country == "ids" and ask.countries and "country" in facets:
|
||||
listed = facets.get("country") or {}
|
||||
known = [c for c in ask.countries if c in listed]
|
||||
missing = [c for c in ask.countries if c not in listed]
|
||||
no_remote = bool(ask.remote) and reads.remote == "ids" and not facets.get("remote")
|
||||
if not (missing or no_remote):
|
||||
return ""
|
||||
if no_remote and not (ask.countries or ask.places) and "remote" not in facets:
|
||||
# Remote alone, on a site that states no remote facet, is read whole.
|
||||
return f"all places, no remote option on this site · {roles}" if lead else ""
|
||||
remote = bool(ask.remote) and not no_remote
|
||||
if not (known or ask.places or remote):
|
||||
if not missing:
|
||||
return "site has no remote option: nothing asked"
|
||||
named = _codes(missing) + (["remote"] if no_remote else [])
|
||||
return f"site lists none of {', '.join(named)}: nothing asked"
|
||||
if not lead:
|
||||
return ""
|
||||
words = [", ".join(_codes(known) + list(ask.places) + (["remote"] if remote else []))]
|
||||
if missing:
|
||||
words.append(f"{', '.join(_codes(missing))} not on this site")
|
||||
if no_remote:
|
||||
words.append("no remote option")
|
||||
return f"{lead}: {' · '.join(w for w in words if w)} · {roles}"
|
||||
|
||||
|
||||
def _roles(ask: readers.Ask) -> str:
|
||||
"""The role-type half of the line: the agents' terms the board is
|
||||
asked for, or all roles."""
|
||||
@@ -363,14 +448,23 @@ def line(found: Plan, sub: Subscription) -> str:
|
||||
· all roles", "set here: DE · all roles", "all places, too many countries
|
||||
to ask · follows agents' terms: engineer", or, for a board read whole,
|
||||
why: "under 300 roles: read whole", "Greenhouse cannot filter: read
|
||||
whole"."""
|
||||
whole". A board with no size yet says it is sized on the next pull. On
|
||||
a host that filters by its own ids, the line names what the site does
|
||||
not know once the runner has asked it (`_site`): "follows agents: DK ·
|
||||
SE not on this site · no remote option · all roles", or "site lists
|
||||
none of DK, SE: nothing asked"."""
|
||||
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||
if module is None:
|
||||
return ""
|
||||
given, why = found.why(sub)
|
||||
if why == "unknown":
|
||||
return f"{_WHOLE[why]}: sized on the next pull"
|
||||
ask = _runner.ask_for(module, sub.params, found.for_subscription(sub))
|
||||
roles = _roles(ask)
|
||||
if _runner.filtered(ask):
|
||||
lead = {"agents": "follows agents", "override": "set here"}.get(why, "")
|
||||
if site := _site(module, ask, found.facets.get(sub.target), lead, roles):
|
||||
return site
|
||||
if why == "countries":
|
||||
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
||||
return f"all countries ({named}) · {roles}"
|
||||
|
||||
+29
-10
@@ -46,10 +46,11 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
|
||||
"""Read company `key` from `board` (`host:slug`): attach it to a company
|
||||
that has none, replace the one it has, or create the company from `name`
|
||||
when there is no row. The board is fetched first and refused if its host
|
||||
is unknown or it does not answer; a board listing zero roles is kept, since
|
||||
a company between openings is still one to follow. `check=False` skips the
|
||||
fetch, for the seed, whose boards were read once when it was written.
|
||||
when there is no row. The board is sized first (`board_size`) and refused
|
||||
if its host is unknown or it does not answer; a board listing zero roles
|
||||
is kept, since a company between openings is still one to follow.
|
||||
`check=False` skips the request, for the seed, whose boards were read
|
||||
once when it was written; such a board is sized by its first pull.
|
||||
|
||||
The row's `ats` and `ats_slug` become the board's host and slug, its
|
||||
`careers_url` the board's page where it was empty or the old board's, and
|
||||
@@ -59,7 +60,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
change in one transaction (`Store.set_company_board`), and replacing a
|
||||
board closes the old one's open roles. Returns the subscription, how many
|
||||
roles the board lists (None unchecked) and how many roles closed."""
|
||||
from plugins.ats import careers_url, fetch_board, is_known_target
|
||||
from plugins.ats import board_size, careers_url, is_known_target, keep_size
|
||||
|
||||
board, name = board.strip(), name.strip()
|
||||
key = key.strip() or company_key(name)
|
||||
@@ -72,10 +73,10 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
raise ValueError(f"no company {key!r}; a new company needs a name")
|
||||
if company_key(name) != key:
|
||||
raise ValueError(f"{name!r} is not the company {key!r}")
|
||||
rows: list[Any] | None = None
|
||||
size: int | None = None
|
||||
if check:
|
||||
try:
|
||||
rows, _complete = fetch_board(board)
|
||||
size = board_size(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
host, _, slug = board.partition(":")
|
||||
@@ -95,7 +96,23 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
assert sub is not None
|
||||
if before is not None and before.target != board:
|
||||
_forget_failing(store, before.target)
|
||||
return sub, None if rows is None else len(rows), closed
|
||||
if size is not None:
|
||||
keep_size(store.state, board, size)
|
||||
return sub, size, closed
|
||||
|
||||
|
||||
def size_board(store: Store, board: str) -> int:
|
||||
"""Size `board` again (`plugins.ats.board_size`) and keep it with the
|
||||
day: how a board's size changes once it was added (a Recheck, `app
|
||||
companies size`). Raises ValueError when it does not answer."""
|
||||
from plugins.ats import board_size, keep_size
|
||||
|
||||
try:
|
||||
size = board_size(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
keep_size(store.state, board, size)
|
||||
return size
|
||||
|
||||
|
||||
def detach_board(store: Store, key: str) -> int:
|
||||
@@ -118,9 +135,11 @@ def _forget_failing(store: Store, target: str) -> None:
|
||||
record of `target` failing would otherwise outlive the board, and count
|
||||
in `app companies check` for good. The `ats` plugin's robots.txt verdict
|
||||
and its lists of roles read and failed in detail go with it, since they
|
||||
are about that board alone and would only grow stale."""
|
||||
are about that board alone and would only grow stale, as would its size,
|
||||
which a board attached again gets anew."""
|
||||
store.state.pop(FAILING + target, None)
|
||||
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
|
||||
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:", "ats:size:",
|
||||
"ats:sizechecked:"):
|
||||
store.state.pop(key + target, None)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user