Merge main into wp-review-limits
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
750d3d1311
62 files changed
+1511
-471
No files matched your search
+10
-2
@@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
|||||||
|
|
||||||
Under `NULL` there is no queue to take the chain, so this raises rather
|
Under `NULL` there is no queue to take the chain, so this raises rather
|
||||||
than report a pull that never happened; `app job pull` claims the run and
|
than report a pull that never happened; `app job pull` claims the run and
|
||||||
starts the chain on a queue of its own instead of calling this.
|
starts the chain on a queue of its own instead of calling this. Anything
|
||||||
|
that raises once the run is claimed, building the chain or handing it
|
||||||
|
over, releases the claim first, or the next pull would wait
|
||||||
|
`pull.ABANDONED` for a run that never started.
|
||||||
"""
|
"""
|
||||||
run = pull.claim_run(store)
|
run = pull.claim_run(store)
|
||||||
if not run:
|
if not run:
|
||||||
since = pull.running(store)
|
since = pull.running(store)
|
||||||
context.notice(already_running(since))
|
context.notice(already_running(since))
|
||||||
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
||||||
if not context.task(pull_chain(run)):
|
try:
|
||||||
|
started = context.task(pull_chain(run))
|
||||||
|
except Exception:
|
||||||
|
pull.release_run(store, run)
|
||||||
|
raise
|
||||||
|
if not started:
|
||||||
pull.release_run(store, run)
|
pull.release_run(store, run)
|
||||||
raise RuntimeError("no queue to start the pull cascade on")
|
raise RuntimeError("no queue to start the pull cascade on")
|
||||||
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
||||||
|
|||||||
+4
-13
@@ -10,7 +10,8 @@ parses) becomes a `board` target the pull can read; any other host is
|
|||||||
recorded as `ats`, with its slug where the URL carries one, so the company is
|
recorded as `ats`, with its slug where the URL carries one, so the company is
|
||||||
kept as one the app cannot read yet rather than dropped. The patterns and
|
kept as one the app cannot read yet rather than dropped. The patterns and
|
||||||
probes of a readable host are its module's; the hosts only named are in
|
probes of a readable host are its module's; the hosts only named are in
|
||||||
`_UNREADABLE` here, and one of them that gains a module is read from then on.
|
`_UNREADABLE` here, and a host that gains a module loses its entry there
|
||||||
|
(`tests/test_discover.py` holds that no entry names a reader).
|
||||||
|
|
||||||
A Teamtailor site on the company's own domain names no slug: the page only
|
A Teamtailor site on the company's own domain names no slug: the page only
|
||||||
loads Teamtailor's CDN. The board there is the site's own host, and it is
|
loads Teamtailor's CDN. The board there is the site's own host, and it is
|
||||||
@@ -65,27 +66,17 @@ READABLE: tuple[str, ...] = tuple(readers.HOSTS)
|
|||||||
#: company by, kept for the day a parser exists; a host with no group is
|
#: company by, kept for the day a parser exists; a host with no group is
|
||||||
#: recognised but its board has no slug the app could use.
|
#: recognised but its board has no slug the app could use.
|
||||||
_UNREADABLE: tuple[tuple[str, str], ...] = (
|
_UNREADABLE: tuple[tuple[str, str], ...] = (
|
||||||
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
|
|
||||||
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
|
|
||||||
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
|
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
|
||||||
("taleo", r"\.taleo\.net"),
|
("taleo", r"\.taleo\.net"),
|
||||||
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
|
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
|
||||||
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
|
|
||||||
("join", r"join\.com/companies/([a-z0-9-]+)"),
|
("join", r"join\.com/companies/([a-z0-9-]+)"),
|
||||||
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
|
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
|
||||||
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
|
|
||||||
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
|
|
||||||
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
|
|
||||||
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
|
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
|
||||||
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
|
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
|
||||||
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
|
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
|
||||||
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
|
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
|
||||||
("eightfold", r"\.eightfold\.ai"),
|
|
||||||
("phenom", r"phenompeople\.com|phenom\.com/careers"),
|
|
||||||
("avature", r"\.avature\.net"),
|
("avature", r"\.avature\.net"),
|
||||||
("oracle", r"oraclecloud\.com/hcmUI"),
|
|
||||||
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
|
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
|
||||||
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -94,8 +85,8 @@ def _hosts(modules: Mapping[str, ModuleType],
|
|||||||
"""Every host's patterns, compiled: the readable hosts' from their
|
"""Every host's patterns, compiled: the readable hosts' from their
|
||||||
modules first, so the first match at one place on a page is the one the
|
modules first, so the first match at one place on a page is the one the
|
||||||
app can read, then the unreadable ones. An unreadable host that has
|
app can read, then the unreadable ones. An unreadable host that has
|
||||||
gained a module is left out here, since its module's patterns replace
|
a module is left out, so its module's patterns win even before its
|
||||||
it."""
|
entry is deleted."""
|
||||||
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
|
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
|
||||||
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
|
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
|
||||||
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
|
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
|
||||||
|
|||||||
+237
-62
@@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see
|
|||||||
`plugins/readers/_runner.py`).
|
`plugins/readers/_runner.py`).
|
||||||
|
|
||||||
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
||||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
|
`app.jobs.reach`, or its subscription's `params`) lists only the roles in
|
||||||
person acted on: those with a vote or an application (`_close_unlisted`). A
|
reach, so its close keeps open the roles a person acted on: those with a vote
|
||||||
filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
or an application (`_close_unlisted`). When the reach wanted remote roles and
|
||||||
under a tenth of what the same read listed last time is recorded by the
|
the reader could not ask for them (`ats:remote_unasked:<target>`), the read
|
||||||
runner as a failure, not as a list to close against.
|
says nothing of the remote roles either, so they stay open too and close only
|
||||||
|
once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
|
||||||
|
that lists under a tenth of what the same read listed last time is recorded
|
||||||
|
by the runner as a failure, not as a list to close against (`_runner.read`).
|
||||||
|
|
||||||
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
||||||
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
||||||
@@ -33,14 +36,21 @@ not swept clears them.
|
|||||||
|
|
||||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||||
the roles matching a word, which says nothing of the roles it did not list,
|
the roles matching a word, which says nothing of the roles it did not list,
|
||||||
so it closes nothing; only an unfiltered or a place-filtered complete read
|
so it closes by list nothing: a role of such a board closes once the board
|
||||||
closes.
|
has not listed it for `STALE_DAYS`, the rule searches have.
|
||||||
|
|
||||||
A search closes the same way when it listed everything that matches (see
|
A search closes the same way when it listed everything that matches (see
|
||||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
roles only once no search has seen them for `STALE_DAYS`. A board or a
|
||||||
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
|
||||||
closes all of its roles, and its next answer reopens what it lists.
|
`search:failing:<target>`) closes all of its roles, and its next answer
|
||||||
|
reopens what it lists. A target that raises in its batch after its read, a
|
||||||
|
row that does not store, is recorded as failing the same way and the batch
|
||||||
|
goes on with the next one (`run_batch`).
|
||||||
|
|
||||||
|
The settle says what needs a look in one notice (`settled`): the targets
|
||||||
|
that did not answer or were read incomplete, and the boards that closed more
|
||||||
|
than `CLOSE_NOTICE` roles or more than half of theirs.
|
||||||
|
|
||||||
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
||||||
set only while the previous run is settled, in one statement), so the
|
set only while the previous run is settled, in one statement), so the
|
||||||
@@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta
|
|||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from pipeline import NULL, RunContext, Tally, Task, job
|
from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
|
||||||
|
|
||||||
from app.config import Config
|
from app.config import Config
|
||||||
from app.jobs.reach import reaches
|
from app.jobs.reach import reaches
|
||||||
@@ -91,6 +101,19 @@ STALE_DAYS = 21
|
|||||||
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
||||||
FAILING = "ats:failing:"
|
FAILING = "ats:failing:"
|
||||||
|
|
||||||
|
#: The same for a search, kept by the `search` plugin.
|
||||||
|
SEARCH_FAILING = "search:failing:"
|
||||||
|
|
||||||
|
#: A board that closes more roles than this in one settle is named in the
|
||||||
|
#: settle's notice, and so is one that closes more than half of its open
|
||||||
|
#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
|
||||||
|
#: one is an ordinary day.
|
||||||
|
CLOSE_NOTICE = 50
|
||||||
|
CLOSE_SHARE_MIN = 10
|
||||||
|
|
||||||
|
#: How many targets a notice names in its text; its detail names them all.
|
||||||
|
NOTICE_NAMES = 10
|
||||||
|
|
||||||
#: The `state` key that records the starter searches were imported.
|
#: The `state` key that records the starter searches were imported.
|
||||||
SEARCHES_SEEDED = "searches:seeded"
|
SEARCHES_SEEDED = "searches:seeded"
|
||||||
|
|
||||||
@@ -142,9 +165,14 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
|
|||||||
`close_boards` compares against. `run` is a marker the caller claimed
|
`close_boards` compares against. `run` is a marker the caller claimed
|
||||||
already (the `pull` job, `app job pull`); without one the finder claims
|
already (the `pull` job, `app job pull`); without one the finder claims
|
||||||
its own, and queues nothing when another run is in flight. A run with
|
its own, and queues nothing when another run is in flight. A run with
|
||||||
nothing to read is settled here, since no settle will see a batch of it."""
|
nothing to read is settled here, since no settle will see a batch of it,
|
||||||
|
and so is a run whose finder raises: no batch and no settle would follow,
|
||||||
|
and the next pull would wait `ABANDONED` for a claim nobody holds."""
|
||||||
|
held = run
|
||||||
|
try:
|
||||||
seed_searches(config, store)
|
seed_searches(config, store)
|
||||||
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
|
subs = {source: store.subscriptions(source=source, fetchable=True)
|
||||||
|
for source in SOURCES}
|
||||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||||
if not any(found for _, found in targets):
|
if not any(found for _, found in targets):
|
||||||
if run:
|
if run:
|
||||||
@@ -155,14 +183,20 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
|
|||||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||||
running(store))
|
running(store))
|
||||||
return []
|
return []
|
||||||
|
held = run
|
||||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||||
reach_of = reaches(store, subs["ats"])
|
reach_of = reaches(store, subs["ats"])
|
||||||
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
batches: list[dict[str, Any]] = [
|
||||||
|
{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||||
for batch in batches:
|
for batch in batches:
|
||||||
if batch["source"] == "ats":
|
if batch["source"] == "ats":
|
||||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||||
return batches
|
return batches
|
||||||
|
except Exception:
|
||||||
|
if held:
|
||||||
|
release_run(store, held)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
@job("pull_batch", "Reading boards", resource="http")
|
@job("pull_batch", "Reading boards", resource="http")
|
||||||
@@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
|||||||
plugin as its `run` option, so its stamps belong to that run whatever the
|
plugin as its `run` option, so its stamps belong to that run whatever the
|
||||||
stored marker says by the time the batch runs. `reach` is `target ->` the
|
stored marker says by the time the batch runs. `reach` is `target ->` the
|
||||||
reach the run computed for a board (`app.jobs.reach`), handed to the
|
reach the run computed for a board (`app.jobs.reach`), handed to the
|
||||||
plugin as its `reach` option.
|
plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
|
||||||
|
which the runner's 10% guard accepts the answer it kept refusing.
|
||||||
|
|
||||||
|
Each target is guarded on its own. A read that fails is the plugin's to
|
||||||
|
record; anything raised after it, a row `upsert_role` refuses or a bug
|
||||||
|
past the read, is recorded here under `<source>:failing:<target>` with
|
||||||
|
its reason, and the batch goes on with the next target. Otherwise one
|
||||||
|
bad row would cost the targets after it in the batch their read on every
|
||||||
|
pull, since batches are stable, and leave no trace on any of them. A
|
||||||
|
Kill (`Cancelled`) still ends the batch.
|
||||||
"""
|
"""
|
||||||
from plugins.ats import ATSSource
|
from plugins.ats import ATSSource
|
||||||
|
from plugins.readers import _runner
|
||||||
from plugins.search import SearchSource
|
from plugins.search import SearchSource
|
||||||
|
|
||||||
# Fetchable now, not when the pull was queued: a board switched off on the
|
# Fetchable now, not when the pull was queued: a board switched off on the
|
||||||
@@ -200,10 +244,14 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
|||||||
if sub is None:
|
if sub is None:
|
||||||
continue
|
continue
|
||||||
context.check()
|
context.check()
|
||||||
options = dict(config.sources.get(source) or {}) | {"run": run}
|
options = dict(config.sources.get(source) or {}) | {"run": run,
|
||||||
|
"stale_days": STALE_DAYS}
|
||||||
|
items: Any = None
|
||||||
|
try:
|
||||||
plugin: ATSSource | SearchSource
|
plugin: ATSSource | SearchSource
|
||||||
if source == "search":
|
if source == "search":
|
||||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
options["searches"] = [{"url": target, "label": sub.label,
|
||||||
|
"params": sub.params}]
|
||||||
plugin = SearchSource(options)
|
plugin = SearchSource(options)
|
||||||
else:
|
else:
|
||||||
# A board's params are its filter; a reader's key is kept with
|
# A board's params are its filter; a reader's key is kept with
|
||||||
@@ -214,10 +262,21 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
|||||||
options["reach"] = (reach or {}).get(target)
|
options["reach"] = (reach or {}).get(target)
|
||||||
plugin = ATSSource(options)
|
plugin = ATSSource(options)
|
||||||
plugin.state = store.state
|
plugin.state = store.state
|
||||||
for item in plugin.fetch():
|
items = plugin.fetch()
|
||||||
|
for item in items:
|
||||||
item.source = source
|
item.source = source
|
||||||
store.upsert_role(item, sub)
|
store.upsert_role(item, sub)
|
||||||
stored += 1
|
stored += 1
|
||||||
|
except Cancelled:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("pull: %s failed after its read (%s); on to the next target",
|
||||||
|
target, exc, exc_info=True)
|
||||||
|
_runner.record_failure(store.state, f"{source}:failing:{target}", exc)
|
||||||
|
finally:
|
||||||
|
# The plugin's generator holds its HTTP client until it is closed.
|
||||||
|
if items is not None and hasattr(items, "close"):
|
||||||
|
items.close()
|
||||||
return {"roles": stored, "run": run}
|
return {"roles": stored, "run": run}
|
||||||
|
|
||||||
|
|
||||||
@@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str:
|
|||||||
def close_boards(config: Config, store: Store, tally: Tally, *,
|
def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||||
context: RunContext = NULL) -> dict[str, Any]:
|
context: RunContext = NULL) -> dict[str, Any]:
|
||||||
"""The run's settle: close every board this run answered in full and
|
"""The run's settle: close every board this run answered in full and
|
||||||
stopped listing a role from, and tally what happened.
|
stopped listing a role from, tally what happened, and say what needs a
|
||||||
|
look.
|
||||||
|
|
||||||
Reads `self.state` rather than the batch jobs' own results, because a
|
Reads `self.state` rather than the batch jobs' own results, because a
|
||||||
`pull_batch` job that raised partway through still leaves the boards it
|
`pull_batch` job that raised partway through still leaves the boards it
|
||||||
@@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||||
whole batch job never got to it — is never closed by its list, and a board
|
whole batch job never got to it — is never closed by its list, and a board
|
||||||
that answered incompletely is not either. A search closes by its list only
|
that answered incompletely is not either. A search closes by its list only
|
||||||
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
|
||||||
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
search that has failed on every fetch since `STALE_DAYS` ago closes all
|
||||||
|
its roles (`failing_boards`). How a read that answered closes is
|
||||||
|
`_close_read`'s.
|
||||||
|
|
||||||
The stamps are compared with this run's own marker, the one its batches
|
The stamps are compared with this run's own marker, the one its batches
|
||||||
carried, never with the stored `pull:run`, which a later run may have
|
carried, never with the stored `pull:run`, which a later run may have
|
||||||
replaced. A settle with no batch behind it (a refused pull, nothing to
|
replaced. A settle with no batch behind it (a refused pull, nothing to
|
||||||
read) closes nothing by list or by age. Last, the run is marked settled,
|
read) closes nothing by list or by age and says nothing. Last, the run is
|
||||||
which lets the next pull start.
|
marked settled, which lets the next pull start.
|
||||||
|
|
||||||
A board or search switched off while its batch was in flight had its
|
A board or search switched off while its batch was in flight had its
|
||||||
roles reopened by that batch, after the switch closed them, so every
|
roles reopened by that batch, after the switch closed them, so every
|
||||||
switched-off one is closed again here.
|
switched-off one is closed again here; those closes are asked for and not
|
||||||
|
named in the notice.
|
||||||
|
|
||||||
|
The counts go into the result (`unanswered` counts boards and searches
|
||||||
|
whose stamp is not this run's), and one notice (`settled`) names every
|
||||||
|
target unanswered, every board read incomplete other than one mid-sweep,
|
||||||
|
and every board that closed more than `CLOSE_NOTICE` roles or more than
|
||||||
|
half the open roles it had, when there is any. The result alone was
|
||||||
|
seen by nobody: a settle closed 12,588 roles with nothing on screen.
|
||||||
|
|
||||||
The `prep` stage that follows maps what this run stored and folds the
|
The `prep` stage that follows maps what this run stored and folds the
|
||||||
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
||||||
and the roles this run added join theirs.
|
and the roles this run added join theirs.
|
||||||
"""
|
"""
|
||||||
answered = searched = failed = incomplete = closed = 0
|
answered = searched = incomplete = closed = 0
|
||||||
|
unanswered: list[str] = []
|
||||||
|
partial: list[str] = []
|
||||||
|
many: list[tuple[str, int, int]] = []
|
||||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||||
failing = failing_boards(store)
|
failing = failing_boards(store)
|
||||||
run_marker = _run_of(tally, context)
|
run_marker = _run_of(tally, context)
|
||||||
|
open_before = _open_counts(store) if run_marker else {}
|
||||||
for source in SOURCES:
|
for source in SOURCES:
|
||||||
for sub in store.subscriptions(source=source):
|
for sub in store.subscriptions(source=source):
|
||||||
if not (sub.subscribed and sub.fetch):
|
if not (sub.subscribed and sub.fetch):
|
||||||
@@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
for sub in store.subscriptions(source=source, fetchable=True):
|
for sub in store.subscriptions(source=source, fetchable=True):
|
||||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||||
if not (stamp and stamp == run_marker):
|
if not (stamp and stamp == run_marker):
|
||||||
failed += 1
|
unanswered.append(sub.target)
|
||||||
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
since = failing.get(sub.target, ("", ""))[0]
|
||||||
if since and since < stale:
|
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
||||||
closed += store.close_subscription(sub.id)
|
else:
|
||||||
continue
|
|
||||||
if source == "search":
|
if source == "search":
|
||||||
searched += 1
|
searched += 1
|
||||||
else:
|
else:
|
||||||
answered += 1
|
answered += 1
|
||||||
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
read = _close_read(store, source, sub, stale)
|
||||||
|
if read is None:
|
||||||
if source == "ats":
|
if source == "ats":
|
||||||
incomplete += 1
|
incomplete += 1
|
||||||
|
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||||
|
partial.append(sub.target)
|
||||||
continue
|
continue
|
||||||
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
|
shut = read
|
||||||
continue
|
closed += shut
|
||||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
before = open_before.get(sub.id, 0)
|
||||||
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
|
if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
|
||||||
if store.state.get(f"{source}:swept:{sub.target}", ""):
|
many.append((sub.target, shut, before))
|
||||||
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
|
|
||||||
continue
|
|
||||||
store.state.pop(f"{source}:missing:{sub.target}", None)
|
|
||||||
if is_filtered:
|
|
||||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
|
||||||
else:
|
|
||||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
|
||||||
if searched:
|
if searched:
|
||||||
closed += store.close_unseen("search", stale)
|
closed += store.close_unseen("search", stale)
|
||||||
if run_marker:
|
if run_marker:
|
||||||
|
if unanswered or partial or many:
|
||||||
|
context.notice(settled(unanswered, partial, many, failing,
|
||||||
|
boards=answered, searches=searched, closed=closed))
|
||||||
release_run(store, run_marker)
|
release_run(store, run_marker)
|
||||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||||
return {"boards": answered, "searches": searched, "failed": failed,
|
return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
|
||||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||||
|
|
||||||
|
|
||||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||||
|
"""Close what one target's read of this run says has closed, and return
|
||||||
|
how many; None when the read was incomplete, which closes nothing.
|
||||||
|
|
||||||
|
A read narrowed by terms closes only the roles it has not listed for
|
||||||
|
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
||||||
|
two sweeps in a row missed (`_close_swept`); any other complete read
|
||||||
|
clears that record and closes what it did not list, all of it unfiltered
|
||||||
|
(`Store.mark_closed`), filtered sparing the roles acted on
|
||||||
|
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
||||||
|
(`remote_unasked`) spares the remote ones as well, and closes those by
|
||||||
|
age instead."""
|
||||||
|
state, target = store.state, sub.target
|
||||||
|
if not state.get(f"{source}:complete:{target}", ""):
|
||||||
|
return None
|
||||||
|
seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
|
||||||
|
if state.get(f"{source}:narrowed:{target}", ""):
|
||||||
|
return _close_aged(store, source, sub.id, seen_ids, stale)
|
||||||
|
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
||||||
|
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
||||||
|
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
||||||
|
if spare_remote else 0)
|
||||||
|
if state.get(f"{source}:swept:{target}", ""):
|
||||||
|
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
||||||
|
spare_remote=spare_remote)
|
||||||
|
state.pop(f"{source}:missing:{target}", None)
|
||||||
|
if is_filtered:
|
||||||
|
return aged + _close_unlisted(store, source, seen_ids, sub.id,
|
||||||
|
spare_remote=spare_remote)
|
||||||
|
return store.mark_closed(source, seen_ids, sub.id)
|
||||||
|
|
||||||
|
|
||||||
|
def _open_counts(store: Store) -> dict[int, int]:
|
||||||
|
"""`subscription id -> open roles`, before the settle closes any, for the
|
||||||
|
share a board closed."""
|
||||||
|
with store.transaction() as conn:
|
||||||
|
return {int(row[0]): int(row[1]) for row in conn.execute(
|
||||||
|
"""SELECT subscription_id, count(*) FROM roles
|
||||||
|
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
||||||
|
GROUP BY subscription_id""")}
|
||||||
|
|
||||||
|
|
||||||
|
def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
|
||||||
|
failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
|
||||||
|
closed: int) -> Notice:
|
||||||
|
"""The notice of a settle that has something to look at: the targets
|
||||||
|
that did not answer (with the reason their failing record gives), the
|
||||||
|
boards read incomplete, and the boards that closed many roles, each
|
||||||
|
as `closed of open`, with the totals. The text names the first
|
||||||
|
`NOTICE_NAMES` of each, the detail all of them, one a line."""
|
||||||
|
def named(target: str) -> str:
|
||||||
|
reason = failing.get(target, ("", ""))[1]
|
||||||
|
return f"{target} ({reason})" if reason else target
|
||||||
|
|
||||||
|
groups = [("Not answering", [named(t) for t in unanswered]),
|
||||||
|
("Read incomplete", list(incomplete)),
|
||||||
|
("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
|
||||||
|
text = [f"The pull read {boards} boards and {searches} searches and closed "
|
||||||
|
f"{closed:,} roles."]
|
||||||
|
detail: list[str] = []
|
||||||
|
for title, names in groups:
|
||||||
|
if not names:
|
||||||
|
continue
|
||||||
|
shown = ", ".join(names[:NOTICE_NAMES])
|
||||||
|
more = len(names) - NOTICE_NAMES
|
||||||
|
text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
|
||||||
|
else "."))
|
||||||
|
detail += [f"{title}: {name}" for name in names]
|
||||||
|
return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
|
||||||
|
text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
|
||||||
|
|
||||||
|
|
||||||
|
#: The roles a partial read never closes by its list: one with a vote (any,
|
||||||
|
#: neutral included) or an application (which holds the notes).
|
||||||
|
_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||||
|
AND uid NOT IN (SELECT role_uid FROM applications
|
||||||
|
WHERE role_uid IS NOT NULL)"""
|
||||||
|
|
||||||
|
#: The remote roles a filtered read that could not ask for them leaves alone.
|
||||||
|
_NOT_REMOTE = " AND remote != 'remote'"
|
||||||
|
|
||||||
|
|
||||||
|
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
|
||||||
|
*, spare_remote: bool = False) -> int:
|
||||||
"""`Store.mark_closed` for a filtered read: close every open role of the
|
"""`Store.mark_closed` for a filtered read: close every open role of the
|
||||||
board it did not list, except a role someone acted on, one with a vote
|
board it did not list, except a role someone acted on (`_ACTED_ON`). A
|
||||||
(any, neutral included) or an application (which holds the notes). A
|
|
||||||
filtered read cannot tell a role gone from a role outside its filter, and
|
filtered read cannot tell a role gone from a role outside its filter, and
|
||||||
a role acted on stays until the board is read without the filter or
|
a role acted on stays until the board is read without the filter or
|
||||||
fails for `STALE_DAYS`."""
|
fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
|
||||||
|
well, since the read could not ask for them."""
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
return conn.execute(
|
return conn.execute(
|
||||||
"""UPDATE roles SET closed_at = ?
|
f"""UPDATE roles SET closed_at = ?
|
||||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
AND source_id NOT IN (SELECT value FROM json_each(?))
|
AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
|
||||||
AND uid NOT IN (SELECT uid FROM role_votes)
|
+ (_NOT_REMOTE if spare_remote else ""),
|
||||||
AND uid NOT IN (SELECT role_uid FROM applications
|
|
||||||
WHERE role_uid IS NOT NULL)""",
|
|
||||||
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
||||||
|
|
||||||
|
|
||||||
|
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
||||||
|
stale: str, *, remote_only: bool = False) -> int:
|
||||||
|
"""Close the open roles of one board that its read did not list and that
|
||||||
|
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
||||||
|
writes on every listing): the rule for what a read cannot speak for. A
|
||||||
|
read narrowed by terms closes all such roles, as a search does; with
|
||||||
|
`remote_only`, a filtered read that could not ask for remote closes only
|
||||||
|
its remote roles so, sparing those acted on as its list does."""
|
||||||
|
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
||||||
|
with store.transaction() as conn:
|
||||||
|
return conn.execute(
|
||||||
|
f"""UPDATE roles SET closed_at = ?
|
||||||
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
|
AND last_seen < ?
|
||||||
|
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
||||||
|
(store.now(), source, subscription_id, stale,
|
||||||
|
json.dumps(list(seen_ids)))).rowcount
|
||||||
|
|
||||||
|
|
||||||
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||||
filtered: bool) -> int:
|
filtered: bool, *, spare_remote: bool = False) -> int:
|
||||||
"""The close of a board whose complete read finished a sweep across
|
"""The close of a board whose complete read finished a sweep across
|
||||||
pulls: its open roles missing from `seen_ids` that also are in
|
pulls: its open roles missing from `seen_ids` that also are in
|
||||||
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
||||||
@@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
|||||||
missed may only have shifted back past the cursor when roles closed
|
missed may only have shifted back past the cursor when roles closed
|
||||||
between pulls, so one sweep is not evidence that it is gone. A filtered
|
between pulls, so one sweep is not evidence that it is gone. A filtered
|
||||||
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
||||||
stays open, and stays in `missing`."""
|
stays open, and stays in `missing`, and so does a remote role with
|
||||||
|
`spare_remote`."""
|
||||||
key = f"{source}:missing:{sub.target}"
|
key = f"{source}:missing:{sub.target}"
|
||||||
try:
|
try:
|
||||||
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
earlier = set()
|
earlier = set()
|
||||||
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
|
||||||
AND uid NOT IN (SELECT role_uid FROM applications
|
|
||||||
WHERE role_uid IS NOT NULL)""" if filtered else ""
|
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
missed = [row[0] for row in conn.execute(
|
missed = [row[0] for row in conn.execute(
|
||||||
"""SELECT source_id FROM roles
|
"""SELECT source_id FROM roles
|
||||||
@@ -365,15 +536,19 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
|||||||
|
|
||||||
|
|
||||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
"""`target -> (since, reason)` for every `ats` board and every search
|
||||||
failed: when the failures began, and what the latest one said."""
|
whose last fetch failed: when the failures began, and what the latest
|
||||||
|
one said. No board host is also a search provider, so a board's target
|
||||||
|
(`host:slug`) and a search's (`provider:name`) never collide in one
|
||||||
|
dict."""
|
||||||
out: dict[str, tuple[str, str]] = {}
|
out: dict[str, tuple[str, str]] = {}
|
||||||
for key, value in store.state_items(FAILING).items():
|
for prefix in (FAILING, SEARCH_FAILING):
|
||||||
|
for key, value in store.state_items(prefix).items():
|
||||||
try:
|
try:
|
||||||
record = json.loads(value)
|
record = json.loads(value)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
continue
|
continue
|
||||||
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
out[key[len(prefix):]] = (str(record.get("since") or ""),
|
||||||
str(record.get("reason") or ""))
|
str(record.get("reason") or ""))
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|||||||
+41
-23
@@ -16,7 +16,9 @@ The allow-list and the classifier are ported from `openings.go` and
|
|||||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||||
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
||||||
its slug is. Every board is read independently; one that fails is logged,
|
its slug is. Every board is read independently; one that fails is logged,
|
||||||
recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
||||||
|
whose read lists under a tenth of what it listed last time (`_runner.read`'s
|
||||||
|
guard), until it has failed that way for the pull's `stale_days`. A board's
|
||||||
filter is the `reach` option the pull hands it (the agents', see
|
filter is the `reach` option the pull hands it (the agents', see
|
||||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||||
@@ -26,7 +28,8 @@ the whole board: it prunes no per-role state.
|
|||||||
A board's size (`board_size`) is set once, when the board is added
|
A board's size (`board_size`) is set once, when the board is added
|
||||||
(`app.jobs.sources.set_board`) or rechecked, and kept under
|
(`app.jobs.sources.set_board`) or rechecked, and kept under
|
||||||
`ats:size:<target>` with the day it was checked under
|
`ats:size:<target>` with the day it was checked under
|
||||||
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small
|
`ats:sizechecked:<target>` (`keep_size`, which also resets the guard's count,
|
||||||
|
so a Recheck accepts a board that shrank); the reach reads it to leave small
|
||||||
boards whole. A pull does not rewrite it. A board with no size yet is sized
|
boards whole. A pull does not rewrite it. A board with no size yet is sized
|
||||||
in its batch before it is read: its reach then says `unsized` and the size
|
in its batch before it is read: its reach then says `unsized` and the size
|
||||||
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
|
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
|
||||||
@@ -106,8 +109,9 @@ _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
|||||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||||
|
|
||||||
|
|
||||||
def _limits() -> _runner.Limits:
|
def _limits(stale_days: int | None = None) -> _runner.Limits:
|
||||||
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
|
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP,
|
||||||
|
stale_days=stale_days)
|
||||||
|
|
||||||
|
|
||||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||||
@@ -339,8 +343,7 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
|||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
module = readers.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
get = _getter(client, host, slug)
|
get = _getter(client, host, slug)
|
||||||
if (_runner.declared(module) and module.READS.listing == "paged"
|
if module.READS.listing == "paged" and module.READS.complete == "stated":
|
||||||
and module.READS.complete == "stated"):
|
|
||||||
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
|
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
|
||||||
if isinstance(total, int):
|
if isinstance(total, int):
|
||||||
return total
|
return total
|
||||||
@@ -348,11 +351,18 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
|||||||
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
||||||
|
|
||||||
|
|
||||||
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
|
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
||||||
|
reset: bool = True) -> None:
|
||||||
"""Keep board `target`'s size and the day it was checked (ISO,
|
"""Keep board `target`'s size and the day it was checked (ISO,
|
||||||
`today`, else today in UTC)."""
|
`today`, else today in UTC). With `reset`, the default and what a
|
||||||
|
Recheck does, the 10% guard's count goes too (`ats:count:<target>`), so
|
||||||
|
the next read of a board that really shrank is taken as it is instead of
|
||||||
|
failing as suspect; the plugin sizing a board as it reads it passes
|
||||||
|
False, since its count is the read it just stamped."""
|
||||||
state[f"ats:size:{target}"] = str(size)
|
state[f"ats:size:{target}"] = str(size)
|
||||||
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
|
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
|
||||||
|
if reset:
|
||||||
|
state.pop(f"ats:count:{target}", None)
|
||||||
|
|
||||||
|
|
||||||
failure_reason = _runner.failure_reason
|
failure_reason = _runner.failure_reason
|
||||||
@@ -400,14 +410,15 @@ class ATSSource(Source):
|
|||||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||||
module = readers.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
get = _getter(client, host, slug)
|
get = _getter(client, host, slug)
|
||||||
ask, unsized = self._ask(client, target, module)
|
ask, unsized, remote = self._ask(client, target, module)
|
||||||
|
stale_days = self.options.get("stale_days")
|
||||||
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
||||||
limits=_limits())
|
limits=_limits(int(stale_days) if stale_days else None),
|
||||||
|
remote=remote)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.warning("ats: %s failed (%s)", target, exc)
|
log.warning("ats: %s failed (%s)", target, exc)
|
||||||
_runner.record_failure(self.state, failing, exc)
|
_runner.record_failure(self.state, failing, exc)
|
||||||
return
|
return
|
||||||
self.state.pop(failing, None)
|
|
||||||
|
|
||||||
def compound(source_id: str) -> str:
|
def compound(source_id: str) -> str:
|
||||||
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
||||||
@@ -435,11 +446,13 @@ class ATSSource(Source):
|
|||||||
yield item_of(row, target, label)
|
yield item_of(row, target, label)
|
||||||
# Stamped only once every row went out, so a board that blew up half
|
# Stamped only once every row went out, so a board that blew up half
|
||||||
# way through is neither marked ok nor has a partial id list treated
|
# way through is neither marked ok nor has a partial id list treated
|
||||||
# as the whole of it.
|
# as the whole of it, and it stays failing until then: a row that
|
||||||
|
# fails to store (`app.jobs.pull.run_batch`) leaves the record in place.
|
||||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||||
[compound(i) for i in listing.ids])
|
[compound(i) for i in listing.ids])
|
||||||
|
self.state.pop(failing, None)
|
||||||
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
||||||
keep_size(self.state, target, len(listing.ids))
|
keep_size(self.state, target, len(listing.ids), reset=False)
|
||||||
if detail_sets is not None:
|
if detail_sets is not None:
|
||||||
# A complete listing drops the ids it no longer lists.
|
# A complete listing drops the ids it no longer lists.
|
||||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||||
@@ -447,9 +460,11 @@ class ATSSource(Source):
|
|||||||
self.state[key] = json.dumps(sorted(ids))
|
self.state[key] = json.dumps(sorted(ids))
|
||||||
|
|
||||||
def _ask(self, client: httpx.Client, target: str,
|
def _ask(self, client: httpx.Client, target: str,
|
||||||
module: Any) -> tuple[readers.Ask, bool]:
|
module: Any) -> tuple[readers.Ask, bool, bool]:
|
||||||
"""What `target` is asked for, from the reach its batch handed it and
|
"""What `target` is asked for, from the reach its batch handed it and
|
||||||
its params, and whether it still needs a size. A board with no size
|
its params, whether it still needs a size, and whether that reach
|
||||||
|
and params want remote roles (`_runner.remote_wanted`, which the
|
||||||
|
runner compares with what it could ask). A board with no size
|
||||||
yet (`unsized` in its reach) whose reach would narrow the read is
|
yet (`unsized` in its reach) whose reach would narrow the read is
|
||||||
sized first (`board_size`, kept by `keep_size`), and read whole under
|
sized first (`board_size`, kept by `keep_size`), and read whole under
|
||||||
`small` roles. One whose read is whole anyway is sized by that read,
|
`small` roles. One whose read is whole anyway is sized by that read,
|
||||||
@@ -457,16 +472,18 @@ class ATSSource(Source):
|
|||||||
given = self.options.get("reach")
|
given = self.options.get("reach")
|
||||||
params, keys = self.params_for(target), self.options.get("keys")
|
params, keys = self.params_for(target), self.options.get("keys")
|
||||||
if not isinstance(given, dict) or not given.get("unsized"):
|
if not isinstance(given, dict) or not given.get("unsized"):
|
||||||
return _runner.ask_for(module, params, given, keys=keys), False
|
return (_runner.ask_for(module, params, given, keys=keys), False,
|
||||||
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
|
_runner.remote_wanted(module, params, given))
|
||||||
ask = _runner.ask_for(module, params, reach or None, keys=keys)
|
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None
|
||||||
|
ask = _runner.ask_for(module, params, reach, keys=keys)
|
||||||
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
|
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
|
||||||
return ask, True
|
return ask, True, _runner.remote_wanted(module, params, reach)
|
||||||
size = board_size(target, client=client)
|
size = board_size(target, client=client)
|
||||||
keep_size(self.state, target, size)
|
keep_size(self.state, target, size, reset=False)
|
||||||
if size < int(given.get("small") or 0):
|
if size < int(given.get("small") or 0):
|
||||||
|
reach = None
|
||||||
ask = _runner.ask_for(module, params, None, keys=keys)
|
ask = _runner.ask_for(module, params, None, keys=keys)
|
||||||
return ask, False
|
return ask, False, _runner.remote_wanted(module, params, reach)
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------ the per-role reads
|
# ------------------------------------------------------ the per-role reads
|
||||||
@@ -567,7 +584,8 @@ def read_board_details(state: Any, target: str, rows: list[Row], *, client: http
|
|||||||
|
|
||||||
def item_of(row: Row, target: str, label: str) -> Item:
|
def item_of(row: Row, target: str, label: str) -> Item:
|
||||||
"""One row of board `target` as the `Item` the store takes: its id
|
"""One row of board `target` as the `Item` the store takes: its id
|
||||||
compound, `host:slug:id`, as `roles.source_id` has it."""
|
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
|
||||||
|
date is dated now, as `Item` itself would date it."""
|
||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
description = row.get("description") or ""
|
description = row.get("description") or ""
|
||||||
return Item(
|
return Item(
|
||||||
@@ -576,7 +594,7 @@ def item_of(row: Row, target: str, label: str) -> Item:
|
|||||||
title=row["title"],
|
title=row["title"],
|
||||||
excerpt=description[:300],
|
excerpt=description[:300],
|
||||||
body=description,
|
body=description,
|
||||||
published_at=row.get("posted"),
|
published_at=row.get("posted") or datetime.now(UTC),
|
||||||
subscription=label,
|
subscription=label,
|
||||||
kind="role",
|
kind="role",
|
||||||
raw=_runner.raw_of(row) | {
|
raw=_runner.raw_of(row) | {
|
||||||
|
|||||||
+109
-77
@@ -3,8 +3,8 @@
|
|||||||
A reader registers by existing: every module in this package whose name does
|
A reader registers by existing: every module in this package whose name does
|
||||||
not start with `_` is loaded when the package is imported, checked against
|
not start with `_` is loaded when the package is imported, checked against
|
||||||
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
|
the contract below, and listed in `READERS` in order of `(RANK, NAME)`;
|
||||||
`HOSTS` holds those of `scope="board"` (and every module without `READS`),
|
`HOSTS` holds those of `scope="board"`, `PROVIDERS` those of
|
||||||
`PROVIDERS` those of `scope="search"`. A module that breaks the contract
|
`scope="search"`. A module that breaks the contract, or declares no `READS`,
|
||||||
fails the import with an ImportError naming the module and every problem,
|
fails the import with an ImportError naming the module and every problem,
|
||||||
rather than a pull that fails board by board. `plugins/ats.py` builds its
|
rather than a pull that fails board by board. `plugins/ats.py` builds its
|
||||||
allow-list, feeds, pages, the Sources page's host select and its
|
allow-list, feeds, pages, the Sources page's host select and its
|
||||||
@@ -14,21 +14,26 @@ since `desk.sources.registry` loads `plugins/*.py` by path and the tests
|
|||||||
patch names on the `ats` module. The types are in `._contract`.
|
patch names on the `ats` module. The types are in `._contract`.
|
||||||
|
|
||||||
A module is named after its source (`plugins/readers/workday.py`) and
|
A module is named after its source (`plugins/readers/workday.py`) and
|
||||||
imports `desk`, `._contract` and `._shared` only, never `plugins.ats`. It
|
imports `desk`, `._contract`, `._shared` and `._countries` only (besides the
|
||||||
holds:
|
standard library, `httpx` and `pydantic`), never `plugins.ats` or another
|
||||||
|
reader; `tests/test_architecture.py` holds that. A board's module holds, as
|
||||||
|
Workday's does:
|
||||||
|
|
||||||
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
||||||
LABEL = "Workday" # what the Sources page's select shows
|
LABEL = "Workday" # what the Sources page's select shows
|
||||||
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||||
RANK = 100 # optional int; lower comes first in the select and the probes
|
RANK = 100 # optional int; lower comes first in the select and the probes
|
||||||
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
KEEP_CASE = True # optional; True keeps a discovered slug's case
|
||||||
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
|
|
||||||
|
|
||||||
READS = Reads(scope="board", listing="paged", page_size=20, ceiling=2000,
|
READS = Reads(
|
||||||
text="detail", complete="stated",
|
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
|
||||||
filters=Filters(country="ids"), places="many",
|
complete="stated", places="many",
|
||||||
fields=Fields(posted="listed"),
|
filters=Filters(country="ids", remote="ids", query="one"),
|
||||||
robots="read: robots.txt allows /wday/cxs/ (checked 2026-09-29)")
|
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
||||||
|
robots=("read: robots.txt allows /<Site>/ and disallows /refreshFacet/ and has no rule "
|
||||||
|
"on /wday/cxs/<tenant>/<site>/, the listing's POST .../jobs and each role's "
|
||||||
|
"JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"),
|
||||||
|
)
|
||||||
|
|
||||||
def slug_ok(slug: str) -> bool # strict, see below
|
def slug_ok(slug: str) -> bool # strict, see below
|
||||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
||||||
@@ -49,24 +54,35 @@ and, optionally:
|
|||||||
|
|
||||||
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
|
def fallback_url(slug: str) -> str # listing="whole": a smaller feed,
|
||||||
# read once when `feed_url` is over the cap
|
# read once when `feed_url` is over the cap
|
||||||
def robots_page(slug: str) -> str # the runtime check's page (goes, see robots.txt)
|
|
||||||
def probe_url(slug: str) -> str # discovery: 2xx only for a board
|
def probe_url(slug: str) -> str # discovery: 2xx only for a board
|
||||||
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
||||||
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
|
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
|
||||||
|
|
||||||
A reader of `scope="search"` needs `NAME`, `LABEL` and `allowed` only; its
|
**Search providers.** A module of `scope="search"` holds `NAME`, `LABEL`,
|
||||||
slug is the provider's own (a search reads no board), and it is not
|
`allowed`, `READS`, the functions its listing needs, and:
|
||||||
discovered. A module without `READS` is read as a board with the older
|
|
||||||
signatures, `feed_url(slug)` and `page(get, slug, cursor)`, and is listed by
|
PARAMS = {"q": "query", "area": "place"} # saved param -> the filter dimension it fills
|
||||||
`undeclared()`; the conformance test marks it until it declares.
|
def clean_params(params: dict[str, Any]) -> dict[str, Any]
|
||||||
|
|
||||||
|
`app/jobs/sources.search_params` keeps the params of `PARAMS` whose
|
||||||
|
dimension `READS.filters` declares and stores what `clean_params` returns;
|
||||||
|
`clean_params` raises ValueError, with a reason a person can read, for a
|
||||||
|
value the provider would not accept. The runner hands the saved params to
|
||||||
|
`page` in `Ask.params`. A search lists many companies, so its rows carry
|
||||||
|
`company`, the employer's name, which `plugins/search.py` stores. Its slug
|
||||||
|
is the provider's own (a search reads no board), and it has no patterns,
|
||||||
|
probes or careers page: it is never discovered.
|
||||||
|
|
||||||
**Declaration.** `READS` states what the reader does; the runner does it.
|
**Declaration.** `READS` states what the reader does; the runner does it.
|
||||||
`problems()` checks it against the functions: `paged` needs `page` and
|
`problems()` checks it against the functions: `paged` needs `page` and
|
||||||
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
|
`rows`, `whole` needs `feed_url` and `parse` and `complete="whole"`,
|
||||||
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
|
`text="detail"` needs `detail` and `detail` needs it, an `ids` filter needs
|
||||||
`facets`, `scope="search"` needs `complete="never"` or `"stated"`, a `key`
|
`facets`, `scope="search"` needs `complete="never"` or `"stated"` and
|
||||||
names a config entry, `robots` is not empty, and each function takes the
|
`PARAMS` naming filter dimensions, a `key` names a config entry, `robots`
|
||||||
arguments above. The traits:
|
is not empty and carries a date, and each function takes the arguments
|
||||||
|
above. `tests/test_readers_contract.py` checks it against the reader's
|
||||||
|
fixtures: each declared field shows on some row, each field declared `none`
|
||||||
|
on none. The traits:
|
||||||
|
|
||||||
- `scope`: `board`, one company's whole list; `search`, many companies,
|
- `scope`: `board`, one company's whole list; `search`, many companies,
|
||||||
never a whole list, so nothing a search stops listing is closed by it.
|
never a whole list, so nothing a search stops listing is closed by it.
|
||||||
@@ -81,11 +97,13 @@ arguments above. The traits:
|
|||||||
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
|
- `filters`: per dimension (`country`, `place`, `remote`, `query`), `none`,
|
||||||
`one` (one value a request), `many` (a list in one request), `ids` (host
|
`one` (one value a request), `many` (a list in one request), `ids` (host
|
||||||
ids looked up with `facets`).
|
ids looked up with `facets`).
|
||||||
- `places`: `one`, `many`, or `copies` when the host publishes one posting
|
- `places`: `one`, `many`, `copies` when the host publishes one posting
|
||||||
per place with its own id.
|
per place with its own id, or `none` when the host states no place.
|
||||||
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
|
- `fields`: `pay` (`figures`, `line`, `none`), `posted` (`trusted`, the
|
||||||
posting date; `listed`, a date the host moves; `none`), and `remote`,
|
posting date; `listed`, a date the host moves; `none`), and `remote`,
|
||||||
`country`, `seniority`, `employment` (`stated` or `none`).
|
`country`, `seniority`, `employment` (`stated` or `none`). A field may
|
||||||
|
come from the listing or from `detail`: BambooHR, Gem and Rippling date a
|
||||||
|
role only on its own page.
|
||||||
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
|
- `key`: `Key(name, fields, env)`, the config entry `sources.search.<name>`
|
||||||
and its environment fallbacks. The runner resolves it and passes it in
|
and its environment fallbacks. The runner resolves it and passes it in
|
||||||
`Ask.params["key"]`.
|
`Ask.params["key"]`.
|
||||||
@@ -93,7 +111,7 @@ arguments above. The traits:
|
|||||||
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
|
- `empty`: `suspect` when an unknown slug answers an empty list, so a read
|
||||||
far under the last count counts as failed.
|
far under the last count counts as failed.
|
||||||
- `title`: `provisional` when the listing may lack the title and `detail`
|
- `title`: `provisional` when the listing may lack the title and `detail`
|
||||||
fills it.
|
fills it. Nothing reads it yet: `_runner.DETAIL_FIELDS` has no `title`.
|
||||||
|
|
||||||
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
|
**Ask.** `Ask(countries, places, remote, query, params)` is what the runner
|
||||||
wants from one series of requests. It sets only dimensions the reader
|
wants from one series of requests. It sets only dimensions the reader
|
||||||
@@ -124,8 +142,7 @@ against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
|
|||||||
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
|
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
|
||||||
paths are case-sensitive (Workday's site). A pattern without a group names
|
paths are case-sensitive (Workday's site). A pattern without a group names
|
||||||
the host with no slug. A host that `discover.py` names in its own
|
the host with no slug. A host that `discover.py` names in its own
|
||||||
`_UNREADABLE` table is taken from its module instead as soon as the module
|
`_UNREADABLE` table loses its entry there when its module is added.
|
||||||
exists.
|
|
||||||
|
|
||||||
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
||||||
name; `_shared.only(host)` is the usual one. Every request goes through
|
name; `_shared.only(host)` is the usual one. Every request goes through
|
||||||
@@ -165,9 +182,8 @@ see `desk.sources.strip_html`), `remote` (True, False, a word or None) and
|
|||||||
`salary_text`; where the host states pay, `salary_min`, `salary_max`
|
`salary_text`; where the host states pay, `salary_min`, `salary_max`
|
||||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
||||||
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
||||||
`employment`; and `teaser`. The runner keeps every stated key in `raw`, so a
|
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
|
||||||
new field needs no change there. Undeclared modules do not state `places`
|
every stated key in `raw`, so a new field needs no change there.
|
||||||
yet.
|
|
||||||
|
|
||||||
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
||||||
remote=...)]`, one entry per place the host states, in the host's order, in
|
remote=...)]`, one entry per place the host states, in the host's order, in
|
||||||
@@ -176,12 +192,14 @@ place and never joins places into one `text`. `remote` is True for a remote
|
|||||||
place, False for an office, None when unsaid. A host that publishes one
|
place, False for an office, None when unsaid. A host that publishes one
|
||||||
posting per place, each with its own id, declares `places="copies"`: each
|
posting per place, each with its own id, declares `places="copies"`: each
|
||||||
copy is a row with its one place, and `dupes.py` folds them into one role
|
copy is a row with its one place, and `dupes.py` folds them into one role
|
||||||
that keeps each copy's link.
|
that keeps each copy's link. A host that states no place declares
|
||||||
|
`places="none"` and hands over `places == []`.
|
||||||
|
|
||||||
**Helpers.** `._shared` holds what more than one host needs. A module maps
|
**Helpers.** `._shared` holds what more than one host needs, `._countries`
|
||||||
no word to the app's vocabulary: it hands on the host's own words under
|
the country names hosts write. A module maps no word to the app's
|
||||||
`seniority`, `employment`, `remote` and `country`, and `app.prep` maps them
|
vocabulary: it hands on the host's own words under `seniority`,
|
||||||
through the host's table in `app/prep/hosts/<name>.py`.
|
`employment`, `remote` and `country`, and `app.prep` maps them through the
|
||||||
|
host's table in `app/prep/hosts/<name>.py`.
|
||||||
|
|
||||||
- `place(text, remote=None)` makes a `Place`: remote when the text has the
|
- `place(text, remote=None)` makes a `Place`: remote when the text has the
|
||||||
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
|
word `remote`, otherwise as `remote` says. `add_place(found, entry)`
|
||||||
@@ -197,38 +215,60 @@ is shown and embedded until `description` arrives. A reader that sees one
|
|||||||
keeps it.
|
keeps it.
|
||||||
|
|
||||||
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
||||||
dict of the fields it found. The shared code calls it for roles not yet read
|
dict of the fields it found. The pull reads listings only; the `enrich`
|
||||||
(`ats:detailed:<target>`), at most `_runner.DETAIL_CAP` (200) a board a
|
stage (`app/jobs/enrich.py`) calls `detail` after scoring, so the requests
|
||||||
pull, `_runner.PAGE_DELAY` apart, the roles never tried before those that
|
go where the ranking is: the open roles without text that an agent ranks
|
||||||
failed (`ats:detailfailed:<target>`). Only `_runner.DETAIL_FIELDS` are taken:
|
among its best `enrich.top` at `enrich.min_score` or above, best first, at
|
||||||
|
most `enrich.per_board` from one board and `enrich.per_run` in all, one
|
||||||
|
batch per board through `ats.read_board_details` and
|
||||||
|
`_runner.read_details`, `_runner.PAGE_DELAY` apart, the roles never tried
|
||||||
|
before those whose read failed. Only `_runner.DETAIL_FIELDS` are taken:
|
||||||
`description`, `location`, `places`, `teaser`, `remote`, `country`,
|
`description`, `location`, `places`, `teaser`, `remote`, `country`,
|
||||||
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the
|
`employment`, `currency`, `salary_min`, `salary_max` and `posted`, the keys
|
||||||
listing's empty value; any other key is left out with a debug log line.
|
whose columns survive a later pull that sends the listing's empty value;
|
||||||
|
any other key is left out with a debug log line, `seniority` among them.
|
||||||
`title`, `url`, `department` and `salary_text` cannot be filled from
|
`title`, `url`, `department` and `salary_text` cannot be filled from
|
||||||
`detail`: the listing rewrites them every pull. A role not read in detail
|
`detail`: the listing rewrites them every pull. A 404 or a 410 counts as read; any
|
||||||
on a pull is sent with an empty description, so the one read earlier stays.
|
other failure is logged and retried on a later run; a read that gave no text
|
||||||
A 404 or a 410 counts as read; any other failure is logged and retried on a
|
is retried after `enrich.retry_days`; a 429 ends the reads of that board for
|
||||||
later pull; a 429 ends the detail reads of that board for the pull.
|
the run. A role read in detail is sent by later pulls without the listing's
|
||||||
A role read in detail on an earlier pull is sent without the
|
place and remote, so what its own page said about them stays. Completeness
|
||||||
listing's place and remote, so what its own page said about them stays.
|
is decided by the listing alone.
|
||||||
Completeness is decided by the listing alone.
|
|
||||||
|
|
||||||
**robots.txt.** Decided when the reader is built, not checked at run time.
|
**robots.txt.** Decided when the reader is built, not checked at run time.
|
||||||
Whoever builds a reader reads the host's robots.txt for the paths the reader
|
Whoever builds a reader reads the robots.txt of every host the reader
|
||||||
requests and reports what it disallows, and the owner decides whether the
|
requests, for the paths it requests (the API's, not the career page a
|
||||||
host is read anyway. `robots` records that decision with its date, e.g.
|
person opens), and the owner decides whether the host is read anyway.
|
||||||
`"read: /wday/cxs/ allowed (checked 2026-09-29)"` or `"read though /api/ is
|
`robots` records what the file says of those paths, the date it was read
|
||||||
disallowed: owner, 2026-09-30"`. Nothing reads robots.txt at run time;
|
and the decision, e.g. `"read: robots.txt has no rule on /wday/cxs/
|
||||||
`robots_page` and `ROBOTS_ANY_CASE`, which fed the former runtime check, are
|
(checked 2026-09-30)"` or `"read though /api/ is disallowed: owner,
|
||||||
read by nothing and a new reader defines neither.
|
2026-09-30"`; a record without a date fails the contract. Nothing reads
|
||||||
|
robots.txt at run time.
|
||||||
|
|
||||||
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
||||||
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
||||||
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
|
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
|
||||||
name is probed as, `(guess,)` without it. A host without `probe_url` is never
|
name is probed as, `(guess,)` without it. A host without `probe_url` is never
|
||||||
probed, only found by a link.
|
probed, only found by a link.
|
||||||
"""
|
|
||||||
|
|
||||||
|
**Adding a reader.** A new host or provider touches:
|
||||||
|
|
||||||
|
- its module here;
|
||||||
|
- `app/prep/hosts/<name>.py` when it states seniority or employment in its
|
||||||
|
own words (the contract test requires the table; a `REMOTE` table when
|
||||||
|
its remote words are not the app's);
|
||||||
|
- its fixtures under `tests/fixtures/ats/` (`tests/fixtures/search/` for a
|
||||||
|
provider), named as `tests/test_readers_contract.py` describes, and
|
||||||
|
`tests/test_ats_<name>.py` and `tests/test_prep_hosts_<name>.py`;
|
||||||
|
- its entry in `discover._UNREADABLE`, deleted, where discovery named the
|
||||||
|
host before;
|
||||||
|
- a section in `plugins/README.md`, and a row in `docs/filter-survey.md`
|
||||||
|
when it filters at the source.
|
||||||
|
|
||||||
|
Nothing else: `HOSTS` and `PROVIDERS`, the Sources page's select,
|
||||||
|
discovery's patterns and probes, and the reach a board is read with are
|
||||||
|
derived from the module.
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
@@ -241,7 +281,7 @@ from . import _contract
|
|||||||
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
|
from ._contract import Ask, Fields, Filters, Key, Place, Reads, Row
|
||||||
|
|
||||||
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
|
__all__ = ["DEFAULT_RANK", "HOSTS", "PROVIDERS", "READERS", "Ask", "Fields", "Filters", "Key",
|
||||||
"Place", "Reads", "Row", "load", "problems", "scope", "undeclared"]
|
"Place", "Reads", "Row", "load", "problems"]
|
||||||
|
|
||||||
#: The rank of a module that states none: after the eight first hosts, whose
|
#: The rank of a module that states none: after the eight first hosts, whose
|
||||||
#: ranks (10 to 80) keep the order they had before there were modules.
|
#: ranks (10 to 80) keep the order they had before there were modules.
|
||||||
@@ -250,27 +290,28 @@ DEFAULT_RANK = 100
|
|||||||
#: What every board module defines.
|
#: What every board module defines.
|
||||||
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
||||||
|
|
||||||
#: What a module of `scope="search"` defines.
|
#: What a module of `scope="search"` defines: `PARAMS` and `clean_params` are
|
||||||
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed")
|
#: what `app/jobs/sources.search_params` stores a search's params with.
|
||||||
|
REQUIRED_SEARCH = ("NAME", "LABEL", "allowed", "PARAMS", "clean_params")
|
||||||
|
|
||||||
#: The functions a module may define; each must be callable if present.
|
#: The functions a module may define; each must be callable if present.
|
||||||
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
||||||
"fallback_url", "robots_page", "detail", "facets", "probe_url", "probe_ok",
|
"fallback_url", "detail", "facets", "clean_params", "probe_url", "probe_ok",
|
||||||
"probe_slugs")
|
"probe_slugs")
|
||||||
|
|
||||||
|
|
||||||
def scope(module: ModuleType) -> str:
|
def _search(module: ModuleType) -> bool:
|
||||||
"""`board` or `search`; `board` for a module without `READS`."""
|
|
||||||
reads = getattr(module, "READS", None)
|
reads = getattr(module, "READS", None)
|
||||||
return reads.scope if isinstance(reads, Reads) else "board"
|
return isinstance(reads, Reads) and reads.scope == "search"
|
||||||
|
|
||||||
|
|
||||||
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
||||||
"""What `module` lacks or gets wrong against the contract, one phrase
|
"""What `module` lacks or gets wrong against the contract, one phrase
|
||||||
each; empty for a module that keeps it. `name` is the module's own name
|
each; empty for a module that keeps it. `name` is the module's own name
|
||||||
in the package, which `NAME` must equal. `READS` is checked when the
|
in the package, which `NAME` must equal. A module without `READS` is
|
||||||
module declares one (`_contract.problems`)."""
|
held to a board's names and fails on the missing declaration; the
|
||||||
required = REQUIRED_SEARCH if scope(module) == "search" else REQUIRED
|
declaration itself is `_contract.problems`."""
|
||||||
|
required = REQUIRED_SEARCH if _search(module) else REQUIRED
|
||||||
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
|
out = [f"no {attr}" for attr in required if not hasattr(module, attr)]
|
||||||
if name is not None and getattr(module, "NAME", name) != name:
|
if name is not None and getattr(module, "NAME", name) != name:
|
||||||
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
||||||
@@ -278,8 +319,6 @@ def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
|||||||
out.append("LABEL is not a non-empty string")
|
out.append("LABEL is not a non-empty string")
|
||||||
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
||||||
out.append("KEEP_CASE is not a bool")
|
out.append("KEEP_CASE is not a bool")
|
||||||
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
|
|
||||||
out.append("ROBOTS_ANY_CASE is not a bool")
|
|
||||||
rank = getattr(module, "RANK", DEFAULT_RANK)
|
rank = getattr(module, "RANK", DEFAULT_RANK)
|
||||||
if not isinstance(rank, int) or isinstance(rank, bool):
|
if not isinstance(rank, int) or isinstance(rank, bool):
|
||||||
out.append("RANK is not an int")
|
out.append("RANK is not an int")
|
||||||
@@ -323,20 +362,13 @@ def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
|
|||||||
return {module.NAME: module for module in modules}
|
return {module.NAME: module for module in modules}
|
||||||
|
|
||||||
|
|
||||||
def undeclared(modules: dict[str, ModuleType] | None = None) -> list[str]:
|
|
||||||
"""The names of the modules (of `READERS` by default) without `READS`,
|
|
||||||
read with the older signatures until they declare."""
|
|
||||||
return [name for name, module in (READERS if modules is None else modules).items()
|
|
||||||
if not hasattr(module, "READS")]
|
|
||||||
|
|
||||||
|
|
||||||
#: name -> module for every reader, in order of `(RANK, NAME)`.
|
#: name -> module for every reader, in order of `(RANK, NAME)`.
|
||||||
READERS: dict[str, ModuleType] = load(__path__, __name__)
|
READERS: dict[str, ModuleType] = load(__path__, __name__)
|
||||||
|
|
||||||
#: host -> its module: the readers of `scope="board"` and the undeclared.
|
#: host -> its module: the readers of `scope="board"`.
|
||||||
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
|
HOSTS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
|
||||||
if scope(module) == "board"}
|
if module.READS.scope == "board"}
|
||||||
|
|
||||||
#: provider -> its module: the readers of `scope="search"`.
|
#: provider -> its module: the readers of `scope="search"`.
|
||||||
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
|
PROVIDERS: dict[str, ModuleType] = {name: module for name, module in READERS.items()
|
||||||
if scope(module) == "search"}
|
if module.READS.scope == "search"}
|
||||||
@@ -20,14 +20,14 @@ Signatures of a module that declares `READS`, with `Get` and `Page` from
|
|||||||
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
|
def rows(decoded: Any, slug: str) -> list[Row] # listing="paged"
|
||||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
|
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
|
||||||
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
|
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
|
||||||
|
def clean_params(params: dict[str, Any]) -> dict[str, Any] # scope="search"
|
||||||
|
|
||||||
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
|
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
|
||||||
`country`, a place name for `place`, any word for `remote`. A dimension the
|
`country`, a place name for `place`, any word for `remote`. A dimension the
|
||||||
host has no facet for is left out, and the runner then reads it unfiltered;
|
host has no facet for is left out, and the runner then reads it unfiltered;
|
||||||
a dimension present without a wanted value means the host has none of it.
|
a dimension present without a wanted value means the host has none of it.
|
||||||
|
|
||||||
A module without `READS` keeps the signatures without `ask` until it
|
A module without `READS` fails the import: every reader declares.
|
||||||
declares; `undeclared()` in the package lists those modules.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -57,6 +57,9 @@ _CONFIG_NAME = re.compile(r"[a-z][a-z0-9_]*")
|
|||||||
#: An environment variable name.
|
#: An environment variable name.
|
||||||
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
|
_ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]*")
|
||||||
|
|
||||||
|
#: The date a robots.txt record must carry: when the file was read.
|
||||||
|
_DATE = re.compile(r"20\d\d-\d\d-\d\d")
|
||||||
|
|
||||||
|
|
||||||
class _Frozen(Model):
|
class _Frozen(Model):
|
||||||
"""A declaration: immutable, extra keys refused."""
|
"""A declaration: immutable, extra keys refused."""
|
||||||
@@ -123,11 +126,13 @@ class Reads(_Frozen):
|
|||||||
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
|
`Row.teaser`. `complete`: `stated` (a total), `end` (an empty page),
|
||||||
`whole` (the one answer), `never` (a search). `places`: `one` (the host
|
`whole` (the one answer), `never` (a search). `places`: `one` (the host
|
||||||
states one), `many` (a list per role), `copies` (the host lists one
|
states one), `many` (a list per role), `copies` (the host lists one
|
||||||
posting per place, each with its own id). `robots`: the robots.txt
|
posting per place, each with its own id), `none` (the host states no
|
||||||
decision, with the date, taken when the reader was built. `empty`:
|
place; every row has `places == []`). `robots`: the robots.txt
|
||||||
`suspect` when an unknown slug answers an empty list rather than an
|
decision for the paths the reader requests, with the date the file was
|
||||||
error. `title`: `provisional` when the listing may not state the title
|
read. `empty`: `suspect` when an unknown slug answers an empty list
|
||||||
(a sitemap), which detail then fills.
|
rather than an error. `title`: `provisional` when the listing may not
|
||||||
|
state the title (a sitemap), which detail then fills; nothing reads it
|
||||||
|
yet, since `_runner.DETAIL_FIELDS` has no `title`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
scope: Literal["board", "search"]
|
scope: Literal["board", "search"]
|
||||||
@@ -138,7 +143,7 @@ class Reads(_Frozen):
|
|||||||
teaser: bool = False
|
teaser: bool = False
|
||||||
complete: Literal["stated", "end", "whole", "never"]
|
complete: Literal["stated", "end", "whole", "never"]
|
||||||
filters: Filters = Filters()
|
filters: Filters = Filters()
|
||||||
places: Literal["one", "many", "copies"]
|
places: Literal["one", "many", "copies", "none"]
|
||||||
fields: Fields = Fields()
|
fields: Fields = Fields()
|
||||||
robots: str
|
robots: str
|
||||||
empty: Literal["trusted", "suspect"] = "trusted"
|
empty: Literal["trusted", "suspect"] = "trusted"
|
||||||
@@ -196,8 +201,10 @@ class Row(TypedDict, total=False):
|
|||||||
place, kept while storage reads it. `posted` is an aware datetime or
|
place, kept while storage reads it. `posted` is an aware datetime or
|
||||||
None; `description` plain text; `remote` True, False, a word or None;
|
None; `description` plain text; `remote` True, False, a word or None;
|
||||||
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
|
pay per `Fields.pay`. `country` (ISO alpha-2), `seniority` and
|
||||||
`employment` only where the host states them, as `Fields` declares. Any
|
`employment` only where the host states them, as `Fields` declares.
|
||||||
other stated key is kept in `raw` by the runner.
|
`company` is the employer's name on a search row, which lists many
|
||||||
|
(`plugins/search.py` stores it); a board row carries none, the board is
|
||||||
|
the company. Any other stated key is kept in `raw` by the runner.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
source_id: Required[str]
|
source_id: Required[str]
|
||||||
@@ -217,6 +224,7 @@ class Row(TypedDict, total=False):
|
|||||||
country: str | None
|
country: str | None
|
||||||
seniority: str | None
|
seniority: str | None
|
||||||
employment: str | None
|
employment: str | None
|
||||||
|
company: str
|
||||||
|
|
||||||
|
|
||||||
#: What a declared module's functions take, positionally.
|
#: What a declared module's functions take, positionally.
|
||||||
@@ -227,6 +235,7 @@ SIGNATURES: dict[str, tuple[str, ...]] = {
|
|||||||
"rows": ("decoded", "slug"),
|
"rows": ("decoded", "slug"),
|
||||||
"detail": ("get", "slug", "row"),
|
"detail": ("get", "slug", "row"),
|
||||||
"facets": ("get", "slug"),
|
"facets": ("get", "slug"),
|
||||||
|
"clean_params": ("params",),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -245,9 +254,9 @@ def _takes(function: Callable[..., Any], count: int) -> bool:
|
|||||||
|
|
||||||
def problems(module: ModuleType) -> list[str]:
|
def problems(module: ModuleType) -> list[str]:
|
||||||
"""What `module`'s `READS` gets wrong, on its own or against the
|
"""What `module`'s `READS` gets wrong, on its own or against the
|
||||||
module's functions; empty for a module that declares none."""
|
module's functions; "no READS" for a module that declares none."""
|
||||||
if not hasattr(module, "READS"):
|
if not hasattr(module, "READS"):
|
||||||
return []
|
return ["no READS: every reader declares what it reads"]
|
||||||
reads = module.READS
|
reads = module.READS
|
||||||
if not isinstance(reads, Reads):
|
if not isinstance(reads, Reads):
|
||||||
return ["READS is not a Reads"]
|
return ["READS is not a Reads"]
|
||||||
@@ -305,6 +314,17 @@ def problems(module: ModuleType) -> list[str]:
|
|||||||
|
|
||||||
if not reads.robots.strip():
|
if not reads.robots.strip():
|
||||||
out.append("robots is empty: record the robots.txt decision and its date")
|
out.append("robots is empty: record the robots.txt decision and its date")
|
||||||
|
elif not _DATE.search(reads.robots):
|
||||||
|
out.append("robots has no date: record when robots.txt was read (YYYY-MM-DD)")
|
||||||
|
|
||||||
|
if reads.scope == "search" and hasattr(module, "PARAMS"):
|
||||||
|
params = module.PARAMS
|
||||||
|
if not isinstance(params, dict) or not params:
|
||||||
|
out.append("PARAMS is not a dict of saved param -> filter dimension")
|
||||||
|
else:
|
||||||
|
dimensions = set(Filters.model_fields)
|
||||||
|
out += [f"PARAMS maps {param!r} to {dimension!r}, not a filter dimension"
|
||||||
|
for param, dimension in params.items() if dimension not in dimensions]
|
||||||
|
|
||||||
for name, params in SIGNATURES.items():
|
for name, params in SIGNATURES.items():
|
||||||
if has(name) and not _takes(getattr(module, name), len(params)):
|
if has(name) and not _takes(getattr(module, name), len(params)):
|
||||||
|
|||||||
+149
-71
@@ -5,23 +5,25 @@ A reader says how its host answers; this module reads it: the guarded fetch
|
|||||||
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
||||||
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
||||||
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
||||||
the sweep of an oversized paged board across pulls, the 10% guard on a
|
the sweep of an oversized paged board across pulls, the 10% guard on every
|
||||||
filtered or `empty="suspect"` read (`read`), the per-role detail reads
|
read that is not a sweep (`read`), the per-role detail reads
|
||||||
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||||
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
||||||
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
||||||
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
||||||
as they were.
|
as they were. Every reader declares `READS`; a module without it is refused
|
||||||
|
by the package before it gets here.
|
||||||
A module without `READS` is read with the older signatures, `feed_url(slug)`
|
|
||||||
and `page(get, slug, cursor)`, and never gets an `Ask`.
|
|
||||||
|
|
||||||
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
||||||
|
|
||||||
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
|
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
|
||||||
"seen", "rows", "whole"}`, continued by the next pull
|
"seen", "rows", "whole"}`, continued by the next pull
|
||||||
count `{"asks", "count"}`, the roles the last whole read listed, for
|
count `{"asks", "count"}`, the roles the last read that was not a
|
||||||
the 10% guard
|
sweep listed, for the 10% guard
|
||||||
|
failing `{"since", "reason"}`, kept by the Source (`record_failure`);
|
||||||
|
the guard gives way once `since` is `Limits.stale_days` old
|
||||||
|
remote_unasked "1" when the reach wanted remote and the read could not
|
||||||
|
ask for it (`Listing.remote_unasked`), else ""
|
||||||
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
||||||
swept "1" when the last stamped listing finished a sweep carried over
|
swept "1" when the last stamped listing finished a sweep carried over
|
||||||
from an earlier pull (`Listing.swept`), else "". Offset paging
|
from an earlier pull (`Listing.swept`), else "". Offset paging
|
||||||
@@ -87,10 +89,16 @@ DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "count
|
|||||||
#: How long a board's `facets` are kept before they are asked again.
|
#: How long a board's `facets` are kept before they are asked again.
|
||||||
FACETS_TTL = timedelta(days=1)
|
FACETS_TTL = timedelta(days=1)
|
||||||
|
|
||||||
#: A filtered or `empty="suspect"` read listing under this share of what the
|
#: A read listing under this share of what the same read listed last time
|
||||||
#: same read listed last time counts as a failure.
|
#: counts as a failure (`read`).
|
||||||
SUSPECT_SHARE = 0.1
|
SUSPECT_SHARE = 0.1
|
||||||
|
|
||||||
|
#: The last count from which the guard holds every read. A read that is
|
||||||
|
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
||||||
|
#: a count of 1, as it was before every read was; a whole read of a board
|
||||||
|
#: that listed a handful of roles can drop to none by itself.
|
||||||
|
SUSPECT_MIN = 10
|
||||||
|
|
||||||
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
||||||
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
|
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
|
||||||
|
|
||||||
@@ -105,8 +113,8 @@ class TooLarge(ValueError):
|
|||||||
|
|
||||||
|
|
||||||
class Suspect(Exception):
|
class Suspect(Exception):
|
||||||
"""A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE`
|
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
|
||||||
of what it listed last time."""
|
(see `read`)."""
|
||||||
|
|
||||||
|
|
||||||
class MissingKey(ValueError):
|
class MissingKey(ValueError):
|
||||||
@@ -115,12 +123,15 @@ class MissingKey(ValueError):
|
|||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class Limits:
|
class Limits:
|
||||||
"""What one Source allows a read: pages, the pause between requests, and
|
"""What one Source allows a read: pages, the pause between requests, the
|
||||||
the detail reads a pull."""
|
detail reads a pull, and the days after which the 10% guard accepts the
|
||||||
|
answer it keeps refusing (`stale_days`, the pull's `STALE_DAYS`; None,
|
||||||
|
for a Source run outside a pull, never)."""
|
||||||
|
|
||||||
max_pages: int = MAX_PAGES
|
max_pages: int = MAX_PAGES
|
||||||
page_delay: float = PAGE_DELAY
|
page_delay: float = PAGE_DELAY
|
||||||
detail_cap: int = DETAIL_CAP
|
detail_cap: int = DETAIL_CAP
|
||||||
|
stale_days: int | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
@dataclass(slots=True)
|
||||||
@@ -134,7 +145,9 @@ class Listing:
|
|||||||
neither are the whole board. `swept` says a complete listing finished a
|
neither are the whole board. `swept` says a complete listing finished a
|
||||||
sweep carried over from an earlier pull: `ids` were read across pulls,
|
sweep carried over from an earlier pull: `ids` were read across pulls,
|
||||||
and roles closing meanwhile may have shifted an offset-paged host's later
|
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||||
roles back past the cursor, so `ids` can miss a role that is still open."""
|
roles back past the cursor, so `ids` can miss a role that is still open.
|
||||||
|
`remote_unasked` says a filtered read was meant to ask for remote roles
|
||||||
|
as well and could not, so its `ids` say nothing of the remote roles."""
|
||||||
|
|
||||||
rows: list[Row]
|
rows: list[Row]
|
||||||
complete: bool
|
complete: bool
|
||||||
@@ -144,12 +157,14 @@ class Listing:
|
|||||||
asks: str = ""
|
asks: str = ""
|
||||||
narrowed: bool = False
|
narrowed: bool = False
|
||||||
swept: bool = False
|
swept: bool = False
|
||||||
|
remote_unasked: bool = False
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- modules
|
# --------------------------------------------------------------- modules
|
||||||
|
|
||||||
def declared(module: ModuleType) -> bool:
|
def declared(module: ModuleType) -> bool:
|
||||||
"""Whether `module` declares `READS`, and so takes `ask`."""
|
"""Whether `module` declares `READS`. Every reader does, so the runner no
|
||||||
|
longer asks; `app.jobs.reach` still does."""
|
||||||
return isinstance(getattr(module, "READS", None), Reads)
|
return isinstance(getattr(module, "READS", None), Reads)
|
||||||
|
|
||||||
|
|
||||||
@@ -334,6 +349,33 @@ def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
|
|||||||
return count
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def _wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None,
|
||||||
|
reach: Mapping[str, Any] | None) -> tuple[dict[str, Any], bool]:
|
||||||
|
"""The dimensions `reach` and then the params set, params winning, before
|
||||||
|
they are narrowed to what the reader declares, and whether the read is
|
||||||
|
wide. A reach of more than `MAX_SERIES` countries on a `one` country
|
||||||
|
filter is dropped whole, which makes the read wide; a `remote` of False
|
||||||
|
asks nothing."""
|
||||||
|
given = reach or {}
|
||||||
|
reached = _dimensions(module, given)
|
||||||
|
wide = bool(given.get("wide"))
|
||||||
|
if module.READS.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
||||||
|
reached = {}
|
||||||
|
wide = True
|
||||||
|
wanted = reached | _dimensions(module, dict(subscription_params or {}))
|
||||||
|
if wanted.get("remote") is False:
|
||||||
|
del wanted["remote"]
|
||||||
|
return wanted, wide
|
||||||
|
|
||||||
|
|
||||||
|
def remote_wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||||
|
reach: Mapping[str, Any] | None = None) -> bool:
|
||||||
|
"""Whether the reach or the params want remote roles, whether or not the
|
||||||
|
reader can ask for them. `read_listing` compares it with what the series
|
||||||
|
asked (`Listing.remote_unasked`)."""
|
||||||
|
return _wanted(module, subscription_params, reach)[0].get("remote") is True
|
||||||
|
|
||||||
|
|
||||||
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||||
reach: Mapping[str, Any] | None = None, *,
|
reach: Mapping[str, Any] | None = None, *,
|
||||||
keys: Mapping[str, Any] | None = None) -> Ask:
|
keys: Mapping[str, Any] | None = None) -> Ask:
|
||||||
@@ -346,7 +388,8 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
|
|||||||
dropped. A `remote` of False asks nothing. A search's params go whole into
|
dropped. A `remote` of False asks nothing. A search's params go whole into
|
||||||
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
||||||
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
||||||
A module without `READS` gets `Ask()`.
|
A remote the reader cannot filter on is dropped here; `remote_wanted`
|
||||||
|
still says it was wanted.
|
||||||
|
|
||||||
Fetch terms (`reach["terms"]`) go only to a board reader that declares
|
Fetch terms (`reach["terms"]`) go only to a board reader that declares
|
||||||
`query="one"`, and only on a wide read: `reach["wide"]` (a company set
|
`query="one"`, and only on a wide read: `reach["wide"]` (a company set
|
||||||
@@ -358,19 +401,10 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
|
|||||||
whole, since remote without its countries would read only the remote
|
whole, since remote without its countries would read only the remote
|
||||||
roles; over it with terms alone, the terms go too and the board is read
|
roles; over it with terms alone, the terms go too and the board is read
|
||||||
unfiltered."""
|
unfiltered."""
|
||||||
if not declared(module):
|
|
||||||
return Ask()
|
|
||||||
reads: Reads = module.READS
|
reads: Reads = module.READS
|
||||||
params = dict(subscription_params or {})
|
params = dict(subscription_params or {})
|
||||||
given = reach or {}
|
given = reach or {}
|
||||||
reached = _dimensions(module, given)
|
wanted, wide = _wanted(module, params, given)
|
||||||
wide = bool(given.get("wide"))
|
|
||||||
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
|
||||||
reached = {}
|
|
||||||
wide = True
|
|
||||||
wanted = reached | _dimensions(module, params)
|
|
||||||
if wanted.get("remote") is False:
|
|
||||||
del wanted["remote"]
|
|
||||||
declared_now = reads.filters.declared()
|
declared_now = reads.filters.declared()
|
||||||
wanted = {name: value for name, value in wanted.items() if name in declared_now}
|
wanted = {name: value for name, value in wanted.items() if name in declared_now}
|
||||||
terms = _terms(given.get("terms"))
|
terms = _terms(given.get("terms"))
|
||||||
@@ -400,7 +434,7 @@ def narrowed(module: ModuleType, ask: Ask) -> bool:
|
|||||||
query. Such a read is never the whole board, nor the board within its
|
query. Such a read is never the whole board, nor the board within its
|
||||||
places, so it closes nothing. A search's query is what
|
places, so it closes nothing. A search's query is what
|
||||||
the search is, and does not count."""
|
the search is, and does not count."""
|
||||||
board = not declared(module) or module.READS.scope == "board"
|
board = module.READS.scope == "board"
|
||||||
return bool(ask.terms) or (board and bool(ask.query))
|
return bool(ask.terms) or (board and bool(ask.query))
|
||||||
|
|
||||||
|
|
||||||
@@ -417,10 +451,7 @@ def series(module: ModuleType, ask: Ask,
|
|||||||
remote part beside places is then left out. Each of those is then asked
|
remote part beside places is then left out. Each of those is then asked
|
||||||
once per fetch term (`ask.terms`), with the term as `query`: the host
|
once per fetch term (`ask.terms`), with the term as `query`: the host
|
||||||
intersects a keyword with its facets, as the survey found on every
|
intersects a keyword with its facets, as the survey found on every
|
||||||
host, so the union by id is "one of the terms, in these places". `[ask]`
|
host, so the union by id is "one of the terms, in these places"."""
|
||||||
for a module without `READS`."""
|
|
||||||
if not declared(module):
|
|
||||||
return [ask]
|
|
||||||
if ask.remote and (ask.countries or ask.places):
|
if ask.remote and (ask.countries or ask.places):
|
||||||
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
||||||
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
||||||
@@ -450,8 +481,8 @@ def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]
|
|||||||
known = known_facets.get(dimension) or {}
|
known = known_facets.get(dimension) or {}
|
||||||
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
|
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
|
||||||
if not mapped:
|
if not mapped:
|
||||||
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
|
log.warning("readers: %s knows none of %s %s; nothing asked", module.NAME,
|
||||||
", ".join(values))
|
dimension, ", ".join(values))
|
||||||
return []
|
return []
|
||||||
if dimension == "country":
|
if dimension == "country":
|
||||||
countries = mapped
|
countries = mapped
|
||||||
@@ -497,7 +528,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
|||||||
for index, ask in enumerate(asks):
|
for index, ask in enumerate(asks):
|
||||||
if index:
|
if index:
|
||||||
time.sleep(limits.page_delay)
|
time.sleep(limits.page_delay)
|
||||||
url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug)
|
url = module.feed_url(slug, ask)
|
||||||
try:
|
try:
|
||||||
body = get(url)
|
body = get(url)
|
||||||
except TooLarge:
|
except TooLarge:
|
||||||
@@ -514,7 +545,8 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
|||||||
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
|
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
|
||||||
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
|
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
|
||||||
can_sweep: bool = False,
|
can_sweep: bool = False,
|
||||||
facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing:
|
facets: Mapping[str, Mapping[str, str]] | None = None,
|
||||||
|
remote: bool = False) -> Listing:
|
||||||
"""One target's listing, read and parsed. Raises on any failure of the
|
"""One target's listing, read and parsed. Raises on any failure of the
|
||||||
first request, and on a reader that does not understand an answer.
|
first request, and on a reader that does not understand an answer.
|
||||||
|
|
||||||
@@ -532,22 +564,35 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
its series and cursor; the sweep is complete when it reaches the end,
|
its series and cursor; the sweep is complete when it reaches the end,
|
||||||
and `ids` then hold every id it listed, and the listing is `swept`. A
|
and `ids` then hold every id it listed, and the listing is `swept`. A
|
||||||
read at the reader's `ceiling` is not swept: the host answers nothing
|
read at the reader's `ceiling` is not swept: the host answers nothing
|
||||||
past it."""
|
past it.
|
||||||
|
|
||||||
|
A filtered read that asks no series, because an `ids` dimension names
|
||||||
|
no value the host's facets know, asked nothing, which is no evidence of
|
||||||
|
anything: it is incomplete and closes nothing. `filtered` is a board's
|
||||||
|
only: a search's places are what the search is, as its query is.
|
||||||
|
`remote` says the reach or the params wanted remote roles; when no series
|
||||||
|
asked for them (a reader without a remote filter, or one whose facets
|
||||||
|
state none beside places) the listing is `remote_unasked`."""
|
||||||
limits = limits or Limits()
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
|
reads: Reads = module.READS
|
||||||
asks = series(module, ask, facets)
|
asks = series(module, ask, facets)
|
||||||
is_filtered = filtered(ask)
|
is_filtered = filtered(ask) and reads.scope == "board"
|
||||||
by_words = narrowed(module, ask)
|
by_words = narrowed(module, ask)
|
||||||
|
unasked = (is_filtered and (remote or bool(ask.remote))
|
||||||
|
and not any(a.remote or a.remote_ids for a in asks))
|
||||||
key = fingerprint(asks)
|
key = fingerprint(asks)
|
||||||
if not asks:
|
if not asks:
|
||||||
return Listing([], True, [], is_filtered, None, key, by_words)
|
log.warning("readers: %s:%s: the filter maps to nothing the host knows; "
|
||||||
|
"nothing asked, not closing", module.NAME, slug)
|
||||||
|
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
||||||
if not hasattr(module, "page"):
|
if not hasattr(module, "page"):
|
||||||
rows = _whole(module, slug, get, asks, limits)
|
whole_rows = _whole(module, slug, get, asks, limits)
|
||||||
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words)
|
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
||||||
|
key, by_words, remote_unasked=unasked)
|
||||||
|
|
||||||
reads: Reads | None = module.READS if declared(module) else None
|
never = reads.complete == "never"
|
||||||
never = reads is not None and reads.complete == "never"
|
ceiling = reads.ceiling
|
||||||
ceiling = reads.ceiling if reads is not None else None
|
|
||||||
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
||||||
resumed = bool(sweep and sweep.get("asks") == key)
|
resumed = bool(sweep and sweep.get("asks") == key)
|
||||||
if sweep and resumed:
|
if sweep and resumed:
|
||||||
@@ -568,10 +613,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
if pages:
|
if pages:
|
||||||
time.sleep(limits.page_delay)
|
time.sleep(limits.page_delay)
|
||||||
try:
|
try:
|
||||||
if reads is not None:
|
|
||||||
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
||||||
else:
|
|
||||||
data, results, total, cursor = module.page(get, slug, cursor)
|
|
||||||
except httpx.HTTPStatusError as exc:
|
except httpx.HTTPStatusError as exc:
|
||||||
if not pages or exc.response.status_code != 429:
|
if not pages or exc.response.status_code != 429:
|
||||||
raise
|
raise
|
||||||
@@ -603,15 +645,17 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
listed = list(dict.fromkeys(earlier + list(rows)))
|
listed = list(dict.fromkeys(earlier + list(rows)))
|
||||||
if not stop:
|
if not stop:
|
||||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
|
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
|
||||||
resumed and whole)
|
resumed and whole, unasked)
|
||||||
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
||||||
module.NAME, slug, pages, stop)
|
module.NAME, slug, pages, stop)
|
||||||
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
||||||
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
||||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words)
|
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words,
|
||||||
|
remote_unasked=unasked)
|
||||||
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
||||||
"rows": listed, "whole": whole}
|
"rows": listed, "whole": whole}
|
||||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words)
|
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words,
|
||||||
|
remote_unasked=unasked)
|
||||||
|
|
||||||
|
|
||||||
def _storable(cursor: Any) -> bool:
|
def _storable(cursor: Any) -> bool:
|
||||||
@@ -637,7 +681,7 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
|||||||
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
|
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
|
||||||
(a country, a place, or remote),
|
(a country, a place, or remote),
|
||||||
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
|
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
|
||||||
if not declared(module) or not hasattr(module, "facets"):
|
if not hasattr(module, "facets"):
|
||||||
return {}
|
return {}
|
||||||
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
|
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
|
||||||
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
|
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
|
||||||
@@ -659,26 +703,56 @@ def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
|||||||
|
|
||||||
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
|
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
|
||||||
slug: str, get: Get, ask: Ask | None = None, *,
|
slug: str, get: Get, ask: Ask | None = None, *,
|
||||||
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
|
limits: Limits | None = None, can_sweep: bool = True, remote: bool = False) -> Listing:
|
||||||
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
||||||
the facets it needs, the sweep it continues, and the 10% guard. A read
|
the facets it needs, the sweep it continues, and the 10% guard. Nothing
|
||||||
that is filtered, narrowed by terms, or whose reader declares
|
is written here but the facets; `stamp` keeps the rest once the rows are
|
||||||
`empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same
|
stored.
|
||||||
read listed last time (the same `fingerprint`, so a term read is only
|
|
||||||
held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
The guard holds every read that is not a sweep: one that listed under
|
||||||
the rest once the rows are stored."""
|
`SUSPECT_SHARE` of what the same read listed last time (the same
|
||||||
|
`fingerprint`, so a term read is only held to a term read with the same
|
||||||
|
terms and places) raises `Suspect`, and the Source records the target
|
||||||
|
as failing, so its roles stay open. A host whose answer changed shape
|
||||||
|
parses to no rows with a 200, and closing a board on that would lose it
|
||||||
|
whole. It holds from a last count of `SUSPECT_MIN`, or of 1 for a read
|
||||||
|
that is filtered, narrowed or of an `empty="suspect"` reader. A board
|
||||||
|
that really shrank is accepted once the target has been failing for
|
||||||
|
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
||||||
|
stamp then resets the count; a Recheck resets it at once
|
||||||
|
(`plugins.ats.keep_size`)."""
|
||||||
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||||
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
||||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets)
|
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
||||||
suspect = declared(module) and module.READS.empty == "suspect"
|
remote=remote)
|
||||||
|
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
||||||
last = _load(state, f"{prefix}:count:{target}")
|
last = _load(state, f"{prefix}:count:{target}")
|
||||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||||
if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count
|
if (listing.sweep is None and count >= (1 if held else SUSPECT_MIN)
|
||||||
and len(listing.ids) < SUSPECT_SHARE * count):
|
and len(listing.ids) < SUSPECT_SHARE * count):
|
||||||
raise Suspect(f"listed {len(listing.ids)} roles, under "
|
why = (f"listed {len(listing.ids)} roles, under "
|
||||||
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
||||||
|
if _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
|
||||||
|
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
|
||||||
|
module.NAME, slug, why, limits.stale_days)
|
||||||
return listing
|
return listing
|
||||||
|
raise Suspect(why)
|
||||||
|
return listing
|
||||||
|
|
||||||
|
|
||||||
|
def _failing_for(state: Mapping[str, str], key: str, days: int | None) -> bool:
|
||||||
|
"""Whether the failure kept under `key` began at least `days` ago; False
|
||||||
|
for `days` None or no record."""
|
||||||
|
since = str(_load(state, key).get("since") or "")
|
||||||
|
if days is None or not since:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
began = datetime.fromisoformat(since)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
return datetime.now(UTC) - began >= timedelta(days=days)
|
||||||
|
|
||||||
|
|
||||||
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||||
@@ -686,19 +760,22 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
|||||||
"""Record a read whose rows are stored: `ok` (the run's marker),
|
"""Record a read whose rows are stored: `ok` (the run's marker),
|
||||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||||
against), whether it finished a sweep from earlier pulls (`swept`), the
|
against), whether it finished a sweep from earlier pulls (`swept`),
|
||||||
|
whether a filtered read left remote unasked (`remote_unasked`), the
|
||||||
sweep to continue or none, and the count for the 10% guard once a read
|
sweep to continue or none, and the count for the 10% guard once a read
|
||||||
is whole."""
|
is not a sweep. A read that asked nothing keeps the count it had."""
|
||||||
state[f"{prefix}:ok:{target}"] = run
|
state[f"{prefix}:ok:{target}"] = run
|
||||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||||
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||||
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
||||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||||
|
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
||||||
if listing.sweep is not None:
|
if listing.sweep is not None:
|
||||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||||
else:
|
return
|
||||||
state.pop(f"{prefix}:sweep:{target}", None)
|
state.pop(f"{prefix}:sweep:{target}", None)
|
||||||
|
if listing.asks != fingerprint([]):
|
||||||
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
|
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
|
||||||
"count": len(listing.ids)})
|
"count": len(listing.ids)})
|
||||||
|
|
||||||
@@ -766,7 +843,7 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
|||||||
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
||||||
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
||||||
name, slug, row["source_id"], ", ".join(ignored))
|
name, slug, row["source_id"], ", ".join(ignored))
|
||||||
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment]
|
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
|
||||||
if taken.get("location") and "places" not in taken:
|
if taken.get("location") and "places" not in taken:
|
||||||
taken["places"] = [{"text": taken["location"], "remote": None}]
|
taken["places"] = [{"text": taken["location"], "remote": None}]
|
||||||
filled[key] = taken
|
filled[key] = taken
|
||||||
@@ -776,12 +853,12 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
|||||||
for row in rows:
|
for row in rows:
|
||||||
key = compound(row)
|
key = compound(row)
|
||||||
if key in filled:
|
if key in filled:
|
||||||
out.append(row | {"description": ""} | filled[key]) # type: ignore[operator]
|
out.append(row | {"description": ""} | filled[key])
|
||||||
elif key in detailed:
|
elif key in detailed:
|
||||||
out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator]
|
out.append(row | {"description": "", "location": "", "places": [],
|
||||||
"remote": None})
|
"remote": None})
|
||||||
else:
|
else:
|
||||||
out.append(row | {"description": ""}) # type: ignore[operator]
|
out.append(row | {"description": ""})
|
||||||
if listed is not None:
|
if listed is not None:
|
||||||
done &= listed
|
done &= listed
|
||||||
failing &= listed
|
failing &= listed
|
||||||
@@ -791,8 +868,9 @@ def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[
|
|||||||
# -------------------------------------------------------------------- raw
|
# -------------------------------------------------------------------- raw
|
||||||
|
|
||||||
def places_of(row: Mapping[str, Any]) -> list[Place]:
|
def places_of(row: Mapping[str, Any]) -> list[Place]:
|
||||||
"""A row's places: its `places`, or, for a row without (an undeclared
|
"""A row's places: its `places`, or, for a row without (a raw row stored
|
||||||
reader's), its `location` as the one place, split nothing."""
|
before readers returned places), its `location` as the one place, split
|
||||||
|
nothing."""
|
||||||
places = row.get("places")
|
places = row.get("places")
|
||||||
if not isinstance(places, list):
|
if not isinstance(places, list):
|
||||||
location = str(row.get("location") or "").strip()
|
location = str(row.get("location") or "").strip()
|
||||||
|
|||||||
@@ -31,7 +31,8 @@ READS = Reads(
|
|||||||
scope="board", listing="whole", text="listing", complete="whole",
|
scope="board", listing="whole", text="listing", complete="whole",
|
||||||
places="many", empty="suspect",
|
places="many", empty="suspect",
|
||||||
fields=Fields(pay="figures", posted="trusted", remote="stated", employment="stated"),
|
fields=Fields(pay="figures", posted="trusted", remote="stated", employment="stated"),
|
||||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
robots=("read: api.ashbyhq.com answers /robots.txt with 401 and no rules, so "
|
||||||
|
"/posting-api/job-board/<slug> is not disallowed (checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
_API = "api.ashbyhq.com"
|
_API = "api.ashbyhq.com"
|
||||||
|
|||||||
@@ -55,7 +55,9 @@ READS = Reads(
|
|||||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||||
places="many", filters=Filters(country="one", query="one"),
|
places="many", filters=Filters(country="one", query="one"),
|
||||||
fields=Fields(posted="listed", remote="stated"),
|
fields=Fields(posted="listed", remote="stated"),
|
||||||
robots="read: robots.txt allows /careers (checked 2026-09-29)",
|
robots=("read: robots.txt disallows / and allows /api/pcsx, which covers "
|
||||||
|
"/api/pcsx/search and /api/pcsx/position_details (jobs.ericsson.com, "
|
||||||
|
"checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
NAME = "eightfold"
|
NAME = "eightfold"
|
||||||
|
|||||||
@@ -35,7 +35,8 @@ READS = Reads(
|
|||||||
scope="board", listing="paged", text="detail", complete="end",
|
scope="board", listing="paged", text="detail", complete="end",
|
||||||
places="many", empty="suspect",
|
places="many", empty="suspect",
|
||||||
fields=Fields(posted="listed", remote="stated", employment="stated"),
|
fields=Fields(posted="listed", remote="stated", employment="stated"),
|
||||||
robots="read: robots.txt allows /<board> (checked 2026-09-29)",
|
robots=("read: POST /api/public/graphql on jobs.gem.com, whose /robots.txt answers "
|
||||||
|
"404, so nothing is disallowed (checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
NAME = "gem"
|
NAME = "gem"
|
||||||
|
|||||||
@@ -29,7 +29,8 @@ READS = Reads(
|
|||||||
scope="board", listing="whole", text="listing", complete="whole",
|
scope="board", listing="whole", text="listing", complete="whole",
|
||||||
places="many",
|
places="many",
|
||||||
fields=Fields(pay="figures", posted="trusted"),
|
fields=Fields(pay="figures", posted="trusted"),
|
||||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
robots=("read: boards-api.greenhouse.io's robots.txt disallows /embed/ only, so "
|
||||||
|
"/v1/boards/<slug>/jobs is allowed (checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
_API = "boards-api.greenhouse.io"
|
_API = "boards-api.greenhouse.io"
|
||||||
|
|||||||
@@ -34,8 +34,8 @@ READS = Reads(
|
|||||||
scope="board", listing="paged", text="listing", complete="end",
|
scope="board", listing="paged", text="listing", complete="end",
|
||||||
places="copies",
|
places="copies",
|
||||||
fields=Fields(pay="figures", posted="listed", remote="stated", employment="stated"),
|
fields=Fields(pay="figures", posted="listed", remote="stated", employment="stated"),
|
||||||
robots="read: robots.txt allows the careers site; its robots.txt is the single-page "
|
robots=("read: /api/job-ad on <slug>.careers.hibob.com, whose /robots.txt answers the "
|
||||||
"app's index page with no rules (checked 2026-09-29)",
|
"single-page app's index page, no rules (dixa, checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
NAME = "hibob"
|
NAME = "hibob"
|
||||||
|
|||||||
@@ -14,9 +14,11 @@ everything for `*` except the application form. The feed is on the API host,
|
|||||||
which has no robots.txt (a 404, which allows); it is the interface HR Manager
|
which has no robots.txt (a 404, which allows); it is the interface HR Manager
|
||||||
publishes for readers like this one, and the portal's HTML list, which the
|
publishes for readers like this one, and the portal's HTML list, which the
|
||||||
robots.txt does shut out, is never read. The role's link is the application
|
robots.txt does shut out, is never read. The role's link is the application
|
||||||
form the robots.txt names as allowed, and is only shown, not fetched. The
|
form the robots.txt names as allowed, and is only shown, not fetched.
|
||||||
declaration says "disallowed" all the same, and the owner decides whether the
|
`READS.robots` records both hosts and the owner's decision.
|
||||||
feed is read.
|
|
||||||
|
The feed names no place (the title or the text may), so every row has
|
||||||
|
`places == []` (`places="none"`).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -34,9 +36,12 @@ from ._shared import Get, Page, is_web_url, only, parse_dt
|
|||||||
|
|
||||||
READS = Reads(
|
READS = Reads(
|
||||||
scope="board", listing="paged", page_size=100, text="listing", complete="end",
|
scope="board", listing="paged", page_size=100, text="listing", complete="end",
|
||||||
places="one",
|
places="none",
|
||||||
fields=Fields(posted="listed"),
|
fields=Fields(posted="listed"),
|
||||||
robots="disallowed by robots.txt; owner to decide",
|
robots=("read: the feed /JobPortal.svc/<customer>/PositionList/rss/ is on "
|
||||||
|
"api.hr-manager.net, whose /robots.txt answers 404 (checked 2026-09-30); the portal "
|
||||||
|
"candidate.hr-manager.net disallows all but /ApplicationInit.aspx? and is not "
|
||||||
|
"read: owner, 2026-09-29"),
|
||||||
)
|
)
|
||||||
|
|
||||||
NAME = "hrmanager"
|
NAME = "hrmanager"
|
||||||
|
|||||||
@@ -46,7 +46,8 @@ READS = Reads(
|
|||||||
scope="board", listing="paged", page_size=100, text="listing", complete="stated",
|
scope="board", listing="paged", page_size=100, text="listing", complete="stated",
|
||||||
places="many",
|
places="many",
|
||||||
fields=Fields(posted="listed", country="stated", employment="stated"),
|
fields=Fields(posted="listed", country="stated", employment="stated"),
|
||||||
robots="read: robots.txt allows /jobs, with crawl-delay 5 not honoured (checked 2026-09-29)",
|
robots=("read: robots.txt allows /, so /api/jobs, with crawl-delay 5 not honoured "
|
||||||
|
"(jobs.booking.com, checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
NAME = "jibe"
|
NAME = "jibe"
|
||||||
|
|||||||
@@ -34,7 +34,10 @@ READS = Reads(
|
|||||||
scope="board", listing="whole", text="detail", teaser=True, complete="whole",
|
scope="board", listing="whole", text="detail", teaser=True, complete="whole",
|
||||||
places="many",
|
places="many",
|
||||||
fields=Fields(posted="trusted", remote="stated", seniority="stated", employment="stated"),
|
fields=Fields(posted="trusted", remote="stated", seniority="stated", employment="stated"),
|
||||||
robots="read: robots.txt allows /jobs/companies/ (checked 2026-09-29)",
|
robots=("read: robots.txt allows / for * and disallows /jobs/*/preview/, so "
|
||||||
|
"/jobs/companies/<id>/embed/v2/ on cdn.jobylon.com and /jobs/<id>-<name>/ on "
|
||||||
|
"emp.jobylon.com are allowed; it disallows / for named job crawlers (JobrapidoBot, "
|
||||||
|
"JoobleBot, AdzunaBot, ...), which this reader's agent is not (checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
NAME = "jobylon"
|
NAME = "jobylon"
|
||||||
|
|||||||
@@ -32,7 +32,8 @@ READS = Reads(
|
|||||||
places="many",
|
places="many",
|
||||||
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated",
|
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated",
|
||||||
employment="stated"),
|
employment="stated"),
|
||||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
robots=("read: robots.txt of api.lever.co and api.eu.lever.co allows /, so "
|
||||||
|
"/v0/postings/<site>, with crawl-delay 1, one request a board (checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
NAME = "lever"
|
NAME = "lever"
|
||||||
|
|||||||
@@ -60,8 +60,9 @@ READS = Reads(
|
|||||||
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
|
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
|
||||||
complete="stated", places="many", filters=Filters(country="ids", remote="one", query="one"),
|
complete="stated", places="many", filters=Filters(country="ids", remote="one", query="one"),
|
||||||
fields=Fields(posted="trusted", remote="stated"),
|
fields=Fields(posted="trusted", remote="stated"),
|
||||||
robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites/<site>/jobs "
|
robots=("read: /hcmRestApi/resources/latest/recruitingCEJobRequisitions and "
|
||||||
"(checked 2026-09-29)"),
|
"recruitingCEJobRequisitionDetails on the tenant's host, whose /robots.txt answers "
|
||||||
|
"404, so nothing is disallowed (four seed tenants, checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: The page size asked for. 200 was answered in full by Nokia (600 roles in
|
#: The page size asked for. 200 was answered in full by Nokia (600 roles in
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ READS = Reads(
|
|||||||
scope="board", listing="whole", text="listing", complete="whole",
|
scope="board", listing="whole", text="listing", complete="whole",
|
||||||
places="many",
|
places="many",
|
||||||
fields=Fields(pay="figures", posted="trusted", seniority="stated", employment="stated"),
|
fields=Fields(pay="figures", posted="trusted", seniority="stated", employment="stated"),
|
||||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
robots=("read: /xml on <slug>.jobs.personio.de, whose /robots.txt answers 404, so "
|
||||||
|
"nothing is disallowed (penneo, checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: Personio's `salaryInformation/type` spellings, to the plain period word
|
#: Personio's `salaryInformation/type` spellings, to the plain period word
|
||||||
|
|||||||
@@ -52,7 +52,10 @@ READS = Reads(
|
|||||||
scope="board", listing="paged", page_size=100, text="detail", teaser=True,
|
scope="board", listing="paged", page_size=100, text="detail", teaser=True,
|
||||||
complete="stated", places="many", filters=Filters(country="many", query="one"),
|
complete="stated", places="many", filters=Filters(country="many", query="one"),
|
||||||
fields=Fields(posted="trusted", remote="stated"),
|
fields=Fields(posted="trusted", remote="stated"),
|
||||||
robots="read: robots.txt allows /<country>/<lang>/search-results (checked 2026-09-29)",
|
robots=("read: robots.txt has no rule on POST /widgets or /<country>/<lang>/job/<id>; "
|
||||||
|
"it disallows */apply, */px-widgets, */jobcart and the like (careers.king.com, "
|
||||||
|
"careers.justeattakeaway.com; www.careers.philips.com redirects /robots.txt to "
|
||||||
|
"its careers page; checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: The page size asked for. 100 is what the site's own search sends; `size`
|
#: The page size asked for. 100 is what the site's own search sends; `size`
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ READS = Reads(
|
|||||||
text="listing", complete="stated",
|
text="listing", complete="stated",
|
||||||
filters=Filters(query="one", place="one"),
|
filters=Filters(query="one", place="one"),
|
||||||
places="many",
|
places="many",
|
||||||
fields=Fields(posted="trusted", country="stated"),
|
fields=Fields(posted="trusted", remote="stated", country="stated"),
|
||||||
robots="read: the host publishes no robots.txt, /robots.txt answers 404 "
|
robots="read: the host publishes no robots.txt, /robots.txt answers 404 "
|
||||||
"(checked 2026-09-30)")
|
"(checked 2026-09-30)")
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,8 @@ READS = Reads(
|
|||||||
places="many",
|
places="many",
|
||||||
fields=Fields(pay="figures", posted="trusted", remote="stated",
|
fields=Fields(pay="figures", posted="trusted", remote="stated",
|
||||||
seniority="stated", employment="stated"),
|
seniority="stated", employment="stated"),
|
||||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
robots=("read: <slug>.recruitee.com's robots.txt disallows /v/ only, so /api/offers/ "
|
||||||
|
"is allowed (famly, checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: What the board writes as a place for a role that is worked anywhere.
|
#: What the board writes as a place for a role that is worked anywhere.
|
||||||
|
|||||||
@@ -49,7 +49,9 @@ READS = Reads(
|
|||||||
places="one", empty="suspect", filters=Filters(country="one", query="one"),
|
places="one", empty="suspect", filters=Filters(country="one", query="one"),
|
||||||
fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated",
|
fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated",
|
||||||
employment="stated"),
|
employment="stated"),
|
||||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
robots=("read though api.smartrecruiters.com's robots.txt disallows / for every agent "
|
||||||
|
"but LinkedInBot, which alone is allowed /v1/companies/, the path of the listing "
|
||||||
|
"and of each posting: owner to decide, checked 2026-09-30"),
|
||||||
)
|
)
|
||||||
|
|
||||||
_API = "api.smartrecruiters.com"
|
_API = "api.smartrecruiters.com"
|
||||||
|
|||||||
@@ -32,7 +32,9 @@ PATTERNS = (
|
|||||||
READS = Reads(
|
READS = Reads(
|
||||||
scope="board", listing="paged", page_size=50, text="listing", complete="end",
|
scope="board", listing="paged", page_size=50, text="listing", complete="end",
|
||||||
places="many", fields=Fields(posted="trusted", remote="stated"),
|
places="many", fields=Fields(posted="trusted", remote="stated"),
|
||||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
robots=("read: a career site's robots.txt disallows /app/, /messages/, /messenger/, "
|
||||||
|
"/facebook/tab/ and /jobs/internal/, so /jobs.rss is allowed (lunar, which "
|
||||||
|
"redirects to jobs.lunar.app, and careers.trackunit.com, checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: The page size, asked for as `per_page`. A feed with neither parameter
|
#: The page size, asked for as `per_page`. A feed with neither parameter
|
||||||
|
|||||||
@@ -37,7 +37,9 @@ READS = Reads(
|
|||||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||||
places="many",
|
places="many",
|
||||||
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
||||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
robots=("read: apply.workable.com's robots.txt disallows nothing (an empty Disallow), so "
|
||||||
|
"POST /api/v3/accounts/<slug>/jobs and /api/v2/accounts/<slug>/jobs/<shortcode> "
|
||||||
|
"are allowed (checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
_API = "apply.workable.com"
|
_API = "apply.workable.com"
|
||||||
|
|||||||
@@ -52,7 +52,9 @@ READS = Reads(
|
|||||||
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
|
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
|
||||||
complete="stated", places="many", filters=Filters(country="ids", remote="ids", query="one"),
|
complete="stated", places="many", filters=Filters(country="ids", remote="ids", query="one"),
|
||||||
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
||||||
robots="read: robots.txt allows /<Site>/ (checked 2026-09-29)",
|
robots=("read: robots.txt allows /<Site>/ and disallows /refreshFacet/ and has no rule "
|
||||||
|
"on /wday/cxs/<tenant>/<site>/, the listing's POST .../jobs and each role's "
|
||||||
|
"JSON (zendesk.wd1, lego.wd103, saxobank.wd3, checked 2026-09-30)"),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
|
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
|
||||||
|
|||||||
+40
-21
@@ -17,14 +17,19 @@ one run is not closed here. A search that read everything that matches was
|
|||||||
complete (`complete="stated"`, as many distinct results as the stated total),
|
complete (`complete="stated"`, as many distinct results as the stated total),
|
||||||
and the pull closes what it stopped listing; the rest close once no search
|
and the pull closes what it stopped listing; the rest close once no search
|
||||||
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
||||||
|
|
||||||
|
In a pull a search is read by `_runner.read` under the prefix `search`, as a
|
||||||
|
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
|
||||||
|
and a search that fails or is suspect is kept as failing under
|
||||||
|
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
|
||||||
|
the settle reads as it reads a failing board's.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
import sys
|
import sys
|
||||||
from collections.abc import Iterator, Mapping
|
from collections.abc import Iterator, Mapping, MutableMapping
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import ModuleType
|
from types import ModuleType
|
||||||
@@ -65,8 +70,9 @@ def _pages(params: Mapping[str, Any]) -> int:
|
|||||||
return DEFAULT_PAGES
|
return DEFAULT_PAGES
|
||||||
|
|
||||||
|
|
||||||
def _limits(params: Mapping[str, Any]) -> _runner.Limits:
|
def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits:
|
||||||
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
|
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY,
|
||||||
|
stale_days=stale_days)
|
||||||
|
|
||||||
|
|
||||||
def provider_of(target: str) -> str:
|
def provider_of(target: str) -> str:
|
||||||
@@ -76,10 +82,13 @@ def provider_of(target: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
||||||
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
|
keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None,
|
||||||
|
stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]:
|
||||||
"""One search read through the runner: its provider and its listing.
|
"""One search read through the runner: its provider and its listing.
|
||||||
Raises ValueError for an unknown provider or a missing key, before any
|
Raises ValueError for an unknown provider or a missing key, before any
|
||||||
request."""
|
request. With `state` (a pull) it is `_runner.read` under `search`, the
|
||||||
|
10% guard included and never swept; without (a search tried before it is
|
||||||
|
saved) a bare `read_listing`, since there is nothing to compare with."""
|
||||||
provider = provider_of(target)
|
provider = provider_of(target)
|
||||||
if not provider:
|
if not provider:
|
||||||
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
||||||
@@ -88,7 +97,11 @@ def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
|||||||
slug = target.partition(":")[2]
|
slug = target.partition(":")[2]
|
||||||
ask = _runner.ask_for(module, params, keys=keys)
|
ask = _runner.ask_for(module, params, keys=keys)
|
||||||
get = _runner.getter(client, module.allowed(slug), provider)
|
get = _runner.getter(client, module.allowed(slug), provider)
|
||||||
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
|
limits = _limits(params, stale_days)
|
||||||
|
if state is None:
|
||||||
|
return module, _runner.read_listing(module, slug, get, ask, limits=limits)
|
||||||
|
return module, _runner.read(state, "search", target, module, slug, get, ask,
|
||||||
|
limits=limits, can_sweep=False)
|
||||||
|
|
||||||
|
|
||||||
def run_search(target: str, params: dict[str, Any],
|
def run_search(target: str, params: dict[str, Any],
|
||||||
@@ -130,29 +143,35 @@ class SearchSource(Source):
|
|||||||
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
||||||
timespec="seconds")
|
timespec="seconds")
|
||||||
for target, label in self.targets():
|
for target, label in self.targets():
|
||||||
|
yield from self._search(client, target, label, run_marker)
|
||||||
|
|
||||||
|
def _search(self, client: httpx.Client, target: str, label: str,
|
||||||
|
run_marker: str) -> Iterator[Item]:
|
||||||
|
"""One search: read, its rows out as items, then stamped. A search
|
||||||
|
that fails, a suspect answer included, is recorded under
|
||||||
|
`search:failing:<target>` and stamps nothing; the record goes once a
|
||||||
|
read is stamped, after every row went out, as for a board."""
|
||||||
|
failing = f"search:failing:{target}"
|
||||||
provider = provider_of(target)
|
provider = provider_of(target)
|
||||||
if not provider:
|
stale_days = self.options.get("stale_days")
|
||||||
log.warning("search: %r is not provider:name; skipped", target)
|
|
||||||
continue
|
|
||||||
try:
|
try:
|
||||||
_module, listing = _read(client, target, self.params_for(target),
|
_module, listing = _read(client, target, self.params_for(target), self.options,
|
||||||
self.options)
|
self.state, int(stale_days) if stale_days else None)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.warning("search: %s failed (%s)", target, exc)
|
log.warning("search: %s failed (%s)", target, exc)
|
||||||
continue
|
_runner.record_failure(self.state, failing, exc)
|
||||||
|
return
|
||||||
|
|
||||||
seen_ids = []
|
now = datetime.now(UTC)
|
||||||
for row in listing.rows:
|
for row in listing.rows:
|
||||||
source_id = f"{provider}:{row['source_id']}"
|
|
||||||
seen_ids.append(source_id)
|
|
||||||
description = row.get("description") or ""
|
description = row.get("description") or ""
|
||||||
yield Item(
|
yield Item(
|
||||||
source_id=source_id,
|
source_id=f"{provider}:{row['source_id']}",
|
||||||
location=row["url"],
|
location=row["url"],
|
||||||
title=row["title"],
|
title=row["title"],
|
||||||
excerpt=description[:300],
|
excerpt=description[:300],
|
||||||
body=description,
|
body=description,
|
||||||
published_at=row.get("posted"),
|
published_at=row.get("posted") or now,
|
||||||
subscription=label,
|
subscription=label,
|
||||||
kind="role",
|
kind="role",
|
||||||
raw=_runner.raw_of(row) | {
|
raw=_runner.raw_of(row) | {
|
||||||
@@ -161,6 +180,6 @@ class SearchSource(Source):
|
|||||||
"search": target,
|
"search": target,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
self.state[f"search:ok:{target}"] = run_marker
|
_runner.stamp(self.state, "search", target, run_marker, listing,
|
||||||
self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
|
[f"{provider}:{i}" for i in listing.ids])
|
||||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
self.state.pop(failing, None)
|
||||||
Vendored
+1
-1
@@ -1 +1 @@
|
|||||||
{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]}
|
{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":50000,"payTransparencyMaxSalary":70000,"payTransparencySalaryCurrency":"EUR","payTransparencySalaryPayPeriod":"Yearly","payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]}
|
||||||
File renamed without changes.
File renamed without changes.
+1
-1
@@ -1 +1 @@
|
|||||||
{"data":[{"id":"581878","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe will help you \u003cstrong\u003egrow your career:\u003c/strong\u003e We help you identify areas of growth for you, whether you want to grow as a specialist in your field or eventually want to go into a leadership/management role.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person from time to time.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will receive an\u003cstrong\u003e individual education budget \u003c/strong\u003eand can educate yourself during work hours: you can spend several hours each month attending courses, reading books, or getting involved in communities.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFull-time position, employed through GI Group, our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eLooking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Job Benefits","compensation":null,"compensation_minimum":null,"compensation_maximum":null,"compensation_currency":null,"compensation_frequency":null,"compensation_visible":false,"deadline_at":null,"description":"\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eData Detective Wanted to join our Data \u0026amp; Analytics Team\u003c/strong\u003e \u003cbr\u003e\u003cbr\u003e\u003cem\u003eLocation: Bucharest, Romania\u003c/em\u003e\u003cbr\u003e\u003cbr\u003eAs a \u003cstrong\u003eData Operations Analyst \u003c/strong\u003eat Xeneta, you’ll learn how to master complex data analytics within the world of supply chain and logistics. Xeneta is one of the world's leading data platforms providing market intelligence to the ocean and air freight industry. We have a big team of skilled Data Analysts that will help you learn the ropes from the ground up, while enabling you to collect and analyze data in one of the world’s greatest container shipping datasets. \u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eBring your analytical mindset and a genuine curiosity for how global shipping works and join our team!\u0026nbsp;\u003c/div\u003e","employment_type":"permanent_full_time","employment_type_text":"Permanent - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eJoin our Liner Data \u0026amp; Analytics Team (LDAT) and own a portfolio of container shipping services for theLine truncated
|
{"data":[{"id":"581878","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe will help you \u003cstrong\u003egrow your career:\u003c/strong\u003e We help you identify areas of growth for you, whether you want to grow as a specialist in your field or eventually want to go into a leadership/management role.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person from time to time.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will receive an\u003cstrong\u003e individual education budget \u003c/strong\u003eand can educate yourself during work hours: you can spend several hours each month attending courses, reading books, or getting involved in communities.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFull-time position, employed through GI Group, our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eLooking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Job Benefits","compensation":null,"compensation_minimum":60000,"compensation_maximum":80000,"compensation_currency":"EUR","compensation_frequency":"annually","compensation_visible":true,"deadline_at":null,"description":"\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eData Detective Wanted to join our Data \u0026amp; Analytics Team\u003c/strong\u003e \u003cbr\u003e\u003cbr\u003e\u003cem\u003eLocation: Bucharest, Romania\u003c/em\u003e\u003cbr\u003e\u003cbr\u003eAs a \u003cstrong\u003eData Operations Analyst \u003c/strong\u003eat Xeneta, you’ll learn how to master complex data analytics within the world of supply chain and logistics. Xeneta is one of the world's leading data platforms providing market intelligence to the ocean and air freight industry. We have a big team of skilled Data Analysts that will help you learn the ropes from the ground up, while enabling you to collect and analyze data in one of the world’s greatest container shipping datasets. \u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eBring your analytical mindset and a genuine curiosity for how global shipping works and join our team!\u0026nbsp;\u003c/div\u003e","employment_type":"permanent_full_time","employment_type_text":"Permanent - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eJoin our Liner Data \u0026amp; Analytics Team (LDAT) and own a portfolio of container shipping servicesLine truncated
|
||||||
File renamed without changes.
File renamed without changes.
+3
-1
@@ -16,7 +16,9 @@
|
|||||||
"fullLocation": "Berlin, Germany"
|
"fullLocation": "Berlin, Germany"
|
||||||
},
|
},
|
||||||
"department": {"id": "2294038", "label": "Engineering, Infrastructure and Operations"},
|
"department": {"id": "2294038", "label": "Engineering, Infrastructure and Operations"},
|
||||||
"function": {"id": "engineering", "label": "Engineering"}
|
"function": {"id": "engineering", "label": "Engineering"},
|
||||||
|
"typeOfEmployment": {"id": "permanent", "label": "Full-time"},
|
||||||
|
"experienceLevel": {"id": "director", "label": "Director"}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "744000147485112",
|
"id": "744000147485112",
|
||||||
|
|||||||
@@ -12,6 +12,11 @@ file holds `app`'s own layers.
|
|||||||
there the door is the tool: `desk.tools.<tool>`, one level, no deeper.
|
there the door is the tool: `desk.tools.<tool>`, one level, no deeper.
|
||||||
3. **Plugins.** A file under `plugins/` imports `desk` and nothing else from
|
3. **Plugins.** A file under `plugins/` imports `desk` and nothing else from
|
||||||
these trees. A plugin is app-agnostic; it never sees `app`.
|
these trees. A plugin is app-agnostic; it never sees `app`.
|
||||||
|
4. **Readers.** A file of `plugins/readers/` imports the standard library,
|
||||||
|
`httpx`, `pydantic`, `desk` through its doors and, within the package,
|
||||||
|
`._contract`, `._shared` and `._countries` only: a reader shares no code
|
||||||
|
with another reader, nor with `_runner`, which reads the readers and so
|
||||||
|
alone may import the package's registry, `plugins.readers`.
|
||||||
|
|
||||||
Tests are held to none of this.
|
Tests are held to none of this.
|
||||||
"""
|
"""
|
||||||
@@ -21,6 +26,7 @@ from __future__ import annotations
|
|||||||
import ast
|
import ast
|
||||||
import importlib
|
import importlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -88,6 +94,13 @@ def _package(path: Path) -> str:
|
|||||||
|
|
||||||
APP_FILES = sorted(p for p in (ROOT / APP).rglob("*.py") if "__pycache__" not in p.parts)
|
APP_FILES = sorted(p for p in (ROOT / APP).rglob("*.py") if "__pycache__" not in p.parts)
|
||||||
PLUGIN_FILES = sorted((ROOT / "plugins").glob("*.py"))
|
PLUGIN_FILES = sorted((ROOT / "plugins").glob("*.py"))
|
||||||
|
READER_FILES = sorted((ROOT / "plugins" / "readers").glob("*.py"))
|
||||||
|
|
||||||
|
#: What a reader may import beside the standard library and `desk`.
|
||||||
|
READER_THIRD_PARTY = frozenset({"httpx", "pydantic"})
|
||||||
|
|
||||||
|
#: The package's own modules a reader may import.
|
||||||
|
READER_SIBLINGS = frozenset({"_contract", "_shared", "_countries"})
|
||||||
|
|
||||||
|
|
||||||
def _outside(here: str, source: str):
|
def _outside(here: str, source: str):
|
||||||
@@ -120,6 +133,38 @@ def plugin_violations(source: str) -> list[str]:
|
|||||||
return wrong
|
return wrong
|
||||||
|
|
||||||
|
|
||||||
|
def reader_violations(source: str, registry: bool = False) -> list[str]:
|
||||||
|
"""Every import in a file of `plugins/readers/` outside rule 4;
|
||||||
|
`registry` allows `from plugins import readers`, for `_runner`."""
|
||||||
|
wrong = []
|
||||||
|
for node in ast.walk(ast.parse(source)):
|
||||||
|
if registry and isinstance(node, ast.ImportFrom) and node.level == 0 \
|
||||||
|
and node.module == "plugins" and [a.name for a in node.names] == ["readers"]:
|
||||||
|
continue
|
||||||
|
if isinstance(node, ast.Import):
|
||||||
|
tops = [(a.name.split(".")[0], a.name) for a in node.names]
|
||||||
|
elif isinstance(node, ast.ImportFrom) and node.level == 0:
|
||||||
|
tops = [((node.module or "").split(".")[0], node.module or "")]
|
||||||
|
elif isinstance(node, ast.ImportFrom):
|
||||||
|
names = {node.module} if node.module else {a.name for a in node.names}
|
||||||
|
if node.level != 1 or not names <= READER_SIBLINGS:
|
||||||
|
wrong.append(f"line {node.lineno}: {'.' * node.level}{node.module or ''} "
|
||||||
|
f"{sorted(names)} (a reader imports {', '.join(sorted(READER_SIBLINGS))})")
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
for top, module in tops:
|
||||||
|
if top == "desk":
|
||||||
|
if isinstance(node, ast.ImportFrom) and (
|
||||||
|
reason := _door_violation(module, [a.name for a in node.names
|
||||||
|
if not _is_module(f"{module}.{a.name}")])):
|
||||||
|
wrong.append(f"line {node.lineno}: {module} ({reason})")
|
||||||
|
elif top not in sys.stdlib_module_names and top not in READER_THIRD_PARTY:
|
||||||
|
wrong.append(f"line {node.lineno}: {module} (a reader imports the standard "
|
||||||
|
f"library, httpx, pydantic and desk)")
|
||||||
|
return wrong
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
|
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
|
||||||
def test_a_package_imports_only_from_below_it(path: Path) -> None:
|
def test_a_package_imports_only_from_below_it(path: Path) -> None:
|
||||||
wrong = direction_violations(_package(path), path.read_text(encoding="utf-8"))
|
wrong = direction_violations(_package(path), path.read_text(encoding="utf-8"))
|
||||||
@@ -141,6 +186,12 @@ def test_a_plugin_knows_only_desk(path: Path) -> None:
|
|||||||
assert not wrong, "\n".join(wrong)
|
assert not wrong, "\n".join(wrong)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("path", READER_FILES, ids=lambda p: p.name)
|
||||||
|
def test_a_reader_knows_only_its_contract_and_helpers(path: Path) -> None:
|
||||||
|
wrong = reader_violations(path.read_text(encoding="utf-8"), path.name == "_runner.py")
|
||||||
|
assert not wrong, "\n".join(wrong)
|
||||||
|
|
||||||
|
|
||||||
def test_the_rules_catch_what_they_are_for() -> None:
|
def test_the_rules_catch_what_they_are_for() -> None:
|
||||||
"""The checker itself, on sources that break each rule."""
|
"""The checker itself, on sources that break each rule."""
|
||||||
assert plugin_violations("from app.storage import Store")
|
assert plugin_violations("from app.storage import Store")
|
||||||
@@ -148,6 +199,23 @@ def test_the_rules_catch_what_they_are_for() -> None:
|
|||||||
assert plugin_violations("from desk.sources.base import Source")
|
assert plugin_violations("from desk.sources.base import Source")
|
||||||
assert not plugin_violations("from desk.sources import Source, Item")
|
assert not plugin_violations("from desk.sources import Source, Item")
|
||||||
|
|
||||||
|
assert reader_violations("from plugins import ats")
|
||||||
|
assert reader_violations("from plugins import readers")
|
||||||
|
assert not reader_violations("from plugins import readers", registry=True)
|
||||||
|
assert reader_violations("from plugins import ats", registry=True)
|
||||||
|
assert reader_violations("from app.prep import words")
|
||||||
|
assert reader_violations("from . import _runner")
|
||||||
|
assert reader_violations("from ._runner import fetch")
|
||||||
|
assert reader_violations("from .lever import parse")
|
||||||
|
assert reader_violations("from .. import ats")
|
||||||
|
assert reader_violations("import requests")
|
||||||
|
assert reader_violations("from desk.sources.base import Source")
|
||||||
|
assert not reader_violations("import json\nfrom urllib.parse import quote\nimport httpx\n"
|
||||||
|
"from pydantic import ConfigDict\n"
|
||||||
|
"from desk.sources import strip_html\n"
|
||||||
|
"from . import _contract\nfrom ._shared import only\n"
|
||||||
|
"from ._countries import code_of")
|
||||||
|
|
||||||
assert direction_violations("storage", "from app.web import TABS")
|
assert direction_violations("storage", "from app.web import TABS")
|
||||||
assert direction_violations("web", "from app.main import create_app")
|
assert direction_violations("web", "from app.main import create_app")
|
||||||
assert not direction_violations("main", "from app.web import pages_router")
|
assert not direction_violations("main", "from app.web import pages_router")
|
||||||
|
|||||||
@@ -35,10 +35,10 @@ def test_the_employment_type_is_the_hosts_word() -> None:
|
|||||||
assert (first["employment"], second["employment"]) == ("FullTime", "Contract")
|
assert (first["employment"], second["employment"]) == ("FullTime", "Contract")
|
||||||
|
|
||||||
|
|
||||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
|
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
|
||||||
reads = ashby.READS
|
reads = ashby.READS
|
||||||
assert (reads.listing, reads.text, reads.places, reads.empty) == (
|
assert (reads.listing, reads.text, reads.places, reads.empty) == (
|
||||||
"whole", "listing", "many", "suspect")
|
"whole", "listing", "many", "suspect")
|
||||||
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
|
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
|
||||||
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
|
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
|
||||||
assert ashby.feed_url("acme", Ask()).endswith("/job-board/acme?includeCompensation=true")
|
assert ashby.feed_url("acme", Ask()).endswith("/job-board/acme?includeCompensation=true")
|
||||||
@@ -156,7 +156,6 @@ def test_it_declares_what_it_reads() -> None:
|
|||||||
assert (reads.listing, reads.text, reads.places, reads.fields.posted) == (
|
assert (reads.listing, reads.text, reads.places, reads.fields.posted) == (
|
||||||
"whole", "detail", "one", "trusted")
|
"whole", "detail", "one", "trusted")
|
||||||
assert reads.robots.startswith("read: robots.txt allows /careers")
|
assert reads.robots.startswith("read: robots.txt allows /careers")
|
||||||
assert not hasattr(bamboohr, "robots_page")
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
||||||
@@ -188,4 +187,3 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None:
|
|||||||
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
|
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
|
||||||
'<a href="https://resources.bamboohr.com/y">z</a>')
|
'<a href="https://resources.bamboohr.com/y">z</a>')
|
||||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
|
||||||
assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
|
||||||
@@ -113,7 +113,6 @@ def test_it_declares_a_whole_listing_read_from_the_page() -> None:
|
|||||||
assert (reads.listing, reads.text, reads.places, reads.complete) == (
|
assert (reads.listing, reads.text, reads.places, reads.complete) == (
|
||||||
"whole", "listing", "many", "whole")
|
"whole", "listing", "many", "whole")
|
||||||
assert reads.fields.pay == "line" and "/api/ is disallowed and not read" in reads.robots
|
assert reads.fields.pay == "line" and "/api/ is disallowed and not read" in reads.robots
|
||||||
assert not hasattr(deel, "robots_page")
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|||||||
@@ -201,7 +201,6 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None
|
|||||||
found = [(h, s) for h, s, _ in discover.boards_in(page)]
|
found = [(h, s) for h, s, _ in discover.boards_in(page)]
|
||||||
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
|
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
|
||||||
assert ("eightfold", "") in found
|
assert ("eightfold", "") in found
|
||||||
assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
|
||||||
|
|
||||||
|
|
||||||
def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None:
|
def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None:
|
||||||
@@ -210,5 +209,4 @@ def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request(
|
|||||||
"paged", 10, "detail", "stated", "many")
|
"paged", 10, "detail", "stated", "many")
|
||||||
assert reads.filters.declared() == {"country": "one", "query": "one"}
|
assert reads.filters.declared() == {"country": "one", "query": "one"}
|
||||||
assert reads.fields.posted == "listed"
|
assert reads.fields.posted == "listed"
|
||||||
assert reads.robots.startswith("read: robots.txt allows /careers")
|
assert "/api/pcsx/search" in reads.robots and "/api/pcsx/position_details" in reads.robots
|
||||||
assert not hasattr(eightfold, "robots_page")
|
|
||||||
@@ -103,8 +103,7 @@ def test_it_declares_one_page_that_may_be_empty_by_mistake() -> None:
|
|||||||
reads = gem.READS
|
reads = gem.READS
|
||||||
assert (reads.listing, reads.text, reads.places, reads.empty) == (
|
assert (reads.listing, reads.text, reads.places, reads.empty) == (
|
||||||
"paged", "detail", "many", "suspect")
|
"paged", "detail", "many", "suspect")
|
||||||
assert reads.robots.startswith("read: robots.txt allows /<board>")
|
assert "POST /api/public/graphql" in reads.robots
|
||||||
assert not hasattr(gem, "robots_page")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(("body", "message"), [
|
@pytest.mark.parametrize(("body", "message"), [
|
||||||
@@ -179,4 +178,3 @@ def test_discovery_finds_the_board_keeping_its_case() -> None:
|
|||||||
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
|
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
|
||||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
|
||||||
("gem", "Productboard"), ("gem", "sennder-gmbh")]
|
("gem", "Productboard"), ("gem", "sennder-gmbh")]
|
||||||
assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
|
||||||
@@ -43,10 +43,10 @@ def test_without_offices_the_name_is_the_place() -> None:
|
|||||||
assert nothing["places"] == [] and nothing["location"] == ""
|
assert nothing["places"] == [] and nothing["location"] == ""
|
||||||
|
|
||||||
|
|
||||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
|
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
|
||||||
reads = greenhouse.READS
|
reads = greenhouse.READS
|
||||||
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
|
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
|
||||||
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
|
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
|
||||||
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
|
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
|
||||||
assert greenhouse.feed_url("acme", Ask()).endswith("/boards/acme/jobs?content=true&pay_transparency=true")
|
assert greenhouse.feed_url("acme", Ask()).endswith("/boards/acme/jobs?content=true&pay_transparency=true")
|
||||||
assert greenhouse.fallback_url("acme").endswith("/boards/acme/jobs")
|
assert greenhouse.fallback_url("acme").endswith("/boards/acme/jobs")
|
||||||
@@ -163,8 +163,7 @@ def test_it_declares_copies_with_a_page_and_a_robots_decision() -> None:
|
|||||||
reads = hibob.READS
|
reads = hibob.READS
|
||||||
assert (reads.listing, reads.text, reads.places, reads.complete) == (
|
assert (reads.listing, reads.text, reads.places, reads.complete) == (
|
||||||
"paged", "listing", "copies", "end")
|
"paged", "listing", "copies", "end")
|
||||||
assert reads.robots.startswith("read: robots.txt allows")
|
assert reads.robots.startswith("read: /api/job-ad")
|
||||||
assert not hasattr(hibob, "robots_page")
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|||||||
@@ -122,9 +122,8 @@ def test_the_feed_states_no_place_and_the_robots_decision_is_the_owners() -> Non
|
|||||||
assert all(r["places"] == [] for r in rows)
|
assert all(r["places"] == [] for r in rows)
|
||||||
reads = hrmanager.READS
|
reads = hrmanager.READS
|
||||||
assert (reads.listing, reads.page_size, reads.text, reads.places) == (
|
assert (reads.listing, reads.page_size, reads.text, reads.places) == (
|
||||||
"paged", 100, "listing", "one")
|
"paged", 100, "listing", "none")
|
||||||
assert reads.robots == "disallowed by robots.txt; owner to decide"
|
assert "api.hr-manager.net" in reads.robots and "owner, 2026-09-29" in reads.robots
|
||||||
assert not hasattr(hrmanager, "robots_page")
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|||||||
@@ -176,7 +176,6 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
|
|||||||
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
|
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
|
||||||
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
|
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
|
||||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
|
||||||
assert "jibe" not in {h for h, _ in discover._UNREADABLE}
|
|
||||||
|
|
||||||
|
|
||||||
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
|
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
|
||||||
@@ -184,4 +183,3 @@ def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
|
|||||||
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
|
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
|
||||||
"paged", 100, "listing", "stated", "many")
|
"paged", 100, "listing", "stated", "many")
|
||||||
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")
|
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")
|
||||||
assert not hasattr(jibe, "robots_page")
|
|
||||||
@@ -87,8 +87,8 @@ def _get(pages: dict[str, bytes], asked: list[str] | None = None) -> Any:
|
|||||||
def test_detail_reads_the_role_pages_json_ld() -> None:
|
def test_detail_reads_the_role_pages_json_ld() -> None:
|
||||||
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
||||||
asked: list[str] = []
|
asked: list[str] = []
|
||||||
pages = {ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
|
pages = {ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
|
||||||
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
|
ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
|
||||||
first = jobylon.detail(_get(pages, asked), "3143", rows["385191"])
|
first = jobylon.detail(_get(pages, asked), "3143", rows["385191"])
|
||||||
assert asked == [ROLE]
|
assert asked == [ROLE]
|
||||||
assert set(first) <= set(ats.DETAIL_FIELDS) | {"places"}
|
assert set(first) <= set(ats.DETAIL_FIELDS) | {"places"}
|
||||||
@@ -116,8 +116,8 @@ def test_detail_without_a_job_posting_fails_the_read() -> None:
|
|||||||
|
|
||||||
def test_the_board_is_one_request_and_each_role_is_read_on_its_own_page() -> None:
|
def test_the_board_is_one_request_and_each_role_is_read_on_its_own_page() -> None:
|
||||||
asked: list[str] = []
|
asked: list[str] = []
|
||||||
pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
|
pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
|
||||||
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
|
ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
|
||||||
get = _get(pages, asked)
|
get = _get(pages, asked)
|
||||||
rows = jobylon.parse(get(jobylon.feed_url("3143", Ask())), "3143")
|
rows = jobylon.parse(get(jobylon.feed_url("3143", Ask())), "3143")
|
||||||
assert len(rows) == 5 and asked == [FEED]
|
assert len(rows) == 5 and asked == [FEED]
|
||||||
@@ -150,8 +150,7 @@ def test_it_declares_a_whole_listing_with_a_teaser_and_detail() -> None:
|
|||||||
assert (reads.listing, reads.text, reads.teaser, reads.places) == (
|
assert (reads.listing, reads.text, reads.teaser, reads.places) == (
|
||||||
"whole", "detail", True, "many")
|
"whole", "detail", True, "many")
|
||||||
assert (reads.fields.posted, reads.fields.seniority) == ("trusted", "stated")
|
assert (reads.fields.posted, reads.fields.seniority) == ("trusted", "stated")
|
||||||
assert reads.robots.startswith("read: robots.txt allows /jobs/companies/")
|
assert "/jobs/companies/<id>/embed/v2/" in reads.robots
|
||||||
assert not hasattr(jobylon, "robots_page")
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|||||||
@@ -33,9 +33,9 @@ def test_the_commitment_is_the_hosts_word() -> None:
|
|||||||
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
|
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
|
||||||
|
|
||||||
|
|
||||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
|
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
|
||||||
reads = lever.READS
|
reads = lever.READS
|
||||||
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
|
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
|
||||||
assert reads.fields.posted == "trusted" and reads.fields.country == "stated"
|
assert reads.fields.posted == "trusted" and reads.fields.country == "stated"
|
||||||
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
|
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
|
||||||
assert lever.feed_url("eu/tomtom", Ask()) == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
|
assert lever.feed_url("eu/tomtom", Ask()) == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
|
||||||
@@ -253,7 +253,6 @@ def test_the_careers_page_is_the_candidate_site() -> None:
|
|||||||
url = f"https://{HOST}/hcmUI/CandidateExperience/en/sites/CX_1/jobs"
|
url = f"https://{HOST}/hcmUI/CandidateExperience/en/sites/CX_1/jobs"
|
||||||
assert oracle.careers_url(SLUG) == ats.careers_url(f"oracle:{SLUG}") == url
|
assert oracle.careers_url(SLUG) == ats.careers_url(f"oracle:{SLUG}") == url
|
||||||
assert oracle.allowed(SLUG)(httpx.URL(url).host)
|
assert oracle.allowed(SLUG)(httpx.URL(url).host)
|
||||||
assert not hasattr(oracle, "robots_page") # decided when built, see READS.robots
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_slug_written_with_capitals_names_the_same_pod() -> None:
|
def test_a_slug_written_with_capitals_names_the_same_pod() -> None:
|
||||||
|
|||||||
@@ -24,8 +24,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
|
|||||||
reads = personio.READS
|
reads = personio.READS
|
||||||
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
|
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
|
||||||
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
|
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
|
||||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
assert reads.robots.startswith("read: /xml")
|
||||||
assert not hasattr(personio, "robots_page")
|
|
||||||
assert personio.feed_url("acme", Ask()) == "https://acme.jobs.personio.de/xml"
|
assert personio.feed_url("acme", Ask()) == "https://acme.jobs.personio.de/xml"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -289,7 +289,6 @@ def test_the_careers_page_is_the_search_page() -> None:
|
|||||||
url = "https://careers.king.com/us/en/search-results"
|
url = "https://careers.king.com/us/en/search-results"
|
||||||
assert phenom.careers_url(SLUG) == ats.careers_url(f"phenom:{SLUG}") == url
|
assert phenom.careers_url(SLUG) == ats.careers_url(f"phenom:{SLUG}") == url
|
||||||
assert phenom.allowed(SLUG)(httpx.URL(url).host)
|
assert phenom.allowed(SLUG)(httpx.URL(url).host)
|
||||||
assert not hasattr(phenom, "robots_page") # decided when built, see READS.robots
|
|
||||||
assert phenom.careers_url("www.careers.philips.com/global/en") == \
|
assert phenom.careers_url("www.careers.philips.com/global/en") == \
|
||||||
"https://www.careers.philips.com/global/en/search-results"
|
"https://www.careers.philips.com/global/en/search-results"
|
||||||
|
|
||||||
|
|||||||
@@ -116,7 +116,6 @@ def test_the_board_is_read_through_the_app_and_is_complete() -> None:
|
|||||||
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
||||||
assert pinpoint.slug_ok("xeneta") and pinpoint.slug_ok("Xeneta") and pinpoint.slug_ok("a-b1")
|
assert pinpoint.slug_ok("xeneta") and pinpoint.slug_ok("Xeneta") and pinpoint.slug_ok("a-b1")
|
||||||
assert pinpoint.careers_url("xeneta") == "https://xeneta.pinpointhq.com/"
|
assert pinpoint.careers_url("xeneta") == "https://xeneta.pinpointhq.com/"
|
||||||
assert not hasattr(pinpoint, "robots_page") # decided when built, see READS.robots
|
|
||||||
assert pinpoint.feed_url("Xeneta") == "https://xeneta.pinpointhq.com/postings.json"
|
assert pinpoint.feed_url("Xeneta") == "https://xeneta.pinpointhq.com/postings.json"
|
||||||
allow = pinpoint.allowed("Xeneta")
|
allow = pinpoint.allowed("Xeneta")
|
||||||
assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com")
|
assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com")
|
||||||
@@ -146,7 +145,6 @@ def test_discovery_finds_the_subdomain() -> None:
|
|||||||
'<script src="https://www.pinpointhq.com/embed.js"></script>')
|
'<script src="https://www.pinpointhq.com/embed.js"></script>')
|
||||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
|
||||||
assert discover.boards_in("https://pinpointhq.com/") == []
|
assert discover.boards_in("https://pinpointhq.com/") == []
|
||||||
assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_pinpoint_board_is_not_probed() -> None:
|
def test_a_pinpoint_board_is_not_probed() -> None:
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
|
|||||||
reads = recruitee.READS
|
reads = recruitee.READS
|
||||||
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
|
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
|
||||||
assert reads.fields.posted == "trusted" and reads.fields.country == "none"
|
assert reads.fields.posted == "trusted" and reads.fields.country == "none"
|
||||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
assert "/api/offers/ is allowed" in reads.robots
|
||||||
assert recruitee.feed_url("acme", Ask()) == "https://acme.recruitee.com/api/offers/"
|
assert recruitee.feed_url("acme", Ask()) == "https://acme.recruitee.com/api/offers/"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ def _get(answer: bytes, asked: list[str] | None = None) -> Any:
|
|||||||
def test_detail_reads_description_places_date_and_a_single_yearly_range() -> None:
|
def test_detail_reads_description_places_date_and_a_single_yearly_range() -> None:
|
||||||
row = {r["source_id"]: r for r in rippling.parse(BOARD, "rippling")}[DIRECTOR]
|
row = {r["source_id"]: r for r in rippling.parse(BOARD, "rippling")}[DIRECTOR]
|
||||||
asked: list[str] = []
|
asked: list[str] = []
|
||||||
found = rippling.detail(_get((FIXTURES / "rippling-role.json").read_bytes(), asked),
|
found = rippling.detail(_get((FIXTURES / "rippling-detail.json").read_bytes(), asked),
|
||||||
"rippling", row)
|
"rippling", row)
|
||||||
assert asked == [f"https://api.rippling.com/platform/api/ats/v1/board/rippling/jobs/{DIRECTOR}"]
|
assert asked == [f"https://api.rippling.com/platform/api/ats/v1/board/rippling/jobs/{DIRECTOR}"]
|
||||||
assert found["description"].startswith("About The Role")
|
assert found["description"].startswith("About The Role")
|
||||||
@@ -96,7 +96,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non
|
|||||||
|
|
||||||
|
|
||||||
def test_detail_without_pay_leaves_the_pay_columns_out() -> None:
|
def test_detail_without_pay_leaves_the_pay_columns_out() -> None:
|
||||||
found = rippling.detail(_get((FIXTURES / "rippling-role-2.json").read_bytes()),
|
found = rippling.detail(_get((FIXTURES / "rippling-detail-2.json").read_bytes()),
|
||||||
"prisma-careers", {"source_id": "x"})
|
"prisma-careers", {"source_id": "x"})
|
||||||
assert found["description"].startswith("What is it like to work at Prisma?")
|
assert found["description"].startswith("What is it like to work at Prisma?")
|
||||||
assert found["location"] == "Remote (Germany)"
|
assert found["location"] == "Remote (Germany)"
|
||||||
@@ -165,7 +165,6 @@ def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
|
|||||||
def test_the_urls() -> None:
|
def test_the_urls() -> None:
|
||||||
assert rippling.feed_url("rippling") == FEED
|
assert rippling.feed_url("rippling") == FEED
|
||||||
assert rippling.careers_url("prisma-careers") == "https://ats.rippling.com/prisma-careers/jobs"
|
assert rippling.careers_url("prisma-careers") == "https://ats.rippling.com/prisma-careers/jobs"
|
||||||
assert not hasattr(rippling, "robots_page") # decided when built, see READS.robots
|
|
||||||
allow = rippling.allowed("rippling")
|
allow = rippling.allowed("rippling")
|
||||||
assert allow("api.rippling.com") and allow("ATS.RIPPLING.COM")
|
assert allow("api.rippling.com") and allow("ATS.RIPPLING.COM")
|
||||||
for host in ("rippling.com", "www.rippling.com", "api.rippling.com.evil.example", "localhost"):
|
for host in ("rippling.com", "www.rippling.com", "api.rippling.com.evil.example", "localhost"):
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ def test_the_declaration_says_what_the_list_gives() -> None:
|
|||||||
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.empty) == \
|
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.empty) == \
|
||||||
("paged", 100, "one", "detail", "suspect")
|
("paged", 100, "one", "detail", "suspect")
|
||||||
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
|
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
|
||||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
assert reads.robots.startswith("read though") and "owner to decide" in reads.robots
|
||||||
|
|
||||||
|
|
||||||
def test_page_reads_by_offset_and_takes_the_ask() -> None:
|
def test_page_reads_by_offset_and_takes_the_ask() -> None:
|
||||||
|
|||||||
@@ -451,7 +451,6 @@ def test_careers_url_is_the_search_page() -> None:
|
|||||||
page = successfactors.careers_url(slug)
|
page = successfactors.careers_url(slug)
|
||||||
assert httpx.URL(page).host == slug.split("/")[0]
|
assert httpx.URL(page).host == slug.split("/")[0]
|
||||||
assert successfactors.allowed(slug)(httpx.URL(page).host)
|
assert successfactors.allowed(slug)(httpx.URL(page).host)
|
||||||
assert not hasattr(successfactors, "robots_page") # decided when built, see READS.robots
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_module_is_case_sensitive_about_the_prefix() -> None:
|
def test_the_module_is_case_sensitive_about_the_prefix() -> None:
|
||||||
|
|||||||
@@ -24,8 +24,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
|
|||||||
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.complete) == \
|
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.complete) == \
|
||||||
("paged", 50, "many", "listing", "end")
|
("paged", 50, "many", "listing", "end")
|
||||||
assert reads.fields.posted == "trusted" and reads.fields.remote == "stated"
|
assert reads.fields.posted == "trusted" and reads.fields.remote == "stated"
|
||||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
assert "/jobs.rss is allowed" in reads.robots
|
||||||
assert not hasattr(teamtailor, "robots_page")
|
|
||||||
|
|
||||||
|
|
||||||
def test_page_takes_the_ask_and_reads_the_same_url() -> None:
|
def test_page_takes_the_ask_and_reads_the_same_url() -> None:
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ def test_the_declaration_says_what_the_list_gives() -> None:
|
|||||||
assert (reads.listing, reads.page_size, reads.places, reads.text) == \
|
assert (reads.listing, reads.page_size, reads.places, reads.text) == \
|
||||||
("paged", 10, "many", "detail")
|
("paged", 10, "many", "detail")
|
||||||
assert reads.fields.posted == "trusted" and reads.fields.employment == "stated"
|
assert reads.fields.posted == "trusted" and reads.fields.employment == "stated"
|
||||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
assert "/api/v3/accounts/<slug>/jobs" in reads.robots
|
||||||
|
|
||||||
|
|
||||||
def test_page_takes_the_ask_and_posts_the_same_body() -> None:
|
def test_page_takes_the_ask_and_posts_the_same_body() -> None:
|
||||||
|
|||||||
@@ -446,10 +446,10 @@ def test_the_case_of_the_site_is_kept() -> None:
|
|||||||
assert "workday" in ats.HOSTS and ats.HOSTS["workday"] == "Workday"
|
assert "workday" in ats.HOSTS and ats.HOSTS["workday"] == "Workday"
|
||||||
|
|
||||||
|
|
||||||
def test_the_runtime_robots_check_has_left_the_module() -> None:
|
def test_the_robots_record_names_the_api_path() -> None:
|
||||||
"""Decided when the reader was built (`READS.robots`): nothing to ask now."""
|
"""Decided when the reader was built (`READS.robots`), for the paths it
|
||||||
assert not hasattr(workday, "robots_page") and not hasattr(workday, "ROBOTS_ANY_CASE")
|
requests, not only the career site's."""
|
||||||
assert "robots.txt allows /<Site>/" in workday.READS.robots
|
assert "/wday/cxs/<tenant>/<site>/" in workday.READS.robots
|
||||||
|
|
||||||
|
|
||||||
# -------------------------------------------------------------- discovery
|
# -------------------------------------------------------------- discovery
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from app.jobs import discover as mod
|
|||||||
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
|
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
|
||||||
from app.jobs.sources import seed_companies
|
from app.jobs.sources import seed_companies
|
||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
|
from plugins import readers
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------ parsing
|
# ------------------------------------------------------------------ parsing
|
||||||
@@ -95,6 +96,12 @@ def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
|
|||||||
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
|
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_unreadable_entry_names_a_host_with_a_module() -> None:
|
||||||
|
"""A new module deletes its host's `_UNREADABLE` entry: its own patterns
|
||||||
|
replace it."""
|
||||||
|
assert not {host for host, _ in mod._UNREADABLE} & set(readers.HOSTS)
|
||||||
|
|
||||||
|
|
||||||
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
|
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
|
||||||
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
|
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
|
||||||
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
|
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
|
||||||
|
|||||||
+231
-6
@@ -7,6 +7,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
|
import types
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -21,8 +22,8 @@ if str(ROOT) not in sys.path:
|
|||||||
from desk.models import Item # noqa: E402
|
from desk.models import Item # noqa: E402
|
||||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||||
from app.models import Application # noqa: E402
|
from app.models import Application # noqa: E402
|
||||||
from plugins import readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import _runner # noqa: E402
|
from plugins.readers import Filters, Reads, _runner # noqa: E402
|
||||||
from plugins.ats import ATSSource # noqa: E402
|
from plugins.ats import ATSSource # noqa: E402
|
||||||
from app import prep # noqa: E402
|
from app import prep # noqa: E402
|
||||||
from app.jobs import cascade, pull # noqa: E402
|
from app.jobs import cascade, pull # noqa: E402
|
||||||
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
|||||||
|
|
||||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||||
|
|
||||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0,
|
||||||
"roles": 1, "closed": 1}
|
"roles": 1, "closed": 1}
|
||||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
||||||
|
|
||||||
@@ -163,7 +164,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
|||||||
|
|
||||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||||
|
|
||||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0,
|
||||||
"roles": 0, "closed": 0}
|
"roles": 0, "closed": 0}
|
||||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||||
|
|
||||||
@@ -256,7 +257,7 @@ def test_a_failing_board_keeps_its_roles_before_stale_days(
|
|||||||
_answer(monkeypatch, ValueError("gone"))
|
_answer(monkeypatch, ValueError("gone"))
|
||||||
result = _pull(acme)
|
result = _pull(acme)
|
||||||
assert _open(acme) == {"workable:acme:1"}
|
assert _open(acme) == {"workable:acme:1"}
|
||||||
assert (result["failed"], result["closed"]) == (1, 0)
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||||
since, reason = pull.failing_boards(acme)["workable:acme"]
|
since, reason = pull.failing_boards(acme)["workable:acme"]
|
||||||
assert since and reason == "ValueError: gone"
|
assert since and reason == "ValueError: gone"
|
||||||
|
|
||||||
@@ -486,7 +487,7 @@ def test_run_a_settle_does_not_close_on_run_b_ids(
|
|||||||
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
||||||
|
|
||||||
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
||||||
assert (result["failed"], result["closed"]) == (1, 0)
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||||
assert pull.running(acme) == b["run"]
|
assert pull.running(acme) == b["run"]
|
||||||
|
|
||||||
@@ -507,3 +508,227 @@ def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> No
|
|||||||
context = RunContext(waited_for=(failed,))
|
context = RunContext(waited_for=(failed,))
|
||||||
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
||||||
assert result["boards"] == 1 and pull.running(acme) == ""
|
assert result["boards"] == 1 and pull.running(acme) == ""
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------- losses, guarded
|
||||||
|
|
||||||
|
def _days_ago(days: float) -> str:
|
||||||
|
return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
|
||||||
|
def _set(store: Store, column: str, value: str, *source_ids: str) -> None:
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?",
|
||||||
|
[(value, i) for i in source_ids])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def big(store: Store) -> Store:
|
||||||
|
store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
||||||
|
board="greenhouse")
|
||||||
|
return store
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails(
|
||||||
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
||||||
|
_pull(big)
|
||||||
|
_answer(monkeypatch, ([], True))
|
||||||
|
result = _pull(big)
|
||||||
|
assert len(_open(big)) == 40
|
||||||
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||||
|
assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_read_of_nothing_after_five_closes_the_board(
|
||||||
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(str(n)) for n in range(5)], True))
|
||||||
|
_pull(big)
|
||||||
|
_answer(monkeypatch, ([], True))
|
||||||
|
assert _pull(big)["closed"] == 5
|
||||||
|
assert _open(big) == set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_suspect_answer_is_accepted_after_stale_days(
|
||||||
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
||||||
|
_pull(big)
|
||||||
|
_answer(monkeypatch, ([_row("1")], True))
|
||||||
|
_pull(big)
|
||||||
|
assert len(_open(big)) == 40
|
||||||
|
record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"])
|
||||||
|
big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps(
|
||||||
|
record | {"since": _days_ago(pull.STALE_DAYS + 1)})
|
||||||
|
result = _pull(big)
|
||||||
|
assert _open(big) == {"greenhouse:big:1"}
|
||||||
|
assert (result["unanswered"], result["closed"]) == (0, 39)
|
||||||
|
assert pull.failing_boards(big) == {}
|
||||||
|
assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]:
|
||||||
|
"""A whole-listing host `fake` that filters by country and, per `remote`,
|
||||||
|
on remote; its feed lists the ids in the returned list."""
|
||||||
|
listed: list[str] = []
|
||||||
|
module: object = types.ModuleType("fake")
|
||||||
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined]
|
||||||
|
module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined]
|
||||||
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined]
|
||||||
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
||||||
|
module.READS = Reads( # type: ignore[attr-defined]
|
||||||
|
scope="board", listing="whole", text="listing", complete="whole", places="one",
|
||||||
|
robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type]
|
||||||
|
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
||||||
|
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined]
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||||
|
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps(
|
||||||
|
listed).encode())
|
||||||
|
return listed
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
listed = _remote_host(monkeypatch, remote="none")
|
||||||
|
store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake")
|
||||||
|
reach = {"fake:acme": {"countries": ["DK"], "remote": True}}
|
||||||
|
|
||||||
|
def pull_once() -> dict[str, object]:
|
||||||
|
run = pull.claim_run(store)
|
||||||
|
result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type]
|
||||||
|
reach=reach)
|
||||||
|
return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
listed[:] = ["1", "2", "3", "4"]
|
||||||
|
pull_once()
|
||||||
|
_set(store, "remote", "remote", "fake:acme:3", "fake:acme:4")
|
||||||
|
listed[:] = ["1"]
|
||||||
|
assert pull_once()["closed"] == 1
|
||||||
|
assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"}
|
||||||
|
assert store.state["ats:remote_unasked:fake:acme"] == "1"
|
||||||
|
|
||||||
|
_set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4")
|
||||||
|
assert pull_once()["closed"] == 1
|
||||||
|
assert _open(store) == {"fake:acme:1", "fake:acme:3"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(i) for i in "123"], True))
|
||||||
|
_pull(acme)
|
||||||
|
|
||||||
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||||
|
return _runner.Listing([_row("1")], True, ["1"], narrowed=True)
|
||||||
|
|
||||||
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||||
|
assert _pull(acme)["closed"] == 0
|
||||||
|
_set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3")
|
||||||
|
assert _pull(acme)["closed"] == 1
|
||||||
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")]
|
||||||
|
for target in targets:
|
||||||
|
slug = target.partition(":")[2]
|
||||||
|
store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse")
|
||||||
|
|
||||||
|
def fake_fetch(self: ATSSource) -> object:
|
||||||
|
[(target, _)] = self.targets()
|
||||||
|
yield _item("greenhouse", target.partition(":")[2], "1")
|
||||||
|
self.state[f"ats:ok:{target}"] = self.options["run"]
|
||||||
|
|
||||||
|
real = Store.upsert_role
|
||||||
|
|
||||||
|
def upsert(self: Store, item: Item, sub: object = None) -> bool:
|
||||||
|
if item.source_id.startswith("greenhouse:b:"):
|
||||||
|
raise ValueError("a row the store refuses")
|
||||||
|
return real(self, item, sub) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||||
|
monkeypatch.setattr(Store, "upsert_role", upsert)
|
||||||
|
result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type]
|
||||||
|
|
||||||
|
assert result["roles"] == 2
|
||||||
|
assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"}
|
||||||
|
assert "ats:ok:greenhouse:b" not in store.state
|
||||||
|
since, reason = pull.failing_boards(store)["greenhouse:b"]
|
||||||
|
assert since and reason == "ValueError: a row the store refuses"
|
||||||
|
assert set(pull.failing_boards(store)) == {"greenhouse:b"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_whose_rows_do_not_store_keeps_its_failing_record(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
since = _days_ago(2)
|
||||||
|
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
||||||
|
{"since": since, "reason": "404 Not Found"})
|
||||||
|
_answer(monkeypatch, ([_row("1")], True))
|
||||||
|
|
||||||
|
def refuse(self: Store, item: Item, sub: object = None) -> bool:
|
||||||
|
raise ValueError("no")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Store, "upsert_role", refuse)
|
||||||
|
_pull(acme)
|
||||||
|
assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no")
|
||||||
|
assert "ats:ok:workable:acme" not in acme.state
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------- the settle speaks
|
||||||
|
|
||||||
|
def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many(
|
||||||
|
store: Store) -> None:
|
||||||
|
sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
||||||
|
board="greenhouse")
|
||||||
|
store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone",
|
||||||
|
board="greenhouse")
|
||||||
|
for n in range(80):
|
||||||
|
store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub)
|
||||||
|
run = pull.claim_run(store)
|
||||||
|
store.state["ats:ok:greenhouse:big"] = run
|
||||||
|
store.state["ats:complete:greenhouse:big"] = "1"
|
||||||
|
store.state["ats:ids:greenhouse:big"] = json.dumps(
|
||||||
|
[f"greenhouse:big:{n}" for n in range(20)])
|
||||||
|
store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps(
|
||||||
|
{"since": _days_ago(1), "reason": "404 Not Found"})
|
||||||
|
|
||||||
|
said: list[object] = []
|
||||||
|
result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type]
|
||||||
|
context=RunContext(on_notice=said.append))
|
||||||
|
assert (result["unanswered"], result["closed"]) == (1, 60)
|
||||||
|
[notice] = said
|
||||||
|
assert notice.code == "pull.settled" # type: ignore[attr-defined]
|
||||||
|
assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined]
|
||||||
|
assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined]
|
||||||
|
assert "closed 60 roles" in notice.text # type: ignore[attr-defined]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
||||||
|
said: list[object] = []
|
||||||
|
results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
|
||||||
|
for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type]
|
||||||
|
pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type]
|
||||||
|
context=RunContext(on_notice=said.append))
|
||||||
|
assert said == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_finder_that_raises_releases_the_pull_claim(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
def broken(store: Store, subs: object) -> dict[str, object]:
|
||||||
|
raise RuntimeError("reach broke")
|
||||||
|
|
||||||
|
monkeypatch.setattr(pull, "reaches", broken)
|
||||||
|
# The worker runs the chain's finder once the job is done; here at once.
|
||||||
|
context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme))
|
||||||
|
with pytest.raises(RuntimeError, match="reach broke"):
|
||||||
|
cascade.run_pull(FakeConfig(), acme, context=context)
|
||||||
|
assert pull.running(acme) == ""
|
||||||
|
|
||||||
|
# And the job itself lets go when the chain cannot be built.
|
||||||
|
def no_chain(run: str) -> object:
|
||||||
|
raise RuntimeError("no chain")
|
||||||
|
|
||||||
|
monkeypatch.setattr(cascade, "pull_chain", no_chain)
|
||||||
|
with pytest.raises(RuntimeError, match="no chain"):
|
||||||
|
cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True))
|
||||||
|
assert pull.running(acme) == ""
|
||||||
+136
-3
@@ -26,7 +26,7 @@ if str(ROOT) not in sys.path:
|
|||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import _runner # noqa: E402
|
from plugins.readers import _runner # noqa: E402
|
||||||
from plugins.readers import lever # noqa: E402
|
from plugins.readers import lever # noqa: E402
|
||||||
from plugins.readers._contract import Ask # noqa: E402
|
from plugins.readers._contract import Ask, Filters, Reads # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -64,8 +64,11 @@ PATTERNS = (r"good\\.example/([a-z]+)",)
|
|||||||
def slug_ok(slug): return True
|
def slug_ok(slug): return True
|
||||||
def allowed(slug): return lambda h: h == "good.example"
|
def allowed(slug): return lambda h: h == "good.example"
|
||||||
def careers_url(slug): return "https://good.example/" + slug
|
def careers_url(slug): return "https://good.example/" + slug
|
||||||
def feed_url(slug): return "https://good.example/" + slug + ".json"
|
def feed_url(slug, ask): return "https://good.example/" + slug + ".json"
|
||||||
def parse(body, slug): return []
|
def parse(body, slug): return []
|
||||||
|
from plugins.readers import Reads
|
||||||
|
READS = Reads(scope="board", listing="whole", text="listing", complete="whole", places="one",
|
||||||
|
robots="read (checked 2026-09-30)")
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
@@ -253,7 +256,9 @@ def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
|
|||||||
module.slug_ok = lambda slug: slug.isalnum()
|
module.slug_ok = lambda slug: slug.isalnum()
|
||||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
module.READS = Reads(scope="board", listing="whole", complete="whole", places="one",
|
||||||
|
text="detail" if detail is not None else "listing", robots="test")
|
||||||
|
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
|
||||||
module.parse = lambda body, slug: [
|
module.parse = lambda body, slug: [
|
||||||
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
||||||
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
||||||
@@ -590,3 +595,131 @@ def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
|
|||||||
|
|
||||||
with _client(handler) as client, pytest.raises(ats._Refused):
|
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||||
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------- the guard, asks, remote unasked
|
||||||
|
|
||||||
|
def _listing_host(**reads: Any) -> Any:
|
||||||
|
"""A declared whole-listing board `fake` whose feed lists `module.ids`,
|
||||||
|
whatever is asked; `reads` override its declaration."""
|
||||||
|
module: Any = types.ModuleType("fake")
|
||||||
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||||
|
module.slug_ok = lambda slug: slug.isalnum()
|
||||||
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||||
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||||
|
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
|
||||||
|
"complete": "whole", "places": "one", "robots": "test"} | reads))
|
||||||
|
module.ids = []
|
||||||
|
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
|
||||||
|
module.parse = lambda body, slug: [
|
||||||
|
{"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
|
||||||
|
"places": [{"text": "Berlin", "remote": None}]} for i in json.loads(body)]
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def _listed(module: Any) -> Callable[..., bytes]:
|
||||||
|
return lambda url, **kwargs: json.dumps(module.ids).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _read_and_stamp(state: dict[str, str], module: Any,
|
||||||
|
limits: _runner.Limits | None = None) -> _runner.Listing:
|
||||||
|
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
|
||||||
|
limits=limits)
|
||||||
|
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||||
|
return listing
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unfiltered_whole_read_of_nothing_after_forty_is_suspect() -> None:
|
||||||
|
"""A Greenhouse-style board, trusted, unfiltered: a changed answer that
|
||||||
|
parses to no rows is refused rather than read as a board that closed."""
|
||||||
|
module = _listing_host()
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
module.ids = [str(n) for n in range(40)]
|
||||||
|
_read_and_stamp(state, module)
|
||||||
|
module.ids = []
|
||||||
|
with pytest.raises(_runner.Suspect, match="listed 0 roles, under 10% of the 40"):
|
||||||
|
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_read_under_suspect_min_is_not_held() -> None:
|
||||||
|
module = _listing_host()
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
module.ids = [str(n) for n in range(5)]
|
||||||
|
_read_and_stamp(state, module)
|
||||||
|
module.ids = []
|
||||||
|
assert _read_and_stamp(state, module).ids == []
|
||||||
|
assert json.loads(state["ats:count:fake:acme"])["count"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_same_suspect_answer_is_accepted_after_stale_days() -> None:
|
||||||
|
module = _listing_host()
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
module.ids = [str(n) for n in range(40)]
|
||||||
|
_read_and_stamp(state, module)
|
||||||
|
module.ids = ["1"]
|
||||||
|
failing = "ats:failing:fake:acme"
|
||||||
|
recent = (datetime.now(UTC) - timedelta(days=20)).isoformat(timespec="seconds")
|
||||||
|
state[failing] = json.dumps({"since": recent, "reason": "Suspect: listed 1 roles"})
|
||||||
|
limits = _runner.Limits(page_delay=0, stale_days=21)
|
||||||
|
with pytest.raises(_runner.Suspect):
|
||||||
|
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits)
|
||||||
|
|
||||||
|
old = (datetime.now(UTC) - timedelta(days=22)).isoformat(timespec="seconds")
|
||||||
|
state[failing] = json.dumps({"since": old, "reason": "Suspect: listed 1 roles"})
|
||||||
|
# Outside a pull (no `stale_days`) the guard never gives way.
|
||||||
|
with pytest.raises(_runner.Suspect):
|
||||||
|
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
|
||||||
|
assert _read_and_stamp(state, module, limits).ids == ["1"]
|
||||||
|
assert json.loads(state["ats:count:fake:acme"])["count"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_keep_size_resets_the_guards_count_unless_told_not_to() -> None:
|
||||||
|
state = {"ats:count:fake:acme": json.dumps({"asks": "[]", "count": 40})}
|
||||||
|
ats.keep_size(state, "fake:acme", 40, reset=False)
|
||||||
|
assert "ats:count:fake:acme" in state
|
||||||
|
ats.keep_size(state, "fake:acme", 3)
|
||||||
|
assert "ats:count:fake:acme" not in state
|
||||||
|
assert state["ats:size:fake:acme"] == "3"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filter_that_maps_to_no_host_id_asks_nothing_and_is_incomplete(
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
module = _listing_host(filters=Filters(country="ids"))
|
||||||
|
module.facets = lambda get, slug: {"country": {}}
|
||||||
|
module.ids = ["1", "2"]
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
|
||||||
|
Ask(countries=("DK",)))
|
||||||
|
assert (listing.rows, listing.ids, listing.complete, listing.filtered) == (
|
||||||
|
[], [], False, True)
|
||||||
|
|
||||||
|
# Through the plugin: stamped, but not complete, so the settle closes nothing.
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||||
|
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b'["1"]')
|
||||||
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
|
||||||
|
"params": {"countries": ["DK"]}}], "run": "r"})
|
||||||
|
assert list(source.fetch()) == []
|
||||||
|
assert (source.state["ats:ok:fake:acme"], source.state["ats:complete:fake:acme"]) == (
|
||||||
|
"r", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filtered_read_that_cannot_ask_remote_says_so() -> None:
|
||||||
|
reach = {"countries": ["DK"], "remote": True}
|
||||||
|
blind = _listing_host(filters=Filters(country="many"))
|
||||||
|
ask = _runner.ask_for(blind, None, reach)
|
||||||
|
assert (ask.countries, ask.remote) == (("DK",), None)
|
||||||
|
assert _runner.remote_wanted(blind, None, reach)
|
||||||
|
listing = _runner.read_listing(blind, "acme", _listed(blind), ask,
|
||||||
|
remote=_runner.remote_wanted(blind, None, reach))
|
||||||
|
assert listing.filtered and listing.remote_unasked
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||||
|
assert state["ats:remote_unasked:fake:acme"] == "1"
|
||||||
|
|
||||||
|
# A host that filters on remote asks it as a series of its own.
|
||||||
|
sees = _listing_host(filters=Filters(country="many", remote="many"))
|
||||||
|
ask = _runner.ask_for(sees, None, reach)
|
||||||
|
listing = _runner.read_listing(sees, "acme", _listed(sees), ask, remote=True)
|
||||||
|
assert listing.filtered and not listing.remote_unasked
|
||||||
|
# A params' remote of False asks nothing and wants nothing.
|
||||||
|
assert not _runner.remote_wanted(blind, {"remote": False}, reach)
|
||||||
+299
-89
@@ -1,34 +1,38 @@
|
|||||||
"""The reader contract, held for every module of `plugins/readers/`.
|
"""The reader contract, held for every module of `plugins/readers/`.
|
||||||
|
|
||||||
Per module: it declares `READS`, the declaration matches its functions, and
|
Per module: it declares `READS`, the declaration matches its functions, and
|
||||||
its rows keep what the contract says they keep. No network: fixtures stand
|
its rows state what the declaration says and nothing it denies. No network:
|
||||||
in for the host.
|
fixtures stand in for the host. A fixture a declaration needs and that is
|
||||||
|
missing fails the test with the path to add.
|
||||||
`UNDECLARED` lists the modules that do not declare `READS` yet; their tests
|
|
||||||
are marked xfail (not strict), and the planner removes a name once the
|
|
||||||
module declares. A module not in the list must pass.
|
|
||||||
|
|
||||||
Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/`
|
Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/`
|
||||||
for searches, named after the module:
|
for searches, named after the module:
|
||||||
|
|
||||||
- `<name>-places.<ext>`: one listing answer with at least one role at two
|
- `<name>-places.<ext>`: one listing answer, required of every reader. For
|
||||||
or more places. For `listing="whole"` it is the body `parse` gets; for
|
`listing="whole"` it is the body `parse` gets; for `listing="paged"` it is
|
||||||
`listing="paged"` it is what every request of the first `page` call
|
what every request of the first `page` call returns, and `rows` reads
|
||||||
returns, and `rows` reads that page. Required by the test of a module
|
that page. Under `places="many"` at least one role is at two or more
|
||||||
that declares `places="many"`, which is skipped until the fixture exists.
|
places.
|
||||||
- `<name>-places-detail.<ext>`: optional, for a host whose places come from
|
- `<name>.<ext>` and `<name>-teaser.<ext>`: more listing answers, read the
|
||||||
the role's own page: what `detail`'s requests return; its `places` replace
|
same way; the teaser one is required under `text="excerpt"` or
|
||||||
the row's.
|
`teaser=True`.
|
||||||
- `<name>-teaser.<ext>`: one listing answer with a teaser, read the same
|
- `<name>-detail*.<ext>` and `<name>-places-detail.<ext>`: what `detail`'s
|
||||||
way. Required by the test of a module with `text="excerpt"` or
|
requests return, one of them required under `text="detail"`. Only
|
||||||
`teaser=True`, skipped until it exists.
|
`_runner.DETAIL_FIELDS` of the answer count, as at run time; the places of
|
||||||
|
`-places-detail` replace those of the `-places` rows.
|
||||||
- `<name>.slug`: optional, the slug the fixtures are read with, one line;
|
- `<name>.slug`: optional, the slug the fixtures are read with, one line;
|
||||||
`acme` without it.
|
`acme` without it. A search's saved params are `SEARCH_PARAMS` below.
|
||||||
|
|
||||||
|
Between them the listing and detail answers show every field the reader
|
||||||
|
declares `stated` (or `figures`, `line`, `trusted`, `listed`): a real
|
||||||
|
fixture that happens to lack one is filled from what the per-host test
|
||||||
|
already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
import textwrap
|
||||||
import types
|
import types
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -41,42 +45,53 @@ ROOT = Path(__file__).resolve().parents[1]
|
|||||||
if str(ROOT) not in sys.path:
|
if str(ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from app.prep.hosts import TABLES # noqa: E402
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402
|
from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402
|
||||||
from plugins.readers import _contract # noqa: E402
|
from plugins.readers import _contract # noqa: E402
|
||||||
|
from plugins.readers._runner import DETAIL_FIELDS # noqa: E402
|
||||||
|
|
||||||
FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" / "search")
|
BOARD_FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
SEARCH_FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||||
|
|
||||||
#: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and
|
#: A search's saved params, which its `page` reads; `{}` for the rest.
|
||||||
#: C (the search providers, which arrive declared) land.
|
SEARCH_PARAMS: dict[str, dict[str, Any]] = {"adzuna": {"q": "backend", "country": "de"}}
|
||||||
UNDECLARED: frozenset[str] = frozenset()
|
|
||||||
|
#: The readers whose listing states no date and whose `posted` comes from
|
||||||
|
#: `detail` alone: the date exists only for roles the enrich stage read.
|
||||||
|
POSTED_BY_DETAIL = frozenset({"bamboohr", "gem", "rippling"})
|
||||||
|
|
||||||
|
#: The fields a host states in its own words, which `app.prep` maps through
|
||||||
|
#: the host's table in `app/prep/hosts/<name>.py`.
|
||||||
|
WORD_FIELDS = ("seniority", "employment")
|
||||||
|
|
||||||
|
|
||||||
def _params() -> list[Any]:
|
def _folder(name: str) -> Path:
|
||||||
return [pytest.param(name, marks=pytest.mark.xfail(reason="no READS yet", strict=False))
|
return SEARCH_FIXTURES if name in readers.PROVIDERS else BOARD_FIXTURES
|
||||||
if name in UNDECLARED else name for name in readers.READERS]
|
|
||||||
|
|
||||||
|
|
||||||
def _declared(name: str) -> tuple[types.ModuleType, Reads]:
|
def _fixtures(name: str, pattern: str) -> list[Path]:
|
||||||
module = readers.READERS[name]
|
return sorted(p for p in _folder(name).glob(pattern.format(name=name))
|
||||||
reads = getattr(module, "READS", None)
|
if p.suffix not in (".slug",))
|
||||||
assert isinstance(reads, Reads), f"{name} declares no READS"
|
|
||||||
return module, reads
|
|
||||||
|
|
||||||
|
|
||||||
def _fixture(name: str, kind: str) -> Path | None:
|
def _fixture(name: str, kind: str) -> Path | None:
|
||||||
for folder in FIXTURE_DIRS:
|
found = _fixtures(name, f"{{name}}-{kind}.*")
|
||||||
found = sorted(folder.glob(f"{name}-{kind}.*"))
|
return found[0] if found else None
|
||||||
if found:
|
|
||||||
return found[0]
|
|
||||||
return None
|
def _need(name: str, kind: str, why: str) -> Path:
|
||||||
|
"""The fixture `<name>-<kind>.*`, or a failure naming the file to add."""
|
||||||
|
found = _fixture(name, kind)
|
||||||
|
if found is None:
|
||||||
|
folder = _folder(name).relative_to(ROOT)
|
||||||
|
pytest.fail(f"add {folder}/{name}-{kind}.<ext>: {why}")
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
def _slug(name: str) -> str:
|
def _slug(name: str) -> str:
|
||||||
for folder in FIXTURE_DIRS:
|
path = _folder(name) / f"{name}.slug"
|
||||||
if (path := folder / f"{name}.slug").exists():
|
return path.read_text(encoding="utf-8").strip() if path.exists() else "acme"
|
||||||
return path.read_text(encoding="utf-8").strip()
|
|
||||||
return "acme"
|
|
||||||
|
|
||||||
|
|
||||||
def _answer(body: bytes) -> Callable[..., bytes]:
|
def _answer(body: bytes) -> Callable[..., bytes]:
|
||||||
@@ -84,7 +99,7 @@ def _answer(body: bytes) -> Callable[..., bytes]:
|
|||||||
|
|
||||||
|
|
||||||
def read_fixture(module: types.ModuleType, body: bytes, slug: str,
|
def read_fixture(module: types.ModuleType, body: bytes, slug: str,
|
||||||
detail_body: bytes | None = None) -> list[dict[str, Any]]:
|
detail_body: bytes | None = None, ask: Ask | None = None) -> list[dict[str, Any]]:
|
||||||
"""The rows `module` makes of one listing answer, as the runner reads
|
"""The rows `module` makes of one listing answer, as the runner reads
|
||||||
them: `parse` for a whole listing, the first `page` and its `rows` for a
|
them: `parse` for a whole listing, the first `page` and its `rows` for a
|
||||||
paged one, and, given `detail_body`, each row's `places` from `detail`."""
|
paged one, and, given `detail_body`, each row's `places` from `detail`."""
|
||||||
@@ -92,7 +107,7 @@ def read_fixture(module: types.ModuleType, body: bytes, slug: str,
|
|||||||
if reads.listing == "whole":
|
if reads.listing == "whole":
|
||||||
rows = list(module.parse(body, slug))
|
rows = list(module.parse(body, slug))
|
||||||
else:
|
else:
|
||||||
decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, Ask())
|
decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, ask or Ask())
|
||||||
rows = list(module.rows(decoded, slug))
|
rows = list(module.rows(decoded, slug))
|
||||||
if detail_body is not None:
|
if detail_body is not None:
|
||||||
for row in rows:
|
for row in rows:
|
||||||
@@ -102,6 +117,38 @@ def read_fixture(module: types.ModuleType, body: bytes, slug: str,
|
|||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _ask(name: str) -> Ask:
|
||||||
|
return Ask(params=SEARCH_PARAMS.get(name, {}))
|
||||||
|
|
||||||
|
|
||||||
|
def listing_rows(name: str) -> list[dict[str, Any]]:
|
||||||
|
"""Every row of the reader's listing fixtures; the `-places` rows with
|
||||||
|
the places of `-places-detail` where there is one."""
|
||||||
|
module, slug = readers.READERS[name], _slug(name)
|
||||||
|
places = _need(name, "places", "one listing answer of the host")
|
||||||
|
detail = _fixture(name, "places-detail")
|
||||||
|
rows = read_fixture(module, places.read_bytes(), slug,
|
||||||
|
detail.read_bytes() if detail else None, _ask(name))
|
||||||
|
for path in _fixtures(name, "{name}.*") + _fixtures(name, "{name}-teaser.*"):
|
||||||
|
rows += read_fixture(module, path.read_bytes(), slug, ask=_ask(name))
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def detail_rows(name: str, listed: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
"""What `detail` hands over for the first listed role, once per detail
|
||||||
|
fixture, cut to `DETAIL_FIELDS` as the runner cuts it."""
|
||||||
|
module = readers.READERS[name]
|
||||||
|
if module.READS.text != "detail":
|
||||||
|
return []
|
||||||
|
paths = _fixtures(name, "{name}-detail*.*") + _fixtures(name, "{name}-places-detail.*")
|
||||||
|
if not paths:
|
||||||
|
_need(name, "detail", "what the role's own page answers, for text=\"detail\"")
|
||||||
|
assert listed, f"{name}: the listing fixtures gave no role to read in detail"
|
||||||
|
return [{key: value for key, value in
|
||||||
|
module.detail(_answer(path.read_bytes()), _slug(name), dict(listed[0])).items()
|
||||||
|
if key in DETAIL_FIELDS} for path in paths]
|
||||||
|
|
||||||
|
|
||||||
def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
|
def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
|
||||||
"""What rows get wrong about `places` under the declared trait."""
|
"""What rows get wrong about `places` under the declared trait."""
|
||||||
out = []
|
out = []
|
||||||
@@ -118,60 +165,138 @@ def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
|
|||||||
out.append(f"{rid}: {place!r} is not a Place")
|
out.append(f"{rid}: {place!r} is not a Place")
|
||||||
if places in ("one", "copies") and len(found) > 1:
|
if places in ("one", "copies") and len(found) > 1:
|
||||||
out.append(f'{rid}: {len(found)} places under places="{places}"')
|
out.append(f'{rid}: {len(found)} places under places="{places}"')
|
||||||
|
if places == "none" and found:
|
||||||
|
out.append(f'{rid}: {len(found)} places under places="none"')
|
||||||
if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows):
|
if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows):
|
||||||
out.append('no row has two places: a reader with places="many" keeps every place')
|
out.append('no row has two places: a reader with places="many" keeps every place')
|
||||||
|
elif places != "none" and not any(row.get("places") for row in rows):
|
||||||
|
out.append(f'no row has a place under places="{places}": declare "none"')
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _states(rows: list[dict[str, Any]], key: str) -> bool:
|
||||||
|
"""Whether some row carries `key` with a value (False counts, "" does not)."""
|
||||||
|
return any(row.get(key) not in (None, "", [], {}) for row in rows)
|
||||||
|
|
||||||
|
|
||||||
|
def field_problems(reads: Reads, listing: list[dict[str, Any]], detail: list[dict[str, Any]],
|
||||||
|
posted_by_detail: bool = False) -> list[str]:
|
||||||
|
"""What the rows get wrong about the declared `fields`, `text` and
|
||||||
|
`teaser`: a trait that promises a key no row carries, or denies one a
|
||||||
|
row carries."""
|
||||||
|
rows = listing + detail
|
||||||
|
fields = reads.fields
|
||||||
|
out: list[str] = []
|
||||||
|
figures = _states(rows, "salary_min") or _states(rows, "salary_max")
|
||||||
|
line = _states(rows, "salary_text")
|
||||||
|
if fields.pay == "figures" and not figures:
|
||||||
|
out.append('pay="figures" and no row has salary_min or salary_max')
|
||||||
|
if fields.pay == "line" and (figures or not line):
|
||||||
|
out.append('pay="line" wants salary_text on some row and figures on none')
|
||||||
|
if fields.pay == "none" and (figures or line):
|
||||||
|
out.append('pay="none" and a row has pay')
|
||||||
|
|
||||||
|
if fields.posted == "none" and _states(rows, "posted"):
|
||||||
|
out.append('posted="none" and a row has posted')
|
||||||
|
elif fields.posted != "none" and posted_by_detail:
|
||||||
|
if _states(listing, "posted") or not _states(detail, "posted"):
|
||||||
|
out.append(f'posted="{fields.posted}" by detail wants posted from detail only')
|
||||||
|
elif fields.posted != "none" and not _states(listing, "posted"):
|
||||||
|
out.append(f'posted="{fields.posted}" and no listed row has posted')
|
||||||
|
|
||||||
|
for key in ("remote", "country", "seniority", "employment"):
|
||||||
|
stated = getattr(fields, key) == "stated"
|
||||||
|
if stated != _states(rows, key):
|
||||||
|
out.append(f'{key}="{getattr(fields, key)}" and '
|
||||||
|
+ ("no row has it" if stated else "a row has it"))
|
||||||
|
|
||||||
|
text = {"listing": _states(listing, "description"),
|
||||||
|
"detail": _states(detail, "description"),
|
||||||
|
"excerpt": _states(listing, "teaser") and not _states(listing, "description"),
|
||||||
|
"none": not _states(rows, "description")}[reads.text]
|
||||||
|
if not text:
|
||||||
|
out.append(f'text="{reads.text}" and the rows do not show it')
|
||||||
|
teaser = reads.teaser or reads.text == "excerpt"
|
||||||
|
if teaser != _states(listing if teaser else rows, "teaser"):
|
||||||
|
out.append("teaser declared and no listed row has one" if teaser
|
||||||
|
else "a row has a teaser the declaration does not name")
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ every module
|
# ------------------------------------------------------------ every module
|
||||||
|
|
||||||
def test_the_undeclared_list_names_only_readers() -> None:
|
@pytest.mark.parametrize("name", readers.READERS)
|
||||||
assert set(readers.READERS) >= UNDECLARED
|
|
||||||
|
|
||||||
|
|
||||||
def test_undeclared_lists_the_modules_without_reads() -> None:
|
|
||||||
assert set(readers.undeclared()) == {name for name, module in readers.READERS.items()
|
|
||||||
if not hasattr(module, "READS")}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("name", _params())
|
|
||||||
def test_a_reader_declares_reads(name: str) -> None:
|
|
||||||
_declared(name)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("name", _params())
|
|
||||||
def test_a_declaration_matches_the_functions(name: str) -> None:
|
def test_a_declaration_matches_the_functions(name: str) -> None:
|
||||||
module, _reads = _declared(name)
|
module = readers.READERS[name]
|
||||||
assert readers.problems(module, name) == []
|
assert readers.problems(module, name) == []
|
||||||
expected = "search" if module.READS.scope == "search" else "board"
|
assert name in (readers.PROVIDERS if module.READS.scope == "search" else readers.HOSTS)
|
||||||
assert name in (readers.PROVIDERS if expected == "search" else readers.HOSTS)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("name", _params())
|
@pytest.mark.parametrize("name", readers.READERS)
|
||||||
def test_a_reader_keeps_every_place(name: str) -> None:
|
def test_a_reader_keeps_every_place(name: str) -> None:
|
||||||
module, reads = _declared(name)
|
listed = listing_rows(name)
|
||||||
fixture = _fixture(name, "places")
|
assert listed, f"{name}: the listing fixtures gave no rows"
|
||||||
if fixture is None:
|
answered = [{"source_id": f"detail {i}", "places": row["places"]}
|
||||||
if reads.places == "many":
|
for i, row in enumerate(detail_rows(name, listed)) if "places" in row]
|
||||||
pytest.skip(f"no fixture {name}-places.* yet")
|
assert place_problems(listed + answered, readers.READERS[name].READS.places) == []
|
||||||
pytest.skip(f'places="{reads.places}" and no fixture')
|
|
||||||
detail = _fixture(name, "places-detail")
|
|
||||||
rows = read_fixture(module, fixture.read_bytes(), _slug(name),
|
|
||||||
detail.read_bytes() if detail else None)
|
|
||||||
assert rows, f"{fixture.name} gave no rows"
|
|
||||||
assert place_problems(rows, reads.places) == []
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("name", _params())
|
@pytest.mark.parametrize("name", readers.READERS)
|
||||||
def test_a_reader_keeps_the_teaser(name: str) -> None:
|
def test_a_reader_states_what_it_declares(name: str) -> None:
|
||||||
module, reads = _declared(name)
|
reads = readers.READERS[name].READS
|
||||||
if not (reads.text == "excerpt" or reads.teaser):
|
if reads.text == "excerpt" or reads.teaser:
|
||||||
pytest.skip("the list carries no teaser")
|
_need(name, "teaser", "a listing answer with a teaser")
|
||||||
fixture = _fixture(name, "teaser")
|
listed = listing_rows(name)
|
||||||
if fixture is None:
|
assert field_problems(reads, listed, detail_rows(name, listed),
|
||||||
pytest.skip(f"no fixture {name}-teaser.* yet")
|
name in POSTED_BY_DETAIL) == []
|
||||||
rows = read_fixture(module, fixture.read_bytes(), _slug(name))
|
|
||||||
assert any(isinstance(row.get("teaser"), str) and row["teaser"].strip() for row in rows)
|
|
||||||
|
@pytest.mark.parametrize("name", [name for name, module in readers.READERS.items()
|
||||||
|
if module.READS.listing == "paged"
|
||||||
|
and module.READS.complete in ("stated", "end")])
|
||||||
|
def test_a_paged_reader_states_a_total_as_declared(name: str) -> None:
|
||||||
|
"""`complete="stated"`: the first page's total is an int; `"end"`: None,
|
||||||
|
since the read ends on an empty page."""
|
||||||
|
module = readers.READERS[name]
|
||||||
|
body = _need(name, "places", "one listing answer of the host").read_bytes()
|
||||||
|
_decoded, _results, total, _cursor = module.page(_answer(body), _slug(name), None,
|
||||||
|
_ask(name))
|
||||||
|
if module.READS.complete == "stated":
|
||||||
|
assert isinstance(total, int) and not isinstance(total, bool), total
|
||||||
|
else:
|
||||||
|
assert total is None, total
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_posted_by_detail_list_names_only_readers_that_date_by_detail() -> None:
|
||||||
|
for name in POSTED_BY_DETAIL:
|
||||||
|
reads = readers.READERS[name].READS
|
||||||
|
assert reads.text == "detail" and reads.fields.posted != "none", name
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ prep tables
|
||||||
|
|
||||||
|
def test_every_prep_table_is_a_readers() -> None:
|
||||||
|
assert set(TABLES) <= set(readers.READERS), set(TABLES) - set(readers.READERS)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name", readers.READERS)
|
||||||
|
def test_a_reader_stating_words_has_a_table_for_them(name: str) -> None:
|
||||||
|
"""A host that states seniority or employment in its own words has a
|
||||||
|
table for that field in `app/prep/hosts/<name>.py`, even an empty one
|
||||||
|
for a host whose words are already the app's (`words.lookup` then
|
||||||
|
takes the word as it is). The table is where a word is known, so a
|
||||||
|
host without one lists every word as unmapped. A table for a field the
|
||||||
|
reader declares `none` is dead."""
|
||||||
|
fields = readers.READERS[name].READS.fields
|
||||||
|
tables = TABLES.get(name, {})
|
||||||
|
for field in WORD_FIELDS:
|
||||||
|
has = field in tables or (field == "employment" and "match" in tables)
|
||||||
|
stated = getattr(fields, field) == "stated"
|
||||||
|
assert has == stated, (f"{name}: {field}={getattr(fields, field)!r} "
|
||||||
|
f"{'without' if stated else 'with'} a table in "
|
||||||
|
f"app/prep/hosts/{name}.py")
|
||||||
|
if "remote" in tables:
|
||||||
|
assert fields.remote == "stated", f"{name}: a remote table and remote=\"none\""
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ the checks
|
# ------------------------------------------------------------ the checks
|
||||||
@@ -219,11 +344,10 @@ def test_a_good_declaration_has_no_problems() -> None:
|
|||||||
assert readers.problems(module, "fake") == []
|
assert readers.problems(module, "fake") == []
|
||||||
|
|
||||||
|
|
||||||
def test_a_module_without_reads_is_checked_as_before() -> None:
|
def test_a_module_without_reads_fails() -> None:
|
||||||
module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: [])
|
module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: [])
|
||||||
assert readers.problems(module, "fake") == []
|
assert readers.problems(module, "fake") == [
|
||||||
assert readers.scope(module) == "board"
|
"no READS: every reader declares what it reads"]
|
||||||
assert readers.undeclared({"fake": module}) == ["fake"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(("module", "reads", "problem"), [
|
@pytest.mark.parametrize(("module", "reads", "problem"), [
|
||||||
@@ -256,6 +380,7 @@ def test_a_module_without_reads_is_checked_as_before() -> None:
|
|||||||
env=("ADZUNA_APP_ID",))},
|
env=("ADZUNA_APP_ID",))},
|
||||||
"key.env does not match key.fields"),
|
"key.env does not match key.fields"),
|
||||||
(_whole(), {"robots": " "}, "robots is empty"),
|
(_whole(), {"robots": " "}, "robots is empty"),
|
||||||
|
(_whole(), {"robots": "read: robots.txt allows /api/"}, "robots has no date"),
|
||||||
(_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {},
|
(_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {},
|
||||||
"feed_url does not take (slug, ask)"),
|
"feed_url does not take (slug, ask)"),
|
||||||
(_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []),
|
(_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []),
|
||||||
@@ -274,22 +399,48 @@ def test_reads_that_is_not_a_reads_is_refused() -> None:
|
|||||||
assert "READS is not a Reads" in readers.problems(module, "fake")
|
assert "READS is not a Reads" in readers.problems(module, "fake")
|
||||||
|
|
||||||
|
|
||||||
def test_a_search_needs_fewer_names_than_a_board() -> None:
|
def _search() -> Any:
|
||||||
module: Any = types.ModuleType("fake")
|
module: Any = types.ModuleType("fake")
|
||||||
module.NAME, module.LABEL = "fake", "Fake"
|
module.NAME, module.LABEL = "fake", "Fake"
|
||||||
module.allowed = lambda slug: lambda host: True
|
module.allowed = lambda slug: lambda host: True
|
||||||
module.page = lambda get, slug, cursor, ask: (None, [], 0, None)
|
module.page = lambda get, slug, cursor, ask: (None, [], 0, None)
|
||||||
module.rows = lambda decoded, slug: []
|
module.rows = lambda decoded, slug: []
|
||||||
|
module.PARAMS = {"q": "query", "where": "place"}
|
||||||
|
module.clean_params = lambda params: dict(params)
|
||||||
module.READS = _reads(scope="search", listing="paged", complete="never",
|
module.READS = _reads(scope="search", listing="paged", complete="never",
|
||||||
filters=Filters(query="one"),
|
filters=Filters(query="one", place="one"),
|
||||||
key=Key(name="adzuna", fields=("app_id", "app_key"),
|
key=Key(name="adzuna", fields=("app_id", "app_key"),
|
||||||
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")))
|
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")))
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_needs_fewer_names_than_a_board() -> None:
|
||||||
|
module = _search()
|
||||||
assert readers.problems(module, "fake") == []
|
assert readers.problems(module, "fake") == []
|
||||||
assert readers.scope(module) == "search"
|
|
||||||
del module.READS
|
del module.READS
|
||||||
assert "no careers_url" in readers.problems(module, "fake")
|
assert "no careers_url" in readers.problems(module, "fake")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_names_its_params_and_cleans_them() -> None:
|
||||||
|
module = _search()
|
||||||
|
del module.PARAMS, module.clean_params
|
||||||
|
assert {"no PARAMS", "no clean_params"} <= set(readers.problems(module, "fake"))
|
||||||
|
module = _search()
|
||||||
|
module.PARAMS = {"q": "query", "area": "town"}
|
||||||
|
assert readers.problems(module, "fake") == [
|
||||||
|
"PARAMS maps 'area' to 'town', not a filter dimension"]
|
||||||
|
module = _search()
|
||||||
|
module.clean_params = lambda: {}
|
||||||
|
assert readers.problems(module, "fake") == ["clean_params does not take (params)"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_search_names_its_params() -> None:
|
||||||
|
for name, module in readers.PROVIDERS.items():
|
||||||
|
declared = module.READS.filters.declared()
|
||||||
|
assert "query" in module.PARAMS.values(), name
|
||||||
|
assert set(module.PARAMS.values()) <= set(declared), name
|
||||||
|
|
||||||
|
|
||||||
def test_a_misspelt_trait_fails_where_it_is_declared() -> None:
|
def test_a_misspelt_trait_fails_where_it_is_declared() -> None:
|
||||||
with pytest.raises(pydantic.ValidationError):
|
with pytest.raises(pydantic.ValidationError):
|
||||||
_reads(listing="pages")
|
_reads(listing="pages")
|
||||||
@@ -349,12 +500,20 @@ def test_the_place_check_passes_a_role_at_two_places() -> None:
|
|||||||
assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == []
|
assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_place_check_holds_a_reader_that_states_none_to_none() -> None:
|
||||||
|
assert place_problems([{"source_id": "1", "places": []}], "none") == []
|
||||||
|
assert place_problems([{"source_id": "1", "places": []}], "one") == [
|
||||||
|
'no row has a place under places="one": declare "none"']
|
||||||
|
assert place_problems([{"source_id": "2", "places": [{"text": "Oslo", "remote": None}]}],
|
||||||
|
"none") == ['2: 1 places under places="none"']
|
||||||
|
|
||||||
|
|
||||||
def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None:
|
def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None:
|
||||||
joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}]
|
joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}]
|
||||||
assert place_problems(joined, "many") == [
|
assert place_problems(joined, "many") == [
|
||||||
'no row has two places: a reader with places="many" keeps every place']
|
'no row has two places: a reader with places="many" keeps every place']
|
||||||
assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [
|
assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [
|
||||||
"2: places is not a list"]
|
"2: places is not a list", 'no row has a place under places="one": declare "none"']
|
||||||
assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [
|
assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [
|
||||||
"3: ('Berlin', False) is not a Place"]
|
"3: ('Berlin', False) is not a Place"]
|
||||||
|
|
||||||
@@ -377,6 +536,57 @@ def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
|
|||||||
assert place_problems(rows, "many") == []
|
assert place_problems(rows, "many") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_contracts_example_is_workdays_declaration() -> None:
|
||||||
|
doc = readers.__doc__ or ""
|
||||||
|
start = doc.index(" READS = Reads(")
|
||||||
|
source = textwrap.dedent(doc[start:doc.index("\n )\n", start) + 6])
|
||||||
|
namespace: dict[str, Any] = {"Reads": Reads, "Filters": Filters, "Fields": Fields}
|
||||||
|
exec(source, namespace)
|
||||||
|
assert namespace["READS"] == readers.READERS["workday"].READS
|
||||||
|
|
||||||
|
|
||||||
def test_the_contract_types_are_what_the_package_exports() -> None:
|
def test_the_contract_types_are_what_the_package_exports() -> None:
|
||||||
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
|
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
|
||||||
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
|
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ the field check
|
||||||
|
|
||||||
|
def test_the_field_check_passes_rows_that_state_what_is_declared() -> None:
|
||||||
|
reads = _reads(text="detail", teaser=True, listing="whole",
|
||||||
|
fields=Fields(pay="figures", posted="trusted", remote="stated"))
|
||||||
|
listed = [{"source_id": "1", "teaser": "Short.", "remote": False, "posted": "2026-09-30"}]
|
||||||
|
detail = [{"description": "Long.", "salary_min": 50000}]
|
||||||
|
assert field_problems(reads, listed, detail) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_field_check_names_a_promise_no_row_keeps_and_a_denial_a_row_breaks() -> None:
|
||||||
|
reads = _reads(fields=Fields(pay="line", employment="stated"))
|
||||||
|
listed = [{"source_id": "1", "description": "Text.", "salary_min": 1, "remote": "hybrid",
|
||||||
|
"posted": "2026-09-30"}]
|
||||||
|
assert field_problems(reads, listed, []) == [
|
||||||
|
'pay="line" wants salary_text on some row and figures on none',
|
||||||
|
'posted="none" and a row has posted',
|
||||||
|
'remote="none" and a row has it',
|
||||||
|
'employment="stated" and no row has it']
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_field_check_takes_a_date_from_detail_only_where_it_is_said() -> None:
|
||||||
|
reads = _reads(text="detail", fields=Fields(posted="trusted"))
|
||||||
|
listed = [{"source_id": "1"}]
|
||||||
|
detail = [{"description": "Text.", "posted": "2026-09-30"}]
|
||||||
|
assert field_problems(reads, listed, detail, posted_by_detail=True) == []
|
||||||
|
assert field_problems(reads, listed, detail) == [
|
||||||
|
'posted="trusted" and no listed row has posted']
|
||||||
|
assert field_problems(reads, [*listed, {"posted": "2026-09-30"}], detail,
|
||||||
|
posted_by_detail=True) == [
|
||||||
|
'posted="trusted" by detail wants posted from detail only']
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_field_check_holds_text_and_teaser_to_the_declaration() -> None:
|
||||||
|
excerpt = _reads(text="excerpt")
|
||||||
|
assert field_problems(excerpt, [{"teaser": "Short."}], []) == []
|
||||||
|
assert field_problems(excerpt, [{"teaser": "Short.", "description": "Long."}], []) == [
|
||||||
|
'text="excerpt" and the rows do not show it']
|
||||||
|
assert field_problems(_reads(), [{"description": "Long.", "teaser": "Short."}], []) == [
|
||||||
|
"a row has a teaser the declaration does not name"]
|
||||||
+7
-17
@@ -103,8 +103,6 @@ def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
|
|||||||
both = _whole(filters=Filters(country="many", remote="many"))
|
both = _whole(filters=Filters(country="many", remote="many"))
|
||||||
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
|
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
|
||||||
assert (ask.countries, ask.remote) == (("DE",), True)
|
assert (ask.countries, ask.remote) == (("DE",), True)
|
||||||
# A module without READS reads the whole board.
|
|
||||||
assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
|
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
|
||||||
@@ -139,11 +137,12 @@ def test_ids_filters_are_mapped_through_the_facets() -> None:
|
|||||||
ask = Ask(countries=("DK", "XX"))
|
ask = Ask(countries=("DK", "XX"))
|
||||||
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
|
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
|
||||||
assert mapped.countries == ("h-1",)
|
assert mapped.countries == ("h-1",)
|
||||||
# None known: the host has nothing in reach, and the read is empty.
|
# None known: nothing is asked, which is no evidence, so the read is
|
||||||
|
# incomplete and closes nothing.
|
||||||
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
|
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
|
||||||
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
|
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
|
||||||
facets={"country": {}})
|
facets={"country": {}})
|
||||||
assert (listing.rows, listing.complete, listing.filtered) == ([], True, True)
|
assert (listing.rows, listing.complete, listing.filtered) == ([], False, True)
|
||||||
|
|
||||||
|
|
||||||
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
|
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
|
||||||
@@ -189,17 +188,6 @@ def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
|
|||||||
_runner.read_listing(module, "acme", web)
|
_runner.read_listing(module, "acme", web)
|
||||||
|
|
||||||
|
|
||||||
def test_a_module_without_reads_gets_the_older_signatures() -> None:
|
|
||||||
module: Any = types.ModuleType("old")
|
|
||||||
module.NAME = "old"
|
|
||||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
|
||||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
|
|
||||||
listing = _runner.read_listing(module, "acme",
|
|
||||||
Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
|
|
||||||
Ask(countries=("DK",)))
|
|
||||||
assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
|
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
|
||||||
pages = [["1", "2"], ["3"]]
|
pages = [["1", "2"], ["3"]]
|
||||||
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
|
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
|
||||||
@@ -333,9 +321,11 @@ def test_a_suspect_read_far_under_the_last_count_fails() -> None:
|
|||||||
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
|
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
|
||||||
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
|
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
|
||||||
_runner.read(state, "ats", "t", module, "acme", web)
|
_runner.read(state, "ats", "t", module, "acme", web)
|
||||||
# A reader whose empty answer is trusted reads it as it is.
|
# A reader whose empty answer is trusted is held too, from a count of
|
||||||
|
# `SUSPECT_MIN`.
|
||||||
module.READS = module.READS.model_copy(update={"empty": "trusted"})
|
module.READS = module.READS.model_copy(update={"empty": "trusted"})
|
||||||
assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"]
|
with pytest.raises(_runner.Suspect):
|
||||||
|
_runner.read(state, "ats", "t", module, "acme", web)
|
||||||
|
|
||||||
|
|
||||||
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
||||||
|
|||||||
+56
-1
@@ -264,8 +264,10 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo
|
|||||||
module: Any = types.ModuleType("fake")
|
module: Any = types.ModuleType("fake")
|
||||||
module.NAME, module.LABEL = "fake", "Fake"
|
module.NAME, module.LABEL = "fake", "Fake"
|
||||||
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
|
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
|
||||||
places="one", fields=Fields(country="stated"), robots="test")
|
places="one", fields=Fields(country="stated"), robots="test (checked 2026-09-30)")
|
||||||
module.allowed = lambda slug: lambda host: host == "search.fake.example"
|
module.allowed = lambda slug: lambda host: host == "search.fake.example"
|
||||||
|
module.PARAMS = {"q": "query"}
|
||||||
|
module.clean_params = lambda params: dict(params)
|
||||||
|
|
||||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||||
assert ask.params["q"]
|
assert ask.params["q"]
|
||||||
@@ -371,3 +373,56 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
|||||||
assert store.roles() == []
|
assert store.roles() == []
|
||||||
# And a batch that starts after the switch reads nothing.
|
# And a batch that starts after the switch reads nothing.
|
||||||
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------- failures and the guard
|
||||||
|
|
||||||
|
def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""The provider `fake` of `_fake_provider`, answering with an error."""
|
||||||
|
_fake_provider(monkeypatch, [], complete=True)
|
||||||
|
|
||||||
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||||
|
raise ValueError("hits renamed")
|
||||||
|
|
||||||
|
readers.PROVIDERS["fake"].page = page
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_whose_provider_raises_is_recorded_failing(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
|
params={"q": "backend"})
|
||||||
|
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||||
|
_pull(store)
|
||||||
|
_broken_provider(monkeypatch)
|
||||||
|
result = _pull(store)
|
||||||
|
assert result["unanswered"] == 1
|
||||||
|
since, reason = pull.failing_boards(store)["fake:cph"]
|
||||||
|
assert since and reason == "ValueError: hits renamed"
|
||||||
|
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
||||||
|
|
||||||
|
# Failing for `STALE_DAYS` closes its roles, as a board's.
|
||||||
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
||||||
|
timespec="seconds")
|
||||||
|
store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason})
|
||||||
|
_pull(store)
|
||||||
|
assert store.roles() == []
|
||||||
|
|
||||||
|
# Its next answer removes the record and reopens what it lists.
|
||||||
|
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||||
|
_pull(store)
|
||||||
|
assert pull.failing_boards(store) == {}
|
||||||
|
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_that_lists_nothing_after_forty_is_suspect(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
|
params={"q": "backend"})
|
||||||
|
forty = [f"h{n}" for n in range(40)]
|
||||||
|
_fake_provider(monkeypatch, forty, complete=True)
|
||||||
|
_pull(store)
|
||||||
|
_fake_provider(monkeypatch, [], complete=True)
|
||||||
|
result = _pull(store)
|
||||||
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||||
|
assert len(store.roles()) == 40
|
||||||
|
assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]
|
||||||
Reference in new issue
Block a user