Let the runner drop and count a row without an id, a title or a link, as ADR Plugins 3.8 has it, in one place for whole, paged and search reads: the count is stamped per target, logged, named on the board's row, and over 10% of the rows fails the read; the 28 readers hand every listing over instead of skipping it

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-06 14:04:33 +02:00
1 parent 38c31c830b
commit 911dfe8584
58 files changed
+479 -203

No files matched your search

+17 -3
View File
@@ -10,7 +10,10 @@ both; nothing here decides how a board is read, so the words never disagree
with the pull as long as they read the keys the pull writes. with the pull as long as they read the keys the pull writes.
A word with a number carries it after a colon ("without text: 12"), and the A word with a number carries it after a colon ("without text: 12"), and the
filter matches the phrase before the colon (`WORDS`). filter matches the phrase before the colon (`WORDS`). One word is outside
`WORDS`: the listings the last read dropped for want of an id, a title or a
link ("3 listings dropped: no title or link", `_runner.valid`), shown on the
board's row and not offered by the filter.
""" """
from __future__ import annotations from __future__ import annotations
@@ -49,7 +52,7 @@ WORDS: dict[str, str] = {
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`. #: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked", KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
"narrowed", "detailfailed") "narrowed", "detailfailed", "dropped")
class Texts(NamedTuple): class Texts(NamedTuple):
@@ -84,7 +87,8 @@ def texts(store: Store) -> dict[int, Texts]:
def words(module: ModuleType, state: Mapping[str, str], target: str, def words(module: ModuleType, state: Mapping[str, str], target: str,
sub: Subscription | None, size: int | None, *, plan: Plan | None = None, sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
line: str = "", counts: Texts | None = None) -> list[str]: line: str = "", counts: Texts | None = None) -> list[str]:
"""The limits of one board, in the order of `WORDS`, those that apply. """The limits of one board, in the order of `WORDS`, those that apply, the
rows the last read dropped after "read incomplete".
`state` holds the board's keys (`state_of`), `size` its kept size, `state` holds the board's keys (`state_of`), `size` its kept size,
`plan` and `sub` the reach it is read with (for "read whole"), `line` the `plan` and `sub` the reach it is read with (for "read whole"), `line` the
@@ -105,6 +109,8 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
out.append("failing") out.append("failing")
if state.get(key("ok")) and not state.get(key("complete")): if state.get(key("ok")) and not state.get(key("complete")):
out.append("read incomplete") out.append("read incomplete")
if dropped := _count(state.get(key("dropped"))):
out.append(f"{dropped} listing{'' if dropped == 1 else 's'} dropped: no title or link")
if state.get(key("swept")) == "1" or key("sweep") in state: if state.get(key("swept")) == "1" or key("sweep") in state:
out.append("swept") out.append("swept")
if state.get(key("filtered")) == "1": if state.get(key("filtered")) == "1":
@@ -130,6 +136,14 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
return out return out
def _count(value: str | None) -> int:
"""A number kept in state, 0 for none or one that does not parse."""
try:
return max(0, int(value or 0))
except ValueError:
return 0
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool: def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
"""Whether the reach leaves the board unread by place: its reader cannot """Whether the reach leaves the board unread by place: its reader cannot
filter, or the board is under `SMALL` roles (the line's two "read whole" filter, or the board is under `SMALL` roles (the line's two "read whole"
+14 -6
View File
@@ -17,8 +17,9 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is read independently; one that fails is logged, its slug is. Every board is read independently; one that fails is logged,
recorded as failing under `ats:failing:<target>`, and skipped, and so is one recorded as failing under `ats:failing:<target>`, and skipped, and so is one
whose read lists under a tenth of what it listed last time (`_runner.read`'s whose read lists under a tenth of what it listed last time, or drops more
guard), until it has failed that way for the pull's `stale_days`. A board's than a tenth of the rows its reader handed over (`_runner.read`'s guards),
until it has failed that way for the pull's `stale_days`. A board's
filter is the `reach` option the pull hands it (the agents', see filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries": `app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`) win, narrowed to what its reader declares ["DK"], "remote": true}`) win, narrowed to what its reader declares
@@ -343,8 +344,10 @@ SIZE_PAGES = 100
def board_size(target: str, *, client: httpx.Client | None = None) -> int: def board_size(target: str, *, client: httpx.Client | None = None) -> int:
"""How many roles board `target` lists, unfiltered and without terms: a """How many roles board `target` lists, unfiltered and without terms: a
whole host's listing, the total a `complete="stated"` host gives on page whole host's listing, the total a `complete="stated"` host gives on page
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises 1, else the listing read to the end, at most `SIZE_PAGES` pages. A read
as `_fetch_rows` does, so a board that does not answer is refused.""" counts the rows the runner dropped as well (`_listed`), as a stated total
does. Raises as `_fetch_rows` does, so a board that does not answer is
refused."""
if not is_known_target(target): if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host") raise ValueError(f"{target!r} is not host:slug for a known ATS host")
if client is None: if client is None:
@@ -358,7 +361,12 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
if isinstance(total, int): if isinstance(total, int):
return total return total
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY) limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
return len(_runner.read_listing(module, slug, get, limits=limits).ids) return _listed(_runner.read_listing(module, slug, get, limits=limits))
def _listed(listing: _runner.Listing) -> int:
"""The listings a read found: the ids it kept and the rows it dropped."""
return len(listing.ids) + listing.dropped
def keep_size(state: Any, target: str, size: int, *, today: str = "", def keep_size(state: Any, target: str, size: int, *, today: str = "",
@@ -462,7 +470,7 @@ class ATSSource(Source):
[compound(i) for i in listing.ids]) [compound(i) for i in listing.ids])
self.state.pop(failing, None) self.state.pop(failing, None)
if unsized and listing.complete and not (listing.filtered or listing.narrowed): if unsized and listing.complete and not (listing.filtered or listing.narrowed):
keep_size(self.state, target, len(listing.ids), reset=False) keep_size(self.state, target, _listed(listing), reset=False)
if detail_sets is not None: if detail_sets is not None:
# A complete listing drops the ids it no longer lists. # A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False): for key, ids in zip(detail_keys(target), detail_sets, strict=False):
+7 -2
View File
@@ -179,7 +179,7 @@ ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's **Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
own id for the role, unique on the board), `title`, `url` (http or https, own id for the role, unique on the board), `title`, `url` (http or https,
checked with `_shared.is_web_url`) and `places`; `location` (the host's which the runner checks with `_shared.is_web_url`) and `places`; `location` (the host's
one-line place, kept while storage reads it), `posted` (an aware datetime one-line place, kept while storage reads it), `posted` (an aware datetime
or None, see `_shared.parse_dt`), `department`, `description` (plain text, or None, see `_shared.parse_dt`), `department`, `description` (plain text,
see `_shared.html_text`), `remote` (True, False, a word or None) and see `_shared.html_text`), `remote` (True, False, a word or None) and
@@ -187,7 +187,12 @@ see `_shared.html_text`), `remote` (True, False, a word or None) and
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`; (yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
where it states them, `country` (ISO 3166 alpha-2), `seniority` and where it states them, `country` (ISO 3166 alpha-2), `seniority` and
`employment`; `teaser`; and, on a search row, `company`. The runner keeps `employment`; `teaser`; and, on a search row, `company`. The runner keeps
every stated key in `raw`, so a new field needs no change there. every stated key in `raw`, so a new field needs no change there. A reader
never drops a listing: one it cannot make a whole row of (no id, no title or
no link) it hands over with what it has, an empty string for what is
missing, and the runner drops it and counts it (`_runner.valid`). A reader
leaves out only what is not a listing of the board: an entry that is not an
object, a second copy of an id, a role of another board.
**Places.** A role has a list of places: `places` is `[Place(text=..., **Places.** A role has a list of places: `places` is `[Place(text=...,
remote=...)]`, one entry per place the host states, in the host's order, in remote=...)]`, one entry per place the host states, in the host's order, in
+2
View File
@@ -205,6 +205,8 @@ class Row(TypedDict, total=False):
`company` is the employer's name on a search row, which lists many `company` is the employer's name on a search row, which lists many
(`plugins/search.py` stores it); a board row carries none, the board is (`plugins/search.py` stores it); a board row carries none, the board is
the company. Any other stated key is kept in `raw` by the runner. the company. Any other stated key is kept in `raw` by the runner.
`source_id`, `title` and `url` come empty from a listing that lacks
them: the runner drops such a row and counts it (`_runner.valid`).
""" """
source_id: Required[str] source_id: Required[str]
+98 -20
View File
@@ -4,10 +4,12 @@ the reader declares in `READS`.
A reader says how its host answers; this module reads it: the guarded fetch A reader says how its host answers; this module reads it: the guarded fetch
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves (`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
to (`ask_for`) and its series (`series`), the listing whole or paged, one to (`ask_for`) and its series (`series`), the listing whole or paged, one
series per `one`-filter value unioned by id (`read_listing`), completeness, series per `one`-filter value unioned by id (`read_listing`), the rows it
the sweep of an oversized paged board across pulls, the 10% guard on every keeps (`valid`: a row without an id, a title or a link is dropped and
read that is not a sweep (`read`), the per-role detail reads counted here, never by the reader), completeness, the sweep of an oversized
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure paged board across pulls, the two 10% guards (`read`: on what a read that
is not a sweep listed, and on what any read dropped), the per-role detail
reads (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
`plugins/search.py` are thin `Source` classes over it: they pass their own `plugins/search.py` are thin `Source` classes over it: they pass their own
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
@@ -30,6 +32,8 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
drifts when roles close between pulls, so `app.jobs.pull` closes drifts when roles close between pulls, so `app.jobs.pull` closes
a role of such a board only when two finished sweeps in a row a role of such a board only when two finished sweeps in a row
missed it (it keeps the ids the last one missed in `missing`) missed it (it keeps the ids the last one missed in `missing`)
dropped the rows the last stamped read dropped (`Listing.dropped`), as a
number, "" for none; `app.jobs.limits` names it on the board's row
""" """
from __future__ import annotations from __future__ import annotations
@@ -52,7 +56,7 @@ import httpx
from desk.sources import USER_AGENT from desk.sources import USER_AGENT
from ._contract import Ask, Key, Place, Reads from ._contract import Ask, Key, Place, Reads
from ._shared import Get, Row from ._shared import Get, Row, is_web_url
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -96,7 +100,8 @@ SUSPECT_SHARE = 0.1
#: The last count from which the guard holds every read. A read that is #: The last count from which the guard holds every read. A read that is
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from #: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
#: a count of 1, as it was before every read was; a whole read of a board #: a count of 1, as it was before every read was; a whole read of a board
#: that listed a handful of roles can drop to none by itself. #: that listed a handful of roles can drop to none by itself. It is also the
#: rows a read must hand over before the share it dropped can fail it.
SUSPECT_MIN = 10 SUSPECT_MIN = 10
#: The keys of a row that an `Item` carries in its own fields, not in `raw`. #: The keys of a row that an `Item` carries in its own fields, not in `raw`.
@@ -113,8 +118,8 @@ class TooLarge(ValueError):
class Suspect(Exception): class Suspect(Exception):
"""A read that listed under `SUSPECT_SHARE` of what it listed last time """A read that listed under `SUSPECT_SHARE` of what it listed last time,
(see `read`).""" or dropped more than that share of its rows (see `read`)."""
class MissingKey(ValueError): class MissingKey(ValueError):
@@ -147,7 +152,9 @@ class Listing:
and roles closing meanwhile may have shifted an offset-paged host's later and roles closing meanwhile may have shifted an offset-paged host's later
roles back past the cursor, so `ids` can miss a role that is still open. roles back past the cursor, so `ids` can miss a role that is still open.
`remote_unasked` says a filtered read was meant to ask for remote roles `remote_unasked` says a filtered read was meant to ask for remote roles
as well and could not, so its `ids` say nothing of the remote roles.""" as well and could not, so its `ids` say nothing of the remote roles.
`handed` counts the rows the reader handed over this pull, `dropped`
those of them `valid` refused, which are in neither `rows` nor `ids`."""
rows: list[Row] rows: list[Row]
complete: bool complete: bool
@@ -158,6 +165,8 @@ class Listing:
narrowed: bool = False narrowed: bool = False
swept: bool = False swept: bool = False
remote_unasked: bool = False remote_unasked: bool = False
handed: int = 0
dropped: int = 0
# --------------------------------------------------------------- modules # --------------------------------------------------------------- modules
@@ -524,8 +533,42 @@ def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
return {str(r.get("id") or r.get("shortcode") or "") for r in results} return {str(r.get("id") or r.get("shortcode") or "") for r in results}
def valid(row: Mapping[str, Any]) -> bool:
"""Whether the runner keeps a row: a non-empty `source_id`, a non-empty
`title` and a web URL (`is_web_url`). A reader hands over every listing
as far as it can make it out; a row without one of the three cannot be
stored or opened, and the runner drops it and counts it."""
try:
link = is_web_url(str(row.get("url") or ""))
except (httpx.InvalidURL, ValueError): # a string httpx cannot read as a URL
link = False
return (link and bool(str(row.get("source_id") or "").strip())
and bool(str(row.get("title") or "").strip()))
@dataclass(slots=True)
class _Tally:
"""The rows a read was handed and dropped, across its series and pages."""
handed: int = 0
dropped: int = 0
def _keep(found: Iterable[Row], rows: dict[str, Row], tally: _Tally) -> None:
"""The rows a reader's `parse` or `rows` returned, into `rows` by
`source_id`, the first of an id kept; those `valid` refuses are counted
in `tally` and left out. The one place a row is checked, for the whole
and the paged listing alike."""
for row in found:
tally.handed += 1
if not valid(row):
tally.dropped += 1
continue
rows.setdefault(row["source_id"], row)
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask], def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
limits: Limits) -> dict[str, Row]: limits: Limits, tally: _Tally) -> dict[str, Row]:
"""A whole listing, one request a series, unioned by `source_id`. A body """A whole listing, one request a series, unioned by `source_id`. A body
over the cap gets one retry at the reader's `fallback_url`.""" over the cap gets one retry at the reader's `fallback_url`."""
rows: dict[str, Row] = {} rows: dict[str, Row] = {}
@@ -541,8 +584,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
log.warning("readers: %s:%s: body over the cap; reading the smaller feed", log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
module.NAME, slug) module.NAME, slug)
body = get(module.fallback_url(slug)) body = get(module.fallback_url(slug))
for row in module.parse(body, slug): _keep(module.parse(body, slug), rows, tally)
rows.setdefault(row["source_id"], row)
return rows return rows
@@ -580,7 +622,29 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
only: a search's places are what the search is, as its query is. only: a search's places are what the search is, as its query is.
`remote` says the reach or the params wanted remote roles; when no series `remote` says the reach or the params wanted remote roles; when no series
asked for them (a reader without a remote filter, or one whose facets asked for them (a reader without a remote filter, or one whose facets
state none beside places) the listing is `remote_unasked`.""" state none beside places) the listing is `remote_unasked`.
Every row `rows` or `parse` returns goes through `valid`: one without a
`source_id`, a title or a web URL is dropped here, counted in
`Listing.dropped` and logged once for the read. Completeness counts the
distinct result ids `page` returns beside its rows, not the rows kept, so
a dropped row still counts as read; a whole listing is complete whenever
it parses, whatever it dropped."""
tally = _Tally()
listing = _listing(module, slug, get, ask, limits=limits, sweep=sweep,
can_sweep=can_sweep, facets=facets, remote=remote, tally=tally)
listing.handed, listing.dropped = tally.handed, tally.dropped
if tally.dropped:
log.warning("readers: %s:%s: dropped %d of %d listings: no id, title or link",
module.NAME, slug, tally.dropped, tally.handed)
return listing
def _listing(module: ModuleType, slug: str, get: Get, ask: Ask | None, *,
limits: Limits | None, sweep: Mapping[str, Any] | None, can_sweep: bool,
facets: Mapping[str, Mapping[str, str]] | None, remote: bool,
tally: _Tally) -> Listing:
"""`read_listing` before its count of dropped rows, which `tally` keeps."""
limits = limits or Limits() limits = limits or Limits()
ask = ask or Ask() ask = ask or Ask()
reads: Reads = module.READS reads: Reads = module.READS
@@ -595,7 +659,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
"nothing asked, not closing", module.NAME, slug) "nothing asked, not closing", module.NAME, slug)
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked) return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
if not hasattr(module, "page"): if not hasattr(module, "page"):
whole_rows = _whole(module, slug, get, asks, limits) whole_rows = _whole(module, slug, get, asks, limits, tally)
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None, return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
key, by_words, remote_unasked=unasked) key, by_words, remote_unasked=unasked)
@@ -632,8 +696,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
pages += 1 pages += 1
before = len(seen) before = len(seen)
seen |= _result_ids(results) seen |= _result_ids(results)
for row in module.rows(data, slug): _keep(module.rows(data, slug), rows, tally)
rows.setdefault(row["source_id"], row)
if cursor is None or (isinstance(total, int) and len(seen) >= total): if cursor is None or (isinstance(total, int) and len(seen) >= total):
ended = True ended = True
break break
@@ -747,13 +810,26 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
that really shrank is accepted once the target has been failing for that really shrank is accepted once the target has been failing for
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its `limits.stale_days` (its `<prefix>:failing:<target>` record), and its
stamp then resets the count; a Recheck resets it at once stamp then resets the count; a Recheck resets it at once
(`plugins.ats.keep_size`).""" (`plugins.ats.keep_size`).
A read, a sweep's included, that dropped more than `SUSPECT_SHARE` of
the rows its reader handed over (`valid`), once it was handed at least
`SUSPECT_MIN`, raises `Suspect` the same way: a page whose shape changed
parses to rows without a title or a link, and the roles it hid would
close. It gives way after `limits.stale_days` of failing as well."""
limits = limits or Limits() limits = limits or Limits()
ask = ask or Ask() ask = ask or Ask()
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}") facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep, listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets, sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
remote=remote) remote=remote)
if listing.handed >= SUSPECT_MIN and listing.dropped > SUSPECT_SHARE * listing.handed:
why = (f"dropped {listing.dropped} of {listing.handed} listings, over "
f"{SUSPECT_SHARE:.0%}: no id, title or link")
if not _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
raise Suspect(why)
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
module.NAME, slug, why, limits.stale_days)
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect" held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
last = _load(state, f"{prefix}:count:{target}") last = _load(state, f"{prefix}:count:{target}")
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0 count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
@@ -788,9 +864,10 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
`complete`, `ids` (the compound ids, what the pull closes against), `complete`, `ids` (the compound ids, what the pull closes against),
`filtered`, `narrowed` (a read by terms, which the pull never closes `filtered`, `narrowed` (a read by terms, which the pull never closes
against), whether it finished a sweep from earlier pulls (`swept`), against), whether it finished a sweep from earlier pulls (`swept`),
whether a filtered read left remote unasked (`remote_unasked`), the whether a filtered read left remote unasked (`remote_unasked`), the rows
sweep to continue or none, and the count for the 10% guard once a read it dropped (`dropped`, "" for none), the sweep to continue or none, and
is not a sweep. A read that asked nothing keeps the count it had.""" the count for the 10% guard once a read is not a sweep. A read that
asked nothing keeps the count it had."""
state[f"{prefix}:ok:{target}"] = run state[f"{prefix}:ok:{target}"] = run
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else "" state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else "" state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
@@ -798,6 +875,7 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else "" state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else "" state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else "" state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
state[f"{prefix}:dropped:{target}"] = str(listing.dropped) if listing.dropped else ""
if listing.sweep is not None: if listing.sweep is not None:
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep) state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
return return
+1 -3
View File
@@ -24,7 +24,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Key, Place, Reads from ._contract import Ask, Fields, Filters, Key, Place, Reads
from ._shared import Get, Page, Row, is_web_url, only, parse_dt from ._shared import Get, Page, Row, only, parse_dt
NAME = "adzuna" NAME = "adzuna"
LABEL = "Adzuna" LABEL = "Adzuna"
@@ -111,8 +111,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
for job in decoded["results"]: for job in decoded["results"]:
title = _text(job.get("title") or "") title = _text(job.get("title") or "")
url = (job.get("redirect_url") or "").strip() url = (job.get("redirect_url") or "").strip()
if not title or not is_web_url(url):
continue
predicted = str(job.get("salary_is_predicted") or "0") == "1" predicted = str(job.get("salary_is_predicted") or "0") == "1"
town = ((job.get("location") or {}).get("display_name") or "").strip() town = ((job.get("location") or {}).get("display_name") or "").strip()
places: list[Place] = [{"text": town, "remote": None}] if town else [] places: list[Place] = [{"text": town, "remote": None}] if town else []
+1 -3
View File
@@ -14,7 +14,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -126,8 +126,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in data.get("jobs") or []: for j in data.get("jobs") or []:
title = (j.get("title") or "").strip() title = (j.get("title") or "").strip()
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip() url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
if not title or not is_web_url(url):
continue
description = j.get("descriptionHtml") or j.get("descriptionPlain") or "" description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
compensation = j.get("compensation") or {} compensation = j.get("compensation") or {}
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or []) salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
-2
View File
@@ -106,8 +106,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in listed: for j in listed:
title = (j.get("jobOpeningName") or "").strip() title = (j.get("jobOpeningName") or "").strip()
source_id = str(j.get("id") or "") source_id = str(j.get("id") or "")
if not title or not source_id:
continue
places = _places(j) places = _places(j)
row: Row = { row: Row = {
"source_id": source_id, "source_id": source_id,
-2
View File
@@ -132,8 +132,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for p in postings: for p in postings:
title = (p.get("title") or "").strip() title = (p.get("title") or "").strip()
post_id = str(p.get("id") or "") post_id = str(p.get("id") or "")
if not title or not post_id:
continue
job = p.get("job") or {} job = p.get("job") or {}
places = [name for loc in job.get("jobLocations") or [] places = [name for loc in job.get("jobLocations") or []
if (name := ((loc.get("location") or {}).get("name") or "").strip())] if (name := ((loc.get("location") or {}).get("name") or "").strip())]
+1 -3
View File
@@ -47,7 +47,7 @@ from urllib.parse import quote
from ._contract import Ask, Fields, Filters, Place, Reads, Row from ._contract import Ask, Fields, Filters, Place, Reads, Row
from ._countries import name_of from ._countries import name_of
from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place from ._shared import Get, Page, add_place, html_text, only, parse_dt, place
READS = Reads( READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated", scope="board", listing="paged", page_size=10, text="detail", complete="stated",
@@ -151,8 +151,6 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
source_id = str(p.get("id") or "") source_id = str(p.get("id") or "")
path = str(p.get("positionUrl") or "").strip() path = str(p.get("positionUrl") or "").strip()
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else "" url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
if not title or not source_id or not is_web_url(url):
continue
places = _places(p.get("locations") or p.get("standardizedLocations")) places = _places(p.get("locations") or p.get("standardizedLocations"))
out.append({ out.append({
"source_id": source_id, "source_id": source_id,
+1 -3
View File
@@ -37,7 +37,7 @@ from urllib.parse import urlencode, urlsplit, urlunsplit
from desk.sources import USER_AGENT from desk.sources import USER_AGENT
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line, from ._shared import (Get, Page, Row, html_text, num, only, parse_dt, pay_line,
yearly_amount) yearly_amount)
NAME = "freehire" NAME = "freehire"
@@ -170,8 +170,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
title = " ".join(html.unescape(str(hit.get("title") or "")).split()) title = " ".join(html.unescape(str(hit.get("title") or "")).split())
source_id = str(hit.get("public_slug") or "").strip() source_id = str(hit.get("public_slug") or "").strip()
url = _url(str(hit.get("url") or "")) url = _url(str(hit.get("url") or ""))
if not title or not source_id or not is_web_url(url):
continue
mode = str(hit.get("work_mode") or "").strip() mode = str(hit.get("work_mode") or "").strip()
places = _places(hit, mode) places = _places(hit, mode)
countries = [str(c) for c in hit.get("countries") or [] if c] countries = [str(c) for c in hit.get("countries") or [] if c]
-2
View File
@@ -116,8 +116,6 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
for p in postings: for p in postings:
title = (p.get("title") or "").strip() title = (p.get("title") or "").strip()
ext_id = str(p.get("extId") or "") ext_id = str(p.get("extId") or "")
if not title or not ext_id:
continue
locations = p.get("locations") or [] locations = p.get("locations") or []
places: list[Place] = [] places: list[Place] = []
for loc in locations: for loc in locations:
+1 -3
View File
@@ -13,7 +13,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place
NAME = "greenhouse" NAME = "greenhouse"
LABEL = "Greenhouse" LABEL = "Greenhouse"
@@ -149,8 +149,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in data.get("jobs") or []: for j in data.get("jobs") or []:
title = (j.get("title") or "").strip() title = (j.get("title") or "").strip()
url = (j.get("absolute_url") or "").strip() url = (j.get("absolute_url") or "").strip()
if not title or not is_web_url(url):
continue
departments = j.get("departments") or [] departments = j.get("departments") or []
ranges = j.get("pay_input_ranges") or [] ranges = j.get("pay_input_ranges") or []
salary_min, salary_max, currency = _salary(ranges) salary_min, salary_max, currency = _salary(ranges)
-2
View File
@@ -90,8 +90,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
for ad in data.get("jobAdDetails") or []: for ad in data.get("jobAdDetails") or []:
title = (ad.get("title") or "").strip() title = (ad.get("title") or "").strip()
ad_id = str(ad.get("id") or "").strip() ad_id = str(ad.get("id") or "").strip()
if not title or not ad_id:
continue
site = str(ad.get("site") or "").strip() site = str(ad.get("site") or "").strip()
country = str(ad.get("country") or "").strip() country = str(ad.get("country") or "").strip()
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \ location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
+1 -3
View File
@@ -30,7 +30,7 @@ from typing import Any
from ._contract import Ask, Fields, Reads, Row from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, Page, html_text, is_web_url, only, parse_dt from ._shared import Get, Page, html_text, only, parse_dt
READS = Reads( READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="end", scope="board", listing="paged", page_size=100, text="listing", complete="end",
@@ -106,8 +106,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
for item in items: for item in items:
title = (item.findtext("title") or "").strip() title = (item.findtext("title") or "").strip()
url = (item.findtext("link") or "").strip() url = (item.findtext("link") or "").strip()
if not title or not is_web_url(url):
continue
out.append({ out.append({
"source_id": (item.findtext("guid") or url).strip(), "source_id": (item.findtext("guid") or url).strip(),
"title": title, "title": title,
-2
View File
@@ -127,8 +127,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
j = job.get("data") or {} j = job.get("data") or {}
title = (j.get("title") or "").strip() title = (j.get("title") or "").strip()
role = str(j.get("slug") or j.get("req_id") or "") role = str(j.get("slug") or j.get("req_id") or "")
if not title or not role:
continue
categories = j.get("categories") or [] categories = j.get("categories") or []
text = "\n".join(p for p in (html_text(j.get("description")), text = "\n".join(p for p in (html_text(j.get("description")),
html_text(j.get("qualifications"))) if p) html_text(j.get("qualifications"))) if p)
+3 -5
View File
@@ -36,7 +36,7 @@ from typing import Any
from urllib.parse import urlencode, urlsplit from urllib.parse import urlencode, urlsplit
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt, place from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt, place
NAME = "jobbank" NAME = "jobbank"
LABEL = "Akademikernes Jobbank (Denmark)" LABEL = "Akademikernes Jobbank (Denmark)"
@@ -132,8 +132,8 @@ def _query(ask: Ask, number: int) -> tuple[str, str]:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search; the cursor is the page number. The next page """One page of a search; the cursor is the page number. The next page
follows a full page (counted before any item is dropped, so a dropped item follows a full page (counted before the runner drops an item, so a
does not pass for the end) while the ceiling is not reached.""" dropped item does not pass for the end) while the ceiling is not reached."""
number = int(cursor or 1) number = int(cursor or 1)
url, remote = _query(ask, number) url, remote = _query(ask, number)
try: try:
@@ -184,8 +184,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = [] out: list[Row] = []
for item in decoded["items"]: for item in decoded["items"]:
link, title = item["link"], item["title"] link, title = item["link"], item["title"]
if not title or not item["id"] or not is_web_url(link):
continue
line = _LINE.fullmatch(item["description"]) line = _LINE.fullmatch(item["description"])
kind = line.group("kind").strip() if line else "" kind = line.group("kind").strip() if line else ""
company, regions, cut = _regions(line.group("rest")) if line else ("", [], False) company, regions, cut = _regions(line.group("rest")) if line else ("", [], False)
+3 -5
View File
@@ -117,9 +117,9 @@ def _query(ask: Ask, number: int) -> str:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of a search, newest first; the cursor is the page number. """One page of a search, newest first; the cursor is the page number.
The next page follows a full page (counted before any item is dropped, so The next page follows a full page (counted before the runner drops an
a dropped item does not pass for the end) while the 200-result ceiling is item, so a dropped item does not pass for the end) while the 200-result
not reached.""" ceiling is not reached."""
number = int(cursor or 1) number = int(cursor or 1)
root = ET.fromstring(get(_query(ask, number))) root = ET.fromstring(get(_query(ask, number)))
channel = root.find("channel") channel = root.find("channel")
@@ -143,8 +143,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
out: list[Row] = [] out: list[Row] = []
for item in decoded["items"]: for item in decoded["items"]:
link, full_title = item["link"], item["title"] link, full_title = item["link"], item["title"]
if not full_title or not is_web_url(link):
continue
description = item["description"] description = item["description"]
h4 = _H4.search(description) h4 = _H4.search(description)
title = _text(h4.group(1)) if h4 else full_title title = _text(h4.group(1)) if h4 else full_title
-2
View File
@@ -138,8 +138,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
for ad in decoded.get("jobAds") or []: for ad in decoded.get("jobAds") or []:
ident = str(ad.get("jobAdId") or "").strip() ident = str(ad.get("jobAdId") or "").strip()
title = _text(ad.get("title")) title = _text(ad.get("title"))
if not ident or not title:
continue
link = str(ad.get("jobAdUrl") or "").strip() link = str(ad.get("jobAdUrl") or "").strip()
town = _text(ad.get("municipality") or ad.get("postalDistrictName")) town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
places: list[Place] = [place(town)] if town else [] places: list[Place] = [place(town)] if town else []
+1 -3
View File
@@ -176,8 +176,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
title = (j.get("title") or "").strip() title = (j.get("title") or "").strip()
job_id = str(j.get("id") or "").strip() job_id = str(j.get("id") or "").strip()
path = str(j.get("url") or "").strip() path = str(j.get("url") or "").strip()
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
continue
function = str(j.get("function") or "").strip() function = str(j.get("function") or "").strip()
places = [place(name) for name in dict.fromkeys( places = [place(name) for name in dict.fromkeys(
str(name).strip() for name in j.get("locations") or [] if str(name).strip())] str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
@@ -186,7 +184,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
"title": title, "title": title,
"places": places, "places": places,
"location": "; ".join(pl["text"] for pl in places), "location": "; ".join(pl["text"] for pl in places),
"url": f"https://{_EMP}{path}", "url": f"https://{_EMP}{path}" if _ROLE_PATH.fullmatch(path) else "",
"posted": _published(j.get("published_date")), "posted": _published(j.get("published_date")),
"department": "" if function == "All functions" else function, "department": "" if function == "All functions" else function,
"description": "", "description": "",
+1 -3
View File
@@ -22,7 +22,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads, Row from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -123,8 +123,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in data: for j in data:
title = (j.get("text") or "").strip() title = (j.get("text") or "").strip()
url = (j.get("hostedUrl") or "").strip() url = (j.get("hostedUrl") or "").strip()
if not title or not is_web_url(url):
continue
categories = j.get("categories") or {} categories = j.get("categories") or {}
description = j.get("description") or j.get("descriptionPlain") or "" description = j.get("description") or j.get("descriptionPlain") or ""
salary_range = j.get("salaryRange") or {} salary_range = j.get("salaryRange") or {}
+4 -4
View File
@@ -198,9 +198,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
for role in search.get("requisitionList") or []: for role in search.get("requisitionList") or []:
role_id = str(role.get("Id") or "").strip() role_id = str(role.get("Id") or "").strip()
title = _text(role.get("Title")) title = _text(role.get("Title"))
# The id goes into a URL, and a listing is external content. # The id goes into a URL, and a listing is external content: an id
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id): # of other characters gets no link, and the runner drops the row.
continue safe = bool(re.fullmatch(r"[A-Za-z0-9_-]+", role_id))
remote = _text(role.get("WorkplaceTypeCode")) or None remote = _text(role.get("WorkplaceTypeCode")) or None
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site") at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
places: list[Place] = [] places: list[Place] = []
@@ -214,7 +214,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
"title": title, "title": title,
"places": places, "places": places,
"location": "; ".join(p["text"] for p in places), "location": "; ".join(p["text"] for p in places),
"url": f"{_base(slug)}/job/{role_id}", "url": f"{_base(slug)}/job/{role_id}" if safe else "",
"posted": parse_dt(role.get("PostedDate")), "posted": parse_dt(role.get("PostedDate")),
"department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction")) "department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction"))
or _text(role.get("Department")) or _text(role.get("Organization"))), or _text(role.get("Department")) or _text(role.get("Organization"))),
-2
View File
@@ -104,8 +104,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for position in root.findall("position"): for position in root.findall("position"):
job_id = (position.findtext("id") or "").strip() job_id = (position.findtext("id") or "").strip()
title = (position.findtext("name") or "").strip() title = (position.findtext("name") or "").strip()
if not job_id or not title:
continue
description = " ".join( description = " ".join(
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value")) (value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
salary_info = position.find("salaryInformation") salary_info = position.find("salaryInformation")
-2
View File
@@ -168,8 +168,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
for job in search.get("data", {}).get("jobs") or []: for job in search.get("data", {}).get("jobs") or []:
key = _key(job) key = _key(job)
title = _text(job.get("title")) title = _text(job.get("title"))
if not title or not key:
continue
places: list[Place] = [] places: list[Place] = []
for text in (job.get("location"), *(job.get("multi_location") or [])): for text in (job.get("location"), *(job.get("multi_location") or [])):
if isinstance(text, str): if isinstance(text, str):
+1 -3
View File
@@ -22,7 +22,7 @@ from typing import Any
from ._contract import Ask, Fields, Reads from ._contract import Ask, Fields, Reads
from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount from ._shared import HEADING, Row, html_text, label_slug_ok, num, only, pay_line, yearly_amount
NAME = "pinpoint" NAME = "pinpoint"
LABEL = "Pinpoint" LABEL = "Pinpoint"
@@ -121,8 +121,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
title = (j.get("title") or "").strip() title = (j.get("title") or "").strip()
url = (j.get("url") or "").strip() url = (j.get("url") or "").strip()
source_id = str(j.get("id") or "") source_id = str(j.get("id") or "")
if not title or not source_id or not is_web_url(url):
continue
job = j.get("job") or {} job = j.get("job") or {}
low, high, currency, salary_text = _pay(j) low, high, currency, salary_text = _pay(j)
workplace = str(j.get("workplace_type") or "").strip() workplace = str(j.get("workplace_type") or "").strip()
+1 -3
View File
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
from desk.sources import strip_html from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt from ._shared import Get, Page, Row, html_text, only, parse_dt
NAME = "platsbanken" NAME = "platsbanken"
LABEL = "Platsbanken" LABEL = "Platsbanken"
@@ -108,8 +108,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
for hit in decoded.get("hits") or []: for hit in decoded.get("hits") or []:
title = _text(hit.get("headline") or "") title = _text(hit.get("headline") or "")
url = (hit.get("webpage_url") or "").strip() url = (hit.get("webpage_url") or "").strip()
if not title or not is_web_url(url):
continue
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}] addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
places = [place for address in addresses if isinstance(address, dict) places = [place for address in addresses if isinstance(address, dict)
if (place := _place(address))] if (place := _place(address))]
+1 -3
View File
@@ -13,7 +13,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads from ._contract import Ask, Fields, Place, Reads
from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
NAME = "recruitee" NAME = "recruitee"
LABEL = "Recruitee" LABEL = "Recruitee"
@@ -105,8 +105,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
for j in data.get("offers") or []: for j in data.get("offers") or []:
title = (j.get("title") or "").strip() title = (j.get("title") or "").strip()
url = (j.get("careers_url") or "").strip() url = (j.get("careers_url") or "").strip()
if not title or not is_web_url(url):
continue
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {} salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
salary_min, salary_max, currency = yearly_amount( salary_min, salary_max, currency = yearly_amount(
salary.get("min"), salary.get("max"), salary.get("currency"), salary.get("min"), salary.get("max"), salary.get("currency"),
+14 -10
View File
@@ -82,27 +82,31 @@ def parse(body: bytes, slug: str) -> list[Row]:
raise ValueError("not a Rippling board: the answer is not a list") raise ValueError("not a Rippling board: the answer is not a list")
roles: dict[str, Row] = {} roles: dict[str, Row] = {}
places: dict[str, list[Place]] = {} places: dict[str, list[Place]] = {}
for entry in data: for index, entry in enumerate(data):
job_id = str(entry.get("uuid") or "").strip() job_id = str(entry.get("uuid") or "").strip()
title = (entry.get("name") or "").strip() title = (entry.get("name") or "").strip()
if not title or not _UUID.fullmatch(job_id): # The id goes into a URL: one of other characters gets no link, and
continue # the runner drops the row. An entry without an id is a role of its
# own, not merged with the others that lack one.
key = job_id or f"#{index}"
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip() place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
places.setdefault(job_id, []) places.setdefault(key, [])
if place: if place:
add_place(places[job_id], _place(place)) add_place(places[key], _place(place))
roles.setdefault(job_id, { roles.setdefault(key, {
"source_id": job_id, "source_id": job_id,
"title": title, "title": title,
"url": f"https://{_SITE}/{slug}/jobs/{job_id}", "url": f"https://{_SITE}/{slug}/jobs/{job_id}" if _UUID.fullmatch(job_id) else "",
"posted": None, "posted": None,
"department": str(((entry.get("department") or {}).get("label")) or "").strip(), "department": str(((entry.get("department") or {}).get("label")) or "").strip(),
"description": "", "description": "",
"salary_text": "", # the board's list states no pay "salary_text": "", # the board's list states no pay
}) })
for job_id, row in roles.items(): if title and not roles[key]["title"]:
row["places"] = places[job_id] roles[key]["title"] = title # an entry of the role that has one
row["location"] = "; ".join(p["text"] for p in places[job_id]) for key, row in roles.items():
row["places"] = places[key]
row["location"] = "; ".join(p["text"] for p in places[key])
return list(roles.values()) return list(roles.values())
-2
View File
@@ -115,8 +115,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
for j in data.get("content") or []: for j in data.get("content") or []:
title = (j.get("name") or "").strip() title = (j.get("name") or "").strip()
job_id = str(j.get("id") or "") job_id = str(j.get("id") or "")
if not title or not job_id:
continue
location = j.get("location") or {} location = j.get("location") or {}
loc_text = (location.get("fullLocation") loc_text = (location.get("fullLocation")
or ", ".join(p for p in (location.get("city"), location.get("country")) if p)) or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
+25 -16
View File
@@ -63,7 +63,7 @@ from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES from ._countries import NAMES
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
NAME = "successfactors" NAME = "successfactors"
LABEL = "SuccessFactors" LABEL = "SuccessFactors"
@@ -247,8 +247,9 @@ def _tile_fields(chunk: str) -> tuple[str, str, str]:
def _entries(text: str) -> list[dict[str, Any]]: def _entries(text: str) -> list[dict[str, Any]]:
"""The roles of one search page, in the order shown, whichever layout it is """The roles of one search page, in the order shown, whichever layout it is
in and before the prefix filter: `id`, `href` (as written, relative), in and before the prefix filter: `id`, `href` (as written, relative),
`title`, `place`, `date` and `department`. A role is listed once, and a row `title`, `place`, `date` and `department`. A role is listed once. The id
without a link to a role is not one.""" and the title are in the role's link, so a row without one is handed over
with an empty `id`, `href` and `title`, for the runner to drop and count."""
table = _ROW.split(text)[1:] table = _ROW.split(text)[1:]
chunks = table or _TILE.split(text)[1:] chunks = table or _TILE.split(text)[1:]
out: list[dict[str, Any]] = [] out: list[dict[str, Any]] = []
@@ -256,10 +257,12 @@ def _entries(text: str) -> list[dict[str, Any]]:
for chunk in chunks: for chunk in chunks:
if table: if table:
chunk = chunk.split("</tr>", 1)[0] chunk = chunk.split("</tr>", 1)[0]
if not (link := _LINK.search(chunk)): if link := _LINK.search(chunk):
continue href, job_id, title = (html.unescape(link.group(1)), link.group(2),
href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3)) _text(link.group(3)))
if job_id in seen: else:
href = job_id = title = ""
if job_id and job_id in seen:
continue continue
seen.add(job_id) seen.add(job_id)
place, date, department = (_table_fields if table else _tile_fields)(chunk) place, date, department = (_table_fields if table else _tile_fields)(chunk)
@@ -283,12 +286,14 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
if not _SEARCH_PAGE.search(text): if not _SEARCH_PAGE.search(text):
raise ValueError("not a SuccessFactors search page") raise ValueError("not a SuccessFactors search page")
found = _entries(text) found = _entries(text)
ids = [entry["id"] for entry in found if entry["id"]]
total = _total(text) total = _total(text)
if found and found[0]["id"] == previous: first = ids[0] if ids else None
if first is not None and first == previous:
return text, [], total, (start, previous) return text, [], total, (start, previous)
more = bool(found) and (total is None or start + len(found) < total) more = bool(found) and (total is None or start + len(found) < total)
return (text, [{"id": entry["id"]} for entry in found], total, return (text, [{"id": job_id} for job_id in ids], total,
(start + len(found), found[0]["id"]) if more else None) (start + len(found), first) if more else None)
def rows(decoded: str, slug: str) -> list[Row]: def rows(decoded: str, slug: str) -> list[Row]:
@@ -296,17 +301,21 @@ def rows(decoded: str, slug: str) -> list[Row]:
slug's prefix when it has one; the id in it is the requisition's `source_id`. slug's prefix when it has one; the id in it is the requisition's `source_id`.
The table and the tile layout give a title, a location, a date and, where The table and the tile layout give a title, a location, a date and, where
the site has the column, a department; nothing else about a role is on the site has the column, a department; nothing else about a role is on
the list, so its description and pay are empty.""" the list, so its description and pay are empty.
A role linked on another host, or outside the prefix, is another board's
(another site, or another brand of a shared site), and is left out: it is
no role of this company. A row with no link or no title is handed over,
and the runner drops it."""
host, prefix = _parts(slug) host, prefix = _parts(slug)
base = f"https://{host}/" base = f"https://{host}/"
out = [] out = []
for entry in _entries(decoded): for entry in _entries(decoded):
url = urljoin(base, entry["href"]) url = urljoin(base, entry["href"]) if entry["href"] else ""
if urlsplit(url).hostname != host or not is_web_url(url): if url and urlsplit(url).hostname != host:
continue continue
if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"): if url and prefix and not urlsplit(url).path.lower().startswith(
continue f"{prefix.lower()}/job/"):
if not entry["title"]:
continue continue
out.append({ out.append({
"source_id": entry["id"], "source_id": entry["id"],
+1 -3
View File
@@ -15,7 +15,7 @@ from typing import Any
from ._contract import Ask, Fields, Place, Reads from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
NAME = "teamtailor" NAME = "teamtailor"
LABEL = "Teamtailor" LABEL = "Teamtailor"
@@ -115,8 +115,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
for item in items: for item in items:
title = (item.findtext("title") or "").strip() title = (item.findtext("title") or "").strip()
url = (item.findtext("link") or "").strip() url = (item.findtext("link") or "").strip()
if not title or not is_web_url(url):
continue
status = (item.findtext("remoteStatus") or "").strip() status = (item.findtext("remoteStatus") or "").strip()
remote = {"fully": True, "hybrid": False, "onsite": False}.get(status) remote = {"fully": True, "hybrid": False, "onsite": False}.get(status)
places: list[Place] = [] places: list[Place] = []
-2
View File
@@ -102,8 +102,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
for j in data.get("results") or []: for j in data.get("results") or []:
title = (j.get("title") or "").strip() title = (j.get("title") or "").strip()
shortcode = (j.get("shortcode") or "").strip() shortcode = (j.get("shortcode") or "").strip()
if not title or not shortcode:
continue
workplace = str(j.get("workplace") or "").strip() workplace = str(j.get("workplace") or "").strip()
remote = bool(j.get("remote")) or workplace == "remote" remote = bool(j.get("remote")) or workplace == "remote"
places = _places(j, remote) places = _places(j, remote)
+6 -6
View File
@@ -124,8 +124,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
postings = data.get("jobPostings") or [] postings = data.get("jobPostings") or []
# Every posting is counted, a path-less one (LEGO lists one that has only # Every posting is counted, a path-less one (LEGO lists one that has only
# `bulletFields`) by its position, so the parser dropping it does not # `bulletFields`) by its position, so the runner dropping its row does
# leave the board short of its total. # not leave the board short of its total.
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)] results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
following = offset + len(postings) following = offset + len(postings)
more = bool(postings) and (total is None or following < total) more = bool(postings) and (total is None or following < total)
@@ -274,13 +274,13 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
`bulletFields`, which lead with the requisition id, and on some tenants `bulletFields`, which lead with the requisition id, and on some tenants
`remoteType`. No description, no date and no country: `detail` has `remoteType`. No description, no date and no country: `detail` has
those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the
path where not; a posting with no title or no path is skipped.""" path where not. A posting with no path has neither an id nor a link, and
the runner drops it."""
out = [] out = []
for p in decoded.get("jobPostings") or []: for p in decoded.get("jobPostings") or []:
title = (p.get("title") or "").strip() title = (p.get("title") or "").strip()
path = (p.get("externalPath") or "").strip() path = (p.get("externalPath") or "").strip()
if not title or not _ROLE_PATH.fullmatch(path): path = path if _ROLE_PATH.fullmatch(path) else ""
continue
remote = _remote(p.get("remoteType")) remote = _remote(p.get("remoteType"))
place = _place(p.get("locationsText"), remote) place = _place(p.get("locationsText"), remote)
out.append({ out.append({
@@ -288,7 +288,7 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
"title": title, "title": title,
"places": [place] if place else [], "places": [place] if place else [],
"location": place["text"] if place else "", "location": place["text"] if place else "",
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}", # type: ignore[index] "url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}" if path else "", # type: ignore[index]
"posted": _posted(p.get("postedOn") or ""), "posted": _posted(p.get("postedOn") or ""),
"department": "", "department": "",
"description": "", "description": "",
+2 -1
View File
@@ -19,7 +19,8 @@ roles close once no search has seen them for `STALE_DAYS` (see
`app/jobs/pull.py`). `app/jobs/pull.py`).
In a pull a search is read by `_runner.read` under the prefix `search`, as a In a pull a search is read by `_runner.read` under the prefix `search`, as a
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it, board is under `ats`: the runner drops a row without an id, a title or a
link, the 10% guards hold it, `_runner.stamp` records it,
and a search that fails or is suspect is kept as failing under and a search that fails or is suspect is kept as failing under
`search:failing:<target>` (`_runner.record_failure`) until it answers, which `search:failing:<target>` (`_runner.record_failure`) until it answers, which
the settle reads as it reads a failing board's. the settle reads as it reads a failing board's.
+15 -8
View File
@@ -24,18 +24,24 @@ from plugins.readers import _runner, smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
def _rows(host: str, slug: str = "acme") -> list[ats.Row]: def _parsed(host: str, slug: str = "acme") -> list[ats.Row]:
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}" path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
body = path.read_bytes() return ats._PARSERS[host](path.read_bytes(), slug)
return ats._PARSERS[host](body, slug)
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
"""The rows of a host's fixture the runner keeps (`_runner.valid`)."""
return [row for row in _parsed(host, slug) if _runner.valid(row)]
# ------------------------------------------------------------------ hosts # ------------------------------------------------------------------ hosts
def test_greenhouse_parses_titles_locations_and_html_description() -> None: def test_greenhouse_parses_titles_locations_and_html_description() -> None:
# The javascript: and data: rows are handed over and the runner drops
# them: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in _parsed("greenhouse")] == [
"1001", "1002", "1003", "1004", "1005", "1006"]
rows = _rows("greenhouse") rows = _rows("greenhouse")
# The javascript: and data: rows are refused, same as the four real ones
# are kept: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"] assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
first = rows[0] first = rows[0]
assert first["title"] == "Senior Backend Engineer" # trimmed assert first["title"] == "Senior Backend Engineer" # trimmed
@@ -269,10 +275,11 @@ TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None: def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry") rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
# the javascript: link is refused like on the other hosts # the javascript: link is handed over and refused by the runner
assert [r["source_id"] for r in rows] == [ assert [r["source_id"] for r in rows] == [
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb", "39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"] "0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"]
assert [_runner.valid(r) for r in rows] == [True, True, True, False]
first = rows[0] first = rows[0]
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer" assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
assert first["department"] == "Engineering" assert first["department"] == "Engineering"
@@ -294,7 +301,7 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None: def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry") rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them
assert rows[1]["title"] == "Läkare" # trimmed assert rows[1]["title"] == "Läkare" # trimmed
assert rows[1]["department"] == "Software Development" assert rows[1]["department"] == "Software Development"
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales" assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
+8 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, bamboohr # noqa: E402 from plugins.readers import Ask, _runner, bamboohr # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -84,11 +84,16 @@ def test_a_page_that_is_not_a_listing_raises_and_an_empty_board_is_no_rows() ->
bamboohr.parse(body, "configit") bamboohr.parse(body, "configit")
def test_a_role_without_a_title_or_an_id_is_left_out() -> None: def test_a_role_without_a_title_or_an_id_reaches_the_runner_and_is_dropped_there() -> None:
body = json.loads(_read("bamboohr.json")) body = json.loads(_read("bamboohr.json"))
body["result"][0]["jobOpeningName"] = " " body["result"][0]["jobOpeningName"] = " "
body["result"][1]["id"] = None body["result"][1]["id"] = None
assert bamboohr.parse(json.dumps(body).encode(), "configit") == [] raw = json.dumps(body).encode()
rows = bamboohr.parse(raw, "configit")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
listing = _runner.read_listing(bamboohr, "configit", lambda url, **kw: raw)
assert (listing.rows, listing.ids, listing.complete) == ([], [], True)
assert (listing.handed, listing.dropped) == (2, 2)
def test_detail_reads_the_text_the_date_and_the_country() -> None: def test_detail_reads_the_text_the_date_and_the_country() -> None:
+5 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, deel # noqa: E402 from plugins.readers import Ask, _runner, deel # noqa: E402
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes() PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
URL = "https://jobs.deel.com/klarna" URL = "https://jobs.deel.com/klarna"
@@ -89,10 +89,12 @@ def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody") deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
def test_a_posting_without_a_title_or_id_is_skipped() -> None: def test_a_posting_without_a_title_or_id_is_handed_over_for_the_runner_to_drop() -> None:
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}}, body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
{"id": "c", "title": "Cook", "job": {}}]} {"id": "c", "title": "Cook", "job": {}}]}
assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"] rows = deel.parse(_page_with(body), "acme")
assert [r["source_id"] for r in rows] == ["a", "", "c"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
# ------------------------------------------------------------ the fetch # ------------------------------------------------------------ the fetch
+5 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, eightfold # noqa: E402 from plugins.readers import Ask, _runner, eightfold # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "jobs.ericsson.com/ericsson.com" SLUG = "jobs.ericsson.com/ericsson.com"
@@ -93,10 +93,12 @@ def test_the_work_location_option_is_the_hosts_word(
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
def test_a_position_without_a_name_an_id_or_a_path_is_left_out() -> None: def test_a_position_without_a_name_an_id_or_a_path_is_handed_over_for_the_runner_to_drop() -> None:
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None}, bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}] _position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
assert eightfold.rows(bad, SLUG) == [] rows = eightfold.rows(bad, SLUG)
assert len(rows) == 4 and not any(map(_runner.valid, rows))
assert [r["url"] for r in rows[2:]] == ["", ""] # no link built off the host
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None: def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
+4 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, gem # noqa: E402 from plugins.readers import Ask, _runner, gem # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
API = "https://jobs.gem.com/api/public/graphql" API = "https://jobs.gem.com/api/public/graphql"
@@ -79,8 +79,9 @@ def test_remote_is_the_location_type_then_the_locations(
assert gem.rows([posting], "acme")[0]["remote"] == remote assert gem.rows([posting], "acme")[0]["remote"] == remote
def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None: def test_a_posting_without_a_title_or_an_ext_id_is_handed_over_for_the_runner_to_drop() -> None:
assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == [] rows = gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
def test_the_board_is_one_post_without_origin_or_referer() -> None: def test_the_board_is_one_post_without_origin_or_referer() -> None:
+5 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, hibob # noqa: E402 from plugins.readers import Ask, _runner, hibob # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hibob.json").read_bytes() BOARD = (FIXTURES / "hibob.json").read_bytes()
@@ -87,10 +87,12 @@ def test_the_place_leaves_the_country_off_where_the_site_names_it() -> None:
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
def test_an_ad_without_a_title_or_id_is_skipped_and_an_empty_board_is_empty() -> None: def test_an_ad_without_a_title_or_id_is_handed_over_and_an_empty_board_is_empty() -> None:
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"}, body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
{"id": "c", "title": "Cook"}]}).encode() {"id": "c", "title": "Cook"}]}).encode()
assert [r["source_id"] for r in hibob.parse(body, "acme")] == ["c"] rows = hibob.parse(body, "acme")
assert [r["source_id"] for r in rows] == ["a", "", "c"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == [] assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
+5 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, hrmanager # noqa: E402 from plugins.readers import Ask, _runner, hrmanager # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hrmanager.xml").read_bytes() BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
@@ -52,12 +52,14 @@ def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_f
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x") hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None: def test_an_item_without_a_title_or_a_web_link_is_handed_over_for_the_runner_to_drop() -> None:
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>" body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>" b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>" b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
b"</channel></rss>") b"</channel></rss>")
assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"] rows = hrmanager.parse(body, "x")
assert [r["source_id"] for r in rows] == ["1", "2", "3"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["3"]
# ------------------------------------------------------------- paging # ------------------------------------------------------------- paging
+4 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, jibe # noqa: E402 from plugins.readers import Ask, _runner, jibe # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SITE = "jobs.booking.com" SITE = "jobs.booking.com"
@@ -72,9 +72,10 @@ def test_the_location_falls_back_to_city_and_country() -> None:
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia" assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None: def test_a_role_without_a_title_or_a_slug_is_handed_over_for_the_runner_to_drop() -> None:
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}] jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
assert jibe.rows({"jobs": jobs}, SITE) == [] rows = jibe.rows({"jobs": jobs}, SITE)
assert len(rows) == 3 and not any(map(_runner.valid, rows))
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None: def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
+5 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, jobylon # noqa: E402 from plugins.readers import Ask, _runner, jobylon # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "jobylon.html").read_bytes() BOARD = (FIXTURES / "jobylon.html").read_bytes()
@@ -65,12 +65,14 @@ def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"] assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None: def test_a_role_whose_link_is_not_a_role_page_gets_no_link_and_is_dropped() -> None:
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'}," body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'}," b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
b"{id: '3', title: 'C', url: '/companies/3/'}," b"{id: '3', title: 'C', url: '/companies/3/'},"
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n") b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"] rows = jobylon.parse(body, "9")
assert [r["url"] for r in rows[:3]] == ["", "", ""]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["4"]
# ---------------------------------------------------------------- detail # ---------------------------------------------------------------- detail
+5 -4
View File
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402 from plugins import ats # noqa: E402
from plugins.readers import oracle # noqa: E402 from plugins.readers import _runner, oracle # noqa: E402
from plugins.readers._shared import is_web_url # noqa: E402 from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -84,14 +84,15 @@ def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""] assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_left_out() -> None: def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_dropped_by_the_runner() -> None:
data = _listing() data = _listing()
roles = data["items"][0]["requisitionList"] roles = data["items"][0]["requisitionList"]
roles[0]["Title"] = " " roles[0]["Title"] = " "
roles[1]["Id"] = "1/../../x" roles[1]["Id"] = "1/../../x"
roles[2]["Id"] = "" roles[2]["Id"] = ""
assert [r["source_id"] for r in oracle.parse(json.dumps(data).encode(), SLUG)] == \ rows = oracle.parse(json.dumps(data).encode(), SLUG)
["40975", "31680"] assert rows[1]["url"] == "" # the id builds no link
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["40975", "31680"]
def test_a_listing_without_the_expand_has_no_roles() -> None: def test_a_listing_without_the_expand_has_no_roles() -> None:
+5 -3
View File
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402 from plugins import ats # noqa: E402
from plugins.readers import phenom # noqa: E402 from plugins.readers import _runner, phenom # noqa: E402
from plugins.readers._shared import is_web_url # noqa: E402 from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -101,13 +101,15 @@ def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS" "https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
def test_a_role_without_title_or_id_is_left_out() -> None: def test_a_role_without_title_or_id_is_dropped_by_the_runner() -> None:
data = _answer() data = _answer()
jobs = data["refineSearch"]["data"]["jobs"] jobs = data["refineSearch"]["data"]["jobs"]
jobs[0]["title"] = "" jobs[0]["title"] = ""
for key in ("jobSeqNo", "jobId", "reqId"): for key in ("jobSeqNo", "jobId", "reqId"):
jobs[1].pop(key, None) jobs[1].pop(key, None)
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:] rows = phenom.parse(json.dumps(data).encode(), SLUG)
assert len(rows) == len(IDS)
assert [r["source_id"] for r in rows if _runner.valid(r)] == IDS[2:]
# ---------------------------------------------------------- the request # ---------------------------------------------------------- the request
+4 -3
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import pinpoint # noqa: E402 from plugins.readers import _runner, pinpoint # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -86,11 +86,12 @@ def test_an_unknown_pay_frequency_keeps_the_text_and_leaves_the_columns() -> Non
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "") assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_left_out() -> None: def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_dropped_by_the_runner() -> None:
body = json.loads(_body()) body = json.loads(_body())
body["data"][0]["title"] = "" body["data"][0]["title"] = ""
body["data"][1]["url"] = "javascript:alert(1)" body["data"][1]["url"] = "javascript:alert(1)"
assert pinpoint.parse(json.dumps(body).encode(), "xeneta") == [] rows = pinpoint.parse(json.dumps(body).encode(), "xeneta")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None: def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
+6 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import rippling # noqa: E402 from plugins.readers import _runner, rippling # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "rippling.json").read_bytes() BOARD = (FIXTURES / "rippling.json").read_bytes()
@@ -61,11 +61,14 @@ def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme") rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
def test_an_entry_without_a_title_or_a_plain_id_is_skipped() -> None: def test_an_entry_without_a_title_or_a_plain_id_is_dropped_by_the_runner() -> None:
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"}, body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
{"uuid": "../../x", "name": "C"}, {"uuid": "../../x", "name": "C"},
{"uuid": "abcdefgh-2", "name": "D"}]).encode() {"uuid": "abcdefgh-2", "name": "D"}]).encode()
assert [r["source_id"] for r in rippling.parse(body, "acme")] == ["abcdefgh-2"] rows = rippling.parse(body, "acme")
assert [r["source_id"] for r in rows] == ["abcdefgh-1", "", "../../x", "abcdefgh-2"]
assert rows[2]["url"] == "" # the id builds no link
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["abcdefgh-2"]
# ---------------------------------------------------------------- detail # ---------------------------------------------------------------- detail
+3 -2
View File
@@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path: if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins.readers import Ask, teamtailor # noqa: E402 from plugins.readers import Ask, _runner, teamtailor # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -40,7 +40,8 @@ def test_page_takes_the_ask_and_reads_the_same_url() -> None:
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None: def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
first, second, third = _rows() first, second, third, refused = _rows()
assert not _runner.valid(refused) # javascript:, for the runner
# "Sverige" (a city that is a country) once, Stockholm's twin once. # "Sverige" (a city that is a country) once, Stockholm's twin once.
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
{"text": "Stockholm, Sweden", "remote": False}] {"text": "Stockholm, Sweden", "remote": False}]
+12 -3
View File
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402 from app.jobs import discover # noqa: E402
from plugins import ats # noqa: E402 from plugins import ats # noqa: E402
from plugins.readers import workday # noqa: E402 from plugins.readers import _runner, workday # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "zendesk.wd1/zendesk" SLUG = "zendesk.wd1/zendesk"
@@ -96,7 +96,7 @@ def test_posted_is_read_only_where_the_text_is_exact(text: str, expected: dateti
assert workday._posted(text) == expected assert workday._posted(text) == expected
def test_a_posting_without_a_title_or_a_path_is_skipped() -> None: def test_a_posting_without_a_title_or_a_path_reaches_the_runner_and_is_dropped() -> None:
page = {"jobPostings": [ page = {"jobPostings": [
{"bulletFields": ["0000036369"]}, # LEGO's, no title {"bulletFields": ["0000036369"]}, # LEGO's, no title
{"title": "No path", "bulletFields": ["R1"]}, {"title": "No path", "bulletFields": ["R1"]},
@@ -105,7 +105,16 @@ def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X", {"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
"postedOn": "Posted Today", "bulletFields": ["R4"]}, "postedOn": "Posted Today", "bulletFields": ["R4"]},
]} ]}
assert [r["title"] for r in workday.rows(page, SLUG)] == ["Whole"] rows = workday.rows(page, SLUG)
assert len(rows) == 5
assert [(r["source_id"], r["url"]) for r in rows[:2]] == [("", ""), ("", "")]
assert [r["title"] for r in rows if _runner.valid(r)] == ["Whole"]
# Through the runner: four dropped and counted, and the read complete,
# since every posting counts by its position towards the stated total.
body = json.dumps(page | {"total": 5}).encode()
listing = _runner.read_listing(workday, SLUG, lambda url, **kw: body)
assert [r["title"] for r in listing.rows] == ["Whole"]
assert (listing.handed, listing.dropped, listing.complete) == (5, 4, True)
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None: def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
+21
View File
@@ -116,6 +116,16 @@ def test_roles_without_text_and_text_reads_that_failed() -> None:
assert words(fake(), state(), counts=limits.Texts(9, 0)) == [] assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
@pytest.mark.parametrize(("dropped", "found"), [
("3", ["3 listings dropped: no title or link"]),
("1", ["1 listing dropped: no title or link"]),
("", []), ("x", [])])
def test_the_listings_the_last_read_dropped_are_named(dropped: str, found: list[str]) -> None:
"""The rows the runner dropped (`_runner.valid`), after "read incomplete"."""
assert words(fake(), state(dropped=dropped)) == found
assert words(fake(), state(dropped=dropped, complete=""))[1:] == found
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None: def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"] assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
assert words(fake(), state(), reach.SMALL) == [] assert words(fake(), state(), reach.SMALL) == []
@@ -199,6 +209,17 @@ def test_companies_carry_their_limits_and_the_boards_own_open_roles(
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"] assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
def test_a_board_that_dropped_listings_says_so_on_its_row(
client: TestClient, store: Store) -> None:
store.add_subscription("ats", "lever:acme", company="Acme", board="lever")
store.add_company("Acme", key="acme", ats="lever")
store.state["ats:ok:lever:acme"] = "run"
store.state["ats:complete:lever:acme"] = "1"
store.state["ats:dropped:lever:acme"] = "3"
[row] = client.get("/api/companies").json()
assert row["limits"][0] == "3 listings dropped: no title or link"
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None: def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
store.add_company("Novo", key="novo", ats="taleo") store.add_company("Novo", key="novo", ats="taleo")
[row] = client.get("/api/companies").json() [row] = client.get("/api/companies").json()
+12 -2
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402 from plugins import readers # noqa: E402
from plugins.readers import Ask, adzuna # noqa: E402 from plugins.readers import Ask, _runner, adzuna # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes() BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin", ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
@@ -39,8 +39,9 @@ def test_it_declares_the_key_and_states_the_robots_finding() -> None:
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None: def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
first, predicted = _rows() # javascript: refused first, predicted, refused = _rows()
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"] assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
assert refused["url"].startswith("javascript:") and not _runner.valid(refused)
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote" assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
assert first["teaser"] == "Build our backend services in Python & Go." assert first["teaser"] == "Build our backend services in Python & Go."
assert "description" not in first assert "description" not in first
@@ -89,3 +90,12 @@ def test_saved_params_need_a_known_country() -> None:
"q": "x", "country": "gb"} "q": "x", "country": "gb"}
with pytest.raises(ValueError, match="country"): with pytest.raises(ValueError, match="country"):
adzuna.clean_params({"q": "x", "country": "dk"}) adzuna.clean_params({"q": "x", "country": "dk"})
def test_the_runner_drops_the_ad_without_a_web_link_and_counts_it() -> None:
"""The reader hands the `javascript:` ad over; the runner drops it, and
the read is complete, since all three result ids were read."""
listing = _runner.read_listing(adzuna, "", lambda url, **kw: BODY, ASK,
limits=_runner.Limits(page_delay=0))
assert listing.ids == ["4412345678", "4412345679"]
assert (listing.handed, listing.dropped, listing.complete) == (3, 1, True)
+7 -4
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402 from plugins import readers # noqa: E402
from plugins.readers import Ask, freehire # noqa: E402 from plugins.readers import Ask, _runner, freehire # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes() BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
@@ -119,14 +119,17 @@ def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None:
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR") assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None: def test_a_hit_without_a_title_an_id_or_a_web_link_is_handed_to_the_runner() -> None:
"""The reader hands every hit over; the runner keeps only the whole one."""
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"}, hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
{"public_slug": "", "title": "B", "url": "https://x.example/b"}, {"public_slug": "", "title": "B", "url": "https://x.example/b"},
{"public_slug": "c", "title": " ", "url": "https://x.example/c"}, {"public_slug": "c", "title": " ", "url": "https://x.example/c"},
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}] {"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
found = freehire.rows({"data": hits}, "") found = freehire.rows({"data": hits}, "")
assert [row["source_id"] for row in found] == ["d"] assert [row["source_id"] for row in found] == ["a", "", "c", "d"]
assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == [] kept = [row for row in found if _runner.valid(row)]
assert [row["source_id"] for row in kept] == ["d"]
assert kept[0]["url"] == "https://x.example/d" and kept[0]["places"] == []
def test_a_title_and_a_company_have_their_entities_decoded() -> None: def test_a_title_and_a_company_have_their_entities_decoded() -> None:
+3 -2
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402 from plugins import readers # noqa: E402
from plugins.readers import Ask, jobbank # noqa: E402 from plugins.readers import Ask, _runner, jobbank # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search" FIXTURES = ROOT / "tests" / "fixtures" / "search"
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes() PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
@@ -170,7 +170,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>") body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask()) decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
assert len(results) == 100 and nxt == 2 assert len(results) == 100 and nxt == 2
assert len(jobbank.rows(decoded, "")) == 99 rows = jobbank.rows(decoded, "")
assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>", @pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
+3 -2
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402 from plugins import readers # noqa: E402
from plugins.readers import Ask, jobindex # noqa: E402 from plugins.readers import Ask, _runner, jobindex # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes() BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
@@ -89,7 +89,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>") body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask()) decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
assert len(results) == 20 and nxt == 2 assert len(results) == 20 and nxt == 2
assert len(jobindex.rows(decoded, "")) == 19 rows = jobindex.rows(decoded, "")
assert len(rows) == 20 and sum(map(_runner.valid, rows)) == 19
def test_an_answer_without_a_channel_raises() -> None: def test_an_answer_without_a_channel_raises() -> None:
+6 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402 from plugins import readers # noqa: E402
from plugins.readers import Ask, jobnet # noqa: E402 from plugins.readers import Ask, _runner, jobnet # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search" FIXTURES = ROOT / "tests" / "fixtures" / "search"
PLACES = (FIXTURES / "jobnet-places.json").read_bytes() PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
@@ -100,9 +100,12 @@ def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None:
assert "applicationDeadline" not in row assert "applicationDeadline" not in row
def test_an_ad_without_an_id_or_a_title_is_skipped() -> None: def test_an_ad_without_an_id_or_a_title_is_handed_over_for_the_runner_to_drop() -> None:
ads = json.loads(OTHER)["jobAds"] ads = json.loads(OTHER)["jobAds"]
assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == [] rows = jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "")
assert [(row["source_id"] == "", row["title"] == "") for row in rows] == [
(False, True), (True, False)]
assert not any(map(_runner.valid, rows))
def test_it_sends_the_header_the_region_and_the_page_number() -> None: def test_it_sends_the_header_the_region_and_the_page_number() -> None:
+5 -4
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402 from plugins import readers # noqa: E402
from plugins.readers import Ask, platsbanken # noqa: E402 from plugins.readers import Ask, _runner, platsbanken # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes() BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
@@ -47,8 +47,9 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask()) decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
rows = platsbanken.rows(decoded, "") rows = platsbanken.rows(decoded, "")
assert total == 32 assert total == 32
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400001", "31400002"]
first, no_town, bare = rows assert not _runner.valid(rows[2]) # javascript:, dropped by the runner
first, no_town, _refused, bare = rows
assert [p["text"] for p in first["places"]] == [ assert [p["text"] for p in first["places"]] == [
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län "Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
assert first["location"] == "Lund" assert first["location"] == "Lund"
@@ -58,7 +59,7 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
def test_it_reads_title_company_date_and_the_full_text() -> None: def test_it_reads_title_company_date_and_the_full_text() -> None:
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask()) decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
first, _no_town, remote = platsbanken.rows(decoded, "") first, _no_town, _refused, remote = platsbanken.rows(decoded, "")
assert first["title"] == "Postdoctoral Researcher" assert first["title"] == "Postdoctoral Researcher"
assert first["company"] == "European Spallation Source Eric" assert first["company"] == "European Spallation Source Eric"
assert first["country"] == "SE" and first["remote"] == "Arbete på plats" assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
+107 -2
View File
@@ -1,7 +1,7 @@
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a """The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
subscription resolves to, series and their union, whole and paged listings, subscription resolves to, series and their union, whole and paged listings,
completeness from the declaration, the sweep across pulls, the 10% guard, completeness from the declaration, the rows kept and dropped, the sweep
the facets kept in state, the detail reads and the `raw` an item carries. No across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
network: `get` is a dict of URL -> body or exception.""" network: `get` is a dict of URL -> body or exception."""
from __future__ import annotations from __future__ import annotations
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
assert filtered.ids == ["1"] and filtered.filtered assert filtered.ids == ["1"] and filtered.filtered
# --------------------------------------------------------- the rows kept
def _handed(i: str) -> dict[str, Any]:
"""A row as a reader hands it over: whole, or, for an id of `-id`,
`-title`, `-link` or `-url`, without that value or with a link that is
no web URL (`javascript:`, or one httpx cannot read)."""
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
return _row(i, **broken.get(i.split(":")[0], {}))
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
assert _runner.valid(_handed("1"))
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
assert not _runner.valid({"source_id": "1", "title": "Role"})
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
caplog: pytest.LogCaptureFixture) -> None:
module = _whole()
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
web = Web({"https://jobs.fake.example/acme/list":
b'["1", "-id", "2", "-title", "-link", "-url"]'})
with caplog.at_level("WARNING", logger=_runner.__name__):
listing = _runner.read_listing(module, "acme", web)
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert (listing.handed, listing.dropped) == (6, 4)
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
caplog: pytest.LogCaptureFixture) -> None:
"""Completeness counts the result ids `page` returns, not the rows kept,
so the dropped rows still count as read; the drop is logged once."""
pages = [["1", "-title:a"], ["2", "-link:b"]]
module = _paged(pages, total=4)
module.rows = lambda ids, slug: [_handed(i) for i in ids]
with caplog.at_level("WARNING", logger=_runner.__name__):
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert (listing.handed, listing.dropped) == (4, 2)
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
def test_the_stamp_keeps_the_rows_dropped() -> None:
state: dict[str, str] = {}
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
assert state["ats:dropped:t"] == "2"
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
assert state["ats:dropped:t"] == ""
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
"""A whole board whose feed hands over `handed` rows, `dropped` of them
without a title."""
module = _whole()
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
handed: int, dropped: int, fails: bool) -> None:
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
a page whose shape changed, which must close nothing."""
module, web = _dropping(handed, dropped)
if not fails:
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
return
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
_runner.read({}, "ats", "t", module, "acme", web)
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
module, web = _dropping(20, 5)
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
listing = _runner.read(state, "ats", "t", module, "acme", web,
limits=_runner.Limits(page_delay=0, stale_days=21))
assert (len(listing.ids), listing.dropped) == (15, 5)
# ------------------------------------------------------ through ATSSource # ------------------------------------------------------ through ATSSource
@pytest.fixture @pytest.fixture
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
assert state["ats:filtered:fake:acme"] == "1" assert state["ats:filtered:fake:acme"] == "1"
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
board.rows = lambda ids, slug: [_handed(i) for i in ids]
board.web.answers = {"https://jobs.fake.example/acme/page/0":
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
board.READS = board.READS.model_copy(update={"complete": "end"})
state: dict[str, str] = {}
assert len(list(_source(state).fetch())) == 19
assert state["ats:dropped:fake:acme"] == "1"
board.web.answers = {"https://jobs.fake.example/acme/page/0":
json.dumps([str(n) for n in range(15)]
+ [f"-id:{n}" for n in range(5)]).encode()}
state = {}
assert list(_source(state).fetch()) == []
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
assert "ats:ok:fake:acme" not in state
# -------------------------------------------------------------------- raw # -------------------------------------------------------------------- raw
def test_raw_keeps_every_stated_key_and_the_places() -> None: def test_raw_keeps_every_stated_key_and_the_places() -> None:
+17
View File
@@ -383,6 +383,23 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch)
assert source.state["search:ids:fake:cph"] == '["fake:h1"]' assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count(
monkeypatch: pytest.MonkeyPatch) -> None:
"""The search driver reads through the same listing code as a board: the
runner drops the row the provider handed over without a title, and the
read is still complete, by the distinct result ids."""
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
module = readers.PROVIDERS["fake"]
handed = module.rows
module.rows = lambda decoded, slug: [
row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)]
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
"params": {"q": "x"}}]})
assert [i.source_id for i in source.fetch()] == ["fake:h1"]
assert source.state["search:complete:fake:cph"] == "1"
assert source.state["search:dropped:fake:cph"] == "1"
def test_a_search_that_fails_is_skipped_and_stamps_nothing( def test_a_search_that_fails_is_skipped_and_stamps_nothing(
monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False) monkeypatch.delenv("ADZUNA_APP_ID", raising=False)