diff --git a/app/jobs/limits.py b/app/jobs/limits.py index 09fa01c..45da689 100644 --- a/app/jobs/limits.py +++ b/app/jobs/limits.py @@ -10,7 +10,10 @@ both; nothing here decides how a board is read, so the words never disagree with the pull as long as they read the keys the pull writes. A word with a number carries it after a colon ("without text: 12"), and the -filter matches the phrase before the colon (`WORDS`). +filter matches the phrase before the colon (`WORDS`). One word is outside +`WORDS`: the listings the last read dropped for want of an id, a title or a +link ("3 listings dropped: no title or link", `_runner.valid`), shown on the +board's row and not offered by the filter. """ from __future__ import annotations @@ -49,7 +52,7 @@ WORDS: dict[str, str] = { #: The `state` keys `words` reads, `::`. KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked", - "narrowed", "detailfailed") + "narrowed", "detailfailed", "dropped") class Texts(NamedTuple): @@ -84,7 +87,8 @@ def texts(store: Store) -> dict[int, Texts]: def words(module: ModuleType, state: Mapping[str, str], target: str, sub: Subscription | None, size: int | None, *, plan: Plan | None = None, line: str = "", counts: Texts | None = None) -> list[str]: - """The limits of one board, in the order of `WORDS`, those that apply. + """The limits of one board, in the order of `WORDS`, those that apply, the + rows the last read dropped after "read incomplete". `state` holds the board's keys (`state_of`), `size` its kept size, `plan` and `sub` the reach it is read with (for "read whole"), `line` the @@ -105,6 +109,8 @@ def words(module: ModuleType, state: Mapping[str, str], target: str, out.append("failing") if state.get(key("ok")) and not state.get(key("complete")): out.append("read incomplete") + if dropped := _count(state.get(key("dropped"))): + out.append(f"{dropped} listing{'' if dropped == 1 else 's'} dropped: no title or link") if state.get(key("swept")) == "1" or key("sweep") in state: out.append("swept") if state.get(key("filtered")) == "1": @@ -130,6 +136,14 @@ def words(module: ModuleType, state: Mapping[str, str], target: str, return out +def _count(value: str | None) -> int: + """A number kept in state, 0 for none or one that does not parse.""" + try: + return max(0, int(value or 0)) + except ValueError: + return 0 + + def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool: """Whether the reach leaves the board unread by place: its reader cannot filter, or the board is under `SMALL` roles (the line's two "read whole" diff --git a/plugins/ats.py b/plugins/ats.py index 2a445ed..54c5735 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -17,8 +17,9 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what its slug is. Every board is read independently; one that fails is logged, recorded as failing under `ats:failing:`, and skipped, and so is one -whose read lists under a tenth of what it listed last time (`_runner.read`'s -guard), until it has failed that way for the pull's `stale_days`. A board's +whose read lists under a tenth of what it listed last time, or drops more +than a tenth of the rows its reader handed over (`_runner.read`'s guards), +until it has failed that way for the pull's `stale_days`. A board's filter is the `reach` option the pull hands it (the agents', see `app/jobs/reach.py`), over which its subscription `params` (`{"countries": ["DK"], "remote": true}`) win, narrowed to what its reader declares @@ -343,8 +344,10 @@ SIZE_PAGES = 100 def board_size(target: str, *, client: httpx.Client | None = None) -> int: """How many roles board `target` lists, unfiltered and without terms: a whole host's listing, the total a `complete="stated"` host gives on page - 1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises - as `_fetch_rows` does, so a board that does not answer is refused.""" + 1, else the listing read to the end, at most `SIZE_PAGES` pages. A read + counts the rows the runner dropped as well (`_listed`), as a stated total + does. Raises as `_fetch_rows` does, so a board that does not answer is + refused.""" if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") if client is None: @@ -358,7 +361,12 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int: if isinstance(total, int): return total limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY) - return len(_runner.read_listing(module, slug, get, limits=limits).ids) + return _listed(_runner.read_listing(module, slug, get, limits=limits)) + + +def _listed(listing: _runner.Listing) -> int: + """The listings a read found: the ids it kept and the rows it dropped.""" + return len(listing.ids) + listing.dropped def keep_size(state: Any, target: str, size: int, *, today: str = "", @@ -462,7 +470,7 @@ class ATSSource(Source): [compound(i) for i in listing.ids]) self.state.pop(failing, None) if unsized and listing.complete and not (listing.filtered or listing.narrowed): - keep_size(self.state, target, len(listing.ids), reset=False) + keep_size(self.state, target, _listed(listing), reset=False) if detail_sets is not None: # A complete listing drops the ids it no longer lists. for key, ids in zip(detail_keys(target), detail_sets, strict=False): diff --git a/plugins/readers/__init__.py b/plugins/readers/__init__.py index 0afb700..768ed2e 100644 --- a/plugins/readers/__init__.py +++ b/plugins/readers/__init__.py @@ -179,7 +179,7 @@ ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most, **Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's own id for the role, unique on the board), `title`, `url` (http or https, -checked with `_shared.is_web_url`) and `places`; `location` (the host's +which the runner checks with `_shared.is_web_url`) and `places`; `location` (the host's one-line place, kept while storage reads it), `posted` (an aware datetime or None, see `_shared.parse_dt`), `department`, `description` (plain text, see `_shared.html_text`), `remote` (True, False, a word or None) and @@ -187,7 +187,12 @@ see `_shared.html_text`), `remote` (True, False, a word or None) and (yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`; where it states them, `country` (ISO 3166 alpha-2), `seniority` and `employment`; `teaser`; and, on a search row, `company`. The runner keeps -every stated key in `raw`, so a new field needs no change there. +every stated key in `raw`, so a new field needs no change there. A reader +never drops a listing: one it cannot make a whole row of (no id, no title or +no link) it hands over with what it has, an empty string for what is +missing, and the runner drops it and counts it (`_runner.valid`). A reader +leaves out only what is not a listing of the board: an entry that is not an +object, a second copy of an id, a role of another board. **Places.** A role has a list of places: `places` is `[Place(text=..., remote=...)]`, one entry per place the host states, in the host's order, in diff --git a/plugins/readers/_contract.py b/plugins/readers/_contract.py index 7fd9706..b3ec5fb 100644 --- a/plugins/readers/_contract.py +++ b/plugins/readers/_contract.py @@ -205,6 +205,8 @@ class Row(TypedDict, total=False): `company` is the employer's name on a search row, which lists many (`plugins/search.py` stores it); a board row carries none, the board is the company. Any other stated key is kept in `raw` by the runner. + `source_id`, `title` and `url` come empty from a listing that lacks + them: the runner drops such a row and counts it (`_runner.valid`). """ source_id: Required[str] diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index 7f9c698..d72dda4 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -4,10 +4,12 @@ the reader declares in `READS`. A reader says how its host answers; this module reads it: the guarded fetch (`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves to (`ask_for`) and its series (`series`), the listing whole or paged, one -series per `one`-filter value unioned by id (`read_listing`), completeness, -the sweep of an oversized paged board across pulls, the 10% guard on every -read that is not a sweep (`read`), the per-role detail reads -(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure +series per `one`-filter value unioned by id (`read_listing`), the rows it +keeps (`valid`: a row without an id, a title or a link is dropped and +counted here, never by the reader), completeness, the sweep of an oversized +paged board across pulls, the two 10% guards (`read`: on what a read that +is not a sweep listed, and on what any read dropped), the per-role detail +reads (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and `plugins/search.py` are thin `Source` classes over it: they pass their own limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay @@ -30,6 +32,8 @@ State keys, under the Source's prefix and the target (`ats:sweep:`): drifts when roles close between pulls, so `app.jobs.pull` closes a role of such a board only when two finished sweeps in a row missed it (it keeps the ids the last one missed in `missing`) + dropped the rows the last stamped read dropped (`Listing.dropped`), as a + number, "" for none; `app.jobs.limits` names it on the board's row """ from __future__ import annotations @@ -52,7 +56,7 @@ import httpx from desk.sources import USER_AGENT from ._contract import Ask, Key, Place, Reads -from ._shared import Get, Row +from ._shared import Get, Row, is_web_url log = logging.getLogger(__name__) @@ -96,7 +100,8 @@ SUSPECT_SHARE = 0.1 #: The last count from which the guard holds every read. A read that is #: filtered, narrowed by terms or of an `empty="suspect"` reader is held from #: a count of 1, as it was before every read was; a whole read of a board -#: that listed a handful of roles can drop to none by itself. +#: that listed a handful of roles can drop to none by itself. It is also the +#: rows a read must hand over before the share it dropped can fail it. SUSPECT_MIN = 10 #: The keys of a row that an `Item` carries in its own fields, not in `raw`. @@ -113,8 +118,8 @@ class TooLarge(ValueError): class Suspect(Exception): - """A read that listed under `SUSPECT_SHARE` of what it listed last time - (see `read`).""" + """A read that listed under `SUSPECT_SHARE` of what it listed last time, + or dropped more than that share of its rows (see `read`).""" class MissingKey(ValueError): @@ -147,7 +152,9 @@ class Listing: and roles closing meanwhile may have shifted an offset-paged host's later roles back past the cursor, so `ids` can miss a role that is still open. `remote_unasked` says a filtered read was meant to ask for remote roles - as well and could not, so its `ids` say nothing of the remote roles.""" + as well and could not, so its `ids` say nothing of the remote roles. + `handed` counts the rows the reader handed over this pull, `dropped` + those of them `valid` refused, which are in neither `rows` nor `ids`.""" rows: list[Row] complete: bool @@ -158,6 +165,8 @@ class Listing: narrowed: bool = False swept: bool = False remote_unasked: bool = False + handed: int = 0 + dropped: int = 0 # --------------------------------------------------------------- modules @@ -524,8 +533,42 @@ def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]: return {str(r.get("id") or r.get("shortcode") or "") for r in results} +def valid(row: Mapping[str, Any]) -> bool: + """Whether the runner keeps a row: a non-empty `source_id`, a non-empty + `title` and a web URL (`is_web_url`). A reader hands over every listing + as far as it can make it out; a row without one of the three cannot be + stored or opened, and the runner drops it and counts it.""" + try: + link = is_web_url(str(row.get("url") or "")) + except (httpx.InvalidURL, ValueError): # a string httpx cannot read as a URL + link = False + return (link and bool(str(row.get("source_id") or "").strip()) + and bool(str(row.get("title") or "").strip())) + + +@dataclass(slots=True) +class _Tally: + """The rows a read was handed and dropped, across its series and pages.""" + + handed: int = 0 + dropped: int = 0 + + +def _keep(found: Iterable[Row], rows: dict[str, Row], tally: _Tally) -> None: + """The rows a reader's `parse` or `rows` returned, into `rows` by + `source_id`, the first of an id kept; those `valid` refuses are counted + in `tally` and left out. The one place a row is checked, for the whole + and the paged listing alike.""" + for row in found: + tally.handed += 1 + if not valid(row): + tally.dropped += 1 + continue + rows.setdefault(row["source_id"], row) + + def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask], - limits: Limits) -> dict[str, Row]: + limits: Limits, tally: _Tally) -> dict[str, Row]: """A whole listing, one request a series, unioned by `source_id`. A body over the cap gets one retry at the reader's `fallback_url`.""" rows: dict[str, Row] = {} @@ -541,8 +584,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask], log.warning("readers: %s:%s: body over the cap; reading the smaller feed", module.NAME, slug) body = get(module.fallback_url(slug)) - for row in module.parse(body, slug): - rows.setdefault(row["source_id"], row) + _keep(module.parse(body, slug), rows, tally) return rows @@ -580,7 +622,29 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None only: a search's places are what the search is, as its query is. `remote` says the reach or the params wanted remote roles; when no series asked for them (a reader without a remote filter, or one whose facets - state none beside places) the listing is `remote_unasked`.""" + state none beside places) the listing is `remote_unasked`. + + Every row `rows` or `parse` returns goes through `valid`: one without a + `source_id`, a title or a web URL is dropped here, counted in + `Listing.dropped` and logged once for the read. Completeness counts the + distinct result ids `page` returns beside its rows, not the rows kept, so + a dropped row still counts as read; a whole listing is complete whenever + it parses, whatever it dropped.""" + tally = _Tally() + listing = _listing(module, slug, get, ask, limits=limits, sweep=sweep, + can_sweep=can_sweep, facets=facets, remote=remote, tally=tally) + listing.handed, listing.dropped = tally.handed, tally.dropped + if tally.dropped: + log.warning("readers: %s:%s: dropped %d of %d listings: no id, title or link", + module.NAME, slug, tally.dropped, tally.handed) + return listing + + +def _listing(module: ModuleType, slug: str, get: Get, ask: Ask | None, *, + limits: Limits | None, sweep: Mapping[str, Any] | None, can_sweep: bool, + facets: Mapping[str, Mapping[str, str]] | None, remote: bool, + tally: _Tally) -> Listing: + """`read_listing` before its count of dropped rows, which `tally` keeps.""" limits = limits or Limits() ask = ask or Ask() reads: Reads = module.READS @@ -595,7 +659,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None "nothing asked, not closing", module.NAME, slug) return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked) if not hasattr(module, "page"): - whole_rows = _whole(module, slug, get, asks, limits) + whole_rows = _whole(module, slug, get, asks, limits, tally) return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None, key, by_words, remote_unasked=unasked) @@ -632,8 +696,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None pages += 1 before = len(seen) seen |= _result_ids(results) - for row in module.rows(data, slug): - rows.setdefault(row["source_id"], row) + _keep(module.rows(data, slug), rows, tally) if cursor is None or (isinstance(total, int) and len(seen) >= total): ended = True break @@ -747,13 +810,26 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu that really shrank is accepted once the target has been failing for `limits.stale_days` (its `:failing:` record), and its stamp then resets the count; a Recheck resets it at once - (`plugins.ats.keep_size`).""" + (`plugins.ats.keep_size`). + + A read, a sweep's included, that dropped more than `SUSPECT_SHARE` of + the rows its reader handed over (`valid`), once it was handed at least + `SUSPECT_MIN`, raises `Suspect` the same way: a page whose shape changed + parses to rows without a title or a link, and the roles it hid would + close. It gives way after `limits.stale_days` of failing as well.""" limits = limits or Limits() ask = ask or Ask() facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}") listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep, sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets, remote=remote) + if listing.handed >= SUSPECT_MIN and listing.dropped > SUSPECT_SHARE * listing.handed: + why = (f"dropped {listing.dropped} of {listing.handed} listings, over " + f"{SUSPECT_SHARE:.0%}: no id, title or link") + if not _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days): + raise Suspect(why) + log.warning("readers: %s:%s: %s, accepted after %d days of failing", + module.NAME, slug, why, limits.stale_days) held = listing.filtered or listing.narrowed or module.READS.empty == "suspect" last = _load(state, f"{prefix}:count:{target}") count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0 @@ -788,9 +864,10 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str, `complete`, `ids` (the compound ids, what the pull closes against), `filtered`, `narrowed` (a read by terms, which the pull never closes against), whether it finished a sweep from earlier pulls (`swept`), - whether a filtered read left remote unasked (`remote_unasked`), the - sweep to continue or none, and the count for the 10% guard once a read - is not a sweep. A read that asked nothing keeps the count it had.""" + whether a filtered read left remote unasked (`remote_unasked`), the rows + it dropped (`dropped`, "" for none), the sweep to continue or none, and + the count for the 10% guard once a read is not a sweep. A read that + asked nothing keeps the count it had.""" state[f"{prefix}:ok:{target}"] = run state[f"{prefix}:complete:{target}"] = "1" if listing.complete else "" state[f"{prefix}:swept:{target}"] = "1" if listing.swept else "" @@ -798,6 +875,7 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str, state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else "" state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else "" state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else "" + state[f"{prefix}:dropped:{target}"] = str(listing.dropped) if listing.dropped else "" if listing.sweep is not None: state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep) return diff --git a/plugins/readers/adzuna.py b/plugins/readers/adzuna.py index 19780a5..92c29f7 100644 --- a/plugins/readers/adzuna.py +++ b/plugins/readers/adzuna.py @@ -24,7 +24,7 @@ from urllib.parse import urlencode from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Key, Place, Reads -from ._shared import Get, Page, Row, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, only, parse_dt NAME = "adzuna" LABEL = "Adzuna" @@ -111,8 +111,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: for job in decoded["results"]: title = _text(job.get("title") or "") url = (job.get("redirect_url") or "").strip() - if not title or not is_web_url(url): - continue predicted = str(job.get("salary_is_predicted") or "0") == "1" town = ((job.get("location") or {}).get("display_name") or "").strip() places: list[Place] = [{"text": town, "remote": None}] if town else [] diff --git a/plugins/readers/ashby.py b/plugins/readers/ashby.py index 9bfa138..b111a77 100644 --- a/plugins/readers/ashby.py +++ b/plugins/readers/ashby.py @@ -14,7 +14,7 @@ from typing import Any from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount +from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount log = logging.getLogger(__name__) @@ -126,8 +126,6 @@ def parse(body: bytes, slug: str) -> list[Row]: for j in data.get("jobs") or []: title = (j.get("title") or "").strip() url = (j.get("jobUrl") or j.get("applyUrl") or "").strip() - if not title or not is_web_url(url): - continue description = j.get("descriptionHtml") or j.get("descriptionPlain") or "" compensation = j.get("compensation") or {} salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or []) diff --git a/plugins/readers/bamboohr.py b/plugins/readers/bamboohr.py index 838beaf..7a789de 100644 --- a/plugins/readers/bamboohr.py +++ b/plugins/readers/bamboohr.py @@ -106,8 +106,6 @@ def parse(body: bytes, slug: str) -> list[Row]: for j in listed: title = (j.get("jobOpeningName") or "").strip() source_id = str(j.get("id") or "") - if not title or not source_id: - continue places = _places(j) row: Row = { "source_id": source_id, diff --git a/plugins/readers/deel.py b/plugins/readers/deel.py index a312a87..61ba611 100644 --- a/plugins/readers/deel.py +++ b/plugins/readers/deel.py @@ -132,8 +132,6 @@ def parse(body: bytes, slug: str) -> list[Row]: for p in postings: title = (p.get("title") or "").strip() post_id = str(p.get("id") or "") - if not title or not post_id: - continue job = p.get("job") or {} places = [name for loc in job.get("jobLocations") or [] if (name := ((loc.get("location") or {}).get("name") or "").strip())] diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py index 3db4577..a16bef7 100644 --- a/plugins/readers/eightfold.py +++ b/plugins/readers/eightfold.py @@ -47,7 +47,7 @@ from urllib.parse import quote from ._contract import Ask, Fields, Filters, Place, Reads, Row from ._countries import name_of -from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place +from ._shared import Get, Page, add_place, html_text, only, parse_dt, place READS = Reads( scope="board", listing="paged", page_size=10, text="detail", complete="stated", @@ -151,8 +151,6 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]: source_id = str(p.get("id") or "") path = str(p.get("positionUrl") or "").strip() url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else "" - if not title or not source_id or not is_web_url(url): - continue places = _places(p.get("locations") or p.get("standardizedLocations")) out.append({ "source_id": source_id, diff --git a/plugins/readers/freehire.py b/plugins/readers/freehire.py index 7eaf616..7d5455e 100644 --- a/plugins/readers/freehire.py +++ b/plugins/readers/freehire.py @@ -37,7 +37,7 @@ from urllib.parse import urlencode, urlsplit, urlunsplit from desk.sources import USER_AGENT from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line, +from ._shared import (Get, Page, Row, html_text, num, only, parse_dt, pay_line, yearly_amount) NAME = "freehire" @@ -170,8 +170,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: title = " ".join(html.unescape(str(hit.get("title") or "")).split()) source_id = str(hit.get("public_slug") or "").strip() url = _url(str(hit.get("url") or "")) - if not title or not source_id or not is_web_url(url): - continue mode = str(hit.get("work_mode") or "").strip() places = _places(hit, mode) countries = [str(c) for c in hit.get("countries") or [] if c] diff --git a/plugins/readers/gem.py b/plugins/readers/gem.py index 696b5a0..c68415d 100644 --- a/plugins/readers/gem.py +++ b/plugins/readers/gem.py @@ -116,8 +116,6 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]: for p in postings: title = (p.get("title") or "").strip() ext_id = str(p.get("extId") or "") - if not title or not ext_id: - continue locations = p.get("locations") or [] places: list[Place] = [] for loc in locations: diff --git a/plugins/readers/greenhouse.py b/plugins/readers/greenhouse.py index e851b04..c5fc22c 100644 --- a/plugins/readers/greenhouse.py +++ b/plugins/readers/greenhouse.py @@ -13,7 +13,7 @@ from typing import Any from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place +from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place NAME = "greenhouse" LABEL = "Greenhouse" @@ -149,8 +149,6 @@ def parse(body: bytes, slug: str) -> list[Row]: for j in data.get("jobs") or []: title = (j.get("title") or "").strip() url = (j.get("absolute_url") or "").strip() - if not title or not is_web_url(url): - continue departments = j.get("departments") or [] ranges = j.get("pay_input_ranges") or [] salary_min, salary_max, currency = _salary(ranges) diff --git a/plugins/readers/hibob.py b/plugins/readers/hibob.py index d87a4ab..b447138 100644 --- a/plugins/readers/hibob.py +++ b/plugins/readers/hibob.py @@ -90,8 +90,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: for ad in data.get("jobAdDetails") or []: title = (ad.get("title") or "").strip() ad_id = str(ad.get("id") or "").strip() - if not title or not ad_id: - continue site = str(ad.get("site") or "").strip() country = str(ad.get("country") or "").strip() location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \ diff --git a/plugins/readers/hrmanager.py b/plugins/readers/hrmanager.py index d1a50e0..bbbf604 100644 --- a/plugins/readers/hrmanager.py +++ b/plugins/readers/hrmanager.py @@ -30,7 +30,7 @@ from typing import Any from ._contract import Ask, Fields, Reads, Row -from ._shared import Get, Page, html_text, is_web_url, only, parse_dt +from ._shared import Get, Page, html_text, only, parse_dt READS = Reads( scope="board", listing="paged", page_size=100, text="listing", complete="end", @@ -106,8 +106,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]: for item in items: title = (item.findtext("title") or "").strip() url = (item.findtext("link") or "").strip() - if not title or not is_web_url(url): - continue out.append({ "source_id": (item.findtext("guid") or url).strip(), "title": title, diff --git a/plugins/readers/jibe.py b/plugins/readers/jibe.py index 03f1a1e..189a448 100644 --- a/plugins/readers/jibe.py +++ b/plugins/readers/jibe.py @@ -127,8 +127,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: j = job.get("data") or {} title = (j.get("title") or "").strip() role = str(j.get("slug") or j.get("req_id") or "") - if not title or not role: - continue categories = j.get("categories") or [] text = "\n".join(p for p in (html_text(j.get("description")), html_text(j.get("qualifications"))) if p) diff --git a/plugins/readers/jobbank.py b/plugins/readers/jobbank.py index b889b2d..de78862 100644 --- a/plugins/readers/jobbank.py +++ b/plugins/readers/jobbank.py @@ -36,7 +36,7 @@ from typing import Any from urllib.parse import urlencode, urlsplit from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt, place +from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt, place NAME = "jobbank" LABEL = "Akademikernes Jobbank (Denmark)" @@ -132,8 +132,8 @@ def _query(ask: Ask, number: int) -> tuple[str, str]: def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """One page of a search; the cursor is the page number. The next page - follows a full page (counted before any item is dropped, so a dropped item - does not pass for the end) while the ceiling is not reached.""" + follows a full page (counted before the runner drops an item, so a + dropped item does not pass for the end) while the ceiling is not reached.""" number = int(cursor or 1) url, remote = _query(ask, number) try: @@ -184,8 +184,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: out: list[Row] = [] for item in decoded["items"]: link, title = item["link"], item["title"] - if not title or not item["id"] or not is_web_url(link): - continue line = _LINE.fullmatch(item["description"]) kind = line.group("kind").strip() if line else "" company, regions, cut = _regions(line.group("rest")) if line else ("", [], False) diff --git a/plugins/readers/jobindex.py b/plugins/readers/jobindex.py index af4f881..ed9743f 100644 --- a/plugins/readers/jobindex.py +++ b/plugins/readers/jobindex.py @@ -117,9 +117,9 @@ def _query(ask: Ask, number: int) -> str: def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """One page of a search, newest first; the cursor is the page number. - The next page follows a full page (counted before any item is dropped, so - a dropped item does not pass for the end) while the 200-result ceiling is - not reached.""" + The next page follows a full page (counted before the runner drops an + item, so a dropped item does not pass for the end) while the 200-result + ceiling is not reached.""" number = int(cursor or 1) root = ET.fromstring(get(_query(ask, number))) channel = root.find("channel") @@ -143,8 +143,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: out: list[Row] = [] for item in decoded["items"]: link, full_title = item["link"], item["title"] - if not full_title or not is_web_url(link): - continue description = item["description"] h4 = _H4.search(description) title = _text(h4.group(1)) if h4 else full_title diff --git a/plugins/readers/jobnet.py b/plugins/readers/jobnet.py index 1d87cd1..3a9f9b2 100644 --- a/plugins/readers/jobnet.py +++ b/plugins/readers/jobnet.py @@ -138,8 +138,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: for ad in decoded.get("jobAds") or []: ident = str(ad.get("jobAdId") or "").strip() title = _text(ad.get("title")) - if not ident or not title: - continue link = str(ad.get("jobAdUrl") or "").strip() town = _text(ad.get("municipality") or ad.get("postalDistrictName")) places: list[Place] = [place(town)] if town else [] diff --git a/plugins/readers/jobylon.py b/plugins/readers/jobylon.py index 477461c..2a13b0e 100644 --- a/plugins/readers/jobylon.py +++ b/plugins/readers/jobylon.py @@ -176,8 +176,6 @@ def parse(body: bytes, slug: str) -> list[Row]: title = (j.get("title") or "").strip() job_id = str(j.get("id") or "").strip() path = str(j.get("url") or "").strip() - if not title or not job_id or not _ROLE_PATH.fullmatch(path): - continue function = str(j.get("function") or "").strip() places = [place(name) for name in dict.fromkeys( str(name).strip() for name in j.get("locations") or [] if str(name).strip())] @@ -186,7 +184,7 @@ def parse(body: bytes, slug: str) -> list[Row]: "title": title, "places": places, "location": "; ".join(pl["text"] for pl in places), - "url": f"https://{_EMP}{path}", + "url": f"https://{_EMP}{path}" if _ROLE_PATH.fullmatch(path) else "", "posted": _published(j.get("published_date")), "department": "" if function == "All functions" else function, "description": "", diff --git a/plugins/readers/lever.py b/plugins/readers/lever.py index b161296..54f1c08 100644 --- a/plugins/readers/lever.py +++ b/plugins/readers/lever.py @@ -22,7 +22,7 @@ from typing import Any from ._contract import Ask, Fields, Place, Reads, Row -from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount +from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount log = logging.getLogger(__name__) @@ -123,8 +123,6 @@ def parse(body: bytes, slug: str) -> list[Row]: for j in data: title = (j.get("text") or "").strip() url = (j.get("hostedUrl") or "").strip() - if not title or not is_web_url(url): - continue categories = j.get("categories") or {} description = j.get("description") or j.get("descriptionPlain") or "" salary_range = j.get("salaryRange") or {} diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py index 43d4f5d..651d64e 100644 --- a/plugins/readers/oracle.py +++ b/plugins/readers/oracle.py @@ -198,9 +198,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: for role in search.get("requisitionList") or []: role_id = str(role.get("Id") or "").strip() title = _text(role.get("Title")) - # The id goes into a URL, and a listing is external content. - if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id): - continue + # The id goes into a URL, and a listing is external content: an id + # of other characters gets no link, and the runner drops the row. + safe = bool(re.fullmatch(r"[A-Za-z0-9_-]+", role_id)) remote = _text(role.get("WorkplaceTypeCode")) or None at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site") places: list[Place] = [] @@ -214,7 +214,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: "title": title, "places": places, "location": "; ".join(p["text"] for p in places), - "url": f"{_base(slug)}/job/{role_id}", + "url": f"{_base(slug)}/job/{role_id}" if safe else "", "posted": parse_dt(role.get("PostedDate")), "department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction")) or _text(role.get("Department")) or _text(role.get("Organization"))), diff --git a/plugins/readers/personio.py b/plugins/readers/personio.py index 070e4fd..b576da6 100644 --- a/plugins/readers/personio.py +++ b/plugins/readers/personio.py @@ -104,8 +104,6 @@ def parse(body: bytes, slug: str) -> list[Row]: for position in root.findall("position"): job_id = (position.findtext("id") or "").strip() title = (position.findtext("name") or "").strip() - if not job_id or not title: - continue description = " ".join( (value.text or "") for value in position.findall("jobDescriptions/jobDescription/value")) salary_info = position.find("salaryInformation") diff --git a/plugins/readers/phenom.py b/plugins/readers/phenom.py index e49a101..419db68 100644 --- a/plugins/readers/phenom.py +++ b/plugins/readers/phenom.py @@ -168,8 +168,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]: for job in search.get("data", {}).get("jobs") or []: key = _key(job) title = _text(job.get("title")) - if not title or not key: - continue places: list[Place] = [] for text in (job.get("location"), *(job.get("multi_location") or [])): if isinstance(text, str): diff --git a/plugins/readers/pinpoint.py b/plugins/readers/pinpoint.py index cd4554c..9e81bf9 100644 --- a/plugins/readers/pinpoint.py +++ b/plugins/readers/pinpoint.py @@ -22,7 +22,7 @@ from typing import Any from ._contract import Ask, Fields, Reads -from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount +from ._shared import HEADING, Row, html_text, label_slug_ok, num, only, pay_line, yearly_amount NAME = "pinpoint" LABEL = "Pinpoint" @@ -121,8 +121,6 @@ def parse(body: bytes, slug: str) -> list[Row]: title = (j.get("title") or "").strip() url = (j.get("url") or "").strip() source_id = str(j.get("id") or "") - if not title or not source_id or not is_web_url(url): - continue job = j.get("job") or {} low, high, currency, salary_text = _pay(j) workplace = str(j.get("workplace_type") or "").strip() diff --git a/plugins/readers/platsbanken.py b/plugins/readers/platsbanken.py index 3382dd1..c42af32 100644 --- a/plugins/readers/platsbanken.py +++ b/plugins/readers/platsbanken.py @@ -22,7 +22,7 @@ from urllib.parse import urlencode from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads -from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, html_text, only, parse_dt NAME = "platsbanken" LABEL = "Platsbanken" @@ -108,8 +108,6 @@ def rows(decoded: Any, slug: str) -> list[Row]: for hit in decoded.get("hits") or []: title = _text(hit.get("headline") or "") url = (hit.get("webpage_url") or "").strip() - if not title or not is_web_url(url): - continue addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}] places = [place for address in addresses if isinstance(address, dict) if (place := _place(address))] diff --git a/plugins/readers/recruitee.py b/plugins/readers/recruitee.py index e4c396c..45659d3 100644 --- a/plugins/readers/recruitee.py +++ b/plugins/readers/recruitee.py @@ -13,7 +13,7 @@ from typing import Any from ._contract import Ask, Fields, Place, Reads -from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount +from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount NAME = "recruitee" LABEL = "Recruitee" @@ -105,8 +105,6 @@ def parse(body: bytes, slug: str) -> list[Row]: for j in data.get("offers") or []: title = (j.get("title") or "").strip() url = (j.get("careers_url") or "").strip() - if not title or not is_web_url(url): - continue salary = j.get("salary") if isinstance(j.get("salary"), dict) else {} salary_min, salary_max, currency = yearly_amount( salary.get("min"), salary.get("max"), salary.get("currency"), diff --git a/plugins/readers/rippling.py b/plugins/readers/rippling.py index 2104fb3..4f63c2b 100644 --- a/plugins/readers/rippling.py +++ b/plugins/readers/rippling.py @@ -82,27 +82,31 @@ def parse(body: bytes, slug: str) -> list[Row]: raise ValueError("not a Rippling board: the answer is not a list") roles: dict[str, Row] = {} places: dict[str, list[Place]] = {} - for entry in data: + for index, entry in enumerate(data): job_id = str(entry.get("uuid") or "").strip() title = (entry.get("name") or "").strip() - if not title or not _UUID.fullmatch(job_id): - continue + # The id goes into a URL: one of other characters gets no link, and + # the runner drops the row. An entry without an id is a role of its + # own, not merged with the others that lack one. + key = job_id or f"#{index}" place = str(((entry.get("workLocation") or {}).get("label")) or "").strip() - places.setdefault(job_id, []) + places.setdefault(key, []) if place: - add_place(places[job_id], _place(place)) - roles.setdefault(job_id, { + add_place(places[key], _place(place)) + roles.setdefault(key, { "source_id": job_id, "title": title, - "url": f"https://{_SITE}/{slug}/jobs/{job_id}", + "url": f"https://{_SITE}/{slug}/jobs/{job_id}" if _UUID.fullmatch(job_id) else "", "posted": None, "department": str(((entry.get("department") or {}).get("label")) or "").strip(), "description": "", "salary_text": "", # the board's list states no pay }) - for job_id, row in roles.items(): - row["places"] = places[job_id] - row["location"] = "; ".join(p["text"] for p in places[job_id]) + if title and not roles[key]["title"]: + roles[key]["title"] = title # an entry of the role that has one + for key, row in roles.items(): + row["places"] = places[key] + row["location"] = "; ".join(p["text"] for p in places[key]) return list(roles.values()) diff --git a/plugins/readers/smartrecruiters.py b/plugins/readers/smartrecruiters.py index e433f0b..0b7a091 100644 --- a/plugins/readers/smartrecruiters.py +++ b/plugins/readers/smartrecruiters.py @@ -115,8 +115,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: for j in data.get("content") or []: title = (j.get("name") or "").strip() job_id = str(j.get("id") or "") - if not title or not job_id: - continue location = j.get("location") or {} loc_text = (location.get("fullLocation") or ", ".join(p for p in (location.get("city"), location.get("country")) if p)) diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py index 3ad376d..bcb4129 100644 --- a/plugins/readers/successfactors.py +++ b/plugins/readers/successfactors.py @@ -63,7 +63,7 @@ from desk.sources import strip_html from ._contract import Ask, Fields, Filters, Place, Reads from ._countries import NAMES -from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt NAME = "successfactors" LABEL = "SuccessFactors" @@ -247,8 +247,9 @@ def _tile_fields(chunk: str) -> tuple[str, str, str]: def _entries(text: str) -> list[dict[str, Any]]: """The roles of one search page, in the order shown, whichever layout it is in and before the prefix filter: `id`, `href` (as written, relative), - `title`, `place`, `date` and `department`. A role is listed once, and a row - without a link to a role is not one.""" + `title`, `place`, `date` and `department`. A role is listed once. The id + and the title are in the role's link, so a row without one is handed over + with an empty `id`, `href` and `title`, for the runner to drop and count.""" table = _ROW.split(text)[1:] chunks = table or _TILE.split(text)[1:] out: list[dict[str, Any]] = [] @@ -256,10 +257,12 @@ def _entries(text: str) -> list[dict[str, Any]]: for chunk in chunks: if table: chunk = chunk.split("", 1)[0] - if not (link := _LINK.search(chunk)): - continue - href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3)) - if job_id in seen: + if link := _LINK.search(chunk): + href, job_id, title = (html.unescape(link.group(1)), link.group(2), + _text(link.group(3))) + else: + href = job_id = title = "" + if job_id and job_id in seen: continue seen.add(job_id) place, date, department = (_table_fields if table else _tile_fields)(chunk) @@ -283,12 +286,14 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: if not _SEARCH_PAGE.search(text): raise ValueError("not a SuccessFactors search page") found = _entries(text) + ids = [entry["id"] for entry in found if entry["id"]] total = _total(text) - if found and found[0]["id"] == previous: + first = ids[0] if ids else None + if first is not None and first == previous: return text, [], total, (start, previous) more = bool(found) and (total is None or start + len(found) < total) - return (text, [{"id": entry["id"]} for entry in found], total, - (start + len(found), found[0]["id"]) if more else None) + return (text, [{"id": job_id} for job_id in ids], total, + (start + len(found), first) if more else None) def rows(decoded: str, slug: str) -> list[Row]: @@ -296,17 +301,21 @@ def rows(decoded: str, slug: str) -> list[Row]: slug's prefix when it has one; the id in it is the requisition's `source_id`. The table and the tile layout give a title, a location, a date and, where the site has the column, a department; nothing else about a role is on - the list, so its description and pay are empty.""" + the list, so its description and pay are empty. + + A role linked on another host, or outside the prefix, is another board's + (another site, or another brand of a shared site), and is left out: it is + no role of this company. A row with no link or no title is handed over, + and the runner drops it.""" host, prefix = _parts(slug) base = f"https://{host}/" out = [] for entry in _entries(decoded): - url = urljoin(base, entry["href"]) - if urlsplit(url).hostname != host or not is_web_url(url): + url = urljoin(base, entry["href"]) if entry["href"] else "" + if url and urlsplit(url).hostname != host: continue - if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"): - continue - if not entry["title"]: + if url and prefix and not urlsplit(url).path.lower().startswith( + f"{prefix.lower()}/job/"): continue out.append({ "source_id": entry["id"], diff --git a/plugins/readers/teamtailor.py b/plugins/readers/teamtailor.py index cb4d7fc..2748ae7 100644 --- a/plugins/readers/teamtailor.py +++ b/plugins/readers/teamtailor.py @@ -15,7 +15,7 @@ from typing import Any from ._contract import Ask, Fields, Place, Reads -from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt +from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt NAME = "teamtailor" LABEL = "Teamtailor" @@ -115,8 +115,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]: for item in items: title = (item.findtext("title") or "").strip() url = (item.findtext("link") or "").strip() - if not title or not is_web_url(url): - continue status = (item.findtext("remoteStatus") or "").strip() remote = {"fully": True, "hybrid": False, "onsite": False}.get(status) places: list[Place] = [] diff --git a/plugins/readers/workable.py b/plugins/readers/workable.py index 28b3d6e..115b6a2 100644 --- a/plugins/readers/workable.py +++ b/plugins/readers/workable.py @@ -102,8 +102,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]: for j in data.get("results") or []: title = (j.get("title") or "").strip() shortcode = (j.get("shortcode") or "").strip() - if not title or not shortcode: - continue workplace = str(j.get("workplace") or "").strip() remote = bool(j.get("remote")) or workplace == "remote" places = _places(j, remote) diff --git a/plugins/readers/workday.py b/plugins/readers/workday.py index 0cbebcf..a7dac62 100644 --- a/plugins/readers/workday.py +++ b/plugins/readers/workday.py @@ -124,8 +124,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None postings = data.get("jobPostings") or [] # Every posting is counted, a path-less one (LEGO lists one that has only - # `bulletFields`) by its position, so the parser dropping it does not - # leave the board short of its total. + # `bulletFields`) by its position, so the runner dropping its row does + # not leave the board short of its total. results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)] following = offset + len(postings) more = bool(postings) and (total is None or following < total) @@ -274,13 +274,13 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]: `bulletFields`, which lead with the requisition id, and on some tenants `remoteType`. No description, no date and no country: `detail` has those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the - path where not; a posting with no title or no path is skipped.""" + path where not. A posting with no path has neither an id nor a link, and + the runner drops it.""" out = [] for p in decoded.get("jobPostings") or []: title = (p.get("title") or "").strip() path = (p.get("externalPath") or "").strip() - if not title or not _ROLE_PATH.fullmatch(path): - continue + path = path if _ROLE_PATH.fullmatch(path) else "" remote = _remote(p.get("remoteType")) place = _place(p.get("locationsText"), remote) out.append({ @@ -288,7 +288,7 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]: "title": title, "places": [place] if place else [], "location": place["text"] if place else "", - "url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}", # type: ignore[index] + "url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}" if path else "", # type: ignore[index] "posted": _posted(p.get("postedOn") or ""), "department": "", "description": "", diff --git a/plugins/search.py b/plugins/search.py index 0a92710..65434a7 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -19,7 +19,8 @@ roles close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). In a pull a search is read by `_runner.read` under the prefix `search`, as a -board is under `ats`: the 10% guard holds it, `_runner.stamp` records it, +board is under `ats`: the runner drops a row without an id, a title or a +link, the 10% guards hold it, `_runner.stamp` records it, and a search that fails or is suspect is kept as failing under `search:failing:` (`_runner.record_failure`) until it answers, which the settle reads as it reads a failing board's. diff --git a/tests/test_ats.py b/tests/test_ats.py index e82808c..a8eeb31 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -24,18 +24,24 @@ from plugins.readers import _runner, smartrecruiters # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" -def _rows(host: str, slug: str = "acme") -> list[ats.Row]: +def _parsed(host: str, slug: str = "acme") -> list[ats.Row]: path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}" - body = path.read_bytes() - return ats._PARSERS[host](body, slug) + return ats._PARSERS[host](path.read_bytes(), slug) + + +def _rows(host: str, slug: str = "acme") -> list[ats.Row]: + """The rows of a host's fixture the runner keeps (`_runner.valid`).""" + return [row for row in _parsed(host, slug) if _runner.valid(row)] # ------------------------------------------------------------------ hosts def test_greenhouse_parses_titles_locations_and_html_description() -> None: + # The javascript: and data: rows are handed over and the runner drops + # them: a feed is external content, never trusted as a URL. + assert [r["source_id"] for r in _parsed("greenhouse")] == [ + "1001", "1002", "1003", "1004", "1005", "1006"] rows = _rows("greenhouse") - # The javascript: and data: rows are refused, same as the four real ones - # are kept: a feed is external content, never trusted as a URL. assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"] first = rows[0] assert first["title"] == "Senior Backend Engineer" # trimmed @@ -269,10 +275,11 @@ TEAMTAILOR = """ def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None: rows = ats._parse_teamtailor(TEAMTAILOR, "kry") - # the javascript: link is refused like on the other hosts + # the javascript: link is handed over and refused by the runner assert [r["source_id"] for r in rows] == [ "39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb", - "0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"] + "0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"] + assert [_runner.valid(r) for r in rows] == [True, True, True, False] first = rows[0] assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer" assert first["department"] == "Engineering" @@ -294,7 +301,7 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None: def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None: rows = ats._parse_teamtailor(TEAMTAILOR, "kry") - assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them + assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them assert rows[1]["title"] == "Läkare" # trimmed assert rows[1]["department"] == "Software Development" assert ats.classify(rows[1]["department"], "Sales Lead") == "sales" diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py index e2c27c6..0fcb236 100644 --- a/tests/test_ats_bamboohr.py +++ b/tests/test_ats_bamboohr.py @@ -17,7 +17,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import Ask, bamboohr # noqa: E402 +from plugins.readers import Ask, _runner, bamboohr # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -84,11 +84,16 @@ def test_a_page_that_is_not_a_listing_raises_and_an_empty_board_is_no_rows() -> bamboohr.parse(body, "configit") -def test_a_role_without_a_title_or_an_id_is_left_out() -> None: +def test_a_role_without_a_title_or_an_id_reaches_the_runner_and_is_dropped_there() -> None: body = json.loads(_read("bamboohr.json")) body["result"][0]["jobOpeningName"] = " " body["result"][1]["id"] = None - assert bamboohr.parse(json.dumps(body).encode(), "configit") == [] + raw = json.dumps(body).encode() + rows = bamboohr.parse(raw, "configit") + assert len(rows) == 2 and not any(map(_runner.valid, rows)) + listing = _runner.read_listing(bamboohr, "configit", lambda url, **kw: raw) + assert (listing.rows, listing.ids, listing.complete) == ([], [], True) + assert (listing.handed, listing.dropped) == (2, 2) def test_detail_reads_the_text_the_date_and_the_country() -> None: diff --git a/tests/test_ats_deel.py b/tests/test_ats_deel.py index 38c230b..1764479 100644 --- a/tests/test_ats_deel.py +++ b/tests/test_ats_deel.py @@ -17,7 +17,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import Ask, deel # noqa: E402 +from plugins.readers import Ask, _runner, deel # noqa: E402 PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes() URL = "https://jobs.deel.com/klarna" @@ -89,10 +89,12 @@ def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board deel.parse(b"Job Board Not Found", "nobody") -def test_a_posting_without_a_title_or_id_is_skipped() -> None: +def test_a_posting_without_a_title_or_id_is_handed_over_for_the_runner_to_drop() -> None: body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}}, {"id": "c", "title": "Cook", "job": {}}]} - assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"] + rows = deel.parse(_page_with(body), "acme") + assert [r["source_id"] for r in rows] == ["a", "", "c"] + assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"] # ------------------------------------------------------------ the fetch diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py index 4d55f56..4bd6aa6 100644 --- a/tests/test_ats_eightfold.py +++ b/tests/test_ats_eightfold.py @@ -18,7 +18,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import Ask, eightfold # noqa: E402 +from plugins.readers import Ask, _runner, eightfold # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SLUG = "jobs.ericsson.com/ericsson.com" @@ -93,10 +93,12 @@ def test_the_work_location_option_is_the_hosts_word( assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote -def test_a_position_without_a_name_an_id_or_a_path_is_left_out() -> None: +def test_a_position_without_a_name_an_id_or_a_path_is_handed_over_for_the_runner_to_drop() -> None: bad = [_position(1) | {"name": " "}, _position(2) | {"id": None}, _position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}] - assert eightfold.rows(bad, SLUG) == [] + rows = eightfold.rows(bad, SLUG) + assert len(rows) == 4 and not any(map(_runner.valid, rows)) + assert [r["url"] for r in rows[2:]] == ["", ""] # no link built off the host def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None: diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py index 96579d0..b3ad268 100644 --- a/tests/test_ats_gem.py +++ b/tests/test_ats_gem.py @@ -16,7 +16,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import Ask, gem # noqa: E402 +from plugins.readers import Ask, _runner, gem # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" API = "https://jobs.gem.com/api/public/graphql" @@ -79,8 +79,9 @@ def test_remote_is_the_location_type_then_the_locations( assert gem.rows([posting], "acme")[0]["remote"] == remote -def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None: - assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == [] +def test_a_posting_without_a_title_or_an_ext_id_is_handed_over_for_the_runner_to_drop() -> None: + rows = gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") + assert len(rows) == 2 and not any(map(_runner.valid, rows)) def test_the_board_is_one_post_without_origin_or_referer() -> None: diff --git a/tests/test_ats_hibob.py b/tests/test_ats_hibob.py index a97e95d..6bb6d92 100644 --- a/tests/test_ats_hibob.py +++ b/tests/test_ats_hibob.py @@ -17,7 +17,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import Ask, hibob # noqa: E402 +from plugins.readers import Ask, _runner, hibob # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "hibob.json").read_bytes() @@ -87,10 +87,12 @@ def test_the_place_leaves_the_country_off_where_the_site_names_it() -> None: assert hibob.parse(_one(**fields), "acme")[0]["location"] == want -def test_an_ad_without_a_title_or_id_is_skipped_and_an_empty_board_is_empty() -> None: +def test_an_ad_without_a_title_or_id_is_handed_over_and_an_empty_board_is_empty() -> None: body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"}, {"id": "c", "title": "Cook"}]}).encode() - assert [r["source_id"] for r in hibob.parse(body, "acme")] == ["c"] + rows = hibob.parse(body, "acme") + assert [r["source_id"] for r in rows] == ["a", "", "c"] + assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"] assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == [] diff --git a/tests/test_ats_hrmanager.py b/tests/test_ats_hrmanager.py index 3198948..3d3cbab 100644 --- a/tests/test_ats_hrmanager.py +++ b/tests/test_ats_hrmanager.py @@ -16,7 +16,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import Ask, hrmanager # noqa: E402 +from plugins.readers import Ask, _runner, hrmanager # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "hrmanager.xml").read_bytes() @@ -52,12 +52,14 @@ def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_f hrmanager.parse(b"404 - File or directory not found.", "x") -def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None: +def test_an_item_without_a_title_or_a_web_link_is_handed_over_for_the_runner_to_drop() -> None: body = (b"1 https://a.example/1" b"2Bjavascript:alert(1)" b"3Chttps://a.example/3" b"") - assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"] + rows = hrmanager.parse(body, "x") + assert [r["source_id"] for r in rows] == ["1", "2", "3"] + assert [r["source_id"] for r in rows if _runner.valid(r)] == ["3"] # ------------------------------------------------------------- paging diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py index 6de4180..30544c3 100644 --- a/tests/test_ats_jibe.py +++ b/tests/test_ats_jibe.py @@ -18,7 +18,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import Ask, jibe # noqa: E402 +from plugins.readers import Ask, _runner, jibe # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SITE = "jobs.booking.com" @@ -72,9 +72,10 @@ def test_the_location_falls_back_to_city_and_country() -> None: assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia" -def test_a_role_without_a_title_or_a_slug_is_left_out() -> None: +def test_a_role_without_a_title_or_a_slug_is_handed_over_for_the_runner_to_drop() -> None: jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}] - assert jibe.rows({"jobs": jobs}, SITE) == [] + rows = jibe.rows({"jobs": jobs}, SITE) + assert len(rows) == 3 and not any(map(_runner.valid, rows)) def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None: diff --git a/tests/test_ats_jobylon.py b/tests/test_ats_jobylon.py index e4ce3a3..7435641 100644 --- a/tests/test_ats_jobylon.py +++ b/tests/test_ats_jobylon.py @@ -16,7 +16,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import Ask, jobylon # noqa: E402 +from plugins.readers import Ask, _runner, jobylon # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "jobylon.html").read_bytes() @@ -65,12 +65,14 @@ def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None: assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"] -def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None: +def test_a_role_whose_link_is_not_a_role_page_gets_no_link_and_is_dropped() -> None: body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'}," b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'}," b"{id: '3', title: 'C', url: '/companies/3/'}," b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n") - assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"] + rows = jobylon.parse(body, "9") + assert [r["url"] for r in rows[:3]] == ["", "", ""] + assert [r["source_id"] for r in rows if _runner.valid(r)] == ["4"] # ---------------------------------------------------------------- detail diff --git a/tests/test_ats_oracle.py b/tests/test_ats_oracle.py index 011cd3e..83d49ad 100644 --- a/tests/test_ats_oracle.py +++ b/tests/test_ats_oracle.py @@ -20,7 +20,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats # noqa: E402 -from plugins.readers import oracle # noqa: E402 +from plugins.readers import _runner, oracle # noqa: E402 from plugins.readers._shared import is_web_url # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -84,14 +84,15 @@ def test_a_department_is_taken_where_a_tenant_fills_one() -> None: assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""] -def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_left_out() -> None: +def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_dropped_by_the_runner() -> None: data = _listing() roles = data["items"][0]["requisitionList"] roles[0]["Title"] = " " roles[1]["Id"] = "1/../../x" roles[2]["Id"] = "" - assert [r["source_id"] for r in oracle.parse(json.dumps(data).encode(), SLUG)] == \ - ["40975", "31680"] + rows = oracle.parse(json.dumps(data).encode(), SLUG) + assert rows[1]["url"] == "" # the id builds no link + assert [r["source_id"] for r in rows if _runner.valid(r)] == ["40975", "31680"] def test_a_listing_without_the_expand_has_no_roles() -> None: diff --git a/tests/test_ats_phenom.py b/tests/test_ats_phenom.py index 97ef8e8..b91b37e 100644 --- a/tests/test_ats_phenom.py +++ b/tests/test_ats_phenom.py @@ -19,7 +19,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats # noqa: E402 -from plugins.readers import phenom # noqa: E402 +from plugins.readers import _runner, phenom # noqa: E402 from plugins.readers._shared import is_web_url # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -101,13 +101,15 @@ def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None "https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS" -def test_a_role_without_title_or_id_is_left_out() -> None: +def test_a_role_without_title_or_id_is_dropped_by_the_runner() -> None: data = _answer() jobs = data["refineSearch"]["data"]["jobs"] jobs[0]["title"] = "" for key in ("jobSeqNo", "jobId", "reqId"): jobs[1].pop(key, None) - assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:] + rows = phenom.parse(json.dumps(data).encode(), SLUG) + assert len(rows) == len(IDS) + assert [r["source_id"] for r in rows if _runner.valid(r)] == IDS[2:] # ---------------------------------------------------------- the request diff --git a/tests/test_ats_pinpoint.py b/tests/test_ats_pinpoint.py index d1756ae..4dc6935 100644 --- a/tests/test_ats_pinpoint.py +++ b/tests/test_ats_pinpoint.py @@ -15,7 +15,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import pinpoint # noqa: E402 +from plugins.readers import _runner, pinpoint # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -86,11 +86,12 @@ def test_an_unknown_pay_frequency_keeps_the_text_and_leaves_the_columns() -> Non assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "") -def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_left_out() -> None: +def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_dropped_by_the_runner() -> None: body = json.loads(_body()) body["data"][0]["title"] = "" body["data"][1]["url"] = "javascript:alert(1)" - assert pinpoint.parse(json.dumps(body).encode(), "xeneta") == [] + rows = pinpoint.parse(json.dumps(body).encode(), "xeneta") + assert len(rows) == 2 and not any(map(_runner.valid, rows)) def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None: diff --git a/tests/test_ats_rippling.py b/tests/test_ats_rippling.py index eb0c932..32e34d3 100644 --- a/tests/test_ats_rippling.py +++ b/tests/test_ats_rippling.py @@ -18,7 +18,7 @@ if str(ROOT) not in sys.path: from app.jobs.discover import boards_in # noqa: E402 from plugins import ats, readers # noqa: E402 -from plugins.readers import rippling # noqa: E402 +from plugins.readers import _runner, rippling # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" BOARD = (FIXTURES / "rippling.json").read_bytes() @@ -61,11 +61,14 @@ def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme") -def test_an_entry_without_a_title_or_a_plain_id_is_skipped() -> None: +def test_an_entry_without_a_title_or_a_plain_id_is_dropped_by_the_runner() -> None: body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"}, {"uuid": "../../x", "name": "C"}, {"uuid": "abcdefgh-2", "name": "D"}]).encode() - assert [r["source_id"] for r in rippling.parse(body, "acme")] == ["abcdefgh-2"] + rows = rippling.parse(body, "acme") + assert [r["source_id"] for r in rows] == ["abcdefgh-1", "", "../../x", "abcdefgh-2"] + assert rows[2]["url"] == "" # the id builds no link + assert [r["source_id"] for r in rows if _runner.valid(r)] == ["abcdefgh-2"] # ---------------------------------------------------------------- detail diff --git a/tests/test_ats_teamtailor.py b/tests/test_ats_teamtailor.py index bd8ed66..f4ee084 100644 --- a/tests/test_ats_teamtailor.py +++ b/tests/test_ats_teamtailor.py @@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from plugins.readers import Ask, teamtailor # noqa: E402 +from plugins.readers import Ask, _runner, teamtailor # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" @@ -40,7 +40,8 @@ def test_page_takes_the_ask_and_reads_the_same_url() -> None: def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None: - first, second, third = _rows() + first, second, third, refused = _rows() + assert not _runner.valid(refused) # javascript:, for the runner # "Sverige" (a city that is a country) once, Stockholm's twin once. assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid {"text": "Stockholm, Sweden", "remote": False}] diff --git a/tests/test_ats_workday.py b/tests/test_ats_workday.py index 9598b75..fb8ef9a 100644 --- a/tests/test_ats_workday.py +++ b/tests/test_ats_workday.py @@ -20,7 +20,7 @@ if str(ROOT) not in sys.path: from app.jobs import discover # noqa: E402 from plugins import ats # noqa: E402 -from plugins.readers import workday # noqa: E402 +from plugins.readers import _runner, workday # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" SLUG = "zendesk.wd1/zendesk" @@ -96,7 +96,7 @@ def test_posted_is_read_only_where_the_text_is_exact(text: str, expected: dateti assert workday._posted(text) == expected -def test_a_posting_without_a_title_or_a_path_is_skipped() -> None: +def test_a_posting_without_a_title_or_a_path_reaches_the_runner_and_is_dropped() -> None: page = {"jobPostings": [ {"bulletFields": ["0000036369"]}, # LEGO's, no title {"title": "No path", "bulletFields": ["R1"]}, @@ -105,7 +105,16 @@ def test_a_posting_without_a_title_or_a_path_is_skipped() -> None: {"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X", "postedOn": "Posted Today", "bulletFields": ["R4"]}, ]} - assert [r["title"] for r in workday.rows(page, SLUG)] == ["Whole"] + rows = workday.rows(page, SLUG) + assert len(rows) == 5 + assert [(r["source_id"], r["url"]) for r in rows[:2]] == [("", ""), ("", "")] + assert [r["title"] for r in rows if _runner.valid(r)] == ["Whole"] + # Through the runner: four dropped and counted, and the read complete, + # since every posting counts by its position towards the stated total. + body = json.dumps(page | {"total": 5}).encode() + listing = _runner.read_listing(workday, SLUG, lambda url, **kw: body) + assert [r["title"] for r in listing.rows] == ["Whole"] + assert (listing.handed, listing.dropped, listing.complete) == (5, 4, True) def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None: diff --git a/tests/test_limits.py b/tests/test_limits.py index 71a2181..dcd72d1 100644 --- a/tests/test_limits.py +++ b/tests/test_limits.py @@ -116,6 +116,16 @@ def test_roles_without_text_and_text_reads_that_failed() -> None: assert words(fake(), state(), counts=limits.Texts(9, 0)) == [] +@pytest.mark.parametrize(("dropped", "found"), [ + ("3", ["3 listings dropped: no title or link"]), + ("1", ["1 listing dropped: no title or link"]), + ("", []), ("x", [])]) +def test_the_listings_the_last_read_dropped_are_named(dropped: str, found: list[str]) -> None: + """The rows the runner dropped (`_runner.valid`), after "read incomplete".""" + assert words(fake(), state(dropped=dropped)) == found + assert words(fake(), state(dropped=dropped, complete=""))[1:] == found + + def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None: assert words(fake(), state(), reach.SMALL - 1) == ["read whole"] assert words(fake(), state(), reach.SMALL) == [] @@ -199,6 +209,17 @@ def test_companies_carry_their_limits_and_the_boards_own_open_roles( assert row["limits"] == ["read incomplete", "without text: 1", "read whole"] +def test_a_board_that_dropped_listings_says_so_on_its_row( + client: TestClient, store: Store) -> None: + store.add_subscription("ats", "lever:acme", company="Acme", board="lever") + store.add_company("Acme", key="acme", ats="lever") + store.state["ats:ok:lever:acme"] = "run" + store.state["ats:complete:lever:acme"] = "1" + store.state["ats:dropped:lever:acme"] = "3" + [row] = client.get("/api/companies").json() + assert row["limits"][0] == "3 listings dropped: no title or link" + + def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None: store.add_company("Novo", key="novo", ats="taleo") [row] = client.get("/api/companies").json() diff --git a/tests/test_readers_adzuna.py b/tests/test_readers_adzuna.py index a50a036..ac05026 100644 --- a/tests/test_readers_adzuna.py +++ b/tests/test_readers_adzuna.py @@ -16,7 +16,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import readers # noqa: E402 -from plugins.readers import Ask, adzuna # noqa: E402 +from plugins.readers import Ask, _runner, adzuna # noqa: E402 BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes() ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin", @@ -39,8 +39,9 @@ def test_it_declares_the_key_and_states_the_robots_finding() -> None: def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None: - first, predicted = _rows() # javascript: refused + first, predicted, refused = _rows() assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"] + assert refused["url"].startswith("javascript:") and not _runner.valid(refused) assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote" assert first["teaser"] == "Build our backend services in Python & Go." assert "description" not in first @@ -89,3 +90,12 @@ def test_saved_params_need_a_known_country() -> None: "q": "x", "country": "gb"} with pytest.raises(ValueError, match="country"): adzuna.clean_params({"q": "x", "country": "dk"}) + + +def test_the_runner_drops_the_ad_without_a_web_link_and_counts_it() -> None: + """The reader hands the `javascript:` ad over; the runner drops it, and + the read is complete, since all three result ids were read.""" + listing = _runner.read_listing(adzuna, "", lambda url, **kw: BODY, ASK, + limits=_runner.Limits(page_delay=0)) + assert listing.ids == ["4412345678", "4412345679"] + assert (listing.handed, listing.dropped, listing.complete) == (3, 1, True) diff --git a/tests/test_readers_freehire.py b/tests/test_readers_freehire.py index 03780cd..a92dc47 100644 --- a/tests/test_readers_freehire.py +++ b/tests/test_readers_freehire.py @@ -16,7 +16,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import readers # noqa: E402 -from plugins.readers import Ask, freehire # noqa: E402 +from plugins.readers import Ask, _runner, freehire # noqa: E402 BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes() @@ -119,14 +119,17 @@ def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None: assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR") -def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None: +def test_a_hit_without_a_title_an_id_or_a_web_link_is_handed_to_the_runner() -> None: + """The reader hands every hit over; the runner keeps only the whole one.""" hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"}, {"public_slug": "", "title": "B", "url": "https://x.example/b"}, {"public_slug": "c", "title": " ", "url": "https://x.example/c"}, {"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}] found = freehire.rows({"data": hits}, "") - assert [row["source_id"] for row in found] == ["d"] - assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == [] + assert [row["source_id"] for row in found] == ["a", "", "c", "d"] + kept = [row for row in found if _runner.valid(row)] + assert [row["source_id"] for row in kept] == ["d"] + assert kept[0]["url"] == "https://x.example/d" and kept[0]["places"] == [] def test_a_title_and_a_company_have_their_entities_decoded() -> None: diff --git a/tests/test_readers_jobbank.py b/tests/test_readers_jobbank.py index bc7d041..a14a60c 100644 --- a/tests/test_readers_jobbank.py +++ b/tests/test_readers_jobbank.py @@ -15,7 +15,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import readers # noqa: E402 -from plugins.readers import Ask, jobbank # noqa: E402 +from plugins.readers import Ask, _runner, jobbank # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "search" PLACES = (FIXTURES / "jobbank-places.xml").read_bytes() @@ -170,7 +170,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None: body = _feed(99).replace(b"", b"No link") decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask()) assert len(results) == 100 and nxt == 2 - assert len(jobbank.rows(decoded, "")) == 99 + rows = jobbank.rows(decoded, "") + assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99 @pytest.mark.parametrize("body", [b"", b"Just a moment...", diff --git a/tests/test_readers_jobindex.py b/tests/test_readers_jobindex.py index b0df886..ce0cb59 100644 --- a/tests/test_readers_jobindex.py +++ b/tests/test_readers_jobindex.py @@ -15,7 +15,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import readers # noqa: E402 -from plugins.readers import Ask, jobindex # noqa: E402 +from plugins.readers import Ask, _runner, jobindex # noqa: E402 BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes() @@ -89,7 +89,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None: body = _feed(19).replace(b"", b"No link") decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask()) assert len(results) == 20 and nxt == 2 - assert len(jobindex.rows(decoded, "")) == 19 + rows = jobindex.rows(decoded, "") + assert len(rows) == 20 and sum(map(_runner.valid, rows)) == 19 def test_an_answer_without_a_channel_raises() -> None: diff --git a/tests/test_readers_jobnet.py b/tests/test_readers_jobnet.py index 870ac85..aeae28c 100644 --- a/tests/test_readers_jobnet.py +++ b/tests/test_readers_jobnet.py @@ -16,7 +16,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import readers # noqa: E402 -from plugins.readers import Ask, jobnet # noqa: E402 +from plugins.readers import Ask, _runner, jobnet # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "search" PLACES = (FIXTURES / "jobnet-places.json").read_bytes() @@ -100,9 +100,12 @@ def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None: assert "applicationDeadline" not in row -def test_an_ad_without_an_id_or_a_title_is_skipped() -> None: +def test_an_ad_without_an_id_or_a_title_is_handed_over_for_the_runner_to_drop() -> None: ads = json.loads(OTHER)["jobAds"] - assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == [] + rows = jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") + assert [(row["source_id"] == "", row["title"] == "") for row in rows] == [ + (False, True), (True, False)] + assert not any(map(_runner.valid, rows)) def test_it_sends_the_header_the_region_and_the_page_number() -> None: diff --git a/tests/test_readers_platsbanken.py b/tests/test_readers_platsbanken.py index 88d5a37..fddc880 100644 --- a/tests/test_readers_platsbanken.py +++ b/tests/test_readers_platsbanken.py @@ -16,7 +16,7 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from plugins import readers # noqa: E402 -from plugins.readers import Ask, platsbanken # noqa: E402 +from plugins.readers import Ask, _runner, platsbanken # noqa: E402 BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes() @@ -47,8 +47,9 @@ def test_it_keeps_every_workplace_of_an_ad() -> None: decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask()) rows = platsbanken.rows(decoded, "") assert total == 32 - assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused - first, no_town, bare = rows + assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400001", "31400002"] + assert not _runner.valid(rows[2]) # javascript:, dropped by the runner + first, no_town, _refused, bare = rows assert [p["text"] for p in first["places"]] == [ "Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län assert first["location"] == "Lund" @@ -58,7 +59,7 @@ def test_it_keeps_every_workplace_of_an_ad() -> None: def test_it_reads_title_company_date_and_the_full_text() -> None: decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask()) - first, _no_town, remote = platsbanken.rows(decoded, "") + first, _no_town, _refused, remote = platsbanken.rows(decoded, "") assert first["title"] == "Postdoctoral Researcher" assert first["company"] == "European Spallation Source Eric" assert first["country"] == "SE" and first["remote"] == "Arbete på plats" diff --git a/tests/test_runner.py b/tests/test_runner.py index 41380c2..51c062c 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,7 +1,7 @@ """The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a subscription resolves to, series and their union, whole and paged listings, -completeness from the declaration, the sweep across pulls, the 10% guard, -the facets kept in state, the detail reads and the `raw` an item carries. No +completeness from the declaration, the rows kept and dropped, the sweep +across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No network: `get` is a dict of URL -> body or exception.""" from __future__ import annotations @@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None: assert filtered.ids == ["1"] and filtered.filtered +# --------------------------------------------------------- the rows kept + +def _handed(i: str) -> dict[str, Any]: + """A row as a reader hands it over: whole, or, for an id of `-id`, + `-title`, `-link` or `-url`, without that value or with a link that is + no web URL (`javascript:`, or one httpx cannot read).""" + broken = {"-id": {"source_id": ""}, "-title": {"title": " "}, + "-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}} + return _row(i, **broken.get(i.split(":")[0], {})) + + +def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None: + assert _runner.valid(_handed("1")) + assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url")) + assert not _runner.valid({"source_id": "1", "title": "Role"}) + + +def test_a_whole_listing_drops_the_rows_without_and_counts_them( + caplog: pytest.LogCaptureFixture) -> None: + module = _whole() + module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)] + web = Web({"https://jobs.fake.example/acme/list": + b'["1", "-id", "2", "-title", "-link", "-url"]'}) + with caplog.at_level("WARNING", logger=_runner.__name__): + listing = _runner.read_listing(module, "acme", web) + assert [r["source_id"] for r in listing.rows] == ["1", "2"] + assert (listing.ids, listing.complete) == (["1", "2"], True) + assert (listing.handed, listing.dropped) == (6, 4) + [line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()] + assert "fake:acme" in line and "dropped 4 of 6 listings" in line + + +def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results( + caplog: pytest.LogCaptureFixture) -> None: + """Completeness counts the result ids `page` returns, not the rows kept, + so the dropped rows still count as read; the drop is logged once.""" + pages = [["1", "-title:a"], ["2", "-link:b"]] + module = _paged(pages, total=4) + module.rows = lambda ids, slug: [_handed(i) for i in ids] + with caplog.at_level("WARNING", logger=_runner.__name__): + listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2"], True) + assert (listing.handed, listing.dropped) == (4, 2) + assert sum("dropped" in r.getMessage() for r in caplog.records) == 1 + + +def test_the_stamp_keeps_the_rows_dropped() -> None: + state: dict[str, str] = {} + listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2) + _runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"]) + assert state["ats:dropped:t"] == "2" + _runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), []) + assert state["ats:dropped:t"] == "" + + +def _dropping(handed: int, dropped: int) -> tuple[Any, Web]: + """A whole board whose feed hands over `handed` rows, `dropped` of them + without a title.""" + module = _whole() + module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)] + ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)] + return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()}) + + +@pytest.mark.parametrize(("handed", "dropped", "fails"), [ + (20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)]) +def test_a_read_that_drops_over_a_tenth_of_its_rows_fails( + handed: int, dropped: int, fails: bool) -> None: + """Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over: + a page whose shape changed, which must close nothing.""" + module, web = _dropping(handed, dropped) + if not fails: + assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped + return + with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"): + _runner.read({}, "ats", "t", module, "acme", web) + + +def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None: + module, web = _dropping(20, 5) + since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds") + state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})} + listing = _runner.read(state, "ats", "t", module, "acme", web, + limits=_runner.Limits(page_delay=0, stale_days=21)) + assert (len(listing.ids), listing.dropped) == (15, 5) + + # ------------------------------------------------------ through ATSSource @pytest.fixture @@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None: assert state["ats:filtered:fake:acme"] == "1" +def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None: + board.rows = lambda ids, slug: [_handed(i) for i in ids] + board.web.answers = {"https://jobs.fake.example/acme/page/0": + json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()} + board.READS = board.READS.model_copy(update={"complete": "end"}) + state: dict[str, str] = {} + assert len(list(_source(state).fetch())) == 19 + assert state["ats:dropped:fake:acme"] == "1" + + board.web.answers = {"https://jobs.fake.example/acme/page/0": + json.dumps([str(n) for n in range(15)] + + [f"-id:{n}" for n in range(5)]).encode()} + state = {} + assert list(_source(state).fetch()) == [] + assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"] + assert "ats:ok:fake:acme" not in state + + # -------------------------------------------------------------------- raw def test_raw_keeps_every_stated_key_and_the_places() -> None: diff --git a/tests/test_search.py b/tests/test_search.py index a19699a..46384b3 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -383,6 +383,23 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) assert source.state["search:ids:fake:cph"] == '["fake:h1"]' +def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count( + monkeypatch: pytest.MonkeyPatch) -> None: + """The search driver reads through the same listing code as a board: the + runner drops the row the provider handed over without a title, and the + read is still complete, by the distinct result ids.""" + _fake_provider(monkeypatch, ["h1", "h2"], complete=True) + module = readers.PROVIDERS["fake"] + handed = module.rows + module.rows = lambda decoded, slug: [ + row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)] + source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen", + "params": {"q": "x"}}]}) + assert [i.source_id for i in source.fetch()] == ["fake:h1"] + assert source.state["search:complete:fake:cph"] == "1" + assert source.state["search:dropped:fake:cph"] == "1" + + def test_a_search_that_fails_is_skipped_and_stamps_nothing( monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("ADZUNA_APP_ID", raising=False)