Let the runner drop and count a row without an id, a title or a link, as ADR Plugins 3.8 has it, in one place for whole, paged and search reads: the count is stamped per target, logged, named on the board's row, and over 10% of the rows fails the read; the 28 readers hand every listing over instead of skipping it
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
38c31c830b
commit
911dfe8584
58 files changed
+479
-203
No files matched your search
+17
-3
@@ -10,7 +10,10 @@ both; nothing here decides how a board is read, so the words never disagree
|
|||||||
with the pull as long as they read the keys the pull writes.
|
with the pull as long as they read the keys the pull writes.
|
||||||
|
|
||||||
A word with a number carries it after a colon ("without text: 12"), and the
|
A word with a number carries it after a colon ("without text: 12"), and the
|
||||||
filter matches the phrase before the colon (`WORDS`).
|
filter matches the phrase before the colon (`WORDS`). One word is outside
|
||||||
|
`WORDS`: the listings the last read dropped for want of an id, a title or a
|
||||||
|
link ("3 listings dropped: no title or link", `_runner.valid`), shown on the
|
||||||
|
board's row and not offered by the filter.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -49,7 +52,7 @@ WORDS: dict[str, str] = {
|
|||||||
|
|
||||||
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
||||||
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
||||||
"narrowed", "detailfailed")
|
"narrowed", "detailfailed", "dropped")
|
||||||
|
|
||||||
|
|
||||||
class Texts(NamedTuple):
|
class Texts(NamedTuple):
|
||||||
@@ -84,7 +87,8 @@ def texts(store: Store) -> dict[int, Texts]:
|
|||||||
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
||||||
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
||||||
line: str = "", counts: Texts | None = None) -> list[str]:
|
line: str = "", counts: Texts | None = None) -> list[str]:
|
||||||
"""The limits of one board, in the order of `WORDS`, those that apply.
|
"""The limits of one board, in the order of `WORDS`, those that apply, the
|
||||||
|
rows the last read dropped after "read incomplete".
|
||||||
|
|
||||||
`state` holds the board's keys (`state_of`), `size` its kept size,
|
`state` holds the board's keys (`state_of`), `size` its kept size,
|
||||||
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
||||||
@@ -105,6 +109,8 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
|
|||||||
out.append("failing")
|
out.append("failing")
|
||||||
if state.get(key("ok")) and not state.get(key("complete")):
|
if state.get(key("ok")) and not state.get(key("complete")):
|
||||||
out.append("read incomplete")
|
out.append("read incomplete")
|
||||||
|
if dropped := _count(state.get(key("dropped"))):
|
||||||
|
out.append(f"{dropped} listing{'' if dropped == 1 else 's'} dropped: no title or link")
|
||||||
if state.get(key("swept")) == "1" or key("sweep") in state:
|
if state.get(key("swept")) == "1" or key("sweep") in state:
|
||||||
out.append("swept")
|
out.append("swept")
|
||||||
if state.get(key("filtered")) == "1":
|
if state.get(key("filtered")) == "1":
|
||||||
@@ -130,6 +136,14 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _count(value: str | None) -> int:
|
||||||
|
"""A number kept in state, 0 for none or one that does not parse."""
|
||||||
|
try:
|
||||||
|
return max(0, int(value or 0))
|
||||||
|
except ValueError:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
||||||
"""Whether the reach leaves the board unread by place: its reader cannot
|
"""Whether the reach leaves the board unread by place: its reader cannot
|
||||||
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
||||||
|
|||||||
+14
-6
@@ -17,8 +17,9 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|||||||
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
||||||
its slug is. Every board is read independently; one that fails is logged,
|
its slug is. Every board is read independently; one that fails is logged,
|
||||||
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
||||||
whose read lists under a tenth of what it listed last time (`_runner.read`'s
|
whose read lists under a tenth of what it listed last time, or drops more
|
||||||
guard), until it has failed that way for the pull's `stale_days`. A board's
|
than a tenth of the rows its reader handed over (`_runner.read`'s guards),
|
||||||
|
until it has failed that way for the pull's `stale_days`. A board's
|
||||||
filter is the `reach` option the pull hands it (the agents', see
|
filter is the `reach` option the pull hands it (the agents', see
|
||||||
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
||||||
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
||||||
@@ -343,8 +344,10 @@ SIZE_PAGES = 100
|
|||||||
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
||||||
"""How many roles board `target` lists, unfiltered and without terms: a
|
"""How many roles board `target` lists, unfiltered and without terms: a
|
||||||
whole host's listing, the total a `complete="stated"` host gives on page
|
whole host's listing, the total a `complete="stated"` host gives on page
|
||||||
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
|
1, else the listing read to the end, at most `SIZE_PAGES` pages. A read
|
||||||
as `_fetch_rows` does, so a board that does not answer is refused."""
|
counts the rows the runner dropped as well (`_listed`), as a stated total
|
||||||
|
does. Raises as `_fetch_rows` does, so a board that does not answer is
|
||||||
|
refused."""
|
||||||
if not is_known_target(target):
|
if not is_known_target(target):
|
||||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||||
if client is None:
|
if client is None:
|
||||||
@@ -358,7 +361,12 @@ def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
|||||||
if isinstance(total, int):
|
if isinstance(total, int):
|
||||||
return total
|
return total
|
||||||
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
|
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
|
||||||
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
return _listed(_runner.read_listing(module, slug, get, limits=limits))
|
||||||
|
|
||||||
|
|
||||||
|
def _listed(listing: _runner.Listing) -> int:
|
||||||
|
"""The listings a read found: the ids it kept and the rows it dropped."""
|
||||||
|
return len(listing.ids) + listing.dropped
|
||||||
|
|
||||||
|
|
||||||
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
||||||
@@ -462,7 +470,7 @@ class ATSSource(Source):
|
|||||||
[compound(i) for i in listing.ids])
|
[compound(i) for i in listing.ids])
|
||||||
self.state.pop(failing, None)
|
self.state.pop(failing, None)
|
||||||
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
||||||
keep_size(self.state, target, len(listing.ids), reset=False)
|
keep_size(self.state, target, _listed(listing), reset=False)
|
||||||
if detail_sets is not None:
|
if detail_sets is not None:
|
||||||
# A complete listing drops the ids it no longer lists.
|
# A complete listing drops the ids it no longer lists.
|
||||||
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ ends the read, incomplete; `_runner.MAX_PAGES` pages a pull at most,
|
|||||||
|
|
||||||
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
|
**Rows.** A `Row` (`_contract.Row`) is a dict with `source_id` (the host's
|
||||||
own id for the role, unique on the board), `title`, `url` (http or https,
|
own id for the role, unique on the board), `title`, `url` (http or https,
|
||||||
checked with `_shared.is_web_url`) and `places`; `location` (the host's
|
which the runner checks with `_shared.is_web_url`) and `places`; `location` (the host's
|
||||||
one-line place, kept while storage reads it), `posted` (an aware datetime
|
one-line place, kept while storage reads it), `posted` (an aware datetime
|
||||||
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
|
or None, see `_shared.parse_dt`), `department`, `description` (plain text,
|
||||||
see `_shared.html_text`), `remote` (True, False, a word or None) and
|
see `_shared.html_text`), `remote` (True, False, a word or None) and
|
||||||
@@ -187,7 +187,12 @@ see `_shared.html_text`), `remote` (True, False, a word or None) and
|
|||||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`;
|
||||||
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
where it states them, `country` (ISO 3166 alpha-2), `seniority` and
|
||||||
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
|
`employment`; `teaser`; and, on a search row, `company`. The runner keeps
|
||||||
every stated key in `raw`, so a new field needs no change there.
|
every stated key in `raw`, so a new field needs no change there. A reader
|
||||||
|
never drops a listing: one it cannot make a whole row of (no id, no title or
|
||||||
|
no link) it hands over with what it has, an empty string for what is
|
||||||
|
missing, and the runner drops it and counts it (`_runner.valid`). A reader
|
||||||
|
leaves out only what is not a listing of the board: an entry that is not an
|
||||||
|
object, a second copy of an id, a role of another board.
|
||||||
|
|
||||||
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
**Places.** A role has a list of places: `places` is `[Place(text=...,
|
||||||
remote=...)]`, one entry per place the host states, in the host's order, in
|
remote=...)]`, one entry per place the host states, in the host's order, in
|
||||||
|
|||||||
@@ -205,6 +205,8 @@ class Row(TypedDict, total=False):
|
|||||||
`company` is the employer's name on a search row, which lists many
|
`company` is the employer's name on a search row, which lists many
|
||||||
(`plugins/search.py` stores it); a board row carries none, the board is
|
(`plugins/search.py` stores it); a board row carries none, the board is
|
||||||
the company. Any other stated key is kept in `raw` by the runner.
|
the company. Any other stated key is kept in `raw` by the runner.
|
||||||
|
`source_id`, `title` and `url` come empty from a listing that lacks
|
||||||
|
them: the runner drops such a row and counts it (`_runner.valid`).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
source_id: Required[str]
|
source_id: Required[str]
|
||||||
|
|||||||
+98
-20
@@ -4,10 +4,12 @@ the reader declares in `READS`.
|
|||||||
A reader says how its host answers; this module reads it: the guarded fetch
|
A reader says how its host answers; this module reads it: the guarded fetch
|
||||||
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
||||||
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
||||||
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
series per `one`-filter value unioned by id (`read_listing`), the rows it
|
||||||
the sweep of an oversized paged board across pulls, the 10% guard on every
|
keeps (`valid`: a row without an id, a title or a link is dropped and
|
||||||
read that is not a sweep (`read`), the per-role detail reads
|
counted here, never by the reader), completeness, the sweep of an oversized
|
||||||
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
paged board across pulls, the two 10% guards (`read`: on what a read that
|
||||||
|
is not a sweep listed, and on what any read dropped), the per-role detail
|
||||||
|
reads (`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
||||||
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
||||||
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
||||||
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
||||||
@@ -30,6 +32,8 @@ State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
|||||||
drifts when roles close between pulls, so `app.jobs.pull` closes
|
drifts when roles close between pulls, so `app.jobs.pull` closes
|
||||||
a role of such a board only when two finished sweeps in a row
|
a role of such a board only when two finished sweeps in a row
|
||||||
missed it (it keeps the ids the last one missed in `missing`)
|
missed it (it keeps the ids the last one missed in `missing`)
|
||||||
|
dropped the rows the last stamped read dropped (`Listing.dropped`), as a
|
||||||
|
number, "" for none; `app.jobs.limits` names it on the board's row
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -52,7 +56,7 @@ import httpx
|
|||||||
from desk.sources import USER_AGENT
|
from desk.sources import USER_AGENT
|
||||||
|
|
||||||
from ._contract import Ask, Key, Place, Reads
|
from ._contract import Ask, Key, Place, Reads
|
||||||
from ._shared import Get, Row
|
from ._shared import Get, Row, is_web_url
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -96,7 +100,8 @@ SUSPECT_SHARE = 0.1
|
|||||||
#: The last count from which the guard holds every read. A read that is
|
#: The last count from which the guard holds every read. A read that is
|
||||||
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
|
||||||
#: a count of 1, as it was before every read was; a whole read of a board
|
#: a count of 1, as it was before every read was; a whole read of a board
|
||||||
#: that listed a handful of roles can drop to none by itself.
|
#: that listed a handful of roles can drop to none by itself. It is also the
|
||||||
|
#: rows a read must hand over before the share it dropped can fail it.
|
||||||
SUSPECT_MIN = 10
|
SUSPECT_MIN = 10
|
||||||
|
|
||||||
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
||||||
@@ -113,8 +118,8 @@ class TooLarge(ValueError):
|
|||||||
|
|
||||||
|
|
||||||
class Suspect(Exception):
|
class Suspect(Exception):
|
||||||
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
|
"""A read that listed under `SUSPECT_SHARE` of what it listed last time,
|
||||||
(see `read`)."""
|
or dropped more than that share of its rows (see `read`)."""
|
||||||
|
|
||||||
|
|
||||||
class MissingKey(ValueError):
|
class MissingKey(ValueError):
|
||||||
@@ -147,7 +152,9 @@ class Listing:
|
|||||||
and roles closing meanwhile may have shifted an offset-paged host's later
|
and roles closing meanwhile may have shifted an offset-paged host's later
|
||||||
roles back past the cursor, so `ids` can miss a role that is still open.
|
roles back past the cursor, so `ids` can miss a role that is still open.
|
||||||
`remote_unasked` says a filtered read was meant to ask for remote roles
|
`remote_unasked` says a filtered read was meant to ask for remote roles
|
||||||
as well and could not, so its `ids` say nothing of the remote roles."""
|
as well and could not, so its `ids` say nothing of the remote roles.
|
||||||
|
`handed` counts the rows the reader handed over this pull, `dropped`
|
||||||
|
those of them `valid` refused, which are in neither `rows` nor `ids`."""
|
||||||
|
|
||||||
rows: list[Row]
|
rows: list[Row]
|
||||||
complete: bool
|
complete: bool
|
||||||
@@ -158,6 +165,8 @@ class Listing:
|
|||||||
narrowed: bool = False
|
narrowed: bool = False
|
||||||
swept: bool = False
|
swept: bool = False
|
||||||
remote_unasked: bool = False
|
remote_unasked: bool = False
|
||||||
|
handed: int = 0
|
||||||
|
dropped: int = 0
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- modules
|
# --------------------------------------------------------------- modules
|
||||||
@@ -524,8 +533,42 @@ def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
|
|||||||
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||||
|
|
||||||
|
|
||||||
|
def valid(row: Mapping[str, Any]) -> bool:
|
||||||
|
"""Whether the runner keeps a row: a non-empty `source_id`, a non-empty
|
||||||
|
`title` and a web URL (`is_web_url`). A reader hands over every listing
|
||||||
|
as far as it can make it out; a row without one of the three cannot be
|
||||||
|
stored or opened, and the runner drops it and counts it."""
|
||||||
|
try:
|
||||||
|
link = is_web_url(str(row.get("url") or ""))
|
||||||
|
except (httpx.InvalidURL, ValueError): # a string httpx cannot read as a URL
|
||||||
|
link = False
|
||||||
|
return (link and bool(str(row.get("source_id") or "").strip())
|
||||||
|
and bool(str(row.get("title") or "").strip()))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class _Tally:
|
||||||
|
"""The rows a read was handed and dropped, across its series and pages."""
|
||||||
|
|
||||||
|
handed: int = 0
|
||||||
|
dropped: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def _keep(found: Iterable[Row], rows: dict[str, Row], tally: _Tally) -> None:
|
||||||
|
"""The rows a reader's `parse` or `rows` returned, into `rows` by
|
||||||
|
`source_id`, the first of an id kept; those `valid` refuses are counted
|
||||||
|
in `tally` and left out. The one place a row is checked, for the whole
|
||||||
|
and the paged listing alike."""
|
||||||
|
for row in found:
|
||||||
|
tally.handed += 1
|
||||||
|
if not valid(row):
|
||||||
|
tally.dropped += 1
|
||||||
|
continue
|
||||||
|
rows.setdefault(row["source_id"], row)
|
||||||
|
|
||||||
|
|
||||||
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
||||||
limits: Limits) -> dict[str, Row]:
|
limits: Limits, tally: _Tally) -> dict[str, Row]:
|
||||||
"""A whole listing, one request a series, unioned by `source_id`. A body
|
"""A whole listing, one request a series, unioned by `source_id`. A body
|
||||||
over the cap gets one retry at the reader's `fallback_url`."""
|
over the cap gets one retry at the reader's `fallback_url`."""
|
||||||
rows: dict[str, Row] = {}
|
rows: dict[str, Row] = {}
|
||||||
@@ -541,8 +584,7 @@ def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
|||||||
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
||||||
module.NAME, slug)
|
module.NAME, slug)
|
||||||
body = get(module.fallback_url(slug))
|
body = get(module.fallback_url(slug))
|
||||||
for row in module.parse(body, slug):
|
_keep(module.parse(body, slug), rows, tally)
|
||||||
rows.setdefault(row["source_id"], row)
|
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@@ -580,7 +622,29 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
only: a search's places are what the search is, as its query is.
|
only: a search's places are what the search is, as its query is.
|
||||||
`remote` says the reach or the params wanted remote roles; when no series
|
`remote` says the reach or the params wanted remote roles; when no series
|
||||||
asked for them (a reader without a remote filter, or one whose facets
|
asked for them (a reader without a remote filter, or one whose facets
|
||||||
state none beside places) the listing is `remote_unasked`."""
|
state none beside places) the listing is `remote_unasked`.
|
||||||
|
|
||||||
|
Every row `rows` or `parse` returns goes through `valid`: one without a
|
||||||
|
`source_id`, a title or a web URL is dropped here, counted in
|
||||||
|
`Listing.dropped` and logged once for the read. Completeness counts the
|
||||||
|
distinct result ids `page` returns beside its rows, not the rows kept, so
|
||||||
|
a dropped row still counts as read; a whole listing is complete whenever
|
||||||
|
it parses, whatever it dropped."""
|
||||||
|
tally = _Tally()
|
||||||
|
listing = _listing(module, slug, get, ask, limits=limits, sweep=sweep,
|
||||||
|
can_sweep=can_sweep, facets=facets, remote=remote, tally=tally)
|
||||||
|
listing.handed, listing.dropped = tally.handed, tally.dropped
|
||||||
|
if tally.dropped:
|
||||||
|
log.warning("readers: %s:%s: dropped %d of %d listings: no id, title or link",
|
||||||
|
module.NAME, slug, tally.dropped, tally.handed)
|
||||||
|
return listing
|
||||||
|
|
||||||
|
|
||||||
|
def _listing(module: ModuleType, slug: str, get: Get, ask: Ask | None, *,
|
||||||
|
limits: Limits | None, sweep: Mapping[str, Any] | None, can_sweep: bool,
|
||||||
|
facets: Mapping[str, Mapping[str, str]] | None, remote: bool,
|
||||||
|
tally: _Tally) -> Listing:
|
||||||
|
"""`read_listing` before its count of dropped rows, which `tally` keeps."""
|
||||||
limits = limits or Limits()
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
reads: Reads = module.READS
|
reads: Reads = module.READS
|
||||||
@@ -595,7 +659,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
"nothing asked, not closing", module.NAME, slug)
|
"nothing asked, not closing", module.NAME, slug)
|
||||||
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
|
||||||
if not hasattr(module, "page"):
|
if not hasattr(module, "page"):
|
||||||
whole_rows = _whole(module, slug, get, asks, limits)
|
whole_rows = _whole(module, slug, get, asks, limits, tally)
|
||||||
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
|
||||||
key, by_words, remote_unasked=unasked)
|
key, by_words, remote_unasked=unasked)
|
||||||
|
|
||||||
@@ -632,8 +696,7 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
pages += 1
|
pages += 1
|
||||||
before = len(seen)
|
before = len(seen)
|
||||||
seen |= _result_ids(results)
|
seen |= _result_ids(results)
|
||||||
for row in module.rows(data, slug):
|
_keep(module.rows(data, slug), rows, tally)
|
||||||
rows.setdefault(row["source_id"], row)
|
|
||||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||||
ended = True
|
ended = True
|
||||||
break
|
break
|
||||||
@@ -747,13 +810,26 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
|
|||||||
that really shrank is accepted once the target has been failing for
|
that really shrank is accepted once the target has been failing for
|
||||||
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
|
||||||
stamp then resets the count; a Recheck resets it at once
|
stamp then resets the count; a Recheck resets it at once
|
||||||
(`plugins.ats.keep_size`)."""
|
(`plugins.ats.keep_size`).
|
||||||
|
|
||||||
|
A read, a sweep's included, that dropped more than `SUSPECT_SHARE` of
|
||||||
|
the rows its reader handed over (`valid`), once it was handed at least
|
||||||
|
`SUSPECT_MIN`, raises `Suspect` the same way: a page whose shape changed
|
||||||
|
parses to rows without a title or a link, and the roles it hid would
|
||||||
|
close. It gives way after `limits.stale_days` of failing as well."""
|
||||||
limits = limits or Limits()
|
limits = limits or Limits()
|
||||||
ask = ask or Ask()
|
ask = ask or Ask()
|
||||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||||
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
||||||
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
|
||||||
remote=remote)
|
remote=remote)
|
||||||
|
if listing.handed >= SUSPECT_MIN and listing.dropped > SUSPECT_SHARE * listing.handed:
|
||||||
|
why = (f"dropped {listing.dropped} of {listing.handed} listings, over "
|
||||||
|
f"{SUSPECT_SHARE:.0%}: no id, title or link")
|
||||||
|
if not _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
|
||||||
|
raise Suspect(why)
|
||||||
|
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
|
||||||
|
module.NAME, slug, why, limits.stale_days)
|
||||||
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
|
||||||
last = _load(state, f"{prefix}:count:{target}")
|
last = _load(state, f"{prefix}:count:{target}")
|
||||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||||
@@ -788,9 +864,10 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
|||||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||||
against), whether it finished a sweep from earlier pulls (`swept`),
|
against), whether it finished a sweep from earlier pulls (`swept`),
|
||||||
whether a filtered read left remote unasked (`remote_unasked`), the
|
whether a filtered read left remote unasked (`remote_unasked`), the rows
|
||||||
sweep to continue or none, and the count for the 10% guard once a read
|
it dropped (`dropped`, "" for none), the sweep to continue or none, and
|
||||||
is not a sweep. A read that asked nothing keeps the count it had."""
|
the count for the 10% guard once a read is not a sweep. A read that
|
||||||
|
asked nothing keeps the count it had."""
|
||||||
state[f"{prefix}:ok:{target}"] = run
|
state[f"{prefix}:ok:{target}"] = run
|
||||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||||
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
|
||||||
@@ -798,6 +875,7 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
|||||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||||
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
|
||||||
|
state[f"{prefix}:dropped:{target}"] = str(listing.dropped) if listing.dropped else ""
|
||||||
if listing.sweep is not None:
|
if listing.sweep is not None:
|
||||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from urllib.parse import urlencode
|
|||||||
from desk.sources import strip_html
|
from desk.sources import strip_html
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Key, Place, Reads
|
from ._contract import Ask, Fields, Filters, Key, Place, Reads
|
||||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
from ._shared import Get, Page, Row, only, parse_dt
|
||||||
|
|
||||||
NAME = "adzuna"
|
NAME = "adzuna"
|
||||||
LABEL = "Adzuna"
|
LABEL = "Adzuna"
|
||||||
@@ -111,8 +111,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
for job in decoded["results"]:
|
for job in decoded["results"]:
|
||||||
title = _text(job.get("title") or "")
|
title = _text(job.get("title") or "")
|
||||||
url = (job.get("redirect_url") or "").strip()
|
url = (job.get("redirect_url") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
||||||
town = ((job.get("location") or {}).get("display_name") or "").strip()
|
town = ((job.get("location") or {}).get("display_name") or "").strip()
|
||||||
places: list[Place] = [{"text": town, "remote": None}] if town else []
|
places: list[Place] = [{"text": town, "remote": None}] if town else []
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads, Row
|
from ._contract import Ask, Fields, Place, Reads, Row
|
||||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -126,8 +126,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in data.get("jobs") or []:
|
for j in data.get("jobs") or []:
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
||||||
compensation = j.get("compensation") or {}
|
compensation = j.get("compensation") or {}
|
||||||
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
|
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
|
||||||
|
|||||||
@@ -106,8 +106,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in listed:
|
for j in listed:
|
||||||
title = (j.get("jobOpeningName") or "").strip()
|
title = (j.get("jobOpeningName") or "").strip()
|
||||||
source_id = str(j.get("id") or "")
|
source_id = str(j.get("id") or "")
|
||||||
if not title or not source_id:
|
|
||||||
continue
|
|
||||||
places = _places(j)
|
places = _places(j)
|
||||||
row: Row = {
|
row: Row = {
|
||||||
"source_id": source_id,
|
"source_id": source_id,
|
||||||
|
|||||||
@@ -132,8 +132,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for p in postings:
|
for p in postings:
|
||||||
title = (p.get("title") or "").strip()
|
title = (p.get("title") or "").strip()
|
||||||
post_id = str(p.get("id") or "")
|
post_id = str(p.get("id") or "")
|
||||||
if not title or not post_id:
|
|
||||||
continue
|
|
||||||
job = p.get("job") or {}
|
job = p.get("job") or {}
|
||||||
places = [name for loc in job.get("jobLocations") or []
|
places = [name for loc in job.get("jobLocations") or []
|
||||||
if (name := ((loc.get("location") or {}).get("name") or "").strip())]
|
if (name := ((loc.get("location") or {}).get("name") or "").strip())]
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ from urllib.parse import quote
|
|||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads, Row
|
from ._contract import Ask, Fields, Filters, Place, Reads, Row
|
||||||
from ._countries import name_of
|
from ._countries import name_of
|
||||||
from ._shared import Get, Page, add_place, html_text, is_web_url, only, parse_dt, place
|
from ._shared import Get, Page, add_place, html_text, only, parse_dt, place
|
||||||
|
|
||||||
READS = Reads(
|
READS = Reads(
|
||||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||||
@@ -151,8 +151,6 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
|
|||||||
source_id = str(p.get("id") or "")
|
source_id = str(p.get("id") or "")
|
||||||
path = str(p.get("positionUrl") or "").strip()
|
path = str(p.get("positionUrl") or "").strip()
|
||||||
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
|
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
|
||||||
if not title or not source_id or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
places = _places(p.get("locations") or p.get("standardizedLocations"))
|
places = _places(p.get("locations") or p.get("standardizedLocations"))
|
||||||
out.append({
|
out.append({
|
||||||
"source_id": source_id,
|
"source_id": source_id,
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ from urllib.parse import urlencode, urlsplit, urlunsplit
|
|||||||
from desk.sources import USER_AGENT
|
from desk.sources import USER_AGENT
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||||
from ._shared import (Get, Page, Row, html_text, is_web_url, num, only, parse_dt, pay_line,
|
from ._shared import (Get, Page, Row, html_text, num, only, parse_dt, pay_line,
|
||||||
yearly_amount)
|
yearly_amount)
|
||||||
|
|
||||||
NAME = "freehire"
|
NAME = "freehire"
|
||||||
@@ -170,8 +170,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
title = " ".join(html.unescape(str(hit.get("title") or "")).split())
|
title = " ".join(html.unescape(str(hit.get("title") or "")).split())
|
||||||
source_id = str(hit.get("public_slug") or "").strip()
|
source_id = str(hit.get("public_slug") or "").strip()
|
||||||
url = _url(str(hit.get("url") or ""))
|
url = _url(str(hit.get("url") or ""))
|
||||||
if not title or not source_id or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
mode = str(hit.get("work_mode") or "").strip()
|
mode = str(hit.get("work_mode") or "").strip()
|
||||||
places = _places(hit, mode)
|
places = _places(hit, mode)
|
||||||
countries = [str(c) for c in hit.get("countries") or [] if c]
|
countries = [str(c) for c in hit.get("countries") or [] if c]
|
||||||
|
|||||||
@@ -116,8 +116,6 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
|||||||
for p in postings:
|
for p in postings:
|
||||||
title = (p.get("title") or "").strip()
|
title = (p.get("title") or "").strip()
|
||||||
ext_id = str(p.get("extId") or "")
|
ext_id = str(p.get("extId") or "")
|
||||||
if not title or not ext_id:
|
|
||||||
continue
|
|
||||||
locations = p.get("locations") or []
|
locations = p.get("locations") or []
|
||||||
places: list[Place] = []
|
places: list[Place] = []
|
||||||
for loc in locations:
|
for loc in locations:
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads, Row
|
from ._contract import Ask, Fields, Place, Reads, Row
|
||||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place
|
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place
|
||||||
|
|
||||||
NAME = "greenhouse"
|
NAME = "greenhouse"
|
||||||
LABEL = "Greenhouse"
|
LABEL = "Greenhouse"
|
||||||
@@ -149,8 +149,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in data.get("jobs") or []:
|
for j in data.get("jobs") or []:
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
url = (j.get("absolute_url") or "").strip()
|
url = (j.get("absolute_url") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
departments = j.get("departments") or []
|
departments = j.get("departments") or []
|
||||||
ranges = j.get("pay_input_ranges") or []
|
ranges = j.get("pay_input_ranges") or []
|
||||||
salary_min, salary_max, currency = _salary(ranges)
|
salary_min, salary_max, currency = _salary(ranges)
|
||||||
|
|||||||
@@ -90,8 +90,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for ad in data.get("jobAdDetails") or []:
|
for ad in data.get("jobAdDetails") or []:
|
||||||
title = (ad.get("title") or "").strip()
|
title = (ad.get("title") or "").strip()
|
||||||
ad_id = str(ad.get("id") or "").strip()
|
ad_id = str(ad.get("id") or "").strip()
|
||||||
if not title or not ad_id:
|
|
||||||
continue
|
|
||||||
site = str(ad.get("site") or "").strip()
|
site = str(ad.get("site") or "").strip()
|
||||||
country = str(ad.get("country") or "").strip()
|
country = str(ad.get("country") or "").strip()
|
||||||
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
|
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Reads, Row
|
from ._contract import Ask, Fields, Reads, Row
|
||||||
from ._shared import Get, Page, html_text, is_web_url, only, parse_dt
|
from ._shared import Get, Page, html_text, only, parse_dt
|
||||||
|
|
||||||
READS = Reads(
|
READS = Reads(
|
||||||
scope="board", listing="paged", page_size=100, text="listing", complete="end",
|
scope="board", listing="paged", page_size=100, text="listing", complete="end",
|
||||||
@@ -106,8 +106,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
|||||||
for item in items:
|
for item in items:
|
||||||
title = (item.findtext("title") or "").strip()
|
title = (item.findtext("title") or "").strip()
|
||||||
url = (item.findtext("link") or "").strip()
|
url = (item.findtext("link") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
out.append({
|
out.append({
|
||||||
"source_id": (item.findtext("guid") or url).strip(),
|
"source_id": (item.findtext("guid") or url).strip(),
|
||||||
"title": title,
|
"title": title,
|
||||||
|
|||||||
@@ -127,8 +127,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
j = job.get("data") or {}
|
j = job.get("data") or {}
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
role = str(j.get("slug") or j.get("req_id") or "")
|
role = str(j.get("slug") or j.get("req_id") or "")
|
||||||
if not title or not role:
|
|
||||||
continue
|
|
||||||
categories = j.get("categories") or []
|
categories = j.get("categories") or []
|
||||||
text = "\n".join(p for p in (html_text(j.get("description")),
|
text = "\n".join(p for p in (html_text(j.get("description")),
|
||||||
html_text(j.get("qualifications"))) if p)
|
html_text(j.get("qualifications"))) if p)
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from typing import Any
|
|||||||
from urllib.parse import urlencode, urlsplit
|
from urllib.parse import urlencode, urlsplit
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt, place
|
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt, place
|
||||||
|
|
||||||
NAME = "jobbank"
|
NAME = "jobbank"
|
||||||
LABEL = "Akademikernes Jobbank (Denmark)"
|
LABEL = "Akademikernes Jobbank (Denmark)"
|
||||||
@@ -132,8 +132,8 @@ def _query(ask: Ask, number: int) -> tuple[str, str]:
|
|||||||
|
|
||||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||||
"""One page of a search; the cursor is the page number. The next page
|
"""One page of a search; the cursor is the page number. The next page
|
||||||
follows a full page (counted before any item is dropped, so a dropped item
|
follows a full page (counted before the runner drops an item, so a
|
||||||
does not pass for the end) while the ceiling is not reached."""
|
dropped item does not pass for the end) while the ceiling is not reached."""
|
||||||
number = int(cursor or 1)
|
number = int(cursor or 1)
|
||||||
url, remote = _query(ask, number)
|
url, remote = _query(ask, number)
|
||||||
try:
|
try:
|
||||||
@@ -184,8 +184,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
out: list[Row] = []
|
out: list[Row] = []
|
||||||
for item in decoded["items"]:
|
for item in decoded["items"]:
|
||||||
link, title = item["link"], item["title"]
|
link, title = item["link"], item["title"]
|
||||||
if not title or not item["id"] or not is_web_url(link):
|
|
||||||
continue
|
|
||||||
line = _LINE.fullmatch(item["description"])
|
line = _LINE.fullmatch(item["description"])
|
||||||
kind = line.group("kind").strip() if line else ""
|
kind = line.group("kind").strip() if line else ""
|
||||||
company, regions, cut = _regions(line.group("rest")) if line else ("", [], False)
|
company, regions, cut = _regions(line.group("rest")) if line else ("", [], False)
|
||||||
|
|||||||
@@ -117,9 +117,9 @@ def _query(ask: Ask, number: int) -> str:
|
|||||||
|
|
||||||
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
||||||
"""One page of a search, newest first; the cursor is the page number.
|
"""One page of a search, newest first; the cursor is the page number.
|
||||||
The next page follows a full page (counted before any item is dropped, so
|
The next page follows a full page (counted before the runner drops an
|
||||||
a dropped item does not pass for the end) while the 200-result ceiling is
|
item, so a dropped item does not pass for the end) while the 200-result
|
||||||
not reached."""
|
ceiling is not reached."""
|
||||||
number = int(cursor or 1)
|
number = int(cursor or 1)
|
||||||
root = ET.fromstring(get(_query(ask, number)))
|
root = ET.fromstring(get(_query(ask, number)))
|
||||||
channel = root.find("channel")
|
channel = root.find("channel")
|
||||||
@@ -143,8 +143,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
out: list[Row] = []
|
out: list[Row] = []
|
||||||
for item in decoded["items"]:
|
for item in decoded["items"]:
|
||||||
link, full_title = item["link"], item["title"]
|
link, full_title = item["link"], item["title"]
|
||||||
if not full_title or not is_web_url(link):
|
|
||||||
continue
|
|
||||||
description = item["description"]
|
description = item["description"]
|
||||||
h4 = _H4.search(description)
|
h4 = _H4.search(description)
|
||||||
title = _text(h4.group(1)) if h4 else full_title
|
title = _text(h4.group(1)) if h4 else full_title
|
||||||
|
|||||||
@@ -138,8 +138,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
for ad in decoded.get("jobAds") or []:
|
for ad in decoded.get("jobAds") or []:
|
||||||
ident = str(ad.get("jobAdId") or "").strip()
|
ident = str(ad.get("jobAdId") or "").strip()
|
||||||
title = _text(ad.get("title"))
|
title = _text(ad.get("title"))
|
||||||
if not ident or not title:
|
|
||||||
continue
|
|
||||||
link = str(ad.get("jobAdUrl") or "").strip()
|
link = str(ad.get("jobAdUrl") or "").strip()
|
||||||
town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
|
town = _text(ad.get("municipality") or ad.get("postalDistrictName"))
|
||||||
places: list[Place] = [place(town)] if town else []
|
places: list[Place] = [place(town)] if town else []
|
||||||
|
|||||||
@@ -176,8 +176,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
job_id = str(j.get("id") or "").strip()
|
job_id = str(j.get("id") or "").strip()
|
||||||
path = str(j.get("url") or "").strip()
|
path = str(j.get("url") or "").strip()
|
||||||
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
|
|
||||||
continue
|
|
||||||
function = str(j.get("function") or "").strip()
|
function = str(j.get("function") or "").strip()
|
||||||
places = [place(name) for name in dict.fromkeys(
|
places = [place(name) for name in dict.fromkeys(
|
||||||
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
|
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
|
||||||
@@ -186,7 +184,7 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
"title": title,
|
"title": title,
|
||||||
"places": places,
|
"places": places,
|
||||||
"location": "; ".join(pl["text"] for pl in places),
|
"location": "; ".join(pl["text"] for pl in places),
|
||||||
"url": f"https://{_EMP}{path}",
|
"url": f"https://{_EMP}{path}" if _ROLE_PATH.fullmatch(path) else "",
|
||||||
"posted": _published(j.get("published_date")),
|
"posted": _published(j.get("published_date")),
|
||||||
"department": "" if function == "All functions" else function,
|
"department": "" if function == "All functions" else function,
|
||||||
"description": "",
|
"description": "",
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads, Row
|
from ._contract import Ask, Fields, Place, Reads, Row
|
||||||
from ._shared import add_place, html_text, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
from ._shared import add_place, html_text, num, only, parse_dt, path_slug_ok, pay_line, place, yearly_amount
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -123,8 +123,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in data:
|
for j in data:
|
||||||
title = (j.get("text") or "").strip()
|
title = (j.get("text") or "").strip()
|
||||||
url = (j.get("hostedUrl") or "").strip()
|
url = (j.get("hostedUrl") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
categories = j.get("categories") or {}
|
categories = j.get("categories") or {}
|
||||||
description = j.get("description") or j.get("descriptionPlain") or ""
|
description = j.get("description") or j.get("descriptionPlain") or ""
|
||||||
salary_range = j.get("salaryRange") or {}
|
salary_range = j.get("salaryRange") or {}
|
||||||
|
|||||||
@@ -198,9 +198,9 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for role in search.get("requisitionList") or []:
|
for role in search.get("requisitionList") or []:
|
||||||
role_id = str(role.get("Id") or "").strip()
|
role_id = str(role.get("Id") or "").strip()
|
||||||
title = _text(role.get("Title"))
|
title = _text(role.get("Title"))
|
||||||
# The id goes into a URL, and a listing is external content.
|
# The id goes into a URL, and a listing is external content: an id
|
||||||
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id):
|
# of other characters gets no link, and the runner drops the row.
|
||||||
continue
|
safe = bool(re.fullmatch(r"[A-Za-z0-9_-]+", role_id))
|
||||||
remote = _text(role.get("WorkplaceTypeCode")) or None
|
remote = _text(role.get("WorkplaceTypeCode")) or None
|
||||||
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
|
at_office = (remote or "").lower() in ("ora_hybrid", "ora_on_site")
|
||||||
places: list[Place] = []
|
places: list[Place] = []
|
||||||
@@ -214,7 +214,7 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
"title": title,
|
"title": title,
|
||||||
"places": places,
|
"places": places,
|
||||||
"location": "; ".join(p["text"] for p in places),
|
"location": "; ".join(p["text"] for p in places),
|
||||||
"url": f"{_base(slug)}/job/{role_id}",
|
"url": f"{_base(slug)}/job/{role_id}" if safe else "",
|
||||||
"posted": parse_dt(role.get("PostedDate")),
|
"posted": parse_dt(role.get("PostedDate")),
|
||||||
"department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction"))
|
"department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction"))
|
||||||
or _text(role.get("Department")) or _text(role.get("Organization"))),
|
or _text(role.get("Department")) or _text(role.get("Organization"))),
|
||||||
|
|||||||
@@ -104,8 +104,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for position in root.findall("position"):
|
for position in root.findall("position"):
|
||||||
job_id = (position.findtext("id") or "").strip()
|
job_id = (position.findtext("id") or "").strip()
|
||||||
title = (position.findtext("name") or "").strip()
|
title = (position.findtext("name") or "").strip()
|
||||||
if not job_id or not title:
|
|
||||||
continue
|
|
||||||
description = " ".join(
|
description = " ".join(
|
||||||
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
||||||
salary_info = position.find("salaryInformation")
|
salary_info = position.find("salaryInformation")
|
||||||
|
|||||||
@@ -168,8 +168,6 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for job in search.get("data", {}).get("jobs") or []:
|
for job in search.get("data", {}).get("jobs") or []:
|
||||||
key = _key(job)
|
key = _key(job)
|
||||||
title = _text(job.get("title"))
|
title = _text(job.get("title"))
|
||||||
if not title or not key:
|
|
||||||
continue
|
|
||||||
places: list[Place] = []
|
places: list[Place] = []
|
||||||
for text in (job.get("location"), *(job.get("multi_location") or [])):
|
for text in (job.get("location"), *(job.get("multi_location") or [])):
|
||||||
if isinstance(text, str):
|
if isinstance(text, str):
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Reads
|
from ._contract import Ask, Fields, Reads
|
||||||
from ._shared import HEADING, Row, html_text, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
|
from ._shared import HEADING, Row, html_text, label_slug_ok, num, only, pay_line, yearly_amount
|
||||||
|
|
||||||
NAME = "pinpoint"
|
NAME = "pinpoint"
|
||||||
LABEL = "Pinpoint"
|
LABEL = "Pinpoint"
|
||||||
@@ -121,8 +121,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
url = (j.get("url") or "").strip()
|
url = (j.get("url") or "").strip()
|
||||||
source_id = str(j.get("id") or "")
|
source_id = str(j.get("id") or "")
|
||||||
if not title or not source_id or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
job = j.get("job") or {}
|
job = j.get("job") or {}
|
||||||
low, high, currency, salary_text = _pay(j)
|
low, high, currency, salary_text = _pay(j)
|
||||||
workplace = str(j.get("workplace_type") or "").strip()
|
workplace = str(j.get("workplace_type") or "").strip()
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from urllib.parse import urlencode
|
|||||||
from desk.sources import strip_html
|
from desk.sources import strip_html
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||||
from ._shared import Get, Page, Row, html_text, is_web_url, only, parse_dt
|
from ._shared import Get, Page, Row, html_text, only, parse_dt
|
||||||
|
|
||||||
NAME = "platsbanken"
|
NAME = "platsbanken"
|
||||||
LABEL = "Platsbanken"
|
LABEL = "Platsbanken"
|
||||||
@@ -108,8 +108,6 @@ def rows(decoded: Any, slug: str) -> list[Row]:
|
|||||||
for hit in decoded.get("hits") or []:
|
for hit in decoded.get("hits") or []:
|
||||||
title = _text(hit.get("headline") or "")
|
title = _text(hit.get("headline") or "")
|
||||||
url = (hit.get("webpage_url") or "").strip()
|
url = (hit.get("webpage_url") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
|
addresses = hit.get("workplace_addresses") or [hit.get("workplace_address") or {}]
|
||||||
places = [place for address in addresses if isinstance(address, dict)
|
places = [place for address in addresses if isinstance(address, dict)
|
||||||
if (place := _place(address))]
|
if (place := _place(address))]
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads
|
from ._contract import Ask, Fields, Place, Reads
|
||||||
from ._shared import Row, add_place, html_text, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
from ._shared import Row, add_place, html_text, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||||
|
|
||||||
NAME = "recruitee"
|
NAME = "recruitee"
|
||||||
LABEL = "Recruitee"
|
LABEL = "Recruitee"
|
||||||
@@ -105,8 +105,6 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
for j in data.get("offers") or []:
|
for j in data.get("offers") or []:
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
url = (j.get("careers_url") or "").strip()
|
url = (j.get("careers_url") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
||||||
salary_min, salary_max, currency = yearly_amount(
|
salary_min, salary_max, currency = yearly_amount(
|
||||||
salary.get("min"), salary.get("max"), salary.get("currency"),
|
salary.get("min"), salary.get("max"), salary.get("currency"),
|
||||||
|
|||||||
+14
-10
@@ -82,27 +82,31 @@ def parse(body: bytes, slug: str) -> list[Row]:
|
|||||||
raise ValueError("not a Rippling board: the answer is not a list")
|
raise ValueError("not a Rippling board: the answer is not a list")
|
||||||
roles: dict[str, Row] = {}
|
roles: dict[str, Row] = {}
|
||||||
places: dict[str, list[Place]] = {}
|
places: dict[str, list[Place]] = {}
|
||||||
for entry in data:
|
for index, entry in enumerate(data):
|
||||||
job_id = str(entry.get("uuid") or "").strip()
|
job_id = str(entry.get("uuid") or "").strip()
|
||||||
title = (entry.get("name") or "").strip()
|
title = (entry.get("name") or "").strip()
|
||||||
if not title or not _UUID.fullmatch(job_id):
|
# The id goes into a URL: one of other characters gets no link, and
|
||||||
continue
|
# the runner drops the row. An entry without an id is a role of its
|
||||||
|
# own, not merged with the others that lack one.
|
||||||
|
key = job_id or f"#{index}"
|
||||||
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
|
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
|
||||||
places.setdefault(job_id, [])
|
places.setdefault(key, [])
|
||||||
if place:
|
if place:
|
||||||
add_place(places[job_id], _place(place))
|
add_place(places[key], _place(place))
|
||||||
roles.setdefault(job_id, {
|
roles.setdefault(key, {
|
||||||
"source_id": job_id,
|
"source_id": job_id,
|
||||||
"title": title,
|
"title": title,
|
||||||
"url": f"https://{_SITE}/{slug}/jobs/{job_id}",
|
"url": f"https://{_SITE}/{slug}/jobs/{job_id}" if _UUID.fullmatch(job_id) else "",
|
||||||
"posted": None,
|
"posted": None,
|
||||||
"department": str(((entry.get("department") or {}).get("label")) or "").strip(),
|
"department": str(((entry.get("department") or {}).get("label")) or "").strip(),
|
||||||
"description": "",
|
"description": "",
|
||||||
"salary_text": "", # the board's list states no pay
|
"salary_text": "", # the board's list states no pay
|
||||||
})
|
})
|
||||||
for job_id, row in roles.items():
|
if title and not roles[key]["title"]:
|
||||||
row["places"] = places[job_id]
|
roles[key]["title"] = title # an entry of the role that has one
|
||||||
row["location"] = "; ".join(p["text"] for p in places[job_id])
|
for key, row in roles.items():
|
||||||
|
row["places"] = places[key]
|
||||||
|
row["location"] = "; ".join(p["text"] for p in places[key])
|
||||||
return list(roles.values())
|
return list(roles.values())
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -115,8 +115,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for j in data.get("content") or []:
|
for j in data.get("content") or []:
|
||||||
title = (j.get("name") or "").strip()
|
title = (j.get("name") or "").strip()
|
||||||
job_id = str(j.get("id") or "")
|
job_id = str(j.get("id") or "")
|
||||||
if not title or not job_id:
|
|
||||||
continue
|
|
||||||
location = j.get("location") or {}
|
location = j.get("location") or {}
|
||||||
loc_text = (location.get("fullLocation")
|
loc_text = (location.get("fullLocation")
|
||||||
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ from desk.sources import strip_html
|
|||||||
|
|
||||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||||
from ._countries import NAMES
|
from ._countries import NAMES
|
||||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
|
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
|
||||||
|
|
||||||
NAME = "successfactors"
|
NAME = "successfactors"
|
||||||
LABEL = "SuccessFactors"
|
LABEL = "SuccessFactors"
|
||||||
@@ -247,8 +247,9 @@ def _tile_fields(chunk: str) -> tuple[str, str, str]:
|
|||||||
def _entries(text: str) -> list[dict[str, Any]]:
|
def _entries(text: str) -> list[dict[str, Any]]:
|
||||||
"""The roles of one search page, in the order shown, whichever layout it is
|
"""The roles of one search page, in the order shown, whichever layout it is
|
||||||
in and before the prefix filter: `id`, `href` (as written, relative),
|
in and before the prefix filter: `id`, `href` (as written, relative),
|
||||||
`title`, `place`, `date` and `department`. A role is listed once, and a row
|
`title`, `place`, `date` and `department`. A role is listed once. The id
|
||||||
without a link to a role is not one."""
|
and the title are in the role's link, so a row without one is handed over
|
||||||
|
with an empty `id`, `href` and `title`, for the runner to drop and count."""
|
||||||
table = _ROW.split(text)[1:]
|
table = _ROW.split(text)[1:]
|
||||||
chunks = table or _TILE.split(text)[1:]
|
chunks = table or _TILE.split(text)[1:]
|
||||||
out: list[dict[str, Any]] = []
|
out: list[dict[str, Any]] = []
|
||||||
@@ -256,10 +257,12 @@ def _entries(text: str) -> list[dict[str, Any]]:
|
|||||||
for chunk in chunks:
|
for chunk in chunks:
|
||||||
if table:
|
if table:
|
||||||
chunk = chunk.split("</tr>", 1)[0]
|
chunk = chunk.split("</tr>", 1)[0]
|
||||||
if not (link := _LINK.search(chunk)):
|
if link := _LINK.search(chunk):
|
||||||
continue
|
href, job_id, title = (html.unescape(link.group(1)), link.group(2),
|
||||||
href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3))
|
_text(link.group(3)))
|
||||||
if job_id in seen:
|
else:
|
||||||
|
href = job_id = title = ""
|
||||||
|
if job_id and job_id in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(job_id)
|
seen.add(job_id)
|
||||||
place, date, department = (_table_fields if table else _tile_fields)(chunk)
|
place, date, department = (_table_fields if table else _tile_fields)(chunk)
|
||||||
@@ -283,12 +286,14 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
|||||||
if not _SEARCH_PAGE.search(text):
|
if not _SEARCH_PAGE.search(text):
|
||||||
raise ValueError("not a SuccessFactors search page")
|
raise ValueError("not a SuccessFactors search page")
|
||||||
found = _entries(text)
|
found = _entries(text)
|
||||||
|
ids = [entry["id"] for entry in found if entry["id"]]
|
||||||
total = _total(text)
|
total = _total(text)
|
||||||
if found and found[0]["id"] == previous:
|
first = ids[0] if ids else None
|
||||||
|
if first is not None and first == previous:
|
||||||
return text, [], total, (start, previous)
|
return text, [], total, (start, previous)
|
||||||
more = bool(found) and (total is None or start + len(found) < total)
|
more = bool(found) and (total is None or start + len(found) < total)
|
||||||
return (text, [{"id": entry["id"]} for entry in found], total,
|
return (text, [{"id": job_id} for job_id in ids], total,
|
||||||
(start + len(found), found[0]["id"]) if more else None)
|
(start + len(found), first) if more else None)
|
||||||
|
|
||||||
|
|
||||||
def rows(decoded: str, slug: str) -> list[Row]:
|
def rows(decoded: str, slug: str) -> list[Row]:
|
||||||
@@ -296,17 +301,21 @@ def rows(decoded: str, slug: str) -> list[Row]:
|
|||||||
slug's prefix when it has one; the id in it is the requisition's `source_id`.
|
slug's prefix when it has one; the id in it is the requisition's `source_id`.
|
||||||
The table and the tile layout give a title, a location, a date and, where
|
The table and the tile layout give a title, a location, a date and, where
|
||||||
the site has the column, a department; nothing else about a role is on
|
the site has the column, a department; nothing else about a role is on
|
||||||
the list, so its description and pay are empty."""
|
the list, so its description and pay are empty.
|
||||||
|
|
||||||
|
A role linked on another host, or outside the prefix, is another board's
|
||||||
|
(another site, or another brand of a shared site), and is left out: it is
|
||||||
|
no role of this company. A row with no link or no title is handed over,
|
||||||
|
and the runner drops it."""
|
||||||
host, prefix = _parts(slug)
|
host, prefix = _parts(slug)
|
||||||
base = f"https://{host}/"
|
base = f"https://{host}/"
|
||||||
out = []
|
out = []
|
||||||
for entry in _entries(decoded):
|
for entry in _entries(decoded):
|
||||||
url = urljoin(base, entry["href"])
|
url = urljoin(base, entry["href"]) if entry["href"] else ""
|
||||||
if urlsplit(url).hostname != host or not is_web_url(url):
|
if url and urlsplit(url).hostname != host:
|
||||||
continue
|
continue
|
||||||
if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"):
|
if url and prefix and not urlsplit(url).path.lower().startswith(
|
||||||
continue
|
f"{prefix.lower()}/job/"):
|
||||||
if not entry["title"]:
|
|
||||||
continue
|
continue
|
||||||
out.append({
|
out.append({
|
||||||
"source_id": entry["id"],
|
"source_id": entry["id"],
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from typing import Any
|
|||||||
|
|
||||||
|
|
||||||
from ._contract import Ask, Fields, Place, Reads
|
from ._contract import Ask, Fields, Place, Reads
|
||||||
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
|
from ._shared import Get, Page, Row, add_place, html_text, only, parse_dt
|
||||||
|
|
||||||
NAME = "teamtailor"
|
NAME = "teamtailor"
|
||||||
LABEL = "Teamtailor"
|
LABEL = "Teamtailor"
|
||||||
@@ -115,8 +115,6 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
|||||||
for item in items:
|
for item in items:
|
||||||
title = (item.findtext("title") or "").strip()
|
title = (item.findtext("title") or "").strip()
|
||||||
url = (item.findtext("link") or "").strip()
|
url = (item.findtext("link") or "").strip()
|
||||||
if not title or not is_web_url(url):
|
|
||||||
continue
|
|
||||||
status = (item.findtext("remoteStatus") or "").strip()
|
status = (item.findtext("remoteStatus") or "").strip()
|
||||||
remote = {"fully": True, "hybrid": False, "onsite": False}.get(status)
|
remote = {"fully": True, "hybrid": False, "onsite": False}.get(status)
|
||||||
places: list[Place] = []
|
places: list[Place] = []
|
||||||
|
|||||||
@@ -102,8 +102,6 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
for j in data.get("results") or []:
|
for j in data.get("results") or []:
|
||||||
title = (j.get("title") or "").strip()
|
title = (j.get("title") or "").strip()
|
||||||
shortcode = (j.get("shortcode") or "").strip()
|
shortcode = (j.get("shortcode") or "").strip()
|
||||||
if not title or not shortcode:
|
|
||||||
continue
|
|
||||||
workplace = str(j.get("workplace") or "").strip()
|
workplace = str(j.get("workplace") or "").strip()
|
||||||
remote = bool(j.get("remote")) or workplace == "remote"
|
remote = bool(j.get("remote")) or workplace == "remote"
|
||||||
places = _places(j, remote)
|
places = _places(j, remote)
|
||||||
|
|||||||
@@ -124,8 +124,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
|||||||
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
|
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
|
||||||
postings = data.get("jobPostings") or []
|
postings = data.get("jobPostings") or []
|
||||||
# Every posting is counted, a path-less one (LEGO lists one that has only
|
# Every posting is counted, a path-less one (LEGO lists one that has only
|
||||||
# `bulletFields`) by its position, so the parser dropping it does not
|
# `bulletFields`) by its position, so the runner dropping its row does
|
||||||
# leave the board short of its total.
|
# not leave the board short of its total.
|
||||||
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
|
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
|
||||||
following = offset + len(postings)
|
following = offset + len(postings)
|
||||||
more = bool(postings) and (total is None or following < total)
|
more = bool(postings) and (total is None or following < total)
|
||||||
@@ -274,13 +274,13 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
`bulletFields`, which lead with the requisition id, and on some tenants
|
`bulletFields`, which lead with the requisition id, and on some tenants
|
||||||
`remoteType`. No description, no date and no country: `detail` has
|
`remoteType`. No description, no date and no country: `detail` has
|
||||||
those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the
|
those, and the places of a role that "2 Locations" hides. The id is the requisition id where the path carries it, and the
|
||||||
path where not; a posting with no title or no path is skipped."""
|
path where not. A posting with no path has neither an id nor a link, and
|
||||||
|
the runner drops it."""
|
||||||
out = []
|
out = []
|
||||||
for p in decoded.get("jobPostings") or []:
|
for p in decoded.get("jobPostings") or []:
|
||||||
title = (p.get("title") or "").strip()
|
title = (p.get("title") or "").strip()
|
||||||
path = (p.get("externalPath") or "").strip()
|
path = (p.get("externalPath") or "").strip()
|
||||||
if not title or not _ROLE_PATH.fullmatch(path):
|
path = path if _ROLE_PATH.fullmatch(path) else ""
|
||||||
continue
|
|
||||||
remote = _remote(p.get("remoteType"))
|
remote = _remote(p.get("remoteType"))
|
||||||
place = _place(p.get("locationsText"), remote)
|
place = _place(p.get("locationsText"), remote)
|
||||||
out.append({
|
out.append({
|
||||||
@@ -288,7 +288,7 @@ def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
|||||||
"title": title,
|
"title": title,
|
||||||
"places": [place] if place else [],
|
"places": [place] if place else [],
|
||||||
"location": place["text"] if place else "",
|
"location": place["text"] if place else "",
|
||||||
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}", # type: ignore[index]
|
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}" if path else "", # type: ignore[index]
|
||||||
"posted": _posted(p.get("postedOn") or ""),
|
"posted": _posted(p.get("postedOn") or ""),
|
||||||
"department": "",
|
"department": "",
|
||||||
"description": "",
|
"description": "",
|
||||||
|
|||||||
+2
-1
@@ -19,7 +19,8 @@ roles close once no search has seen them for `STALE_DAYS` (see
|
|||||||
`app/jobs/pull.py`).
|
`app/jobs/pull.py`).
|
||||||
|
|
||||||
In a pull a search is read by `_runner.read` under the prefix `search`, as a
|
In a pull a search is read by `_runner.read` under the prefix `search`, as a
|
||||||
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
|
board is under `ats`: the runner drops a row without an id, a title or a
|
||||||
|
link, the 10% guards hold it, `_runner.stamp` records it,
|
||||||
and a search that fails or is suspect is kept as failing under
|
and a search that fails or is suspect is kept as failing under
|
||||||
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
|
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
|
||||||
the settle reads as it reads a failing board's.
|
the settle reads as it reads a failing board's.
|
||||||
|
|||||||
+15
-8
@@ -24,18 +24,24 @@ from plugins.readers import _runner, smartrecruiters # noqa: E402
|
|||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
|
|
||||||
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
def _parsed(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||||
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
||||||
body = path.read_bytes()
|
return ats._PARSERS[host](path.read_bytes(), slug)
|
||||||
return ats._PARSERS[host](body, slug)
|
|
||||||
|
|
||||||
|
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||||
|
"""The rows of a host's fixture the runner keeps (`_runner.valid`)."""
|
||||||
|
return [row for row in _parsed(host, slug) if _runner.valid(row)]
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------ hosts
|
# ------------------------------------------------------------------ hosts
|
||||||
|
|
||||||
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
||||||
|
# The javascript: and data: rows are handed over and the runner drops
|
||||||
|
# them: a feed is external content, never trusted as a URL.
|
||||||
|
assert [r["source_id"] for r in _parsed("greenhouse")] == [
|
||||||
|
"1001", "1002", "1003", "1004", "1005", "1006"]
|
||||||
rows = _rows("greenhouse")
|
rows = _rows("greenhouse")
|
||||||
# The javascript: and data: rows are refused, same as the four real ones
|
|
||||||
# are kept: a feed is external content, never trusted as a URL.
|
|
||||||
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
||||||
first = rows[0]
|
first = rows[0]
|
||||||
assert first["title"] == "Senior Backend Engineer" # trimmed
|
assert first["title"] == "Senior Backend Engineer" # trimmed
|
||||||
@@ -269,10 +275,11 @@ TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
|
|||||||
|
|
||||||
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
||||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||||
# the javascript: link is refused like on the other hosts
|
# the javascript: link is handed over and refused by the runner
|
||||||
assert [r["source_id"] for r in rows] == [
|
assert [r["source_id"] for r in rows] == [
|
||||||
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
|
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
|
||||||
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"]
|
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"]
|
||||||
|
assert [_runner.valid(r) for r in rows] == [True, True, True, False]
|
||||||
first = rows[0]
|
first = rows[0]
|
||||||
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
||||||
assert first["department"] == "Engineering"
|
assert first["department"] == "Engineering"
|
||||||
@@ -294,7 +301,7 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
|
|||||||
|
|
||||||
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
||||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||||
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
|
assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them
|
||||||
assert rows[1]["title"] == "Läkare" # trimmed
|
assert rows[1]["title"] == "Läkare" # trimmed
|
||||||
assert rows[1]["department"] == "Software Development"
|
assert rows[1]["department"] == "Software Development"
|
||||||
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, bamboohr # noqa: E402
|
from plugins.readers import Ask, _runner, bamboohr # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -84,11 +84,16 @@ def test_a_page_that_is_not_a_listing_raises_and_an_empty_board_is_no_rows() ->
|
|||||||
bamboohr.parse(body, "configit")
|
bamboohr.parse(body, "configit")
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_without_a_title_or_an_id_is_left_out() -> None:
|
def test_a_role_without_a_title_or_an_id_reaches_the_runner_and_is_dropped_there() -> None:
|
||||||
body = json.loads(_read("bamboohr.json"))
|
body = json.loads(_read("bamboohr.json"))
|
||||||
body["result"][0]["jobOpeningName"] = " "
|
body["result"][0]["jobOpeningName"] = " "
|
||||||
body["result"][1]["id"] = None
|
body["result"][1]["id"] = None
|
||||||
assert bamboohr.parse(json.dumps(body).encode(), "configit") == []
|
raw = json.dumps(body).encode()
|
||||||
|
rows = bamboohr.parse(raw, "configit")
|
||||||
|
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||||
|
listing = _runner.read_listing(bamboohr, "configit", lambda url, **kw: raw)
|
||||||
|
assert (listing.rows, listing.ids, listing.complete) == ([], [], True)
|
||||||
|
assert (listing.handed, listing.dropped) == (2, 2)
|
||||||
|
|
||||||
|
|
||||||
def test_detail_reads_the_text_the_date_and_the_country() -> None:
|
def test_detail_reads_the_text_the_date_and_the_country() -> None:
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, deel # noqa: E402
|
from plugins.readers import Ask, _runner, deel # noqa: E402
|
||||||
|
|
||||||
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
|
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
|
||||||
URL = "https://jobs.deel.com/klarna"
|
URL = "https://jobs.deel.com/klarna"
|
||||||
@@ -89,10 +89,12 @@ def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board
|
|||||||
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
|
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
|
||||||
|
|
||||||
|
|
||||||
def test_a_posting_without_a_title_or_id_is_skipped() -> None:
|
def test_a_posting_without_a_title_or_id_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
|
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
|
||||||
{"id": "c", "title": "Cook", "job": {}}]}
|
{"id": "c", "title": "Cook", "job": {}}]}
|
||||||
assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"]
|
rows = deel.parse(_page_with(body), "acme")
|
||||||
|
assert [r["source_id"] for r in rows] == ["a", "", "c"]
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ the fetch
|
# ------------------------------------------------------------ the fetch
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, eightfold # noqa: E402
|
from plugins.readers import Ask, _runner, eightfold # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SLUG = "jobs.ericsson.com/ericsson.com"
|
SLUG = "jobs.ericsson.com/ericsson.com"
|
||||||
@@ -93,10 +93,12 @@ def test_the_work_location_option_is_the_hosts_word(
|
|||||||
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
|
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
|
||||||
|
|
||||||
|
|
||||||
def test_a_position_without_a_name_an_id_or_a_path_is_left_out() -> None:
|
def test_a_position_without_a_name_an_id_or_a_path_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
|
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
|
||||||
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
|
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
|
||||||
assert eightfold.rows(bad, SLUG) == []
|
rows = eightfold.rows(bad, SLUG)
|
||||||
|
assert len(rows) == 4 and not any(map(_runner.valid, rows))
|
||||||
|
assert [r["url"] for r in rows[2:]] == ["", ""] # no link built off the host
|
||||||
|
|
||||||
|
|
||||||
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
|
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, gem # noqa: E402
|
from plugins.readers import Ask, _runner, gem # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
API = "https://jobs.gem.com/api/public/graphql"
|
API = "https://jobs.gem.com/api/public/graphql"
|
||||||
@@ -79,8 +79,9 @@ def test_remote_is_the_location_type_then_the_locations(
|
|||||||
assert gem.rows([posting], "acme")[0]["remote"] == remote
|
assert gem.rows([posting], "acme")[0]["remote"] == remote
|
||||||
|
|
||||||
|
|
||||||
def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None:
|
def test_a_posting_without_a_title_or_an_ext_id_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == []
|
rows = gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme")
|
||||||
|
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_the_board_is_one_post_without_origin_or_referer() -> None:
|
def test_the_board_is_one_post_without_origin_or_referer() -> None:
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, hibob # noqa: E402
|
from plugins.readers import Ask, _runner, hibob # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "hibob.json").read_bytes()
|
BOARD = (FIXTURES / "hibob.json").read_bytes()
|
||||||
@@ -87,10 +87,12 @@ def test_the_place_leaves_the_country_off_where_the_site_names_it() -> None:
|
|||||||
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
|
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
|
||||||
|
|
||||||
|
|
||||||
def test_an_ad_without_a_title_or_id_is_skipped_and_an_empty_board_is_empty() -> None:
|
def test_an_ad_without_a_title_or_id_is_handed_over_and_an_empty_board_is_empty() -> None:
|
||||||
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
|
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
|
||||||
{"id": "c", "title": "Cook"}]}).encode()
|
{"id": "c", "title": "Cook"}]}).encode()
|
||||||
assert [r["source_id"] for r in hibob.parse(body, "acme")] == ["c"]
|
rows = hibob.parse(body, "acme")
|
||||||
|
assert [r["source_id"] for r in rows] == ["a", "", "c"]
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
|
||||||
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
|
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, hrmanager # noqa: E402
|
from plugins.readers import Ask, _runner, hrmanager # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
||||||
@@ -52,12 +52,14 @@ def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_f
|
|||||||
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
|
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
|
||||||
|
|
||||||
|
|
||||||
def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None:
|
def test_an_item_without_a_title_or_a_web_link_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
|
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
|
||||||
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
|
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
|
||||||
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
|
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
|
||||||
b"</channel></rss>")
|
b"</channel></rss>")
|
||||||
assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"]
|
rows = hrmanager.parse(body, "x")
|
||||||
|
assert [r["source_id"] for r in rows] == ["1", "2", "3"]
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["3"]
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- paging
|
# ------------------------------------------------------------- paging
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, jibe # noqa: E402
|
from plugins.readers import Ask, _runner, jibe # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SITE = "jobs.booking.com"
|
SITE = "jobs.booking.com"
|
||||||
@@ -72,9 +72,10 @@ def test_the_location_falls_back_to_city_and_country() -> None:
|
|||||||
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
|
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
|
def test_a_role_without_a_title_or_a_slug_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
|
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
|
||||||
assert jibe.rows({"jobs": jobs}, SITE) == []
|
rows = jibe.rows({"jobs": jobs}, SITE)
|
||||||
|
assert len(rows) == 3 and not any(map(_runner.valid, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import Ask, jobylon # noqa: E402
|
from plugins.readers import Ask, _runner, jobylon # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
||||||
@@ -65,12 +65,14 @@ def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
|
|||||||
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
|
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None:
|
def test_a_role_whose_link_is_not_a_role_page_gets_no_link_and_is_dropped() -> None:
|
||||||
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
|
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
|
||||||
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
|
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
|
||||||
b"{id: '3', title: 'C', url: '/companies/3/'},"
|
b"{id: '3', title: 'C', url: '/companies/3/'},"
|
||||||
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
|
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
|
||||||
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"]
|
rows = jobylon.parse(body, "9")
|
||||||
|
assert [r["url"] for r in rows[:3]] == ["", "", ""]
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["4"]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- detail
|
# ---------------------------------------------------------------- detail
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.readers import oracle # noqa: E402
|
from plugins.readers import _runner, oracle # noqa: E402
|
||||||
from plugins.readers._shared import is_web_url # noqa: E402
|
from plugins.readers._shared import is_web_url # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
@@ -84,14 +84,15 @@ def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
|
|||||||
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
|
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_left_out() -> None:
|
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_dropped_by_the_runner() -> None:
|
||||||
data = _listing()
|
data = _listing()
|
||||||
roles = data["items"][0]["requisitionList"]
|
roles = data["items"][0]["requisitionList"]
|
||||||
roles[0]["Title"] = " "
|
roles[0]["Title"] = " "
|
||||||
roles[1]["Id"] = "1/../../x"
|
roles[1]["Id"] = "1/../../x"
|
||||||
roles[2]["Id"] = ""
|
roles[2]["Id"] = ""
|
||||||
assert [r["source_id"] for r in oracle.parse(json.dumps(data).encode(), SLUG)] == \
|
rows = oracle.parse(json.dumps(data).encode(), SLUG)
|
||||||
["40975", "31680"]
|
assert rows[1]["url"] == "" # the id builds no link
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["40975", "31680"]
|
||||||
|
|
||||||
|
|
||||||
def test_a_listing_without_the_expand_has_no_roles() -> None:
|
def test_a_listing_without_the_expand_has_no_roles() -> None:
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.readers import phenom # noqa: E402
|
from plugins.readers import _runner, phenom # noqa: E402
|
||||||
from plugins.readers._shared import is_web_url # noqa: E402
|
from plugins.readers._shared import is_web_url # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
@@ -101,13 +101,15 @@ def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None
|
|||||||
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
|
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
|
||||||
|
|
||||||
|
|
||||||
def test_a_role_without_title_or_id_is_left_out() -> None:
|
def test_a_role_without_title_or_id_is_dropped_by_the_runner() -> None:
|
||||||
data = _answer()
|
data = _answer()
|
||||||
jobs = data["refineSearch"]["data"]["jobs"]
|
jobs = data["refineSearch"]["data"]["jobs"]
|
||||||
jobs[0]["title"] = ""
|
jobs[0]["title"] = ""
|
||||||
for key in ("jobSeqNo", "jobId", "reqId"):
|
for key in ("jobSeqNo", "jobId", "reqId"):
|
||||||
jobs[1].pop(key, None)
|
jobs[1].pop(key, None)
|
||||||
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
|
rows = phenom.parse(json.dumps(data).encode(), SLUG)
|
||||||
|
assert len(rows) == len(IDS)
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == IDS[2:]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------- the request
|
# ---------------------------------------------------------- the request
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import pinpoint # noqa: E402
|
from plugins.readers import _runner, pinpoint # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -86,11 +86,12 @@ def test_an_unknown_pay_frequency_keeps_the_text_and_leaves_the_columns() -> Non
|
|||||||
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
|
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
|
||||||
|
|
||||||
|
|
||||||
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_left_out() -> None:
|
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_dropped_by_the_runner() -> None:
|
||||||
body = json.loads(_body())
|
body = json.loads(_body())
|
||||||
body["data"][0]["title"] = ""
|
body["data"][0]["title"] = ""
|
||||||
body["data"][1]["url"] = "javascript:alert(1)"
|
body["data"][1]["url"] = "javascript:alert(1)"
|
||||||
assert pinpoint.parse(json.dumps(body).encode(), "xeneta") == []
|
rows = pinpoint.parse(json.dumps(body).encode(), "xeneta")
|
||||||
|
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
|
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import rippling # noqa: E402
|
from plugins.readers import _runner, rippling # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "rippling.json").read_bytes()
|
BOARD = (FIXTURES / "rippling.json").read_bytes()
|
||||||
@@ -61,11 +61,14 @@ def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None
|
|||||||
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
|
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
|
||||||
|
|
||||||
|
|
||||||
def test_an_entry_without_a_title_or_a_plain_id_is_skipped() -> None:
|
def test_an_entry_without_a_title_or_a_plain_id_is_dropped_by_the_runner() -> None:
|
||||||
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
|
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
|
||||||
{"uuid": "../../x", "name": "C"},
|
{"uuid": "../../x", "name": "C"},
|
||||||
{"uuid": "abcdefgh-2", "name": "D"}]).encode()
|
{"uuid": "abcdefgh-2", "name": "D"}]).encode()
|
||||||
assert [r["source_id"] for r in rippling.parse(body, "acme")] == ["abcdefgh-2"]
|
rows = rippling.parse(body, "acme")
|
||||||
|
assert [r["source_id"] for r in rows] == ["abcdefgh-1", "", "../../x", "abcdefgh-2"]
|
||||||
|
assert rows[2]["url"] == "" # the id builds no link
|
||||||
|
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["abcdefgh-2"]
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- detail
|
# ---------------------------------------------------------------- detail
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
|||||||
if str(ROOT) not in sys.path:
|
if str(ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins.readers import Ask, teamtailor # noqa: E402
|
from plugins.readers import Ask, _runner, teamtailor # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -40,7 +40,8 @@ def test_page_takes_the_ask_and_reads_the_same_url() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
|
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
|
||||||
first, second, third = _rows()
|
first, second, third, refused = _rows()
|
||||||
|
assert not _runner.valid(refused) # javascript:, for the runner
|
||||||
# "Sverige" (a city that is a country) once, Stockholm's twin once.
|
# "Sverige" (a city that is a country) once, Stockholm's twin once.
|
||||||
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
|
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
|
||||||
{"text": "Stockholm, Sweden", "remote": False}]
|
{"text": "Stockholm, Sweden", "remote": False}]
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.readers import workday # noqa: E402
|
from plugins.readers import _runner, workday # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SLUG = "zendesk.wd1/zendesk"
|
SLUG = "zendesk.wd1/zendesk"
|
||||||
@@ -96,7 +96,7 @@ def test_posted_is_read_only_where_the_text_is_exact(text: str, expected: dateti
|
|||||||
assert workday._posted(text) == expected
|
assert workday._posted(text) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
|
def test_a_posting_without_a_title_or_a_path_reaches_the_runner_and_is_dropped() -> None:
|
||||||
page = {"jobPostings": [
|
page = {"jobPostings": [
|
||||||
{"bulletFields": ["0000036369"]}, # LEGO's, no title
|
{"bulletFields": ["0000036369"]}, # LEGO's, no title
|
||||||
{"title": "No path", "bulletFields": ["R1"]},
|
{"title": "No path", "bulletFields": ["R1"]},
|
||||||
@@ -105,7 +105,16 @@ def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
|
|||||||
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
|
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
|
||||||
"postedOn": "Posted Today", "bulletFields": ["R4"]},
|
"postedOn": "Posted Today", "bulletFields": ["R4"]},
|
||||||
]}
|
]}
|
||||||
assert [r["title"] for r in workday.rows(page, SLUG)] == ["Whole"]
|
rows = workday.rows(page, SLUG)
|
||||||
|
assert len(rows) == 5
|
||||||
|
assert [(r["source_id"], r["url"]) for r in rows[:2]] == [("", ""), ("", "")]
|
||||||
|
assert [r["title"] for r in rows if _runner.valid(r)] == ["Whole"]
|
||||||
|
# Through the runner: four dropped and counted, and the read complete,
|
||||||
|
# since every posting counts by its position towards the stated total.
|
||||||
|
body = json.dumps(page | {"total": 5}).encode()
|
||||||
|
listing = _runner.read_listing(workday, SLUG, lambda url, **kw: body)
|
||||||
|
assert [r["title"] for r in listing.rows] == ["Whole"]
|
||||||
|
assert (listing.handed, listing.dropped, listing.complete) == (5, 4, True)
|
||||||
|
|
||||||
|
|
||||||
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
|
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
|
||||||
|
|||||||
@@ -116,6 +116,16 @@ def test_roles_without_text_and_text_reads_that_failed() -> None:
|
|||||||
assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
|
assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("dropped", "found"), [
|
||||||
|
("3", ["3 listings dropped: no title or link"]),
|
||||||
|
("1", ["1 listing dropped: no title or link"]),
|
||||||
|
("", []), ("x", [])])
|
||||||
|
def test_the_listings_the_last_read_dropped_are_named(dropped: str, found: list[str]) -> None:
|
||||||
|
"""The rows the runner dropped (`_runner.valid`), after "read incomplete"."""
|
||||||
|
assert words(fake(), state(dropped=dropped)) == found
|
||||||
|
assert words(fake(), state(dropped=dropped, complete=""))[1:] == found
|
||||||
|
|
||||||
|
|
||||||
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
|
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
|
||||||
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
|
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
|
||||||
assert words(fake(), state(), reach.SMALL) == []
|
assert words(fake(), state(), reach.SMALL) == []
|
||||||
@@ -199,6 +209,17 @@ def test_companies_carry_their_limits_and_the_boards_own_open_roles(
|
|||||||
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
|
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_that_dropped_listings_says_so_on_its_row(
|
||||||
|
client: TestClient, store: Store) -> None:
|
||||||
|
store.add_subscription("ats", "lever:acme", company="Acme", board="lever")
|
||||||
|
store.add_company("Acme", key="acme", ats="lever")
|
||||||
|
store.state["ats:ok:lever:acme"] = "run"
|
||||||
|
store.state["ats:complete:lever:acme"] = "1"
|
||||||
|
store.state["ats:dropped:lever:acme"] = "3"
|
||||||
|
[row] = client.get("/api/companies").json()
|
||||||
|
assert row["limits"][0] == "3 listings dropped: no title or link"
|
||||||
|
|
||||||
|
|
||||||
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
|
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
|
||||||
store.add_company("Novo", key="novo", ats="taleo")
|
store.add_company("Novo", key="novo", ats="taleo")
|
||||||
[row] = client.get("/api/companies").json()
|
[row] = client.get("/api/companies").json()
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, adzuna # noqa: E402
|
from plugins.readers import Ask, _runner, adzuna # noqa: E402
|
||||||
|
|
||||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
|
||||||
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
|
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
|
||||||
@@ -39,8 +39,9 @@ def test_it_declares_the_key_and_states_the_robots_finding() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
|
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
|
||||||
first, predicted = _rows() # javascript: refused
|
first, predicted, refused = _rows()
|
||||||
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
|
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
|
||||||
|
assert refused["url"].startswith("javascript:") and not _runner.valid(refused)
|
||||||
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
||||||
assert first["teaser"] == "Build our backend services in Python & Go."
|
assert first["teaser"] == "Build our backend services in Python & Go."
|
||||||
assert "description" not in first
|
assert "description" not in first
|
||||||
@@ -89,3 +90,12 @@ def test_saved_params_need_a_known_country() -> None:
|
|||||||
"q": "x", "country": "gb"}
|
"q": "x", "country": "gb"}
|
||||||
with pytest.raises(ValueError, match="country"):
|
with pytest.raises(ValueError, match="country"):
|
||||||
adzuna.clean_params({"q": "x", "country": "dk"})
|
adzuna.clean_params({"q": "x", "country": "dk"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_runner_drops_the_ad_without_a_web_link_and_counts_it() -> None:
|
||||||
|
"""The reader hands the `javascript:` ad over; the runner drops it, and
|
||||||
|
the read is complete, since all three result ids were read."""
|
||||||
|
listing = _runner.read_listing(adzuna, "", lambda url, **kw: BODY, ASK,
|
||||||
|
limits=_runner.Limits(page_delay=0))
|
||||||
|
assert listing.ids == ["4412345678", "4412345679"]
|
||||||
|
assert (listing.handed, listing.dropped, listing.complete) == (3, 1, True)
|
||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, freehire # noqa: E402
|
from plugins.readers import Ask, _runner, freehire # noqa: E402
|
||||||
|
|
||||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
|
||||||
|
|
||||||
@@ -119,14 +119,17 @@ def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None:
|
|||||||
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
|
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
|
||||||
|
|
||||||
|
|
||||||
def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None:
|
def test_a_hit_without_a_title_an_id_or_a_web_link_is_handed_to_the_runner() -> None:
|
||||||
|
"""The reader hands every hit over; the runner keeps only the whole one."""
|
||||||
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
|
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
|
||||||
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
|
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
|
||||||
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
|
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
|
||||||
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
|
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
|
||||||
found = freehire.rows({"data": hits}, "")
|
found = freehire.rows({"data": hits}, "")
|
||||||
assert [row["source_id"] for row in found] == ["d"]
|
assert [row["source_id"] for row in found] == ["a", "", "c", "d"]
|
||||||
assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == []
|
kept = [row for row in found if _runner.valid(row)]
|
||||||
|
assert [row["source_id"] for row in kept] == ["d"]
|
||||||
|
assert kept[0]["url"] == "https://x.example/d" and kept[0]["places"] == []
|
||||||
|
|
||||||
|
|
||||||
def test_a_title_and_a_company_have_their_entities_decoded() -> None:
|
def test_a_title_and_a_company_have_their_entities_decoded() -> None:
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, jobbank # noqa: E402
|
from plugins.readers import Ask, _runner, jobbank # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||||
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
|
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
|
||||||
@@ -170,7 +170,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
|||||||
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||||
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
|
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
|
||||||
assert len(results) == 100 and nxt == 2
|
assert len(results) == 100 and nxt == 2
|
||||||
assert len(jobbank.rows(decoded, "")) == 99
|
rows = jobbank.rows(decoded, "")
|
||||||
|
assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
|
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, jobindex # noqa: E402
|
from plugins.readers import Ask, _runner, jobindex # noqa: E402
|
||||||
|
|
||||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
|
||||||
|
|
||||||
@@ -89,7 +89,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
|||||||
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||||
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
|
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
|
||||||
assert len(results) == 20 and nxt == 2
|
assert len(results) == 20 and nxt == 2
|
||||||
assert len(jobindex.rows(decoded, "")) == 19
|
rows = jobindex.rows(decoded, "")
|
||||||
|
assert len(rows) == 20 and sum(map(_runner.valid, rows)) == 19
|
||||||
|
|
||||||
|
|
||||||
def test_an_answer_without_a_channel_raises() -> None:
|
def test_an_answer_without_a_channel_raises() -> None:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, jobnet # noqa: E402
|
from plugins.readers import Ask, _runner, jobnet # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||||
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
|
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
|
||||||
@@ -100,9 +100,12 @@ def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None:
|
|||||||
assert "applicationDeadline" not in row
|
assert "applicationDeadline" not in row
|
||||||
|
|
||||||
|
|
||||||
def test_an_ad_without_an_id_or_a_title_is_skipped() -> None:
|
def test_an_ad_without_an_id_or_a_title_is_handed_over_for_the_runner_to_drop() -> None:
|
||||||
ads = json.loads(OTHER)["jobAds"]
|
ads = json.loads(OTHER)["jobAds"]
|
||||||
assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == []
|
rows = jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "")
|
||||||
|
assert [(row["source_id"] == "", row["title"] == "") for row in rows] == [
|
||||||
|
(False, True), (True, False)]
|
||||||
|
assert not any(map(_runner.valid, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
|
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import readers # noqa: E402
|
from plugins import readers # noqa: E402
|
||||||
from plugins.readers import Ask, platsbanken # noqa: E402
|
from plugins.readers import Ask, _runner, platsbanken # noqa: E402
|
||||||
|
|
||||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
|
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
|
||||||
|
|
||||||
@@ -47,8 +47,9 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
|
|||||||
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||||
rows = platsbanken.rows(decoded, "")
|
rows = platsbanken.rows(decoded, "")
|
||||||
assert total == 32
|
assert total == 32
|
||||||
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400001", "31400002"]
|
||||||
first, no_town, bare = rows
|
assert not _runner.valid(rows[2]) # javascript:, dropped by the runner
|
||||||
|
first, no_town, _refused, bare = rows
|
||||||
assert [p["text"] for p in first["places"]] == [
|
assert [p["text"] for p in first["places"]] == [
|
||||||
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
|
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
|
||||||
assert first["location"] == "Lund"
|
assert first["location"] == "Lund"
|
||||||
@@ -58,7 +59,7 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
|
|||||||
|
|
||||||
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
||||||
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||||
first, _no_town, remote = platsbanken.rows(decoded, "")
|
first, _no_town, _refused, remote = platsbanken.rows(decoded, "")
|
||||||
assert first["title"] == "Postdoctoral Researcher"
|
assert first["title"] == "Postdoctoral Researcher"
|
||||||
assert first["company"] == "European Spallation Source Eric"
|
assert first["company"] == "European Spallation Source Eric"
|
||||||
assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
|
assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
|
||||||
|
|||||||
+107
-2
@@ -1,7 +1,7 @@
|
|||||||
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
||||||
subscription resolves to, series and their union, whole and paged listings,
|
subscription resolves to, series and their union, whole and paged listings,
|
||||||
completeness from the declaration, the sweep across pulls, the 10% guard,
|
completeness from the declaration, the rows kept and dropped, the sweep
|
||||||
the facets kept in state, the detail reads and the `raw` an item carries. No
|
across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||||
network: `get` is a dict of URL -> body or exception."""
|
network: `get` is a dict of URL -> body or exception."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
|||||||
assert filtered.ids == ["1"] and filtered.filtered
|
assert filtered.ids == ["1"] and filtered.filtered
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------- the rows kept
|
||||||
|
|
||||||
|
def _handed(i: str) -> dict[str, Any]:
|
||||||
|
"""A row as a reader hands it over: whole, or, for an id of `-id`,
|
||||||
|
`-title`, `-link` or `-url`, without that value or with a link that is
|
||||||
|
no web URL (`javascript:`, or one httpx cannot read)."""
|
||||||
|
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
|
||||||
|
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
|
||||||
|
return _row(i, **broken.get(i.split(":")[0], {}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
|
||||||
|
assert _runner.valid(_handed("1"))
|
||||||
|
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
|
||||||
|
assert not _runner.valid({"source_id": "1", "title": "Role"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
|
||||||
|
caplog: pytest.LogCaptureFixture) -> None:
|
||||||
|
module = _whole()
|
||||||
|
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||||
|
web = Web({"https://jobs.fake.example/acme/list":
|
||||||
|
b'["1", "-id", "2", "-title", "-link", "-url"]'})
|
||||||
|
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||||
|
listing = _runner.read_listing(module, "acme", web)
|
||||||
|
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
|
||||||
|
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||||
|
assert (listing.handed, listing.dropped) == (6, 4)
|
||||||
|
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
|
||||||
|
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
|
||||||
|
caplog: pytest.LogCaptureFixture) -> None:
|
||||||
|
"""Completeness counts the result ids `page` returns, not the rows kept,
|
||||||
|
so the dropped rows still count as read; the drop is logged once."""
|
||||||
|
pages = [["1", "-title:a"], ["2", "-link:b"]]
|
||||||
|
module = _paged(pages, total=4)
|
||||||
|
module.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||||
|
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||||
|
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||||
|
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||||
|
assert (listing.handed, listing.dropped) == (4, 2)
|
||||||
|
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_stamp_keeps_the_rows_dropped() -> None:
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
|
||||||
|
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
|
||||||
|
assert state["ats:dropped:t"] == "2"
|
||||||
|
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
|
||||||
|
assert state["ats:dropped:t"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
|
||||||
|
"""A whole board whose feed hands over `handed` rows, `dropped` of them
|
||||||
|
without a title."""
|
||||||
|
module = _whole()
|
||||||
|
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||||
|
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
|
||||||
|
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
|
||||||
|
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
|
||||||
|
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
|
||||||
|
handed: int, dropped: int, fails: bool) -> None:
|
||||||
|
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
|
||||||
|
a page whose shape changed, which must close nothing."""
|
||||||
|
module, web = _dropping(handed, dropped)
|
||||||
|
if not fails:
|
||||||
|
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
|
||||||
|
return
|
||||||
|
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
|
||||||
|
_runner.read({}, "ats", "t", module, "acme", web)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
|
||||||
|
module, web = _dropping(20, 5)
|
||||||
|
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
|
||||||
|
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
|
||||||
|
listing = _runner.read(state, "ats", "t", module, "acme", web,
|
||||||
|
limits=_runner.Limits(page_delay=0, stale_days=21))
|
||||||
|
assert (len(listing.ids), listing.dropped) == (15, 5)
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------ through ATSSource
|
# ------------------------------------------------------ through ATSSource
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
|
|||||||
assert state["ats:filtered:fake:acme"] == "1"
|
assert state["ats:filtered:fake:acme"] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
|
||||||
|
board.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||||
|
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||||
|
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
|
||||||
|
board.READS = board.READS.model_copy(update={"complete": "end"})
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
assert len(list(_source(state).fetch())) == 19
|
||||||
|
assert state["ats:dropped:fake:acme"] == "1"
|
||||||
|
|
||||||
|
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||||
|
json.dumps([str(n) for n in range(15)]
|
||||||
|
+ [f"-id:{n}" for n in range(5)]).encode()}
|
||||||
|
state = {}
|
||||||
|
assert list(_source(state).fetch()) == []
|
||||||
|
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
|
||||||
|
assert "ats:ok:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
# -------------------------------------------------------------------- raw
|
# -------------------------------------------------------------------- raw
|
||||||
|
|
||||||
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
||||||
|
|||||||
@@ -383,6 +383,23 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch)
|
|||||||
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count(
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""The search driver reads through the same listing code as a board: the
|
||||||
|
runner drops the row the provider handed over without a title, and the
|
||||||
|
read is still complete, by the distinct result ids."""
|
||||||
|
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
||||||
|
module = readers.PROVIDERS["fake"]
|
||||||
|
handed = module.rows
|
||||||
|
module.rows = lambda decoded, slug: [
|
||||||
|
row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)]
|
||||||
|
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
|
||||||
|
"params": {"q": "x"}}]})
|
||||||
|
assert [i.source_id for i in source.fetch()] == ["fake:h1"]
|
||||||
|
assert source.state["search:complete:fake:cph"] == "1"
|
||||||
|
assert source.state["search:dropped:fake:cph"] == "1"
|
||||||
|
|
||||||
|
|
||||||
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
||||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||||
|
|||||||
Reference in new issue
Block a user