Rename plugins/atshosts to plugins/readers
Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
af337ef558
commit
73dec9028e
48 files changed
+104
-104
No files matched your search
+19
-19
@@ -1,5 +1,5 @@
|
||||
"""ATS boards: a company's own careers page on one of the hosts in
|
||||
`plugins/atshosts/`, one module per host.
|
||||
`plugins/readers/`, one module per host.
|
||||
|
||||
Every one of these hosts serves a company's own list of roles as public,
|
||||
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
||||
@@ -49,7 +49,7 @@ from desk.sources import (
|
||||
)
|
||||
|
||||
try:
|
||||
from plugins import atshosts
|
||||
from plugins import readers
|
||||
except ModuleNotFoundError:
|
||||
# `desk.sources.registry` loads this file by its path, under a name of its
|
||||
# own, and the app puts the repo root on `sys.path` before it does; a
|
||||
@@ -57,16 +57,16 @@ except ModuleNotFoundError:
|
||||
# does not, and the host modules are found as the package beside this
|
||||
# file. So the root goes on the path then, and only then.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plugins import atshosts
|
||||
from plugins.atshosts._shared import Get, Row
|
||||
from plugins import readers
|
||||
from plugins.readers._shared import Get, Row
|
||||
|
||||
# The names the parsers had here before they moved to their host modules,
|
||||
# kept for the callers and the tests that still reach for them on `ats`.
|
||||
from plugins.atshosts._shared import ( # noqa: F401
|
||||
from plugins.readers._shared import ( # noqa: F401
|
||||
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
|
||||
)
|
||||
from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
||||
from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401
|
||||
from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
||||
from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -101,7 +101,7 @@ DEADLINE = 60.0
|
||||
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
||||
#: the order a select offers them. What the Sources page's host select is
|
||||
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
||||
HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()}
|
||||
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
|
||||
|
||||
|
||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
@@ -110,7 +110,7 @@ def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
company's own site. Anything else is refused, at the first hop and at
|
||||
every redirect: a feed is external content, and a board row must never
|
||||
be able to point this fetch at an internal address."""
|
||||
module = atshosts.HOSTS.get(host)
|
||||
module = readers.HOSTS.get(host)
|
||||
return module.allowed(slug) if module is not None else lambda h: False
|
||||
|
||||
|
||||
@@ -234,7 +234,7 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
||||
#: host -> parser, for the hosts whose module parses a whole body. Every one
|
||||
#: takes the raw response body and the slug.
|
||||
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
||||
name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")}
|
||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||
|
||||
|
||||
# --------------------------------------------------------- the classifier
|
||||
@@ -365,11 +365,11 @@ def classify(department: str, title: str) -> str:
|
||||
|
||||
|
||||
def is_known_target(target: str) -> bool:
|
||||
"""`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's
|
||||
"""`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's
|
||||
`slug_ok` accepts. What the target validator refuses: `app companies add`
|
||||
and the pull job both call this before touching the network."""
|
||||
host, sep, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
module = readers.HOSTS.get(host)
|
||||
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
|
||||
|
||||
|
||||
@@ -377,7 +377,7 @@ def careers_url(target: str) -> str:
|
||||
"""The company's own careers page for `host:slug`, or "" for an unknown
|
||||
host. What `app companies add` stores alongside the board."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
module = readers.HOSTS.get(host)
|
||||
return module.careers_url(slug) if module is not None else ""
|
||||
|
||||
|
||||
@@ -404,7 +404,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
A 429 after the first page stops the read with what it has, incomplete;
|
||||
one on the first page raises like any other failure."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
rows: dict[str, Row] = {}
|
||||
seen: set[str] = set()
|
||||
@@ -449,7 +449,7 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
module = readers.HOSTS[host]
|
||||
if hasattr(module, "page"):
|
||||
return _paged(client, target)
|
||||
get = _getter(client, host, slug)
|
||||
@@ -566,7 +566,7 @@ def _check_robots(client: httpx.Client, target: str,
|
||||
kept, so the next pull asks again. A 429 counts as unreachable although
|
||||
it is a 4xx, since it says "not now" rather than "there is none"."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
module = readers.HOSTS.get(host)
|
||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||
return
|
||||
page = module.robots_page(slug)
|
||||
@@ -657,7 +657,7 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
|
||||
after a complete listing, so one cut short does not make the next pull
|
||||
read its missing roles again."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
done, failing = set(detailed), set(failed) - set(detailed)
|
||||
|
||||
@@ -804,7 +804,7 @@ class ATSSource(Source):
|
||||
|
||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||
detail_sets: tuple[set[str], set[str]] | None = None
|
||||
if hasattr(atshosts.HOSTS.get(host), "detail"):
|
||||
if hasattr(readers.HOSTS.get(host), "detail"):
|
||||
detailed, failed = (self._id_set(key) for key in detail_keys)
|
||||
rows, detailed, failed = _read_details(client, target, rows, complete,
|
||||
detailed, failed)
|
||||
@@ -833,7 +833,7 @@ class ATSSource(Source):
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
# Every parser sets salary_text, empty or not (see
|
||||
# "salary_text" in `atshosts/_shared.py`); only
|
||||
# "salary_text" in `readers/_shared.py`); only
|
||||
# the hosts that state structured pay also set the
|
||||
# normalised figures, so the rest leave them at
|
||||
# row.get's default of no figure.
|
||||
|
||||
Reference in new issue
Block a user