Rename plugins/atshosts to plugins/readers
Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
af337ef558
commit
73dec9028e
48 files changed
+104
-104
No files matched your search
+2
-2
@@ -1,8 +1,8 @@
|
||||
# Where roles come from
|
||||
|
||||
This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a host module is the docstring of `plugins/atshosts/__init__.py`, not this file.
|
||||
This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a host module is the docstring of `plugins/readers/__init__.py`, not this file.
|
||||
|
||||
The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/atshosts/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
|
||||
The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/readers/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
|
||||
|
||||
## The three ways a source is read
|
||||
|
||||
|
||||
+2
-2
@@ -4,7 +4,7 @@ A plugin file imports `desk` only — see `tests/test_architecture.py` — and
|
||||
its own helpers in this directory. This `__init__.py` makes the directory
|
||||
importable as `plugins.<name>`, for the rest of roledesk (`jobs/pull.py`,
|
||||
`cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the
|
||||
package `plugins/atshosts/`. `desk.sources.registry` still finds `ats.py` by
|
||||
package `plugins/readers/`. `desk.sources.registry` still finds `ats.py` by
|
||||
path, not by this package; `ats.py` puts the repo root on `sys.path` itself
|
||||
when the app has not, so that `plugins.atshosts` imports either way.
|
||||
when the app has not, so that `plugins.readers` imports either way.
|
||||
"""
|
||||
+19
-19
@@ -1,5 +1,5 @@
|
||||
"""ATS boards: a company's own careers page on one of the hosts in
|
||||
`plugins/atshosts/`, one module per host.
|
||||
`plugins/readers/`, one module per host.
|
||||
|
||||
Every one of these hosts serves a company's own list of roles as public,
|
||||
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
||||
@@ -49,7 +49,7 @@ from desk.sources import (
|
||||
)
|
||||
|
||||
try:
|
||||
from plugins import atshosts
|
||||
from plugins import readers
|
||||
except ModuleNotFoundError:
|
||||
# `desk.sources.registry` loads this file by its path, under a name of its
|
||||
# own, and the app puts the repo root on `sys.path` before it does; a
|
||||
@@ -57,16 +57,16 @@ except ModuleNotFoundError:
|
||||
# does not, and the host modules are found as the package beside this
|
||||
# file. So the root goes on the path then, and only then.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plugins import atshosts
|
||||
from plugins.atshosts._shared import Get, Row
|
||||
from plugins import readers
|
||||
from plugins.readers._shared import Get, Row
|
||||
|
||||
# The names the parsers had here before they moved to their host modules,
|
||||
# kept for the callers and the tests that still reach for them on `ats`.
|
||||
from plugins.atshosts._shared import ( # noqa: F401
|
||||
from plugins.readers._shared import ( # noqa: F401
|
||||
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
|
||||
)
|
||||
from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
||||
from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401
|
||||
from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
||||
from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -101,7 +101,7 @@ DEADLINE = 60.0
|
||||
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
||||
#: the order a select offers them. What the Sources page's host select is
|
||||
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
||||
HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()}
|
||||
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
|
||||
|
||||
|
||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
@@ -110,7 +110,7 @@ def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
company's own site. Anything else is refused, at the first hop and at
|
||||
every redirect: a feed is external content, and a board row must never
|
||||
be able to point this fetch at an internal address."""
|
||||
module = atshosts.HOSTS.get(host)
|
||||
module = readers.HOSTS.get(host)
|
||||
return module.allowed(slug) if module is not None else lambda h: False
|
||||
|
||||
|
||||
@@ -234,7 +234,7 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
||||
#: host -> parser, for the hosts whose module parses a whole body. Every one
|
||||
#: takes the raw response body and the slug.
|
||||
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
||||
name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")}
|
||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||
|
||||
|
||||
# --------------------------------------------------------- the classifier
|
||||
@@ -365,11 +365,11 @@ def classify(department: str, title: str) -> str:
|
||||
|
||||
|
||||
def is_known_target(target: str) -> bool:
|
||||
"""`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's
|
||||
"""`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's
|
||||
`slug_ok` accepts. What the target validator refuses: `app companies add`
|
||||
and the pull job both call this before touching the network."""
|
||||
host, sep, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
module = readers.HOSTS.get(host)
|
||||
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
|
||||
|
||||
|
||||
@@ -377,7 +377,7 @@ def careers_url(target: str) -> str:
|
||||
"""The company's own careers page for `host:slug`, or "" for an unknown
|
||||
host. What `app companies add` stores alongside the board."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
module = readers.HOSTS.get(host)
|
||||
return module.careers_url(slug) if module is not None else ""
|
||||
|
||||
|
||||
@@ -404,7 +404,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
A 429 after the first page stops the read with what it has, incomplete;
|
||||
one on the first page raises like any other failure."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
rows: dict[str, Row] = {}
|
||||
seen: set[str] = set()
|
||||
@@ -449,7 +449,7 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
module = readers.HOSTS[host]
|
||||
if hasattr(module, "page"):
|
||||
return _paged(client, target)
|
||||
get = _getter(client, host, slug)
|
||||
@@ -566,7 +566,7 @@ def _check_robots(client: httpx.Client, target: str,
|
||||
kept, so the next pull asks again. A 429 counts as unreachable although
|
||||
it is a 4xx, since it says "not now" rather than "there is none"."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
module = readers.HOSTS.get(host)
|
||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||
return
|
||||
page = module.robots_page(slug)
|
||||
@@ -657,7 +657,7 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
|
||||
after a complete listing, so one cut short does not make the next pull
|
||||
read its missing roles again."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
done, failing = set(detailed), set(failed) - set(detailed)
|
||||
|
||||
@@ -804,7 +804,7 @@ class ATSSource(Source):
|
||||
|
||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||
detail_sets: tuple[set[str], set[str]] | None = None
|
||||
if hasattr(atshosts.HOSTS.get(host), "detail"):
|
||||
if hasattr(readers.HOSTS.get(host), "detail"):
|
||||
detailed, failed = (self._id_set(key) for key in detail_keys)
|
||||
rows, detailed, failed = _read_details(client, target, rows, complete,
|
||||
detailed, failed)
|
||||
@@ -833,7 +833,7 @@ class ATSSource(Source):
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
# Every parser sets salary_text, empty or not (see
|
||||
# "salary_text" in `atshosts/_shared.py`); only
|
||||
# "salary_text" in `readers/_shared.py`); only
|
||||
# the hosts that state structured pay also set the
|
||||
# normalised figures, so the rest leave them at
|
||||
# row.get's default of no figure.
|
||||
|
||||
@@ -11,7 +11,7 @@ patterns and probes. The package sits beside `ats.py` rather than being
|
||||
`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and
|
||||
the tests patch names on the `ats` module.
|
||||
|
||||
A module is named after its host (`plugins/atshosts/workday.py`) and imports
|
||||
A module is named after its host (`plugins/readers/workday.py`) and imports
|
||||
`desk` and `._shared` only, never `plugins.ats`. It holds:
|
||||
|
||||
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
||||
@@ -3,7 +3,7 @@ handed, and the date, URL and salary helpers more than one host needs.
|
||||
|
||||
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
|
||||
import it without either importing the other. The leading underscore keeps
|
||||
`plugins.atshosts` from listing it as a host.
|
||||
`plugins.readers` from listing it as a host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Reference in new issue
Block a user