Rename plugins/atshosts to plugins/readers
Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
af337ef558
commit
73dec9028e
48 files changed
+104
-104
No files matched your search
@@ -1,6 +1,6 @@
|
|||||||
# roledesk
|
# roledesk
|
||||||
|
|
||||||
A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor, Workday, SuccessFactors, Oracle, Phenom, Eightfold and a dozen smaller hosts, one module each in `plugins/atshosts/`) and runs saved searches on boards that span companies (Jobindex, Adzuna), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make.
|
A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor, Workday, SuccessFactors, Oracle, Phenom, Eightfold and a dozen smaller hosts, one module each in `plugins/readers/`) and runs saved searches on boards that span companies (Jobindex, Adzuna), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make.
|
||||||
|
|
||||||
Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeline/`), password (`auth/`) and deploy kit (`deploy/`), plus the plugin contract and tools from news (`desk/`). FastAPI, Jinja2, SQLite, `uv`, no build step.
|
Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeline/`), password (`auth/`) and deploy kit (`deploy/`), plus the plugin contract and tools from news (`desk/`). FastAPI, Jinja2, SQLite, `uv`, no build step.
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ The seed list and the LinkedIn flow both start from a company name and a
|
|||||||
website, not from a board. This module gets from the one to the other: it
|
website, not from a board. This module gets from the one to the other: it
|
||||||
fetches the site's careers page, or the pages the home page links to as
|
fetches the site's careers page, or the pages the home page links to as
|
||||||
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
|
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
|
||||||
host (one with a module in `plugins/atshosts/`, which the `ats` plugin
|
host (one with a module in `plugins/readers/`, which the `ats` plugin
|
||||||
parses) becomes a `board` target the pull can read; any other host is
|
parses) becomes a `board` target the pull can read; any other host is
|
||||||
recorded as `ats`, with its slug where the URL carries one, so the company is
|
recorded as `ats`, with its slug where the URL carries one, so the company is
|
||||||
kept as one the app cannot read yet rather than dropped. The patterns and
|
kept as one the app cannot read yet rather than dropped. The patterns and
|
||||||
@@ -48,7 +48,7 @@ from urllib.parse import urljoin
|
|||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from desk.sources import USER_AGENT
|
from desk.sources import USER_AGENT
|
||||||
from plugins import atshosts
|
from plugins import readers
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -58,7 +58,7 @@ MAX_BYTES = 2 << 20
|
|||||||
MAX_PAGES = 6
|
MAX_PAGES = 6
|
||||||
|
|
||||||
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
|
#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
|
||||||
READABLE: tuple[str, ...] = tuple(atshosts.HOSTS)
|
READABLE: tuple[str, ...] = tuple(readers.HOSTS)
|
||||||
|
|
||||||
#: host -> pattern over a URL, for the hosts discovery names but the app
|
#: host -> pattern over a URL, for the hosts discovery names but the app
|
||||||
#: cannot read. Group 1, where there is one, is the slug the host names the
|
#: cannot read. Group 1, where there is one, is the slug the host names the
|
||||||
@@ -104,14 +104,14 @@ def _hosts(modules: Mapping[str, ModuleType],
|
|||||||
#: host -> pattern over a URL, every host discovery can name. The slug is
|
#: host -> pattern over a URL, every host discovery can name. The slug is
|
||||||
#: group 1, or, for a pattern with more groups, the groups that matched
|
#: group 1, or, for a pattern with more groups, the groups that matched
|
||||||
#: joined by "/" (Lever's `eu/<site>`).
|
#: joined by "/" (Lever's `eu/<site>`).
|
||||||
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(atshosts.HOSTS, _UNREADABLE)
|
HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = _hosts(readers.HOSTS, _UNREADABLE)
|
||||||
|
|
||||||
#: The hosts whose slugs keep the case the URL wrote them in, by their
|
#: The hosts whose slugs keep the case the URL wrote them in, by their
|
||||||
#: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its
|
#: module's `KEEP_CASE`: a Workday site is `LEGO_External`, and both its
|
||||||
#: listing and its robots.txt paths are case-sensitive. Every other slug is
|
#: listing and its robots.txt paths are case-sensitive. Every other slug is
|
||||||
#: lowercased, which is how a page that writes `BoschGroup` and one that
|
#: lowercased, which is how a page that writes `BoschGroup` and one that
|
||||||
#: writes `boschgroup` name one board.
|
#: writes `boschgroup` name one board.
|
||||||
_KEEP_CASE = frozenset(name for name, module in atshosts.HOSTS.items()
|
_KEEP_CASE = frozenset(name for name, module in readers.HOSTS.items()
|
||||||
if getattr(module, "KEEP_CASE", False))
|
if getattr(module, "KEEP_CASE", False))
|
||||||
|
|
||||||
#: What a Teamtailor feed carries and a page that only looks like one does not:
|
#: What a Teamtailor feed carries and a page that only looks like one does not:
|
||||||
@@ -257,14 +257,14 @@ def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
|
|||||||
#: rate limit that a run of guesses trips, and a Workable site links its
|
#: rate limit that a run of guesses trips, and a Workable site links its
|
||||||
#: board plainly anyway.
|
#: board plainly anyway.
|
||||||
_PROBES: dict[str, Callable[[str], str]] = {
|
_PROBES: dict[str, Callable[[str], str]] = {
|
||||||
name: module.probe_url for name, module in atshosts.HOSTS.items()
|
name: module.probe_url for name, module in readers.HOSTS.items()
|
||||||
if hasattr(module, "probe_url")}
|
if hasattr(module, "probe_url")}
|
||||||
|
|
||||||
|
|
||||||
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
|
def _probe_slugs(host: str, guess: str) -> tuple[str, ...]:
|
||||||
"""The slugs a guessed name is probed as on `host`: the name itself, and
|
"""The slugs a guessed name is probed as on `host`: the name itself, and
|
||||||
for Lever the name on its EU instance too (the module's `probe_slugs`)."""
|
for Lever the name on its EU instance too (the module's `probe_slugs`)."""
|
||||||
module = atshosts.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
|
return module.probe_slugs(guess) if hasattr(module, "probe_slugs") else (guess,)
|
||||||
|
|
||||||
|
|
||||||
@@ -281,7 +281,7 @@ def probe(client: httpx.Client, host: str, slug: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
if not response.is_success:
|
if not response.is_success:
|
||||||
return False
|
return False
|
||||||
module = atshosts.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
|
return module.probe_ok(response.content) if hasattr(module, "probe_ok") else True
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -663,7 +663,7 @@ HAND_REASONS: tuple[str, ...] = get_args(HandReason)
|
|||||||
#: read as when a role was posted, checked against live boards on
|
#: read as when a role was posted, checked against live boards on
|
||||||
#: 2026-09-29:
|
#: 2026-09-29:
|
||||||
#:
|
#:
|
||||||
#: - `greenhouse`: `plugins.atshosts.greenhouse.parse` already prefers
|
#: - `greenhouse`: `plugins.readers.greenhouse.parse` already prefers
|
||||||
#: `first_published` over `updated_at`. A live board (adyen) showed a job
|
#: `first_published` over `updated_at`. A live board (adyen) showed a job
|
||||||
#: with `first_published` five months before `updated_at`, proving the
|
#: with `first_published` five months before `updated_at`, proving the
|
||||||
#: first stays fixed while the second moves on an edit — the resettable
|
#: first stays fixed while the second moves on an edit — the resettable
|
||||||
|
|||||||
+1
-1
@@ -100,7 +100,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
|||||||
|
|
||||||
**A board that fails on every fetch for 21 days closes its roles.** A board whose fetch raised was only logged, so a company that moved ATS kept its roles open for good behind a 404. The `ats` plugin now records `ats:failing:<target>` in `state` with the time of the first failure in a row and the latest reason, and a fetch that answers removes it. Once the first failure is `STALE_DAYS` old, the same window a search role gets, the settle closes the board's roles; the next answer reopens what it lists through `upsert_role`, as for a board switched back on. The record is plugin state rather than a column because the plugin only sees `self.state`. `/api/companies` returns it as `failing_since` and `failing_reason`, the Companies list shows "not answering since", and `app companies check` prints it beside its live result.
|
**A board that fails on every fetch for 21 days closes its roles.** A board whose fetch raised was only logged, so a company that moved ATS kept its roles open for good behind a 404. The `ats` plugin now records `ats:failing:<target>` in `state` with the time of the first failure in a row and the latest reason, and a fetch that answers removes it. Once the first failure is `STALE_DAYS` old, the same window a search role gets, the settle closes the board's roles; the next answer reopens what it lists through `upsert_role`, as for a board switched back on. The record is plugin state rather than a column because the plugin only sees `self.state`. `/api/companies` returns it as `failing_since` and `failing_reason`, the Companies list shows "not answering since", and `app companies check` prints it beside its live result.
|
||||||
|
|
||||||
**A host is a module in `plugins/atshosts/`, and registers by existing.** `plugins/ats.py` held eight hosts in six parallel tables and `app/jobs/discover.py` three more, so two people adding a host edited the same lines. Each host is now one file that states its name, label, URL patterns, slug check, allowed host names and either a feed and a parser or a pager; `ats.py` and `discover.py` build their tables from the package, ordered by the module's `RANK` so the select and the probes kept their order. The package sits beside `ats.py` and is not `plugins/ats/`, since `desk.sources.registry` globs `plugins/*.py` and the tests patch names on the `ats` module. The contract is the docstring of `plugins/atshosts/__init__.py`, and a module that breaks it fails the import by name.
|
**A host is a module in `plugins/readers/`, and registers by existing.** `plugins/ats.py` held eight hosts in six parallel tables and `app/jobs/discover.py` three more, so two people adding a host edited the same lines. Each host is now one file that states its name, label, URL patterns, slug check, allowed host names and either a feed and a parser or a pager; `ats.py` and `discover.py` build their tables from the package, ordered by the module's `RANK` so the select and the probes kept their order. The package sits beside `ats.py` and is not `plugins/ats/`, since `desk.sources.registry` globs `plugins/*.py` and the tests patch names on the `ats` module. The contract is the docstring of `plugins/readers/__init__.py`, and a module that breaks it fails the import by name.
|
||||||
|
|
||||||
**A host is handed the fetch.** A module imports `desk` and `._shared` only and receives `get`, which `ats.py` binds to its own `_get` at call time. The allow-list, the redirect rule, the byte cap and the deadline stay in one place, and a test that patches `ats._get` reaches every host. `_get` refuses an IP address, a local or numeric host name and any port but 443 before it asks, because a Teamtailor or SuccessFactors slug is a host name somebody typed.
|
**A host is handed the fetch.** A module imports `desk` and `._shared` only and receives `get`, which `ats.py` binds to its own `_get` at call time. The allow-list, the redirect rule, the byte cap and the deadline stay in one place, and a test that patches `ats._get` reaches every host. `_get` refuses an IP address, a local or numeric host name and any port but 443 before it asks, because a Teamtailor or SuccessFactors slug is a host name somebody typed.
|
||||||
|
|
||||||
|
|||||||
@@ -28,11 +28,11 @@ The survey's record of each company is in `docs/plan-hosts/survey-own.json` (cla
|
|||||||
|
|
||||||
## Work packages
|
## Work packages
|
||||||
|
|
||||||
Each owns its modules in `plugins/atshosts/`, `tests/test_ats_<name>.py` and `tests/fixtures/ats/<name>*`, and nothing else. The common rules of wave 1 apply: the live request limits, robots.txt read first, no key from a script, a final report with the targets to attach.
|
Each owns its modules in `plugins/readers/`, `tests/test_ats_<name>.py` and `tests/fixtures/ats/<name>*`, and nothing else. The common rules of wave 1 apply: the live request limits, robots.txt read first, no key from a script, a final report with the targets to attach.
|
||||||
|
|
||||||
| WP | Model | Builds | Needs |
|
| WP | Model | Builds | Needs |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| 9 Provisional title | Opus | the row key, `upsert_role`'s rule and its tests; owns `plugins/ats.py`, `plugins/atshosts/__init__.py`, `app/storage/store.py`, `tests/test_atshosts.py`, `tests/test_storage.py` | the first decision |
|
| 9 Provisional title | Opus | the row key, `upsert_role`'s rule and its tests; owns `plugins/ats.py`, `plugins/readers/__init__.py`, `app/storage/store.py`, `tests/test_readers.py`, `tests/test_storage.py` | the first decision |
|
||||||
| 10 Sitemap reader | Sonnet | `sitemap`: job URLs from a sitemap, the role from the page's JSON-LD or microdata | WP9 |
|
| 10 Sitemap reader | Sonnet | `sitemap`: job URLs from a sitemap, the role from the page's JSON-LD or microdata | WP9 |
|
||||||
| 11 Board filters | Opus | `params` on a board subscription, handed to `page`; owns `plugins/ats.py`, `app/jobs/pull.py`, `app/jobs/sources.py`, `app/web/api.py`, `app/static/companies.js` and their tests | the second decision |
|
| 11 Board filters | Opus | `params` on a board subscription, handed to `page`; owns `plugins/ats.py`, `app/jobs/pull.py`, `app/jobs/sources.py`, `app/web/api.py`, `app/static/companies.js` and their tests | the second decision |
|
||||||
| 12 Own APIs | Sonnet | Ørsted, Amazon, Vinted, Dynatrace, RELEX, Siteimprove, Bending Spoons | |
|
| 12 Own APIs | Sonnet | Ørsted, Amazon, Vinted, Dynatrace, RELEX, Siteimprove, Bending Spoons | |
|
||||||
|
|||||||
+2
-2
@@ -1,8 +1,8 @@
|
|||||||
# Where roles come from
|
# Where roles come from
|
||||||
|
|
||||||
This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a host module is the docstring of `plugins/atshosts/__init__.py`, not this file.
|
This file lists every source roledesk reads roles from, how each is read and what it cannot deliver, as of the code and the live database of 2026-09-29. The contract for writing a host module is the docstring of `plugins/readers/__init__.py`, not this file.
|
||||||
|
|
||||||
The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/atshosts/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
|
The code has two plugins: `ats` (`plugins/ats.py`) reads the board of a company someone listed, with one module per host in `plugins/readers/`; `search` (`plugins/search.py`) runs saved queries on sites that list roles across companies. Below, a module of `ats` is a "host" and a module of `search` is a "provider". A company's list of roles on a host is its "board"; a board is attached as a target `host:slug`, e.g. `workday:lego.wd103/LEGO_External`.
|
||||||
|
|
||||||
## The three ways a source is read
|
## The three ways a source is read
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -4,7 +4,7 @@ A plugin file imports `desk` only — see `tests/test_architecture.py` — and
|
|||||||
its own helpers in this directory. This `__init__.py` makes the directory
|
its own helpers in this directory. This `__init__.py` makes the directory
|
||||||
importable as `plugins.<name>`, for the rest of roledesk (`jobs/pull.py`,
|
importable as `plugins.<name>`, for the rest of roledesk (`jobs/pull.py`,
|
||||||
`cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the
|
`cli.py`, the tests) and for `ats.py`, whose hosts are the modules of the
|
||||||
package `plugins/atshosts/`. `desk.sources.registry` still finds `ats.py` by
|
package `plugins/readers/`. `desk.sources.registry` still finds `ats.py` by
|
||||||
path, not by this package; `ats.py` puts the repo root on `sys.path` itself
|
path, not by this package; `ats.py` puts the repo root on `sys.path` itself
|
||||||
when the app has not, so that `plugins.atshosts` imports either way.
|
when the app has not, so that `plugins.readers` imports either way.
|
||||||
"""
|
"""
|
||||||
+19
-19
@@ -1,5 +1,5 @@
|
|||||||
"""ATS boards: a company's own careers page on one of the hosts in
|
"""ATS boards: a company's own careers page on one of the hosts in
|
||||||
`plugins/atshosts/`, one module per host.
|
`plugins/readers/`, one module per host.
|
||||||
|
|
||||||
Every one of these hosts serves a company's own list of roles as public,
|
Every one of these hosts serves a company's own list of roles as public,
|
||||||
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
||||||
@@ -49,7 +49,7 @@ from desk.sources import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from plugins import atshosts
|
from plugins import readers
|
||||||
except ModuleNotFoundError:
|
except ModuleNotFoundError:
|
||||||
# `desk.sources.registry` loads this file by its path, under a name of its
|
# `desk.sources.registry` loads this file by its path, under a name of its
|
||||||
# own, and the app puts the repo root on `sys.path` before it does; a
|
# own, and the app puts the repo root on `sys.path` before it does; a
|
||||||
@@ -57,16 +57,16 @@ except ModuleNotFoundError:
|
|||||||
# does not, and the host modules are found as the package beside this
|
# does not, and the host modules are found as the package beside this
|
||||||
# file. So the root goes on the path then, and only then.
|
# file. So the root goes on the path then, and only then.
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
from plugins import atshosts
|
from plugins import readers
|
||||||
from plugins.atshosts._shared import Get, Row
|
from plugins.readers._shared import Get, Row
|
||||||
|
|
||||||
# The names the parsers had here before they moved to their host modules,
|
# The names the parsers had here before they moved to their host modules,
|
||||||
# kept for the callers and the tests that still reach for them on `ats`.
|
# kept for the callers and the tests that still reach for them on `ats`.
|
||||||
from plugins.atshosts._shared import ( # noqa: F401
|
from plugins.readers._shared import ( # noqa: F401
|
||||||
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
|
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
|
||||||
)
|
)
|
||||||
from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
||||||
from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401
|
from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -101,7 +101,7 @@ DEADLINE = 60.0
|
|||||||
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
||||||
#: the order a select offers them. What the Sources page's host select is
|
#: the order a select offers them. What the Sources page's host select is
|
||||||
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
||||||
HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()}
|
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
|
||||||
|
|
||||||
|
|
||||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||||
@@ -110,7 +110,7 @@ def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
|||||||
company's own site. Anything else is refused, at the first hop and at
|
company's own site. Anything else is refused, at the first hop and at
|
||||||
every redirect: a feed is external content, and a board row must never
|
every redirect: a feed is external content, and a board row must never
|
||||||
be able to point this fetch at an internal address."""
|
be able to point this fetch at an internal address."""
|
||||||
module = atshosts.HOSTS.get(host)
|
module = readers.HOSTS.get(host)
|
||||||
return module.allowed(slug) if module is not None else lambda h: False
|
return module.allowed(slug) if module is not None else lambda h: False
|
||||||
|
|
||||||
|
|
||||||
@@ -234,7 +234,7 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
|||||||
#: host -> parser, for the hosts whose module parses a whole body. Every one
|
#: host -> parser, for the hosts whose module parses a whole body. Every one
|
||||||
#: takes the raw response body and the slug.
|
#: takes the raw response body and the slug.
|
||||||
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
||||||
name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")}
|
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------- the classifier
|
# --------------------------------------------------------- the classifier
|
||||||
@@ -365,11 +365,11 @@ def classify(department: str, title: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def is_known_target(target: str) -> bool:
|
def is_known_target(target: str) -> bool:
|
||||||
"""`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's
|
"""`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's
|
||||||
`slug_ok` accepts. What the target validator refuses: `app companies add`
|
`slug_ok` accepts. What the target validator refuses: `app companies add`
|
||||||
and the pull job both call this before touching the network."""
|
and the pull job both call this before touching the network."""
|
||||||
host, sep, slug = target.partition(":")
|
host, sep, slug = target.partition(":")
|
||||||
module = atshosts.HOSTS.get(host)
|
module = readers.HOSTS.get(host)
|
||||||
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
|
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
|
||||||
|
|
||||||
|
|
||||||
@@ -377,7 +377,7 @@ def careers_url(target: str) -> str:
|
|||||||
"""The company's own careers page for `host:slug`, or "" for an unknown
|
"""The company's own careers page for `host:slug`, or "" for an unknown
|
||||||
host. What `app companies add` stores alongside the board."""
|
host. What `app companies add` stores alongside the board."""
|
||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
module = atshosts.HOSTS.get(host)
|
module = readers.HOSTS.get(host)
|
||||||
return module.careers_url(slug) if module is not None else ""
|
return module.careers_url(slug) if module is not None else ""
|
||||||
|
|
||||||
|
|
||||||
@@ -404,7 +404,7 @@ def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|||||||
A 429 after the first page stops the read with what it has, incomplete;
|
A 429 after the first page stops the read with what it has, incomplete;
|
||||||
one on the first page raises like any other failure."""
|
one on the first page raises like any other failure."""
|
||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
module = atshosts.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
get = _getter(client, host, slug)
|
get = _getter(client, host, slug)
|
||||||
rows: dict[str, Row] = {}
|
rows: dict[str, Row] = {}
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
@@ -449,7 +449,7 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|||||||
if not is_known_target(target):
|
if not is_known_target(target):
|
||||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
module = atshosts.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
if hasattr(module, "page"):
|
if hasattr(module, "page"):
|
||||||
return _paged(client, target)
|
return _paged(client, target)
|
||||||
get = _getter(client, host, slug)
|
get = _getter(client, host, slug)
|
||||||
@@ -566,7 +566,7 @@ def _check_robots(client: httpx.Client, target: str,
|
|||||||
kept, so the next pull asks again. A 429 counts as unreachable although
|
kept, so the next pull asks again. A 429 counts as unreachable although
|
||||||
it is a 4xx, since it says "not now" rather than "there is none"."""
|
it is a 4xx, since it says "not now" rather than "there is none"."""
|
||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
module = atshosts.HOSTS.get(host)
|
module = readers.HOSTS.get(host)
|
||||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||||
return
|
return
|
||||||
page = module.robots_page(slug)
|
page = module.robots_page(slug)
|
||||||
@@ -657,7 +657,7 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
|
|||||||
after a complete listing, so one cut short does not make the next pull
|
after a complete listing, so one cut short does not make the next pull
|
||||||
read its missing roles again."""
|
read its missing roles again."""
|
||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
module = atshosts.HOSTS[host]
|
module = readers.HOSTS[host]
|
||||||
get = _getter(client, host, slug)
|
get = _getter(client, host, slug)
|
||||||
done, failing = set(detailed), set(failed) - set(detailed)
|
done, failing = set(detailed), set(failed) - set(detailed)
|
||||||
|
|
||||||
@@ -804,7 +804,7 @@ class ATSSource(Source):
|
|||||||
|
|
||||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||||
detail_sets: tuple[set[str], set[str]] | None = None
|
detail_sets: tuple[set[str], set[str]] | None = None
|
||||||
if hasattr(atshosts.HOSTS.get(host), "detail"):
|
if hasattr(readers.HOSTS.get(host), "detail"):
|
||||||
detailed, failed = (self._id_set(key) for key in detail_keys)
|
detailed, failed = (self._id_set(key) for key in detail_keys)
|
||||||
rows, detailed, failed = _read_details(client, target, rows, complete,
|
rows, detailed, failed = _read_details(client, target, rows, complete,
|
||||||
detailed, failed)
|
detailed, failed)
|
||||||
@@ -833,7 +833,7 @@ class ATSSource(Source):
|
|||||||
"function": classify(row["department"], row["title"]),
|
"function": classify(row["department"], row["title"]),
|
||||||
"remote": row["remote"],
|
"remote": row["remote"],
|
||||||
# Every parser sets salary_text, empty or not (see
|
# Every parser sets salary_text, empty or not (see
|
||||||
# "salary_text" in `atshosts/_shared.py`); only
|
# "salary_text" in `readers/_shared.py`); only
|
||||||
# the hosts that state structured pay also set the
|
# the hosts that state structured pay also set the
|
||||||
# normalised figures, so the rest leave them at
|
# normalised figures, so the rest leave them at
|
||||||
# row.get's default of no figure.
|
# row.get's default of no figure.
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ patterns and probes. The package sits beside `ats.py` rather than being
|
|||||||
`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and
|
`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and
|
||||||
the tests patch names on the `ats` module.
|
the tests patch names on the `ats` module.
|
||||||
|
|
||||||
A module is named after its host (`plugins/atshosts/workday.py`) and imports
|
A module is named after its host (`plugins/readers/workday.py`) and imports
|
||||||
`desk` and `._shared` only, never `plugins.ats`. It holds:
|
`desk` and `._shared` only, never `plugins.ats`. It holds:
|
||||||
|
|
||||||
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
||||||
@@ -3,7 +3,7 @@ handed, and the date, URL and salary helpers more than one host needs.
|
|||||||
|
|
||||||
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
|
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
|
||||||
import it without either importing the other. The leading underscore keeps
|
import it without either importing the other. The leading underscore keeps
|
||||||
`plugins.atshosts` from listing it as a host.
|
`plugins.readers` from listing it as a host.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+1
-1
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.atshosts import smartrecruiters # noqa: E402
|
from plugins.readers import smartrecruiters # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import bamboohr # noqa: E402
|
from plugins.readers import bamboohr # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -161,7 +161,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
|
|||||||
assert allow("configit.bamboohr.com") and allow("CONFIGIT.bamboohr.com")
|
assert allow("configit.bamboohr.com") and allow("CONFIGIT.bamboohr.com")
|
||||||
assert not allow("bamboohr.com") and not allow("other.bamboohr.com")
|
assert not allow("bamboohr.com") and not allow("other.bamboohr.com")
|
||||||
assert not allow("configit.bamboohr.com.evil.example") and not allow("localhost")
|
assert not allow("configit.bamboohr.com.evil.example") and not allow("localhost")
|
||||||
assert atshosts.HOSTS["bamboohr"] is bamboohr and ats.HOSTS["bamboohr"] == "BambooHR"
|
assert readers.HOSTS["bamboohr"] is bamboohr and ats.HOSTS["bamboohr"] == "BambooHR"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("slug", [
|
@pytest.mark.parametrize("slug", [
|
||||||
@@ -182,4 +182,4 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None:
|
|||||||
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
|
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
|
||||||
'<a href="https://resources.bamboohr.com/y">z</a>')
|
'<a href="https://resources.bamboohr.com/y">z</a>')
|
||||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
|
||||||
assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS}
|
assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
||||||
@@ -16,8 +16,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import deel # noqa: E402
|
from plugins.readers import deel # noqa: E402
|
||||||
|
|
||||||
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
|
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
|
||||||
URL = "https://jobs.deel.com/klarna"
|
URL = "https://jobs.deel.com/klarna"
|
||||||
@@ -126,7 +126,7 @@ def test_a_page_that_cannot_be_read_fails_the_board(web: dict[str, Any]) -> None
|
|||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|
||||||
def test_the_module_is_registered_as_a_feed_host() -> None:
|
def test_the_module_is_registered_as_a_feed_host() -> None:
|
||||||
assert atshosts.HOSTS["deel"] is deel and ats.HOSTS["deel"] == "Deel"
|
assert readers.HOSTS["deel"] is deel and ats.HOSTS["deel"] == "Deel"
|
||||||
assert deel.NAME == "deel" and deel.KEEP_CASE
|
assert deel.NAME == "deel" and deel.KEEP_CASE
|
||||||
assert not hasattr(deel, "page") and not hasattr(deel, "detail")
|
assert not hasattr(deel, "page") and not hasattr(deel, "detail")
|
||||||
|
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import eightfold # noqa: E402
|
from plugins.readers import eightfold # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SLUG = "jobs.ericsson.com/ericsson.com"
|
SLUG = "jobs.ericsson.com/ericsson.com"
|
||||||
@@ -169,7 +169,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
|
|||||||
assert allow("jobs.ericsson.com") and allow("JOBS.ERICSSON.COM")
|
assert allow("jobs.ericsson.com") and allow("JOBS.ERICSSON.COM")
|
||||||
assert not allow("ericsson.com") and not allow("evil.example")
|
assert not allow("ericsson.com") and not allow("evil.example")
|
||||||
assert not allow("jobs.ericsson.com.evil.example") and not allow("localhost")
|
assert not allow("jobs.ericsson.com.evil.example") and not allow("localhost")
|
||||||
assert atshosts.HOSTS["eightfold"] is eightfold and ats.HOSTS["eightfold"] == "Eightfold"
|
assert readers.HOSTS["eightfold"] is eightfold and ats.HOSTS["eightfold"] == "Eightfold"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("slug", [
|
@pytest.mark.parametrize("slug", [
|
||||||
@@ -192,4 +192,4 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None
|
|||||||
found = [(h, s) for h, s, _ in discover.boards_in(page)]
|
found = [(h, s) for h, s, _ in discover.boards_in(page)]
|
||||||
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
|
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
|
||||||
assert ("eightfold", "") in found
|
assert ("eightfold", "") in found
|
||||||
assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS}
|
assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
||||||
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import gem # noqa: E402
|
from plugins.readers import gem # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
API = "https://jobs.gem.com/api/public/graphql"
|
API = "https://jobs.gem.com/api/public/graphql"
|
||||||
@@ -154,7 +154,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
|
|||||||
assert allow("jobs.gem.com") and allow("JOBS.gem.com")
|
assert allow("jobs.gem.com") and allow("JOBS.gem.com")
|
||||||
assert not allow("gem.com") and not allow("productboard.gem.com")
|
assert not allow("gem.com") and not allow("productboard.gem.com")
|
||||||
assert not allow("jobs.gem.com.evil.example") and not allow("localhost")
|
assert not allow("jobs.gem.com.evil.example") and not allow("localhost")
|
||||||
assert atshosts.HOSTS["gem"] is gem and ats.HOSTS["gem"] == "Gem"
|
assert readers.HOSTS["gem"] is gem and ats.HOSTS["gem"] == "Gem"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("slug", [
|
@pytest.mark.parametrize("slug", [
|
||||||
@@ -177,4 +177,4 @@ def test_discovery_finds_the_board_keeping_its_case() -> None:
|
|||||||
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
|
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
|
||||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
|
||||||
("gem", "Productboard"), ("gem", "sennder-gmbh")]
|
("gem", "Productboard"), ("gem", "sennder-gmbh")]
|
||||||
assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS}
|
assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
||||||
@@ -16,8 +16,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import hibob # noqa: E402
|
from plugins.readers import hibob # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "hibob.json").read_bytes()
|
BOARD = (FIXTURES / "hibob.json").read_bytes()
|
||||||
@@ -171,7 +171,7 @@ def test_the_robots_txt_that_is_the_apps_own_page_allows(web: dict[str, Any]) ->
|
|||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|
||||||
def test_the_module_is_registered_as_a_paged_host() -> None:
|
def test_the_module_is_registered_as_a_paged_host() -> None:
|
||||||
assert atshosts.HOSTS["hibob"] is hibob and ats.HOSTS["hibob"] == "HiBob"
|
assert readers.HOSTS["hibob"] is hibob and ats.HOSTS["hibob"] == "HiBob"
|
||||||
assert hibob.NAME == "hibob" and not getattr(hibob, "KEEP_CASE", False)
|
assert hibob.NAME == "hibob" and not getattr(hibob, "KEEP_CASE", False)
|
||||||
assert hasattr(hibob, "page") and not hasattr(hibob, "detail")
|
assert hasattr(hibob, "page") and not hasattr(hibob, "detail")
|
||||||
|
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import hrmanager # noqa: E402
|
from plugins.readers import hrmanager # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
||||||
@@ -155,7 +155,7 @@ def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide(
|
|||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|
||||||
def test_the_module_is_registered_as_a_paged_host() -> None:
|
def test_the_module_is_registered_as_a_paged_host() -> None:
|
||||||
assert atshosts.HOSTS["hrmanager"] is hrmanager and ats.HOSTS["hrmanager"] == "HR Manager"
|
assert readers.HOSTS["hrmanager"] is hrmanager and ats.HOSTS["hrmanager"] == "HR Manager"
|
||||||
assert hrmanager.NAME == "hrmanager" and not getattr(hrmanager, "KEEP_CASE", False)
|
assert hrmanager.NAME == "hrmanager" and not getattr(hrmanager, "KEEP_CASE", False)
|
||||||
assert hasattr(hrmanager, "page") and not hasattr(hrmanager, "detail")
|
assert hasattr(hrmanager, "page") and not hasattr(hrmanager, "detail")
|
||||||
|
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import jibe # noqa: E402
|
from plugins.readers import jibe # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SITE = "jobs.booking.com"
|
SITE = "jobs.booking.com"
|
||||||
@@ -152,7 +152,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
|
|||||||
assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM")
|
assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM")
|
||||||
assert not allow("booking.com") and not allow("evil.example")
|
assert not allow("booking.com") and not allow("evil.example")
|
||||||
assert not allow("jobs.booking.com.evil.example") and not allow("localhost")
|
assert not allow("jobs.booking.com.evil.example") and not allow("localhost")
|
||||||
assert atshosts.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe"
|
assert readers.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("slug", [
|
@pytest.mark.parametrize("slug", [
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import jobylon # noqa: E402
|
from plugins.readers import jobylon # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
||||||
@@ -158,7 +158,7 @@ def test_an_empty_board_is_complete(web: dict[str, Any]) -> None:
|
|||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|
||||||
def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
|
def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
|
||||||
assert atshosts.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon"
|
assert readers.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon"
|
||||||
assert jobylon.NAME == "jobylon" and not getattr(jobylon, "KEEP_CASE", False)
|
assert jobylon.NAME == "jobylon" and not getattr(jobylon, "KEEP_CASE", False)
|
||||||
assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page")
|
assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page")
|
||||||
|
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.atshosts import oracle # noqa: E402
|
from plugins.readers import oracle # noqa: E402
|
||||||
from plugins.atshosts._shared import is_web_url # noqa: E402
|
from plugins.readers._shared import is_web_url # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
HOST = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com"
|
HOST = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com"
|
||||||
|
|||||||
@@ -19,8 +19,8 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.atshosts import phenom # noqa: E402
|
from plugins.readers import phenom # noqa: E402
|
||||||
from plugins.atshosts._shared import is_web_url # noqa: E402
|
from plugins.readers._shared import is_web_url # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SLUG = "careers.king.com/us/en"
|
SLUG = "careers.king.com/us/en"
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import pinpoint # noqa: E402
|
from plugins.readers import pinpoint # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -122,7 +122,7 @@ def test_the_slug_the_urls_and_the_allow_list() -> None:
|
|||||||
assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com")
|
assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com")
|
||||||
assert not allow("pinpointhq.com") and not allow("other.pinpointhq.com")
|
assert not allow("pinpointhq.com") and not allow("other.pinpointhq.com")
|
||||||
assert not allow("xeneta.pinpointhq.com.evil.example") and not allow("localhost")
|
assert not allow("xeneta.pinpointhq.com.evil.example") and not allow("localhost")
|
||||||
assert atshosts.HOSTS["pinpoint"] is pinpoint and ats.HOSTS["pinpoint"] == "Pinpoint"
|
assert readers.HOSTS["pinpoint"] is pinpoint and ats.HOSTS["pinpoint"] == "Pinpoint"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("slug", [
|
@pytest.mark.parametrize("slug", [
|
||||||
@@ -146,7 +146,7 @@ def test_discovery_finds_the_subdomain() -> None:
|
|||||||
'<script src="https://www.pinpointhq.com/embed.js"></script>')
|
'<script src="https://www.pinpointhq.com/embed.js"></script>')
|
||||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
|
||||||
assert discover.boards_in("https://pinpointhq.com/") == []
|
assert discover.boards_in("https://pinpointhq.com/") == []
|
||||||
assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in atshosts.HOSTS}
|
assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
||||||
|
|
||||||
|
|
||||||
def test_a_pinpoint_board_is_not_probed() -> None:
|
def test_a_pinpoint_board_is_not_probed() -> None:
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import rippling # noqa: E402
|
from plugins.readers import rippling # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
BOARD = (FIXTURES / "rippling.json").read_bytes()
|
BOARD = (FIXTURES / "rippling.json").read_bytes()
|
||||||
@@ -168,7 +168,7 @@ def test_robots_are_read_from_the_site_and_disallow_only_internal(
|
|||||||
# ------------------------------------------------------- slugs and hosts
|
# ------------------------------------------------------- slugs and hosts
|
||||||
|
|
||||||
def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
|
def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
|
||||||
assert atshosts.HOSTS["rippling"] is rippling and ats.HOSTS["rippling"] == "Rippling"
|
assert readers.HOSTS["rippling"] is rippling and ats.HOSTS["rippling"] == "Rippling"
|
||||||
assert rippling.NAME == "rippling" and rippling.KEEP_CASE
|
assert rippling.NAME == "rippling" and rippling.KEEP_CASE
|
||||||
assert hasattr(rippling, "detail") and not hasattr(rippling, "page")
|
assert hasattr(rippling, "detail") and not hasattr(rippling, "page")
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""The SuccessFactors host: `plugins/atshosts/successfactors.py` against saved
|
"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved
|
||||||
career-site pages. No network. The fixtures are trimmed pages of Vestas and
|
career-site pages. No network. The fixtures are trimmed pages of Vestas and
|
||||||
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
|
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
|
||||||
with a path prefix), and the job pages of Vestas (`addressCountry`),
|
with a path prefix), and the job pages of Vestas (`addressCountry`),
|
||||||
@@ -19,8 +19,8 @@ if str(ROOT) not in sys.path:
|
|||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import successfactors # noqa: E402
|
from plugins.readers import successfactors # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -439,7 +439,7 @@ def test_careers_url_and_robots_page_are_the_search_page() -> None:
|
|||||||
|
|
||||||
def test_the_module_is_case_sensitive_about_the_prefix() -> None:
|
def test_the_module_is_case_sensitive_about_the_prefix() -> None:
|
||||||
assert successfactors.KEEP_CASE is True
|
assert successfactors.KEEP_CASE is True
|
||||||
assert atshosts.HOSTS["successfactors"] is successfactors
|
assert readers.HOSTS["successfactors"] is successfactors
|
||||||
assert successfactors._parts("jobs.teradyne.com/MiR") == ("jobs.teradyne.com", "/MiR")
|
assert successfactors._parts("jobs.teradyne.com/MiR") == ("jobs.teradyne.com", "/MiR")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
|||||||
|
|
||||||
from app.jobs import discover # noqa: E402
|
from app.jobs import discover # noqa: E402
|
||||||
from plugins import ats # noqa: E402
|
from plugins import ats # noqa: E402
|
||||||
from plugins.atshosts import workday # noqa: E402
|
from plugins.readers import workday # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
SLUG = "zendesk.wd1/zendesk"
|
SLUG = "zendesk.wd1/zendesk"
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for
|
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
||||||
every host: the registry and its contract check, the fetch a host is handed,
|
every host: the registry and its contract check, the fetch a host is handed,
|
||||||
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
||||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||||
@@ -22,8 +22,8 @@ ROOT = Path(__file__).resolve().parents[1]
|
|||||||
if str(ROOT) not in sys.path:
|
if str(ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
from plugins import ats, atshosts # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.atshosts import lever # noqa: E402
|
from plugins.readers import lever # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
@@ -31,14 +31,14 @@ FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|||||||
# --------------------------------------------------------------- registry
|
# --------------------------------------------------------------- registry
|
||||||
|
|
||||||
def test_the_registry_finds_every_module() -> None:
|
def test_the_registry_finds_every_module() -> None:
|
||||||
names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py")
|
names = sorted(p.stem for p in (ROOT / "plugins" / "readers").glob("*.py")
|
||||||
if not p.stem.startswith("_"))
|
if not p.stem.startswith("_"))
|
||||||
assert sorted(atshosts.HOSTS) == names
|
assert sorted(readers.HOSTS) == names
|
||||||
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||||
"recruitee", "personio", "teamtailor"}
|
"recruitee", "personio", "teamtailor"}
|
||||||
assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items())
|
assert all(not readers.problems(module, name) for name, module in readers.HOSTS.items())
|
||||||
# The Sources page's select and the plugin's description follow the modules.
|
# The Sources page's select and the plugin's description follow the modules.
|
||||||
assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS
|
assert {name: module.LABEL for name, module in readers.HOSTS.items()} == ats.HOSTS
|
||||||
assert "Teamtailor" in ats.ATSSource.description
|
assert "Teamtailor" in ats.ATSSource.description
|
||||||
|
|
||||||
|
|
||||||
@@ -86,7 +86,7 @@ def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
|
|||||||
path = _package(tmp_path, monkeypatch, "hostsok", {
|
path = _package(tmp_path, monkeypatch, "hostsok", {
|
||||||
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
|
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
|
||||||
"_helpers": "X = 1\n"})
|
"_helpers": "X = 1\n"})
|
||||||
assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"]
|
assert list(readers.load(path, "hostsok")) == ["alpha", "zeta"]
|
||||||
|
|
||||||
|
|
||||||
def test_the_registry_orders_by_rank_then_name(
|
def test_the_registry_orders_by_rank_then_name(
|
||||||
@@ -95,7 +95,7 @@ def test_the_registry_orders_by_rank_then_name(
|
|||||||
"alpha": _GOOD.format(name="alpha"), # the default, 100
|
"alpha": _GOOD.format(name="alpha"), # the default, 100
|
||||||
"beta": _GOOD.format(name="beta") + "RANK = 5\n",
|
"beta": _GOOD.format(name="beta") + "RANK = 5\n",
|
||||||
"gamma": _GOOD.format(name="gamma") + "RANK = 100\n"})
|
"gamma": _GOOD.format(name="gamma") + "RANK = 100\n"})
|
||||||
assert list(atshosts.load(path, "hostsranked")) == ["beta", "alpha", "gamma"]
|
assert list(readers.load(path, "hostsranked")) == ["beta", "alpha", "gamma"]
|
||||||
|
|
||||||
|
|
||||||
def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
|
def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
|
||||||
@@ -107,8 +107,8 @@ def test_the_eight_first_hosts_keep_the_order_they_had_before_modules() -> None:
|
|||||||
assert ats.ATSSource.description.startswith(
|
assert ats.ATSSource.description.startswith(
|
||||||
"Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, "
|
"Company job boards: Greenhouse, Lever, Ashby, Workable, SmartRecruiters, "
|
||||||
"Recruitee, Personio, Teamtailor")
|
"Recruitee, Personio, Teamtailor")
|
||||||
assert all(module.RANK < atshosts.DEFAULT_RANK
|
assert all(module.RANK < readers.DEFAULT_RANK
|
||||||
for name, module in atshosts.HOSTS.items() if name in first)
|
for name, module in readers.HOSTS.items() if name in first)
|
||||||
|
|
||||||
|
|
||||||
def test_the_registry_refuses_a_module_missing_a_required_name(
|
def test_the_registry_refuses_a_module_missing_a_required_name(
|
||||||
@@ -116,7 +116,7 @@ def test_the_registry_refuses_a_module_missing_a_required_name(
|
|||||||
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
|
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
|
||||||
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
|
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
|
||||||
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
|
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
|
||||||
atshosts.load(path, "hostsbad")
|
readers.load(path, "hostsbad")
|
||||||
|
|
||||||
|
|
||||||
def test_the_contract_check_names_each_problem() -> None:
|
def test_the_contract_check_names_each_problem() -> None:
|
||||||
@@ -126,7 +126,7 @@ def test_the_contract_check_names_each_problem() -> None:
|
|||||||
module.page = lambda get, slug, cursor: None
|
module.page = lambda get, slug, cursor: None
|
||||||
module.probe_ok = lambda body: True
|
module.probe_ok = lambda body: True
|
||||||
module.RANK, module.KEEP_CASE = "1", "yes"
|
module.RANK, module.KEEP_CASE = "1", "yes"
|
||||||
found = atshosts.problems(module, "odd")
|
found = readers.problems(module, "odd")
|
||||||
assert "NAME is 'other', not the module's name 'odd'" in found
|
assert "NAME is 'other', not the module's name 'odd'" in found
|
||||||
assert "LABEL is not a non-empty string" in found
|
assert "LABEL is not a non-empty string" in found
|
||||||
assert any(p.startswith("PATTERNS has '('") for p in found)
|
assert any(p.startswith("PATTERNS has '('") for p in found)
|
||||||
@@ -294,7 +294,7 @@ def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
|||||||
|
|
||||||
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
||||||
state: dict[str, str] | None = None) -> tuple[list[Any], Any]:
|
state: dict[str, str] | None = None) -> tuple[list[Any], Any]:
|
||||||
monkeypatch.setitem(atshosts.HOSTS, "fake", module)
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||||
if state is not None:
|
if state is not None:
|
||||||
source.state = state
|
source.state = state
|
||||||
@@ -411,7 +411,7 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
|||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||||
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
||||||
atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
||||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||||
@@ -422,7 +422,7 @@ def test_fetch_board_reads_the_listing_only(
|
|||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
|
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
|
||||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
||||||
rows, complete = ats.fetch_board("fake:acme")
|
rows, complete = ats.fetch_board("fake:acme")
|
||||||
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
|
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
|
||||||
assert web["asked"] == [f"{base}/list"]
|
assert web["asked"] == [f"{base}/list"]
|
||||||
@@ -534,7 +534,7 @@ def _paged_host() -> types.ModuleType:
|
|||||||
|
|
||||||
module.page = page
|
module.page = page
|
||||||
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
|
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
|
||||||
assert not atshosts.problems(module, "fake")
|
assert not readers.problems(module, "fake")
|
||||||
return module
|
return module
|
||||||
|
|
||||||
|
|
||||||
@@ -574,7 +574,7 @@ def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
|||||||
stored, is read again on the next pull."""
|
stored, is read again on the next pull."""
|
||||||
base = "https://jobs.fake.example/acme"
|
base = "https://jobs.fake.example/acme"
|
||||||
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
|
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
||||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||||
roles: Any = source.fetch() # a generator, which `close` stops
|
roles: Any = source.fetch() # a generator, which `close` stops
|
||||||
assert next(roles).body == "one"
|
assert next(roles).body == "one"
|
||||||
@@ -728,7 +728,7 @@ def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
|
|||||||
|
|
||||||
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
||||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||||
pytest.raises(ats._Disallowed):
|
pytest.raises(ats._Disallowed):
|
||||||
ats._check_robots(client, "fake:acme", {})
|
ats._check_robots(client, "fake:acme", {})
|
||||||
@@ -740,13 +740,13 @@ def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
|||||||
])
|
])
|
||||||
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
||||||
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
||||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||||
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
||||||
ats._check_robots(client, "fake:acme", {})
|
ats._check_robots(client, "fake:acme", {})
|
||||||
|
|
||||||
|
|
||||||
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||||
pytest.raises(ats._Refused):
|
pytest.raises(ats._Refused):
|
||||||
ats._get(client, ROBOTS, "fake", "acme")
|
ats._get(client, ROBOTS, "fake", "acme")
|
||||||
@@ -756,7 +756,7 @@ def test_fetch_board_honours_robots_txt(
|
|||||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||||
with pytest.raises(ats._Disallowed) as caught:
|
with pytest.raises(ats._Disallowed) as caught:
|
||||||
ats.fetch_board("fake:acme")
|
ats.fetch_board("fake:acme")
|
||||||
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
||||||
Reference in new issue
Block a user