diff --git a/docs/plan-hosts-and-by-hand.md b/docs/plan-hosts-and-by-hand.md index e5c5cf3..47505eb 100644 --- a/docs/plan-hosts-and-by-hand.md +++ b/docs/plan-hosts-and-by-hand.md @@ -46,32 +46,52 @@ The 49 were missed because the slug is not the company's name (`BoschGroup`, `ji ## Contracts between packages -Host module, `plugins/atshosts/.py`: +Host module, `plugins/atshosts/.py`, as built by WP0. The docstring of `plugins/atshosts/__init__.py` holds the same contract and is the one to keep current. A module imports `desk` and `._shared` only, never `plugins.ats`. ```python -NAME = "workday" # the host of a target `workday:` -LABEL = "Workday" # the Sources page's select -PATTERNS = (r"https?://...",) # discover: over a URL, group 1 is the slug +NAME = "workday" # the host of a target `workday:`; equals the module's file name +LABEL = "Workday" # the Sources page's select +PATTERNS = (r"https?://...",) # discovery: regexes over a URL, compiled with IGNORECASE +RANK = 100 # optional int, default 100; lower comes first +KEEP_CASE = True # optional, default False; keep a discovered slug's case -def slug_ok(slug: str) -> bool: ... # strict; a slug that fails is no target -def allowed(slug: str) -> Callable[[str], bool]: ... # hostnames a fetch may reach -def careers_url(slug: str) -> str: ... # the page a person opens -def robots_page(slug: str) -> str: ... # optional; the URL robots.txt must allow +def slug_ok(slug: str) -> bool: ... # strict, re.fullmatch +def allowed(slug: str) -> Callable[[str], bool]: ... # host names a fetch may reach +def careers_url(slug: str) -> str: ... # the page a person opens # one of the two -def feed_url(slug: str) -> str: ... +def feed_url(slug: str) -> str: ... # the whole board in one answer def parse(body: bytes, slug: str) -> list[Row]: ... -def page(get: Get, slug: str, cursor: Any) -> Page: ... +def page(get: Get, slug: str, cursor: Any) -> Page: ... # a paged board, one page a call def rows(decoded: Any, slug: str) -> list[Row]: ... # optional -def detail(get: Get, slug: str, row: Row) -> Row: ... -def probe_url(slug: str) -> str: ... -def probe_ok(body: bytes) -> bool: ... +def fallback_url(slug: str) -> str: ... # feed hosts: a smaller feed, read once when feed_url is over the byte cap +def robots_page(slug: str) -> str: ... # the URL robots.txt must allow; on a host allowed(slug) admits +def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: ... # fields from the role's own page +def probe_url(slug: str) -> str: ... # discovery: answers 2xx only for a board +def probe_ok(body: bytes) -> bool: ... # discovery: when 2xx is not enough +def probe_slugs(guess: str) -> tuple[str, ...]: ... # the slugs a guessed name is probed as; default (guess,) ``` -`Get` is `get(url, *, method="GET", payload=None, headers=None) -> bytes`. `Row` and `Page` are as in `plugins/ats.py` today. `plugins/atshosts/__init__.py` lists the modules with `pkgutil.iter_modules`, sorted by name, so a new host registers by existing. +`Get`, `Row` and `Page` are in `plugins/atshosts/_shared.py`, with the helpers several hosts share: `only(host)` for `allowed`, `path_slug_ok` and `label_slug_ok` for `slug_ok`, `parse_dt`, `is_web_url`, `num`, `yearly_amount` and `pay_line`. `plugins/atshosts/__init__.py` lists the modules with `pkgutil.iter_modules`, checks each against the contract at import (an ImportError names the module and what it lacks), and orders `HOSTS` by `(RANK, NAME)`. The eight first hosts hold ranks 10 to 80, which keeps the order they had before the modules: the select opens on Greenhouse, and a guessed slug on two hosts goes to the one probed first. + +Order and names. `RANK` orders the Sources page's select, the plugin's description and discovery's probes. `NAME` equals the file name. A host that `discover.py` names in its own `_UNREADABLE` table is taken from its module as soon as the module exists, so a wave 1 package does not edit `discover.py`. + +Patterns. The slug is group 1, or, when a pattern has more groups, the groups that matched joined by "/" (`https?://([a-z0-9-]+\.wd\d+)\.myworkdayjobs\.com/(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)` gives `lego.wd103/LEGO_External`). Only group 1 is checked against discovery's words that name no board (`api`, `jobs`, `embed`, ...). The slug is lowercased unless the module sets `KEEP_CASE = True`, which a host with case-sensitive paths needs, since robots.txt paths are case-sensitive too. A pattern without a group names the host with no slug. + +Slugs. `slug_ok` uses `re.fullmatch` (a `$` lets a trailing newline through) and refuses anything that could add a path, query, fragment, port or user to the URLs built from it. A slug of several parts joins them with "/": Lever's EU boards are `lever:eu/`. + +`get(url, *, method="GET", payload=None, headers=None) -> bytes` is `ats._get` bound to the board. A POST sends `payload` as JSON; `headers` are added to the default User-Agent and Accept and replace either where they name it. It fetches https on port 443 only, never an IP address or a local name (`localhost`, `.local`, `.internal`, `.lan`, `.home.arpa`), follows at most three redirects each checked against `allowed(slug)`, cuts the body at `MAX_BYTES` and ends within `DEADLINE`. It returns the body and raises `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError` on a timeout or a connection failure, and another exception, `ValueError` among them, for a refused host, an oversized body or a broken redirect. A module lets these through. + +`Page` is `(decoded, results, total, cursor)`: the decoded page handed to `rows`, its raw results as dicts with an `id` (counted by distinct id for completeness), the total the host states or None, and the next cursor or None after the last page. The first call gets cursor None. A board with a total is complete when as many distinct ids were read, one without when the read ended on an empty page; a 429 after the first page ends the read, incomplete. A paged module may also have `parse(body, slug)` for one page's body, for its tests. + +A `Row` has `source_id`, `title`, `location`, `url`, `posted` (aware datetime or None), `department`, `description` (plain text), `remote` (True, False, a word or None) and `salary_text`, and, where the host states pay, `salary_min`, `salary_max` (yearly integers) and `currency`. + +`detail` returns a dict of what the role's own page states. The shared code calls it for roles not yet read (`ats:detailed:`), at most `DETAIL_CAP` (200) per board per pull, `PAGE_DELAY` apart, the roles never tried before those whose detail failed (`ats:detailfailed:`). Only `DETAIL_FIELDS` are taken: `description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` and `posted`, the keys whose columns survive a later pull that sends the listing's empty value (`upsert_role`'s keep-filled columns, the salary figures, and `posted_at`, which keeps the earliest date). Any other key is dropped with a debug log line: `title`, `url`, `department` and `salary_text` cannot be filled from `detail`, since the listing rewrites them every pull. A role not read in detail on a pull is sent with an empty description. A 404 or 410 counts as read, any other failure is retried on a later pull, and a 429 ends the detail reads for that board and pull. Completeness is decided by the listing alone. + +robots.txt. For a module with `robots_page`, the shared code reads `https:///robots.txt` before the listing, at most once a day per board (`ats:robots:`), following a redirect to the host's `www.` twin or back and no other. It fails the board with "robots.txt disallows it" when the rules for `*` or for the app's own token disallow the page, matched per RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a final `$` in paths, paths case-sensitive, an empty Disallow allows. A 4xx other than 429 means there are no rules, which allows. A 5xx, a 429, a timeout, a connection error or a redirect elsewhere fails the board with "robots.txt could not be read", and the next pull asks again. Slugs: Workday `lego.wd103/LEGO_External`; SuccessFactors, Phenom and the sitemap reader take the site's hostname, with a path where the site has one (`jobs.teradyne.com/MiR`); Oracle `/`. diff --git a/plugins/atshosts/__init__.py b/plugins/atshosts/__init__.py index 0538693..efe0623 100644 --- a/plugins/atshosts/__init__.py +++ b/plugins/atshosts/__init__.py @@ -1,48 +1,135 @@ """The hosts the `ats` plugin reads, one module each. A host registers by existing: every module in this package whose name does -not start with `_` is loaded, in order of its name, and checked against the -contract below when the package is imported, so a module that breaks it fails -the import with a message naming the module and what it lacks, rather than a -pull that fails board by board. `plugins/ats.py` builds its allow-list, -feeds and pages from `HOSTS`, and `app/jobs/discover.py` its patterns and -probes. The package sits beside `ats.py` rather than being `plugins/ats/`, -since `desk.sources.registry` loads `plugins/*.py` by path and the tests -patch names on the `ats` module. +not start with `_` is loaded when the package is imported, checked against +the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A +module that breaks the contract fails the import with an ImportError naming +the module and what it lacks, rather than a pull that fails board by board. +`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host +select and its description from `HOSTS`, and `app/jobs/discover.py` its +patterns and probes. The package sits beside `ats.py` rather than being +`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and +the tests patch names on the `ats` module. -A module is named after its host and holds: +A module is named after its host (`plugins/atshosts/workday.py`) and imports +`desk` and `._shared` only, never `plugins.ats`. It holds: - NAME = "lever" # the host of a target `lever:`, and the module's name - LABEL = "Lever" # what the Sources page's select shows - PATTERNS = (r"...",) # discovery: over a URL; the slug is group 1, or the - # non-empty groups joined by "/" when there are more + NAME = "workday" # the host of a target `workday:`; equals the module's name + LABEL = "Workday" # what the Sources page's select shows + PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE + RANK = 100 # optional int; lower comes first in the select and the probes + KEEP_CASE = False # optional; True keeps a discovered slug's case - def slug_ok(slug) -> bool # strict; a slug that fails is no target - def allowed(slug) -> Callable[[str], bool] # the host names a fetch may reach - def careers_url(slug) -> str # the page a person opens + def slug_ok(slug: str) -> bool # strict, see below + def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach + def careers_url(slug: str) -> str # the page a person opens and one of two ways to read a board: - def feed_url(slug) -> str # the whole board in one answer, - def parse(body: bytes, slug) -> list[Row] # read with `parse` + def feed_url(slug: str) -> str # the whole board in one answer, + def parse(body: bytes, slug: str) -> list[Row] # read with `parse` - def page(get: Get, slug, cursor) -> Page # a paged board, one page a call, - def rows(decoded, slug) -> list[Row] # its rows from `page`'s decoded page + def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call, + def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page and, optionally: - def fallback_url(slug) -> str # a smaller feed, read once when - # `feed_url`'s answer is over the byte cap - def robots_page(slug) -> str # the URL the site's robots.txt must allow - def detail(get: Get, slug, row: Row) -> Row # the row filled from the role's own page - def probe_url(slug) -> str # discovery: answers 2xx only for a board - def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough - def probe_slugs(guess) -> tuple[str, ...] # the slugs a guessed name is probed as + def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read + # once when `feed_url` is over the cap + def robots_page(slug: str) -> str # the URL robots.txt must allow + def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page + def probe_url(slug: str) -> str # discovery: 2xx only for a board + def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough + def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as -`Get`, `Row` and `Page` are in `_shared`, with the helpers several hosts use. -A host module imports nothing from `plugins.ats`: the fetch it is handed is -`ats._get` bound to the board, so every request a host makes passes the same -allow-list, redirect rule, byte cap and deadline. +**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts +hold 10 to 80, which keeps the order they had before there were modules, so +a new host comes after them unless it says otherwise. The order decides the +select's default and which host wins when discovery's guessed slug exists on +two. + +**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`, +which lets a trailing newline through) and refuses anything that could add a +path, a query, a fragment, a port or a user to the URLs built from it. +`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok` +(one DNS label) cover the usual cases; a slug with several parts joins them +with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`). + +**Patterns.** The slug is group 1, or, when a pattern has more than one +group, the groups that matched joined by "/". Only group 1 is checked +against discovery's words that name no board (`api`, `jobs`, `embed`, ...). +The slug is lowercased unless `KEEP_CASE` is True: set it when the host's +paths are case-sensitive (Workday's site), since robots.txt paths are too. A +pattern without a group names the host with no slug. A host that +`discover.py` names in its own `_UNREADABLE` table is taken from its module +instead as soon as the module exists. + +**Fetching.** `allowed(slug)` returns a predicate over a lowercased host +name; `_shared.only(host)` is the usual one. Every request goes through +`get`, which is `ats._get` bound to the board: + + get(url, *, method="GET", payload=None, headers=None) -> bytes + +A POST sends `payload` as JSON (`{}` when None); `headers` are added to the +default User-Agent and `Accept: application/json, text/xml`, and replace +either where they name it. `get` is https only on port 443, never to an IP +address or a local name (`localhost`, `.local`, `.internal`, ...), follows at +most three redirects each checked against `allowed(slug)`, cuts the body at +`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns +the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read +`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection +failure, and another exception (`ValueError` among them) for a refused host, +an oversized body or a broken redirect. A module lets these through; the +shared code logs, records and decides. + +**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page +handed to `rows`, its raw results as dicts with an `id` (counted, by distinct +id, to decide completeness), the total the host states or None, and the +cursor of the next page or None after the last. The first call gets cursor +None. A board with a total is complete when as many distinct ids were read; +one without, when the read ended on an empty page. A 429 after the first page +ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY` +apart. A paged module may also have `parse(body, slug)` for one page's body. + +**Rows.** A `Row` is a dict with `source_id` (the host's own id for the +role, unique on the board), `title`, `location`, `url` (http or https, +checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see +`_shared.parse_dt`), `department`, `description` (plain text, see +`desk.sources.strip_html`), `remote` (True, False, a word or None) and +`salary_text`; and, where the host states pay, `salary_min`, `salary_max` +(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`. + +**Detail.** `detail(get, slug, row)` reads one role's own page and returns a +dict of the fields it found. The shared code calls it for roles not yet read +(`ats:detailed:`), at most `ats.DETAIL_CAP` (200) a board a pull, +`ats.PAGE_DELAY` apart, the roles never tried before those that failed +(`ats:detailfailed:`). Only `ats.DETAIL_FIELDS` are taken: +`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max` +and `posted`, the keys whose columns survive a later pull that sends the +listing's empty value; any other key is left out with a debug log line. +`title`, `url`, `department` and `salary_text` cannot be filled from +`detail`: the listing rewrites them every pull. A role not read in detail +on a pull is sent with an empty description, so the one read earlier stays. +A 404 or a 410 counts as read; any other failure is logged and retried on a +later pull; a 429 ends the detail reads of that board for the pull. +Completeness is decided by the listing alone. + +**robots.txt.** With `robots_page(slug)`, the shared code reads +`https:///robots.txt` through `get` before the listing, at most +once a day per board (`ats:robots:`), following a redirect to the +host's `www.` twin or back and no other. The board fails with "robots.txt +disallows it" when the rules for `*` or for the app's own token disallow the +page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a +final `$` in paths, case-sensitive paths). A 4xx other than 429 means no +rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails +the board with "robots.txt could not be read" and is asked again next pull. +`robots_page` must be on a host `allowed(slug)` admits. + +**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when +the slug is a board; `probe_ok(body)` narrows that when a host answers an +empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed +name is probed as, `(guess,)` without it. A host without `probe_url` is never +probed, only found by a link. """ from __future__ import annotations diff --git a/plugins/atshosts/_shared.py b/plugins/atshosts/_shared.py index 280f5d5..c31441f 100644 --- a/plugins/atshosts/_shared.py +++ b/plugins/atshosts/_shared.py @@ -34,7 +34,8 @@ class Get(Protocol): allow-list, the redirect rule, the byte cap and the deadline apply to every request a host makes. A POST sends `payload` as JSON; `headers` are added to (and may replace) the default User-Agent and Accept. Returns the - body, and raises on a non-2xx answer as `httpx` does.""" + body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see + "Fetching" in the package's docstring for the rest.""" def __call__(self, url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,