Merge main into wp-review-limits

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 16:01:30 +02:00
commit 750d3d1311
62 files changed
+1578 -538

No files matched your search

+1 -1
View File
@@ -1 +1 @@
{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]}
{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":50000,"payTransparencyMaxSalary":70000,"payTransparencySalaryCurrency":"EUR","payTransparencySalaryPayPeriod":"Yearly","payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]}
File renamed without changes.
File renamed without changes.
+1 -1
View File
@@ -1 +1 @@
{"data":[{"id":"581878","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe will help you \u003cstrong\u003egrow your career:\u003c/strong\u003e We help you identify areas of growth for you, whether you want to grow as a specialist in your field or eventually want to go into a leadership/management role.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person from time to time.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will receive an\u003cstrong\u003e individual education budget \u003c/strong\u003eand can educate yourself during work hours: you can spend several hours each month attending courses, reading books, or getting involved in communities.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFull-time position, employed through GI Group, our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eLooking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Job Benefits","compensation":null,"compensation_minimum":null,"compensation_maximum":null,"compensation_currency":null,"compensation_frequency":null,"compensation_visible":false,"deadline_at":null,"description":"\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eData Detective Wanted to join our Data \u0026amp; Analytics Team\u003c/strong\u003e \u003cbr\u003e\u003cbr\u003e\u003cem\u003eLocation: Bucharest, Romania\u003c/em\u003e\u003cbr\u003e\u003cbr\u003eAs a \u003cstrong\u003eData Operations Analyst \u003c/strong\u003eat Xeneta, you’ll learn how to master complex data analytics within the world of supply chain and logistics. Xeneta is one of the world's leading data platforms providing market intelligence to the ocean and air freight industry. We have a big team of skilled Data Analysts that will help you learn the ropes from the ground up, while enabling you to collect and analyze data in one of the world’s greatest container shipping datasets. \u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eBring your analytical mindset and a genuine curiosity for how global shipping works and join our team!\u0026nbsp;\u003c/div\u003e","employment_type":"permanent_full_time","employment_type_text":"Permanent - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eJoin our Liner Data \u0026amp; Analytics Team (LDAT) and own a portfolio of container shipping services for theLine truncated
{"data":[{"id":"581878","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe will help you \u003cstrong\u003egrow your career:\u003c/strong\u003e We help you identify areas of growth for you, whether you want to grow as a specialist in your field or eventually want to go into a leadership/management role.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person from time to time.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will receive an\u003cstrong\u003e individual education budget \u003c/strong\u003eand can educate yourself during work hours: you can spend several hours each month attending courses, reading books, or getting involved in communities.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFull-time position, employed through GI Group, our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eLooking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Job Benefits","compensation":null,"compensation_minimum":60000,"compensation_maximum":80000,"compensation_currency":"EUR","compensation_frequency":"annually","compensation_visible":true,"deadline_at":null,"description":"\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eData Detective Wanted to join our Data \u0026amp; Analytics Team\u003c/strong\u003e \u003cbr\u003e\u003cbr\u003e\u003cem\u003eLocation: Bucharest, Romania\u003c/em\u003e\u003cbr\u003e\u003cbr\u003eAs a \u003cstrong\u003eData Operations Analyst \u003c/strong\u003eat Xeneta, you’ll learn how to master complex data analytics within the world of supply chain and logistics. Xeneta is one of the world's leading data platforms providing market intelligence to the ocean and air freight industry. We have a big team of skilled Data Analysts that will help you learn the ropes from the ground up, while enabling you to collect and analyze data in one of the world’s greatest container shipping datasets. \u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eBring your analytical mindset and a genuine curiosity for how global shipping works and join our team!\u0026nbsp;\u003c/div\u003e","employment_type":"permanent_full_time","employment_type_text":"Permanent - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eJoin our Liner Data \u0026amp; Analytics Team (LDAT) and own a portfolio of container shipping servicesLine truncated
File renamed without changes.
File renamed without changes.
+3 -1
View File
@@ -16,7 +16,9 @@
"fullLocation": "Berlin, Germany"
},
"department": {"id": "2294038", "label": "Engineering, Infrastructure and Operations"},
"function": {"id": "engineering", "label": "Engineering"}
"function": {"id": "engineering", "label": "Engineering"},
"typeOfEmployment": {"id": "permanent", "label": "Full-time"},
"experienceLevel": {"id": "director", "label": "Director"}
},
{
"id": "744000147485112",
+68
View File
@@ -12,6 +12,11 @@ file holds `app`'s own layers.
there the door is the tool: `desk.tools.<tool>`, one level, no deeper.
3. **Plugins.** A file under `plugins/` imports `desk` and nothing else from
these trees. A plugin is app-agnostic; it never sees `app`.
4. **Readers.** A file of `plugins/readers/` imports the standard library,
`httpx`, `pydantic`, `desk` through its doors and, within the package,
`._contract`, `._shared` and `._countries` only: a reader shares no code
with another reader, nor with `_runner`, which reads the readers and so
alone may import the package's registry, `plugins.readers`.
Tests are held to none of this.
"""
@@ -21,6 +26,7 @@ from __future__ import annotations
import ast
import importlib
import importlib.util
import sys
from pathlib import Path
import pytest
@@ -88,6 +94,13 @@ def _package(path: Path) -> str:
APP_FILES = sorted(p for p in (ROOT / APP).rglob("*.py") if "__pycache__" not in p.parts)
PLUGIN_FILES = sorted((ROOT / "plugins").glob("*.py"))
READER_FILES = sorted((ROOT / "plugins" / "readers").glob("*.py"))
#: What a reader may import beside the standard library and `desk`.
READER_THIRD_PARTY = frozenset({"httpx", "pydantic"})
#: The package's own modules a reader may import.
READER_SIBLINGS = frozenset({"_contract", "_shared", "_countries"})
def _outside(here: str, source: str):
@@ -120,6 +133,38 @@ def plugin_violations(source: str) -> list[str]:
return wrong
def reader_violations(source: str, registry: bool = False) -> list[str]:
"""Every import in a file of `plugins/readers/` outside rule 4;
`registry` allows `from plugins import readers`, for `_runner`."""
wrong = []
for node in ast.walk(ast.parse(source)):
if registry and isinstance(node, ast.ImportFrom) and node.level == 0 \
and node.module == "plugins" and [a.name for a in node.names] == ["readers"]:
continue
if isinstance(node, ast.Import):
tops = [(a.name.split(".")[0], a.name) for a in node.names]
elif isinstance(node, ast.ImportFrom) and node.level == 0:
tops = [((node.module or "").split(".")[0], node.module or "")]
elif isinstance(node, ast.ImportFrom):
names = {node.module} if node.module else {a.name for a in node.names}
if node.level != 1 or not names <= READER_SIBLINGS:
wrong.append(f"line {node.lineno}: {'.' * node.level}{node.module or ''} "
f"{sorted(names)} (a reader imports {', '.join(sorted(READER_SIBLINGS))})")
continue
else:
continue
for top, module in tops:
if top == "desk":
if isinstance(node, ast.ImportFrom) and (
reason := _door_violation(module, [a.name for a in node.names
if not _is_module(f"{module}.{a.name}")])):
wrong.append(f"line {node.lineno}: {module} ({reason})")
elif top not in sys.stdlib_module_names and top not in READER_THIRD_PARTY:
wrong.append(f"line {node.lineno}: {module} (a reader imports the standard "
f"library, httpx, pydantic and desk)")
return wrong
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
def test_a_package_imports_only_from_below_it(path: Path) -> None:
wrong = direction_violations(_package(path), path.read_text(encoding="utf-8"))
@@ -141,6 +186,12 @@ def test_a_plugin_knows_only_desk(path: Path) -> None:
assert not wrong, "\n".join(wrong)
@pytest.mark.parametrize("path", READER_FILES, ids=lambda p: p.name)
def test_a_reader_knows_only_its_contract_and_helpers(path: Path) -> None:
wrong = reader_violations(path.read_text(encoding="utf-8"), path.name == "_runner.py")
assert not wrong, "\n".join(wrong)
def test_the_rules_catch_what_they_are_for() -> None:
"""The checker itself, on sources that break each rule."""
assert plugin_violations("from app.storage import Store")
@@ -148,6 +199,23 @@ def test_the_rules_catch_what_they_are_for() -> None:
assert plugin_violations("from desk.sources.base import Source")
assert not plugin_violations("from desk.sources import Source, Item")
assert reader_violations("from plugins import ats")
assert reader_violations("from plugins import readers")
assert not reader_violations("from plugins import readers", registry=True)
assert reader_violations("from plugins import ats", registry=True)
assert reader_violations("from app.prep import words")
assert reader_violations("from . import _runner")
assert reader_violations("from ._runner import fetch")
assert reader_violations("from .lever import parse")
assert reader_violations("from .. import ats")
assert reader_violations("import requests")
assert reader_violations("from desk.sources.base import Source")
assert not reader_violations("import json\nfrom urllib.parse import quote\nimport httpx\n"
"from pydantic import ConfigDict\n"
"from desk.sources import strip_html\n"
"from . import _contract\nfrom ._shared import only\n"
"from ._countries import code_of")
assert direction_violations("storage", "from app.web import TABS")
assert direction_violations("web", "from app.main import create_app")
assert not direction_violations("main", "from app.web import pages_router")
+2 -2
View File
@@ -35,10 +35,10 @@ def test_the_employment_type_is_the_hosts_word() -> None:
assert (first["employment"], second["employment"]) == ("FullTime", "Contract")
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
reads = ashby.READS
assert (reads.listing, reads.text, reads.places, reads.empty) == (
"whole", "listing", "many", "suspect")
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
assert ashby.feed_url("acme", Ask()).endswith("/job-board/acme?includeCompensation=true")
-2
View File
@@ -156,7 +156,6 @@ def test_it_declares_what_it_reads() -> None:
assert (reads.listing, reads.text, reads.places, reads.fields.posted) == (
"whole", "detail", "one", "trusted")
assert reads.robots.startswith("read: robots.txt allows /careers")
assert not hasattr(bamboohr, "robots_page")
def test_the_slug_the_urls_and_the_allow_list() -> None:
@@ -188,4 +187,3 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None:
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
'<a href="https://resources.bamboohr.com/y">z</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
-1
View File
@@ -113,7 +113,6 @@ def test_it_declares_a_whole_listing_read_from_the_page() -> None:
assert (reads.listing, reads.text, reads.places, reads.complete) == (
"whole", "listing", "many", "whole")
assert reads.fields.pay == "line" and "/api/ is disallowed and not read" in reads.robots
assert not hasattr(deel, "robots_page")
# ------------------------------------------------------- slugs and hosts
+1 -3
View File
@@ -201,7 +201,6 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None
found = [(h, s) for h, s, _ in discover.boards_in(page)]
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
assert ("eightfold", "") in found
assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None:
@@ -210,5 +209,4 @@ def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request(
"paged", 10, "detail", "stated", "many")
assert reads.filters.declared() == {"country": "one", "query": "one"}
assert reads.fields.posted == "listed"
assert reads.robots.startswith("read: robots.txt allows /careers")
assert not hasattr(eightfold, "robots_page")
assert "/api/pcsx/search" in reads.robots and "/api/pcsx/position_details" in reads.robots
+1 -3
View File
@@ -103,8 +103,7 @@ def test_it_declares_one_page_that_may_be_empty_by_mistake() -> None:
reads = gem.READS
assert (reads.listing, reads.text, reads.places, reads.empty) == (
"paged", "detail", "many", "suspect")
assert reads.robots.startswith("read: robots.txt allows /<board>")
assert not hasattr(gem, "robots_page")
assert "POST /api/public/graphql" in reads.robots
@pytest.mark.parametrize(("body", "message"), [
@@ -179,4 +178,3 @@ def test_discovery_finds_the_board_keeping_its_case() -> None:
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
("gem", "Productboard"), ("gem", "sennder-gmbh")]
assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
+2 -2
View File
@@ -43,10 +43,10 @@ def test_without_offices_the_name_is_the_place() -> None:
assert nothing["places"] == [] and nothing["location"] == ""
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
reads = greenhouse.READS
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
assert greenhouse.feed_url("acme", Ask()).endswith("/boards/acme/jobs?content=true&pay_transparency=true")
assert greenhouse.fallback_url("acme").endswith("/boards/acme/jobs")
+1 -2
View File
@@ -163,8 +163,7 @@ def test_it_declares_copies_with_a_page_and_a_robots_decision() -> None:
reads = hibob.READS
assert (reads.listing, reads.text, reads.places, reads.complete) == (
"paged", "listing", "copies", "end")
assert reads.robots.startswith("read: robots.txt allows")
assert not hasattr(hibob, "robots_page")
assert reads.robots.startswith("read: /api/job-ad")
# ------------------------------------------------------- slugs and hosts
+2 -3
View File
@@ -122,9 +122,8 @@ def test_the_feed_states_no_place_and_the_robots_decision_is_the_owners() -> Non
assert all(r["places"] == [] for r in rows)
reads = hrmanager.READS
assert (reads.listing, reads.page_size, reads.text, reads.places) == (
"paged", 100, "listing", "one")
assert reads.robots == "disallowed by robots.txt; owner to decide"
assert not hasattr(hrmanager, "robots_page")
"paged", 100, "listing", "none")
assert "api.hr-manager.net" in reads.robots and "owner, 2026-09-29" in reads.robots
# ------------------------------------------------------- slugs and hosts
-2
View File
@@ -176,7 +176,6 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
assert "jibe" not in {h for h, _ in discover._UNREADABLE}
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
@@ -184,4 +183,3 @@ def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
"paged", 100, "listing", "stated", "many")
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")
assert not hasattr(jibe, "robots_page")
+5 -6
View File
@@ -87,8 +87,8 @@ def _get(pages: dict[str, bytes], asked: list[str] | None = None) -> Any:
def test_detail_reads_the_role_pages_json_ld() -> None:
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
asked: list[str] = []
pages = {ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
pages = {ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
first = jobylon.detail(_get(pages, asked), "3143", rows["385191"])
assert asked == [ROLE]
assert set(first) <= set(ats.DETAIL_FIELDS) | {"places"}
@@ -116,8 +116,8 @@ def test_detail_without_a_job_posting_fails_the_read() -> None:
def test_the_board_is_one_request_and_each_role_is_read_on_its_own_page() -> None:
asked: list[str] = []
pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
get = _get(pages, asked)
rows = jobylon.parse(get(jobylon.feed_url("3143", Ask())), "3143")
assert len(rows) == 5 and asked == [FEED]
@@ -150,8 +150,7 @@ def test_it_declares_a_whole_listing_with_a_teaser_and_detail() -> None:
assert (reads.listing, reads.text, reads.teaser, reads.places) == (
"whole", "detail", True, "many")
assert (reads.fields.posted, reads.fields.seniority) == ("trusted", "stated")
assert reads.robots.startswith("read: robots.txt allows /jobs/companies/")
assert not hasattr(jobylon, "robots_page")
assert "/jobs/companies/<id>/embed/v2/" in reads.robots
# ------------------------------------------------------- slugs and hosts
+2 -2
View File
@@ -33,9 +33,9 @@ def test_the_commitment_is_the_hosts_word() -> None:
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
reads = lever.READS
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
assert reads.fields.posted == "trusted" and reads.fields.country == "stated"
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
assert lever.feed_url("eu/tomtom", Ask()) == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
-1
View File
@@ -253,7 +253,6 @@ def test_the_careers_page_is_the_candidate_site() -> None:
url = f"https://{HOST}/hcmUI/CandidateExperience/en/sites/CX_1/jobs"
assert oracle.careers_url(SLUG) == ats.careers_url(f"oracle:{SLUG}") == url
assert oracle.allowed(SLUG)(httpx.URL(url).host)
assert not hasattr(oracle, "robots_page") # decided when built, see READS.robots
def test_a_slug_written_with_capitals_names_the_same_pod() -> None:
+1 -2
View File
@@ -24,8 +24,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
reads = personio.READS
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
assert reads.robots.startswith("api: no robots.txt check needed")
assert not hasattr(personio, "robots_page")
assert reads.robots.startswith("read: /xml")
assert personio.feed_url("acme", Ask()) == "https://acme.jobs.personio.de/xml"
-1
View File
@@ -289,7 +289,6 @@ def test_the_careers_page_is_the_search_page() -> None:
url = "https://careers.king.com/us/en/search-results"
assert phenom.careers_url(SLUG) == ats.careers_url(f"phenom:{SLUG}") == url
assert phenom.allowed(SLUG)(httpx.URL(url).host)
assert not hasattr(phenom, "robots_page") # decided when built, see READS.robots
assert phenom.careers_url("www.careers.philips.com/global/en") == \
"https://www.careers.philips.com/global/en/search-results"
-2
View File
@@ -116,7 +116,6 @@ def test_the_board_is_read_through_the_app_and_is_complete() -> None:
def test_the_slug_the_urls_and_the_allow_list() -> None:
assert pinpoint.slug_ok("xeneta") and pinpoint.slug_ok("Xeneta") and pinpoint.slug_ok("a-b1")
assert pinpoint.careers_url("xeneta") == "https://xeneta.pinpointhq.com/"
assert not hasattr(pinpoint, "robots_page") # decided when built, see READS.robots
assert pinpoint.feed_url("Xeneta") == "https://xeneta.pinpointhq.com/postings.json"
allow = pinpoint.allowed("Xeneta")
assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com")
@@ -146,7 +145,6 @@ def test_discovery_finds_the_subdomain() -> None:
'<script src="https://www.pinpointhq.com/embed.js"></script>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
assert discover.boards_in("https://pinpointhq.com/") == []
assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_a_pinpoint_board_is_not_probed() -> None:
+1 -1
View File
@@ -25,7 +25,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
reads = recruitee.READS
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
assert reads.fields.posted == "trusted" and reads.fields.country == "none"
assert reads.robots.startswith("api: no robots.txt check needed")
assert "/api/offers/ is allowed" in reads.robots
assert recruitee.feed_url("acme", Ask()) == "https://acme.recruitee.com/api/offers/"
+2 -3
View File
@@ -82,7 +82,7 @@ def _get(answer: bytes, asked: list[str] | None = None) -> Any:
def test_detail_reads_description_places_date_and_a_single_yearly_range() -> None:
row = {r["source_id"]: r for r in rippling.parse(BOARD, "rippling")}[DIRECTOR]
asked: list[str] = []
found = rippling.detail(_get((FIXTURES / "rippling-role.json").read_bytes(), asked),
found = rippling.detail(_get((FIXTURES / "rippling-detail.json").read_bytes(), asked),
"rippling", row)
assert asked == [f"https://api.rippling.com/platform/api/ats/v1/board/rippling/jobs/{DIRECTOR}"]
assert found["description"].startswith("About The Role")
@@ -96,7 +96,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non
def test_detail_without_pay_leaves_the_pay_columns_out() -> None:
found = rippling.detail(_get((FIXTURES / "rippling-role-2.json").read_bytes()),
found = rippling.detail(_get((FIXTURES / "rippling-detail-2.json").read_bytes()),
"prisma-careers", {"source_id": "x"})
assert found["description"].startswith("What is it like to work at Prisma?")
assert found["location"] == "Remote (Germany)"
@@ -165,7 +165,6 @@ def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
def test_the_urls() -> None:
assert rippling.feed_url("rippling") == FEED
assert rippling.careers_url("prisma-careers") == "https://ats.rippling.com/prisma-careers/jobs"
assert not hasattr(rippling, "robots_page") # decided when built, see READS.robots
allow = rippling.allowed("rippling")
assert allow("api.rippling.com") and allow("ATS.RIPPLING.COM")
for host in ("rippling.com", "www.rippling.com", "api.rippling.com.evil.example", "localhost"):
+1 -1
View File
@@ -35,7 +35,7 @@ def test_the_declaration_says_what_the_list_gives() -> None:
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.empty) == \
("paged", 100, "one", "detail", "suspect")
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
assert reads.robots.startswith("api: no robots.txt check needed")
assert reads.robots.startswith("read though") and "owner to decide" in reads.robots
def test_page_reads_by_offset_and_takes_the_ask() -> None:
-1
View File
@@ -451,7 +451,6 @@ def test_careers_url_is_the_search_page() -> None:
page = successfactors.careers_url(slug)
assert httpx.URL(page).host == slug.split("/")[0]
assert successfactors.allowed(slug)(httpx.URL(page).host)
assert not hasattr(successfactors, "robots_page") # decided when built, see READS.robots
def test_the_module_is_case_sensitive_about_the_prefix() -> None:
+1 -2
View File
@@ -24,8 +24,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.complete) == \
("paged", 50, "many", "listing", "end")
assert reads.fields.posted == "trusted" and reads.fields.remote == "stated"
assert reads.robots.startswith("api: no robots.txt check needed")
assert not hasattr(teamtailor, "robots_page")
assert "/jobs.rss is allowed" in reads.robots
def test_page_takes_the_ask_and_reads_the_same_url() -> None:
+1 -1
View File
@@ -30,7 +30,7 @@ def test_the_declaration_says_what_the_list_gives() -> None:
assert (reads.listing, reads.page_size, reads.places, reads.text) == \
("paged", 10, "many", "detail")
assert reads.fields.posted == "trusted" and reads.fields.employment == "stated"
assert reads.robots.startswith("api: no robots.txt check needed")
assert "/api/v3/accounts/<slug>/jobs" in reads.robots
def test_page_takes_the_ask_and_posts_the_same_body() -> None:
+4 -4
View File
@@ -446,10 +446,10 @@ def test_the_case_of_the_site_is_kept() -> None:
assert "workday" in ats.HOSTS and ats.HOSTS["workday"] == "Workday"
def test_the_runtime_robots_check_has_left_the_module() -> None:
"""Decided when the reader was built (`READS.robots`): nothing to ask now."""
assert not hasattr(workday, "robots_page") and not hasattr(workday, "ROBOTS_ANY_CASE")
assert "robots.txt allows /<Site>/" in workday.READS.robots
def test_the_robots_record_names_the_api_path() -> None:
"""Decided when the reader was built (`READS.robots`), for the paths it
requests, not only the career site's."""
assert "/wday/cxs/<tenant>/<site>/" in workday.READS.robots
# -------------------------------------------------------------- discovery
+7
View File
@@ -17,6 +17,7 @@ from app.jobs import discover as mod
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
from app.jobs.sources import seed_companies
from app.storage import Store
from plugins import readers
# ------------------------------------------------------------------ parsing
@@ -95,6 +96,12 @@ def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
def test_no_unreadable_entry_names_a_host_with_a_module() -> None:
"""A new module deletes its host's `_UNREADABLE` entry: its own patterns
replace it."""
assert not {host for host, _ in mod._UNREADABLE} & set(readers.HOSTS)
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
+231 -6
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import json
import sys
import threading
import types
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -21,8 +22,8 @@ if str(ROOT) not in sys.path:
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext, Tally # noqa: E402
from app.models import Application # noqa: E402
from plugins import readers # noqa: E402
from plugins.readers import _runner # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Filters, Reads, _runner # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app import prep # noqa: E402
from app.jobs import cascade, pull # noqa: E402
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0,
"roles": 1, "closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -163,7 +164,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0,
"roles": 0, "closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
@@ -256,7 +257,7 @@ def test_a_failing_board_keeps_its_roles_before_stale_days(
_answer(monkeypatch, ValueError("gone"))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1"}
assert (result["failed"], result["closed"]) == (1, 0)
assert (result["unanswered"], result["closed"]) == (1, 0)
since, reason = pull.failing_boards(acme)["workable:acme"]
assert since and reason == "ValueError: gone"
@@ -486,7 +487,7 @@ def test_run_a_settle_does_not_close_on_run_b_ids(
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
assert (result["failed"], result["closed"]) == (1, 0)
assert (result["unanswered"], result["closed"]) == (1, 0)
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
assert pull.running(acme) == b["run"]
@@ -507,3 +508,227 @@ def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> No
context = RunContext(waited_for=(failed,))
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
assert result["boards"] == 1 and pull.running(acme) == ""
# -------------------------------------------------------- losses, guarded
def _days_ago(days: float) -> str:
return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds")
def _set(store: Store, column: str, value: str, *source_ids: str) -> None:
with store.transaction() as conn:
conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?",
[(value, i) for i in source_ids])
@pytest.fixture
def big(store: Store) -> Store:
store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
board="greenhouse")
return store
def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails(
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
_pull(big)
_answer(monkeypatch, ([], True))
result = _pull(big)
assert len(_open(big)) == 40
assert (result["unanswered"], result["closed"]) == (1, 0)
assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1]
def test_a_whole_read_of_nothing_after_five_closes_the_board(
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(str(n)) for n in range(5)], True))
_pull(big)
_answer(monkeypatch, ([], True))
assert _pull(big)["closed"] == 5
assert _open(big) == set()
def test_a_suspect_answer_is_accepted_after_stale_days(
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
_pull(big)
_answer(monkeypatch, ([_row("1")], True))
_pull(big)
assert len(_open(big)) == 40
record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"])
big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps(
record | {"since": _days_ago(pull.STALE_DAYS + 1)})
result = _pull(big)
assert _open(big) == {"greenhouse:big:1"}
assert (result["unanswered"], result["closed"]) == (0, 39)
assert pull.failing_boards(big) == {}
assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1
def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]:
"""A whole-listing host `fake` that filters by country and, per `remote`,
on remote; its feed lists the ids in the returned list."""
listed: list[str] = []
module: object = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined]
module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined]
module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined]
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
module.READS = Reads( # type: ignore[attr-defined]
scope="board", listing="whole", text="listing", complete="whole", places="one",
robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type]
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined]
monkeypatch.setitem(readers.HOSTS, "fake", module)
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps(
listed).encode())
return listed
def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
listed = _remote_host(monkeypatch, remote="none")
store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake")
reach = {"fake:acme": {"countries": ["DK"], "remote": True}}
def pull_once() -> dict[str, object]:
run = pull.claim_run(store)
result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type]
reach=reach)
return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
listed[:] = ["1", "2", "3", "4"]
pull_once()
_set(store, "remote", "remote", "fake:acme:3", "fake:acme:4")
listed[:] = ["1"]
assert pull_once()["closed"] == 1
assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"}
assert store.state["ats:remote_unasked:fake:acme"] == "1"
_set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4")
assert pull_once()["closed"] == 1
assert _open(store) == {"fake:acme:1", "fake:acme:3"}
def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(i) for i in "123"], True))
_pull(acme)
def fake(*args: object, **kwargs: object) -> _runner.Listing:
return _runner.Listing([_row("1")], True, ["1"], narrowed=True)
monkeypatch.setattr(_runner, "read_listing", fake)
assert _pull(acme)["closed"] == 0
_set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3")
assert _pull(acme)["closed"] == 1
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")]
for target in targets:
slug = target.partition(":")[2]
store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse")
def fake_fetch(self: ATSSource) -> object:
[(target, _)] = self.targets()
yield _item("greenhouse", target.partition(":")[2], "1")
self.state[f"ats:ok:{target}"] = self.options["run"]
real = Store.upsert_role
def upsert(self: Store, item: Item, sub: object = None) -> bool:
if item.source_id.startswith("greenhouse:b:"):
raise ValueError("a row the store refuses")
return real(self, item, sub) # type: ignore[arg-type]
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
monkeypatch.setattr(Store, "upsert_role", upsert)
result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type]
assert result["roles"] == 2
assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"}
assert "ats:ok:greenhouse:b" not in store.state
since, reason = pull.failing_boards(store)["greenhouse:b"]
assert since and reason == "ValueError: a row the store refuses"
assert set(pull.failing_boards(store)) == {"greenhouse:b"}
def test_a_board_whose_rows_do_not_store_keeps_its_failing_record(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
since = _days_ago(2)
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
{"since": since, "reason": "404 Not Found"})
_answer(monkeypatch, ([_row("1")], True))
def refuse(self: Store, item: Item, sub: object = None) -> bool:
raise ValueError("no")
monkeypatch.setattr(Store, "upsert_role", refuse)
_pull(acme)
assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no")
assert "ats:ok:workable:acme" not in acme.state
# ------------------------------------------------------- the settle speaks
def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many(
store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
board="greenhouse")
store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone",
board="greenhouse")
for n in range(80):
store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub)
run = pull.claim_run(store)
store.state["ats:ok:greenhouse:big"] = run
store.state["ats:complete:greenhouse:big"] = "1"
store.state["ats:ids:greenhouse:big"] = json.dumps(
[f"greenhouse:big:{n}" for n in range(20)])
store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps(
{"since": _days_ago(1), "reason": "404 Not Found"})
said: list[object] = []
result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type]
context=RunContext(on_notice=said.append))
assert (result["unanswered"], result["closed"]) == (1, 60)
[notice] = said
assert notice.code == "pull.settled" # type: ignore[attr-defined]
assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined]
assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined]
assert "closed 60 roles" in notice.text # type: ignore[attr-defined]
def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row(i) for i in "12"], True))
said: list[object] = []
results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type]
pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type]
context=RunContext(on_notice=said.append))
assert said == []
def test_a_finder_that_raises_releases_the_pull_claim(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
def broken(store: Store, subs: object) -> dict[str, object]:
raise RuntimeError("reach broke")
monkeypatch.setattr(pull, "reaches", broken)
# The worker runs the chain's finder once the job is done; here at once.
context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme))
with pytest.raises(RuntimeError, match="reach broke"):
cascade.run_pull(FakeConfig(), acme, context=context)
assert pull.running(acme) == ""
# And the job itself lets go when the chain cannot be built.
def no_chain(run: str) -> object:
raise RuntimeError("no chain")
monkeypatch.setattr(cascade, "pull_chain", no_chain)
with pytest.raises(RuntimeError, match="no chain"):
cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True))
assert pull.running(acme) == ""
+136 -3
View File
@@ -26,7 +26,7 @@ if str(ROOT) not in sys.path:
from plugins import ats, readers # noqa: E402
from plugins.readers import _runner # noqa: E402
from plugins.readers import lever # noqa: E402
from plugins.readers._contract import Ask # noqa: E402
from plugins.readers._contract import Ask, Filters, Reads # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -64,8 +64,11 @@ PATTERNS = (r"good\\.example/([a-z]+)",)
def slug_ok(slug): return True
def allowed(slug): return lambda h: h == "good.example"
def careers_url(slug): return "https://good.example/" + slug
def feed_url(slug): return "https://good.example/" + slug + ".json"
def feed_url(slug, ask): return "https://good.example/" + slug + ".json"
def parse(body, slug): return []
from plugins.readers import Reads
READS = Reads(scope="board", listing="whole", text="listing", complete="whole", places="one",
robots="read (checked 2026-09-30)")
"""
@@ -253,7 +256,9 @@ def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
module.READS = Reads(scope="board", listing="whole", complete="whole", places="one",
text="detail" if detail is not None else "listing", robots="test")
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
@@ -590,3 +595,131 @@ def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
with _client(handler) as client, pytest.raises(ats._Refused):
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
# ------------------------------------------- the guard, asks, remote unasked
def _listing_host(**reads: Any) -> Any:
"""A declared whole-listing board `fake` whose feed lists `module.ids`,
whatever is asked; `reads` override its declaration."""
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
module.slug_ok = lambda slug: slug.isalnum()
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
"complete": "whole", "places": "one", "robots": "test"} | reads))
module.ids = []
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [
{"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
"places": [{"text": "Berlin", "remote": None}]} for i in json.loads(body)]
return module
def _listed(module: Any) -> Callable[..., bytes]:
return lambda url, **kwargs: json.dumps(module.ids).encode()
def _read_and_stamp(state: dict[str, str], module: Any,
limits: _runner.Limits | None = None) -> _runner.Listing:
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
return listing
def test_an_unfiltered_whole_read_of_nothing_after_forty_is_suspect() -> None:
"""A Greenhouse-style board, trusted, unfiltered: a changed answer that
parses to no rows is refused rather than read as a board that closed."""
module = _listing_host()
state: dict[str, str] = {}
module.ids = [str(n) for n in range(40)]
_read_and_stamp(state, module)
module.ids = []
with pytest.raises(_runner.Suspect, match="listed 0 roles, under 10% of the 40"):
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
def test_a_whole_read_under_suspect_min_is_not_held() -> None:
module = _listing_host()
state: dict[str, str] = {}
module.ids = [str(n) for n in range(5)]
_read_and_stamp(state, module)
module.ids = []
assert _read_and_stamp(state, module).ids == []
assert json.loads(state["ats:count:fake:acme"])["count"] == 0
def test_the_same_suspect_answer_is_accepted_after_stale_days() -> None:
module = _listing_host()
state: dict[str, str] = {}
module.ids = [str(n) for n in range(40)]
_read_and_stamp(state, module)
module.ids = ["1"]
failing = "ats:failing:fake:acme"
recent = (datetime.now(UTC) - timedelta(days=20)).isoformat(timespec="seconds")
state[failing] = json.dumps({"since": recent, "reason": "Suspect: listed 1 roles"})
limits = _runner.Limits(page_delay=0, stale_days=21)
with pytest.raises(_runner.Suspect):
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits)
old = (datetime.now(UTC) - timedelta(days=22)).isoformat(timespec="seconds")
state[failing] = json.dumps({"since": old, "reason": "Suspect: listed 1 roles"})
# Outside a pull (no `stale_days`) the guard never gives way.
with pytest.raises(_runner.Suspect):
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
assert _read_and_stamp(state, module, limits).ids == ["1"]
assert json.loads(state["ats:count:fake:acme"])["count"] == 1
def test_keep_size_resets_the_guards_count_unless_told_not_to() -> None:
state = {"ats:count:fake:acme": json.dumps({"asks": "[]", "count": 40})}
ats.keep_size(state, "fake:acme", 40, reset=False)
assert "ats:count:fake:acme" in state
ats.keep_size(state, "fake:acme", 3)
assert "ats:count:fake:acme" not in state
assert state["ats:size:fake:acme"] == "3"
def test_a_filter_that_maps_to_no_host_id_asks_nothing_and_is_incomplete(
monkeypatch: pytest.MonkeyPatch) -> None:
module = _listing_host(filters=Filters(country="ids"))
module.facets = lambda get, slug: {"country": {}}
module.ids = ["1", "2"]
state: dict[str, str] = {}
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
Ask(countries=("DK",)))
assert (listing.rows, listing.ids, listing.complete, listing.filtered) == (
[], [], False, True)
# Through the plugin: stamped, but not complete, so the settle closes nothing.
monkeypatch.setitem(readers.HOSTS, "fake", module)
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b'["1"]')
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
"params": {"countries": ["DK"]}}], "run": "r"})
assert list(source.fetch()) == []
assert (source.state["ats:ok:fake:acme"], source.state["ats:complete:fake:acme"]) == (
"r", "")
def test_a_filtered_read_that_cannot_ask_remote_says_so() -> None:
reach = {"countries": ["DK"], "remote": True}
blind = _listing_host(filters=Filters(country="many"))
ask = _runner.ask_for(blind, None, reach)
assert (ask.countries, ask.remote) == (("DK",), None)
assert _runner.remote_wanted(blind, None, reach)
listing = _runner.read_listing(blind, "acme", _listed(blind), ask,
remote=_runner.remote_wanted(blind, None, reach))
assert listing.filtered and listing.remote_unasked
state: dict[str, str] = {}
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
assert state["ats:remote_unasked:fake:acme"] == "1"
# A host that filters on remote asks it as a series of its own.
sees = _listing_host(filters=Filters(country="many", remote="many"))
ask = _runner.ask_for(sees, None, reach)
listing = _runner.read_listing(sees, "acme", _listed(sees), ask, remote=True)
assert listing.filtered and not listing.remote_unasked
# A params' remote of False asks nothing and wants nothing.
assert not _runner.remote_wanted(blind, {"remote": False}, reach)
+299 -89
View File
@@ -1,34 +1,38 @@
"""The reader contract, held for every module of `plugins/readers/`.
Per module: it declares `READS`, the declaration matches its functions, and
its rows keep what the contract says they keep. No network: fixtures stand
in for the host.
`UNDECLARED` lists the modules that do not declare `READS` yet; their tests
are marked xfail (not strict), and the planner removes a name once the
module declares. A module not in the list must pass.
its rows state what the declaration says and nothing it denies. No network:
fixtures stand in for the host. A fixture a declaration needs and that is
missing fails the test with the path to add.
Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/`
for searches, named after the module:
- `<name>-places.<ext>`: one listing answer with at least one role at two
or more places. For `listing="whole"` it is the body `parse` gets; for
`listing="paged"` it is what every request of the first `page` call
returns, and `rows` reads that page. Required by the test of a module
that declares `places="many"`, which is skipped until the fixture exists.
- `<name>-places-detail.<ext>`: optional, for a host whose places come from
the role's own page: what `detail`'s requests return; its `places` replace
the row's.
- `<name>-teaser.<ext>`: one listing answer with a teaser, read the same
way. Required by the test of a module with `text="excerpt"` or
`teaser=True`, skipped until it exists.
- `<name>-places.<ext>`: one listing answer, required of every reader. For
`listing="whole"` it is the body `parse` gets; for `listing="paged"` it is
what every request of the first `page` call returns, and `rows` reads
that page. Under `places="many"` at least one role is at two or more
places.
- `<name>.<ext>` and `<name>-teaser.<ext>`: more listing answers, read the
same way; the teaser one is required under `text="excerpt"` or
`teaser=True`.
- `<name>-detail*.<ext>` and `<name>-places-detail.<ext>`: what `detail`'s
requests return, one of them required under `text="detail"`. Only
`_runner.DETAIL_FIELDS` of the answer count, as at run time; the places of
`-places-detail` replace those of the `-places` rows.
- `<name>.slug`: optional, the slug the fixtures are read with, one line;
`acme` without it.
`acme` without it. A search's saved params are `SEARCH_PARAMS` below.
Between them the listing and detail answers show every field the reader
declares `stated` (or `figures`, `line`, `trusted`, `listed`): a real
fixture that happens to lack one is filled from what the per-host test
already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels).
"""
from __future__ import annotations
import sys
import textwrap
import types
from collections.abc import Callable
from pathlib import Path
@@ -41,42 +45,53 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.prep.hosts import TABLES # noqa: E402
from plugins import readers # noqa: E402
from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402
from plugins.readers import _contract # noqa: E402
from plugins.readers._runner import DETAIL_FIELDS # noqa: E402
FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" / "search")
BOARD_FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SEARCH_FIXTURES = ROOT / "tests" / "fixtures" / "search"
#: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and
#: C (the search providers, which arrive declared) land.
UNDECLARED: frozenset[str] = frozenset()
#: A search's saved params, which its `page` reads; `{}` for the rest.
SEARCH_PARAMS: dict[str, dict[str, Any]] = {"adzuna": {"q": "backend", "country": "de"}}
#: The readers whose listing states no date and whose `posted` comes from
#: `detail` alone: the date exists only for roles the enrich stage read.
POSTED_BY_DETAIL = frozenset({"bamboohr", "gem", "rippling"})
#: The fields a host states in its own words, which `app.prep` maps through
#: the host's table in `app/prep/hosts/<name>.py`.
WORD_FIELDS = ("seniority", "employment")
def _params() -> list[Any]:
return [pytest.param(name, marks=pytest.mark.xfail(reason="no READS yet", strict=False))
if name in UNDECLARED else name for name in readers.READERS]
def _folder(name: str) -> Path:
return SEARCH_FIXTURES if name in readers.PROVIDERS else BOARD_FIXTURES
def _declared(name: str) -> tuple[types.ModuleType, Reads]:
module = readers.READERS[name]
reads = getattr(module, "READS", None)
assert isinstance(reads, Reads), f"{name} declares no READS"
return module, reads
def _fixtures(name: str, pattern: str) -> list[Path]:
return sorted(p for p in _folder(name).glob(pattern.format(name=name))
if p.suffix not in (".slug",))
def _fixture(name: str, kind: str) -> Path | None:
for folder in FIXTURE_DIRS:
found = sorted(folder.glob(f"{name}-{kind}.*"))
if found:
return found[0]
return None
found = _fixtures(name, f"{{name}}-{kind}.*")
return found[0] if found else None
def _need(name: str, kind: str, why: str) -> Path:
"""The fixture `<name>-<kind>.*`, or a failure naming the file to add."""
found = _fixture(name, kind)
if found is None:
folder = _folder(name).relative_to(ROOT)
pytest.fail(f"add {folder}/{name}-{kind}.<ext>: {why}")
return found
def _slug(name: str) -> str:
for folder in FIXTURE_DIRS:
if (path := folder / f"{name}.slug").exists():
return path.read_text(encoding="utf-8").strip()
return "acme"
path = _folder(name) / f"{name}.slug"
return path.read_text(encoding="utf-8").strip() if path.exists() else "acme"
def _answer(body: bytes) -> Callable[..., bytes]:
@@ -84,7 +99,7 @@ def _answer(body: bytes) -> Callable[..., bytes]:
def read_fixture(module: types.ModuleType, body: bytes, slug: str,
detail_body: bytes | None = None) -> list[dict[str, Any]]:
detail_body: bytes | None = None, ask: Ask | None = None) -> list[dict[str, Any]]:
"""The rows `module` makes of one listing answer, as the runner reads
them: `parse` for a whole listing, the first `page` and its `rows` for a
paged one, and, given `detail_body`, each row's `places` from `detail`."""
@@ -92,7 +107,7 @@ def read_fixture(module: types.ModuleType, body: bytes, slug: str,
if reads.listing == "whole":
rows = list(module.parse(body, slug))
else:
decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, Ask())
decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, ask or Ask())
rows = list(module.rows(decoded, slug))
if detail_body is not None:
for row in rows:
@@ -102,6 +117,38 @@ def read_fixture(module: types.ModuleType, body: bytes, slug: str,
return rows
def _ask(name: str) -> Ask:
return Ask(params=SEARCH_PARAMS.get(name, {}))
def listing_rows(name: str) -> list[dict[str, Any]]:
"""Every row of the reader's listing fixtures; the `-places` rows with
the places of `-places-detail` where there is one."""
module, slug = readers.READERS[name], _slug(name)
places = _need(name, "places", "one listing answer of the host")
detail = _fixture(name, "places-detail")
rows = read_fixture(module, places.read_bytes(), slug,
detail.read_bytes() if detail else None, _ask(name))
for path in _fixtures(name, "{name}.*") + _fixtures(name, "{name}-teaser.*"):
rows += read_fixture(module, path.read_bytes(), slug, ask=_ask(name))
return rows
def detail_rows(name: str, listed: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""What `detail` hands over for the first listed role, once per detail
fixture, cut to `DETAIL_FIELDS` as the runner cuts it."""
module = readers.READERS[name]
if module.READS.text != "detail":
return []
paths = _fixtures(name, "{name}-detail*.*") + _fixtures(name, "{name}-places-detail.*")
if not paths:
_need(name, "detail", "what the role's own page answers, for text=\"detail\"")
assert listed, f"{name}: the listing fixtures gave no role to read in detail"
return [{key: value for key, value in
module.detail(_answer(path.read_bytes()), _slug(name), dict(listed[0])).items()
if key in DETAIL_FIELDS} for path in paths]
def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
"""What rows get wrong about `places` under the declared trait."""
out = []
@@ -118,60 +165,138 @@ def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
out.append(f"{rid}: {place!r} is not a Place")
if places in ("one", "copies") and len(found) > 1:
out.append(f'{rid}: {len(found)} places under places="{places}"')
if places == "none" and found:
out.append(f'{rid}: {len(found)} places under places="none"')
if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows):
out.append('no row has two places: a reader with places="many" keeps every place')
elif places != "none" and not any(row.get("places") for row in rows):
out.append(f'no row has a place under places="{places}": declare "none"')
return out
def _states(rows: list[dict[str, Any]], key: str) -> bool:
"""Whether some row carries `key` with a value (False counts, "" does not)."""
return any(row.get(key) not in (None, "", [], {}) for row in rows)
def field_problems(reads: Reads, listing: list[dict[str, Any]], detail: list[dict[str, Any]],
posted_by_detail: bool = False) -> list[str]:
"""What the rows get wrong about the declared `fields`, `text` and
`teaser`: a trait that promises a key no row carries, or denies one a
row carries."""
rows = listing + detail
fields = reads.fields
out: list[str] = []
figures = _states(rows, "salary_min") or _states(rows, "salary_max")
line = _states(rows, "salary_text")
if fields.pay == "figures" and not figures:
out.append('pay="figures" and no row has salary_min or salary_max')
if fields.pay == "line" and (figures or not line):
out.append('pay="line" wants salary_text on some row and figures on none')
if fields.pay == "none" and (figures or line):
out.append('pay="none" and a row has pay')
if fields.posted == "none" and _states(rows, "posted"):
out.append('posted="none" and a row has posted')
elif fields.posted != "none" and posted_by_detail:
if _states(listing, "posted") or not _states(detail, "posted"):
out.append(f'posted="{fields.posted}" by detail wants posted from detail only')
elif fields.posted != "none" and not _states(listing, "posted"):
out.append(f'posted="{fields.posted}" and no listed row has posted')
for key in ("remote", "country", "seniority", "employment"):
stated = getattr(fields, key) == "stated"
if stated != _states(rows, key):
out.append(f'{key}="{getattr(fields, key)}" and '
+ ("no row has it" if stated else "a row has it"))
text = {"listing": _states(listing, "description"),
"detail": _states(detail, "description"),
"excerpt": _states(listing, "teaser") and not _states(listing, "description"),
"none": not _states(rows, "description")}[reads.text]
if not text:
out.append(f'text="{reads.text}" and the rows do not show it')
teaser = reads.teaser or reads.text == "excerpt"
if teaser != _states(listing if teaser else rows, "teaser"):
out.append("teaser declared and no listed row has one" if teaser
else "a row has a teaser the declaration does not name")
return out
# ------------------------------------------------------------ every module
def test_the_undeclared_list_names_only_readers() -> None:
assert set(readers.READERS) >= UNDECLARED
def test_undeclared_lists_the_modules_without_reads() -> None:
assert set(readers.undeclared()) == {name for name, module in readers.READERS.items()
if not hasattr(module, "READS")}
@pytest.mark.parametrize("name", _params())
def test_a_reader_declares_reads(name: str) -> None:
_declared(name)
@pytest.mark.parametrize("name", _params())
@pytest.mark.parametrize("name", readers.READERS)
def test_a_declaration_matches_the_functions(name: str) -> None:
module, _reads = _declared(name)
module = readers.READERS[name]
assert readers.problems(module, name) == []
expected = "search" if module.READS.scope == "search" else "board"
assert name in (readers.PROVIDERS if expected == "search" else readers.HOSTS)
assert name in (readers.PROVIDERS if module.READS.scope == "search" else readers.HOSTS)
@pytest.mark.parametrize("name", _params())
@pytest.mark.parametrize("name", readers.READERS)
def test_a_reader_keeps_every_place(name: str) -> None:
module, reads = _declared(name)
fixture = _fixture(name, "places")
if fixture is None:
if reads.places == "many":
pytest.skip(f"no fixture {name}-places.* yet")
pytest.skip(f'places="{reads.places}" and no fixture')
detail = _fixture(name, "places-detail")
rows = read_fixture(module, fixture.read_bytes(), _slug(name),
detail.read_bytes() if detail else None)
assert rows, f"{fixture.name} gave no rows"
assert place_problems(rows, reads.places) == []
listed = listing_rows(name)
assert listed, f"{name}: the listing fixtures gave no rows"
answered = [{"source_id": f"detail {i}", "places": row["places"]}
for i, row in enumerate(detail_rows(name, listed)) if "places" in row]
assert place_problems(listed + answered, readers.READERS[name].READS.places) == []
@pytest.mark.parametrize("name", _params())
def test_a_reader_keeps_the_teaser(name: str) -> None:
module, reads = _declared(name)
if not (reads.text == "excerpt" or reads.teaser):
pytest.skip("the list carries no teaser")
fixture = _fixture(name, "teaser")
if fixture is None:
pytest.skip(f"no fixture {name}-teaser.* yet")
rows = read_fixture(module, fixture.read_bytes(), _slug(name))
assert any(isinstance(row.get("teaser"), str) and row["teaser"].strip() for row in rows)
@pytest.mark.parametrize("name", readers.READERS)
def test_a_reader_states_what_it_declares(name: str) -> None:
reads = readers.READERS[name].READS
if reads.text == "excerpt" or reads.teaser:
_need(name, "teaser", "a listing answer with a teaser")
listed = listing_rows(name)
assert field_problems(reads, listed, detail_rows(name, listed),
name in POSTED_BY_DETAIL) == []
@pytest.mark.parametrize("name", [name for name, module in readers.READERS.items()
if module.READS.listing == "paged"
and module.READS.complete in ("stated", "end")])
def test_a_paged_reader_states_a_total_as_declared(name: str) -> None:
"""`complete="stated"`: the first page's total is an int; `"end"`: None,
since the read ends on an empty page."""
module = readers.READERS[name]
body = _need(name, "places", "one listing answer of the host").read_bytes()
_decoded, _results, total, _cursor = module.page(_answer(body), _slug(name), None,
_ask(name))
if module.READS.complete == "stated":
assert isinstance(total, int) and not isinstance(total, bool), total
else:
assert total is None, total
def test_the_posted_by_detail_list_names_only_readers_that_date_by_detail() -> None:
for name in POSTED_BY_DETAIL:
reads = readers.READERS[name].READS
assert reads.text == "detail" and reads.fields.posted != "none", name
# ------------------------------------------------------------ prep tables
def test_every_prep_table_is_a_readers() -> None:
assert set(TABLES) <= set(readers.READERS), set(TABLES) - set(readers.READERS)
@pytest.mark.parametrize("name", readers.READERS)
def test_a_reader_stating_words_has_a_table_for_them(name: str) -> None:
"""A host that states seniority or employment in its own words has a
table for that field in `app/prep/hosts/<name>.py`, even an empty one
for a host whose words are already the app's (`words.lookup` then
takes the word as it is). The table is where a word is known, so a
host without one lists every word as unmapped. A table for a field the
reader declares `none` is dead."""
fields = readers.READERS[name].READS.fields
tables = TABLES.get(name, {})
for field in WORD_FIELDS:
has = field in tables or (field == "employment" and "match" in tables)
stated = getattr(fields, field) == "stated"
assert has == stated, (f"{name}: {field}={getattr(fields, field)!r} "
f"{'without' if stated else 'with'} a table in "
f"app/prep/hosts/{name}.py")
if "remote" in tables:
assert fields.remote == "stated", f"{name}: a remote table and remote=\"none\""
# ------------------------------------------------------------ the checks
@@ -219,11 +344,10 @@ def test_a_good_declaration_has_no_problems() -> None:
assert readers.problems(module, "fake") == []
def test_a_module_without_reads_is_checked_as_before() -> None:
def test_a_module_without_reads_fails() -> None:
module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: [])
assert readers.problems(module, "fake") == []
assert readers.scope(module) == "board"
assert readers.undeclared({"fake": module}) == ["fake"]
assert readers.problems(module, "fake") == [
"no READS: every reader declares what it reads"]
@pytest.mark.parametrize(("module", "reads", "problem"), [
@@ -256,6 +380,7 @@ def test_a_module_without_reads_is_checked_as_before() -> None:
env=("ADZUNA_APP_ID",))},
"key.env does not match key.fields"),
(_whole(), {"robots": " "}, "robots is empty"),
(_whole(), {"robots": "read: robots.txt allows /api/"}, "robots has no date"),
(_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {},
"feed_url does not take (slug, ask)"),
(_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []),
@@ -274,22 +399,48 @@ def test_reads_that_is_not_a_reads_is_refused() -> None:
assert "READS is not a Reads" in readers.problems(module, "fake")
def test_a_search_needs_fewer_names_than_a_board() -> None:
def _search() -> Any:
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL = "fake", "Fake"
module.allowed = lambda slug: lambda host: True
module.page = lambda get, slug, cursor, ask: (None, [], 0, None)
module.rows = lambda decoded, slug: []
module.PARAMS = {"q": "query", "where": "place"}
module.clean_params = lambda params: dict(params)
module.READS = _reads(scope="search", listing="paged", complete="never",
filters=Filters(query="one"),
filters=Filters(query="one", place="one"),
key=Key(name="adzuna", fields=("app_id", "app_key"),
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")))
return module
def test_a_search_needs_fewer_names_than_a_board() -> None:
module = _search()
assert readers.problems(module, "fake") == []
assert readers.scope(module) == "search"
del module.READS
assert "no careers_url" in readers.problems(module, "fake")
def test_a_search_names_its_params_and_cleans_them() -> None:
module = _search()
del module.PARAMS, module.clean_params
assert {"no PARAMS", "no clean_params"} <= set(readers.problems(module, "fake"))
module = _search()
module.PARAMS = {"q": "query", "area": "town"}
assert readers.problems(module, "fake") == [
"PARAMS maps 'area' to 'town', not a filter dimension"]
module = _search()
module.clean_params = lambda: {}
assert readers.problems(module, "fake") == ["clean_params does not take (params)"]
def test_every_search_names_its_params() -> None:
for name, module in readers.PROVIDERS.items():
declared = module.READS.filters.declared()
assert "query" in module.PARAMS.values(), name
assert set(module.PARAMS.values()) <= set(declared), name
def test_a_misspelt_trait_fails_where_it_is_declared() -> None:
with pytest.raises(pydantic.ValidationError):
_reads(listing="pages")
@@ -349,12 +500,20 @@ def test_the_place_check_passes_a_role_at_two_places() -> None:
assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == []
def test_the_place_check_holds_a_reader_that_states_none_to_none() -> None:
assert place_problems([{"source_id": "1", "places": []}], "none") == []
assert place_problems([{"source_id": "1", "places": []}], "one") == [
'no row has a place under places="one": declare "none"']
assert place_problems([{"source_id": "2", "places": [{"text": "Oslo", "remote": None}]}],
"none") == ['2: 1 places under places="none"']
def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None:
joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}]
assert place_problems(joined, "many") == [
'no row has two places: a reader with places="many" keeps every place']
assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [
"2: places is not a list"]
"2: places is not a list", 'no row has a place under places="one": declare "none"']
assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [
"3: ('Berlin', False) is not a Place"]
@@ -377,6 +536,57 @@ def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
assert place_problems(rows, "many") == []
def test_the_contracts_example_is_workdays_declaration() -> None:
doc = readers.__doc__ or ""
start = doc.index(" READS = Reads(")
source = textwrap.dedent(doc[start:doc.index("\n )\n", start) + 6])
namespace: dict[str, Any] = {"Reads": Reads, "Filters": Filters, "Fields": Fields}
exec(source, namespace)
assert namespace["READS"] == readers.READERS["workday"].READS
def test_the_contract_types_are_what_the_package_exports() -> None:
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
# ------------------------------------------------------------ the field check
def test_the_field_check_passes_rows_that_state_what_is_declared() -> None:
reads = _reads(text="detail", teaser=True, listing="whole",
fields=Fields(pay="figures", posted="trusted", remote="stated"))
listed = [{"source_id": "1", "teaser": "Short.", "remote": False, "posted": "2026-09-30"}]
detail = [{"description": "Long.", "salary_min": 50000}]
assert field_problems(reads, listed, detail) == []
def test_the_field_check_names_a_promise_no_row_keeps_and_a_denial_a_row_breaks() -> None:
reads = _reads(fields=Fields(pay="line", employment="stated"))
listed = [{"source_id": "1", "description": "Text.", "salary_min": 1, "remote": "hybrid",
"posted": "2026-09-30"}]
assert field_problems(reads, listed, []) == [
'pay="line" wants salary_text on some row and figures on none',
'posted="none" and a row has posted',
'remote="none" and a row has it',
'employment="stated" and no row has it']
def test_the_field_check_takes_a_date_from_detail_only_where_it_is_said() -> None:
reads = _reads(text="detail", fields=Fields(posted="trusted"))
listed = [{"source_id": "1"}]
detail = [{"description": "Text.", "posted": "2026-09-30"}]
assert field_problems(reads, listed, detail, posted_by_detail=True) == []
assert field_problems(reads, listed, detail) == [
'posted="trusted" and no listed row has posted']
assert field_problems(reads, [*listed, {"posted": "2026-09-30"}], detail,
posted_by_detail=True) == [
'posted="trusted" by detail wants posted from detail only']
def test_the_field_check_holds_text_and_teaser_to_the_declaration() -> None:
excerpt = _reads(text="excerpt")
assert field_problems(excerpt, [{"teaser": "Short."}], []) == []
assert field_problems(excerpt, [{"teaser": "Short.", "description": "Long."}], []) == [
'text="excerpt" and the rows do not show it']
assert field_problems(_reads(), [{"description": "Long.", "teaser": "Short."}], []) == [
"a row has a teaser the declaration does not name"]
+7 -17
View File
@@ -103,8 +103,6 @@ def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
both = _whole(filters=Filters(country="many", remote="many"))
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
assert (ask.countries, ask.remote) == (("DE",), True)
# A module without READS reads the whole board.
assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
@@ -139,11 +137,12 @@ def test_ids_filters_are_mapped_through_the_facets() -> None:
ask = Ask(countries=("DK", "XX"))
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
assert mapped.countries == ("h-1",)
# None known: the host has nothing in reach, and the read is empty.
# None known: nothing is asked, which is no evidence, so the read is
# incomplete and closes nothing.
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
facets={"country": {}})
assert (listing.rows, listing.complete, listing.filtered) == ([], True, True)
assert (listing.rows, listing.complete, listing.filtered) == ([], False, True)
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
@@ -189,17 +188,6 @@ def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
_runner.read_listing(module, "acme", web)
def test_a_module_without_reads_gets_the_older_signatures() -> None:
module: Any = types.ModuleType("old")
module.NAME = "old"
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
listing = _runner.read_listing(module, "acme",
Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
Ask(countries=("DK",)))
assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
pages = [["1", "2"], ["3"]]
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
@@ -333,9 +321,11 @@ def test_a_suspect_read_far_under_the_last_count_fails() -> None:
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
_runner.read(state, "ats", "t", module, "acme", web)
# A reader whose empty answer is trusted reads it as it is.
# A reader whose empty answer is trusted is held too, from a count of
# `SUSPECT_MIN`.
module.READS = module.READS.model_copy(update={"empty": "trusted"})
assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"]
with pytest.raises(_runner.Suspect):
_runner.read(state, "ats", "t", module, "acme", web)
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
+56 -1
View File
@@ -264,8 +264,10 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo
module: Any = types.ModuleType("fake")
module.NAME, module.LABEL = "fake", "Fake"
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
places="one", fields=Fields(country="stated"), robots="test")
places="one", fields=Fields(country="stated"), robots="test (checked 2026-09-30)")
module.allowed = lambda slug: lambda host: host == "search.fake.example"
module.PARAMS = {"q": "query"}
module.clean_params = lambda params: dict(params)
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
assert ask.params["q"]
@@ -371,3 +373,56 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
assert store.roles() == []
# And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
# ------------------------------------------------- failures and the guard
def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None:
"""The provider `fake` of `_fake_provider`, answering with an error."""
_fake_provider(monkeypatch, [], complete=True)
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
raise ValueError("hits renamed")
readers.PROVIDERS["fake"].page = page
def test_a_search_whose_provider_raises_is_recorded_failing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_fake_provider(monkeypatch, ["h1"], complete=True)
_pull(store)
_broken_provider(monkeypatch)
result = _pull(store)
assert result["unanswered"] == 1
since, reason = pull.failing_boards(store)["fake:cph"]
assert since and reason == "ValueError: hits renamed"
assert {r.source_id for r in store.roles()} == {"fake:h1"}
# Failing for `STALE_DAYS` closes its roles, as a board's.
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
timespec="seconds")
store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason})
_pull(store)
assert store.roles() == []
# Its next answer removes the record and reopens what it lists.
_fake_provider(monkeypatch, ["h1"], complete=True)
_pull(store)
assert pull.failing_boards(store) == {}
assert {r.source_id for r in store.roles()} == {"fake:h1"}
def test_a_search_that_lists_nothing_after_forty_is_suspect(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
forty = [f"h{n}" for n in range(40)]
_fake_provider(monkeypatch, forty, complete=True)
_pull(store)
_fake_provider(monkeypatch, [], complete=True)
result = _pull(store)
assert (result["unanswered"], result["closed"]) == (1, 0)
assert len(store.roles()) == 40
assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]