Merge main into wp-review-limits
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
750d3d1311
62 files changed
+1578
-538
No files matched your search
Vendored
+1
-1
@@ -1 +1 @@
|
||||
{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]}
|
||||
{"filterGroups":{"departments":[{"serverId":"265204383","value":"General application"}],"sites":[{"serverId":"2556616","value":"Copenhagen"},{"serverId":"2556618","value":"London"}],"jobTypes":[{"serverId":"Permanent","value":"Permanent"}],"workspaceTypes":[{"serverId":"hybrid","value":"Hybrid"},{"serverId":"on_site","value":"On site"}]},"jobAdDetails":[{"id":"340dbb78-164e-4c82-b7aa-6b88d2661eea","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556616,"site":"Copenhagen","country":"Denmark","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":null,"responsibilities":null,"benefits":null,"sectionLabels":null,"publishedAt":"2025-10-14T12:58:32.138726405Z","workspaceTypeId":"hybrid","workspaceType":"Hybrid","payTransparencyMinSalary":50000,"payTransparencyMaxSalary":70000,"payTransparencySalaryCurrency":"EUR","payTransparencySalaryPayPeriod":"Yearly","payTransparencySalaryAdditionalInfo":null},{"id":"b94be0ce-4a55-462f-99a6-5246395b3c08","title":"Expression of interest","departmentId":"265204383","department":"General application","employmentTypeId":"Permanent","employmentType":"Permanent","siteId":2556618,"site":"London","country":"United Kingdom","language":"en","description":"We're always excited to connect with talented professionals who share our vision. If you believe your skills and experience would be a great fit for Dixa, we'd love to hear from you. Submit your resume below, and our team will reach out when an opportunity aligns with your expertise.","requirements":"","responsibilities":"","benefits":"","sectionLabels":null,"publishedAt":"2026-02-27T15:42:31.845195914Z","workspaceTypeId":"on_site","workspaceType":"On site","payTransparencyMinSalary":null,"payTransparencyMaxSalary":null,"payTransparencySalaryCurrency":null,"payTransparencySalaryPayPeriod":null,"payTransparencySalaryAdditionalInfo":null}]}
|
||||
File renamed without changes.
File renamed without changes.
+1
-1
@@ -1 +1 @@
|
||||
{"data":[{"id":"581878","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe will help you \u003cstrong\u003egrow your career:\u003c/strong\u003e We help you identify areas of growth for you, whether you want to grow as a specialist in your field or eventually want to go into a leadership/management role.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person from time to time.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will receive an\u003cstrong\u003e individual education budget \u003c/strong\u003eand can educate yourself during work hours: you can spend several hours each month attending courses, reading books, or getting involved in communities.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFull-time position, employed through GI Group, our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eLooking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Job Benefits","compensation":null,"compensation_minimum":null,"compensation_maximum":null,"compensation_currency":null,"compensation_frequency":null,"compensation_visible":false,"deadline_at":null,"description":"\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eData Detective Wanted to join our Data \u0026amp; Analytics Team\u003c/strong\u003e \u003cbr\u003e\u003cbr\u003e\u003cem\u003eLocation: Bucharest, Romania\u003c/em\u003e\u003cbr\u003e\u003cbr\u003eAs a \u003cstrong\u003eData Operations Analyst \u003c/strong\u003eat Xeneta, you’ll learn how to master complex data analytics within the world of supply chain and logistics. Xeneta is one of the world's leading data platforms providing market intelligence to the ocean and air freight industry. We have a big team of skilled Data Analysts that will help you learn the ropes from the ground up, while enabling you to collect and analyze data in one of the world’s greatest container shipping datasets. \u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eBring your analytical mindset and a genuine curiosity for how global shipping works and join our team!\u0026nbsp;\u003c/div\u003e","employment_type":"permanent_full_time","employment_type_text":"Permanent - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eJoin our Liner Data \u0026amp; Analytics Team (LDAT) and own a portfolio of container shipping services for theLine truncated
|
||||
{"data":[{"id":"581878","benefits":"\u003cul\u003e\u003cli\u003e\u003c!--block--\u003eWe will help you \u003cstrong\u003egrow your career:\u003c/strong\u003e We help you identify areas of growth for you, whether you want to grow as a specialist in your field or eventually want to go into a leadership/management role.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have a \u003cstrong\u003etransparent and open work environment\u003c/strong\u003e where everyone is empowered to make an impact and encouraged to voice their ideas.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eKeeping true to our value, “\u003cstrong\u003eXeneta is one”\u003c/strong\u003e, we regularly bring colleagues together across locations. We organize an annual global company trip, and we aim to create opportunities for people to meet in person from time to time.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will receive an\u003cstrong\u003e individual education budget \u003c/strong\u003eand can educate yourself during work hours: you can spend several hours each month attending courses, reading books, or getting involved in communities.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eYou will enjoy our\u003cstrong\u003e generous vacation plan\u003c/strong\u003e: 26 paid days of vacation per year.\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe value \u003cstrong\u003ework-life balance \u003c/strong\u003eand want to make sure you can spend quality time with your family or enjoy your hobbies when your workday is over.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eWe have \u003cstrong\u003eflexible working hours\u003c/strong\u003e (core time with meetings is 11:00 to 16:00 EET, the rest is up to you).\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003eFull-time position, employed through GI Group, our staffing partner in Romania.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eWhy you will love it here\u003c/div\u003e\u003cul\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Impact\u003c/strong\u003e: We’re category leaders digitalizing the way ocean and air freight is procured bringing on-demand actionable data to all industry stakeholders. We’re moving fast.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Technology\u003c/strong\u003e: Deep data analytics and integrations coupled with a scalable, sleek and beautiful user interface.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Team\u003c/strong\u003e: Shipping, consulting and tech all-stars on our roster.\u0026nbsp;\u003c/li\u003e\u003cli\u003e\u003c!--block--\u003e\u003cstrong\u003eOur Values\u003c/strong\u003e: Xeneta is One. Transparency builds trust. Ownership drives Impact. Modernization through data. Diversity is our Superpower.\u003c/li\u003e\u003c/ul\u003e\u003cdiv\u003e\u003c!--block--\u003eMake sure to highlight your past achievements that you think are relevant. Be sure to give us an idea of who you are as a real live person and not just your professional experience. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eXeneta is an equal opportunity employer. We celebrate diversity and are committed to creating an inclusive environment for all employees. \u0026nbsp;\u003cbr\u003e\u003cbr\u003e\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003eLooking forward to hearing from you!\u0026nbsp;\u003c/div\u003e","benefits_header":"Job Benefits","compensation":null,"compensation_minimum":60000,"compensation_maximum":80000,"compensation_currency":"EUR","compensation_frequency":"annually","compensation_visible":true,"deadline_at":null,"description":"\u003cdiv\u003e\u003c!--block--\u003e\u003cstrong\u003eData Detective Wanted to join our Data \u0026amp; Analytics Team\u003c/strong\u003e \u003cbr\u003e\u003cbr\u003e\u003cem\u003eLocation: Bucharest, Romania\u003c/em\u003e\u003cbr\u003e\u003cbr\u003eAs a \u003cstrong\u003eData Operations Analyst \u003c/strong\u003eat Xeneta, you’ll learn how to master complex data analytics within the world of supply chain and logistics. Xeneta is one of the world's leading data platforms providing market intelligence to the ocean and air freight industry. We have a big team of skilled Data Analysts that will help you learn the ropes from the ground up, while enabling you to collect and analyze data in one of the world’s greatest container shipping datasets. \u0026nbsp;\u003c/div\u003e\u003cdiv\u003e\u003c!--block--\u003e\u003cbr\u003eBring your analytical mindset and a genuine curiosity for how global shipping works and join our team!\u0026nbsp;\u003c/div\u003e","employment_type":"permanent_full_time","employment_type_text":"Permanent - Full Time","key_responsibilities":"\u003cdiv\u003e\u003c!--block--\u003eJoin our Liner Data \u0026amp; Analytics Team (LDAT) and own a portfolio of container shipping servicesLine truncated
|
||||
File renamed without changes.
File renamed without changes.
+3
-1
@@ -16,7 +16,9 @@
|
||||
"fullLocation": "Berlin, Germany"
|
||||
},
|
||||
"department": {"id": "2294038", "label": "Engineering, Infrastructure and Operations"},
|
||||
"function": {"id": "engineering", "label": "Engineering"}
|
||||
"function": {"id": "engineering", "label": "Engineering"},
|
||||
"typeOfEmployment": {"id": "permanent", "label": "Full-time"},
|
||||
"experienceLevel": {"id": "director", "label": "Director"}
|
||||
},
|
||||
{
|
||||
"id": "744000147485112",
|
||||
|
||||
@@ -12,6 +12,11 @@ file holds `app`'s own layers.
|
||||
there the door is the tool: `desk.tools.<tool>`, one level, no deeper.
|
||||
3. **Plugins.** A file under `plugins/` imports `desk` and nothing else from
|
||||
these trees. A plugin is app-agnostic; it never sees `app`.
|
||||
4. **Readers.** A file of `plugins/readers/` imports the standard library,
|
||||
`httpx`, `pydantic`, `desk` through its doors and, within the package,
|
||||
`._contract`, `._shared` and `._countries` only: a reader shares no code
|
||||
with another reader, nor with `_runner`, which reads the readers and so
|
||||
alone may import the package's registry, `plugins.readers`.
|
||||
|
||||
Tests are held to none of this.
|
||||
"""
|
||||
@@ -21,6 +26,7 @@ from __future__ import annotations
|
||||
import ast
|
||||
import importlib
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -88,6 +94,13 @@ def _package(path: Path) -> str:
|
||||
|
||||
APP_FILES = sorted(p for p in (ROOT / APP).rglob("*.py") if "__pycache__" not in p.parts)
|
||||
PLUGIN_FILES = sorted((ROOT / "plugins").glob("*.py"))
|
||||
READER_FILES = sorted((ROOT / "plugins" / "readers").glob("*.py"))
|
||||
|
||||
#: What a reader may import beside the standard library and `desk`.
|
||||
READER_THIRD_PARTY = frozenset({"httpx", "pydantic"})
|
||||
|
||||
#: The package's own modules a reader may import.
|
||||
READER_SIBLINGS = frozenset({"_contract", "_shared", "_countries"})
|
||||
|
||||
|
||||
def _outside(here: str, source: str):
|
||||
@@ -120,6 +133,38 @@ def plugin_violations(source: str) -> list[str]:
|
||||
return wrong
|
||||
|
||||
|
||||
def reader_violations(source: str, registry: bool = False) -> list[str]:
|
||||
"""Every import in a file of `plugins/readers/` outside rule 4;
|
||||
`registry` allows `from plugins import readers`, for `_runner`."""
|
||||
wrong = []
|
||||
for node in ast.walk(ast.parse(source)):
|
||||
if registry and isinstance(node, ast.ImportFrom) and node.level == 0 \
|
||||
and node.module == "plugins" and [a.name for a in node.names] == ["readers"]:
|
||||
continue
|
||||
if isinstance(node, ast.Import):
|
||||
tops = [(a.name.split(".")[0], a.name) for a in node.names]
|
||||
elif isinstance(node, ast.ImportFrom) and node.level == 0:
|
||||
tops = [((node.module or "").split(".")[0], node.module or "")]
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
names = {node.module} if node.module else {a.name for a in node.names}
|
||||
if node.level != 1 or not names <= READER_SIBLINGS:
|
||||
wrong.append(f"line {node.lineno}: {'.' * node.level}{node.module or ''} "
|
||||
f"{sorted(names)} (a reader imports {', '.join(sorted(READER_SIBLINGS))})")
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
for top, module in tops:
|
||||
if top == "desk":
|
||||
if isinstance(node, ast.ImportFrom) and (
|
||||
reason := _door_violation(module, [a.name for a in node.names
|
||||
if not _is_module(f"{module}.{a.name}")])):
|
||||
wrong.append(f"line {node.lineno}: {module} ({reason})")
|
||||
elif top not in sys.stdlib_module_names and top not in READER_THIRD_PARTY:
|
||||
wrong.append(f"line {node.lineno}: {module} (a reader imports the standard "
|
||||
f"library, httpx, pydantic and desk)")
|
||||
return wrong
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", APP_FILES, ids=lambda p: str(p.relative_to(ROOT)))
|
||||
def test_a_package_imports_only_from_below_it(path: Path) -> None:
|
||||
wrong = direction_violations(_package(path), path.read_text(encoding="utf-8"))
|
||||
@@ -141,6 +186,12 @@ def test_a_plugin_knows_only_desk(path: Path) -> None:
|
||||
assert not wrong, "\n".join(wrong)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", READER_FILES, ids=lambda p: p.name)
|
||||
def test_a_reader_knows_only_its_contract_and_helpers(path: Path) -> None:
|
||||
wrong = reader_violations(path.read_text(encoding="utf-8"), path.name == "_runner.py")
|
||||
assert not wrong, "\n".join(wrong)
|
||||
|
||||
|
||||
def test_the_rules_catch_what_they_are_for() -> None:
|
||||
"""The checker itself, on sources that break each rule."""
|
||||
assert plugin_violations("from app.storage import Store")
|
||||
@@ -148,6 +199,23 @@ def test_the_rules_catch_what_they_are_for() -> None:
|
||||
assert plugin_violations("from desk.sources.base import Source")
|
||||
assert not plugin_violations("from desk.sources import Source, Item")
|
||||
|
||||
assert reader_violations("from plugins import ats")
|
||||
assert reader_violations("from plugins import readers")
|
||||
assert not reader_violations("from plugins import readers", registry=True)
|
||||
assert reader_violations("from plugins import ats", registry=True)
|
||||
assert reader_violations("from app.prep import words")
|
||||
assert reader_violations("from . import _runner")
|
||||
assert reader_violations("from ._runner import fetch")
|
||||
assert reader_violations("from .lever import parse")
|
||||
assert reader_violations("from .. import ats")
|
||||
assert reader_violations("import requests")
|
||||
assert reader_violations("from desk.sources.base import Source")
|
||||
assert not reader_violations("import json\nfrom urllib.parse import quote\nimport httpx\n"
|
||||
"from pydantic import ConfigDict\n"
|
||||
"from desk.sources import strip_html\n"
|
||||
"from . import _contract\nfrom ._shared import only\n"
|
||||
"from ._countries import code_of")
|
||||
|
||||
assert direction_violations("storage", "from app.web import TABS")
|
||||
assert direction_violations("web", "from app.main import create_app")
|
||||
assert not direction_violations("main", "from app.web import pages_router")
|
||||
|
||||
@@ -35,10 +35,10 @@ def test_the_employment_type_is_the_hosts_word() -> None:
|
||||
assert (first["employment"], second["employment"]) == ("FullTime", "Contract")
|
||||
|
||||
|
||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
|
||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
|
||||
reads = ashby.READS
|
||||
assert (reads.listing, reads.text, reads.places, reads.empty) == (
|
||||
"whole", "listing", "many", "suspect")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
|
||||
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
|
||||
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
|
||||
assert ashby.feed_url("acme", Ask()).endswith("/job-board/acme?includeCompensation=true")
|
||||
@@ -156,7 +156,6 @@ def test_it_declares_what_it_reads() -> None:
|
||||
assert (reads.listing, reads.text, reads.places, reads.fields.posted) == (
|
||||
"whole", "detail", "one", "trusted")
|
||||
assert reads.robots.startswith("read: robots.txt allows /careers")
|
||||
assert not hasattr(bamboohr, "robots_page")
|
||||
|
||||
|
||||
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
||||
@@ -188,4 +187,3 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None:
|
||||
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
|
||||
'<a href="https://resources.bamboohr.com/y">z</a>')
|
||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
|
||||
assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
||||
@@ -113,7 +113,6 @@ def test_it_declares_a_whole_listing_read_from_the_page() -> None:
|
||||
assert (reads.listing, reads.text, reads.places, reads.complete) == (
|
||||
"whole", "listing", "many", "whole")
|
||||
assert reads.fields.pay == "line" and "/api/ is disallowed and not read" in reads.robots
|
||||
assert not hasattr(deel, "robots_page")
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
@@ -201,7 +201,6 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None
|
||||
found = [(h, s) for h, s, _ in discover.boards_in(page)]
|
||||
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
|
||||
assert ("eightfold", "") in found
|
||||
assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
||||
|
||||
|
||||
def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None:
|
||||
@@ -210,5 +209,4 @@ def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request(
|
||||
"paged", 10, "detail", "stated", "many")
|
||||
assert reads.filters.declared() == {"country": "one", "query": "one"}
|
||||
assert reads.fields.posted == "listed"
|
||||
assert reads.robots.startswith("read: robots.txt allows /careers")
|
||||
assert not hasattr(eightfold, "robots_page")
|
||||
assert "/api/pcsx/search" in reads.robots and "/api/pcsx/position_details" in reads.robots
|
||||
@@ -103,8 +103,7 @@ def test_it_declares_one_page_that_may_be_empty_by_mistake() -> None:
|
||||
reads = gem.READS
|
||||
assert (reads.listing, reads.text, reads.places, reads.empty) == (
|
||||
"paged", "detail", "many", "suspect")
|
||||
assert reads.robots.startswith("read: robots.txt allows /<board>")
|
||||
assert not hasattr(gem, "robots_page")
|
||||
assert "POST /api/public/graphql" in reads.robots
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("body", "message"), [
|
||||
@@ -179,4 +178,3 @@ def test_discovery_finds_the_board_keeping_its_case() -> None:
|
||||
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
|
||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
|
||||
("gem", "Productboard"), ("gem", "sennder-gmbh")]
|
||||
assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
||||
@@ -43,10 +43,10 @@ def test_without_offices_the_name_is_the_place() -> None:
|
||||
assert nothing["places"] == [] and nothing["location"] == ""
|
||||
|
||||
|
||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
|
||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
|
||||
reads = greenhouse.READS
|
||||
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.pay == "figures"
|
||||
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
|
||||
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
|
||||
assert greenhouse.feed_url("acme", Ask()).endswith("/boards/acme/jobs?content=true&pay_transparency=true")
|
||||
assert greenhouse.fallback_url("acme").endswith("/boards/acme/jobs")
|
||||
@@ -163,8 +163,7 @@ def test_it_declares_copies_with_a_page_and_a_robots_decision() -> None:
|
||||
reads = hibob.READS
|
||||
assert (reads.listing, reads.text, reads.places, reads.complete) == (
|
||||
"paged", "listing", "copies", "end")
|
||||
assert reads.robots.startswith("read: robots.txt allows")
|
||||
assert not hasattr(hibob, "robots_page")
|
||||
assert reads.robots.startswith("read: /api/job-ad")
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
@@ -122,9 +122,8 @@ def test_the_feed_states_no_place_and_the_robots_decision_is_the_owners() -> Non
|
||||
assert all(r["places"] == [] for r in rows)
|
||||
reads = hrmanager.READS
|
||||
assert (reads.listing, reads.page_size, reads.text, reads.places) == (
|
||||
"paged", 100, "listing", "one")
|
||||
assert reads.robots == "disallowed by robots.txt; owner to decide"
|
||||
assert not hasattr(hrmanager, "robots_page")
|
||||
"paged", 100, "listing", "none")
|
||||
assert "api.hr-manager.net" in reads.robots and "owner, 2026-09-29" in reads.robots
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
@@ -176,7 +176,6 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
|
||||
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
|
||||
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
|
||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
|
||||
assert "jibe" not in {h for h, _ in discover._UNREADABLE}
|
||||
|
||||
|
||||
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
|
||||
@@ -184,4 +183,3 @@ def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
|
||||
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
|
||||
"paged", 100, "listing", "stated", "many")
|
||||
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")
|
||||
assert not hasattr(jibe, "robots_page")
|
||||
@@ -87,8 +87,8 @@ def _get(pages: dict[str, bytes], asked: list[str] | None = None) -> Any:
|
||||
def test_detail_reads_the_role_pages_json_ld() -> None:
|
||||
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
||||
asked: list[str] = []
|
||||
pages = {ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
|
||||
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
|
||||
pages = {ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
|
||||
ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
|
||||
first = jobylon.detail(_get(pages, asked), "3143", rows["385191"])
|
||||
assert asked == [ROLE]
|
||||
assert set(first) <= set(ats.DETAIL_FIELDS) | {"places"}
|
||||
@@ -116,8 +116,8 @@ def test_detail_without_a_job_posting_fails_the_read() -> None:
|
||||
|
||||
def test_the_board_is_one_request_and_each_role_is_read_on_its_own_page() -> None:
|
||||
asked: list[str] = []
|
||||
pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
|
||||
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
|
||||
pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
|
||||
ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
|
||||
get = _get(pages, asked)
|
||||
rows = jobylon.parse(get(jobylon.feed_url("3143", Ask())), "3143")
|
||||
assert len(rows) == 5 and asked == [FEED]
|
||||
@@ -150,8 +150,7 @@ def test_it_declares_a_whole_listing_with_a_teaser_and_detail() -> None:
|
||||
assert (reads.listing, reads.text, reads.teaser, reads.places) == (
|
||||
"whole", "detail", True, "many")
|
||||
assert (reads.fields.posted, reads.fields.seniority) == ("trusted", "stated")
|
||||
assert reads.robots.startswith("read: robots.txt allows /jobs/companies/")
|
||||
assert not hasattr(jobylon, "robots_page")
|
||||
assert "/jobs/companies/<id>/embed/v2/" in reads.robots
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
@@ -33,9 +33,9 @@ def test_the_commitment_is_the_hosts_word() -> None:
|
||||
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
|
||||
|
||||
|
||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_no_robots_check() -> None:
|
||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
|
||||
reads = lever.READS
|
||||
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.country == "stated"
|
||||
assert reads.robots == "api: no robots.txt check needed (public board API), 2026-09-30"
|
||||
assert reads.robots.startswith("read: ") and "(checked 2026-09-30)" in reads.robots
|
||||
assert lever.feed_url("eu/tomtom", Ask()) == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
|
||||
@@ -253,7 +253,6 @@ def test_the_careers_page_is_the_candidate_site() -> None:
|
||||
url = f"https://{HOST}/hcmUI/CandidateExperience/en/sites/CX_1/jobs"
|
||||
assert oracle.careers_url(SLUG) == ats.careers_url(f"oracle:{SLUG}") == url
|
||||
assert oracle.allowed(SLUG)(httpx.URL(url).host)
|
||||
assert not hasattr(oracle, "robots_page") # decided when built, see READS.robots
|
||||
|
||||
|
||||
def test_a_slug_written_with_capitals_names_the_same_pod() -> None:
|
||||
|
||||
@@ -24,8 +24,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
|
||||
reads = personio.READS
|
||||
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
|
||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
||||
assert not hasattr(personio, "robots_page")
|
||||
assert reads.robots.startswith("read: /xml")
|
||||
assert personio.feed_url("acme", Ask()) == "https://acme.jobs.personio.de/xml"
|
||||
|
||||
|
||||
|
||||
@@ -289,7 +289,6 @@ def test_the_careers_page_is_the_search_page() -> None:
|
||||
url = "https://careers.king.com/us/en/search-results"
|
||||
assert phenom.careers_url(SLUG) == ats.careers_url(f"phenom:{SLUG}") == url
|
||||
assert phenom.allowed(SLUG)(httpx.URL(url).host)
|
||||
assert not hasattr(phenom, "robots_page") # decided when built, see READS.robots
|
||||
assert phenom.careers_url("www.careers.philips.com/global/en") == \
|
||||
"https://www.careers.philips.com/global/en/search-results"
|
||||
|
||||
|
||||
@@ -116,7 +116,6 @@ def test_the_board_is_read_through_the_app_and_is_complete() -> None:
|
||||
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
||||
assert pinpoint.slug_ok("xeneta") and pinpoint.slug_ok("Xeneta") and pinpoint.slug_ok("a-b1")
|
||||
assert pinpoint.careers_url("xeneta") == "https://xeneta.pinpointhq.com/"
|
||||
assert not hasattr(pinpoint, "robots_page") # decided when built, see READS.robots
|
||||
assert pinpoint.feed_url("Xeneta") == "https://xeneta.pinpointhq.com/postings.json"
|
||||
allow = pinpoint.allowed("Xeneta")
|
||||
assert allow("xeneta.pinpointhq.com") and allow("XENETA.pinpointhq.com")
|
||||
@@ -146,7 +145,6 @@ def test_discovery_finds_the_subdomain() -> None:
|
||||
'<script src="https://www.pinpointhq.com/embed.js"></script>')
|
||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
|
||||
assert discover.boards_in("https://pinpointhq.com/") == []
|
||||
assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
|
||||
|
||||
|
||||
def test_a_pinpoint_board_is_not_probed() -> None:
|
||||
|
||||
@@ -25,7 +25,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
|
||||
reads = recruitee.READS
|
||||
assert (reads.listing, reads.places, reads.text) == ("whole", "many", "listing")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.country == "none"
|
||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
||||
assert "/api/offers/ is allowed" in reads.robots
|
||||
assert recruitee.feed_url("acme", Ask()) == "https://acme.recruitee.com/api/offers/"
|
||||
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ def _get(answer: bytes, asked: list[str] | None = None) -> Any:
|
||||
def test_detail_reads_description_places_date_and_a_single_yearly_range() -> None:
|
||||
row = {r["source_id"]: r for r in rippling.parse(BOARD, "rippling")}[DIRECTOR]
|
||||
asked: list[str] = []
|
||||
found = rippling.detail(_get((FIXTURES / "rippling-role.json").read_bytes(), asked),
|
||||
found = rippling.detail(_get((FIXTURES / "rippling-detail.json").read_bytes(), asked),
|
||||
"rippling", row)
|
||||
assert asked == [f"https://api.rippling.com/platform/api/ats/v1/board/rippling/jobs/{DIRECTOR}"]
|
||||
assert found["description"].startswith("About The Role")
|
||||
@@ -96,7 +96,7 @@ def test_detail_reads_description_places_date_and_a_single_yearly_range() -> Non
|
||||
|
||||
|
||||
def test_detail_without_pay_leaves_the_pay_columns_out() -> None:
|
||||
found = rippling.detail(_get((FIXTURES / "rippling-role-2.json").read_bytes()),
|
||||
found = rippling.detail(_get((FIXTURES / "rippling-detail-2.json").read_bytes()),
|
||||
"prisma-careers", {"source_id": "x"})
|
||||
assert found["description"].startswith("What is it like to work at Prisma?")
|
||||
assert found["location"] == "Remote (Germany)"
|
||||
@@ -165,7 +165,6 @@ def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
|
||||
def test_the_urls() -> None:
|
||||
assert rippling.feed_url("rippling") == FEED
|
||||
assert rippling.careers_url("prisma-careers") == "https://ats.rippling.com/prisma-careers/jobs"
|
||||
assert not hasattr(rippling, "robots_page") # decided when built, see READS.robots
|
||||
allow = rippling.allowed("rippling")
|
||||
assert allow("api.rippling.com") and allow("ATS.RIPPLING.COM")
|
||||
for host in ("rippling.com", "www.rippling.com", "api.rippling.com.evil.example", "localhost"):
|
||||
|
||||
@@ -35,7 +35,7 @@ def test_the_declaration_says_what_the_list_gives() -> None:
|
||||
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.empty) == \
|
||||
("paged", 100, "one", "detail", "suspect")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
|
||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
||||
assert reads.robots.startswith("read though") and "owner to decide" in reads.robots
|
||||
|
||||
|
||||
def test_page_reads_by_offset_and_takes_the_ask() -> None:
|
||||
|
||||
@@ -451,7 +451,6 @@ def test_careers_url_is_the_search_page() -> None:
|
||||
page = successfactors.careers_url(slug)
|
||||
assert httpx.URL(page).host == slug.split("/")[0]
|
||||
assert successfactors.allowed(slug)(httpx.URL(page).host)
|
||||
assert not hasattr(successfactors, "robots_page") # decided when built, see READS.robots
|
||||
|
||||
|
||||
def test_the_module_is_case_sensitive_about_the_prefix() -> None:
|
||||
|
||||
@@ -24,8 +24,7 @@ def test_the_declaration_says_what_the_feed_gives() -> None:
|
||||
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.complete) == \
|
||||
("paged", 50, "many", "listing", "end")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.remote == "stated"
|
||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
||||
assert not hasattr(teamtailor, "robots_page")
|
||||
assert "/jobs.rss is allowed" in reads.robots
|
||||
|
||||
|
||||
def test_page_takes_the_ask_and_reads_the_same_url() -> None:
|
||||
|
||||
@@ -30,7 +30,7 @@ def test_the_declaration_says_what_the_list_gives() -> None:
|
||||
assert (reads.listing, reads.page_size, reads.places, reads.text) == \
|
||||
("paged", 10, "many", "detail")
|
||||
assert reads.fields.posted == "trusted" and reads.fields.employment == "stated"
|
||||
assert reads.robots.startswith("api: no robots.txt check needed")
|
||||
assert "/api/v3/accounts/<slug>/jobs" in reads.robots
|
||||
|
||||
|
||||
def test_page_takes_the_ask_and_posts_the_same_body() -> None:
|
||||
|
||||
@@ -446,10 +446,10 @@ def test_the_case_of_the_site_is_kept() -> None:
|
||||
assert "workday" in ats.HOSTS and ats.HOSTS["workday"] == "Workday"
|
||||
|
||||
|
||||
def test_the_runtime_robots_check_has_left_the_module() -> None:
|
||||
"""Decided when the reader was built (`READS.robots`): nothing to ask now."""
|
||||
assert not hasattr(workday, "robots_page") and not hasattr(workday, "ROBOTS_ANY_CASE")
|
||||
assert "robots.txt allows /<Site>/" in workday.READS.robots
|
||||
def test_the_robots_record_names_the_api_path() -> None:
|
||||
"""Decided when the reader was built (`READS.robots`), for the paths it
|
||||
requests, not only the career site's."""
|
||||
assert "/wday/cxs/<tenant>/<site>/" in workday.READS.robots
|
||||
|
||||
|
||||
# -------------------------------------------------------------- discovery
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.jobs import discover as mod
|
||||
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
|
||||
from app.jobs.sources import seed_companies
|
||||
from app.storage import Store
|
||||
from plugins import readers
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ parsing
|
||||
@@ -95,6 +96,12 @@ def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
|
||||
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
|
||||
|
||||
|
||||
def test_no_unreadable_entry_names_a_host_with_a_module() -> None:
|
||||
"""A new module deletes its host's `_UNREADABLE` entry: its own patterns
|
||||
replace it."""
|
||||
assert not {host for host, _ in mod._UNREADABLE} & set(readers.HOSTS)
|
||||
|
||||
|
||||
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
|
||||
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
|
||||
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
|
||||
|
||||
+231
-6
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -21,8 +22,8 @@ if str(ROOT) not in sys.path:
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||
from app.models import Application # noqa: E402
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Filters, Reads, _runner # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app import prep # noqa: E402
|
||||
from app.jobs import cascade, pull # noqa: E402
|
||||
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
||||
assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0,
|
||||
"roles": 1, "closed": 1}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
||||
|
||||
@@ -163,7 +164,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
||||
assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0,
|
||||
"roles": 0, "closed": 0}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||
|
||||
@@ -256,7 +257,7 @@ def test_a_failing_board_keeps_its_roles_before_stale_days(
|
||||
_answer(monkeypatch, ValueError("gone"))
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
assert (result["failed"], result["closed"]) == (1, 0)
|
||||
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||
since, reason = pull.failing_boards(acme)["workable:acme"]
|
||||
assert since and reason == "ValueError: gone"
|
||||
|
||||
@@ -486,7 +487,7 @@ def test_run_a_settle_does_not_close_on_run_b_ids(
|
||||
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
||||
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
||||
assert (result["failed"], result["closed"]) == (1, 0)
|
||||
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||
assert pull.running(acme) == b["run"]
|
||||
|
||||
@@ -507,3 +508,227 @@ def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> No
|
||||
context = RunContext(waited_for=(failed,))
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
||||
assert result["boards"] == 1 and pull.running(acme) == ""
|
||||
|
||||
|
||||
# -------------------------------------------------------- losses, guarded
|
||||
|
||||
def _days_ago(days: float) -> str:
|
||||
return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _set(store: Store, column: str, value: str, *source_ids: str) -> None:
|
||||
with store.transaction() as conn:
|
||||
conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?",
|
||||
[(value, i) for i in source_ids])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def big(store: Store) -> Store:
|
||||
store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
||||
board="greenhouse")
|
||||
return store
|
||||
|
||||
|
||||
def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails(
|
||||
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
||||
_pull(big)
|
||||
_answer(monkeypatch, ([], True))
|
||||
result = _pull(big)
|
||||
assert len(_open(big)) == 40
|
||||
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||
assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1]
|
||||
|
||||
|
||||
def test_a_whole_read_of_nothing_after_five_closes_the_board(
|
||||
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(str(n)) for n in range(5)], True))
|
||||
_pull(big)
|
||||
_answer(monkeypatch, ([], True))
|
||||
assert _pull(big)["closed"] == 5
|
||||
assert _open(big) == set()
|
||||
|
||||
|
||||
def test_a_suspect_answer_is_accepted_after_stale_days(
|
||||
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
||||
_pull(big)
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(big)
|
||||
assert len(_open(big)) == 40
|
||||
record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"])
|
||||
big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps(
|
||||
record | {"since": _days_ago(pull.STALE_DAYS + 1)})
|
||||
result = _pull(big)
|
||||
assert _open(big) == {"greenhouse:big:1"}
|
||||
assert (result["unanswered"], result["closed"]) == (0, 39)
|
||||
assert pull.failing_boards(big) == {}
|
||||
assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1
|
||||
|
||||
|
||||
def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]:
|
||||
"""A whole-listing host `fake` that filters by country and, per `remote`,
|
||||
on remote; its feed lists the ids in the returned list."""
|
||||
listed: list[str] = []
|
||||
module: object = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined]
|
||||
module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined]
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined]
|
||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
||||
module.READS = Reads( # type: ignore[attr-defined]
|
||||
scope="board", listing="whole", text="listing", complete="whole", places="one",
|
||||
robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type]
|
||||
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps(
|
||||
listed).encode())
|
||||
return listed
|
||||
|
||||
|
||||
def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
listed = _remote_host(monkeypatch, remote="none")
|
||||
store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake")
|
||||
reach = {"fake:acme": {"countries": ["DK"], "remote": True}}
|
||||
|
||||
def pull_once() -> dict[str, object]:
|
||||
run = pull.claim_run(store)
|
||||
result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type]
|
||||
reach=reach)
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
||||
|
||||
listed[:] = ["1", "2", "3", "4"]
|
||||
pull_once()
|
||||
_set(store, "remote", "remote", "fake:acme:3", "fake:acme:4")
|
||||
listed[:] = ["1"]
|
||||
assert pull_once()["closed"] == 1
|
||||
assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"}
|
||||
assert store.state["ats:remote_unasked:fake:acme"] == "1"
|
||||
|
||||
_set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4")
|
||||
assert pull_once()["closed"] == 1
|
||||
assert _open(store) == {"fake:acme:1", "fake:acme:3"}
|
||||
|
||||
|
||||
def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(i) for i in "123"], True))
|
||||
_pull(acme)
|
||||
|
||||
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||
return _runner.Listing([_row("1")], True, ["1"], narrowed=True)
|
||||
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
assert _pull(acme)["closed"] == 0
|
||||
_set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3")
|
||||
assert _pull(acme)["closed"] == 1
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||
|
||||
|
||||
def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")]
|
||||
for target in targets:
|
||||
slug = target.partition(":")[2]
|
||||
store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse")
|
||||
|
||||
def fake_fetch(self: ATSSource) -> object:
|
||||
[(target, _)] = self.targets()
|
||||
yield _item("greenhouse", target.partition(":")[2], "1")
|
||||
self.state[f"ats:ok:{target}"] = self.options["run"]
|
||||
|
||||
real = Store.upsert_role
|
||||
|
||||
def upsert(self: Store, item: Item, sub: object = None) -> bool:
|
||||
if item.source_id.startswith("greenhouse:b:"):
|
||||
raise ValueError("a row the store refuses")
|
||||
return real(self, item, sub) # type: ignore[arg-type]
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
monkeypatch.setattr(Store, "upsert_role", upsert)
|
||||
result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type]
|
||||
|
||||
assert result["roles"] == 2
|
||||
assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"}
|
||||
assert "ats:ok:greenhouse:b" not in store.state
|
||||
since, reason = pull.failing_boards(store)["greenhouse:b"]
|
||||
assert since and reason == "ValueError: a row the store refuses"
|
||||
assert set(pull.failing_boards(store)) == {"greenhouse:b"}
|
||||
|
||||
|
||||
def test_a_board_whose_rows_do_not_store_keeps_its_failing_record(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
since = _days_ago(2)
|
||||
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
||||
{"since": since, "reason": "404 Not Found"})
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
|
||||
def refuse(self: Store, item: Item, sub: object = None) -> bool:
|
||||
raise ValueError("no")
|
||||
|
||||
monkeypatch.setattr(Store, "upsert_role", refuse)
|
||||
_pull(acme)
|
||||
assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no")
|
||||
assert "ats:ok:workable:acme" not in acme.state
|
||||
|
||||
|
||||
# ------------------------------------------------------- the settle speaks
|
||||
|
||||
def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many(
|
||||
store: Store) -> None:
|
||||
sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
||||
board="greenhouse")
|
||||
store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone",
|
||||
board="greenhouse")
|
||||
for n in range(80):
|
||||
store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub)
|
||||
run = pull.claim_run(store)
|
||||
store.state["ats:ok:greenhouse:big"] = run
|
||||
store.state["ats:complete:greenhouse:big"] = "1"
|
||||
store.state["ats:ids:greenhouse:big"] = json.dumps(
|
||||
[f"greenhouse:big:{n}" for n in range(20)])
|
||||
store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps(
|
||||
{"since": _days_ago(1), "reason": "404 Not Found"})
|
||||
|
||||
said: list[object] = []
|
||||
result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type]
|
||||
context=RunContext(on_notice=said.append))
|
||||
assert (result["unanswered"], result["closed"]) == (1, 60)
|
||||
[notice] = said
|
||||
assert notice.code == "pull.settled" # type: ignore[attr-defined]
|
||||
assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined]
|
||||
assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined]
|
||||
assert "closed 60 roles" in notice.text # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
||||
said: list[object] = []
|
||||
results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type]
|
||||
pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type]
|
||||
context=RunContext(on_notice=said.append))
|
||||
assert said == []
|
||||
|
||||
|
||||
def test_a_finder_that_raises_releases_the_pull_claim(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def broken(store: Store, subs: object) -> dict[str, object]:
|
||||
raise RuntimeError("reach broke")
|
||||
|
||||
monkeypatch.setattr(pull, "reaches", broken)
|
||||
# The worker runs the chain's finder once the job is done; here at once.
|
||||
context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme))
|
||||
with pytest.raises(RuntimeError, match="reach broke"):
|
||||
cascade.run_pull(FakeConfig(), acme, context=context)
|
||||
assert pull.running(acme) == ""
|
||||
|
||||
# And the job itself lets go when the chain cannot be built.
|
||||
def no_chain(run: str) -> object:
|
||||
raise RuntimeError("no chain")
|
||||
|
||||
monkeypatch.setattr(cascade, "pull_chain", no_chain)
|
||||
with pytest.raises(RuntimeError, match="no chain"):
|
||||
cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True))
|
||||
assert pull.running(acme) == ""
|
||||
+136
-3
@@ -26,7 +26,7 @@ if str(ROOT) not in sys.path:
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins.readers import lever # noqa: E402
|
||||
from plugins.readers._contract import Ask # noqa: E402
|
||||
from plugins.readers._contract import Ask, Filters, Reads # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -64,8 +64,11 @@ PATTERNS = (r"good\\.example/([a-z]+)",)
|
||||
def slug_ok(slug): return True
|
||||
def allowed(slug): return lambda h: h == "good.example"
|
||||
def careers_url(slug): return "https://good.example/" + slug
|
||||
def feed_url(slug): return "https://good.example/" + slug + ".json"
|
||||
def feed_url(slug, ask): return "https://good.example/" + slug + ".json"
|
||||
def parse(body, slug): return []
|
||||
from plugins.readers import Reads
|
||||
READS = Reads(scope="board", listing="whole", text="listing", complete="whole", places="one",
|
||||
robots="read (checked 2026-09-30)")
|
||||
"""
|
||||
|
||||
|
||||
@@ -253,7 +256,9 @@ def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
module.READS = Reads(scope="board", listing="whole", complete="whole", places="one",
|
||||
text="detail" if detail is not None else "listing", robots="test")
|
||||
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
|
||||
module.parse = lambda body, slug: [
|
||||
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
||||
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
||||
@@ -590,3 +595,131 @@ def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
|
||||
|
||||
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|
||||
|
||||
|
||||
# ------------------------------------------- the guard, asks, remote unasked
|
||||
|
||||
def _listing_host(**reads: Any) -> Any:
|
||||
"""A declared whole-listing board `fake` whose feed lists `module.ids`,
|
||||
whatever is asked; `reads` override its declaration."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||
module.READS = Reads(**({"scope": "board", "listing": "whole", "text": "listing",
|
||||
"complete": "whole", "places": "one", "robots": "test"} | reads))
|
||||
module.ids = []
|
||||
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}/list"
|
||||
module.parse = lambda body, slug: [
|
||||
{"source_id": i, "title": f"Role {i}", "url": f"https://jobs.fake.example/{i}",
|
||||
"places": [{"text": "Berlin", "remote": None}]} for i in json.loads(body)]
|
||||
return module
|
||||
|
||||
|
||||
def _listed(module: Any) -> Callable[..., bytes]:
|
||||
return lambda url, **kwargs: json.dumps(module.ids).encode()
|
||||
|
||||
|
||||
def _read_and_stamp(state: dict[str, str], module: Any,
|
||||
limits: _runner.Limits | None = None) -> _runner.Listing:
|
||||
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
|
||||
limits=limits)
|
||||
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||
return listing
|
||||
|
||||
|
||||
def test_an_unfiltered_whole_read_of_nothing_after_forty_is_suspect() -> None:
|
||||
"""A Greenhouse-style board, trusted, unfiltered: a changed answer that
|
||||
parses to no rows is refused rather than read as a board that closed."""
|
||||
module = _listing_host()
|
||||
state: dict[str, str] = {}
|
||||
module.ids = [str(n) for n in range(40)]
|
||||
_read_and_stamp(state, module)
|
||||
module.ids = []
|
||||
with pytest.raises(_runner.Suspect, match="listed 0 roles, under 10% of the 40"):
|
||||
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
|
||||
|
||||
|
||||
def test_a_whole_read_under_suspect_min_is_not_held() -> None:
|
||||
module = _listing_host()
|
||||
state: dict[str, str] = {}
|
||||
module.ids = [str(n) for n in range(5)]
|
||||
_read_and_stamp(state, module)
|
||||
module.ids = []
|
||||
assert _read_and_stamp(state, module).ids == []
|
||||
assert json.loads(state["ats:count:fake:acme"])["count"] == 0
|
||||
|
||||
|
||||
def test_the_same_suspect_answer_is_accepted_after_stale_days() -> None:
|
||||
module = _listing_host()
|
||||
state: dict[str, str] = {}
|
||||
module.ids = [str(n) for n in range(40)]
|
||||
_read_and_stamp(state, module)
|
||||
module.ids = ["1"]
|
||||
failing = "ats:failing:fake:acme"
|
||||
recent = (datetime.now(UTC) - timedelta(days=20)).isoformat(timespec="seconds")
|
||||
state[failing] = json.dumps({"since": recent, "reason": "Suspect: listed 1 roles"})
|
||||
limits = _runner.Limits(page_delay=0, stale_days=21)
|
||||
with pytest.raises(_runner.Suspect):
|
||||
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module), limits=limits)
|
||||
|
||||
old = (datetime.now(UTC) - timedelta(days=22)).isoformat(timespec="seconds")
|
||||
state[failing] = json.dumps({"since": old, "reason": "Suspect: listed 1 roles"})
|
||||
# Outside a pull (no `stale_days`) the guard never gives way.
|
||||
with pytest.raises(_runner.Suspect):
|
||||
_runner.read(state, "ats", "fake:acme", module, "acme", _listed(module))
|
||||
assert _read_and_stamp(state, module, limits).ids == ["1"]
|
||||
assert json.loads(state["ats:count:fake:acme"])["count"] == 1
|
||||
|
||||
|
||||
def test_keep_size_resets_the_guards_count_unless_told_not_to() -> None:
|
||||
state = {"ats:count:fake:acme": json.dumps({"asks": "[]", "count": 40})}
|
||||
ats.keep_size(state, "fake:acme", 40, reset=False)
|
||||
assert "ats:count:fake:acme" in state
|
||||
ats.keep_size(state, "fake:acme", 3)
|
||||
assert "ats:count:fake:acme" not in state
|
||||
assert state["ats:size:fake:acme"] == "3"
|
||||
|
||||
|
||||
def test_a_filter_that_maps_to_no_host_id_asks_nothing_and_is_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
module = _listing_host(filters=Filters(country="ids"))
|
||||
module.facets = lambda get, slug: {"country": {}}
|
||||
module.ids = ["1", "2"]
|
||||
state: dict[str, str] = {}
|
||||
listing = _runner.read(state, "ats", "fake:acme", module, "acme", _listed(module),
|
||||
Ask(countries=("DK",)))
|
||||
assert (listing.rows, listing.ids, listing.complete, listing.filtered) == (
|
||||
[], [], False, True)
|
||||
|
||||
# Through the plugin: stamped, but not complete, so the settle closes nothing.
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: b'["1"]')
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme",
|
||||
"params": {"countries": ["DK"]}}], "run": "r"})
|
||||
assert list(source.fetch()) == []
|
||||
assert (source.state["ats:ok:fake:acme"], source.state["ats:complete:fake:acme"]) == (
|
||||
"r", "")
|
||||
|
||||
|
||||
def test_a_filtered_read_that_cannot_ask_remote_says_so() -> None:
|
||||
reach = {"countries": ["DK"], "remote": True}
|
||||
blind = _listing_host(filters=Filters(country="many"))
|
||||
ask = _runner.ask_for(blind, None, reach)
|
||||
assert (ask.countries, ask.remote) == (("DK",), None)
|
||||
assert _runner.remote_wanted(blind, None, reach)
|
||||
listing = _runner.read_listing(blind, "acme", _listed(blind), ask,
|
||||
remote=_runner.remote_wanted(blind, None, reach))
|
||||
assert listing.filtered and listing.remote_unasked
|
||||
state: dict[str, str] = {}
|
||||
_runner.stamp(state, "ats", "fake:acme", "r", listing, listing.ids)
|
||||
assert state["ats:remote_unasked:fake:acme"] == "1"
|
||||
|
||||
# A host that filters on remote asks it as a series of its own.
|
||||
sees = _listing_host(filters=Filters(country="many", remote="many"))
|
||||
ask = _runner.ask_for(sees, None, reach)
|
||||
listing = _runner.read_listing(sees, "acme", _listed(sees), ask, remote=True)
|
||||
assert listing.filtered and not listing.remote_unasked
|
||||
# A params' remote of False asks nothing and wants nothing.
|
||||
assert not _runner.remote_wanted(blind, {"remote": False}, reach)
|
||||
+299
-89
@@ -1,34 +1,38 @@
|
||||
"""The reader contract, held for every module of `plugins/readers/`.
|
||||
|
||||
Per module: it declares `READS`, the declaration matches its functions, and
|
||||
its rows keep what the contract says they keep. No network: fixtures stand
|
||||
in for the host.
|
||||
|
||||
`UNDECLARED` lists the modules that do not declare `READS` yet; their tests
|
||||
are marked xfail (not strict), and the planner removes a name once the
|
||||
module declares. A module not in the list must pass.
|
||||
its rows state what the declaration says and nothing it denies. No network:
|
||||
fixtures stand in for the host. A fixture a declaration needs and that is
|
||||
missing fails the test with the path to add.
|
||||
|
||||
Fixtures, in `tests/fixtures/ats/` for boards and `tests/fixtures/search/`
|
||||
for searches, named after the module:
|
||||
|
||||
- `<name>-places.<ext>`: one listing answer with at least one role at two
|
||||
or more places. For `listing="whole"` it is the body `parse` gets; for
|
||||
`listing="paged"` it is what every request of the first `page` call
|
||||
returns, and `rows` reads that page. Required by the test of a module
|
||||
that declares `places="many"`, which is skipped until the fixture exists.
|
||||
- `<name>-places-detail.<ext>`: optional, for a host whose places come from
|
||||
the role's own page: what `detail`'s requests return; its `places` replace
|
||||
the row's.
|
||||
- `<name>-teaser.<ext>`: one listing answer with a teaser, read the same
|
||||
way. Required by the test of a module with `text="excerpt"` or
|
||||
`teaser=True`, skipped until it exists.
|
||||
- `<name>-places.<ext>`: one listing answer, required of every reader. For
|
||||
`listing="whole"` it is the body `parse` gets; for `listing="paged"` it is
|
||||
what every request of the first `page` call returns, and `rows` reads
|
||||
that page. Under `places="many"` at least one role is at two or more
|
||||
places.
|
||||
- `<name>.<ext>` and `<name>-teaser.<ext>`: more listing answers, read the
|
||||
same way; the teaser one is required under `text="excerpt"` or
|
||||
`teaser=True`.
|
||||
- `<name>-detail*.<ext>` and `<name>-places-detail.<ext>`: what `detail`'s
|
||||
requests return, one of them required under `text="detail"`. Only
|
||||
`_runner.DETAIL_FIELDS` of the answer count, as at run time; the places of
|
||||
`-places-detail` replace those of the `-places` rows.
|
||||
- `<name>.slug`: optional, the slug the fixtures are read with, one line;
|
||||
`acme` without it.
|
||||
`acme` without it. A search's saved params are `SEARCH_PARAMS` below.
|
||||
|
||||
Between them the listing and detail answers show every field the reader
|
||||
declares `stated` (or `figures`, `line`, `trusted`, `listed`): a real
|
||||
fixture that happens to lack one is filled from what the per-host test
|
||||
already builds (Pinpoint's and HiBob's pay, SmartRecruiters' levels).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import textwrap
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
@@ -41,42 +45,53 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.prep.hosts import TABLES # noqa: E402
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, Fields, Filters, Key, Reads # noqa: E402
|
||||
from plugins.readers import _contract # noqa: E402
|
||||
from plugins.readers._runner import DETAIL_FIELDS # noqa: E402
|
||||
|
||||
FIXTURE_DIRS = (ROOT / "tests" / "fixtures" / "ats", ROOT / "tests" / "fixtures" / "search")
|
||||
BOARD_FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
SEARCH_FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||
|
||||
#: The modules without `READS` yet. Shrinks as D1 (a to l), D2 (o to w) and
|
||||
#: C (the search providers, which arrive declared) land.
|
||||
UNDECLARED: frozenset[str] = frozenset()
|
||||
#: A search's saved params, which its `page` reads; `{}` for the rest.
|
||||
SEARCH_PARAMS: dict[str, dict[str, Any]] = {"adzuna": {"q": "backend", "country": "de"}}
|
||||
|
||||
#: The readers whose listing states no date and whose `posted` comes from
|
||||
#: `detail` alone: the date exists only for roles the enrich stage read.
|
||||
POSTED_BY_DETAIL = frozenset({"bamboohr", "gem", "rippling"})
|
||||
|
||||
#: The fields a host states in its own words, which `app.prep` maps through
|
||||
#: the host's table in `app/prep/hosts/<name>.py`.
|
||||
WORD_FIELDS = ("seniority", "employment")
|
||||
|
||||
|
||||
def _params() -> list[Any]:
|
||||
return [pytest.param(name, marks=pytest.mark.xfail(reason="no READS yet", strict=False))
|
||||
if name in UNDECLARED else name for name in readers.READERS]
|
||||
def _folder(name: str) -> Path:
|
||||
return SEARCH_FIXTURES if name in readers.PROVIDERS else BOARD_FIXTURES
|
||||
|
||||
|
||||
def _declared(name: str) -> tuple[types.ModuleType, Reads]:
|
||||
module = readers.READERS[name]
|
||||
reads = getattr(module, "READS", None)
|
||||
assert isinstance(reads, Reads), f"{name} declares no READS"
|
||||
return module, reads
|
||||
def _fixtures(name: str, pattern: str) -> list[Path]:
|
||||
return sorted(p for p in _folder(name).glob(pattern.format(name=name))
|
||||
if p.suffix not in (".slug",))
|
||||
|
||||
|
||||
def _fixture(name: str, kind: str) -> Path | None:
|
||||
for folder in FIXTURE_DIRS:
|
||||
found = sorted(folder.glob(f"{name}-{kind}.*"))
|
||||
if found:
|
||||
return found[0]
|
||||
return None
|
||||
found = _fixtures(name, f"{{name}}-{kind}.*")
|
||||
return found[0] if found else None
|
||||
|
||||
|
||||
def _need(name: str, kind: str, why: str) -> Path:
|
||||
"""The fixture `<name>-<kind>.*`, or a failure naming the file to add."""
|
||||
found = _fixture(name, kind)
|
||||
if found is None:
|
||||
folder = _folder(name).relative_to(ROOT)
|
||||
pytest.fail(f"add {folder}/{name}-{kind}.<ext>: {why}")
|
||||
return found
|
||||
|
||||
|
||||
def _slug(name: str) -> str:
|
||||
for folder in FIXTURE_DIRS:
|
||||
if (path := folder / f"{name}.slug").exists():
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
return "acme"
|
||||
path = _folder(name) / f"{name}.slug"
|
||||
return path.read_text(encoding="utf-8").strip() if path.exists() else "acme"
|
||||
|
||||
|
||||
def _answer(body: bytes) -> Callable[..., bytes]:
|
||||
@@ -84,7 +99,7 @@ def _answer(body: bytes) -> Callable[..., bytes]:
|
||||
|
||||
|
||||
def read_fixture(module: types.ModuleType, body: bytes, slug: str,
|
||||
detail_body: bytes | None = None) -> list[dict[str, Any]]:
|
||||
detail_body: bytes | None = None, ask: Ask | None = None) -> list[dict[str, Any]]:
|
||||
"""The rows `module` makes of one listing answer, as the runner reads
|
||||
them: `parse` for a whole listing, the first `page` and its `rows` for a
|
||||
paged one, and, given `detail_body`, each row's `places` from `detail`."""
|
||||
@@ -92,7 +107,7 @@ def read_fixture(module: types.ModuleType, body: bytes, slug: str,
|
||||
if reads.listing == "whole":
|
||||
rows = list(module.parse(body, slug))
|
||||
else:
|
||||
decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, Ask())
|
||||
decoded, _results, _total, _cursor = module.page(_answer(body), slug, None, ask or Ask())
|
||||
rows = list(module.rows(decoded, slug))
|
||||
if detail_body is not None:
|
||||
for row in rows:
|
||||
@@ -102,6 +117,38 @@ def read_fixture(module: types.ModuleType, body: bytes, slug: str,
|
||||
return rows
|
||||
|
||||
|
||||
def _ask(name: str) -> Ask:
|
||||
return Ask(params=SEARCH_PARAMS.get(name, {}))
|
||||
|
||||
|
||||
def listing_rows(name: str) -> list[dict[str, Any]]:
|
||||
"""Every row of the reader's listing fixtures; the `-places` rows with
|
||||
the places of `-places-detail` where there is one."""
|
||||
module, slug = readers.READERS[name], _slug(name)
|
||||
places = _need(name, "places", "one listing answer of the host")
|
||||
detail = _fixture(name, "places-detail")
|
||||
rows = read_fixture(module, places.read_bytes(), slug,
|
||||
detail.read_bytes() if detail else None, _ask(name))
|
||||
for path in _fixtures(name, "{name}.*") + _fixtures(name, "{name}-teaser.*"):
|
||||
rows += read_fixture(module, path.read_bytes(), slug, ask=_ask(name))
|
||||
return rows
|
||||
|
||||
|
||||
def detail_rows(name: str, listed: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""What `detail` hands over for the first listed role, once per detail
|
||||
fixture, cut to `DETAIL_FIELDS` as the runner cuts it."""
|
||||
module = readers.READERS[name]
|
||||
if module.READS.text != "detail":
|
||||
return []
|
||||
paths = _fixtures(name, "{name}-detail*.*") + _fixtures(name, "{name}-places-detail.*")
|
||||
if not paths:
|
||||
_need(name, "detail", "what the role's own page answers, for text=\"detail\"")
|
||||
assert listed, f"{name}: the listing fixtures gave no role to read in detail"
|
||||
return [{key: value for key, value in
|
||||
module.detail(_answer(path.read_bytes()), _slug(name), dict(listed[0])).items()
|
||||
if key in DETAIL_FIELDS} for path in paths]
|
||||
|
||||
|
||||
def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
|
||||
"""What rows get wrong about `places` under the declared trait."""
|
||||
out = []
|
||||
@@ -118,60 +165,138 @@ def place_problems(rows: list[dict[str, Any]], places: str) -> list[str]:
|
||||
out.append(f"{rid}: {place!r} is not a Place")
|
||||
if places in ("one", "copies") and len(found) > 1:
|
||||
out.append(f'{rid}: {len(found)} places under places="{places}"')
|
||||
if places == "none" and found:
|
||||
out.append(f'{rid}: {len(found)} places under places="none"')
|
||||
if places == "many" and not any(len(row.get("places") or ()) >= 2 for row in rows):
|
||||
out.append('no row has two places: a reader with places="many" keeps every place')
|
||||
elif places != "none" and not any(row.get("places") for row in rows):
|
||||
out.append(f'no row has a place under places="{places}": declare "none"')
|
||||
return out
|
||||
|
||||
|
||||
def _states(rows: list[dict[str, Any]], key: str) -> bool:
|
||||
"""Whether some row carries `key` with a value (False counts, "" does not)."""
|
||||
return any(row.get(key) not in (None, "", [], {}) for row in rows)
|
||||
|
||||
|
||||
def field_problems(reads: Reads, listing: list[dict[str, Any]], detail: list[dict[str, Any]],
|
||||
posted_by_detail: bool = False) -> list[str]:
|
||||
"""What the rows get wrong about the declared `fields`, `text` and
|
||||
`teaser`: a trait that promises a key no row carries, or denies one a
|
||||
row carries."""
|
||||
rows = listing + detail
|
||||
fields = reads.fields
|
||||
out: list[str] = []
|
||||
figures = _states(rows, "salary_min") or _states(rows, "salary_max")
|
||||
line = _states(rows, "salary_text")
|
||||
if fields.pay == "figures" and not figures:
|
||||
out.append('pay="figures" and no row has salary_min or salary_max')
|
||||
if fields.pay == "line" and (figures or not line):
|
||||
out.append('pay="line" wants salary_text on some row and figures on none')
|
||||
if fields.pay == "none" and (figures or line):
|
||||
out.append('pay="none" and a row has pay')
|
||||
|
||||
if fields.posted == "none" and _states(rows, "posted"):
|
||||
out.append('posted="none" and a row has posted')
|
||||
elif fields.posted != "none" and posted_by_detail:
|
||||
if _states(listing, "posted") or not _states(detail, "posted"):
|
||||
out.append(f'posted="{fields.posted}" by detail wants posted from detail only')
|
||||
elif fields.posted != "none" and not _states(listing, "posted"):
|
||||
out.append(f'posted="{fields.posted}" and no listed row has posted')
|
||||
|
||||
for key in ("remote", "country", "seniority", "employment"):
|
||||
stated = getattr(fields, key) == "stated"
|
||||
if stated != _states(rows, key):
|
||||
out.append(f'{key}="{getattr(fields, key)}" and '
|
||||
+ ("no row has it" if stated else "a row has it"))
|
||||
|
||||
text = {"listing": _states(listing, "description"),
|
||||
"detail": _states(detail, "description"),
|
||||
"excerpt": _states(listing, "teaser") and not _states(listing, "description"),
|
||||
"none": not _states(rows, "description")}[reads.text]
|
||||
if not text:
|
||||
out.append(f'text="{reads.text}" and the rows do not show it')
|
||||
teaser = reads.teaser or reads.text == "excerpt"
|
||||
if teaser != _states(listing if teaser else rows, "teaser"):
|
||||
out.append("teaser declared and no listed row has one" if teaser
|
||||
else "a row has a teaser the declaration does not name")
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------ every module
|
||||
|
||||
def test_the_undeclared_list_names_only_readers() -> None:
|
||||
assert set(readers.READERS) >= UNDECLARED
|
||||
|
||||
|
||||
def test_undeclared_lists_the_modules_without_reads() -> None:
|
||||
assert set(readers.undeclared()) == {name for name, module in readers.READERS.items()
|
||||
if not hasattr(module, "READS")}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _params())
|
||||
def test_a_reader_declares_reads(name: str) -> None:
|
||||
_declared(name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _params())
|
||||
@pytest.mark.parametrize("name", readers.READERS)
|
||||
def test_a_declaration_matches_the_functions(name: str) -> None:
|
||||
module, _reads = _declared(name)
|
||||
module = readers.READERS[name]
|
||||
assert readers.problems(module, name) == []
|
||||
expected = "search" if module.READS.scope == "search" else "board"
|
||||
assert name in (readers.PROVIDERS if expected == "search" else readers.HOSTS)
|
||||
assert name in (readers.PROVIDERS if module.READS.scope == "search" else readers.HOSTS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _params())
|
||||
@pytest.mark.parametrize("name", readers.READERS)
|
||||
def test_a_reader_keeps_every_place(name: str) -> None:
|
||||
module, reads = _declared(name)
|
||||
fixture = _fixture(name, "places")
|
||||
if fixture is None:
|
||||
if reads.places == "many":
|
||||
pytest.skip(f"no fixture {name}-places.* yet")
|
||||
pytest.skip(f'places="{reads.places}" and no fixture')
|
||||
detail = _fixture(name, "places-detail")
|
||||
rows = read_fixture(module, fixture.read_bytes(), _slug(name),
|
||||
detail.read_bytes() if detail else None)
|
||||
assert rows, f"{fixture.name} gave no rows"
|
||||
assert place_problems(rows, reads.places) == []
|
||||
listed = listing_rows(name)
|
||||
assert listed, f"{name}: the listing fixtures gave no rows"
|
||||
answered = [{"source_id": f"detail {i}", "places": row["places"]}
|
||||
for i, row in enumerate(detail_rows(name, listed)) if "places" in row]
|
||||
assert place_problems(listed + answered, readers.READERS[name].READS.places) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _params())
|
||||
def test_a_reader_keeps_the_teaser(name: str) -> None:
|
||||
module, reads = _declared(name)
|
||||
if not (reads.text == "excerpt" or reads.teaser):
|
||||
pytest.skip("the list carries no teaser")
|
||||
fixture = _fixture(name, "teaser")
|
||||
if fixture is None:
|
||||
pytest.skip(f"no fixture {name}-teaser.* yet")
|
||||
rows = read_fixture(module, fixture.read_bytes(), _slug(name))
|
||||
assert any(isinstance(row.get("teaser"), str) and row["teaser"].strip() for row in rows)
|
||||
@pytest.mark.parametrize("name", readers.READERS)
|
||||
def test_a_reader_states_what_it_declares(name: str) -> None:
|
||||
reads = readers.READERS[name].READS
|
||||
if reads.text == "excerpt" or reads.teaser:
|
||||
_need(name, "teaser", "a listing answer with a teaser")
|
||||
listed = listing_rows(name)
|
||||
assert field_problems(reads, listed, detail_rows(name, listed),
|
||||
name in POSTED_BY_DETAIL) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", [name for name, module in readers.READERS.items()
|
||||
if module.READS.listing == "paged"
|
||||
and module.READS.complete in ("stated", "end")])
|
||||
def test_a_paged_reader_states_a_total_as_declared(name: str) -> None:
|
||||
"""`complete="stated"`: the first page's total is an int; `"end"`: None,
|
||||
since the read ends on an empty page."""
|
||||
module = readers.READERS[name]
|
||||
body = _need(name, "places", "one listing answer of the host").read_bytes()
|
||||
_decoded, _results, total, _cursor = module.page(_answer(body), _slug(name), None,
|
||||
_ask(name))
|
||||
if module.READS.complete == "stated":
|
||||
assert isinstance(total, int) and not isinstance(total, bool), total
|
||||
else:
|
||||
assert total is None, total
|
||||
|
||||
|
||||
def test_the_posted_by_detail_list_names_only_readers_that_date_by_detail() -> None:
|
||||
for name in POSTED_BY_DETAIL:
|
||||
reads = readers.READERS[name].READS
|
||||
assert reads.text == "detail" and reads.fields.posted != "none", name
|
||||
|
||||
|
||||
# ------------------------------------------------------------ prep tables
|
||||
|
||||
def test_every_prep_table_is_a_readers() -> None:
|
||||
assert set(TABLES) <= set(readers.READERS), set(TABLES) - set(readers.READERS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", readers.READERS)
|
||||
def test_a_reader_stating_words_has_a_table_for_them(name: str) -> None:
|
||||
"""A host that states seniority or employment in its own words has a
|
||||
table for that field in `app/prep/hosts/<name>.py`, even an empty one
|
||||
for a host whose words are already the app's (`words.lookup` then
|
||||
takes the word as it is). The table is where a word is known, so a
|
||||
host without one lists every word as unmapped. A table for a field the
|
||||
reader declares `none` is dead."""
|
||||
fields = readers.READERS[name].READS.fields
|
||||
tables = TABLES.get(name, {})
|
||||
for field in WORD_FIELDS:
|
||||
has = field in tables or (field == "employment" and "match" in tables)
|
||||
stated = getattr(fields, field) == "stated"
|
||||
assert has == stated, (f"{name}: {field}={getattr(fields, field)!r} "
|
||||
f"{'without' if stated else 'with'} a table in "
|
||||
f"app/prep/hosts/{name}.py")
|
||||
if "remote" in tables:
|
||||
assert fields.remote == "stated", f"{name}: a remote table and remote=\"none\""
|
||||
|
||||
|
||||
# ------------------------------------------------------------ the checks
|
||||
@@ -219,11 +344,10 @@ def test_a_good_declaration_has_no_problems() -> None:
|
||||
assert readers.problems(module, "fake") == []
|
||||
|
||||
|
||||
def test_a_module_without_reads_is_checked_as_before() -> None:
|
||||
def test_a_module_without_reads_fails() -> None:
|
||||
module = _reader(feed_url=lambda slug: "", parse=lambda body, slug: [])
|
||||
assert readers.problems(module, "fake") == []
|
||||
assert readers.scope(module) == "board"
|
||||
assert readers.undeclared({"fake": module}) == ["fake"]
|
||||
assert readers.problems(module, "fake") == [
|
||||
"no READS: every reader declares what it reads"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("module", "reads", "problem"), [
|
||||
@@ -256,6 +380,7 @@ def test_a_module_without_reads_is_checked_as_before() -> None:
|
||||
env=("ADZUNA_APP_ID",))},
|
||||
"key.env does not match key.fields"),
|
||||
(_whole(), {"robots": " "}, "robots is empty"),
|
||||
(_whole(), {"robots": "read: robots.txt allows /api/"}, "robots has no date"),
|
||||
(_reader(feed_url=lambda slug: "", parse=lambda b, s: []), {},
|
||||
"feed_url does not take (slug, ask)"),
|
||||
(_reader(page=lambda get, slug, cursor: None, rows=lambda d, s: []),
|
||||
@@ -274,22 +399,48 @@ def test_reads_that_is_not_a_reads_is_refused() -> None:
|
||||
assert "READS is not a Reads" in readers.problems(module, "fake")
|
||||
|
||||
|
||||
def test_a_search_needs_fewer_names_than_a_board() -> None:
|
||||
def _search() -> Any:
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL = "fake", "Fake"
|
||||
module.allowed = lambda slug: lambda host: True
|
||||
module.page = lambda get, slug, cursor, ask: (None, [], 0, None)
|
||||
module.rows = lambda decoded, slug: []
|
||||
module.PARAMS = {"q": "query", "where": "place"}
|
||||
module.clean_params = lambda params: dict(params)
|
||||
module.READS = _reads(scope="search", listing="paged", complete="never",
|
||||
filters=Filters(query="one"),
|
||||
filters=Filters(query="one", place="one"),
|
||||
key=Key(name="adzuna", fields=("app_id", "app_key"),
|
||||
env=("ADZUNA_APP_ID", "ADZUNA_APP_KEY")))
|
||||
return module
|
||||
|
||||
|
||||
def test_a_search_needs_fewer_names_than_a_board() -> None:
|
||||
module = _search()
|
||||
assert readers.problems(module, "fake") == []
|
||||
assert readers.scope(module) == "search"
|
||||
del module.READS
|
||||
assert "no careers_url" in readers.problems(module, "fake")
|
||||
|
||||
|
||||
def test_a_search_names_its_params_and_cleans_them() -> None:
|
||||
module = _search()
|
||||
del module.PARAMS, module.clean_params
|
||||
assert {"no PARAMS", "no clean_params"} <= set(readers.problems(module, "fake"))
|
||||
module = _search()
|
||||
module.PARAMS = {"q": "query", "area": "town"}
|
||||
assert readers.problems(module, "fake") == [
|
||||
"PARAMS maps 'area' to 'town', not a filter dimension"]
|
||||
module = _search()
|
||||
module.clean_params = lambda: {}
|
||||
assert readers.problems(module, "fake") == ["clean_params does not take (params)"]
|
||||
|
||||
|
||||
def test_every_search_names_its_params() -> None:
|
||||
for name, module in readers.PROVIDERS.items():
|
||||
declared = module.READS.filters.declared()
|
||||
assert "query" in module.PARAMS.values(), name
|
||||
assert set(module.PARAMS.values()) <= set(declared), name
|
||||
|
||||
|
||||
def test_a_misspelt_trait_fails_where_it_is_declared() -> None:
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
_reads(listing="pages")
|
||||
@@ -349,12 +500,20 @@ def test_the_place_check_passes_a_role_at_two_places() -> None:
|
||||
assert place_problems(read_fixture(_fake_many(rows), b"", "acme"), "many") == []
|
||||
|
||||
|
||||
def test_the_place_check_holds_a_reader_that_states_none_to_none() -> None:
|
||||
assert place_problems([{"source_id": "1", "places": []}], "none") == []
|
||||
assert place_problems([{"source_id": "1", "places": []}], "one") == [
|
||||
'no row has a place under places="one": declare "none"']
|
||||
assert place_problems([{"source_id": "2", "places": [{"text": "Oslo", "remote": None}]}],
|
||||
"none") == ['2: 1 places under places="none"']
|
||||
|
||||
|
||||
def test_the_place_check_fails_a_reader_that_drops_or_joins_places() -> None:
|
||||
joined = [{"source_id": "1", "places": [{"text": "Berlin; Munich", "remote": None}]}]
|
||||
assert place_problems(joined, "many") == [
|
||||
'no row has two places: a reader with places="many" keeps every place']
|
||||
assert place_problems([{"source_id": "2", "location": "Berlin"}], "one") == [
|
||||
"2: places is not a list"]
|
||||
"2: places is not a list", 'no row has a place under places="one": declare "none"']
|
||||
assert place_problems([{"source_id": "3", "places": [("Berlin", False)]}], "one") == [
|
||||
"3: ('Berlin', False) is not a Place"]
|
||||
|
||||
@@ -377,6 +536,57 @@ def test_the_fixture_reader_drives_a_paged_module_and_detail() -> None:
|
||||
assert place_problems(rows, "many") == []
|
||||
|
||||
|
||||
def test_the_contracts_example_is_workdays_declaration() -> None:
|
||||
doc = readers.__doc__ or ""
|
||||
start = doc.index(" READS = Reads(")
|
||||
source = textwrap.dedent(doc[start:doc.index("\n )\n", start) + 6])
|
||||
namespace: dict[str, Any] = {"Reads": Reads, "Filters": Filters, "Fields": Fields}
|
||||
exec(source, namespace)
|
||||
assert namespace["READS"] == readers.READERS["workday"].READS
|
||||
|
||||
|
||||
def test_the_contract_types_are_what_the_package_exports() -> None:
|
||||
assert readers.Reads is _contract.Reads and readers.Row is _contract.Row
|
||||
assert _contract.Place(text="Berlin", remote=None) == {"text": "Berlin", "remote": None}
|
||||
|
||||
|
||||
# ------------------------------------------------------------ the field check
|
||||
|
||||
def test_the_field_check_passes_rows_that_state_what_is_declared() -> None:
|
||||
reads = _reads(text="detail", teaser=True, listing="whole",
|
||||
fields=Fields(pay="figures", posted="trusted", remote="stated"))
|
||||
listed = [{"source_id": "1", "teaser": "Short.", "remote": False, "posted": "2026-09-30"}]
|
||||
detail = [{"description": "Long.", "salary_min": 50000}]
|
||||
assert field_problems(reads, listed, detail) == []
|
||||
|
||||
|
||||
def test_the_field_check_names_a_promise_no_row_keeps_and_a_denial_a_row_breaks() -> None:
|
||||
reads = _reads(fields=Fields(pay="line", employment="stated"))
|
||||
listed = [{"source_id": "1", "description": "Text.", "salary_min": 1, "remote": "hybrid",
|
||||
"posted": "2026-09-30"}]
|
||||
assert field_problems(reads, listed, []) == [
|
||||
'pay="line" wants salary_text on some row and figures on none',
|
||||
'posted="none" and a row has posted',
|
||||
'remote="none" and a row has it',
|
||||
'employment="stated" and no row has it']
|
||||
|
||||
|
||||
def test_the_field_check_takes_a_date_from_detail_only_where_it_is_said() -> None:
|
||||
reads = _reads(text="detail", fields=Fields(posted="trusted"))
|
||||
listed = [{"source_id": "1"}]
|
||||
detail = [{"description": "Text.", "posted": "2026-09-30"}]
|
||||
assert field_problems(reads, listed, detail, posted_by_detail=True) == []
|
||||
assert field_problems(reads, listed, detail) == [
|
||||
'posted="trusted" and no listed row has posted']
|
||||
assert field_problems(reads, [*listed, {"posted": "2026-09-30"}], detail,
|
||||
posted_by_detail=True) == [
|
||||
'posted="trusted" by detail wants posted from detail only']
|
||||
|
||||
|
||||
def test_the_field_check_holds_text_and_teaser_to_the_declaration() -> None:
|
||||
excerpt = _reads(text="excerpt")
|
||||
assert field_problems(excerpt, [{"teaser": "Short."}], []) == []
|
||||
assert field_problems(excerpt, [{"teaser": "Short.", "description": "Long."}], []) == [
|
||||
'text="excerpt" and the rows do not show it']
|
||||
assert field_problems(_reads(), [{"description": "Long.", "teaser": "Short."}], []) == [
|
||||
"a row has a teaser the declaration does not name"]
|
||||
+7
-17
@@ -103,8 +103,6 @@ def test_ask_for_narrows_the_params_to_the_declared_filters() -> None:
|
||||
both = _whole(filters=Filters(country="many", remote="many"))
|
||||
ask = _runner.ask_for(both, {"countries": ["DE"]}, {"countries": ["DK"], "remote": True})
|
||||
assert (ask.countries, ask.remote) == (("DE",), True)
|
||||
# A module without READS reads the whole board.
|
||||
assert _runner.ask_for(types.ModuleType("old"), {"countries": ["DK"]}) == Ask()
|
||||
|
||||
|
||||
def test_a_searchs_params_go_whole_into_the_ask_with_its_key(
|
||||
@@ -139,11 +137,12 @@ def test_ids_filters_are_mapped_through_the_facets() -> None:
|
||||
ask = Ask(countries=("DK", "XX"))
|
||||
[mapped] = _runner.series(module, ask, {"country": {"DK": "h-1", "SE": "h-2"}})
|
||||
assert mapped.countries == ("h-1",)
|
||||
# None known: the host has nothing in reach, and the read is empty.
|
||||
# None known: nothing is asked, which is no evidence, so the read is
|
||||
# incomplete and closes nothing.
|
||||
assert _runner.series(module, Ask(countries=("XX",)), {"country": {"DK": "h-1"}}) == []
|
||||
listing = _runner.read_listing(module, "acme", Web(), Ask(countries=("XX",)),
|
||||
facets={"country": {}})
|
||||
assert (listing.rows, listing.complete, listing.filtered) == ([], True, True)
|
||||
assert (listing.rows, listing.complete, listing.filtered) == ([], False, True)
|
||||
|
||||
|
||||
def test_facets_are_asked_once_a_day_and_only_for_an_ids_filter() -> None:
|
||||
@@ -189,17 +188,6 @@ def test_a_body_over_the_cap_is_read_again_at_the_fallback_feed() -> None:
|
||||
_runner.read_listing(module, "acme", web)
|
||||
|
||||
|
||||
def test_a_module_without_reads_gets_the_older_signatures() -> None:
|
||||
module: Any = types.ModuleType("old")
|
||||
module.NAME = "old"
|
||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)]
|
||||
listing = _runner.read_listing(module, "acme",
|
||||
Web({"https://jobs.fake.example/acme/list": b'["1"]'}),
|
||||
Ask(countries=("DK",)))
|
||||
assert [r["source_id"] for r in listing.rows] == ["1"] and listing.complete
|
||||
|
||||
|
||||
def test_a_paged_board_with_a_total_is_complete_when_it_read_as_many() -> None:
|
||||
pages = [["1", "2"], ["3"]]
|
||||
listing = _runner.read_listing(_paged(pages, total=3), "acme", _pages_web(pages),
|
||||
@@ -333,9 +321,11 @@ def test_a_suspect_read_far_under_the_last_count_fails() -> None:
|
||||
web.answers["https://jobs.fake.example/acme/list"] = b'["1"]'
|
||||
with pytest.raises(_runner.Suspect, match="under 10% of the 20"):
|
||||
_runner.read(state, "ats", "t", module, "acme", web)
|
||||
# A reader whose empty answer is trusted reads it as it is.
|
||||
# A reader whose empty answer is trusted is held too, from a count of
|
||||
# `SUSPECT_MIN`.
|
||||
module.READS = module.READS.model_copy(update={"empty": "trusted"})
|
||||
assert _runner.read(state, "ats", "t", module, "acme", web).ids == ["1"]
|
||||
with pytest.raises(_runner.Suspect):
|
||||
_runner.read(state, "ats", "t", module, "acme", web)
|
||||
|
||||
|
||||
def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
||||
|
||||
+56
-1
@@ -264,8 +264,10 @@ def _fake_provider(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bo
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL = "fake", "Fake"
|
||||
module.READS = Reads(scope="search", listing="paged", text="listing", complete="stated",
|
||||
places="one", fields=Fields(country="stated"), robots="test")
|
||||
places="one", fields=Fields(country="stated"), robots="test (checked 2026-09-30)")
|
||||
module.allowed = lambda slug: lambda host: host == "search.fake.example"
|
||||
module.PARAMS = {"q": "query"}
|
||||
module.clean_params = lambda params: dict(params)
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
assert ask.params["q"]
|
||||
@@ -371,3 +373,56 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
||||
assert store.roles() == []
|
||||
# And a batch that starts after the switch reads nothing.
|
||||
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ------------------------------------------------- failures and the guard
|
||||
|
||||
def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The provider `fake` of `_fake_provider`, answering with an error."""
|
||||
_fake_provider(monkeypatch, [], complete=True)
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
raise ValueError("hits renamed")
|
||||
|
||||
readers.PROVIDERS["fake"].page = page
|
||||
|
||||
|
||||
def test_a_search_whose_provider_raises_is_recorded_failing(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||
params={"q": "backend"})
|
||||
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||
_pull(store)
|
||||
_broken_provider(monkeypatch)
|
||||
result = _pull(store)
|
||||
assert result["unanswered"] == 1
|
||||
since, reason = pull.failing_boards(store)["fake:cph"]
|
||||
assert since and reason == "ValueError: hits renamed"
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
||||
|
||||
# Failing for `STALE_DAYS` closes its roles, as a board's.
|
||||
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
||||
timespec="seconds")
|
||||
store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason})
|
||||
_pull(store)
|
||||
assert store.roles() == []
|
||||
|
||||
# Its next answer removes the record and reopens what it lists.
|
||||
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||
_pull(store)
|
||||
assert pull.failing_boards(store) == {}
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
||||
|
||||
|
||||
def test_a_search_that_lists_nothing_after_forty_is_suspect(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||
params={"q": "backend"})
|
||||
forty = [f"h{n}" for n in range(40)]
|
||||
_fake_provider(monkeypatch, forty, complete=True)
|
||||
_pull(store)
|
||||
_fake_provider(monkeypatch, [], complete=True)
|
||||
result = _pull(store)
|
||||
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||
assert len(store.roles()) == 40
|
||||
assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]
|
||||
Reference in new issue
Block a user