Merge the runner dropping and counting rows without an id, title or link, so no reader skips a listing (wp-rows)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-06 14:05:15 +02:00
commit 136047023e
58 files changed
+479 -203

No files matched your search

+15 -8
View File
@@ -24,18 +24,24 @@ from plugins.readers import _runner, smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
def _parsed(host: str, slug: str = "acme") -> list[ats.Row]:
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
body = path.read_bytes()
return ats._PARSERS[host](body, slug)
return ats._PARSERS[host](path.read_bytes(), slug)
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
"""The rows of a host's fixture the runner keeps (`_runner.valid`)."""
return [row for row in _parsed(host, slug) if _runner.valid(row)]
# ------------------------------------------------------------------ hosts
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
# The javascript: and data: rows are handed over and the runner drops
# them: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in _parsed("greenhouse")] == [
"1001", "1002", "1003", "1004", "1005", "1006"]
rows = _rows("greenhouse")
# The javascript: and data: rows are refused, same as the four real ones
# are kept: a feed is external content, never trusted as a URL.
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
first = rows[0]
assert first["title"] == "Senior Backend Engineer" # trimmed
@@ -269,10 +275,11 @@ TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
# the javascript: link is refused like on the other hosts
# the javascript: link is handed over and refused by the runner
assert [r["source_id"] for r in rows] == [
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"]
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"]
assert [_runner.valid(r) for r in rows] == [True, True, True, False]
first = rows[0]
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
assert first["department"] == "Engineering"
@@ -294,7 +301,7 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them
assert rows[1]["title"] == "Läkare" # trimmed
assert rows[1]["department"] == "Software Development"
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
+8 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, bamboohr # noqa: E402
from plugins.readers import Ask, _runner, bamboohr # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -84,11 +84,16 @@ def test_a_page_that_is_not_a_listing_raises_and_an_empty_board_is_no_rows() ->
bamboohr.parse(body, "configit")
def test_a_role_without_a_title_or_an_id_is_left_out() -> None:
def test_a_role_without_a_title_or_an_id_reaches_the_runner_and_is_dropped_there() -> None:
body = json.loads(_read("bamboohr.json"))
body["result"][0]["jobOpeningName"] = " "
body["result"][1]["id"] = None
assert bamboohr.parse(json.dumps(body).encode(), "configit") == []
raw = json.dumps(body).encode()
rows = bamboohr.parse(raw, "configit")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
listing = _runner.read_listing(bamboohr, "configit", lambda url, **kw: raw)
assert (listing.rows, listing.ids, listing.complete) == ([], [], True)
assert (listing.handed, listing.dropped) == (2, 2)
def test_detail_reads_the_text_the_date_and_the_country() -> None:
+5 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, deel # noqa: E402
from plugins.readers import Ask, _runner, deel # noqa: E402
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
URL = "https://jobs.deel.com/klarna"
@@ -89,10 +89,12 @@ def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
def test_a_posting_without_a_title_or_id_is_skipped() -> None:
def test_a_posting_without_a_title_or_id_is_handed_over_for_the_runner_to_drop() -> None:
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
{"id": "c", "title": "Cook", "job": {}}]}
assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"]
rows = deel.parse(_page_with(body), "acme")
assert [r["source_id"] for r in rows] == ["a", "", "c"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
# ------------------------------------------------------------ the fetch
+5 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, eightfold # noqa: E402
from plugins.readers import Ask, _runner, eightfold # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "jobs.ericsson.com/ericsson.com"
@@ -93,10 +93,12 @@ def test_the_work_location_option_is_the_hosts_word(
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
def test_a_position_without_a_name_an_id_or_a_path_is_left_out() -> None:
def test_a_position_without_a_name_an_id_or_a_path_is_handed_over_for_the_runner_to_drop() -> None:
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
assert eightfold.rows(bad, SLUG) == []
rows = eightfold.rows(bad, SLUG)
assert len(rows) == 4 and not any(map(_runner.valid, rows))
assert [r["url"] for r in rows[2:]] == ["", ""] # no link built off the host
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
+4 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, gem # noqa: E402
from plugins.readers import Ask, _runner, gem # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
API = "https://jobs.gem.com/api/public/graphql"
@@ -79,8 +79,9 @@ def test_remote_is_the_location_type_then_the_locations(
assert gem.rows([posting], "acme")[0]["remote"] == remote
def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None:
assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == []
def test_a_posting_without_a_title_or_an_ext_id_is_handed_over_for_the_runner_to_drop() -> None:
rows = gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
def test_the_board_is_one_post_without_origin_or_referer() -> None:
+5 -3
View File
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, hibob # noqa: E402
from plugins.readers import Ask, _runner, hibob # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hibob.json").read_bytes()
@@ -87,10 +87,12 @@ def test_the_place_leaves_the_country_off_where_the_site_names_it() -> None:
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
def test_an_ad_without_a_title_or_id_is_skipped_and_an_empty_board_is_empty() -> None:
def test_an_ad_without_a_title_or_id_is_handed_over_and_an_empty_board_is_empty() -> None:
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
{"id": "c", "title": "Cook"}]}).encode()
assert [r["source_id"] for r in hibob.parse(body, "acme")] == ["c"]
rows = hibob.parse(body, "acme")
assert [r["source_id"] for r in rows] == ["a", "", "c"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
+5 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, hrmanager # noqa: E402
from plugins.readers import Ask, _runner, hrmanager # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
@@ -52,12 +52,14 @@ def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_f
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None:
def test_an_item_without_a_title_or_a_web_link_is_handed_over_for_the_runner_to_drop() -> None:
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
b"</channel></rss>")
assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"]
rows = hrmanager.parse(body, "x")
assert [r["source_id"] for r in rows] == ["1", "2", "3"]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["3"]
# ------------------------------------------------------------- paging
+4 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, jibe # noqa: E402
from plugins.readers import Ask, _runner, jibe # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SITE = "jobs.booking.com"
@@ -72,9 +72,10 @@ def test_the_location_falls_back_to_city_and_country() -> None:
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
def test_a_role_without_a_title_or_a_slug_is_handed_over_for_the_runner_to_drop() -> None:
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
assert jibe.rows({"jobs": jobs}, SITE) == []
rows = jibe.rows({"jobs": jobs}, SITE)
assert len(rows) == 3 and not any(map(_runner.valid, rows))
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
+5 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, jobylon # noqa: E402
from plugins.readers import Ask, _runner, jobylon # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "jobylon.html").read_bytes()
@@ -65,12 +65,14 @@ def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None:
def test_a_role_whose_link_is_not_a_role_page_gets_no_link_and_is_dropped() -> None:
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
b"{id: '3', title: 'C', url: '/companies/3/'},"
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"]
rows = jobylon.parse(body, "9")
assert [r["url"] for r in rows[:3]] == ["", "", ""]
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["4"]
# ---------------------------------------------------------------- detail
+5 -4
View File
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import oracle # noqa: E402
from plugins.readers import _runner, oracle # noqa: E402
from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -84,14 +84,15 @@ def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_left_out() -> None:
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_dropped_by_the_runner() -> None:
data = _listing()
roles = data["items"][0]["requisitionList"]
roles[0]["Title"] = " "
roles[1]["Id"] = "1/../../x"
roles[2]["Id"] = ""
assert [r["source_id"] for r in oracle.parse(json.dumps(data).encode(), SLUG)] == \
["40975", "31680"]
rows = oracle.parse(json.dumps(data).encode(), SLUG)
assert rows[1]["url"] == "" # the id builds no link
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["40975", "31680"]
def test_a_listing_without_the_expand_has_no_roles() -> None:
+5 -3
View File
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import phenom # noqa: E402
from plugins.readers import _runner, phenom # noqa: E402
from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -101,13 +101,15 @@ def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
def test_a_role_without_title_or_id_is_left_out() -> None:
def test_a_role_without_title_or_id_is_dropped_by_the_runner() -> None:
data = _answer()
jobs = data["refineSearch"]["data"]["jobs"]
jobs[0]["title"] = ""
for key in ("jobSeqNo", "jobId", "reqId"):
jobs[1].pop(key, None)
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
rows = phenom.parse(json.dumps(data).encode(), SLUG)
assert len(rows) == len(IDS)
assert [r["source_id"] for r in rows if _runner.valid(r)] == IDS[2:]
# ---------------------------------------------------------- the request
+4 -3
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import pinpoint # noqa: E402
from plugins.readers import _runner, pinpoint # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -86,11 +86,12 @@ def test_an_unknown_pay_frequency_keeps_the_text_and_leaves_the_columns() -> Non
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_left_out() -> None:
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_dropped_by_the_runner() -> None:
body = json.loads(_body())
body["data"][0]["title"] = ""
body["data"][1]["url"] = "javascript:alert(1)"
assert pinpoint.parse(json.dumps(body).encode(), "xeneta") == []
rows = pinpoint.parse(json.dumps(body).encode(), "xeneta")
assert len(rows) == 2 and not any(map(_runner.valid, rows))
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
+6 -3
View File
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import rippling # noqa: E402
from plugins.readers import _runner, rippling # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "rippling.json").read_bytes()
@@ -61,11 +61,14 @@ def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
def test_an_entry_without_a_title_or_a_plain_id_is_skipped() -> None:
def test_an_entry_without_a_title_or_a_plain_id_is_dropped_by_the_runner() -> None:
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
{"uuid": "../../x", "name": "C"},
{"uuid": "abcdefgh-2", "name": "D"}]).encode()
assert [r["source_id"] for r in rippling.parse(body, "acme")] == ["abcdefgh-2"]
rows = rippling.parse(body, "acme")
assert [r["source_id"] for r in rows] == ["abcdefgh-1", "", "../../x", "abcdefgh-2"]
assert rows[2]["url"] == "" # the id builds no link
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["abcdefgh-2"]
# ---------------------------------------------------------------- detail
+3 -2
View File
@@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins.readers import Ask, teamtailor # noqa: E402
from plugins.readers import Ask, _runner, teamtailor # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -40,7 +40,8 @@ def test_page_takes_the_ask_and_reads_the_same_url() -> None:
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
first, second, third = _rows()
first, second, third, refused = _rows()
assert not _runner.valid(refused) # javascript:, for the runner
# "Sverige" (a city that is a country) once, Stockholm's twin once.
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
{"text": "Stockholm, Sweden", "remote": False}]
+12 -3
View File
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
from app.jobs import discover # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import workday # noqa: E402
from plugins.readers import _runner, workday # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "zendesk.wd1/zendesk"
@@ -96,7 +96,7 @@ def test_posted_is_read_only_where_the_text_is_exact(text: str, expected: dateti
assert workday._posted(text) == expected
def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
def test_a_posting_without_a_title_or_a_path_reaches_the_runner_and_is_dropped() -> None:
page = {"jobPostings": [
{"bulletFields": ["0000036369"]}, # LEGO's, no title
{"title": "No path", "bulletFields": ["R1"]},
@@ -105,7 +105,16 @@ def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
"postedOn": "Posted Today", "bulletFields": ["R4"]},
]}
assert [r["title"] for r in workday.rows(page, SLUG)] == ["Whole"]
rows = workday.rows(page, SLUG)
assert len(rows) == 5
assert [(r["source_id"], r["url"]) for r in rows[:2]] == [("", ""), ("", "")]
assert [r["title"] for r in rows if _runner.valid(r)] == ["Whole"]
# Through the runner: four dropped and counted, and the read complete,
# since every posting counts by its position towards the stated total.
body = json.dumps(page | {"total": 5}).encode()
listing = _runner.read_listing(workday, SLUG, lambda url, **kw: body)
assert [r["title"] for r in listing.rows] == ["Whole"]
assert (listing.handed, listing.dropped, listing.complete) == (5, 4, True)
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
+21
View File
@@ -116,6 +116,16 @@ def test_roles_without_text_and_text_reads_that_failed() -> None:
assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
@pytest.mark.parametrize(("dropped", "found"), [
("3", ["3 listings dropped: no title or link"]),
("1", ["1 listing dropped: no title or link"]),
("", []), ("x", [])])
def test_the_listings_the_last_read_dropped_are_named(dropped: str, found: list[str]) -> None:
"""The rows the runner dropped (`_runner.valid`), after "read incomplete"."""
assert words(fake(), state(dropped=dropped)) == found
assert words(fake(), state(dropped=dropped, complete=""))[1:] == found
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
assert words(fake(), state(), reach.SMALL) == []
@@ -199,6 +209,17 @@ def test_companies_carry_their_limits_and_the_boards_own_open_roles(
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
def test_a_board_that_dropped_listings_says_so_on_its_row(
client: TestClient, store: Store) -> None:
store.add_subscription("ats", "lever:acme", company="Acme", board="lever")
store.add_company("Acme", key="acme", ats="lever")
store.state["ats:ok:lever:acme"] = "run"
store.state["ats:complete:lever:acme"] = "1"
store.state["ats:dropped:lever:acme"] = "3"
[row] = client.get("/api/companies").json()
assert row["limits"][0] == "3 listings dropped: no title or link"
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
store.add_company("Novo", key="novo", ats="taleo")
[row] = client.get("/api/companies").json()
+12 -2
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, adzuna # noqa: E402
from plugins.readers import Ask, _runner, adzuna # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
@@ -39,8 +39,9 @@ def test_it_declares_the_key_and_states_the_robots_finding() -> None:
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
first, predicted = _rows() # javascript: refused
first, predicted, refused = _rows()
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
assert refused["url"].startswith("javascript:") and not _runner.valid(refused)
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
assert first["teaser"] == "Build our backend services in Python & Go."
assert "description" not in first
@@ -89,3 +90,12 @@ def test_saved_params_need_a_known_country() -> None:
"q": "x", "country": "gb"}
with pytest.raises(ValueError, match="country"):
adzuna.clean_params({"q": "x", "country": "dk"})
def test_the_runner_drops_the_ad_without_a_web_link_and_counts_it() -> None:
"""The reader hands the `javascript:` ad over; the runner drops it, and
the read is complete, since all three result ids were read."""
listing = _runner.read_listing(adzuna, "", lambda url, **kw: BODY, ASK,
limits=_runner.Limits(page_delay=0))
assert listing.ids == ["4412345678", "4412345679"]
assert (listing.handed, listing.dropped, listing.complete) == (3, 1, True)
+7 -4
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, freehire # noqa: E402
from plugins.readers import Ask, _runner, freehire # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
@@ -119,14 +119,17 @@ def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None:
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None:
def test_a_hit_without_a_title_an_id_or_a_web_link_is_handed_to_the_runner() -> None:
"""The reader hands every hit over; the runner keeps only the whole one."""
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
found = freehire.rows({"data": hits}, "")
assert [row["source_id"] for row in found] == ["d"]
assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == []
assert [row["source_id"] for row in found] == ["a", "", "c", "d"]
kept = [row for row in found if _runner.valid(row)]
assert [row["source_id"] for row in kept] == ["d"]
assert kept[0]["url"] == "https://x.example/d" and kept[0]["places"] == []
def test_a_title_and_a_company_have_their_entities_decoded() -> None:
+3 -2
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, jobbank # noqa: E402
from plugins.readers import Ask, _runner, jobbank # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
@@ -170,7 +170,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
assert len(results) == 100 and nxt == 2
assert len(jobbank.rows(decoded, "")) == 99
rows = jobbank.rows(decoded, "")
assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
+3 -2
View File
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, jobindex # noqa: E402
from plugins.readers import Ask, _runner, jobindex # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
@@ -89,7 +89,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
assert len(results) == 20 and nxt == 2
assert len(jobindex.rows(decoded, "")) == 19
rows = jobindex.rows(decoded, "")
assert len(rows) == 20 and sum(map(_runner.valid, rows)) == 19
def test_an_answer_without_a_channel_raises() -> None:
+6 -3
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, jobnet # noqa: E402
from plugins.readers import Ask, _runner, jobnet # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
@@ -100,9 +100,12 @@ def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None:
assert "applicationDeadline" not in row
def test_an_ad_without_an_id_or_a_title_is_skipped() -> None:
def test_an_ad_without_an_id_or_a_title_is_handed_over_for_the_runner_to_drop() -> None:
ads = json.loads(OTHER)["jobAds"]
assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == []
rows = jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "")
assert [(row["source_id"] == "", row["title"] == "") for row in rows] == [
(False, True), (True, False)]
assert not any(map(_runner.valid, rows))
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
+5 -4
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins import readers # noqa: E402
from plugins.readers import Ask, platsbanken # noqa: E402
from plugins.readers import Ask, _runner, platsbanken # noqa: E402
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
@@ -47,8 +47,9 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
rows = platsbanken.rows(decoded, "")
assert total == 32
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
first, no_town, bare = rows
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400001", "31400002"]
assert not _runner.valid(rows[2]) # javascript:, dropped by the runner
first, no_town, _refused, bare = rows
assert [p["text"] for p in first["places"]] == [
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
assert first["location"] == "Lund"
@@ -58,7 +59,7 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
def test_it_reads_title_company_date_and_the_full_text() -> None:
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
first, _no_town, remote = platsbanken.rows(decoded, "")
first, _no_town, _refused, remote = platsbanken.rows(decoded, "")
assert first["title"] == "Postdoctoral Researcher"
assert first["company"] == "European Spallation Source Eric"
assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
+107 -2
View File
@@ -1,7 +1,7 @@
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
subscription resolves to, series and their union, whole and paged listings,
completeness from the declaration, the sweep across pulls, the 10% guard,
the facets kept in state, the detail reads and the `raw` an item carries. No
completeness from the declaration, the rows kept and dropped, the sweep
across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
network: `get` is a dict of URL -> body or exception."""
from __future__ import annotations
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
assert filtered.ids == ["1"] and filtered.filtered
# --------------------------------------------------------- the rows kept
def _handed(i: str) -> dict[str, Any]:
"""A row as a reader hands it over: whole, or, for an id of `-id`,
`-title`, `-link` or `-url`, without that value or with a link that is
no web URL (`javascript:`, or one httpx cannot read)."""
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
return _row(i, **broken.get(i.split(":")[0], {}))
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
assert _runner.valid(_handed("1"))
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
assert not _runner.valid({"source_id": "1", "title": "Role"})
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
caplog: pytest.LogCaptureFixture) -> None:
module = _whole()
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
web = Web({"https://jobs.fake.example/acme/list":
b'["1", "-id", "2", "-title", "-link", "-url"]'})
with caplog.at_level("WARNING", logger=_runner.__name__):
listing = _runner.read_listing(module, "acme", web)
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert (listing.handed, listing.dropped) == (6, 4)
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
caplog: pytest.LogCaptureFixture) -> None:
"""Completeness counts the result ids `page` returns, not the rows kept,
so the dropped rows still count as read; the drop is logged once."""
pages = [["1", "-title:a"], ["2", "-link:b"]]
module = _paged(pages, total=4)
module.rows = lambda ids, slug: [_handed(i) for i in ids]
with caplog.at_level("WARNING", logger=_runner.__name__):
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert (listing.handed, listing.dropped) == (4, 2)
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
def test_the_stamp_keeps_the_rows_dropped() -> None:
state: dict[str, str] = {}
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
assert state["ats:dropped:t"] == "2"
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
assert state["ats:dropped:t"] == ""
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
"""A whole board whose feed hands over `handed` rows, `dropped` of them
without a title."""
module = _whole()
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
handed: int, dropped: int, fails: bool) -> None:
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
a page whose shape changed, which must close nothing."""
module, web = _dropping(handed, dropped)
if not fails:
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
return
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
_runner.read({}, "ats", "t", module, "acme", web)
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
module, web = _dropping(20, 5)
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
listing = _runner.read(state, "ats", "t", module, "acme", web,
limits=_runner.Limits(page_delay=0, stale_days=21))
assert (len(listing.ids), listing.dropped) == (15, 5)
# ------------------------------------------------------ through ATSSource
@pytest.fixture
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
assert state["ats:filtered:fake:acme"] == "1"
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
board.rows = lambda ids, slug: [_handed(i) for i in ids]
board.web.answers = {"https://jobs.fake.example/acme/page/0":
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
board.READS = board.READS.model_copy(update={"complete": "end"})
state: dict[str, str] = {}
assert len(list(_source(state).fetch())) == 19
assert state["ats:dropped:fake:acme"] == "1"
board.web.answers = {"https://jobs.fake.example/acme/page/0":
json.dumps([str(n) for n in range(15)]
+ [f"-id:{n}" for n in range(5)]).encode()}
state = {}
assert list(_source(state).fetch()) == []
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
assert "ats:ok:fake:acme" not in state
# -------------------------------------------------------------------- raw
def test_raw_keeps_every_stated_key_and_the_places() -> None:
+17
View File
@@ -383,6 +383,23 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch)
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count(
monkeypatch: pytest.MonkeyPatch) -> None:
"""The search driver reads through the same listing code as a board: the
runner drops the row the provider handed over without a title, and the
read is still complete, by the distinct result ids."""
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
module = readers.PROVIDERS["fake"]
handed = module.rows
module.rows = lambda decoded, slug: [
row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)]
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
"params": {"q": "x"}}]})
assert [i.source_id for i in source.fetch()] == ["fake:h1"]
assert source.state["search:complete:fake:cph"] == "1"
assert source.state["search:dropped:fake:cph"] == "1"
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)