Merge the runner dropping and counting rows without an id, title or link, so no reader skips a listing (wp-rows)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
136047023e
58 files changed
+479
-203
No files matched your search
+15
-8
@@ -24,18 +24,24 @@ from plugins.readers import _runner, smartrecruiters # noqa: E402
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
|
||||
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||
def _parsed(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||
path = FIXTURES / f"{host}.{'xml' if host == 'personio' else 'json'}"
|
||||
body = path.read_bytes()
|
||||
return ats._PARSERS[host](body, slug)
|
||||
return ats._PARSERS[host](path.read_bytes(), slug)
|
||||
|
||||
|
||||
def _rows(host: str, slug: str = "acme") -> list[ats.Row]:
|
||||
"""The rows of a host's fixture the runner keeps (`_runner.valid`)."""
|
||||
return [row for row in _parsed(host, slug) if _runner.valid(row)]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ hosts
|
||||
|
||||
def test_greenhouse_parses_titles_locations_and_html_description() -> None:
|
||||
# The javascript: and data: rows are handed over and the runner drops
|
||||
# them: a feed is external content, never trusted as a URL.
|
||||
assert [r["source_id"] for r in _parsed("greenhouse")] == [
|
||||
"1001", "1002", "1003", "1004", "1005", "1006"]
|
||||
rows = _rows("greenhouse")
|
||||
# The javascript: and data: rows are refused, same as the four real ones
|
||||
# are kept: a feed is external content, never trusted as a URL.
|
||||
assert [r["source_id"] for r in rows] == ["1001", "1002", "1005", "1006"]
|
||||
first = rows[0]
|
||||
assert first["title"] == "Senior Backend Engineer" # trimmed
|
||||
@@ -269,10 +275,11 @@ TEAMTAILOR = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
def test_teamtailor_reads_the_rss_feed_with_its_own_namespace() -> None:
|
||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||
# the javascript: link is refused like on the other hosts
|
||||
# the javascript: link is handed over and refused by the runner
|
||||
assert [r["source_id"] for r in rows] == [
|
||||
"39ac1fca-a904-4905-b780-18c599e99b07", "9787a509-ff84-400a-ad68-84e22c4cf2fb",
|
||||
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb"]
|
||||
"0a0a0a0a-ff84-400a-ad68-84e22c4cf2fb", "bad"]
|
||||
assert [_runner.valid(r) for r in rows] == [True, True, True, False]
|
||||
first = rows[0]
|
||||
assert first["url"] == "https://career.kry.se/jobs/8440833-software-engineer"
|
||||
assert first["department"] == "Engineering"
|
||||
@@ -294,7 +301,7 @@ def test_teamtailor_joins_the_distinct_locations_for_country_of() -> None:
|
||||
|
||||
def test_teamtailor_hands_on_remote_status_and_falls_back_to_role_for_department() -> None:
|
||||
rows = ats._parse_teamtailor(TEAMTAILOR, "kry")
|
||||
assert [r["remote"] for r in rows] == ["hybrid", "fully", "none"] # prep maps them
|
||||
assert [r["remote"] for r in rows[:3]] == ["hybrid", "fully", "none"] # prep maps them
|
||||
assert rows[1]["title"] == "Läkare" # trimmed
|
||||
assert rows[1]["department"] == "Software Development"
|
||||
assert ats.classify(rows[1]["department"], "Sales Lead") == "sales"
|
||||
|
||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, bamboohr # noqa: E402
|
||||
from plugins.readers import Ask, _runner, bamboohr # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -84,11 +84,16 @@ def test_a_page_that_is_not_a_listing_raises_and_an_empty_board_is_no_rows() ->
|
||||
bamboohr.parse(body, "configit")
|
||||
|
||||
|
||||
def test_a_role_without_a_title_or_an_id_is_left_out() -> None:
|
||||
def test_a_role_without_a_title_or_an_id_reaches_the_runner_and_is_dropped_there() -> None:
|
||||
body = json.loads(_read("bamboohr.json"))
|
||||
body["result"][0]["jobOpeningName"] = " "
|
||||
body["result"][1]["id"] = None
|
||||
assert bamboohr.parse(json.dumps(body).encode(), "configit") == []
|
||||
raw = json.dumps(body).encode()
|
||||
rows = bamboohr.parse(raw, "configit")
|
||||
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||
listing = _runner.read_listing(bamboohr, "configit", lambda url, **kw: raw)
|
||||
assert (listing.rows, listing.ids, listing.complete) == ([], [], True)
|
||||
assert (listing.handed, listing.dropped) == (2, 2)
|
||||
|
||||
|
||||
def test_detail_reads_the_text_the_date_and_the_country() -> None:
|
||||
|
||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, deel # noqa: E402
|
||||
from plugins.readers import Ask, _runner, deel # noqa: E402
|
||||
|
||||
PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes()
|
||||
URL = "https://jobs.deel.com/klarna"
|
||||
@@ -89,10 +89,12 @@ def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board
|
||||
deel.parse(b"<html><title>Job Board Not Found</title></html>", "nobody")
|
||||
|
||||
|
||||
def test_a_posting_without_a_title_or_id_is_skipped() -> None:
|
||||
def test_a_posting_without_a_title_or_id_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}},
|
||||
{"id": "c", "title": "Cook", "job": {}}]}
|
||||
assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"]
|
||||
rows = deel.parse(_page_with(body), "acme")
|
||||
assert [r["source_id"] for r in rows] == ["a", "", "c"]
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------ the fetch
|
||||
|
||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, eightfold # noqa: E402
|
||||
from plugins.readers import Ask, _runner, eightfold # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
SLUG = "jobs.ericsson.com/ericsson.com"
|
||||
@@ -93,10 +93,12 @@ def test_the_work_location_option_is_the_hosts_word(
|
||||
assert eightfold.rows([_position(1) | {"workLocationOption": option}], SLUG)[0]["remote"] == remote
|
||||
|
||||
|
||||
def test_a_position_without_a_name_an_id_or_a_path_is_left_out() -> None:
|
||||
def test_a_position_without_a_name_an_id_or_a_path_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
bad = [_position(1) | {"name": " "}, _position(2) | {"id": None},
|
||||
_position(3) | {"positionUrl": ""}, _position(4) | {"positionUrl": "//evil.example/x"}]
|
||||
assert eightfold.rows(bad, SLUG) == []
|
||||
rows = eightfold.rows(bad, SLUG)
|
||||
assert len(rows) == 4 and not any(map(_runner.valid, rows))
|
||||
assert [r["url"] for r in rows[2:]] == ["", ""] # no link built off the host
|
||||
|
||||
|
||||
def test_the_standardized_location_stands_in_when_there_is_no_raw_one() -> None:
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, gem # noqa: E402
|
||||
from plugins.readers import Ask, _runner, gem # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
API = "https://jobs.gem.com/api/public/graphql"
|
||||
@@ -79,8 +79,9 @@ def test_remote_is_the_location_type_then_the_locations(
|
||||
assert gem.rows([posting], "acme")[0]["remote"] == remote
|
||||
|
||||
|
||||
def test_a_posting_without_a_title_or_an_ext_id_is_left_out() -> None:
|
||||
assert gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme") == []
|
||||
def test_a_posting_without_a_title_or_an_ext_id_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
rows = gem.rows([{"extId": "1", "title": " "}, {"extId": "", "title": "Role"}], "acme")
|
||||
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||
|
||||
|
||||
def test_the_board_is_one_post_without_origin_or_referer() -> None:
|
||||
|
||||
@@ -17,7 +17,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, hibob # noqa: E402
|
||||
from plugins.readers import Ask, _runner, hibob # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "hibob.json").read_bytes()
|
||||
@@ -87,10 +87,12 @@ def test_the_place_leaves_the_country_off_where_the_site_names_it() -> None:
|
||||
assert hibob.parse(_one(**fields), "acme")[0]["location"] == want
|
||||
|
||||
|
||||
def test_an_ad_without_a_title_or_id_is_skipped_and_an_empty_board_is_empty() -> None:
|
||||
def test_an_ad_without_a_title_or_id_is_handed_over_and_an_empty_board_is_empty() -> None:
|
||||
body = json.dumps({"jobAdDetails": [{"id": "a", "title": " "}, {"title": "x"},
|
||||
{"id": "c", "title": "Cook"}]}).encode()
|
||||
assert [r["source_id"] for r in hibob.parse(body, "acme")] == ["c"]
|
||||
rows = hibob.parse(body, "acme")
|
||||
assert [r["source_id"] for r in rows] == ["a", "", "c"]
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["c"]
|
||||
assert hibob.parse(b'{"filterGroups": {}, "jobAdDetails": []}', "acme") == []
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, hrmanager # noqa: E402
|
||||
from plugins.readers import Ask, _runner, hrmanager # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
|
||||
@@ -52,12 +52,14 @@ def test_a_channel_with_no_items_is_an_empty_board_and_another_answer_is_not_a_f
|
||||
hrmanager.parse(b"<html><body>404 - File or directory not found.</body></html>", "x")
|
||||
|
||||
|
||||
def test_an_item_without_a_title_or_a_web_link_is_skipped() -> None:
|
||||
def test_an_item_without_a_title_or_a_web_link_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
body = (b"<rss><channel><item><guid>1</guid><title> </title><link>https://a.example/1</link></item>"
|
||||
b"<item><guid>2</guid><title>B</title><link>javascript:alert(1)</link></item>"
|
||||
b"<item><guid>3</guid><title>C</title><link>https://a.example/3</link></item>"
|
||||
b"</channel></rss>")
|
||||
assert [r["source_id"] for r in hrmanager.parse(body, "x")] == ["3"]
|
||||
rows = hrmanager.parse(body, "x")
|
||||
assert [r["source_id"] for r in rows] == ["1", "2", "3"]
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["3"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------- paging
|
||||
|
||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, jibe # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jibe # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
SITE = "jobs.booking.com"
|
||||
@@ -72,9 +72,10 @@ def test_the_location_falls_back_to_city_and_country() -> None:
|
||||
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
|
||||
|
||||
|
||||
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
|
||||
def test_a_role_without_a_title_or_a_slug_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
|
||||
assert jibe.rows({"jobs": jobs}, SITE) == []
|
||||
rows = jibe.rows({"jobs": jobs}, SITE)
|
||||
assert len(rows) == 3 and not any(map(_runner.valid, rows))
|
||||
|
||||
|
||||
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Ask, jobylon # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jobylon # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
||||
@@ -65,12 +65,14 @@ def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
|
||||
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
|
||||
|
||||
|
||||
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None:
|
||||
def test_a_role_whose_link_is_not_a_role_page_gets_no_link_and_is_dropped() -> None:
|
||||
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
|
||||
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
|
||||
b"{id: '3', title: 'C', url: '/companies/3/'},"
|
||||
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
|
||||
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"]
|
||||
rows = jobylon.parse(body, "9")
|
||||
assert [r["url"] for r in rows[:3]] == ["", "", ""]
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["4"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import oracle # noqa: E402
|
||||
from plugins.readers import _runner, oracle # noqa: E402
|
||||
from plugins.readers._shared import is_web_url # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -84,14 +84,15 @@ def test_a_department_is_taken_where_a_tenant_fills_one() -> None:
|
||||
assert [r["department"] for r in rows[:3]] == ["Supply Chain", "Finance", ""]
|
||||
|
||||
|
||||
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_left_out() -> None:
|
||||
def test_a_role_without_a_title_or_with_an_id_that_is_no_id_is_dropped_by_the_runner() -> None:
|
||||
data = _listing()
|
||||
roles = data["items"][0]["requisitionList"]
|
||||
roles[0]["Title"] = " "
|
||||
roles[1]["Id"] = "1/../../x"
|
||||
roles[2]["Id"] = ""
|
||||
assert [r["source_id"] for r in oracle.parse(json.dumps(data).encode(), SLUG)] == \
|
||||
["40975", "31680"]
|
||||
rows = oracle.parse(json.dumps(data).encode(), SLUG)
|
||||
assert rows[1]["url"] == "" # the id builds no link
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["40975", "31680"]
|
||||
|
||||
|
||||
def test_a_listing_without_the_expand_has_no_roles() -> None:
|
||||
|
||||
@@ -19,7 +19,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import phenom # noqa: E402
|
||||
from plugins.readers import _runner, phenom # noqa: E402
|
||||
from plugins.readers._shared import is_web_url # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -101,13 +101,15 @@ def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None
|
||||
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
|
||||
|
||||
|
||||
def test_a_role_without_title_or_id_is_left_out() -> None:
|
||||
def test_a_role_without_title_or_id_is_dropped_by_the_runner() -> None:
|
||||
data = _answer()
|
||||
jobs = data["refineSearch"]["data"]["jobs"]
|
||||
jobs[0]["title"] = ""
|
||||
for key in ("jobSeqNo", "jobId", "reqId"):
|
||||
jobs[1].pop(key, None)
|
||||
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
|
||||
rows = phenom.parse(json.dumps(data).encode(), SLUG)
|
||||
assert len(rows) == len(IDS)
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == IDS[2:]
|
||||
|
||||
|
||||
# ---------------------------------------------------------- the request
|
||||
|
||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import pinpoint # noqa: E402
|
||||
from plugins.readers import _runner, pinpoint # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -86,11 +86,12 @@ def test_an_unknown_pay_frequency_keeps_the_text_and_leaves_the_columns() -> Non
|
||||
assert (row["salary_min"], row["salary_max"], row["currency"]) == (None, None, "")
|
||||
|
||||
|
||||
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_left_out() -> None:
|
||||
def test_a_posting_without_a_title_or_an_id_or_a_web_url_is_dropped_by_the_runner() -> None:
|
||||
body = json.loads(_body())
|
||||
body["data"][0]["title"] = ""
|
||||
body["data"][1]["url"] = "javascript:alert(1)"
|
||||
assert pinpoint.parse(json.dumps(body).encode(), "xeneta") == []
|
||||
rows = pinpoint.parse(json.dumps(body).encode(), "xeneta")
|
||||
assert len(rows) == 2 and not any(map(_runner.valid, rows))
|
||||
|
||||
|
||||
def test_an_empty_board_is_no_rows_and_a_page_that_is_not_a_listing_raises() -> None:
|
||||
|
||||
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import rippling # noqa: E402
|
||||
from plugins.readers import _runner, rippling # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "rippling.json").read_bytes()
|
||||
@@ -61,11 +61,14 @@ def test_an_empty_board_is_no_roles_and_an_error_answer_is_not_a_board() -> None
|
||||
rippling.parse(b'{"error_code":"RESOURCE_NOT_FOUND"}', "acme")
|
||||
|
||||
|
||||
def test_an_entry_without_a_title_or_a_plain_id_is_skipped() -> None:
|
||||
def test_an_entry_without_a_title_or_a_plain_id_is_dropped_by_the_runner() -> None:
|
||||
body = json.dumps([{"uuid": "abcdefgh-1", "name": " "}, {"name": "B"},
|
||||
{"uuid": "../../x", "name": "C"},
|
||||
{"uuid": "abcdefgh-2", "name": "D"}]).encode()
|
||||
assert [r["source_id"] for r in rippling.parse(body, "acme")] == ["abcdefgh-2"]
|
||||
rows = rippling.parse(body, "acme")
|
||||
assert [r["source_id"] for r in rows] == ["abcdefgh-1", "", "../../x", "abcdefgh-2"]
|
||||
assert rows[2]["url"] == "" # the id builds no link
|
||||
assert [r["source_id"] for r in rows if _runner.valid(r)] == ["abcdefgh-2"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
@@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins.readers import Ask, teamtailor # noqa: E402
|
||||
from plugins.readers import Ask, _runner, teamtailor # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
@@ -40,7 +40,8 @@ def test_page_takes_the_ask_and_reads_the_same_url() -> None:
|
||||
|
||||
|
||||
def test_every_distinct_location_is_a_place_with_the_roles_work_mode() -> None:
|
||||
first, second, third = _rows()
|
||||
first, second, third, refused = _rows()
|
||||
assert not _runner.valid(refused) # javascript:, for the runner
|
||||
# "Sverige" (a city that is a country) once, Stockholm's twin once.
|
||||
assert first["places"] == [{"text": "Sverige, Sweden", "remote": False}, # hybrid
|
||||
{"text": "Stockholm, Sweden", "remote": False}]
|
||||
|
||||
@@ -20,7 +20,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.readers import workday # noqa: E402
|
||||
from plugins.readers import _runner, workday # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
SLUG = "zendesk.wd1/zendesk"
|
||||
@@ -96,7 +96,7 @@ def test_posted_is_read_only_where_the_text_is_exact(text: str, expected: dateti
|
||||
assert workday._posted(text) == expected
|
||||
|
||||
|
||||
def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
|
||||
def test_a_posting_without_a_title_or_a_path_reaches_the_runner_and_is_dropped() -> None:
|
||||
page = {"jobPostings": [
|
||||
{"bulletFields": ["0000036369"]}, # LEGO's, no title
|
||||
{"title": "No path", "bulletFields": ["R1"]},
|
||||
@@ -105,7 +105,16 @@ def test_a_posting_without_a_title_or_a_path_is_skipped() -> None:
|
||||
{"title": "Whole", "externalPath": "/job/X/Whole_R4", "locationsText": "X",
|
||||
"postedOn": "Posted Today", "bulletFields": ["R4"]},
|
||||
]}
|
||||
assert [r["title"] for r in workday.rows(page, SLUG)] == ["Whole"]
|
||||
rows = workday.rows(page, SLUG)
|
||||
assert len(rows) == 5
|
||||
assert [(r["source_id"], r["url"]) for r in rows[:2]] == [("", ""), ("", "")]
|
||||
assert [r["title"] for r in rows if _runner.valid(r)] == ["Whole"]
|
||||
# Through the runner: four dropped and counted, and the read complete,
|
||||
# since every posting counts by its position towards the stated total.
|
||||
body = json.dumps(page | {"total": 5}).encode()
|
||||
listing = _runner.read_listing(workday, SLUG, lambda url, **kw: body)
|
||||
assert [r["title"] for r in listing.rows] == ["Whole"]
|
||||
assert (listing.handed, listing.dropped, listing.complete) == (5, 4, True)
|
||||
|
||||
|
||||
def test_the_id_is_the_path_where_the_requisition_id_is_not_in_it() -> None:
|
||||
|
||||
@@ -116,6 +116,16 @@ def test_roles_without_text_and_text_reads_that_failed() -> None:
|
||||
assert words(fake(), state(), counts=limits.Texts(9, 0)) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("dropped", "found"), [
|
||||
("3", ["3 listings dropped: no title or link"]),
|
||||
("1", ["1 listing dropped: no title or link"]),
|
||||
("", []), ("x", [])])
|
||||
def test_the_listings_the_last_read_dropped_are_named(dropped: str, found: list[str]) -> None:
|
||||
"""The rows the runner dropped (`_runner.valid`), after "read incomplete"."""
|
||||
assert words(fake(), state(dropped=dropped)) == found
|
||||
assert words(fake(), state(dropped=dropped, complete=""))[1:] == found
|
||||
|
||||
|
||||
def test_a_board_is_read_whole_under_the_bar_or_when_its_host_cannot_filter() -> None:
|
||||
assert words(fake(), state(), reach.SMALL - 1) == ["read whole"]
|
||||
assert words(fake(), state(), reach.SMALL) == []
|
||||
@@ -199,6 +209,17 @@ def test_companies_carry_their_limits_and_the_boards_own_open_roles(
|
||||
assert row["limits"] == ["read incomplete", "without text: 1", "read whole"]
|
||||
|
||||
|
||||
def test_a_board_that_dropped_listings_says_so_on_its_row(
|
||||
client: TestClient, store: Store) -> None:
|
||||
store.add_subscription("ats", "lever:acme", company="Acme", board="lever")
|
||||
store.add_company("Acme", key="acme", ats="lever")
|
||||
store.state["ats:ok:lever:acme"] = "run"
|
||||
store.state["ats:complete:lever:acme"] = "1"
|
||||
store.state["ats:dropped:lever:acme"] = "3"
|
||||
[row] = client.get("/api/companies").json()
|
||||
assert row["limits"][0] == "3 listings dropped: no title or link"
|
||||
|
||||
|
||||
def test_a_company_without_a_board_has_no_limits(client: TestClient, store: Store) -> None:
|
||||
store.add_company("Novo", key="novo", ats="taleo")
|
||||
[row] = client.get("/api/companies").json()
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, adzuna # noqa: E402
|
||||
from plugins.readers import Ask, _runner, adzuna # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "adzuna.json").read_bytes()
|
||||
ASK = Ask(params={"q": "backend", "country": "de", "where": "Berlin",
|
||||
@@ -39,8 +39,9 @@ def test_it_declares_the_key_and_states_the_robots_finding() -> None:
|
||||
|
||||
|
||||
def test_it_strips_highlights_keeps_the_excerpt_as_teaser_and_drops_predicted_pay() -> None:
|
||||
first, predicted = _rows() # javascript: refused
|
||||
first, predicted, refused = _rows()
|
||||
assert [r["source_id"] for r in (first, predicted)] == ["4412345678", "4412345679"]
|
||||
assert refused["url"].startswith("javascript:") and not _runner.valid(refused)
|
||||
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
|
||||
assert first["teaser"] == "Build our backend services in Python & Go."
|
||||
assert "description" not in first
|
||||
@@ -89,3 +90,12 @@ def test_saved_params_need_a_known_country() -> None:
|
||||
"q": "x", "country": "gb"}
|
||||
with pytest.raises(ValueError, match="country"):
|
||||
adzuna.clean_params({"q": "x", "country": "dk"})
|
||||
|
||||
|
||||
def test_the_runner_drops_the_ad_without_a_web_link_and_counts_it() -> None:
|
||||
"""The reader hands the `javascript:` ad over; the runner drops it, and
|
||||
the read is complete, since all three result ids were read."""
|
||||
listing = _runner.read_listing(adzuna, "", lambda url, **kw: BODY, ASK,
|
||||
limits=_runner.Limits(page_delay=0))
|
||||
assert listing.ids == ["4412345678", "4412345679"]
|
||||
assert (listing.handed, listing.dropped, listing.complete) == (3, 1, True)
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, freehire # noqa: E402
|
||||
from plugins.readers import Ask, _runner, freehire # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "freehire-places.json").read_bytes()
|
||||
|
||||
@@ -119,14 +119,17 @@ def test_a_salary_of_another_period_is_a_line_and_no_figures() -> None:
|
||||
assert (year["salary_min"], year["salary_max"], year["currency"]) == (60000, None, "EUR")
|
||||
|
||||
|
||||
def test_it_refuses_a_hit_without_a_title_an_id_or_a_web_link() -> None:
|
||||
def test_a_hit_without_a_title_an_id_or_a_web_link_is_handed_to_the_runner() -> None:
|
||||
"""The reader hands every hit over; the runner keeps only the whole one."""
|
||||
hits = [{"public_slug": "a", "title": "A", "url": "javascript:alert(1)"},
|
||||
{"public_slug": "", "title": "B", "url": "https://x.example/b"},
|
||||
{"public_slug": "c", "title": " ", "url": "https://x.example/c"},
|
||||
{"public_slug": "d", "title": "D", "url": "https://x.example/d?utm_source=freehire.me"}]
|
||||
found = freehire.rows({"data": hits}, "")
|
||||
assert [row["source_id"] for row in found] == ["d"]
|
||||
assert found[0]["url"] == "https://x.example/d" and found[0]["places"] == []
|
||||
assert [row["source_id"] for row in found] == ["a", "", "c", "d"]
|
||||
kept = [row for row in found if _runner.valid(row)]
|
||||
assert [row["source_id"] for row in kept] == ["d"]
|
||||
assert kept[0]["url"] == "https://x.example/d" and kept[0]["places"] == []
|
||||
|
||||
|
||||
def test_a_title_and_a_company_have_their_entities_decoded() -> None:
|
||||
|
||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, jobbank # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jobbank # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||
PLACES = (FIXTURES / "jobbank-places.xml").read_bytes()
|
||||
@@ -170,7 +170,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
||||
body = _feed(99).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||
decoded, results, _total, nxt = jobbank.page(lambda url, **kw: body, "", None, Ask())
|
||||
assert len(results) == 100 and nxt == 2
|
||||
assert len(jobbank.rows(decoded, "")) == 99
|
||||
rows = jobbank.rows(decoded, "")
|
||||
assert len(rows) == 100 and sum(map(_runner.valid, rows)) == 99
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [b"<rss/>", b"<html><body>Just a moment...</body>",
|
||||
|
||||
@@ -15,7 +15,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, jobindex # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jobindex # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "jobindex.xml").read_bytes()
|
||||
|
||||
@@ -89,7 +89,8 @@ def test_a_dropped_item_does_not_pass_for_the_end() -> None:
|
||||
body = _feed(19).replace(b"</channel>", b"<item><title>No link</title></item></channel>")
|
||||
decoded, results, _total, nxt = jobindex.page(lambda url, **kw: body, "", None, Ask())
|
||||
assert len(results) == 20 and nxt == 2
|
||||
assert len(jobindex.rows(decoded, "")) == 19
|
||||
rows = jobindex.rows(decoded, "")
|
||||
assert len(rows) == 20 and sum(map(_runner.valid, rows)) == 19
|
||||
|
||||
|
||||
def test_an_answer_without_a_channel_raises() -> None:
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, jobnet # noqa: E402
|
||||
from plugins.readers import Ask, _runner, jobnet # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "search"
|
||||
PLACES = (FIXTURES / "jobnet-places.json").read_bytes()
|
||||
@@ -100,9 +100,12 @@ def test_the_deadline_is_kept_in_raw_and_1900_means_none() -> None:
|
||||
assert "applicationDeadline" not in row
|
||||
|
||||
|
||||
def test_an_ad_without_an_id_or_a_title_is_skipped() -> None:
|
||||
def test_an_ad_without_an_id_or_a_title_is_handed_over_for_the_runner_to_drop() -> None:
|
||||
ads = json.loads(OTHER)["jobAds"]
|
||||
assert jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "") == []
|
||||
rows = jobnet.rows({"jobAds": [ads[0] | {"title": " "}, ads[1] | {"jobAdId": ""}]}, "")
|
||||
assert [(row["source_id"] == "", row["title"] == "") for row in rows] == [
|
||||
(False, True), (True, False)]
|
||||
assert not any(map(_runner.valid, rows))
|
||||
|
||||
|
||||
def test_it_sends_the_header_the_region_and_the_page_number() -> None:
|
||||
|
||||
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import Ask, platsbanken # noqa: E402
|
||||
from plugins.readers import Ask, _runner, platsbanken # noqa: E402
|
||||
|
||||
BODY = (ROOT / "tests" / "fixtures" / "search" / "platsbanken-places.json").read_bytes()
|
||||
|
||||
@@ -47,8 +47,9 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
|
||||
decoded, _found, total, _cursor = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||
rows = platsbanken.rows(decoded, "")
|
||||
assert total == 32
|
||||
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400002"] # javascript: refused
|
||||
first, no_town, bare = rows
|
||||
assert [r["source_id"] for r in rows] == ["31457942", "31435504", "31400001", "31400002"]
|
||||
assert not _runner.valid(rows[2]) # javascript:, dropped by the runner
|
||||
first, no_town, _refused, bare = rows
|
||||
assert [p["text"] for p in first["places"]] == [
|
||||
"Lund", "Stockholm", "Västra Götalands län"] # municipality, else the län
|
||||
assert first["location"] == "Lund"
|
||||
@@ -58,7 +59,7 @@ def test_it_keeps_every_workplace_of_an_ad() -> None:
|
||||
|
||||
def test_it_reads_title_company_date_and_the_full_text() -> None:
|
||||
decoded, *_ = platsbanken.page(lambda url, **kw: BODY, "", None, Ask())
|
||||
first, _no_town, remote = platsbanken.rows(decoded, "")
|
||||
first, _no_town, _refused, remote = platsbanken.rows(decoded, "")
|
||||
assert first["title"] == "Postdoctoral Researcher"
|
||||
assert first["company"] == "European Spallation Source Eric"
|
||||
assert first["country"] == "SE" and first["remote"] == "Arbete på plats"
|
||||
|
||||
+107
-2
@@ -1,7 +1,7 @@
|
||||
"""The runner, `plugins/readers/_runner.py`, over fake readers: the `Ask` a
|
||||
subscription resolves to, series and their union, whole and paged listings,
|
||||
completeness from the declaration, the sweep across pulls, the 10% guard,
|
||||
the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||
completeness from the declaration, the rows kept and dropped, the sweep
|
||||
across pulls, the 10% guards, the facets kept in state, the detail reads and the `raw` an item carries. No
|
||||
network: `get` is a dict of URL -> body or exception."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -426,6 +426,93 @@ def test_the_guard_compares_only_reads_that_asked_the_same() -> None:
|
||||
assert filtered.ids == ["1"] and filtered.filtered
|
||||
|
||||
|
||||
# --------------------------------------------------------- the rows kept
|
||||
|
||||
def _handed(i: str) -> dict[str, Any]:
|
||||
"""A row as a reader hands it over: whole, or, for an id of `-id`,
|
||||
`-title`, `-link` or `-url`, without that value or with a link that is
|
||||
no web URL (`javascript:`, or one httpx cannot read)."""
|
||||
broken = {"-id": {"source_id": ""}, "-title": {"title": " "},
|
||||
"-link": {"url": "javascript:alert(1)"}, "-url": {"url": "http://[::1"}}
|
||||
return _row(i, **broken.get(i.split(":")[0], {}))
|
||||
|
||||
|
||||
def test_a_row_is_kept_with_an_id_a_title_and_a_web_link() -> None:
|
||||
assert _runner.valid(_handed("1"))
|
||||
assert not any(_runner.valid(_handed(i)) for i in ("-id", "-title", "-link", "-url"))
|
||||
assert not _runner.valid({"source_id": "1", "title": "Role"})
|
||||
|
||||
|
||||
def test_a_whole_listing_drops_the_rows_without_and_counts_them(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
module = _whole()
|
||||
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||
web = Web({"https://jobs.fake.example/acme/list":
|
||||
b'["1", "-id", "2", "-title", "-link", "-url"]'})
|
||||
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||
listing = _runner.read_listing(module, "acme", web)
|
||||
assert [r["source_id"] for r in listing.rows] == ["1", "2"]
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
assert (listing.handed, listing.dropped) == (6, 4)
|
||||
[line] = [r.getMessage() for r in caplog.records if "dropped" in r.getMessage()]
|
||||
assert "fake:acme" in line and "dropped 4 of 6 listings" in line
|
||||
|
||||
|
||||
def test_a_paged_listing_drops_on_every_page_and_counts_distinct_results(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""Completeness counts the result ids `page` returns, not the rows kept,
|
||||
so the dropped rows still count as read; the drop is logged once."""
|
||||
pages = [["1", "-title:a"], ["2", "-link:b"]]
|
||||
module = _paged(pages, total=4)
|
||||
module.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||
with caplog.at_level("WARNING", logger=_runner.__name__):
|
||||
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||
assert (listing.handed, listing.dropped) == (4, 2)
|
||||
assert sum("dropped" in r.getMessage() for r in caplog.records) == 1
|
||||
|
||||
|
||||
def test_the_stamp_keeps_the_rows_dropped() -> None:
|
||||
state: dict[str, str] = {}
|
||||
listing = _runner.Listing([_row("1")], True, ["1"], handed=3, dropped=2)
|
||||
_runner.stamp(state, "ats", "t", "r", listing, ["fake:acme:1"])
|
||||
assert state["ats:dropped:t"] == "2"
|
||||
_runner.stamp(state, "ats", "t", "r", _runner.Listing([], True, []), [])
|
||||
assert state["ats:dropped:t"] == ""
|
||||
|
||||
|
||||
def _dropping(handed: int, dropped: int) -> tuple[Any, Web]:
|
||||
"""A whole board whose feed hands over `handed` rows, `dropped` of them
|
||||
without a title."""
|
||||
module = _whole()
|
||||
module.parse = lambda body, slug: [_handed(i) for i in json.loads(body)]
|
||||
ids = [f"-title:{n}" for n in range(dropped)] + [str(n) for n in range(handed - dropped)]
|
||||
return module, Web({"https://jobs.fake.example/acme/list": json.dumps(ids).encode()})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("handed", "dropped", "fails"), [
|
||||
(20, 3, True), (20, 2, False), (9, 5, False), (10, 2, True)])
|
||||
def test_a_read_that_drops_over_a_tenth_of_its_rows_fails(
|
||||
handed: int, dropped: int, fails: bool) -> None:
|
||||
"""Over `SUSPECT_SHARE` of the rows, once `SUSPECT_MIN` were handed over:
|
||||
a page whose shape changed, which must close nothing."""
|
||||
module, web = _dropping(handed, dropped)
|
||||
if not fails:
|
||||
assert _runner.read({}, "ats", "t", module, "acme", web).dropped == dropped
|
||||
return
|
||||
with pytest.raises(_runner.Suspect, match=f"dropped {dropped} of {handed} listings"):
|
||||
_runner.read({}, "ats", "t", module, "acme", web)
|
||||
|
||||
|
||||
def test_a_read_that_drops_too_many_is_accepted_after_the_stale_days() -> None:
|
||||
module, web = _dropping(20, 5)
|
||||
since = (datetime.now(UTC) - timedelta(days=30)).isoformat(timespec="seconds")
|
||||
state = {"ats:failing:t": json.dumps({"since": since, "reason": "x"})}
|
||||
listing = _runner.read(state, "ats", "t", module, "acme", web,
|
||||
limits=_runner.Limits(page_delay=0, stale_days=21))
|
||||
assert (len(listing.ids), listing.dropped) == (15, 5)
|
||||
|
||||
|
||||
# ------------------------------------------------------ through ATSSource
|
||||
|
||||
@pytest.fixture
|
||||
@@ -479,6 +566,24 @@ def test_ats_source_hands_the_board_params_to_the_reader(board: Any) -> None:
|
||||
assert state["ats:filtered:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_ats_source_stamps_the_rows_dropped_and_fails_on_too_many(board: Any) -> None:
|
||||
board.rows = lambda ids, slug: [_handed(i) for i in ids]
|
||||
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||
json.dumps([str(n) for n in range(19)] + ["-link:x"]).encode()}
|
||||
board.READS = board.READS.model_copy(update={"complete": "end"})
|
||||
state: dict[str, str] = {}
|
||||
assert len(list(_source(state).fetch())) == 19
|
||||
assert state["ats:dropped:fake:acme"] == "1"
|
||||
|
||||
board.web.answers = {"https://jobs.fake.example/acme/page/0":
|
||||
json.dumps([str(n) for n in range(15)]
|
||||
+ [f"-id:{n}" for n in range(5)]).encode()}
|
||||
state = {}
|
||||
assert list(_source(state).fetch()) == []
|
||||
assert "dropped 5 of 20" in json.loads(state["ats:failing:fake:acme"])["reason"]
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- raw
|
||||
|
||||
def test_raw_keeps_every_stated_key_and_the_places() -> None:
|
||||
|
||||
@@ -383,6 +383,23 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch)
|
||||
assert source.state["search:ids:fake:cph"] == '["fake:h1"]'
|
||||
|
||||
|
||||
def test_a_search_drops_a_hit_without_a_title_and_stamps_the_count(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The search driver reads through the same listing code as a board: the
|
||||
runner drops the row the provider handed over without a title, and the
|
||||
read is still complete, by the distinct result ids."""
|
||||
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
||||
module = readers.PROVIDERS["fake"]
|
||||
handed = module.rows
|
||||
module.rows = lambda decoded, slug: [
|
||||
row | ({"title": ""} if row["source_id"] == "h2" else {}) for row in handed(decoded, slug)]
|
||||
source = SearchSource({"searches": [{"url": "fake:cph", "label": "Copenhagen",
|
||||
"params": {"q": "x"}}]})
|
||||
assert [i.source_id for i in source.fetch()] == ["fake:h1"]
|
||||
assert source.state["search:complete:fake:cph"] == "1"
|
||||
assert source.state["search:dropped:fake:cph"] == "1"
|
||||
|
||||
|
||||
def test_a_search_that_fails_is_skipped_and_stamps_nothing(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
|
||||
|
||||
Reference in new issue
Block a user