Delete the _UNREADABLE entries of hosts that have a module

Ten entries were shadowed by their modules. A test now holds that no entry
names a reader, which replaces the per-host checks.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:51:48 +02:00
1 parent 5f1b9e93e1
commit 531c8bc337
7 files changed
+11 -18

No files matched your search

+4 -13
View File
@@ -10,7 +10,8 @@ parses) becomes a `board` target the pull can read; any other host is
recorded as `ats`, with its slug where the URL carries one, so the company is recorded as `ats`, with its slug where the URL carries one, so the company is
kept as one the app cannot read yet rather than dropped. The patterns and kept as one the app cannot read yet rather than dropped. The patterns and
probes of a readable host are its module's; the hosts only named are in probes of a readable host are its module's; the hosts only named are in
`_UNREADABLE` here, and one of them that gains a module is read from then on. `_UNREADABLE` here, and a host that gains a module loses its entry there
(`tests/test_discover.py` holds that no entry names a reader).
A Teamtailor site on the company's own domain names no slug: the page only A Teamtailor site on the company's own domain names no slug: the page only
loads Teamtailor's CDN. The board there is the site's own host, and it is loads Teamtailor's CDN. The board there is the site's own host, and it is
@@ -65,27 +66,17 @@ READABLE: tuple[str, ...] = tuple(readers.HOSTS)
#: company by, kept for the day a parser exists; a host with no group is #: company by, kept for the day a parser exists; a host with no group is
#: recognised but its board has no slug the app could use. #: recognised but its board has no slug the app could use.
_UNREADABLE: tuple[tuple[str, str], ...] = ( _UNREADABLE: tuple[tuple[str, str], ...] = (
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"), ("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"), ("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"), ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"), ("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"), ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"), ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"), ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"), ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"), ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
("eightfold", r"\.eightfold\.ai"),
("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"), ("avature", r"\.avature\.net"),
("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
) )
@@ -94,8 +85,8 @@ def _hosts(modules: Mapping[str, ModuleType],
"""Every host's patterns, compiled: the readable hosts' from their """Every host's patterns, compiled: the readable hosts' from their
modules first, so the first match at one place on a page is the one the modules first, so the first match at one place on a page is the one the
app can read, then the unreadable ones. An unreadable host that has app can read, then the unreadable ones. An unreadable host that has
gained a module is left out here, since its module's patterns replace a module is left out, so its module's patterns win even before its
it.""" entry is deleted."""
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS] out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
out += [(host, pattern) for host, pattern in unreadable if host not in modules] out += [(host, pattern) for host, pattern in unreadable if host not in modules]
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out) return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
-1
View File
@@ -187,4 +187,3 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None:
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>' '<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
'<a href="https://resources.bamboohr.com/y">z</a>') '<a href="https://resources.bamboohr.com/y">z</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")] assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
-1
View File
@@ -201,7 +201,6 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None
found = [(h, s) for h, s, _ in discover.boards_in(page)] found = [(h, s) for h, s, _ in discover.boards_in(page)]
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
assert ("eightfold", "") in found assert ("eightfold", "") in found
assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None: def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None:
-1
View File
@@ -178,4 +178,3 @@ def test_discovery_finds_the_board_keeping_its_case() -> None:
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>') '<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [ assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
("gem", "Productboard"), ("gem", "sennder-gmbh")] ("gem", "Productboard"), ("gem", "sennder-gmbh")]
assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
-1
View File
@@ -176,7 +176,6 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">' page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>') '<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")] assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
assert "jibe" not in {h for h, _ in discover._UNREADABLE}
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None: def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
-1
View File
@@ -145,7 +145,6 @@ def test_discovery_finds_the_subdomain() -> None:
'<script src="https://www.pinpointhq.com/embed.js"></script>') '<script src="https://www.pinpointhq.com/embed.js"></script>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")] assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
assert discover.boards_in("https://pinpointhq.com/") == [] assert discover.boards_in("https://pinpointhq.com/") == []
assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_a_pinpoint_board_is_not_probed() -> None: def test_a_pinpoint_board_is_not_probed() -> None:
+7
View File
@@ -17,6 +17,7 @@ from app.jobs import discover as mod
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
from app.jobs.sources import seed_companies from app.jobs.sources import seed_companies
from app.storage import Store from app.storage import Store
from plugins import readers
# ------------------------------------------------------------------ parsing # ------------------------------------------------------------------ parsing
@@ -95,6 +96,12 @@ def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
def test_no_unreadable_entry_names_a_host_with_a_module() -> None:
"""A new module deletes its host's `_UNREADABLE` entry: its own patterns
replace it."""
assert not {host for host, _ in mod._UNREADABLE} & set(readers.HOSTS)
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None: def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>' page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>') '<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')