Delete the _UNREADABLE entries of hosts that have a module

Ten entries were shadowed by their modules. A test now holds that no entry
names a reader, which replaces the per-host checks.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:51:48 +02:00
1 parent 5f1b9e93e1
commit 531c8bc337
7 files changed
+11 -18

No files matched your search

+4 -13
View File
@@ -10,7 +10,8 @@ parses) becomes a `board` target the pull can read; any other host is
recorded as `ats`, with its slug where the URL carries one, so the company is
kept as one the app cannot read yet rather than dropped. The patterns and
probes of a readable host are its module's; the hosts only named are in
`_UNREADABLE` here, and one of them that gains a module is read from then on.
`_UNREADABLE` here, and a host that gains a module loses its entry there
(`tests/test_discover.py` holds that no entry names a reader).
A Teamtailor site on the company's own domain names no slug: the page only
loads Teamtailor's CDN. The board there is the site's own host, and it is
@@ -65,27 +66,17 @@ READABLE: tuple[str, ...] = tuple(readers.HOSTS)
#: company by, kept for the day a parser exists; a host with no group is
#: recognised but its board has no slug the app could use.
_UNREADABLE: tuple[tuple[str, str], ...] = (
("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
("taleo", r"\.taleo\.net"),
("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
("join", r"join\.com/companies/([a-z0-9-]+)"),
("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
("eightfold", r"\.eightfold\.ai"),
("phenom", r"phenompeople\.com|phenom\.com/careers"),
("avature", r"\.avature\.net"),
("oracle", r"oraclecloud\.com/hcmUI"),
("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
)
@@ -94,8 +85,8 @@ def _hosts(modules: Mapping[str, ModuleType],
"""Every host's patterns, compiled: the readable hosts' from their
modules first, so the first match at one place on a page is the one the
app can read, then the unreadable ones. An unreadable host that has
gained a module is left out here, since its module's patterns replace
it."""
a module is left out, so its module's patterns win even before its
entry is deleted."""
out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS]
out += [(host, pattern) for host, pattern in unreadable if host not in modules]
return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out)
-1
View File
@@ -187,4 +187,3 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None:
'<a href="https://www.bamboohr.com/">x</a><a href="https://app.bamboohr.com/x">y</a>'
'<a href="https://resources.bamboohr.com/y">z</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")]
assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
-1
View File
@@ -201,7 +201,6 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None
found = [(h, s) for h, s, _ in discover.boards_in(page)]
assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found
assert ("eightfold", "") in found
assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None:
-1
View File
@@ -178,4 +178,3 @@ def test_discovery_finds_the_board_keeping_its_case() -> None:
'<a href="https://jobs.gem.com/sennder-gmbh/8080046002">y</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [
("gem", "Productboard"), ("gem", "sennder-gmbh")]
assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
-1
View File
@@ -176,7 +176,6 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
assert "jibe" not in {h for h, _ in discover._UNREADABLE}
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
-1
View File
@@ -145,7 +145,6 @@ def test_discovery_finds_the_subdomain() -> None:
'<script src="https://www.pinpointhq.com/embed.js"></script>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")]
assert discover.boards_in("https://pinpointhq.com/") == []
assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS}
def test_a_pinpoint_board_is_not_probed() -> None:
+7
View File
@@ -17,6 +17,7 @@ from app.jobs import discover as mod
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
from app.jobs.sources import seed_companies
from app.storage import Store
from plugins import readers
# ------------------------------------------------------------------ parsing
@@ -95,6 +96,12 @@ def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None:
assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE
def test_no_unreadable_entry_names_a_host_with_a_module() -> None:
"""A new module deletes its host's `_UNREADABLE` entry: its own patterns
replace it."""
assert not {host for host, _ in mod._UNREADABLE} & set(readers.HOSTS)
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')