diff --git a/app/jobs/discover.py b/app/jobs/discover.py index c4bd671..121c282 100644 --- a/app/jobs/discover.py +++ b/app/jobs/discover.py @@ -10,7 +10,8 @@ parses) becomes a `board` target the pull can read; any other host is recorded as `ats`, with its slug where the URL carries one, so the company is kept as one the app cannot read yet rather than dropped. The patterns and probes of a readable host are its module's; the hosts only named are in -`_UNREADABLE` here, and one of them that gains a module is read from then on. +`_UNREADABLE` here, and a host that gains a module loses its entry there +(`tests/test_discover.py` holds that no entry names a reader). A Teamtailor site on the company's own domain names no slug: the page only loads Teamtailor's CDN. The board there is the site's own host, and it is @@ -65,27 +66,17 @@ READABLE: tuple[str, ...] = tuple(readers.HOSTS) #: company by, kept for the day a parser exists; a host with no group is #: recognised but its board has no slug the app could use. _UNREADABLE: tuple[tuple[str, str], ...] = ( - ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"), - ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"), ("icims", r"https?://([a-z0-9-]+)\.icims\.com"), ("taleo", r"\.taleo\.net"), ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"), - ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"), ("join", r"join\.com/companies/([a-z0-9-]+)"), ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"), - ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"), - ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"), - ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"), ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"), ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"), ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"), ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"), - ("eightfold", r"\.eightfold\.ai"), - ("phenom", r"phenompeople\.com|phenom\.com/careers"), ("avature", r"\.avature\.net"), - ("oracle", r"oraclecloud\.com/hcmUI"), ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"), - ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"), ) @@ -94,8 +85,8 @@ def _hosts(modules: Mapping[str, ModuleType], """Every host's patterns, compiled: the readable hosts' from their modules first, so the first match at one place on a page is the one the app can read, then the unreadable ones. An unreadable host that has - gained a module is left out here, since its module's patterns replace - it.""" + a module is left out, so its module's patterns win even before its + entry is deleted.""" out = [(name, pattern) for name, module in modules.items() for pattern in module.PATTERNS] out += [(host, pattern) for host, pattern in unreadable if host not in modules] return tuple((host, re.compile(pattern, re.IGNORECASE)) for host, pattern in out) diff --git a/tests/test_ats_bamboohr.py b/tests/test_ats_bamboohr.py index cdb773d..09f17d4 100644 --- a/tests/test_ats_bamboohr.py +++ b/tests/test_ats_bamboohr.py @@ -187,4 +187,3 @@ def test_discovery_finds_the_subdomain_and_not_bamboohrs_own_hosts() -> None: 'xy' 'z') assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("bamboohr", "configit")] - assert "bamboohr" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py index 171ac68..266098b 100644 --- a/tests/test_ats_eightfold.py +++ b/tests/test_ats_eightfold.py @@ -201,7 +201,6 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None found = [(h, s) for h, s, _ in discover.boards_in(page)] assert ("eightfold", "infineon.eightfold.ai/infineon.com") in found assert ("eightfold", "") in found - assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None: diff --git a/tests/test_ats_gem.py b/tests/test_ats_gem.py index dee67af..c5f76bf 100644 --- a/tests/test_ats_gem.py +++ b/tests/test_ats_gem.py @@ -178,4 +178,3 @@ def test_discovery_finds_the_board_keeping_its_case() -> None: 'y') assert [(h, s) for h, s, _ in discover.boards_in(page)] == [ ("gem", "Productboard"), ("gem", "sennder-gmbh")] - assert "gem" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} diff --git a/tests/test_ats_jibe.py b/tests/test_ats_jibe.py index 61e8bc7..05568c4 100644 --- a/tests/test_ats_jibe.py +++ b/tests/test_ats_jibe.py @@ -176,7 +176,6 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None: page = ('' 'apply') assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")] - assert "jibe" not in {h for h, _ in discover._UNREADABLE} def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None: diff --git a/tests/test_ats_pinpoint.py b/tests/test_ats_pinpoint.py index e568e3e..b3741de 100644 --- a/tests/test_ats_pinpoint.py +++ b/tests/test_ats_pinpoint.py @@ -145,7 +145,6 @@ def test_discovery_finds_the_subdomain() -> None: '') assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("pinpoint", "xeneta")] assert discover.boards_in("https://pinpointhq.com/") == [] - assert "pinpoint" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} def test_a_pinpoint_board_is_not_probed() -> None: diff --git a/tests/test_discover.py b/tests/test_discover.py index 2e072f4..2440f25 100644 --- a/tests/test_discover.py +++ b/tests/test_discover.py @@ -17,6 +17,7 @@ from app.jobs import discover as mod from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses from app.jobs.sources import seed_companies from app.storage import Store +from plugins import readers # ------------------------------------------------------------------ parsing @@ -95,6 +96,12 @@ def test_a_host_that_gains_a_module_replaces_its_unreadable_pattern() -> None: assert "taleo" not in mod.READABLE and ("taleo", r"\.taleo\.net") in mod._UNREADABLE +def test_no_unreadable_entry_names_a_host_with_a_module() -> None: + """A new module deletes its host's `_UNREADABLE` entry: its own patterns + replace it.""" + assert not {host for host, _ in mod._UNREADABLE} & set(readers.HOSTS) + + def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None: page = ('a' 'b')