Files
jobboard/tests/test_discover.py
T
WolfandClaude Opus 5.5 3b209d628a Every way a company gets a board is set_board, and an unreadable host keeps its slug
`app.jobs.sources.set_board` attaches a board to an existing company,
replaces the one it has (closing the old board's open roles) or creates
the company, with row and subscription changed in one transaction by
`Store.set_company_board`. A board that lists zero roles is kept. `add`,
`discover --add`, the seed importer (moved out of app/cli.py) and the new
`POST|DELETE /api/companies/{key}/board` all go through it; a seed re-run
attaches a board to a row that never had one. A company's subscription is
rewritten in place, and detaching hides it.

`companies.ats_slug` (migration 9) keeps the slug of a host the app
cannot read yet, and discovery returns such a host before probing slugs.
The Companies rows get Set board / Change, the host select is rendered
from `plugins.ats.HOSTS`, and a row with nothing found reads "board not
found" and is in the "Not readable yet" filter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 14:04:52 +02:00

235 lines
10 KiB
Python

"""Board discovery: which ATS a careers site names, and in what order the
site's pages are read."""
from __future__ import annotations
import json
from pathlib import Path
import httpx
import pytest
from app.jobs import discover as mod
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
from app.jobs.sources import seed_companies
from app.storage import Store
# ------------------------------------------------------------------ parsing
def test_boards_in_names_readable_hosts_with_their_slug() -> None:
page = """
<a href="https://job-boards.greenhouse.io/pleo">Open roles</a>
<script src="https://boards.greenhouse.io/embed/job_board/js?for=pleo"></script>
<a href="https://jobs.lever.co/spotify/abc-123">One role</a>
<iframe src="https://jobs.ashbyhq.com/DeepL/embed"></iframe>
<a href="https://apply.workable.com/acme/">Workable</a>
<a href="https://jobs.smartrecruiters.com/Bosch/123">SR</a>
<a href="https://acme.recruitee.com/o/backend">Recruitee</a>
<a href="https://acme-gmbh.jobs.personio.de/job/1">Personio</a>
"""
assert [(h, s) for h, s, _url in boards_in(page)] == [
("greenhouse", "pleo"), ("lever", "spotify"), ("ashby", "deepl"),
("workable", "acme"), ("smartrecruiters", "bosch"), ("recruitee", "acme"),
("personio", "acme-gmbh")]
def test_boards_in_keeps_hosts_the_app_cannot_read() -> None:
page = """
<a href="https://novo.wd3.myworkdayjobs.com/en-US/careers">Workday</a>
<a href="https://performancemanager.successfactors.eu/career?company=x">SF</a>
<a href="https://acme.teamtailor.com/jobs">TT</a>
"""
assert boards_in(page) == [
("workday", "novo", "https://novo.wd3.myworkdayjobs.com"),
("successfactors", "", "performancemanager.successfactors.eu"),
("teamtailor", "acme", "https://acme.teamtailor.com")]
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [("greenhouse", "acme")]
assert boards_in("nothing here") == []
def test_careers_links_keeps_on_site_links_that_read_as_careers() -> None:
html = """
<a href="/about">About</a>
<a href="/careers">Careers</a>
<a href="https://jobs.acme.com/">We are hiring</a>
<a href="https://other.example/jobs">Partner jobs</a>
<a href="/en/karriere#top"><span>Karriere</span></a>
<a href="mailto:jobs@acme.com">Mail</a>
<a href="/careers">Careers again</a>
"""
assert careers_links(html, "https://www.acme.com/") == [
"https://www.acme.com/careers", "https://jobs.acme.com/",
"https://www.acme.com/en/karriere"]
def test_slug_guesses_come_from_the_key_and_the_domain() -> None:
assert slug_guesses("Trade Republic Bank GmbH", "www.traderepublic.com") == [
"trade-republic-bank", "traderepublicbank", "traderepublic"]
assert slug_guesses("Pleo", "pleo.io") == ["pleo"]
# ------------------------------------------------------------------- discover
class Site:
"""A fake web: `pages` maps a URL to `(final URL, html)`; anything else
is a page that did not answer. Remembers the order of fetches."""
def __init__(self, pages: dict[str, tuple[str, str] | str],
boards: set[tuple[str, str]] = frozenset()) -> None:
self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()}
self.boards = boards
self.fetched: list[str] = []
self.probed: list[tuple[str, str]] = []
def fetch(self, client: httpx.Client, url: str) -> tuple[str, str]:
self.fetched.append(url)
return self.pages.get(url, ("", ""))
def probe(self, client: httpx.Client, host: str, slug: str) -> bool:
self.probed.append((host, slug))
return (host, slug) in self.boards
def run(site: Site, name: str = "Acme", domain: str = "acme.com") -> Discovery:
with httpx.Client() as client:
return discover(name, domain, fetch=site.fetch, probe_slug=site.probe, client=client)
def test_a_board_named_on_the_home_page_ends_discovery_there() -> None:
site = Site({"https://acme.com/": '<a href="https://jobs.lever.co/acme">Jobs</a>'})
assert run(site) == Discovery(board="lever:acme", ats="lever", ats_slug="acme",
careers_url="https://acme.com/",
evidence="jobs.lever.co/acme")
assert site.fetched == ["https://acme.com/"]
assert site.probed == []
def test_the_home_pages_own_careers_link_is_read_before_the_usual_paths() -> None:
site = Site({
"https://acme.com/": '<a href="/company/join-us">Join us</a>',
"https://acme.com/company/join-us": ('https://acme.com/company/join-us',
'<a href="https://acme.recruitee.com/">Open</a>'),
"https://acme.com/careers": '<a href="https://jobs.lever.co/wrong">no</a>',
})
found = run(site)
assert found.board == "recruitee:acme"
assert found.careers_url == "https://acme.com/company/join-us"
assert site.fetched == ["https://acme.com/", "https://acme.com/company/join-us"]
def test_a_careers_pages_own_link_is_followed_one_level_down() -> None:
site = Site({
"https://acme.com/": "<p>no links</p>",
"https://acme.com/careers": '<a href="/careers/open-positions">See open positions</a>',
"https://acme.com/careers/open-positions":
'<iframe src="https://jobs.ashbyhq.com/acme"></iframe>',
})
assert run(site).board == "ashby:acme"
assert site.fetched[:3] == ["https://acme.com/", "https://acme.com/careers",
"https://acme.com/careers/open-positions"]
def test_an_unreadable_host_named_by_a_page_is_the_answer_and_nothing_is_probed() -> None:
site = Site({
"https://acme.com/": "<p>welcome</p>",
"https://acme.com/careers": '<a href="https://acme.wd3.myworkdayjobs.com/x">Jobs</a>',
}, boards={("greenhouse", "acme")})
found = run(site)
assert found == Discovery(ats="workday", ats_slug="acme",
careers_url="https://acme.com/careers",
evidence="https://acme.wd3.myworkdayjobs.com")
assert not found.readable
# The page named the host: a probe could only find another tenant's
# board of the same name, like the greenhouse:acme this fake web has.
assert site.probed == []
def test_a_readable_board_on_a_later_page_still_beats_an_unreadable_host() -> None:
site = Site({
"https://acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>'
'<a href="/careers">Careers</a>',
"https://acme.com/careers": '<a href="https://jobs.lever.co/acme">Jobs</a>',
})
assert run(site).board == "lever:acme"
assert site.probed == []
def test_an_unreadable_hosts_slug_is_kept_when_a_later_link_names_it() -> None:
"""The CDN names teamtailor before any link does; the slug comes from the
first link that carries one, the acme of acme.teamtailor.com."""
site = Site({
"https://acme.com/": '<script src="https://cdn.teamtailor.com/x.js"></script>',
"https://acme.com/careers": '<a href="https://acme-jobs.teamtailor.com/jobs">Jobs</a>',
})
found = run(site)
assert (found.ats, found.ats_slug, found.board) == ("teamtailor", "acme-jobs", "")
assert found.evidence == "https://acme-jobs.teamtailor.com"
assert found.careers_url == "https://acme.com/"
def test_slug_probes_find_a_board_the_pages_never_link() -> None:
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>",
"https://acme.com/jobs": "<div id=app></div>"},
boards={("ashby", "acme")})
found = run(site)
assert found.board == "ashby:acme"
assert found.careers_url == "https://acme.com/jobs"
assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme"
def test_a_site_that_never_answers_gives_an_empty_discovery() -> None:
site = Site({})
assert run(site) == Discovery()
assert len(site.fetched) <= mod.MAX_PAGES + 1
def test_a_domain_is_folded_to_its_site() -> None:
site = Site({"https://acme.com/": '<a href="https://jobs.lever.co/acme">Jobs</a>'})
with httpx.Client() as client:
found = discover("Acme", "https://www.acme.com/some/page", fetch=site.fetch,
probe_slug=site.probe, client=client)
assert found.board == "lever:acme"
assert discover("Acme", "", fetch=site.fetch, probe_slug=site.probe) == Discovery()
# ---------------------------------------------------------------- the seed
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "t.db")
def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Path) -> None:
seed = tmp_path / "companies.json"
seed.write_text(json.dumps({"companies": [
{"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen",
"board": "ashby:pleo"},
{"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK",
"city": "Bagsværd", "ats": "successfactors",
"careers_url": "https://www.novonordisk.com/careers.html"},
{"name": "Odd", "board": "nosuch:odd"},
]}))
result = seed_companies(store, seed)
assert result == {"added": 3, "boards": 1, "skipped": 0, "total": 3,
"problems": ["odd: 'nosuch:odd' is not a known board; kept without it"]}
pleo = store.company("pleo")
assert pleo is not None
assert (pleo.ats, pleo.ats_slug, pleo.country, pleo.city, pleo.careers_url) == (
"ashby", "pleo", "DK", "Copenhagen", "https://jobs.ashbyhq.com/pleo")
[sub] = store.subscriptions(source="ats")
assert (sub.target, sub.company) == ("ashby:pleo", "Pleo")
novo = store.company("novo-nordisk")
assert novo is not None
assert (novo.ats, novo.careers_url) == ("successfactors",
"https://www.novonordisk.com/careers.html")
odd = store.company("odd")
assert odd is not None and odd.ats == ""
# Re-running adds nothing and edits nothing.
assert seed_companies(store, seed)["skipped"] == 3