A European seed of 362 employers, readable or not; discover finds a company's board from its site
The seed was jobhunt-hq's 186 US boards, chosen because they could be read. It is now a hand-picked list of European tech employers, with `board` filled by the new `app.jobs.discover` from each company's own careers site and every board fetched once before it landed: 175 have one, and the rest keep the ATS host their site runs on (Teamtailor 32, Workday 12, SuccessFactors 9, ...) and a careers URL, as rows the app cannot read yet. Companies shows them as "not readable yet" with a link, behind a filter, so the next parser to write is the host that filter shows most. `app companies discover NAME DOMAIN [--add]` reads the home page, its careers links and the usual paths for a link into a known host, one level down on a careers page, and probes Greenhouse, Lever, Ashby, SmartRecruiters, Recruitee and Personio by slug when the pages link nothing. It is what the LinkedIn flow will call. Migration 7 adds companies.ats, country and city. The Workable parser now reads the v3 shape its list endpoint actually sends (location object, department list, shortcode and no URL or text); the first populated board showed it. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
c4f2bcc0c9
commit
51d1e67ff9
19 files changed
+3362
-1394
No files matched your search
Vendored
+34
-25
@@ -2,33 +2,42 @@
|
||||
"total": 2,
|
||||
"results": [
|
||||
{
|
||||
"id": "4001",
|
||||
"title": "Backend Engineer",
|
||||
"department": "Engineering",
|
||||
"city": "Berlin",
|
||||
"region": "",
|
||||
"country": "Germany",
|
||||
"telecommute": true,
|
||||
"published_on": "2026-09-01",
|
||||
"created_at": "2026-08-30T10:00:00.000Z",
|
||||
"url": "https://workable.example/1",
|
||||
"shortlink": "https://workable.example/short/1",
|
||||
"description": "<p>Ship our API.</p>",
|
||||
"requirements": "<p>5+ years.</p>",
|
||||
"benefits": ""
|
||||
"id": 6125026,
|
||||
"shortcode": "DB4D7C0EC8",
|
||||
"title": "Senior Open-Source Python Engineer, ML Developer Tools - EMEA Remote",
|
||||
"remote": true,
|
||||
"location": {
|
||||
"country": "France",
|
||||
"countryCode": "FR",
|
||||
"city": "",
|
||||
"region": null
|
||||
},
|
||||
"state": "published",
|
||||
"published": "2026-09-21T00:00:00.000Z",
|
||||
"type": "full",
|
||||
"department": [
|
||||
"Product"
|
||||
],
|
||||
"workplace": "remote"
|
||||
},
|
||||
{
|
||||
"id": "4002",
|
||||
"shortcode": "ABCDEF",
|
||||
"title": "Account Manager",
|
||||
"department": "Sales",
|
||||
"city": "Munich",
|
||||
"region": "",
|
||||
"country": "Germany",
|
||||
"telecommute": false,
|
||||
"published_on": "2026-08-15",
|
||||
"url": "https://workable.example/2",
|
||||
"description": ""
|
||||
"id": 6125041,
|
||||
"shortcode": "F88446C814",
|
||||
"title": "Senior Open-Source Python Engineer, ML Developer Tools - US Remote",
|
||||
"remote": false,
|
||||
"location": {
|
||||
"country": "Germany",
|
||||
"countryCode": "DE",
|
||||
"city": "Berlin",
|
||||
"region": null
|
||||
},
|
||||
"state": "published",
|
||||
"published": "2026-09-21T00:00:00.000Z",
|
||||
"type": "full",
|
||||
"department": [
|
||||
"Product"
|
||||
],
|
||||
"workplace": "hybrid"
|
||||
}
|
||||
]
|
||||
}
|
||||
+9
-5
@@ -59,12 +59,16 @@ def test_ashby_parses_is_remote_and_compensation() -> None:
|
||||
assert rows[1]["remote"] is True
|
||||
|
||||
|
||||
def test_workable_parses_location_parts_and_telecommute() -> None:
|
||||
rows = _rows("workable")
|
||||
assert [r["source_id"] for r in rows] == ["4001", "4002"]
|
||||
assert rows[0]["location"] == "Berlin, Germany"
|
||||
def test_workable_builds_the_job_url_and_reads_location_and_remote() -> None:
|
||||
rows = _rows("workable", slug="huggingface")
|
||||
assert [r["source_id"] for r in rows] == ["6125026", "6125041"]
|
||||
assert rows[0]["url"] == "https://apply.workable.com/huggingface/j/DB4D7C0EC8/"
|
||||
assert rows[0]["location"] == "France"
|
||||
assert rows[0]["remote"] is True
|
||||
assert "Ship our API." in rows[0]["description"]
|
||||
assert rows[0]["department"] == "Product"
|
||||
assert rows[1]["location"] == "Berlin, Germany"
|
||||
assert rows[1]["remote"] is False
|
||||
assert rows[0]["description"] == "" # the list endpoint carries no text
|
||||
|
||||
|
||||
def test_smartrecruiters_builds_the_job_url_from_id_and_slug() -> None:
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Board discovery: which ATS a careers site names, and in what order the
|
||||
site's pages are read."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.cli import seed_companies
|
||||
from app.jobs import discover as mod
|
||||
from app.jobs.discover import Discovery, boards_in, careers_links, discover, slug_guesses
|
||||
from app.storage import Store
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ parsing
|
||||
|
||||
def test_boards_in_names_readable_hosts_with_their_slug() -> None:
|
||||
page = """
|
||||
<a href="https://job-boards.greenhouse.io/pleo">Open roles</a>
|
||||
<script src="https://boards.greenhouse.io/embed/job_board/js?for=pleo"></script>
|
||||
<a href="https://jobs.lever.co/spotify/abc-123">One role</a>
|
||||
<iframe src="https://jobs.ashbyhq.com/DeepL/embed"></iframe>
|
||||
<a href="https://apply.workable.com/acme/">Workable</a>
|
||||
<a href="https://jobs.smartrecruiters.com/Bosch/123">SR</a>
|
||||
<a href="https://acme.recruitee.com/o/backend">Recruitee</a>
|
||||
<a href="https://acme-gmbh.jobs.personio.de/job/1">Personio</a>
|
||||
"""
|
||||
assert [(h, s) for h, s, _url in boards_in(page)] == [
|
||||
("greenhouse", "pleo"), ("lever", "spotify"), ("ashby", "deepl"),
|
||||
("workable", "acme"), ("smartrecruiters", "bosch"), ("recruitee", "acme"),
|
||||
("personio", "acme-gmbh")]
|
||||
|
||||
|
||||
def test_boards_in_keeps_hosts_the_app_cannot_read() -> None:
|
||||
page = """
|
||||
<a href="https://novo.wd3.myworkdayjobs.com/en-US/careers">Workday</a>
|
||||
<a href="https://performancemanager.successfactors.eu/career?company=x">SF</a>
|
||||
<a href="https://acme.teamtailor.com/jobs">TT</a>
|
||||
"""
|
||||
assert boards_in(page) == [
|
||||
("workday", "novo", "https://novo.wd3.myworkdayjobs.com"),
|
||||
("successfactors", "", "performancemanager.successfactors.eu"),
|
||||
("teamtailor", "acme", "https://acme.teamtailor.com")]
|
||||
|
||||
|
||||
def test_boards_in_dedupes_and_skips_the_api_paths_own_words() -> None:
|
||||
page = ('<a href="https://boards.greenhouse.io/embed/job_board?for=acme">a</a>'
|
||||
'<a href="https://boards-api.greenhouse.io/v1/boards/acme/jobs">b</a>')
|
||||
assert [(h, s) for h, s, _ in boards_in(page)] == [("greenhouse", "acme")]
|
||||
assert boards_in("nothing here") == []
|
||||
|
||||
|
||||
def test_careers_links_keeps_on_site_links_that_read_as_careers() -> None:
|
||||
html = """
|
||||
<a href="/about">About</a>
|
||||
<a href="/careers">Careers</a>
|
||||
<a href="https://jobs.acme.com/">We are hiring</a>
|
||||
<a href="https://other.example/jobs">Partner jobs</a>
|
||||
<a href="/en/karriere#top"><span>Karriere</span></a>
|
||||
<a href="mailto:jobs@acme.com">Mail</a>
|
||||
<a href="/careers">Careers again</a>
|
||||
"""
|
||||
assert careers_links(html, "https://www.acme.com/") == [
|
||||
"https://www.acme.com/careers", "https://jobs.acme.com/",
|
||||
"https://www.acme.com/en/karriere"]
|
||||
|
||||
|
||||
def test_slug_guesses_come_from_the_key_and_the_domain() -> None:
|
||||
assert slug_guesses("Trade Republic Bank GmbH", "www.traderepublic.com") == [
|
||||
"trade-republic-bank", "traderepublicbank", "traderepublic"]
|
||||
assert slug_guesses("Pleo", "pleo.io") == ["pleo"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- discover
|
||||
|
||||
class Site:
|
||||
"""A fake web: `pages` maps a URL to `(final URL, html)`; anything else
|
||||
is a page that did not answer. Remembers the order of fetches."""
|
||||
|
||||
def __init__(self, pages: dict[str, tuple[str, str] | str],
|
||||
boards: set[tuple[str, str]] = frozenset()) -> None:
|
||||
self.pages = {k: (v if isinstance(v, tuple) else (k, v)) for k, v in pages.items()}
|
||||
self.boards = boards
|
||||
self.fetched: list[str] = []
|
||||
self.probed: list[tuple[str, str]] = []
|
||||
|
||||
def fetch(self, client: httpx.Client, url: str) -> tuple[str, str]:
|
||||
self.fetched.append(url)
|
||||
return self.pages.get(url, ("", ""))
|
||||
|
||||
def probe(self, client: httpx.Client, host: str, slug: str) -> bool:
|
||||
self.probed.append((host, slug))
|
||||
return (host, slug) in self.boards
|
||||
|
||||
|
||||
def run(site: Site, name: str = "Acme", domain: str = "acme.com") -> Discovery:
|
||||
with httpx.Client() as client:
|
||||
return discover(name, domain, fetch=site.fetch, probe_slug=site.probe, client=client)
|
||||
|
||||
|
||||
def test_a_board_named_on_the_home_page_ends_discovery_there() -> None:
|
||||
site = Site({"https://acme.com/": '<a href="https://jobs.lever.co/acme">Jobs</a>'})
|
||||
assert run(site) == Discovery(board="lever:acme", ats="lever",
|
||||
careers_url="https://acme.com/",
|
||||
evidence="jobs.lever.co/acme")
|
||||
assert site.fetched == ["https://acme.com/"]
|
||||
assert site.probed == []
|
||||
|
||||
|
||||
def test_the_home_pages_own_careers_link_is_read_before_the_usual_paths() -> None:
|
||||
site = Site({
|
||||
"https://acme.com/": '<a href="/company/join-us">Join us</a>',
|
||||
"https://acme.com/company/join-us": ('https://acme.com/company/join-us',
|
||||
'<a href="https://acme.recruitee.com/">Open</a>'),
|
||||
"https://acme.com/careers": '<a href="https://jobs.lever.co/wrong">no</a>',
|
||||
})
|
||||
found = run(site)
|
||||
assert found.board == "recruitee:acme"
|
||||
assert found.careers_url == "https://acme.com/company/join-us"
|
||||
assert site.fetched == ["https://acme.com/", "https://acme.com/company/join-us"]
|
||||
|
||||
|
||||
def test_a_careers_pages_own_link_is_followed_one_level_down() -> None:
|
||||
site = Site({
|
||||
"https://acme.com/": "<p>no links</p>",
|
||||
"https://acme.com/careers": '<a href="/careers/open-positions">See open positions</a>',
|
||||
"https://acme.com/careers/open-positions":
|
||||
'<iframe src="https://jobs.ashbyhq.com/acme"></iframe>',
|
||||
})
|
||||
assert run(site).board == "ashby:acme"
|
||||
assert site.fetched[:3] == ["https://acme.com/", "https://acme.com/careers",
|
||||
"https://acme.com/careers/open-positions"]
|
||||
|
||||
|
||||
def test_an_unreadable_host_is_kept_when_nothing_readable_turns_up() -> None:
|
||||
site = Site({
|
||||
"https://acme.com/": "<p>welcome</p>",
|
||||
"https://acme.com/careers": '<a href="https://acme.wd3.myworkdayjobs.com/x">Jobs</a>',
|
||||
})
|
||||
found = run(site)
|
||||
assert found == Discovery(ats="workday", careers_url="https://acme.com/careers",
|
||||
evidence="https://acme.wd3.myworkdayjobs.com")
|
||||
assert not found.readable
|
||||
# Every slug was still tried on the hosts with a probe, in case a readable
|
||||
# board exists that the pages do not link.
|
||||
assert ("greenhouse", "acme") in site.probed
|
||||
|
||||
|
||||
def test_slug_probes_find_a_board_the_pages_never_link() -> None:
|
||||
site = Site({"https://acme.com/": "<p>a script draws the jobs</p>",
|
||||
"https://acme.com/jobs": "<div id=app></div>"},
|
||||
boards={("ashby", "acme")})
|
||||
found = run(site)
|
||||
assert found.board == "ashby:acme"
|
||||
assert found.careers_url == "https://acme.com/jobs"
|
||||
assert found.evidence == "https://api.ashbyhq.com/posting-api/job-board/acme"
|
||||
|
||||
|
||||
def test_a_site_that_never_answers_gives_an_empty_discovery() -> None:
|
||||
site = Site({})
|
||||
assert run(site) == Discovery()
|
||||
assert len(site.fetched) <= mod.MAX_PAGES + 1
|
||||
|
||||
|
||||
def test_a_domain_is_folded_to_its_site() -> None:
|
||||
site = Site({"https://acme.com/": '<a href="https://jobs.lever.co/acme">Jobs</a>'})
|
||||
with httpx.Client() as client:
|
||||
found = discover("Acme", "https://www.acme.com/some/page", fetch=site.fetch,
|
||||
probe_slug=site.probe, client=client)
|
||||
assert found.board == "lever:acme"
|
||||
assert discover("Acme", "", fetch=site.fetch, probe_slug=site.probe) == Discovery()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the seed
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "t.db")
|
||||
|
||||
|
||||
def test_the_seed_keeps_a_company_the_app_cannot_read(store: Store, tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str]) -> None:
|
||||
seed = tmp_path / "companies.json"
|
||||
seed.write_text(json.dumps({"companies": [
|
||||
{"name": "Pleo", "domain": "pleo.io", "country": "DK", "city": "Copenhagen",
|
||||
"board": "ashby:pleo"},
|
||||
{"name": "Novo Nordisk", "domain": "novonordisk.com", "country": "DK",
|
||||
"city": "Bagsværd", "ats": "successfactors",
|
||||
"careers_url": "https://www.novonordisk.com/careers.html"},
|
||||
{"name": "Odd", "board": "nosuch:odd"},
|
||||
]}))
|
||||
assert seed_companies(store, str(seed)) == {"added": 3, "readable": 1, "skipped": 0,
|
||||
"total": 3}
|
||||
pleo = store.company("pleo")
|
||||
assert pleo is not None
|
||||
assert (pleo.ats, pleo.country, pleo.city, pleo.careers_url) == (
|
||||
"ashby", "DK", "Copenhagen", "https://jobs.ashbyhq.com/pleo")
|
||||
[sub] = store.subscriptions(source="ats")
|
||||
assert (sub.target, sub.company) == ("ashby:pleo", "Pleo")
|
||||
novo = store.company("novo-nordisk")
|
||||
assert novo is not None
|
||||
assert (novo.ats, novo.careers_url) == ("successfactors",
|
||||
"https://www.novonordisk.com/careers.html")
|
||||
odd = store.company("odd")
|
||||
assert odd is not None and odd.ats == ""
|
||||
assert "not a known board" in capsys.readouterr().err
|
||||
# Re-running adds nothing and edits nothing.
|
||||
assert seed_companies(store, str(seed))["skipped"] == 3
|
||||
@@ -93,6 +93,8 @@ def test_a_v1_file_migrates_to_the_shape_a_new_file_has(tmp_path: Path) -> None:
|
||||
# v1 had affinity on the company; the vote columns v4 added and v6
|
||||
# drops again are not in a fresh file, so there is nothing to put back.
|
||||
conn.execute("ALTER TABLE companies ADD COLUMN affinity INTEGER NOT NULL DEFAULT 0")
|
||||
for column in ("ats", "country", "city"): # v7
|
||||
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
||||
for table in ("vectors", "role_tags", "agent_companies", "role_votes"):
|
||||
conn.execute(f"DROP TABLE {table}")
|
||||
conn.execute("PRAGMA user_version = 1")
|
||||
@@ -113,6 +115,8 @@ def _downgrade_to_v4(path: Path) -> None:
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN vote INTEGER CHECK (vote IN (-1, 0, 1))")
|
||||
conn.execute("ALTER TABLE roles ADD COLUMN voted_at TEXT")
|
||||
conn.execute("ALTER TABLE companies ADD COLUMN affinity INTEGER NOT NULL DEFAULT 0")
|
||||
for column in ("ats", "country", "city"): # v7
|
||||
conn.execute(f"ALTER TABLE companies DROP COLUMN {column}")
|
||||
conn.execute("DROP TABLE agent_companies")
|
||||
conn.execute("DROP TABLE role_votes")
|
||||
conn.execute("PRAGMA user_version = 4")
|
||||
@@ -139,7 +143,7 @@ def test_v6_moves_the_votes_and_affinities_to_the_default_agent(tmp_path: Path)
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == [5, 6]
|
||||
assert migrated.migrations_applied == list(range(5, LATEST + 1))
|
||||
assert [a.name for a in migrated.agents()] == ["First", "Second"] # none added
|
||||
assert migrated.votes(first.id) == {posting("liked").uid: 1, posting("passed").uid: -1}
|
||||
assert migrated.votes(second.id) == {}
|
||||
@@ -181,7 +185,7 @@ def test_v6_makes_no_agent_when_there_is_nothing_to_move(tmp_path: Path) -> None
|
||||
store.add_company("Acme", key="acme")
|
||||
_downgrade_to_v4(path)
|
||||
|
||||
assert Store(path).migrations_applied == [5, 6]
|
||||
assert Store(path).migrations_applied == list(range(5, LATEST + 1))
|
||||
assert Store(path).agents() == []
|
||||
|
||||
|
||||
@@ -191,7 +195,7 @@ def test_a_v4_file_migrates_read_state_and_tags(tmp_path: Path) -> None:
|
||||
Store(old)
|
||||
_downgrade_to_v4(old)
|
||||
|
||||
assert Store(old).migrations_applied == [5, 6]
|
||||
assert Store(old).migrations_applied == list(range(5, LATEST + 1))
|
||||
assert sorted(_columns(old, "roles")) == sorted(_columns(fresh, "roles"))
|
||||
with sqlite3.connect(old) as conn:
|
||||
tables = {r[0] for r in conn.execute(
|
||||
|
||||
+10
-3
@@ -384,17 +384,24 @@ def test_the_roles_page_offers_the_judged_filter_and_hidden_companies(
|
||||
def test_companies_lists_board_and_open_role_count(
|
||||
client: TestClient, store: Store) -> None:
|
||||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
||||
store.add_company("Acme", key="acme")
|
||||
store.add_company("Acme", key="acme", ats="greenhouse", country="DK")
|
||||
store.upsert_role(posting("1", company="Acme"), sub)
|
||||
store.add_company("Other", key="other") # known only from a search: no board
|
||||
store.add_company("Novo", key="novo", ats="successfactors", # followed, not readable
|
||||
careers_url="https://novo.example/careers")
|
||||
|
||||
res = client.get("/api/companies")
|
||||
assert res.status_code == 200
|
||||
[row] = [c for c in res.json() if c["key"] == "acme"]
|
||||
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
|
||||
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1}
|
||||
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
|
||||
"ats": "greenhouse", "careers_url": "", "country": "DK"}
|
||||
[other] = [c for c in res.json() if c["key"] == "other"]
|
||||
assert (other["board"], other["subscription_id"], other["on"]) == ("", None, None)
|
||||
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
|
||||
"", None, None, "")
|
||||
[novo] = [c for c in res.json() if c["key"] == "novo"]
|
||||
assert (novo["board"], novo["subscription_id"], novo["ats"], novo["careers_url"]) == (
|
||||
"", None, "successfactors", "https://novo.example/careers")
|
||||
|
||||
|
||||
def test_set_affinity_validates_updates_and_enqueues_a_score(
|
||||
|
||||
Reference in new issue
Block a user