Merge branch 'worktree-agent-a7e6bf7e67aa710c3'
This commit is contained in:
commit
0ea6c10d80
18 files changed
+1038
-178
No files matched your search
+30
-14
@@ -6,16 +6,20 @@ website, not from a board. This module gets from the one to the other: it
|
||||
fetches the site's careers page, or the pages the home page links to as
|
||||
careers, and looks for a link into one of the hosts in `HOSTS`. A readable
|
||||
host (one the `ats` plugin parses) becomes a `board` target the pull can
|
||||
read; any other host is recorded as `ats`, so the company is kept as one the
|
||||
app cannot read yet rather than dropped.
|
||||
read; any other host is recorded as `ats`, with its slug where the URL
|
||||
carries one, so the company is kept as one the app cannot read yet rather
|
||||
than dropped.
|
||||
|
||||
When no page names a host, the three hosts with a cheap per-slug endpoint
|
||||
(Greenhouse, Lever, Ashby) are probed with the company's own slug, because a
|
||||
site that draws its jobs from an iframe or a script leaves no link to find.
|
||||
Only when no page names a host at all are the hosts with a cheap per-slug
|
||||
endpoint (`_PROBES`) probed with the company's own slug, because a site that
|
||||
draws its jobs from an iframe or a script leaves no link to find. A page that
|
||||
names an unreadable host is the answer: a probe on a common name can hit
|
||||
another tenant's board.
|
||||
|
||||
found = discover("Pleo", "pleo.io")
|
||||
found.board # "greenhouse:pleo", or "" when nothing readable
|
||||
found.ats # "greenhouse", "workday", ... or "" when unknown
|
||||
found.ats_slug # the name that host knows the company by, or ""
|
||||
found.careers_url # the page that named the host, or the careers page
|
||||
found.evidence # the URL that named it
|
||||
|
||||
@@ -30,7 +34,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
@@ -112,9 +116,10 @@ _ANCHOR = re.compile(r"<a\b[^>]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)
|
||||
_TAGS = re.compile(r"<[^>]+>")
|
||||
|
||||
#: Words a pattern's slug group can catch that name no board: the hosts'
|
||||
#: own path segments, and the analytics subdomain Recruitee sites embed.
|
||||
#: own path segments, the analytics subdomain Recruitee sites embed, and
|
||||
#: the CDN Teamtailor sites load from (cdn.teamtailor.com).
|
||||
_NOT_SLUGS = frozenset({"embed", "boards", "v1", "api", "www", "jobs", "careers", "apply",
|
||||
"careers-analytics"})
|
||||
"careers-analytics", "cdn"})
|
||||
|
||||
Found = tuple[str, str, str] # (host, slug, the URL that named it)
|
||||
|
||||
@@ -125,6 +130,7 @@ class Discovery:
|
||||
|
||||
board: str = ""
|
||||
ats: str = ""
|
||||
ats_slug: str = ""
|
||||
careers_url: str = ""
|
||||
evidence: str = ""
|
||||
|
||||
@@ -301,14 +307,22 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
|
||||
careers_page = ""
|
||||
|
||||
def first_readable(found: list[Found], page: str) -> Discovery | None:
|
||||
"""The first readable board on a page. Otherwise the first host named
|
||||
is remembered, with the first slug any page gives that host: a page
|
||||
may name teamtailor by its CDN before it links acme.teamtailor.com."""
|
||||
nonlocal unreadable
|
||||
for host, slug, url in found:
|
||||
if host in READABLE and slug:
|
||||
return Discovery(board=f"{host}:{slug}", ats=host, careers_url=page,
|
||||
evidence=url)
|
||||
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
|
||||
careers_url=page, evidence=url)
|
||||
if unreadable is None and found:
|
||||
host, slug, url = found[0]
|
||||
unreadable = Discovery(ats=host, careers_url=page, evidence=url)
|
||||
unreadable = Discovery(ats=host, ats_slug=slug, careers_url=page, evidence=url)
|
||||
if unreadable is not None and not unreadable.ats_slug:
|
||||
for host, slug, url in found:
|
||||
if host == unreadable.ats and slug:
|
||||
unreadable = replace(unreadable, ats_slug=slug, evidence=url)
|
||||
break
|
||||
return None
|
||||
|
||||
if home and (hit := first_readable(boards_in(home), home_url)):
|
||||
@@ -333,11 +347,13 @@ def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
|
||||
if hit := first_readable(boards_in(text), final):
|
||||
return hit
|
||||
queue[:0] = [link for link in careers_links(text, final)[:2] if link not in seen]
|
||||
# A page named a host the app cannot read: that is the answer, and a
|
||||
# probe could only add another tenant's board of the same name.
|
||||
if unreadable is not None:
|
||||
return unreadable
|
||||
for slug in slug_guesses(name, site):
|
||||
for host in _PROBES:
|
||||
if probe_slug(client, host, slug):
|
||||
return Discovery(board=f"{host}:{slug}", ats=host,
|
||||
return Discovery(board=f"{host}:{slug}", ats=host, ats_slug=slug,
|
||||
careers_url=careers_page, evidence=_PROBES[host].format(slug=slug))
|
||||
if unreadable is not None:
|
||||
return unreadable
|
||||
return Discovery(careers_url=careers_page)
|
||||
+153
-33
@@ -1,6 +1,8 @@
|
||||
"""What a pull reads, changed by hand: add a board or a search, switch one on
|
||||
or off. `app companies`, `app searches` and the settings pages all call these,
|
||||
so the command line and the pages cannot disagree about what "off" means.
|
||||
or off, attach, replace or detach a company's board, import the seed.
|
||||
`app companies`, `app searches` and the settings pages all call these, so the
|
||||
command line and the pages cannot disagree about what "off" means. Every way
|
||||
a company gets a board is `set_board`.
|
||||
|
||||
A board or search switched off has its open roles closed at once. Nothing
|
||||
else would ever close them, since only a pull that reads a target closes its
|
||||
@@ -9,7 +11,9 @@ roles; switched back on, the next pull reopens every role it still lists.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app.config import Config
|
||||
@@ -32,55 +36,171 @@ def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int
|
||||
return sub, closed
|
||||
|
||||
|
||||
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
||||
"""Follow one company's ATS board: fetch it first, and refuse if the host
|
||||
is unknown, the company exists, the board does not answer or lists
|
||||
nothing. Returns the subscription and how many roles the board lists."""
|
||||
def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
|
||||
"""Read company `key` from `board` (`host:slug`): attach it to a company
|
||||
that has none, replace the one it has, or create the company from `name`
|
||||
when there is no row. The board is fetched first and refused if its host
|
||||
is unknown or it does not answer; a board listing zero roles is kept, since
|
||||
a company between openings is still one to follow. `check=False` skips the
|
||||
fetch, for the seed, whose boards were read once when it was written.
|
||||
|
||||
The row's `ats` and `ats_slug` become the board's host and slug, and its
|
||||
`careers_url` the board's page where it was empty or the old board's.
|
||||
`fields` (domain, country, city, ...) fill only columns that are empty, so
|
||||
nothing a person or the seed wrote is overwritten. Row and subscription
|
||||
change in one transaction (`Store.set_company_board`), and replacing a
|
||||
board closes the old one's open roles. Returns the subscription, how many
|
||||
roles the board lists (None unchecked) and how many roles closed."""
|
||||
from plugins.ats import careers_url, fetch_board, is_known_target
|
||||
|
||||
board, name = board.strip(), name.strip()
|
||||
if not name:
|
||||
raise ValueError("a company needs a name")
|
||||
key = key.strip() or company_key(name)
|
||||
if not is_known_target(board):
|
||||
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
|
||||
"e.g. greenhouse:acme or ashby:acme")
|
||||
key = company_key(name)
|
||||
if store.company(key) is not None:
|
||||
raise ValueError(f"{name} is already followed")
|
||||
try:
|
||||
rows = fetch_board(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
if not rows:
|
||||
raise ValueError(f"{board} answered but lists no roles; nothing added")
|
||||
host = board.partition(":")[0]
|
||||
store.add_company(name, key=key, careers_url=careers_url(board))
|
||||
sub = store.add_subscription("ats", board, label=name, company=name, board=host)
|
||||
return sub, len(rows)
|
||||
company = store.company(key)
|
||||
if company is None:
|
||||
if not name:
|
||||
raise ValueError(f"no company {key!r}; a new company needs a name")
|
||||
if company_key(name) != key:
|
||||
raise ValueError(f"{name!r} is not the company {key!r}")
|
||||
rows: list[Any] | None = None
|
||||
if check:
|
||||
try:
|
||||
rows = fetch_board(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
host, _, slug = board.partition(":")
|
||||
writes: dict[str, str] = {k: v for k, v in fields.items() if v}
|
||||
if company is not None:
|
||||
writes = {k: v for k, v in writes.items() if not getattr(company, k)}
|
||||
old = store.company_subscription(key)
|
||||
if not company.careers_url or (old is not None
|
||||
and company.careers_url == careers_url(old.target)):
|
||||
writes["careers_url"] = careers_url(board)
|
||||
else:
|
||||
writes.setdefault("careers_url", careers_url(board))
|
||||
writes.update(ats=host, ats_slug=slug)
|
||||
_company, sub, closed = store.set_company_board(key, board, host=host, name=name,
|
||||
fields=writes)
|
||||
assert sub is not None
|
||||
return sub, None if rows is None else len(rows), closed
|
||||
|
||||
|
||||
def detach_board(store: Store, key: str) -> int:
|
||||
"""Stop reading company `key`'s board: its open roles close and the row
|
||||
stays, with its `ats` and `ats_slug`, as one the app does not read.
|
||||
Returns how many roles closed. Raises KeyError for an unknown company and
|
||||
ValueError for one with no board."""
|
||||
if store.company(key) is None:
|
||||
raise KeyError(key)
|
||||
sub = store.company_subscription(key)
|
||||
if sub is None or not sub.show:
|
||||
raise ValueError(f"{key} has no board to detach")
|
||||
_company, _sub, closed = store.set_company_board(key, "")
|
||||
return closed
|
||||
|
||||
|
||||
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
||||
"""Follow one company by its ATS board: `set_board` keyed by `name`, so
|
||||
a company that exists gets the board attached or replaced. Returns the
|
||||
subscription and how many roles the board lists."""
|
||||
name = name.strip()
|
||||
if not name:
|
||||
raise ValueError("a company needs a name")
|
||||
sub, roles, _closed = set_board(store, company_key(name), board, name=name)
|
||||
return sub, roles or 0
|
||||
|
||||
|
||||
def add_discovered(store: Store, name: str, domain: str,
|
||||
found: Discovery) -> tuple[str, int]:
|
||||
"""Follow a company from what `app.jobs.discover` found: through
|
||||
`add_board` when the board is readable, else as a row with its `ats` host
|
||||
and careers page, so the company is kept as one the app cannot read yet.
|
||||
Returns the company key and how many roles its board lists (0 with no
|
||||
board). Refuses a company that exists, as `add_board` does."""
|
||||
"""Follow a company from what `app.jobs.discover` found. A readable
|
||||
board goes through `set_board`, which attaches it, replaces the one the
|
||||
company has, or creates the company. Otherwise the row records `ats`,
|
||||
`ats_slug` and the careers page, as one the app cannot read yet; a
|
||||
company that already has a board keeps it and this raises, since
|
||||
discovery finding nothing readable is weaker evidence than a board that
|
||||
answers. Returns the company key and how many roles its board lists (0
|
||||
with no board)."""
|
||||
name = name.strip()
|
||||
if not name:
|
||||
raise ValueError("a company needs a name")
|
||||
key = company_key(name)
|
||||
if found.board:
|
||||
_sub, roles = add_board(store, found.board, name)
|
||||
store.update_company(key, domain=domain, ats=found.ats)
|
||||
return key, roles
|
||||
if store.company(key) is not None:
|
||||
raise ValueError(f"{name} is already followed")
|
||||
store.add_company(name, key=key, domain=domain, ats=found.ats,
|
||||
careers_url=found.careers_url)
|
||||
_sub, roles, _closed = set_board(store, key, found.board, name=name, domain=domain)
|
||||
return key, roles or 0
|
||||
company = store.company(key)
|
||||
if company is None:
|
||||
store.add_company(name, key=key, domain=domain, ats=found.ats,
|
||||
ats_slug=found.ats_slug, careers_url=found.careers_url)
|
||||
return key, 0
|
||||
sub = store.company_subscription(key)
|
||||
if sub is not None and sub.show:
|
||||
raise ValueError(f"{name} is read from {sub.target} and discovery found no "
|
||||
"readable board; nothing changed")
|
||||
store.update_company(key, ats=found.ats, ats_slug=found.ats_slug,
|
||||
careers_url=found.careers_url or company.careers_url,
|
||||
**({"domain": domain} if not company.domain else {}))
|
||||
return key, 0
|
||||
|
||||
|
||||
def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
|
||||
"""Import `seed/companies.json`: one `companies` row per entry, and its
|
||||
`board` through `set_board` where the entry names one. An entry without
|
||||
a board is the point of the list as much as one with: a company whose
|
||||
careers site the app cannot read yet keeps its `ats` host, `ats_slug`
|
||||
and `careers_url`, and the Companies section says so.
|
||||
|
||||
Re-running adds what is missing: a new entry becomes a row, and an
|
||||
existing row that never had a board gets the one the seed now names, so a
|
||||
seed update that gains boards reaches a database made before it. A row's
|
||||
board, once it has one (detached counts), and its affinity, notes and
|
||||
status are never touched. The seed's boards are not fetched again: they
|
||||
were read once when the seed was written, and `app companies check`
|
||||
finds the dead ones. Returns counts, and `problems`, one line per entry
|
||||
that could not land whole."""
|
||||
from plugins.ats import is_known_target
|
||||
|
||||
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
entries = data.get("companies", [])
|
||||
added = boards = skipped = 0
|
||||
problems: list[str] = []
|
||||
for row in entries:
|
||||
name = row["name"].strip()
|
||||
key = row.get("key") or company_key(name)
|
||||
board = (row.get("board") or "").strip()
|
||||
if board and not is_known_target(board):
|
||||
problems.append(f"{key}: {board!r} is not a known board; kept without it")
|
||||
board = ""
|
||||
facts = {f: str(row.get(f) or "") for f in ("domain", "country", "city",
|
||||
"careers_url", "ats", "ats_slug")}
|
||||
exists = store.company(key) is not None
|
||||
if exists and (not board or store.company_subscription(key) is not None):
|
||||
skipped += 1
|
||||
continue
|
||||
if board:
|
||||
try:
|
||||
set_board(store, key, board, name=name, check=False,
|
||||
**{f: v for f, v in facts.items() if f not in ("ats", "ats_slug")})
|
||||
boards += 1
|
||||
added += not exists
|
||||
continue
|
||||
except ValueError as exc:
|
||||
problems.append(f"{key}: {exc}")
|
||||
if exists:
|
||||
skipped += 1
|
||||
continue
|
||||
# No board, or one that clashed: the company is still a row, and a
|
||||
# clashing board's host and slug are what discovery found.
|
||||
if board:
|
||||
facts["ats"], _, facts["ats_slug"] = board.partition(":")
|
||||
store.add_company(name, key=key, **facts)
|
||||
added += 1
|
||||
return {"added": added, "boards": boards, "skipped": skipped, "total": len(entries),
|
||||
"problems": problems}
|
||||
|
||||
|
||||
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
||||
where: str = "", pages: int = 0) -> dict[str, Any]:
|
||||
"""The `params` a search subscription stores, with only the keys its
|
||||
|
||||
Reference in new issue
Block a user