diff --git a/README.md b/README.md
index 18d93d1..308ddb2 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeli
```bash
uv sync --all-extras
-uv run app companies seed # 186 companies from seed/companies.json, one board each
+uv run app companies seed # 362 European employers from seed/companies.json, 175 with a readable board
uv run app job pull # read every board and search, embed and score them, extract the best, rescore, summarise
uv run app serve # http://127.0.0.1:8790
```
@@ -24,8 +24,11 @@ uv run app searches add jobindex "Copenhagen golang" --q "golang go-udvikler" --
uv run app searches add adzuna "Berlin backend" --q backend --country de --where Berlin
uv run app searches list
uv run app companies prune --in DK,SE,NO,EU # lists; add --apply to unsubscribe
+uv run app companies discover "Pleo" pleo.io --add # find its board on its own site, then follow it
```
+The seed is a hand-picked list of European tech employers, not a list of boards: a company whose careers site runs on something the app cannot read yet (Workday, SuccessFactors, Teamtailor, its own pages) is still a row, shown under Companies as "not readable yet" with a link to its careers page, so the gap is visible and one parser at a time closes it. `companies discover` reads a company's site for a link into one of the boards it knows, probes the hosts with a cheap per-slug endpoint when the pages link nothing, and with `--add` follows what it found the same way the seed does. A probe hit is a guess by name: check the roles it lists before trusting it for a common name.
+
A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. Switching a board or search off closes its open roles; the next pull after switching it back on reopens the ones it still lists. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys.
## The tabs
diff --git a/app/cli.py b/app/cli.py
index e51ee1e..5541c86 100644
--- a/app/cli.py
+++ b/app/cli.py
@@ -3,7 +3,7 @@
app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle]
- app companies seed|add|check|prune
+ app companies seed|add|discover|check|prune
app searches add|list|remove
app db init
@@ -67,11 +67,17 @@ def build_parser() -> argparse.ArgumentParser:
companies = sub.add_parser("companies", help="the target company list")
companies_sub = companies.add_subparsers(dest="action", required=True)
seed = companies_sub.add_parser(
- "seed", help="import seed/companies.json, subscribing to each board")
+ "seed", help="import seed/companies.json, subscribing to each readable board")
seed.add_argument("--path", default="seed/companies.json")
add = companies_sub.add_parser("add", help="add one company from its board")
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
add.add_argument("name")
+ discover = companies_sub.add_parser(
+ "discover", help="find a company's board from its website; --add follows it")
+ discover.add_argument("name")
+ discover.add_argument("domain", help="its website, e.g. pleo.io")
+ discover.add_argument("--add", action="store_true",
+ help="add the company, with its board when one is readable")
companies_sub.add_parser("check", help="report boards that do not answer")
prune = companies_sub.add_parser(
"prune", help="unsubscribe every board with no open role in the region")
@@ -234,42 +240,76 @@ def run_companies(args: argparse.Namespace) -> int:
return 0
if args.action == "add":
return add_company(store, args.board, args.name)
+ if args.action == "discover":
+ return discover_company(store, args.name, args.domain, add=args.add)
if args.action == "prune":
return prune_companies(store, args.region, apply=args.apply)
return check_companies(store)
def seed_companies(store: Store, path: str) -> dict[str, int]:
- """Import `seed/companies.json`: one `companies` row and one `ats`
- subscription per entry, skipping a key that already exists. Not
- idempotent on edits to an existing row — re-running it only ever adds
- what is missing."""
- from plugins.ats import is_known_target
+ """Import `seed/companies.json`: one `companies` row per entry, and one
+ `ats` subscription for each entry with a readable `board`. An entry
+ without one is the point of the list as much as one with: a company worth
+ following whose careers site the app cannot read yet keeps its `ats`
+ host and `careers_url`, and the Companies section says so. A key that
+ already exists is skipped, so re-running only ever adds what is missing;
+ it never edits a row."""
+ from app.models import company_key
+ from plugins.ats import careers_url, is_known_target
data = json.loads(Path(path).read_text(encoding="utf-8"))
- added = skipped = 0
+ added = readable = skipped = 0
for row in data.get("companies", []):
- key, name, board = row["key"], row["name"], row["board"]
+ name = row["name"]
+ key = row.get("key") or company_key(name)
if store.company(key) is not None:
skipped += 1
continue
- if not is_known_target(board):
- print(f"companies seed: {key}: {board!r} is not a known board; skipped",
+ board = row.get("board") or ""
+ if board and not is_known_target(board):
+ print(f"companies seed: {key}: {board!r} is not a known board; kept without it",
file=sys.stderr)
- skipped += 1
+ board = ""
+ host = board.partition(":")[0]
+ store.add_company(
+ name, key=key, domain=row.get("domain", ""),
+ careers_url=row.get("careers_url") or (careers_url(board) if board else ""),
+ ats=row.get("ats") or host, country=row.get("country", ""),
+ city=row.get("city", ""))
+ added += 1
+ if not board:
continue
- # A seed's `affinity` is not read: affinity is per agent now, and a
- # seed has no agent to set it for.
- store.add_company(name, key=key, careers_url=row.get("careers_url", ""))
- host, _, _slug = board.partition(":")
try:
store.add_subscription("ats", board, label=name, company=name, board=host)
+ readable += 1
except ValueError as exc:
# The company row landed; the subscription is the only thing
# missing, and worth saying rather than losing silently.
print(f"companies seed: {key}: {exc}", file=sys.stderr)
- added += 1
- return {"added": added, "skipped": skipped, "total": len(data.get("companies", []))}
+ return {"added": added, "readable": readable, "skipped": skipped,
+ "total": len(data.get("companies", []))}
+
+
+def discover_company(store: Store, name: str, domain: str, *, add: bool) -> int:
+ """`app companies discover "Pleo" pleo.io [--add]`: what the company's
+ site says its board is, as JSON. With `--add`, the company is added the
+ way the seed adds it: with its board when one is readable, as a row the
+ app cannot read yet otherwise."""
+ from app import jobs
+
+ found = jobs.discover.discover(name, domain)
+ out: dict[str, Any] = {"name": name, "domain": domain, "board": found.board,
+ "ats": found.ats, "careers_url": found.careers_url,
+ "evidence": found.evidence}
+ if add:
+ try:
+ out["key"], out["roles"] = jobs.sources.add_discovered(store, name, domain, found)
+ except ValueError as exc:
+ print(f"companies discover: {exc}", file=sys.stderr)
+ return 1
+ print(json.dumps(out))
+ return 0
def add_company(store: Store, board: str, name: str) -> int:
diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py
index d61b14b..8182e1f 100644
--- a/app/jobs/__init__.py
+++ b/app/jobs/__init__.py
@@ -10,13 +10,14 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
offers, the `pull` job a schedule fires, and that schedule
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off, pruned by region
+ discover.py which ATS a company's careers site runs on, from its pages
"""
from __future__ import annotations
from typing import Any
-from app.jobs import cascade, embed, extract, pull, score, sources, summarise
+from app.jobs import cascade, discover, embed, extract, pull, score, sources, summarise
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -35,5 +36,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
return config, Store(config.database)
-__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score",
- "search_target", "seed_agents", "seed_searches", "sources", "summarise"]
+__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
+ "score", "search_target", "seed_agents", "seed_searches", "sources", "summarise"]
diff --git a/app/jobs/discover.py b/app/jobs/discover.py
new file mode 100644
index 0000000..b934755
--- /dev/null
+++ b/app/jobs/discover.py
@@ -0,0 +1,343 @@
+"""Board discovery: which applicant tracking system a company's careers site
+runs on, read off its own pages.
+
+The seed list and the LinkedIn flow both start from a company name and a
+website, not from a board. This module gets from the one to the other: it
+fetches the site's careers page, or the pages the home page links to as
+careers, and looks for a link into one of the hosts in `HOSTS`. A readable
+host (one the `ats` plugin parses) becomes a `board` target the pull can
+read; any other host is recorded as `ats`, so the company is kept as one the
+app cannot read yet rather than dropped.
+
+When no page names a host, the three hosts with a cheap per-slug endpoint
+(Greenhouse, Lever, Ashby) are probed with the company's own slug, because a
+site that draws its jobs from an iframe or a script leaves no link to find.
+
+ found = discover("Pleo", "pleo.io")
+ found.board # "greenhouse:pleo", or "" when nothing readable
+ found.ats # "greenhouse", "workday", ... or "" when unknown
+ found.careers_url # the page that named the host, or the careers page
+ found.evidence # the URL that named it
+
+Only https is fetched, redirects are followed at most five hops, a body is
+cut at `MAX_BYTES`, and every failure is one page skipped. A company's site
+is content the app was pointed at by its owner, not an ATS API, so nothing
+here trusts what it finds beyond a host name matching a fixed pattern.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from urllib.parse import urljoin
+
+import httpx
+
+from desk.sources import USER_AGENT
+
+log = logging.getLogger(__name__)
+
+TIMEOUT = 10.0
+MAX_BYTES = 2 << 20
+#: How many pages of one site are read before giving up on links.
+MAX_PAGES = 6
+
+#: The hosts the `ats` plugin parses; a match on one of these is a `board`.
+READABLE = ("greenhouse", "lever", "ashby", "workable", "smartrecruiters",
+ "recruitee", "personio")
+
+#: host -> pattern over a URL. Group 1, where there is one, is the slug the
+#: host names the company by; a host with no group is recognised but its
+#: board has no slug the app could use. Order matters only for readability:
+#: the readable hosts come first so the first match on a page is the one the
+#: app can read.
+HOSTS: tuple[tuple[str, re.Pattern[str]], ...] = tuple(
+ (host, re.compile(pattern, re.IGNORECASE)) for host, pattern in (
+ ("greenhouse", r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)"),
+ ("greenhouse", r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)"),
+ ("lever", r"jobs\.(?:eu\.)?lever\.co/([A-Za-z0-9_-]+)"),
+ ("lever", r"api\.(?:eu\.)?lever\.co/v0/postings/([A-Za-z0-9_-]+)"),
+ ("ashby", r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)"),
+ ("ashby", r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)"),
+ ("workable", r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)"),
+ ("workable", r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com"),
+ ("smartrecruiters", r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)"),
+ ("recruitee", r"https?://([a-z0-9-]+)\.recruitee\.com"),
+ ("personio", r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)"),
+ # Not readable. The slug, where the URL has one, is kept for the day
+ # a parser exists.
+ ("workday", r"https?://([a-z0-9-]+)\.wd\d+\.myworkdayjobs\.com"),
+ ("successfactors", r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"),
+ ("teamtailor", r"https?://([a-z0-9-]+)\.teamtailor\.com"),
+ ("teamtailor", r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets"),
+ ("icims", r"https?://([a-z0-9-]+)\.icims\.com"),
+ ("taleo", r"\.taleo\.net"),
+ ("jobvite", r"jobs\.jobvite\.com/([a-z0-9-]+)"),
+ ("bamboohr", r"https?://([a-z0-9-]+)\.bamboohr\.com"),
+ ("join", r"join\.com/companies/([a-z0-9-]+)"),
+ ("softgarden", r"https?://([a-z0-9-]+)\.softgarden\.io"),
+ ("pinpoint", r"https?://([a-z0-9-]+)\.pinpointhq\.com"),
+ ("rippling", r"ats\.rippling\.com/([a-z0-9-]+)"),
+ ("gem", r"jobs\.gem\.com/([a-z0-9-]+)"),
+ ("breezy", r"https?://([a-z0-9-]+)\.breezy\.hr"),
+ ("homerun", r"https?://([a-z0-9-]+)\.homerun\.co"),
+ ("jazzhr", r"https?://([a-z0-9-]+)\.applytojob\.com"),
+ ("welcometothejungle", r"welcometothejungle\.com/[a-z]{2}/companies/([a-z0-9-]+)"),
+ ("eightfold", r"\.eightfold\.ai"),
+ ("phenom", r"phenompeople\.com|phenom\.com/careers"),
+ ("avature", r"\.avature\.net"),
+ ("oracle", r"oraclecloud\.com/hcmUI"),
+ ("comeet", r"comeet\.com/jobs/([a-z0-9-]+)"),
+ ("hibob", r"https?://([a-z0-9-]+)\.hibob\.com"),
+ ))
+
+#: Paths a careers page usually lives at, tried after the home page's own
+#: links so the site's word for it wins over a guess.
+CAREERS_PATHS = ("/careers", "/careers/", "/jobs", "/en/careers", "/company/careers",
+ "/about/careers", "/join-us", "/careers/jobs", "/en/jobs")
+#: Subdomains careers pages move to.
+CAREERS_HOSTS = ("careers.", "jobs.", "career.")
+
+#: A link text or href that points at a careers page.
+_CAREERS_LINK = re.compile(
+ r"career|jobs|join[-_ ]?us|work[-_ ]?with[-_ ]?us|open[-_ ]?positions|vacanc|"
+ r"we[-_ ]?are[-_ ]?hiring|ledige[-_ ]?stillinger|stellen|karriere|lediga[-_ ]?jobb",
+ re.IGNORECASE)
+_HREF = re.compile(r"""(?:href|src|action|content)\s*=\s*["']([^"'#][^"']*)["']""",
+ re.IGNORECASE)
+_ANCHOR = re.compile(r"]*href\s*=\s*[\"']([^\"'#][^\"']*)[\"'][^>]*>(.*?)",
+ re.IGNORECASE | re.DOTALL)
+_TAGS = re.compile(r"<[^>]+>")
+
+#: Words a pattern's slug group can catch that name no board: the hosts'
+#: own path segments, and the analytics subdomain Recruitee sites embed.
+_NOT_SLUGS = frozenset({"embed", "boards", "v1", "api", "www", "jobs", "careers", "apply",
+ "careers-analytics"})
+
+Found = tuple[str, str, str] # (host, slug, the URL that named it)
+
+
+@dataclass(frozen=True)
+class Discovery:
+ """What discovery learned about one company; every field may be empty."""
+
+ board: str = ""
+ ats: str = ""
+ careers_url: str = ""
+ evidence: str = ""
+
+ @property
+ def readable(self) -> bool:
+ return bool(self.board)
+
+
+# ------------------------------------------------------------------ parsing
+
+def boards_in(text: str) -> list[Found]:
+ """Every ATS host named in `text` (a page, a script, a mail), in order of
+ first appearance, once each. `slug` is "" for a host whose URL carries
+ none."""
+ seen: set[tuple[str, str]] = set()
+ out: list[Found] = []
+ hits: list[tuple[int, str, str, str]] = []
+ for host, pattern in HOSTS:
+ for m in pattern.finditer(text):
+ slug = (m.group(1) if m.groups() and m.group(1) else "").lower()
+ if slug in _NOT_SLUGS:
+ continue
+ hits.append((m.start(), host, slug, m.group(0)))
+ for _pos, host, slug, url in sorted(hits):
+ if (host, slug) in seen:
+ continue
+ seen.add((host, slug))
+ out.append((host, slug, url))
+ return out
+
+
+def careers_links(html: str, base: str) -> list[str]:
+ """The links on a page that read as its careers page, absolute, in page
+ order, once each. Nothing off-site is followed: a careers page on another
+ domain is what `boards_in` finds by its host."""
+ out: list[str] = []
+ site = httpx.URL(base).host or ""
+ for href, label in _ANCHOR.findall(html):
+ text = _TAGS.sub(" ", label)
+ if not (_CAREERS_LINK.search(href) or _CAREERS_LINK.search(text)):
+ continue
+ url = urljoin(base, href.strip())
+ parsed = httpx.URL(url)
+ if parsed.scheme not in ("http", "https"):
+ continue
+ host = parsed.host or ""
+ if host != site and not host.endswith("." + _apex(site)):
+ continue
+ url = str(parsed.copy_with(scheme="https", fragment=None))
+ if url not in out:
+ out.append(url)
+ return out
+
+
+def _apex(host: str) -> str:
+ parts = host.split(".")
+ return ".".join(parts[-2:]) if len(parts) > 2 else host
+
+
+def slug_guesses(name: str, domain: str) -> list[str]:
+ """The slugs a host most likely knows the company by: its key, the key
+ without hyphens, and the domain's own label."""
+ from app.models import company_key
+
+ key = company_key(name)
+ label = domain.lower().split("/")[0]
+ for prefix in ("www.",):
+ label = label.removeprefix(prefix)
+ label = label.split(".")[0]
+ out = []
+ for guess in (key, key.replace("-", ""), label, label.replace("-", "")):
+ if guess and guess not in out:
+ out.append(guess)
+ return out
+
+
+# ----------------------------------------------------------------- fetching
+
+def fetch_page(client: httpx.Client, url: str) -> tuple[str, str]:
+ """`(final URL, text)` for one page, or `("", "")` when it did not answer
+ with a 2xx text body within the limits. Never raises."""
+ try:
+ response = client.get(
+ url, timeout=TIMEOUT, follow_redirects=True,
+ headers={"User-Agent": USER_AGENT,
+ "Accept": "text/html,application/xhtml+xml,*/*;q=0.5"})
+ if not response.is_success:
+ return "", ""
+ body = response.content[:MAX_BYTES]
+ return str(response.url), body.decode(response.encoding or "utf-8", "replace")
+ except Exception as exc: # one page skipped, whatever went wrong
+ log.debug("discover: %s: %s", url, exc)
+ return "", ""
+
+
+#: host -> a cheap URL that answers 2xx only when the slug is a board there.
+#: Workable is left out: its API sits behind a rate limit that a run of
+#: guesses trips, and a Workable site links its board plainly anyway.
+_PROBES = {
+ "greenhouse": "https://boards-api.greenhouse.io/v1/boards/{slug}",
+ "lever": "https://api.lever.co/v0/postings/{slug}?limit=1&mode=json",
+ "ashby": "https://api.ashbyhq.com/posting-api/job-board/{slug}",
+ "smartrecruiters": "https://api.smartrecruiters.com/v1/companies/{slug}/postings?limit=1",
+ "recruitee": "https://{slug}.recruitee.com/api/offers/",
+ "personio": "https://{slug}.jobs.personio.de/xml",
+}
+
+
+def probe(client: httpx.Client, host: str, slug: str) -> bool:
+ """Whether `host` has a board called `slug`, by its cheapest endpoint.
+ Ashby and SmartRecruiters answer an empty listing for an unknown slug,
+ so there a listing without jobs counts as no board."""
+ url = _PROBES[host].format(slug=slug)
+ try:
+ response = client.get(url, timeout=TIMEOUT, follow_redirects=False,
+ headers={"User-Agent": USER_AGENT})
+ except Exception:
+ return False
+ if not response.is_success:
+ return False
+ body = response.content
+ if host == "ashby":
+ return b'"jobs":[' in body and b'"jobs":[]' not in body
+ if host == "smartrecruiters":
+ return b'"totalFound":' in body and b'"totalFound":0' not in body
+ if host == "recruitee":
+ return b'"offers":[' in body and b'"offers":[]' not in body
+ if host == "personio":
+ return b"" in body
+ return True
+
+
+Fetch = Callable[[httpx.Client, str], tuple[str, str]]
+Probe = Callable[[httpx.Client, str, str], bool]
+
+
+def _site(domain: str) -> str:
+ domain = domain.strip().lower()
+ domain = re.sub(r"^https?://", "", domain).split("/")[0]
+ return domain.removeprefix("www.")
+
+
+def _pages(domain: str, home_links: Iterable[str]) -> list[str]:
+ site = _site(domain)
+ urls = list(home_links)
+ urls += [f"https://{site}{path}" for path in CAREERS_PATHS]
+ urls += [f"https://{sub}{site}/" for sub in CAREERS_HOSTS]
+ out: list[str] = []
+ for url in urls:
+ if url not in out:
+ out.append(url)
+ return out
+
+
+def discover(name: str, domain: str, *, fetch: Fetch = fetch_page,
+ probe_slug: Probe = probe, client: httpx.Client | None = None) -> Discovery:
+ """Where `name`'s careers site draws its jobs from; see the module
+ docstring for the order pages are read in."""
+ site = _site(domain)
+ if not site:
+ return Discovery()
+ own = client or httpx.Client()
+ try:
+ return _discover(name, site, own, fetch, probe_slug)
+ finally:
+ if client is None:
+ own.close()
+
+
+def _discover(name: str, site: str, client: httpx.Client, fetch: Fetch,
+ probe_slug: Probe) -> Discovery:
+ home_url, home = fetch(client, f"https://{site}/")
+ unreadable: Discovery | None = None
+ careers_page = ""
+
+ def first_readable(found: list[Found], page: str) -> Discovery | None:
+ nonlocal unreadable
+ for host, slug, url in found:
+ if host in READABLE and slug:
+ return Discovery(board=f"{host}:{slug}", ats=host, careers_url=page,
+ evidence=url)
+ if unreadable is None and found:
+ host, slug, url = found[0]
+ unreadable = Discovery(ats=host, careers_url=page, evidence=url)
+ return None
+
+ if home and (hit := first_readable(boards_in(home), home_url)):
+ return hit
+ # The home page's own careers links, then the usual paths; a careers page's
+ # own links (a "See open positions" one level down) go to the front of the
+ # queue. `MAX_PAGES` fetches in all, whatever the queue holds.
+ queue = _pages(site, careers_links(home, home_url) if home else [])
+ seen = {home_url, f"https://{site}/"}
+ budget = MAX_PAGES
+ while queue and budget > 0:
+ url = queue.pop(0)
+ if url in seen:
+ continue
+ seen.add(url)
+ budget -= 1
+ final, text = fetch(client, url)
+ if not text:
+ continue
+ seen.add(final)
+ careers_page = careers_page or final
+ if hit := first_readable(boards_in(text), final):
+ return hit
+ queue[:0] = [link for link in careers_links(text, final)[:2] if link not in seen]
+ for slug in slug_guesses(name, site):
+ for host in _PROBES:
+ if probe_slug(client, host, slug):
+ return Discovery(board=f"{host}:{slug}", ats=host,
+ careers_url=careers_page, evidence=_PROBES[host].format(slug=slug))
+ if unreadable is not None:
+ return unreadable
+ return Discovery(careers_url=careers_page)
diff --git a/app/jobs/sources.py b/app/jobs/sources.py
index 470705d..3a6a79b 100644
--- a/app/jobs/sources.py
+++ b/app/jobs/sources.py
@@ -11,13 +11,16 @@ roles; switched back on, the next pull reopens every role it still lists.
from __future__ import annotations
import os
-from typing import Any
+from typing import TYPE_CHECKING, Any
from app.config import Config
from app.models import EUROPE, Subscription, company_key
from app.storage import Store
from app.jobs.pull import search_target
+if TYPE_CHECKING:
+ from app.jobs.discover import Discovery
+
#: The providers `plugins.search` runs, and which of them needs a key.
PROVIDERS = ("jobindex", "adzuna")
@@ -91,6 +94,28 @@ def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
return sub, len(rows)
+def add_discovered(store: Store, name: str, domain: str,
+ found: Discovery) -> tuple[str, int]:
+ """Follow a company from what `app.jobs.discover` found: through
+ `add_board` when the board is readable, else as a row with its `ats` host
+ and careers page, so the company is kept as one the app cannot read yet.
+ Returns the company key and how many roles its board lists (0 with no
+ board). Refuses a company that exists, as `add_board` does."""
+ name = name.strip()
+ if not name:
+ raise ValueError("a company needs a name")
+ key = company_key(name)
+ if found.board:
+ _sub, roles = add_board(store, found.board, name)
+ store.update_company(key, domain=domain, ats=found.ats)
+ return key, roles
+ if store.company(key) is not None:
+ raise ValueError(f"{name} is already followed")
+ store.add_company(name, key=key, domain=domain, ats=found.ats,
+ careers_url=found.careers_url)
+ return key, 0
+
+
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its
diff --git a/app/models.py b/app/models.py
index 93331f1..e741afa 100644
--- a/app/models.py
+++ b/app/models.py
@@ -314,6 +314,12 @@ class Company(Model):
status: str = "researching"
careers_url: str = ""
notes: str = ""
+ #: The host the careers site runs on, readable or not; "" when unknown.
+ #: A company with an `ats` and no board subscription is one the app
+ #: cannot read yet. See `app.jobs.discover`.
+ ats: str = ""
+ country: str = ""
+ city: str = ""
created_at: str
updated_at: str
diff --git a/app/static/companies.js b/app/static/companies.js
index 9606bf4..7f16956 100644
--- a/app/static/companies.js
+++ b/app/static/companies.js
@@ -54,9 +54,25 @@
badge.className = "badge";
badge.textContent = company.board;
sub.appendChild(badge);
+ } else if (company.ats || company.careers_url) {
+ // Followed, but its careers site runs on nothing the app reads yet.
+ const badge = document.createElement("span");
+ badge.className = "badge";
+ badge.textContent = company.ats || "own site";
+ sub.appendChild(badge);
+ sub.append(" · not readable yet");
} else {
sub.append("from job boards");
}
+ if (company.careers_url && !company.board) {
+ const link = document.createElement("a");
+ link.href = company.careers_url;
+ link.target = "_blank";
+ link.rel = "noopener";
+ link.textContent = "careers page";
+ sub.append(" · ");
+ sub.appendChild(link);
+ }
sub.append(` · ${count(company.open_roles, "open role")}`);
return sub;
}
@@ -94,6 +110,7 @@
case "hidden": return company.affinity === -1;
case "on": return company.on === true;
case "off": return company.on === false;
+ case "unreadable": return company.subscription_id == null && Boolean(company.ats || company.careers_url);
default: return true;
}
}
@@ -102,7 +119,8 @@
const q = $("#c-q").value.trim().toLowerCase();
const filter = $("#c-filter").value;
const rows = [...companies.values()]
- .filter((c) => !q || c.name.toLowerCase().includes(q) || c.board.includes(q))
+ .filter((c) => !q || c.name.toLowerCase().includes(q) || c.board.includes(q)
+ || (c.ats || "").includes(q) || (c.country || "").toLowerCase() === q)
.filter((c) => matchesFilter(c, filter))
.sort((a, b) => b.open_roles - a.open_roles || a.name.localeCompare(b.name));
const body = $("#companies-list");
diff --git a/app/storage/schema.py b/app/storage/schema.py
index 224df95..f23a8cf 100644
--- a/app/storage/schema.py
+++ b/app/storage/schema.py
@@ -91,6 +91,10 @@ CREATE TABLE IF NOT EXISTS role_tags (
-- A company as a target, whether or not any board of it is read. key is
-- models.company_key(name), the slug roles.company_key holds.
+-- ats is the host the careers site runs on, readable by the ats plugin or
+-- not (workday, successfactors, ...): what discovery found, '' unknown. A
+-- company with an ats and no board subscription is one the app cannot read
+-- yet. country is an ISO code and city the head office's.
CREATE TABLE IF NOT EXISTS companies (
key TEXT PRIMARY KEY,
name TEXT NOT NULL,
@@ -99,7 +103,10 @@ CREATE TABLE IF NOT EXISTS companies (
careers_url TEXT NOT NULL DEFAULT '',
notes TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
- updated_at TEXT NOT NULL
+ updated_at TEXT NOT NULL,
+ ats TEXT NOT NULL DEFAULT '',
+ country TEXT NOT NULL DEFAULT '',
+ city TEXT NOT NULL DEFAULT ''
);
-- The targets desk's runner hands to plugins. newsdesk's shape, with its two
@@ -319,6 +326,11 @@ SELECT (SELECT id FROM agents ORDER BY enabled DESC, id LIMIT 1), key, affinity,
ALTER TABLE roles DROP COLUMN vote;
ALTER TABLE roles DROP COLUMN voted_at;
ALTER TABLE companies DROP COLUMN affinity;
+"""),
+ (7, "companies.ats, country and city: a company the app cannot read yet is still a row", """
+ALTER TABLE companies ADD COLUMN ats TEXT NOT NULL DEFAULT '';
+ALTER TABLE companies ADD COLUMN country TEXT NOT NULL DEFAULT '';
+ALTER TABLE companies ADD COLUMN city TEXT NOT NULL DEFAULT '';
"""),
)
diff --git a/app/storage/store.py b/app/storage/store.py
index b75f957..59ac571 100644
--- a/app/storage/store.py
+++ b/app/storage/store.py
@@ -69,7 +69,7 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
#: nothing, so `vote` and `voted_at` are NULL: one query shape either way.
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
-_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes")
+_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city")
#: A company as the store serves it: the row and the asked-for agent's affinity.
_COMPANY_SELECT = """SELECT c.*, COALESCE(ac.affinity, 0) AS affinity FROM companies c
@@ -967,7 +967,8 @@ class Store:
def add_company(self, name: str, *, key: str = "", domain: str = "",
status: str = "researching", careers_url: str = "",
- notes: str = "") -> Company:
+ notes: str = "", ats: str = "", country: str = "",
+ city: str = "") -> Company:
"""Raises ValueError on a key already taken."""
name = name.strip()
key = key or company_key(name)
@@ -977,9 +978,10 @@ class Store:
try:
self._write(
"""INSERT INTO companies (key, name, domain, status, careers_url, notes,
- created_at, updated_at)
- VALUES (?,?,?,?,?,?,?,?)""",
- (key, name, domain, status, careers_url, notes, stamp, stamp))
+ ats, country, city, created_at, updated_at)
+ VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
+ (key, name, domain, status, careers_url, notes, ats, country, city,
+ stamp, stamp))
except sqlite3.IntegrityError as exc:
raise ValueError(f"company {key!r} already exists") from exc
return self._company(key)
diff --git a/app/templates/settings/sources.html b/app/templates/settings/sources.html
index 007c3c8..1806a2e 100644
--- a/app/templates/settings/sources.html
+++ b/app/templates/settings/sources.html
@@ -86,7 +86,7 @@