From 047f7cd8addd602f1cb33e1b385d19feec198623 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 23 Sep 2026 19:55:50 +0200 Subject: [PATCH] Find roles by search, not only by company: Jobindex and Adzuna searches The ats boards only read companies someone listed, and the seed list is US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs saved queries on boards that span companies, pulled in the same task as the boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU, free key, built to its documented API). Jobnet and Jooble were checked and left out. `country` is now filled from the board's location line by a lookup, with EU for Europe-wide roles, so country gates work without extract. `app companies prune --in` unsubscribes boards with no open role in a region. The starter agents and searches are Copenhagen and Europe. Co-Authored-By: Claude Opus 5.5 --- README.md | 17 +- app/cli.py | 125 ++++++++++- app/config.py | 16 +- app/jobs/__init__.py | 5 +- app/jobs/pull.py | 160 +++++++++---- app/main.py | 3 +- app/models.py | 130 ++++++++++- app/storage/store.py | 43 +++- config.example.yaml | 17 +- config.yaml | 45 +++- docs/decisions.md | 12 + plugins/search.py | 345 +++++++++++++++++++++++++++++ tests/fixtures/search/adzuna.json | 42 ++++ tests/fixtures/search/jobindex.xml | 125 +++++++++++ tests/test_pull.py | 7 +- tests/test_score.py | 11 +- tests/test_search.py | 262 ++++++++++++++++++++++ 17 files changed, 1293 insertions(+), 72 deletions(-) create mode 100644 plugins/search.py create mode 100644 tests/fixtures/search/adzuna.json create mode 100644 tests/fixtures/search/jobindex.xml create mode 100644 tests/test_search.py diff --git a/README.md b/README.md index f618b8a..3a78774 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # roledesk -A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make. +A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio) and runs saved searches on boards that span companies (Jobindex, Adzuna), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make. Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeline/`), password (`auth/`) and deploy kit (`deploy/`), plus the plugin contract and tools from news (`desk/`). FastAPI, Jinja2, SQLite, `uv`, no build step. @@ -9,11 +9,22 @@ Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeli ```bash uv sync --all-extras uv run app companies seed # 186 companies from seed/companies.json, one board each -uv run app job pull # read every board, then extract, embed and score +uv run app job pull # read every board and search, then extract, embed and score uv run app serve # http://127.0.0.1:8790 ``` -`uv run app worker` drains the queue without the web app, and `uv run pytest` runs the app's tests and the copied modules' own. `config.yaml` is the committed default, with two starter search agents; `config.example.yaml` documents every key. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys. +`uv run app worker` drains the queue without the web app, and `uv run pytest` runs the app's tests and the copied modules' own. `config.yaml` is the committed default, with three starter agents and four Copenhagen searches on Jobindex; `config.example.yaml` documents every key. + +## Searches and companies + +```bash +uv run app searches add jobindex "Copenhagen golang" --q "golang go-udvikler" --area 1 +uv run app searches add adzuna "Berlin backend" --q backend --country de --where Berlin +uv run app searches list +uv run app companies prune --in DK,SE,NO,EU # lists; add --apply to unsubscribe +``` + +A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys. ## The tabs diff --git a/app/cli.py b/app/cli.py index 7d785a0..84d32d6 100644 --- a/app/cli.py +++ b/app/cli.py @@ -3,7 +3,8 @@ app serve [--host H] [--port P] [--reload] app job pull|extract|embed|score [--payload JSON] app worker [--workers http=8,cpu=1] [--until-idle] - app companies seed|add|check + app companies seed|add|check|prune + app searches add|list|remove app db init `app job` queues the work on a queue of its own and drains it here, so a @@ -72,6 +73,32 @@ def build_parser() -> argparse.ArgumentParser: add.add_argument("board", help="host:slug, e.g. greenhouse:acme") add.add_argument("name") companies_sub.add_parser("check", help="report boards that do not answer") + prune = companies_sub.add_parser( + "prune", help="unsubscribe every board with no open role in the region") + prune.add_argument("--in", dest="region", default="europe", + help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)") + prune.add_argument("--apply", action="store_true", + help="unsubscribe and close their roles; without it, only list them") + + searches = sub.add_parser("searches", help="saved searches on cross-company job boards") + searches_sub = searches.add_subparsers(dest="action", required=True) + search_add = searches_sub.add_parser( + "add", help="run a search once, then save it for every pull") + search_add.add_argument("provider", choices=["jobindex", "adzuna"]) + search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"') + search_add.add_argument("--q", required=True, + help="the query, in the provider's own syntax") + search_add.add_argument("--area", type=int, default=0, + help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus") + search_add.add_argument("--country", default="", + help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch") + search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin") + search_add.add_argument("--pages", type=int, default=0, + help="result pages per pull (default 5, at most 10)") + searches_sub.add_parser("list", help="every saved search and its open roles") + search_remove = searches_sub.add_parser( + "remove", help="stop running a search; its roles stay until they go stale") + search_remove.add_argument("name") db = sub.add_parser("db", help="database housekeeping") db.add_argument("action", choices=["init"]) @@ -91,6 +118,8 @@ def main(argv: list[str] | None = None) -> int: return run_worker(args) if args.command == "companies": return run_companies(args) + if args.command == "searches": + return run_searches(args) return init_db(args) except (FileNotFoundError, config_mod.ConfigError) as exc: print(exc, file=sys.stderr) @@ -205,6 +234,8 @@ def run_companies(args: argparse.Namespace) -> int: return 0 if args.action == "add": return add_company(store, args.board, args.name) + if args.action == "prune": + return prune_companies(store, args.region, apply=args.apply) return check_companies(store) @@ -293,6 +324,98 @@ def check_companies(store: Store) -> int: return 1 if dead else 0 +def prune_companies(store: Store, region: str, *, apply: bool) -> int: + """`app companies prune --in DK,EU`: every subscribed board none of whose + open roles is in the region, by `country`, which this fills from `place` + first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles, + since a board no longer pulled would otherwise keep them open for good. + `app companies seed` never brings one back: it skips a company that exists. + + A board whose open roles all name no country is left alone: it says + nothing about where that company hires.""" + from app.models import EUROPE + + filled = store.backfill_country() + countries = store.open_countries_by_subscription() + wanted = {c.strip().upper() for c in region.split(",") if c.strip()} + if "EUROPE" in wanted: + wanted = (wanted - {"EUROPE"}) | set(EUROPE) + pruned = [(sub, sum(countries[sub.id].values())) + for sub in store.subscriptions(source="ats", fetchable=True) + if set(countries.get(sub.id, {})) - {""} + and not wanted.intersection(countries[sub.id])] + closed = 0 + for sub, count in sorted(pruned, key=lambda p: -p[1]): + print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}") + if apply: + store.unsubscribe(sub.id) + closed += store.close_subscription(sub.id) + kept = len(store.subscriptions(source="ats", fetchable=True)) + print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply, + "roles_closed": closed, "boards_left": kept})) + return 0 + + +def run_searches(args: argparse.Namespace) -> int: + from app.storage import Store + + config = config_mod.load(args.config) + store = Store(config.database) + if args.action == "add": + return add_search(config, store, args) + if args.action == "remove": + sub = store.subscription_for("search", args.name) + if sub is None: + print(f"searches remove: no search named {args.name!r}", file=sys.stderr) + return 1 + store.unsubscribe(sub.id) + print(json.dumps({"removed": sub.label})) + return 0 + countries = store.open_countries_by_subscription() + for sub in store.subscriptions(source="search"): + removed = "" if sub.subscribed and sub.fetch else " (removed)" + open_roles = sum(countries.get(sub.id, {}).values()) + print(f"{sub.label:30} {sub.target:40} {open_roles:5} open " + f"{json.dumps(sub.params, ensure_ascii=False)}{removed}") + return 0 + + +def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int: + """`app searches add jobindex "Copenhagen backend" --q '"backend developer"' + --area 1`: run the search once and refuse if it fails or finds nothing, so + a typo in a query is found now rather than by an empty tab later.""" + from app.jobs import search_target + from plugins.search import run_search + + params: dict[str, Any] = {"q": args.q} + if args.provider == "jobindex" and args.area: + params["area"] = args.area + if args.provider == "adzuna": + params["country"] = args.country.lower() + if args.where: + params["where"] = args.where + if args.pages: + params["pages"] = args.pages + target = search_target(args.provider, args.name) + try: + rows, complete = run_search(target, params, dict(config.sources.get("search") or {})) + except Exception as exc: + print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr) + return 1 + if not rows: + print("searches add: the search found nothing; not saved", file=sys.stderr) + return 1 + try: + store.add_subscription("search", target, label=args.name, board=args.provider, + params=params) + except ValueError as exc: + print(f"searches add: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"search": args.name, "target": target, "roles": len(rows), + "complete": complete}, ensure_ascii=False)) + return 0 + + def init_db(args: argparse.Namespace) -> int: """Create the file, or bring it to the current schema.""" from app.storage import LATEST, Store diff --git a/app/config.py b/app/config.py index 657f6f6..be00784 100644 --- a/app/config.py +++ b/app/config.py @@ -17,7 +17,7 @@ from __future__ import annotations import os from pathlib import Path -from typing import Any +from typing import Any, Literal import yaml from pydantic import Field, ValidationError, model_validator @@ -72,6 +72,17 @@ class AgentSeed(Section): enabled: bool = True +class SearchSeed(Section): + """A saved search as `config.yaml` lists it: one `search` subscription, + imported once like the agents. `params` are the provider's own, see + plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`, + `where` and `pages` for adzuna.""" + + name: str + provider: Literal["jobindex", "adzuna"] + params: dict[str, Any] = Field(default_factory=dict) + + class Config(Section): """The whole file.""" @@ -101,6 +112,9 @@ class Config(Section): embed: EmbedConfig = Field(default_factory=EmbedConfig) #: Starter agents, imported once into an empty table. agents: list[AgentSeed] = Field(default_factory=list) + #: Starter searches, imported once into an empty set of `search` + #: subscriptions. + searches: list[SearchSeed] = Field(default_factory=list) #: One block per plugin: `enabled`, and whatever that plugin reads. sources: dict[str, Any] = Field(default_factory=dict) diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index 21cbabe..14aaa60 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -1,7 +1,7 @@ """The work roledesk queues. Importing this package registers every job, every task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. - pull.py read every configured plugin and store the roles it yields + pull.py read every board and search and store the roles they yield extract.py title and description into the gate columns, on Haiku embed.py one vector per open role, for fit score.py every open role scored for every enabled search agent @@ -14,6 +14,7 @@ from __future__ import annotations from typing import Any from app.jobs import cascade, embed, extract, pull, score +from app.jobs.pull import search_target, seed_searches from app.jobs.score import seed_agents #: What `JobQueue(bootstrap=...)` and `pipeline.worker --bootstrap` are given. @@ -32,4 +33,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]: __all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score", - "seed_agents"] + "search_target", "seed_agents", "seed_searches"] diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 6863ba5..950ee4f 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -1,28 +1,34 @@ -"""`pull`: fetch every active `ats` board and store the roles it lists. +"""`pull`: fetch every active `ats` board and `search`, and store the roles +they list. -A task, not a single job: ten boards per `pull_batch` job on the `http` -resource pool, because a board is one HTTP round trip and that pool is sized -for exactly this. The final job's `settle` then closes every board a +A task, not a single job: ten boards or searches per `pull_batch` job on the +`http` resource pool, because a board is one HTTP round trip and that pool is +sized for exactly this. The final job's `settle` then closes every board a complete fetch of it did not list again — see `close_boards`. -Closure is read from `self.state`, which the `ats` plugin itself stamps per -board on a successful parse (`ats:ok:`, `ats:ids:`), rather -than from what each `pull_batch` job happens to return. A batch job that -raises partway through still leaves the boards it had already finished -stamped, so one bad board in a batch of ten does not cost the other nine -their closure. +Closure is read from `self.state`, which each plugin itself stamps per target +on a successful parse (`ats:ok:`, `ats:ids:`, and the same +under `search:`), rather than from what each `pull_batch` job happens to +return. A batch job that raises partway through still leaves the boards it had +already finished stamped, so one bad board in a batch of ten does not cost the +other nine their closure. + +A search closes the same way when its results ran out before its page limit, +since it then listed everything that matches. One that hit the limit did not, +so its roles close only once no search has seen them for `STALE_DAYS`. """ from __future__ import annotations import json import logging -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Any from pipeline import NULL, RunContext, Tally, Task, job from app.config import Config +from app.models import company_key from app.storage import Store log = logging.getLogger(__name__) @@ -34,49 +40,74 @@ BATCH_SIZE = 10 SETTLE = "app.jobs.pull:close_boards" +#: The plugins a pull reads, by the `source` their subscriptions carry. +SOURCES = ("ats", "search") + +#: How long a search role stays open once no search lists it any more. +STALE_DAYS = 21 + +#: The `state` key that records the starter searches were imported. +SEARCHES_SEEDED = "searches:seeded" + def _now() -> str: return datetime.now(UTC).isoformat(timespec="seconds") def _boards(config: Config, store: Store) -> list[dict[str, Any]]: - """Every fetchable `ats` subscription, ten to a batch. Stamps the run - marker `close_boards` will look for, before any batch runs — finders run - synchronously, in whoever starts the task, ahead of every job it queues.""" - subs = store.subscriptions(source="ats", fetchable=True) - if not subs: - return [] - store.state["ats:run"] = _now() - return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]} - for i in range(0, len(subs), BATCH_SIZE)] + """Every fetchable `ats` and `search` subscription, ten to a batch of one + source. Stamps the run marker `close_boards` will look for, before any + batch runs — finders run synchronously, in whoever starts the task, ahead + of every job it queues.""" + seed_searches(config, store) + batches: list[dict[str, Any]] = [] + for source in SOURCES: + subs = store.subscriptions(source=source, fetchable=True) + if not subs: + continue + store.state[f"{source}:run"] = _now() + batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source} + for i in range(0, len(subs), BATCH_SIZE)] + return batches @job("pull_batch", "Reading boards", resource="http") -def run_batch(config: Config, store: Store, *, targets: list[str], +def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats", context: RunContext = NULL) -> dict[str, Any]: - """Fetch this batch's boards and `upsert_role` every item they list. + """Fetch this batch's boards or searches and `upsert_role` every item + they list. `targets` is keyword-only without a default, which is what makes it a required payload key to the queue: the Jobs tab offers no button for a batch that could only fail without one. - Builds the `ats` plugin scoped to just these targets — `Source.targets()` - reads `options[subscription_field]`, so handing it only this batch's - entries is what keeps one job from re-fetching every other job's boards. + Builds the plugin scoped to just these targets — `Source.targets()` reads + `options[subscription_field]`, so handing it only this batch's entries is + what keeps one job from re-fetching every other job's boards. """ from plugins.ats import ATSSource + from plugins.search import SearchSource - subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets} - options = dict(config.sources.get("ats") or {}) - options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label} - for target in targets if target in subs] - source = ATSSource(options) - source.state = store.state + subs = {s.target: s for s in store.subscriptions(source=source) if s.target in targets} + options = dict(config.sources.get(source) or {}) + plugin: ATSSource | SearchSource + if source == "search": + options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params} + for t in targets if t in subs] + plugin = SearchSource(options) + else: + options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label} + for t in targets if t in subs] + plugin = ATSSource(options) + plugin.state = store.state stored = 0 - for item in source.fetch(): - item.source = "ats" - target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}" + for item in plugin.fetch(): + item.source = source + if source == "search": + target = str(item.raw.get("search", "")) + else: + target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}" store.upsert_role(item, subs.get(target)) stored += 1 return {"roles": stored} @@ -91,20 +122,57 @@ def close_boards(config: Config, store: Store, tally: Tally, *, `pull_batch` job that raised partway through still leaves the boards it had already parsed stamped — see the module docstring. A board whose `ats:ok:` was not written during *this* run — it failed, or its - whole batch job never got to it — is never closed. + whole batch job never got to it — is never closed. A search closes by its + list only when that list was complete, and otherwise by `STALE_DAYS`. """ - run_marker = store.state.get("ats:run", "") - answered = failed = closed = 0 - for sub in store.subscriptions(source="ats", fetchable=True): - stamp = store.state.get(f"ats:ok:{sub.target}", "") - if stamp and stamp == run_marker: - answered += 1 - seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]")) - closed += store.mark_closed("ats", seen_ids, sub.id) - else: - failed += 1 + answered = searched = failed = closed = 0 + for source in SOURCES: + run_marker = store.state.get(f"{source}:run", "") + for sub in store.subscriptions(source=source, fetchable=True): + stamp = store.state.get(f"{source}:ok:{sub.target}", "") + if not (stamp and stamp == run_marker): + failed += 1 + continue + if source == "search": + searched += 1 + if not store.state.get(f"search:complete:{sub.target}", ""): + continue + else: + answered += 1 + seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) + closed += store.mark_closed(source, seen_ids, sub.id) + if searched: + stale = datetime.now(UTC) - timedelta(days=STALE_DAYS) + closed += store.close_unseen("search", stale.isoformat(timespec="seconds")) roles = sum(int(result.get("roles", 0)) for result in tally.results) - return {"boards": answered, "failed": failed, "roles": roles, "closed": closed} + return {"boards": answered, "searches": searched, "failed": failed, "roles": roles, + "closed": closed} + + +def search_target(provider: str, name: str) -> str: + """The subscription target of a saved search: `jobindex:copenhagen-backend`. + The query itself lives in `params`, so editing it keeps the target.""" + return f"{provider}:{company_key(name)}" + + +def seed_searches(config: Config, store: Store) -> list[str]: + """Import `config.yaml`'s `searches:` as `search` subscriptions, once, + and only while there are none: the same rule as `seed_agents`.""" + if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"): + return [] + added = [] + for seed in config.searches: + try: + store.add_subscription("search", search_target(seed.provider, seed.name), + label=seed.name, board=seed.provider, params=seed.params) + except ValueError as exc: + log.warning("searches: %s", exc) + continue + added.append(seed.name) + if added: + store.state[SEARCHES_SEEDED] = store.now() + log.info("imported %d searches from %s", len(added), config.path or "the defaults") + return added def pull_task() -> Task: diff --git a/app/main.py b/app/main.py index 727326e..ee112fd 100644 --- a/app/main.py +++ b/app/main.py @@ -45,9 +45,10 @@ def create_app(config: Config | None = None) -> FastAPI: # auth add their own tables to it. store = Store(config.database) jobs.seed_agents(config, store) + jobs.seed_searches(config, store) app = FastAPI(title="roledesk", docs_url="/api/docs", - summary="ATS boards in, roles scored per search agent, " + summary="ATS boards and job searches in, roles scored per search agent, " "applications tracked.") app.state.config = config app.state.store = store diff --git a/app/models.py b/app/models.py index 14b9bd1..8c32203 100644 --- a/app/models.py +++ b/app/models.py @@ -34,7 +34,7 @@ _GENDER = re.compile( #: board and "celonis" on another share a key. _LEGAL = re.compile( r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|" - r"co|bv|b\.v|sa|sas|srl|oy|ab|as|aps))+\.?$", + r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$", re.IGNORECASE, ) @@ -64,6 +64,134 @@ def company_key(company: str) -> str: return _fold(name).replace(" ", "-") +# ---------------------------------------------------------------- geography + +#: The EU, the EEA, the UK and Switzerland: what "a European role" means to a +#: gate on `country`. `EU` is ISO's reserved code, used here for a role open +#: anywhere in Europe ("Remote - Europe", "EMEA") that names no one country. +EUROPE: tuple[str, ...] = ( + "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", + "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", + "IS", "LI", "NO", "GB", "CH", "EU", +) + +#: Folded names a board writes in its location line, to an ISO code: every +#: European country in English and its own language, the cities the boards +#: name most, and the non-European places that fill the seed boards, so those +#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest. +_PLACES: dict[str, str] = { + **dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"), + **dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"), + **dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp", + "ghent"), "BE"), + **dict.fromkeys(("bulgaria", "sofia"), "BG"), + **dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"), + **dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"), + **dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"), + # Folding keeps ø and æ, which are letters of their own rather than an o + # or an a with a mark, so København is listed beside Kobenhavn. + **dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus", + "århus", "odense", "aalborg", "region hovedstaden", "frederiksberg", + "lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup", + "bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv", + "birkerød"), "DK"), + **dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"), + **dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"), + **dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes", + "bordeaux", "lille"), "FR"), + **dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg", + "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig", + "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe", + "heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"), + **dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"), + **dict.fromkeys(("hungary", "budapest"), "HU"), + **dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"), + **dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin", + "torino", "bologna", "florence"), "IT"), + **dict.fromkeys(("latvia", "riga"), "LV"), + **dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"), + **dict.fromkeys(("luxembourg",), "LU"), + **dict.fromkeys(("malta", "valletta"), "MT"), + **dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam", + "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", + "delft"), "NL"), + **dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw", + "gdansk", "poznan", "lodz"), "PL"), + **dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"), + **dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), + "RO"), + **dict.fromkeys(("slovakia", "bratislava"), "SK"), + **dict.fromkeys(("slovenia", "ljubljana"), "SI"), + **dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville", + "malaga", "bilbao"), "ES"), + **dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo", + "uppsala"), "SE"), + **dict.fromkeys(("iceland", "reykjavik"), "IS"), + **dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"), + **dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland", + "wales", "northern ireland", "london", "manchester", "edinburgh", + "glasgow", "bristol", "cambridge uk", "oxford", "leeds", + "birmingham uk", "belfast"), "GB"), + **dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve", + "basel", "lausanne", "bern", "zug"), "CH"), + **dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city", + "nyc", "san francisco", "seattle", "boston", "chicago", "austin", + "los angeles", "palo alto", "mountain view", "san mateo", + "foster city", "sunnyvale", "san jose", "menlo park", "redwood city", + "costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas", + "houston", "phoenix", "tempe", "portland", "salt lake city", + "pittsburgh", "philadelphia", "detroit", "san diego", "raleigh", + "nashville", "minneapolis"), "US"), + **dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary", + "waterloo"), "CA"), + **dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai", + "chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), + **dict.fromkeys(("singapore",), "SG"), + **dict.fromkeys(("japan", "tokyo", "osaka"), "JP"), + **dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"), + **dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), + **dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"), + **dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"), + **dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"), + **dict.fromkeys(("south korea", "korea", "seoul"), "KR"), +} + +#: Keyed the way a place is folded before it is looked up, so a name can be +#: written as it is spelt. +_PLACES = {_fold(name): code for name, code in _PLACES.items()} + +#: The longest name first, so "new york city" wins over "york" and "northern +#: ireland" over "ireland". +_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True) + +#: US state codes after a comma, the way US boards write a city: "Boston, MA". +#: Read only when no name matched, since "DE" and "IN" are countries too. +_US_STATE = re.compile( + r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|" + r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)" + r"\b") + + +def country_of(place: str) -> str: + """The one country a board's location line names, as an ISO code, or '' + when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is + US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe" + is EU, and "London, EMEA" is GB: a country beats the region it is in.""" + folded = f" {_fold(place or '')} " + found: set[str] = set() + for name in _PLACE_NAMES: + needle = f" {name} " + if needle in folded: + found.add(_PLACES[name]) + folded = folded.replace(needle, " ") + if not found and _US_STATE.search(place or ""): + # Only when nothing else named a country: "Berlin, DE" is Germany. + found.add("US") + if len(found) > 1: + found.discard("EU") + return found.pop() if len(found) == 1 else "" + + # --------------------------------------------------------------- vocabulary Remote = Literal["", "remote", "hybrid", "onsite"] diff --git a/app/storage/store.py b/app/storage/store.py index eb6950f..c1318ed 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -22,8 +22,8 @@ import numpy as np from desk.models import Item from app.models import ( AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate, - Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, place_key, - title_key, + Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, country_of, + place_key, title_key, ) from app.storage import schema @@ -212,7 +212,8 @@ class Store: `first_seen` and `created_at` are written on insert only; `last_seen` on every call, and a role seen again is open again. The gate columns come from `item.raw` under their own names, with `company` falling back to - the subscription's; an empty value never overwrites a filled column. + the subscription's and `country` to what `place` names; an empty value + never overwrites a filled column. """ if not item.source: raise ValueError(f"item {item.source_id!r} has no source; the runner sets it") @@ -241,6 +242,7 @@ class Store: "created_at": stamp, } row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT} + row["country"] = row["country"] or country_of(place) row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT} keep_filled = ("company", "company_key", "place_key", "remote", "description", @@ -341,6 +343,31 @@ class Store: AND source_id NOT IN (SELECT value FROM json_each(?))""", (self.now(), source, subscription_id, json.dumps(list(seen_ids)))) + def close_unseen(self, source: str, before: str) -> int: + """Close every open role of `source` last seen before `before`. For a + source whose fetch is never a complete list, a search, where a role + dropping out of one run's results is not the role closing.""" + return self._write( + """UPDATE roles SET closed_at = ? + WHERE source = ? AND closed_at IS NULL AND last_seen < ?""", + (self.now(), source, before)) + + def close_subscription(self, subscription_id: int) -> int: + """Close every open role of one subscription, for a board that is no + longer fetched: nothing would ever close them otherwise.""" + return self._write( + "UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL", + (self.now(), subscription_id)) + + def backfill_country(self) -> int: + """Fill `country` from `place` on every role that has none, the way + `upsert_role` does for a new one. Returns how many it filled.""" + rows = self._rows("SELECT uid, place FROM roles WHERE country = '' AND place != ''") + filled = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))] + with self.transaction() as conn: + conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", filled) + return len(filled) + # -------------------------------------------------------------- extract def extract_pending(self) -> int: @@ -614,6 +641,16 @@ class Store: "SELECT company_key, COUNT(*) AS n FROM roles " "WHERE closed_at IS NULL GROUP BY company_key")} + def open_countries_by_subscription(self) -> dict[int, dict[str, int]]: + """Per subscription id, its open roles counted by `country`.""" + rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles + WHERE closed_at IS NULL AND subscription_id IS NOT NULL + GROUP BY subscription_id, country""") + out: dict[int, dict[str, int]] = {} + for r in rows: + out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"] + return out + def company_boards(self) -> dict[str, str]: """`company_key -> ats host`, read off each company's own subscription. `Subscription.company` is the display name it and the diff --git a/config.example.yaml b/config.example.yaml index 549f463..5a08873 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -60,12 +60,23 @@ embed: model: sentence-transformers/all-MiniLM-L6-v2 per_run: 50000 -# One block per plugin, keyed by the plugin's name. The ats plugin reads its -# boards from the subscriptions table, so it needs none. +# One block per plugin, keyed by the plugin's name. The ats and search plugins +# read their boards and searches from the subscriptions table, so they need +# none, except the key an adzuna search needs (or ADZUNA_APP_ID and +# ADZUNA_APP_KEY in the environment, which keeps it out of this file). sources: {} +# search: +# adzuna: {app_id: "...", app_key: "..."} # Starter search agents, imported once into an empty `agents` table. Each gate # is {field, values, mode: require|prefer, weight}; fields are place, country, # remote, seniority, function, salary_min, currency, permit, languages and -# company. weights default to {fit: 0.6, recency: 0.2, affinity: 0.2}. +# company. weights default to {fit: 0.6, recency: 0.2, affinity: 0.2}. A +# country is an ISO code, filled from the board's location line; EU is a role +# open anywhere in Europe. agents: [] + +# Starter searches, imported once while there are no `search` subscriptions. +# provider is jobindex (Denmark, no key: q, area, pages) or adzuna (the UK and +# most of the EU, a free key: q, country, where, pages). See plugins/search.py. +searches: [] diff --git a/config.yaml b/config.yaml index 86f72ea..a09993b 100644 --- a/config.yaml +++ b/config.yaml @@ -21,21 +21,50 @@ embed: model: sentence-transformers/all-MiniLM-L6-v2 sources: {} # Starter search agents, imported once into an empty table. Edit them on the -# Agents tab afterwards; this list is not read again. +# Agents tab afterwards; this list is not read again. EU in a country gate is a +# role open anywhere in Europe ("Remote - Europe", "EMEA"). agents: - - name: Berlin backend + - name: Copenhagen engineering intent: >- - Senior backend engineer. Python and Go services, PostgreSQL, event-driven - systems, APIs and data pipelines on Kubernetes in the cloud. Wants - ownership of production systems in a product company. + Senior backend or platform engineer. Python and Go services, PostgreSQL, + event-driven systems, APIs and data pipelines on Kubernetes in the cloud. + Wants ownership of production systems in a product company. gates: - - {field: place, values: [Berlin], mode: require} + - {field: country, values: [DK], mode: require} - {field: remote, values: [hybrid, remote], mode: prefer, weight: 0.05} - - name: Remote EU stretch + - name: Remote from Europe intent: >- Staff or lead engineer role, fully remote from Europe. Platform and infrastructure work, distributed systems, developer tooling, technical leadership across teams. gates: - {field: remote, values: [remote], mode: require} - - {field: country, values: [DE, NL, AT, CH], mode: prefer, weight: 0.05} + - {field: country, values: [AT, BE, BG, HR, CY, CZ, DK, EE, FI, FR, DE, GR, HU, IE, IT, + LV, LT, LU, MT, NL, PL, PT, RO, SK, SI, ES, SE, IS, LI, NO, + GB, CH, EU], mode: require} + - name: Europe hybrid + intent: >- + Senior backend or platform engineer in a European tech hub, hybrid or + remote-first. Python and Go, distributed systems, cloud infrastructure. + gates: + - {field: country, values: [AT, BE, BG, HR, CY, CZ, DK, EE, FI, FR, DE, GR, HU, IE, IT, + LV, LT, LU, MT, NL, PL, PT, RO, SK, SI, ES, SE, IS, LI, NO, + GB, CH, EU], mode: require} + - {field: remote, values: [hybrid, remote], mode: prefer, weight: 0.05} +# Starter searches, imported once while there are none. Each is a query on a +# job board that spans companies (plugins/search.py); `app searches add` adds +# more. area 1 is Jobindex's Region Hovedstaden. An adzuna search needs a key: +# - {name: Berlin backend, provider: adzuna, params: {q: backend, country: de, where: Berlin}} +searches: + - name: Copenhagen backend + provider: jobindex + params: {q: backend, area: 1, pages: 10} + - name: Copenhagen software engineer + provider: jobindex + params: {q: '"software engineer" "software developer" softwareudvikler', area: 1, pages: 10} + - name: Copenhagen platform + provider: jobindex + params: {q: 'devops "platform engineer" kubernetes', area: 1, pages: 10} + - name: Copenhagen data engineer + provider: jobindex + params: {q: '"data engineer"', area: 1, pages: 10} diff --git a/docs/decisions.md b/docs/decisions.md index 04f31c8..86c1f7d 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -26,6 +26,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`extract` reads the board's location too.** The model gets title, the board's `place` text, and the first 3000 characters of the description. Greenhouse's location line ("Berlin, Germany") is often the only place a country is stated, and it is part of the posting, so the "only what the posting states" contract still holds. +**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the folded location line to an ISO code from a table of European countries and cities, plus the non-European places that fill the seed boards, and returns nothing when a line names two countries. It resolves about 90% of the seed boards' roles for free, where `extract` would be a paid call per role for the same answer. `EU`, ISO's reserved code, stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it. A US state after a comma counts only when no name matched, because DE and IN are countries too. + **`permit` and `languages` are columns, and so is `extract_tries`.** `permit` answers the plan's open item 1 (visa sponsorship as a gate field) while nothing depends on it yet. newsdesk keeps give-up counts in `raw`; here `upsert_role` rewrites `raw` on every pull, which would reset the count, so the counter is its own column. **Postings are persisted.** Storing every posting is what gives `first_seen`, which is the only honest posted date since Greenhouse resets `updated_at` on every edit, and `closed_at`, set by `mark_closed` when a complete fetch of a board no longer lists a role. A role listed again is open again. @@ -38,6 +40,16 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker and the ids it saw, and the pull's settle closes a board only when its stamp matches the marker written before any batch ran. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list. +**Boards follow companies; searches find them.** `ats` only reads companies someone listed, so it cannot surface a role at a company nobody knew of, and the seed list came from a US-centred project: of 29,000 open roles, 31 were in Denmark. The `search` plugin runs saved queries on boards that span companies, as `search` subscriptions whose `params` are the query, and pulls them in the same task as the boards. + +**Jobindex and Adzuna, not Jobnet or Jooble.** Jobindex, Denmark's largest board, has an RSS twin for every search page, with phrase and required-word syntax and a region id; no key. Adzuna has a documented API with a free key for the UK and most of the EU, but no Denmark. Jobnet's search endpoint answers 401 outside its own site's session, and Jooble gives a key only on a manual application and answered a Cloudflare challenge. LinkedIn and Indeed have no public search API. + +**A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. A search whose results ran out before its page limit did list everything, and closes like a board; the rest close once no search has listed them for 21 days. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists. + +**Jobindex ads are read from the feed, not followed to the employer.** Each item links to Jobindex's own page, and its `/c?t=` redirect leads to the employer's site or ATS. Following it per role would count as a click in Jobindex's statistics and costs a request per role, so discovering a company's ATS board from a search result is left for later. + +**`app companies prune` cuts the list by where it hires.** A board none of whose open roles is in the region (`--in`, Europe by default, or codes such as `DK,SE,EU`) is unsubscribed and its open roles closed, since a board no longer pulled would keep them open for good. A board whose roles name no country is kept, because it says nothing about where the company hires. + **SmartRecruiters' board list has no description.** Its `/postings` endpoint gives title, location and department; the job text is only on the per-posting endpoint, and one extra request per role would multiply a board's request count by its role count. `description` stays empty for this host. **The body cap is 96 MB, with one retry for Greenhouse.** openings.go's 8 MB cap was written against arbitrary URLs, while `ats` only fetches a fixed allow-list of seven ATS hosts, and the real run found `?content=true` boards between 9 and 41 MB. A board that still exceeds 96 MB retries once without `?content=true`, trading the description for the listing. diff --git a/plugins/search.py b/plugins/search.py new file mode 100644 index 0000000..bce4af5 --- /dev/null +++ b/plugins/search.py @@ -0,0 +1,345 @@ +"""Job searches: a query against a job board that spans companies. + +`ats` reads the boards of companies someone already chose. This plugin finds +roles at companies nobody has listed yet, by asking a cross-company board a +saved question: "backend developer" in Region Hovedstaden, "python" in Berlin. +A subscription is one saved search; its `params` are the question. + +Two boards, both checked live on 2026-09-23: + + jobindex Denmark's largest job board. Every search result page has an RSS + twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's + own syntax: words are OR, `"a b"` a phrase, `+word` required. + `area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region + Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. + 20 results a page. + adzuna An aggregator with a documented API for most of Europe, the UK + and the US, but not Denmark (adzuna.dk does not exist). Needs a + free key from developer.adzuna.com, as `sources.search.adzuna: + {app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built + against Adzuna's documented response and not yet run live. + +Considered and left out: Jobnet (its search endpoint answers 401 outside the +browser session of its own site) and Jooble (a key needs a manual +application, and its API host answered a Cloudflare challenge). + +A search result list is never a board's complete list, so a role missing from +one run is not closed here. A search that ran out of results before its page +limit was complete, and the pull closes what it stopped listing; the rest +close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). +""" + +from __future__ import annotations + +import html +import json +import logging +import os +import re +import xml.etree.ElementTree as ET +from collections.abc import Callable, Iterator +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime +from typing import Any + +import httpx + +from desk.sources import ( + API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, + SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html, +) + +log = logging.getLogger(__name__) + +TIMEOUT = 15.0 + +#: Pages fetched per search unless its params say otherwise, and the most a +#: search may ask for. A search that needs more than ten pages is too broad to +#: read anyway. +DEFAULT_PAGES = 5 +MAX_PAGES = 10 + +PROVIDERS = ("jobindex", "adzuna") + +#: What every provider turns a result into. +Row = dict[str, Any] + + +def _web_url(url: str) -> bool: + parsed = httpx.URL((url or "").strip()) + return parsed.scheme in ("http", "https") and bool(parsed.host) + + +def _pages(params: dict[str, Any]) -> int: + try: + return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES)) + except (TypeError, ValueError): + return DEFAULT_PAGES + + +def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes: + response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True, + headers={"User-Agent": USER_AGENT}) + response.raise_for_status() + return response.content + + +# ---------------------------------------------------------------- jobindex + +JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss" +JOBINDEX_PAGE = 20 + +_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE) +_H4 = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) +_AREA = re.compile(r'class="jix_robotjob--area">(.*?)', re.IGNORECASE | re.DOTALL) +_PARAGRAPHS = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) +#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden". +_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende") + + +#: What a Danish ad's town line says about where the work is done: +#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". +_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) +_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE) + + +def _remote(text: str) -> str | None: + if _REMOTE.search(text): + return "remote" + return "hybrid" if _HYBRID.search(text) else None + + +def _text(fragment: str) -> str: + """HTML to plain text. Jobindex breaks titles after a slash with a + zero-width space, which would otherwise end up in `title_key`.""" + return html.unescape(strip_html(fragment)).replace("​", "").strip() + + +def parse_jobindex(body: bytes) -> list[Row]: + """One page of a Jobindex RSS search. An item is `` "Role, Company", + a `<link>` to Jobindex's own page for it, and a `<description>` of HTML + whose `<h4>` is the title alone, whose logo's `alt` is the company on a + paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad + ("robot job") has no logo, so its company is what follows the title's + last comma, or '' when there is none.""" + root = ET.fromstring(body) + channel = root.find("channel") + if channel is None: + return [] + area_match = _SEARCH_AREA.search(channel.findtext("title") or "") + search_area = area_match.group(1).strip() if area_match else "" + rows = [] + for item in channel.findall("item"): + link = (item.findtext("link") or "").strip() + full_title = (item.findtext("title") or "").strip() + if not full_title or not _web_url(link): + continue + description = item.findtext("description") or "" + h4 = _H4.search(description) + title = _text(h4.group(1)) if h4 else full_title + alt = _ALT.search(description) + company = html.unescape(alt.group(1)).strip() if alt else "" + if not company and full_title.startswith(title + ", "): + company = full_title[len(title) + 2:].strip() + area = _AREA.search(description) + posted = None + if pub := item.findtext("pubDate"): + try: + posted = parsedate_to_datetime(pub) + except (TypeError, ValueError): + posted = None + place = _text(area.group(1)) if area else search_area + rows.append({ + "id": link.rstrip("/").rsplit("/", 1)[-1], + "title": title or full_title, + "company": company, + "place": place, + "country": "DK", + "remote": _remote(f"{place} {title}"), + "url": link, + "posted": posted, + "description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)), + "salary_min": None, "salary_max": None, "currency": "", + }) + return rows + + +def search_jobindex(client: httpx.Client, params: dict[str, Any], + keys: dict[str, Any]) -> tuple[list[Row], bool]: + """Every page of one search, newest first, until a short page or the page + limit. Returns the rows and whether the results ran out before the limit.""" + query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"} + if params.get("area"): + query["geoareaid"] = params["area"] + rows: list[Row] = [] + for page in range(1, _pages(params) + 1): + got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page})) + rows += got + if len(got) < JOBINDEX_PAGE: + return rows, True + return rows, False + + +# ------------------------------------------------------------------ adzuna + +ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}" +ADZUNA_PAGE = 50 + +#: Adzuna's countries in Europe, and the currency its salaries are in. +ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR", + "be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"} + + +def parse_adzuna(body: bytes, country: str) -> list[Row]: + """One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles + and snippets carry `<strong>` highlights. A salary Adzuna estimated + (`salary_is_predicted`) is not the posting's, so it is dropped.""" + data = json.loads(body) + rows = [] + for job in data.get("results") or []: + title = _text(job.get("title") or "") + url = (job.get("redirect_url") or "").strip() + if not title or not _web_url(url): + continue + predicted = str(job.get("salary_is_predicted") or "0") == "1" + rows.append({ + "id": str(job.get("id") or ""), + "title": title, + "company": ((job.get("company") or {}).get("display_name") or "").strip(), + "place": ((job.get("location") or {}).get("display_name") or "").strip(), + "country": country.upper(), + "remote": _remote(title), + "url": url, + "posted": _parse_iso(job.get("created")), + "description": _text(job.get("description") or ""), + "salary_min": None if predicted else job.get("salary_min"), + "salary_max": None if predicted else job.get("salary_max"), + "currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""), + }) + return rows + + +def _parse_iso(value: Any) -> datetime | None: + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) + + +def search_adzuna(client: httpx.Client, params: dict[str, Any], + keys: dict[str, Any]) -> tuple[list[Row], bool]: + app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "") + app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "") + if not (app_id and app_key): + raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, " + "or ADZUNA_APP_ID and ADZUNA_APP_KEY") + country = str(params.get("country") or "").lower() + if country not in ADZUNA_COUNTRIES: + raise ValueError(f"adzuna country {country!r} is not one of " + f"{', '.join(sorted(ADZUNA_COUNTRIES))}") + query: dict[str, Any] = {"app_id": app_id, "app_key": app_key, + "results_per_page": ADZUNA_PAGE, "sort_by": "date", + "what": str(params.get("q") or "")} + if params.get("where"): + query["where"] = params["where"] + rows: list[Row] = [] + for page in range(1, _pages(params) + 1): + url = ADZUNA_SEARCH.format(country=country, page=page) + got = parse_adzuna(_get(client, url, query), country) + rows += got + if len(got) < ADZUNA_PAGE: + return rows, True + return rows, False + + +_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]], + tuple[list[Row], bool]]] = { + "jobindex": search_jobindex, + "adzuna": search_adzuna, +} + + +def provider_of(target: str) -> str: + """`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one.""" + provider, sep, name = target.partition(":") + return provider if sep and name.strip() and provider in _SEARCHES else "" + + +def run_search(target: str, params: dict[str, Any], + keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]: + """One search, outside any pull: for `app searches add`, which runs a + search once before saving it.""" + provider = provider_of(target) + if not provider: + raise ValueError(f"{target!r} is not provider:name with provider one of " + f"{', '.join(PROVIDERS)}") + with httpx.Client() as client: + return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {}) + + +# -------------------------------------------------------------- the plugin + +class SearchSource(Source): + name = "search" + description = "Saved searches on cross-company job boards: Jobindex, Adzuna." + #: Entries are `{"url": target, "label": name, "params": {...}}`. + subscription_field = "searches" + + spec = PluginSpec( + api_version=API_VERSION, id=name, version="1", label="Job searches", + targets=TargetSpec(mode="editable", kind="search", defaults=[]), + profiles={"default": Profile( + kind="role", + tasks=Tasks( + summary=SummaryPolicy(enabled=False), + tagging=TaggingPolicy(enabled=False), + ), + presentation=ARTICLE_READER, + )}, + ) + + def fetch(self) -> Iterator[Item]: + client = httpx.Client() + run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat( + timespec="seconds") + for target, label in self.targets(): + provider = provider_of(target) + if not provider: + log.warning("search: %r is not provider:name; skipped", target) + continue + try: + rows, complete = _SEARCHES[provider]( + client, self.params_for(target), self.options.get(provider) or {}) + except Exception as exc: + log.warning("search: %s failed (%s)", target, exc) + continue + + seen_ids = [] + for row in rows: + source_id = f"{provider}:{row['id']}" + seen_ids.append(source_id) + yield Item( + source_id=source_id, + location=row["url"], + title=row["title"], + excerpt=(row["description"] or "")[:300], + body=row["description"], + published_at=row["posted"], + subscription=label, + kind="role", + raw={ + "company": row["company"], + "place": row["place"], + "country": row["country"], + "remote": row["remote"], + "salary_min": row["salary_min"], + "salary_max": row["salary_max"], + "currency": row["currency"], + "provider": provider, + "search": target, + }, + ) + self.state[f"search:ok:{target}"] = run_marker + self.state[f"search:complete:{target}"] = "1" if complete else "" + self.state[f"search:ids:{target}"] = json.dumps(seen_ids) diff --git a/tests/fixtures/search/adzuna.json b/tests/fixtures/search/adzuna.json new file mode 100644 index 0000000..28bfa41 --- /dev/null +++ b/tests/fixtures/search/adzuna.json @@ -0,0 +1,42 @@ +{ + "__CLASS__": "Adzuna::API::Response::JobSearchResults", + "count": 3, + "mean": 65000, + "results": [ + { + "__CLASS__": "Adzuna::API::Response::Job", + "id": "4412345678", + "title": "Senior <strong>Backend</strong> Engineer (m/w/d) - Remote", + "description": "Build our <strong>backend</strong> services in Python & Go.", + "created": "2026-09-20T08:15:00Z", + "redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345678?se=x", + "company": {"__CLASS__": "Adzuna::API::Response::Company", "display_name": "Acme GmbH"}, + "location": {"__CLASS__": "Adzuna::API::Response::Location", + "display_name": "Berlin, Deutschland", "area": ["Deutschland", "Berlin"]}, + "category": {"__CLASS__": "Adzuna::API::Response::Category", + "label": "IT-Stellen", "tag": "it-jobs"}, + "salary_min": 70000, + "salary_max": 90000, + "salary_is_predicted": "0", + "contract_time": "full_time" + }, + { + "__CLASS__": "Adzuna::API::Response::Job", + "id": "4412345679", + "title": "Platform Engineer", + "description": "Kubernetes and Terraform.", + "created": "2026-09-19T10:00:00Z", + "redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345679", + "company": {"display_name": "Beta AG"}, + "location": {"display_name": "Hamburg"}, + "salary_min": 64000, + "salary_max": 64000, + "salary_is_predicted": "1" + }, + { + "id": "4412345680", + "title": "Refused", + "redirect_url": "javascript:alert(1)" + } + ] +} diff --git a/tests/fixtures/search/jobindex.xml b/tests/fixtures/search/jobindex.xml new file mode 100644 index 0000000..f03d6d2 --- /dev/null +++ b/tests/fixtures/search/jobindex.xml @@ -0,0 +1,125 @@ +<?xml version="1.0" encoding="ISO-8859-1"?> + +<rss version="2.0" + xmlns:blogChannel="http://backend.userland.com/blogChannelModule" +> + +<channel> +<title>Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer +https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date +Danmarks største jobmarked +da +http://blogs.law.harvard.edu/tech/rss +support@jobindex.dk (Jobindex) +1440 + + +Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer +https://www.jobindex.dk/img/jobindex130x30gray.gif +https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&sort=date + + +Dangerous Goods Compliance Manager – ensure global safety, DSV +https://www.jobindex.dk/vis-job/h1701165 + + +<div class="jobsearch-result"> +<div class="PaidJob"> + + + <div class="PaidJob-inner"> + + +<CENTER style="margin-bottom: 15px;"><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/img/brand/40_spo_top_dsv_1812_20250718.png" ALT="DSV" style="border: 0px; margin: 0;"></a></CENTER> +<h4><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank">Dangerous Goods Compliance Manager – ensure global safety</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">Hedehusene</span> + + </div> + +<p>Do you want to minimise workplace accidents and damage to property, while helping ensure that dangerous goods and hazardous materials are handled correctly at our nearly 3,000 locations worldwide?</p><p>“One minute, you’ll help a forwarding agent in Holland figure out what to do with a pallet. The next, you’ll help shape our strategic decisions after the merger with DB Schenker,” says Martin Guldhammer.</p> +<center><A HREF="https://www.jobindex.dk/vis-job/h1701165" rel="noopener" target="_blank"><img src="https://www.jobindex.dk/api/imagescaler/v1/brand/8_spo_bund_dsv_1812_20250718.png?max_width=525" alt="DSV" style="border: 0px; margin-top: 5px;"></a></center> + + </div> + +</div> +</div> + +Logistik og spedition +https://www.jobindex.dk/1701165 +Wed, 23 Sep 2026 00:00:00 +0200 + + +Senior Account Executive +https://www.jobindex.dk/vis-job/r14011701 + + +<div class=""> +<div class="jobsearch-result"> + <div class="jix_robotjob" data-jobad-tid="r14011701" data-robotjob-abtest-enabled=""> + + + <div class="jix_robotjob-inner"> + + +<h4><A HREF="https://www.jobindex.dk/vis-job/r14011701" rel="noopener" target="_blank"> + Senior Account Executive +</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">København</span> + + </div> + +<p>All Gravy is the AI-native people operating platform for hospitality and retail.</p> + + </div> + + + </div> +</div> +</div> + +Salg +https://www.jobindex.dk/14011701 +Wed, 23 Sep 2026 00:00:00 +0200 + + +Marketing Content Creator, Clerk.IO ApS +https://www.jobindex.dk/vis-job/r14011706 + + +<div class=""> +<div class="jobsearch-result"> + <div class="jix_robotjob" data-jobad-tid="r14011706" data-robotjob-abtest-enabled=""> + + + <div class="jix_robotjob-inner"> + + +<h4><A HREF="https://www.jobindex.dk/vis-job/r14011706" rel="noopener" target="_blank"> + Marketing Content Creator +</a></h4> + + <div class="jobad-element-area"> + <span class="jix_robotjob--area">København</span> + + </div> + +<p>Your marketing degree gave you the foundation.</p> + + </div> + + + </div> +</div> +</div> + +Studiepraktik +Marketing +https://www.jobindex.dk/14011706 +Wed, 23 Sep 2026 00:00:00 +0200 + + + diff --git a/tests/test_pull.py b/tests/test_pull.py index f92889c..6e3176e 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -23,6 +23,7 @@ from app.storage import Store # noqa: E402 class FakeConfig: sources: ClassVar[dict[str, object]] = {} + searches: ClassVar[list[object]] = [] def _item(board: str, slug: str, job_id: str) -> Item: @@ -102,7 +103,8 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert result == {"boards": 1, "failed": 1, "roles": 1, "closed": 1} + assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1, + "closed": 1} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} @@ -123,5 +125,6 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert result == {"boards": 0, "failed": 1, "roles": 0, "closed": 0} + assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0, + "closed": 0} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} diff --git a/tests/test_score.py b/tests/test_score.py index 85b98e6..970bf93 100644 --- a/tests/test_score.py +++ b/tests/test_score.py @@ -331,11 +331,20 @@ def test_the_shipped_starter_agents_validate() -> None: from app import config as config_mod agents = config_mod.load(ROOT / "config.yaml").agents - assert [a.name for a in agents] == ["Berlin backend", "Remote EU stretch"] + assert [a.name for a in agents] == ["Copenhagen engineering", "Remote from Europe", + "Europe hybrid"] # Small enough that a prefer gate breaks ties rather than overrides fit. assert {g.weight for a in agents for g in a.gates if g.mode == "prefer"} == {0.05} +def test_the_shipped_starter_searches_validate() -> None: + from app import config as config_mod + + searches = config_mod.load(ROOT / "config.yaml").searches + assert searches and {s.provider for s in searches} == {"jobindex"} + assert all(s.params.get("q") for s in searches) + + # ---------------------------------------------------------------- cascade def test_the_pull_job_runs_pull_extract_embed_then_score( diff --git a/tests/test_search.py b/tests/test_search.py new file mode 100644 index 0000000..84dbeca --- /dev/null +++ b/tests/test_search.py @@ -0,0 +1,262 @@ +"""The `search` plugin, `country_of`, and what the pull and the CLI do with +searches. No network: Jobindex's feed is a trimmed copy of a real one +(2026-09-23), Adzuna's is written from its documented response.""" + +from __future__ import annotations + +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, ClassVar + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from desk.models import Item # noqa: E402 +from plugins import search # noqa: E402 +from plugins.search import SearchSource # noqa: E402 +from app import cli # noqa: E402 +from app.config import SearchSeed # noqa: E402 +from app.jobs import pull # noqa: E402 +from app.models import country_of # noqa: E402 +from app.storage import Store # noqa: E402 + +FIXTURES = ROOT / "tests" / "fixtures" / "search" + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +# ----------------------------------------------------------------- country + +@pytest.mark.parametrize(("place", "code"), [ + ("Berlin, Germany", "DE"), + ("København K", "DK"), + ("Kongens Lyngby", "DK"), + ("Boston, MA", "US"), + ("Berlin, DE", "DE"), # a state code only when nothing else matched + ("Remote - US", "US"), + ("London | Remote", "GB"), + ("Northern Ireland", "GB"), # the longer name wins over "ireland" + ("Remote - Europe", "EU"), + ("Home based - EMEA", "EU"), + ("London, EMEA", "GB"), # a country beats the region it is in + ("London or New York", ""), + ("Remote", ""), + ("", ""), +]) +def test_country_of(place: str, code: str) -> None: + assert country_of(place) == code + + +def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None: + def item(source_id: str, **raw: Any) -> Item: + it = Item(source_id=source_id, location="https://x.example/1", title="Engineer", + body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw) + it.source = "ats" + return it + + store.upsert_role(item("a", place="Copenhagen, Denmark")) + store.upsert_role(item("b", place="Remote", country="EU")) + store.upsert_role(item("c", place="Somewhere")) + assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""} + + +# ----------------------------------------------------------------- parsers + +def test_jobindex_reads_title_company_town_and_date() -> None: + rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes()) + assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"] + paid, bare, robot = rows + # A paid ad: the company is its logo's alt text, the title its

. + assert paid["company"] == "DSV" + assert paid["title"].startswith("Dangerous Goods Compliance Manager") + assert paid["place"] == "Hedehusene" + assert paid["country"] == "DK" + assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23" + assert paid["description"].startswith("Do you want to minimise") + # A crawled ad with no company after the title's comma has none. + assert bare["company"] == "" + assert bare["place"] == "København" + # A crawled ad names its company after the title. + assert robot["title"] == "Marketing Content Creator" + assert robot["company"] == "Clerk.IO ApS" + + +def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None: + assert search._remote("Birkerød, Hybrid position") == "hybrid" + assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid" + assert search._remote("Remote, Danmark") == "remote" + assert search._remote("Hedehusene") is None + + +def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None: + rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de") + assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused + first, predicted = rows + assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote" + assert first["description"] == "Build our backend services in Python & Go." + assert first["company"] == "Acme GmbH" + assert first["place"] == "Berlin, Deutschland" + assert first["country"] == "DE" + assert first["remote"] == "remote" + assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR") + assert (predicted["salary_min"], predicted["currency"]) == (None, "") + + +def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ADZUNA_APP_ID", raising=False) + monkeypatch.delenv("ADZUNA_APP_KEY", raising=False) + with pytest.raises(ValueError, match="app_id"): + search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] + + +def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None: + pages: list[int] = [] + full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)] + + def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: + pages.append(params["page"]) + return b"short" if params["page"] == 3 else b"full" + + monkeypatch.setattr(search, "_get", fake_get) + monkeypatch.setattr(search, "parse_jobindex", + lambda body: full[:5] if body == b"short" else full) + rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type] + assert (pages, len(rows), complete) == ([1, 2, 3], 45, True) + + pages.clear() + rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] + assert (pages, complete) == ([1, 2], False) + + +def test_provider_of() -> None: + assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" + assert search.provider_of("adzuna:berlin") == "adzuna" + assert search.provider_of("indeed:x") == "" + assert search.provider_of("jobindex:") == "" + + +# ------------------------------------------------------------------- pull + +class FakeConfig: + sources: ClassVar[dict[str, object]] = {} + searches: ClassVar[list[SearchSeed]] = [] + path = None + + +def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None: + def fake(client: object, params: dict[str, Any], + keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: + return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København", + "country": "DK", "remote": None, "url": f"https://x.example/{i}", + "posted": None, "description": "", "salary_min": None, + "salary_max": None, "currency": ""} for i in ids], complete + + monkeypatch.setitem(search._SEARCHES, "jobindex", fake) + + +def _pull(store: Store) -> dict[str, Any]: + batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] + for batch in batches: + pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + + class Tally: + results: ClassVar[list[dict[str, Any]]] = [] + + return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type] + + +def test_a_complete_search_closes_what_it_stopped_listing( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", + params={"q": "backend", "area": 1}) + _stub_search(monkeypatch, ["h1", "h2"], complete=True) + _pull(store) + roles = {r.source_id: r for r in store.roles()} + assert set(roles) == {"jobindex:h1", "jobindex:h2"} + assert roles["jobindex:h1"].country == "DK" + assert roles["jobindex:h1"].subscription_id is not None + + _stub_search(monkeypatch, ["h2"], complete=True) + result = _pull(store) + assert {r.source_id for r in store.roles()} == {"jobindex:h2"} + assert (result["searches"], result["closed"]) == (1, 1) + + +def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", + params={"q": "backend"}) + _stub_search(monkeypatch, ["h1", "h2"], complete=False) + _pull(store) + _stub_search(monkeypatch, ["h2"], complete=False) + _pull(store) + assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"} + + old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() + with store.transaction() as conn: + conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,)) + _pull(store) + assert {r.source_id for r in store.roles()} == {"jobindex:h2"} + + +def test_seed_searches_imports_once(store: Store) -> None: + config = FakeConfig() + config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex", + params={"q": "backend", "area": 1})] + assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type] + sub = store.subscriptions(source="search")[0] + assert (sub.target, sub.params) == ("jobindex:copenhagen-backend", + {"q": "backend", "area": 1}) + store.unsubscribe(sub.id) + with store.transaction() as conn: + conn.execute("DELETE FROM subscriptions") + assert pull.seed_searches(config, store) == [] # type: ignore[arg-type] + + +# -------------------------------------------------------------------- cli + +def test_prune_unsubscribes_boards_with_no_role_in_the_region( + store: Store, capsys: pytest.CaptureFixture[str]) -> None: + def board(slug: str, *places: str) -> int: + sub = store.add_subscription("ats", f"greenhouse:{slug}", label=slug, company=slug, + board="greenhouse") + for n, place in enumerate(places): + it = Item(source_id=f"greenhouse:{slug}:{n}", location="https://x.example/1", + title=f"Engineer {n}", body="", raw={"company": slug, "place": place}) + it.source = "ats" + store.upsert_role(it, sub) + return sub.id + + us = board("usonly", "Boston, MA", "Remote - US") + both = board("both", "San Francisco, CA", "London, UK") + vague = board("vague", "Remote", "Hybrid") + + cli.prune_companies(store, "europe", apply=False) + assert "usonly" in capsys.readouterr().out + assert len(store.subscriptions(source="ats", fetchable=True)) == 3 + + cli.prune_companies(store, "europe", apply=True) + fetchable = {s.id for s in store.subscriptions(source="ats", fetchable=True)} + assert fetchable == {both, vague} + assert all(r.subscription_id != us for r in store.roles()) + + cli.prune_companies(store, "DK,EU", apply=True) + assert {s.id for s in store.subscriptions(source="ats", fetchable=True)} == {vague} + + +def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_search(monkeypatch, ["h1"], complete=True) + source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", + "params": {"q": "x"}}]}) + items = list(source.fetch()) + assert [i.source_id for i in items] == ["jobindex:h1"] + assert items[0].raw["search"] == "jobindex:cph" + assert source.state["search:complete:jobindex:cph"] == "1" + assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'