Find roles by search, not only by company: Jobindex and Adzuna searches

The ats boards only read companies someone listed, and the seed list is
US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs
saved queries on boards that span companies, pulled in the same task as the
boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU,
free key, built to its documented API). Jobnet and Jooble were checked and
left out.

`country` is now filled from the board's location line by a lookup, with EU
for Europe-wide roles, so country gates work without extract. `app companies
prune --in` unsubscribes boards with no open role in a region. The starter
agents and searches are Copenhagen and Europe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 19:55:50 +02:00
1 parent bfab68c878
commit 047f7cd8ad
17 files changed
+1293 -72

No files matched your search

+124 -1
View File
@@ -3,7 +3,8 @@
app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle]
app companies seed|add|check
app companies seed|add|check|prune
app searches add|list|remove
app db init
`app job` queues the work on a queue of its own and drains it here, so a
@@ -72,6 +73,32 @@ def build_parser() -> argparse.ArgumentParser:
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
add.add_argument("name")
companies_sub.add_parser("check", help="report boards that do not answer")
prune = companies_sub.add_parser(
"prune", help="unsubscribe every board with no open role in the region")
prune.add_argument("--in", dest="region", default="europe",
help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)")
prune.add_argument("--apply", action="store_true",
help="unsubscribe and close their roles; without it, only list them")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
search_add = searches_sub.add_parser(
"add", help="run a search once, then save it for every pull")
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
search_add.add_argument("--q", required=True,
help="the query, in the provider's own syntax")
search_add.add_argument("--area", type=int, default=0,
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
search_add.add_argument("--country", default="",
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
search_add.add_argument("--pages", type=int, default=0,
help="result pages per pull (default 5, at most 10)")
searches_sub.add_parser("list", help="every saved search and its open roles")
search_remove = searches_sub.add_parser(
"remove", help="stop running a search; its roles stay until they go stale")
search_remove.add_argument("name")
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
@@ -91,6 +118,8 @@ def main(argv: list[str] | None = None) -> int:
return run_worker(args)
if args.command == "companies":
return run_companies(args)
if args.command == "searches":
return run_searches(args)
return init_db(args)
except (FileNotFoundError, config_mod.ConfigError) as exc:
print(exc, file=sys.stderr)
@@ -205,6 +234,8 @@ def run_companies(args: argparse.Namespace) -> int:
return 0
if args.action == "add":
return add_company(store, args.board, args.name)
if args.action == "prune":
return prune_companies(store, args.region, apply=args.apply)
return check_companies(store)
@@ -293,6 +324,98 @@ def check_companies(store: Store) -> int:
return 1 if dead else 0
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
"""`app companies prune --in DK,EU`: every subscribed board none of whose
open roles is in the region, by `country`, which this fills from `place`
first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles,
since a board no longer pulled would otherwise keep them open for good.
`app companies seed` never brings one back: it skips a company that exists.
A board whose open roles all name no country is left alone: it says
nothing about where that company hires."""
from app.models import EUROPE
filled = store.backfill_country()
countries = store.open_countries_by_subscription()
wanted = {c.strip().upper() for c in region.split(",") if c.strip()}
if "EUROPE" in wanted:
wanted = (wanted - {"EUROPE"}) | set(EUROPE)
pruned = [(sub, sum(countries[sub.id].values()))
for sub in store.subscriptions(source="ats", fetchable=True)
if set(countries.get(sub.id, {})) - {""}
and not wanted.intersection(countries[sub.id])]
closed = 0
for sub, count in sorted(pruned, key=lambda p: -p[1]):
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
if apply:
store.unsubscribe(sub.id)
closed += store.close_subscription(sub.id)
kept = len(store.subscriptions(source="ats", fetchable=True))
print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply,
"roles_closed": closed, "boards_left": kept}))
return 0
def run_searches(args: argparse.Namespace) -> int:
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "add":
return add_search(config, store, args)
if args.action == "remove":
sub = store.subscription_for("search", args.name)
if sub is None:
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
return 1
store.unsubscribe(sub.id)
print(json.dumps({"removed": sub.label}))
return 0
countries = store.open_countries_by_subscription()
for sub in store.subscriptions(source="search"):
removed = "" if sub.subscribed and sub.fetch else " (removed)"
open_roles = sum(countries.get(sub.id, {}).values())
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
return 0
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
--area 1`: run the search once and refuse if it fails or finds nothing, so
a typo in a query is found now rather than by an empty tab later."""
from app.jobs import search_target
from plugins.search import run_search
params: dict[str, Any] = {"q": args.q}
if args.provider == "jobindex" and args.area:
params["area"] = args.area
if args.provider == "adzuna":
params["country"] = args.country.lower()
if args.where:
params["where"] = args.where
if args.pages:
params["pages"] = args.pages
target = search_target(args.provider, args.name)
try:
rows, complete = run_search(target, params, dict(config.sources.get("search") or {}))
except Exception as exc:
print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr)
return 1
if not rows:
print("searches add: the search found nothing; not saved", file=sys.stderr)
return 1
try:
store.add_subscription("search", target, label=args.name, board=args.provider,
params=params)
except ValueError as exc:
print(f"searches add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"search": args.name, "target": target, "roles": len(rows),
"complete": complete}, ensure_ascii=False))
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from app.storage import LATEST, Store