Find roles by search, not only by company: Jobindex and Adzuna searches
The ats boards only read companies someone listed, and the seed list is US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs saved queries on boards that span companies, pulled in the same task as the boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU, free key, built to its documented API). Jobnet and Jooble were checked and left out. `country` is now filled from the board's location line by a lookup, with EU for Europe-wide roles, so country gates work without extract. `app companies prune --in` unsubscribes boards with no open role in a region. The starter agents and searches are Copenhagen and Europe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bfab68c878
commit
047f7cd8ad
17 files changed
+1293
-72
No files matched your search
+124
-1
@@ -3,7 +3,8 @@
|
||||
app serve [--host H] [--port P] [--reload]
|
||||
app job pull|extract|embed|score [--payload JSON]
|
||||
app worker [--workers http=8,cpu=1] [--until-idle]
|
||||
app companies seed|add|check
|
||||
app companies seed|add|check|prune
|
||||
app searches add|list|remove
|
||||
app db init
|
||||
|
||||
`app job` queues the work on a queue of its own and drains it here, so a
|
||||
@@ -72,6 +73,32 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
|
||||
add.add_argument("name")
|
||||
companies_sub.add_parser("check", help="report boards that do not answer")
|
||||
prune = companies_sub.add_parser(
|
||||
"prune", help="unsubscribe every board with no open role in the region")
|
||||
prune.add_argument("--in", dest="region", default="europe",
|
||||
help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)")
|
||||
prune.add_argument("--apply", action="store_true",
|
||||
help="unsubscribe and close their roles; without it, only list them")
|
||||
|
||||
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
||||
searches_sub = searches.add_subparsers(dest="action", required=True)
|
||||
search_add = searches_sub.add_parser(
|
||||
"add", help="run a search once, then save it for every pull")
|
||||
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
|
||||
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
|
||||
search_add.add_argument("--q", required=True,
|
||||
help="the query, in the provider's own syntax")
|
||||
search_add.add_argument("--area", type=int, default=0,
|
||||
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
|
||||
search_add.add_argument("--country", default="",
|
||||
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
|
||||
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
|
||||
search_add.add_argument("--pages", type=int, default=0,
|
||||
help="result pages per pull (default 5, at most 10)")
|
||||
searches_sub.add_parser("list", help="every saved search and its open roles")
|
||||
search_remove = searches_sub.add_parser(
|
||||
"remove", help="stop running a search; its roles stay until they go stale")
|
||||
search_remove.add_argument("name")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
@@ -91,6 +118,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return run_worker(args)
|
||||
if args.command == "companies":
|
||||
return run_companies(args)
|
||||
if args.command == "searches":
|
||||
return run_searches(args)
|
||||
return init_db(args)
|
||||
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
@@ -205,6 +234,8 @@ def run_companies(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
if args.action == "add":
|
||||
return add_company(store, args.board, args.name)
|
||||
if args.action == "prune":
|
||||
return prune_companies(store, args.region, apply=args.apply)
|
||||
return check_companies(store)
|
||||
|
||||
|
||||
@@ -293,6 +324,98 @@ def check_companies(store: Store) -> int:
|
||||
return 1 if dead else 0
|
||||
|
||||
|
||||
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
|
||||
"""`app companies prune --in DK,EU`: every subscribed board none of whose
|
||||
open roles is in the region, by `country`, which this fills from `place`
|
||||
first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles,
|
||||
since a board no longer pulled would otherwise keep them open for good.
|
||||
`app companies seed` never brings one back: it skips a company that exists.
|
||||
|
||||
A board whose open roles all name no country is left alone: it says
|
||||
nothing about where that company hires."""
|
||||
from app.models import EUROPE
|
||||
|
||||
filled = store.backfill_country()
|
||||
countries = store.open_countries_by_subscription()
|
||||
wanted = {c.strip().upper() for c in region.split(",") if c.strip()}
|
||||
if "EUROPE" in wanted:
|
||||
wanted = (wanted - {"EUROPE"}) | set(EUROPE)
|
||||
pruned = [(sub, sum(countries[sub.id].values()))
|
||||
for sub in store.subscriptions(source="ats", fetchable=True)
|
||||
if set(countries.get(sub.id, {})) - {""}
|
||||
and not wanted.intersection(countries[sub.id])]
|
||||
closed = 0
|
||||
for sub, count in sorted(pruned, key=lambda p: -p[1]):
|
||||
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
|
||||
if apply:
|
||||
store.unsubscribe(sub.id)
|
||||
closed += store.close_subscription(sub.id)
|
||||
kept = len(store.subscriptions(source="ats", fetchable=True))
|
||||
print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply,
|
||||
"roles_closed": closed, "boards_left": kept}))
|
||||
return 0
|
||||
|
||||
|
||||
def run_searches(args: argparse.Namespace) -> int:
|
||||
from app.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
if args.action == "add":
|
||||
return add_search(config, store, args)
|
||||
if args.action == "remove":
|
||||
sub = store.subscription_for("search", args.name)
|
||||
if sub is None:
|
||||
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
|
||||
return 1
|
||||
store.unsubscribe(sub.id)
|
||||
print(json.dumps({"removed": sub.label}))
|
||||
return 0
|
||||
countries = store.open_countries_by_subscription()
|
||||
for sub in store.subscriptions(source="search"):
|
||||
removed = "" if sub.subscribed and sub.fetch else " (removed)"
|
||||
open_roles = sum(countries.get(sub.id, {}).values())
|
||||
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
|
||||
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
|
||||
return 0
|
||||
|
||||
|
||||
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
|
||||
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
|
||||
--area 1`: run the search once and refuse if it fails or finds nothing, so
|
||||
a typo in a query is found now rather than by an empty tab later."""
|
||||
from app.jobs import search_target
|
||||
from plugins.search import run_search
|
||||
|
||||
params: dict[str, Any] = {"q": args.q}
|
||||
if args.provider == "jobindex" and args.area:
|
||||
params["area"] = args.area
|
||||
if args.provider == "adzuna":
|
||||
params["country"] = args.country.lower()
|
||||
if args.where:
|
||||
params["where"] = args.where
|
||||
if args.pages:
|
||||
params["pages"] = args.pages
|
||||
target = search_target(args.provider, args.name)
|
||||
try:
|
||||
rows, complete = run_search(target, params, dict(config.sources.get("search") or {}))
|
||||
except Exception as exc:
|
||||
print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not rows:
|
||||
print("searches add: the search found nothing; not saved", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
store.add_subscription("search", target, label=args.name, board=args.provider,
|
||||
params=params)
|
||||
except ValueError as exc:
|
||||
print(f"searches add: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({"search": args.name, "target": target, "roles": len(rows),
|
||||
"complete": complete}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from app.storage import LATEST, Store
|
||||
|
||||
Reference in new issue
Block a user