Sources tab: searches, company boards and the prune by region, in the app
What a pull reads can now be changed on a page instead of only the command line: try a search and see its first ten results before saving it, edit or remove one, follow a company's ATS board, switch any board or search on or off, and preview and apply a prune by region. Each row shows its open roles and the countries they are in. The page and `app companies`/`app searches` call the same functions in app/jobs/sources.py. Switching a target off closes its open roles, and the pull's settle closes them again if a batch in flight reopened them. Roles pulled before `country` existed are backfilled at startup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
047f7cd8ad
commit
6b40702a9a
16 files changed
+1068
-92
No files matched your search
+34
-78
@@ -83,7 +83,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
||||
searches_sub = searches.add_subparsers(dest="action", required=True)
|
||||
search_add = searches_sub.add_parser(
|
||||
"add", help="run a search once, then save it for every pull")
|
||||
"add", help="run a search once, then save it for every pull (or update it)")
|
||||
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
|
||||
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
|
||||
search_add.add_argument("--q", required=True,
|
||||
@@ -97,7 +97,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="result pages per pull (default 5, at most 10)")
|
||||
searches_sub.add_parser("list", help="every saved search and its open roles")
|
||||
search_remove = searches_sub.add_parser(
|
||||
"remove", help="stop running a search; its roles stay until they go stale")
|
||||
"remove", help="stop running a search and close its open roles")
|
||||
search_remove.add_argument("name")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
@@ -275,34 +275,15 @@ def add_company(store: Store, board: str, name: str) -> int:
|
||||
"""`app companies add greenhouse:acme "Acme"`: fetch the board first
|
||||
and refuse if it does not answer, is not a known host, or the company
|
||||
already exists. Port of seedgen's `-add`."""
|
||||
from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target
|
||||
from app import jobs
|
||||
from app.models import company_key
|
||||
|
||||
if not is_known_target(board):
|
||||
print(f"companies add: {board!r} is not host:slug for a known ATS host",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
key = company_key(name)
|
||||
if store.company(key) is not None:
|
||||
print(f"companies add: {key!r} already exists", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
rows = fetch_board(board)
|
||||
except Exception as exc:
|
||||
print(f"companies add: {board} did not answer: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not rows:
|
||||
print(f"companies add: {board} answered but lists no roles; nothing added",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
host, _, _slug = board.partition(":")
|
||||
store.add_company(name, key=key, careers_url=careers_url_of(board))
|
||||
try:
|
||||
store.add_subscription("ats", board, label=name, company=name, board=host)
|
||||
sub, found = jobs.sources.add_board(store, board, name)
|
||||
except ValueError as exc:
|
||||
print(f"companies add: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({"key": key, "board": board, "roles": len(rows)}))
|
||||
print(json.dumps({"key": company_key(sub.company), "board": sub.target, "roles": found}))
|
||||
return 0
|
||||
|
||||
|
||||
@@ -325,38 +306,29 @@ def check_companies(store: Store) -> int:
|
||||
|
||||
|
||||
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
|
||||
"""`app companies prune --in DK,EU`: every subscribed board none of whose
|
||||
open roles is in the region, by `country`, which this fills from `place`
|
||||
first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles,
|
||||
since a board no longer pulled would otherwise keep them open for good.
|
||||
`app companies seed` never brings one back: it skips a company that exists.
|
||||
"""`app companies prune --in DK,EU`: every fetched board none of whose open
|
||||
roles is in the region, which is `europe` unless named. `--apply`
|
||||
switches them off, which closes their open roles. `app companies seed`
|
||||
never brings one back: it skips a company that exists. See
|
||||
`app.jobs.sources.prune_plan` for which boards count."""
|
||||
from app import jobs
|
||||
|
||||
A board whose open roles all name no country is left alone: it says
|
||||
nothing about where that company hires."""
|
||||
from app.models import EUROPE
|
||||
|
||||
filled = store.backfill_country()
|
||||
countries = store.open_countries_by_subscription()
|
||||
wanted = {c.strip().upper() for c in region.split(",") if c.strip()}
|
||||
if "EUROPE" in wanted:
|
||||
wanted = (wanted - {"EUROPE"}) | set(EUROPE)
|
||||
pruned = [(sub, sum(countries[sub.id].values()))
|
||||
for sub in store.subscriptions(source="ats", fetchable=True)
|
||||
if set(countries.get(sub.id, {})) - {""}
|
||||
and not wanted.intersection(countries[sub.id])]
|
||||
closed = 0
|
||||
for sub, count in sorted(pruned, key=lambda p: -p[1]):
|
||||
try:
|
||||
filled, plan = jobs.sources.prune_plan(store, region)
|
||||
except ValueError as exc:
|
||||
print(f"companies prune: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
for sub, count in plan:
|
||||
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
|
||||
if apply:
|
||||
store.unsubscribe(sub.id)
|
||||
closed += store.close_subscription(sub.id)
|
||||
kept = len(store.subscriptions(source="ats", fetchable=True))
|
||||
print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply,
|
||||
"roles_closed": closed, "boards_left": kept}))
|
||||
result: dict[str, Any] = {"countries_filled": filled, "pruned": len(plan), "applied": apply}
|
||||
if apply:
|
||||
result |= jobs.sources.prune(store, region)
|
||||
print(json.dumps(result))
|
||||
return 0
|
||||
|
||||
|
||||
def run_searches(args: argparse.Namespace) -> int:
|
||||
from app import jobs
|
||||
from app.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
@@ -368,12 +340,12 @@ def run_searches(args: argparse.Namespace) -> int:
|
||||
if sub is None:
|
||||
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
|
||||
return 1
|
||||
store.unsubscribe(sub.id)
|
||||
print(json.dumps({"removed": sub.label}))
|
||||
_, closed = jobs.sources.remove_search(store, sub.id)
|
||||
print(json.dumps({"removed": sub.label, "roles_closed": closed}))
|
||||
return 0
|
||||
countries = store.open_countries_by_subscription()
|
||||
for sub in store.subscriptions(source="search"):
|
||||
removed = "" if sub.subscribed and sub.fetch else " (removed)"
|
||||
removed = "" if sub.subscribed and sub.fetch else " (off)"
|
||||
open_roles = sum(countries.get(sub.id, {}).values())
|
||||
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
|
||||
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
|
||||
@@ -382,36 +354,20 @@ def run_searches(args: argparse.Namespace) -> int:
|
||||
|
||||
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
|
||||
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
|
||||
--area 1`: run the search once and refuse if it fails or finds nothing, so
|
||||
a typo in a query is found now rather than by an empty tab later."""
|
||||
from app.jobs import search_target
|
||||
from plugins.search import run_search
|
||||
--area 1`: run the search once and save it only if it finds something.
|
||||
A search of the same name is updated in place."""
|
||||
from app import jobs
|
||||
|
||||
params: dict[str, Any] = {"q": args.q}
|
||||
if args.provider == "jobindex" and args.area:
|
||||
params["area"] = args.area
|
||||
if args.provider == "adzuna":
|
||||
params["country"] = args.country.lower()
|
||||
if args.where:
|
||||
params["where"] = args.where
|
||||
if args.pages:
|
||||
params["pages"] = args.pages
|
||||
target = search_target(args.provider, args.name)
|
||||
try:
|
||||
rows, complete = run_search(target, params, dict(config.sources.get("search") or {}))
|
||||
except Exception as exc:
|
||||
print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not rows:
|
||||
print("searches add: the search found nothing; not saved", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
store.add_subscription("search", target, label=args.name, board=args.provider,
|
||||
params=params)
|
||||
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
|
||||
country=args.country, where=args.where,
|
||||
pages=args.pages)
|
||||
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
|
||||
params)
|
||||
except ValueError as exc:
|
||||
print(f"searches add: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({"search": args.name, "target": target, "roles": len(rows),
|
||||
print(json.dumps({"search": sub.label, "target": sub.target, "roles": found,
|
||||
"complete": complete}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
Reference in new issue
Block a user