Find roles by search, not only by company: Jobindex and Adzuna searches
The ats boards only read companies someone listed, and the seed list is US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs saved queries on boards that span companies, pulled in the same task as the boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU, free key, built to its documented API). Jobnet and Jooble were checked and left out. `country` is now filled from the board's location line by a lookup, with EU for Europe-wide roles, so country gates work without extract. `app companies prune --in` unsubscribes boards with no open role in a region. The starter agents and searches are Copenhagen and Europe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bfab68c878
commit
047f7cd8ad
17 files changed
+1293
-72
No files matched your search
+124
-1
@@ -3,7 +3,8 @@
|
||||
app serve [--host H] [--port P] [--reload]
|
||||
app job pull|extract|embed|score [--payload JSON]
|
||||
app worker [--workers http=8,cpu=1] [--until-idle]
|
||||
app companies seed|add|check
|
||||
app companies seed|add|check|prune
|
||||
app searches add|list|remove
|
||||
app db init
|
||||
|
||||
`app job` queues the work on a queue of its own and drains it here, so a
|
||||
@@ -72,6 +73,32 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
|
||||
add.add_argument("name")
|
||||
companies_sub.add_parser("check", help="report boards that do not answer")
|
||||
prune = companies_sub.add_parser(
|
||||
"prune", help="unsubscribe every board with no open role in the region")
|
||||
prune.add_argument("--in", dest="region", default="europe",
|
||||
help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)")
|
||||
prune.add_argument("--apply", action="store_true",
|
||||
help="unsubscribe and close their roles; without it, only list them")
|
||||
|
||||
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
||||
searches_sub = searches.add_subparsers(dest="action", required=True)
|
||||
search_add = searches_sub.add_parser(
|
||||
"add", help="run a search once, then save it for every pull")
|
||||
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
|
||||
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
|
||||
search_add.add_argument("--q", required=True,
|
||||
help="the query, in the provider's own syntax")
|
||||
search_add.add_argument("--area", type=int, default=0,
|
||||
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
|
||||
search_add.add_argument("--country", default="",
|
||||
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
|
||||
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
|
||||
search_add.add_argument("--pages", type=int, default=0,
|
||||
help="result pages per pull (default 5, at most 10)")
|
||||
searches_sub.add_parser("list", help="every saved search and its open roles")
|
||||
search_remove = searches_sub.add_parser(
|
||||
"remove", help="stop running a search; its roles stay until they go stale")
|
||||
search_remove.add_argument("name")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
@@ -91,6 +118,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return run_worker(args)
|
||||
if args.command == "companies":
|
||||
return run_companies(args)
|
||||
if args.command == "searches":
|
||||
return run_searches(args)
|
||||
return init_db(args)
|
||||
except (FileNotFoundError, config_mod.ConfigError) as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
@@ -205,6 +234,8 @@ def run_companies(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
if args.action == "add":
|
||||
return add_company(store, args.board, args.name)
|
||||
if args.action == "prune":
|
||||
return prune_companies(store, args.region, apply=args.apply)
|
||||
return check_companies(store)
|
||||
|
||||
|
||||
@@ -293,6 +324,98 @@ def check_companies(store: Store) -> int:
|
||||
return 1 if dead else 0
|
||||
|
||||
|
||||
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
|
||||
"""`app companies prune --in DK,EU`: every subscribed board none of whose
|
||||
open roles is in the region, by `country`, which this fills from `place`
|
||||
first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles,
|
||||
since a board no longer pulled would otherwise keep them open for good.
|
||||
`app companies seed` never brings one back: it skips a company that exists.
|
||||
|
||||
A board whose open roles all name no country is left alone: it says
|
||||
nothing about where that company hires."""
|
||||
from app.models import EUROPE
|
||||
|
||||
filled = store.backfill_country()
|
||||
countries = store.open_countries_by_subscription()
|
||||
wanted = {c.strip().upper() for c in region.split(",") if c.strip()}
|
||||
if "EUROPE" in wanted:
|
||||
wanted = (wanted - {"EUROPE"}) | set(EUROPE)
|
||||
pruned = [(sub, sum(countries[sub.id].values()))
|
||||
for sub in store.subscriptions(source="ats", fetchable=True)
|
||||
if set(countries.get(sub.id, {})) - {""}
|
||||
and not wanted.intersection(countries[sub.id])]
|
||||
closed = 0
|
||||
for sub, count in sorted(pruned, key=lambda p: -p[1]):
|
||||
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
|
||||
if apply:
|
||||
store.unsubscribe(sub.id)
|
||||
closed += store.close_subscription(sub.id)
|
||||
kept = len(store.subscriptions(source="ats", fetchable=True))
|
||||
print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply,
|
||||
"roles_closed": closed, "boards_left": kept}))
|
||||
return 0
|
||||
|
||||
|
||||
def run_searches(args: argparse.Namespace) -> int:
|
||||
from app.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
if args.action == "add":
|
||||
return add_search(config, store, args)
|
||||
if args.action == "remove":
|
||||
sub = store.subscription_for("search", args.name)
|
||||
if sub is None:
|
||||
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
|
||||
return 1
|
||||
store.unsubscribe(sub.id)
|
||||
print(json.dumps({"removed": sub.label}))
|
||||
return 0
|
||||
countries = store.open_countries_by_subscription()
|
||||
for sub in store.subscriptions(source="search"):
|
||||
removed = "" if sub.subscribed and sub.fetch else " (removed)"
|
||||
open_roles = sum(countries.get(sub.id, {}).values())
|
||||
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
|
||||
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
|
||||
return 0
|
||||
|
||||
|
||||
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
|
||||
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
|
||||
--area 1`: run the search once and refuse if it fails or finds nothing, so
|
||||
a typo in a query is found now rather than by an empty tab later."""
|
||||
from app.jobs import search_target
|
||||
from plugins.search import run_search
|
||||
|
||||
params: dict[str, Any] = {"q": args.q}
|
||||
if args.provider == "jobindex" and args.area:
|
||||
params["area"] = args.area
|
||||
if args.provider == "adzuna":
|
||||
params["country"] = args.country.lower()
|
||||
if args.where:
|
||||
params["where"] = args.where
|
||||
if args.pages:
|
||||
params["pages"] = args.pages
|
||||
target = search_target(args.provider, args.name)
|
||||
try:
|
||||
rows, complete = run_search(target, params, dict(config.sources.get("search") or {}))
|
||||
except Exception as exc:
|
||||
print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not rows:
|
||||
print("searches add: the search found nothing; not saved", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
store.add_subscription("search", target, label=args.name, board=args.provider,
|
||||
params=params)
|
||||
except ValueError as exc:
|
||||
print(f"searches add: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({"search": args.name, "target": target, "roles": len(rows),
|
||||
"complete": complete}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from app.storage import LATEST, Store
|
||||
|
||||
+15
-1
@@ -17,7 +17,7 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import yaml
|
||||
from pydantic import Field, ValidationError, model_validator
|
||||
@@ -72,6 +72,17 @@ class AgentSeed(Section):
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
class SearchSeed(Section):
|
||||
"""A saved search as `config.yaml` lists it: one `search` subscription,
|
||||
imported once like the agents. `params` are the provider's own, see
|
||||
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
|
||||
`where` and `pages` for adzuna."""
|
||||
|
||||
name: str
|
||||
provider: Literal["jobindex", "adzuna"]
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class Config(Section):
|
||||
"""The whole file."""
|
||||
|
||||
@@ -101,6 +112,9 @@ class Config(Section):
|
||||
embed: EmbedConfig = Field(default_factory=EmbedConfig)
|
||||
#: Starter agents, imported once into an empty table.
|
||||
agents: list[AgentSeed] = Field(default_factory=list)
|
||||
#: Starter searches, imported once into an empty set of `search`
|
||||
#: subscriptions.
|
||||
searches: list[SearchSeed] = Field(default_factory=list)
|
||||
#: One block per plugin: `enabled`, and whatever that plugin reads.
|
||||
sources: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""The work roledesk queues. Importing this package registers every job, every
|
||||
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||
|
||||
pull.py read every configured plugin and store the roles it yields
|
||||
pull.py read every board and search and store the roles they yield
|
||||
extract.py title and description into the gate columns, on Haiku
|
||||
embed.py one vector per open role, for fit
|
||||
score.py every open role scored for every enabled search agent
|
||||
@@ -14,6 +14,7 @@ from __future__ import annotations
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import cascade, embed, extract, pull, score
|
||||
from app.jobs.pull import search_target, seed_searches
|
||||
from app.jobs.score import seed_agents
|
||||
|
||||
#: What `JobQueue(bootstrap=...)` and `pipeline.worker --bootstrap` are given.
|
||||
@@ -32,4 +33,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
|
||||
|
||||
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score",
|
||||
"seed_agents"]
|
||||
"search_target", "seed_agents", "seed_searches"]
|
||||
+114
-46
@@ -1,28 +1,34 @@
|
||||
"""`pull`: fetch every active `ats` board and store the roles it lists.
|
||||
"""`pull`: fetch every active `ats` board and `search`, and store the roles
|
||||
they list.
|
||||
|
||||
A task, not a single job: ten boards per `pull_batch` job on the `http`
|
||||
resource pool, because a board is one HTTP round trip and that pool is sized
|
||||
for exactly this. The final job's `settle` then closes every board a
|
||||
A task, not a single job: ten boards or searches per `pull_batch` job on the
|
||||
`http` resource pool, because a board is one HTTP round trip and that pool is
|
||||
sized for exactly this. The final job's `settle` then closes every board a
|
||||
complete fetch of it did not list again — see `close_boards`.
|
||||
|
||||
Closure is read from `self.state`, which the `ats` plugin itself stamps per
|
||||
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
|
||||
than from what each `pull_batch` job happens to return. A batch job that
|
||||
raises partway through still leaves the boards it had already finished
|
||||
stamped, so one bad board in a batch of ten does not cost the other nine
|
||||
their closure.
|
||||
Closure is read from `self.state`, which each plugin itself stamps per target
|
||||
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
|
||||
under `search:`), rather than from what each `pull_batch` job happens to
|
||||
return. A batch job that raises partway through still leaves the boards it had
|
||||
already finished stamped, so one bad board in a batch of ten does not cost the
|
||||
other nine their closure.
|
||||
|
||||
A search closes the same way when its results ran out before its page limit,
|
||||
since it then listed everything that matches. One that hit the limit did not,
|
||||
so its roles close only once no search has seen them for `STALE_DAYS`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, Tally, Task, job
|
||||
|
||||
from app.config import Config
|
||||
from app.models import company_key
|
||||
from app.storage import Store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -34,49 +40,74 @@ BATCH_SIZE = 10
|
||||
|
||||
SETTLE = "app.jobs.pull:close_boards"
|
||||
|
||||
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
||||
SOURCES = ("ats", "search")
|
||||
|
||||
#: How long a search role stays open once no search lists it any more.
|
||||
STALE_DAYS = 21
|
||||
|
||||
#: The `state` key that records the starter searches were imported.
|
||||
SEARCHES_SEEDED = "searches:seeded"
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
||||
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
|
||||
marker `close_boards` will look for, before any batch runs — finders run
|
||||
synchronously, in whoever starts the task, ahead of every job it queues."""
|
||||
subs = store.subscriptions(source="ats", fetchable=True)
|
||||
if not subs:
|
||||
return []
|
||||
store.state["ats:run"] = _now()
|
||||
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
|
||||
for i in range(0, len(subs), BATCH_SIZE)]
|
||||
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
|
||||
source. Stamps the run marker `close_boards` will look for, before any
|
||||
batch runs — finders run synchronously, in whoever starts the task, ahead
|
||||
of every job it queues."""
|
||||
seed_searches(config, store)
|
||||
batches: list[dict[str, Any]] = []
|
||||
for source in SOURCES:
|
||||
subs = store.subscriptions(source=source, fetchable=True)
|
||||
if not subs:
|
||||
continue
|
||||
store.state[f"{source}:run"] = _now()
|
||||
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
|
||||
for i in range(0, len(subs), BATCH_SIZE)]
|
||||
return batches
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
def run_batch(config: Config, store: Store, *, targets: list[str],
|
||||
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Fetch this batch's boards and `upsert_role` every item they list.
|
||||
"""Fetch this batch's boards or searches and `upsert_role` every item
|
||||
they list.
|
||||
|
||||
`targets` is keyword-only without a default, which is what makes it a
|
||||
required payload key to the queue: the Jobs tab offers no button for a
|
||||
batch that could only fail without one.
|
||||
|
||||
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
|
||||
reads `options[subscription_field]`, so handing it only this batch's
|
||||
entries is what keeps one job from re-fetching every other job's boards.
|
||||
Builds the plugin scoped to just these targets — `Source.targets()` reads
|
||||
`options[subscription_field]`, so handing it only this batch's entries is
|
||||
what keeps one job from re-fetching every other job's boards.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.search import SearchSource
|
||||
|
||||
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
|
||||
options = dict(config.sources.get("ats") or {})
|
||||
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
|
||||
for target in targets if target in subs]
|
||||
source = ATSSource(options)
|
||||
source.state = store.state
|
||||
subs = {s.target: s for s in store.subscriptions(source=source) if s.target in targets}
|
||||
options = dict(config.sources.get(source) or {})
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
|
||||
for t in targets if t in subs]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
|
||||
for t in targets if t in subs]
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
|
||||
stored = 0
|
||||
for item in source.fetch():
|
||||
item.source = "ats"
|
||||
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
if source == "search":
|
||||
target = str(item.raw.get("search", ""))
|
||||
else:
|
||||
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
||||
store.upsert_role(item, subs.get(target))
|
||||
stored += 1
|
||||
return {"roles": stored}
|
||||
@@ -91,20 +122,57 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
had already parsed stamped — see the module docstring. A board whose
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed.
|
||||
whole batch job never got to it — is never closed. A search closes by its
|
||||
list only when that list was complete, and otherwise by `STALE_DAYS`.
|
||||
"""
|
||||
run_marker = store.state.get("ats:run", "")
|
||||
answered = failed = closed = 0
|
||||
for sub in store.subscriptions(source="ats", fetchable=True):
|
||||
stamp = store.state.get(f"ats:ok:{sub.target}", "")
|
||||
if stamp and stamp == run_marker:
|
||||
answered += 1
|
||||
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed("ats", seen_ids, sub.id)
|
||||
else:
|
||||
failed += 1
|
||||
answered = searched = failed = closed = 0
|
||||
for source in SOURCES:
|
||||
run_marker = store.state.get(f"{source}:run", "")
|
||||
for sub in store.subscriptions(source=source, fetchable=True):
|
||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||
if not (stamp and stamp == run_marker):
|
||||
failed += 1
|
||||
continue
|
||||
if source == "search":
|
||||
searched += 1
|
||||
if not store.state.get(f"search:complete:{sub.target}", ""):
|
||||
continue
|
||||
else:
|
||||
answered += 1
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if searched:
|
||||
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
|
||||
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
|
||||
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
|
||||
"closed": closed}
|
||||
|
||||
|
||||
def search_target(provider: str, name: str) -> str:
|
||||
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
|
||||
The query itself lives in `params`, so editing it keeps the target."""
|
||||
return f"{provider}:{company_key(name)}"
|
||||
|
||||
|
||||
def seed_searches(config: Config, store: Store) -> list[str]:
|
||||
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
|
||||
and only while there are none: the same rule as `seed_agents`."""
|
||||
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
|
||||
return []
|
||||
added = []
|
||||
for seed in config.searches:
|
||||
try:
|
||||
store.add_subscription("search", search_target(seed.provider, seed.name),
|
||||
label=seed.name, board=seed.provider, params=seed.params)
|
||||
except ValueError as exc:
|
||||
log.warning("searches: %s", exc)
|
||||
continue
|
||||
added.append(seed.name)
|
||||
if added:
|
||||
store.state[SEARCHES_SEEDED] = store.now()
|
||||
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
|
||||
return added
|
||||
|
||||
|
||||
def pull_task() -> Task:
|
||||
|
||||
+2
-1
@@ -45,9 +45,10 @@ def create_app(config: Config | None = None) -> FastAPI:
|
||||
# auth add their own tables to it.
|
||||
store = Store(config.database)
|
||||
jobs.seed_agents(config, store)
|
||||
jobs.seed_searches(config, store)
|
||||
|
||||
app = FastAPI(title="roledesk", docs_url="/api/docs",
|
||||
summary="ATS boards in, roles scored per search agent, "
|
||||
summary="ATS boards and job searches in, roles scored per search agent, "
|
||||
"applications tracked.")
|
||||
app.state.config = config
|
||||
app.state.store = store
|
||||
|
||||
+129
-1
@@ -34,7 +34,7 @@ _GENDER = re.compile(
|
||||
#: board and "celonis" on another share a key.
|
||||
_LEGAL = re.compile(
|
||||
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
|
||||
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|aps))+\.?$",
|
||||
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
@@ -64,6 +64,134 @@ def company_key(company: str) -> str:
|
||||
return _fold(name).replace(" ", "-")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- geography
|
||||
|
||||
#: The EU, the EEA, the UK and Switzerland: what "a European role" means to a
|
||||
#: gate on `country`. `EU` is ISO's reserved code, used here for a role open
|
||||
#: anywhere in Europe ("Remote - Europe", "EMEA") that names no one country.
|
||||
EUROPE: tuple[str, ...] = (
|
||||
"AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
|
||||
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
|
||||
"IS", "LI", "NO", "GB", "CH", "EU",
|
||||
)
|
||||
|
||||
#: Folded names a board writes in its location line, to an ISO code: every
|
||||
#: European country in English and its own language, the cities the boards
|
||||
#: name most, and the non-European places that fill the seed boards, so those
|
||||
#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
|
||||
_PLACES: dict[str, str] = {
|
||||
**dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
|
||||
**dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
|
||||
**dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
|
||||
"ghent"), "BE"),
|
||||
**dict.fromkeys(("bulgaria", "sofia"), "BG"),
|
||||
**dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
|
||||
**dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
|
||||
**dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
|
||||
# Folding keeps ø and æ, which are letters of their own rather than an o
|
||||
# or an a with a mark, so København is listed beside Kobenhavn.
|
||||
**dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
|
||||
"århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
|
||||
"lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
|
||||
"bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
|
||||
"birkerød"), "DK"),
|
||||
**dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
|
||||
**dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
|
||||
**dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
|
||||
"bordeaux", "lille"), "FR"),
|
||||
**dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
|
||||
"frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
|
||||
"dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
|
||||
"heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
|
||||
**dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
|
||||
**dict.fromkeys(("hungary", "budapest"), "HU"),
|
||||
**dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
|
||||
**dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
|
||||
"torino", "bologna", "florence"), "IT"),
|
||||
**dict.fromkeys(("latvia", "riga"), "LV"),
|
||||
**dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
|
||||
**dict.fromkeys(("luxembourg",), "LU"),
|
||||
**dict.fromkeys(("malta", "valletta"), "MT"),
|
||||
**dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
|
||||
"rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
||||
"delft"), "NL"),
|
||||
**dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
|
||||
"gdansk", "poznan", "lodz"), "PL"),
|
||||
**dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
|
||||
**dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
|
||||
"RO"),
|
||||
**dict.fromkeys(("slovakia", "bratislava"), "SK"),
|
||||
**dict.fromkeys(("slovenia", "ljubljana"), "SI"),
|
||||
**dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
|
||||
"malaga", "bilbao"), "ES"),
|
||||
**dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
|
||||
"uppsala"), "SE"),
|
||||
**dict.fromkeys(("iceland", "reykjavik"), "IS"),
|
||||
**dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
|
||||
**dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
|
||||
"wales", "northern ireland", "london", "manchester", "edinburgh",
|
||||
"glasgow", "bristol", "cambridge uk", "oxford", "leeds",
|
||||
"birmingham uk", "belfast"), "GB"),
|
||||
**dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
|
||||
"basel", "lausanne", "bern", "zug"), "CH"),
|
||||
**dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
|
||||
"nyc", "san francisco", "seattle", "boston", "chicago", "austin",
|
||||
"los angeles", "palo alto", "mountain view", "san mateo",
|
||||
"foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
|
||||
"costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
|
||||
"houston", "phoenix", "tempe", "portland", "salt lake city",
|
||||
"pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
|
||||
"nashville", "minneapolis"), "US"),
|
||||
**dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
|
||||
"waterloo"), "CA"),
|
||||
**dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
|
||||
"chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
||||
**dict.fromkeys(("singapore",), "SG"),
|
||||
**dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
|
||||
**dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
|
||||
**dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
||||
**dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
|
||||
**dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
||||
**dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
|
||||
**dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
|
||||
}
|
||||
|
||||
#: Keyed the way a place is folded before it is looked up, so a name can be
|
||||
#: written as it is spelt.
|
||||
_PLACES = {_fold(name): code for name, code in _PLACES.items()}
|
||||
|
||||
#: The longest name first, so "new york city" wins over "york" and "northern
|
||||
#: ireland" over "ireland".
|
||||
_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
|
||||
|
||||
#: US state codes after a comma, the way US boards write a city: "Boston, MA".
|
||||
#: Read only when no name matched, since "DE" and "IN" are countries too.
|
||||
_US_STATE = re.compile(
|
||||
r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
|
||||
r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
|
||||
r"\b")
|
||||
|
||||
|
||||
def country_of(place: str) -> str:
|
||||
"""The one country a board's location line names, as an ISO code, or ''
|
||||
when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
|
||||
US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
|
||||
is EU, and "London, EMEA" is GB: a country beats the region it is in."""
|
||||
folded = f" {_fold(place or '')} "
|
||||
found: set[str] = set()
|
||||
for name in _PLACE_NAMES:
|
||||
needle = f" {name} "
|
||||
if needle in folded:
|
||||
found.add(_PLACES[name])
|
||||
folded = folded.replace(needle, " ")
|
||||
if not found and _US_STATE.search(place or ""):
|
||||
# Only when nothing else named a country: "Berlin, DE" is Germany.
|
||||
found.add("US")
|
||||
if len(found) > 1:
|
||||
found.discard("EU")
|
||||
return found.pop() if len(found) == 1 else ""
|
||||
|
||||
|
||||
# --------------------------------------------------------------- vocabulary
|
||||
|
||||
Remote = Literal["", "remote", "hybrid", "onsite"]
|
||||
|
||||
+40
-3
@@ -22,8 +22,8 @@ import numpy as np
|
||||
from desk.models import Item
|
||||
from app.models import (
|
||||
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
|
||||
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, place_key,
|
||||
title_key,
|
||||
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, country_of,
|
||||
place_key, title_key,
|
||||
)
|
||||
from app.storage import schema
|
||||
|
||||
@@ -212,7 +212,8 @@ class Store:
|
||||
`first_seen` and `created_at` are written on insert only; `last_seen` on
|
||||
every call, and a role seen again is open again. The gate columns come
|
||||
from `item.raw` under their own names, with `company` falling back to
|
||||
the subscription's; an empty value never overwrites a filled column.
|
||||
the subscription's and `country` to what `place` names; an empty value
|
||||
never overwrites a filled column.
|
||||
"""
|
||||
if not item.source:
|
||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||
@@ -241,6 +242,7 @@ class Store:
|
||||
"created_at": stamp,
|
||||
}
|
||||
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
|
||||
row["country"] = row["country"] or country_of(place)
|
||||
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
|
||||
|
||||
keep_filled = ("company", "company_key", "place_key", "remote", "description",
|
||||
@@ -341,6 +343,31 @@ class Store:
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
||||
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
|
||||
|
||||
def close_unseen(self, source: str, before: str) -> int:
|
||||
"""Close every open role of `source` last seen before `before`. For a
|
||||
source whose fetch is never a complete list, a search, where a role
|
||||
dropping out of one run's results is not the role closing."""
|
||||
return self._write(
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND closed_at IS NULL AND last_seen < ?""",
|
||||
(self.now(), source, before))
|
||||
|
||||
def close_subscription(self, subscription_id: int) -> int:
|
||||
"""Close every open role of one subscription, for a board that is no
|
||||
longer fetched: nothing would ever close them otherwise."""
|
||||
return self._write(
|
||||
"UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL",
|
||||
(self.now(), subscription_id))
|
||||
|
||||
def backfill_country(self) -> int:
|
||||
"""Fill `country` from `place` on every role that has none, the way
|
||||
`upsert_role` does for a new one. Returns how many it filled."""
|
||||
rows = self._rows("SELECT uid, place FROM roles WHERE country = '' AND place != ''")
|
||||
filled = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))]
|
||||
with self.transaction() as conn:
|
||||
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", filled)
|
||||
return len(filled)
|
||||
|
||||
# -------------------------------------------------------------- extract
|
||||
|
||||
def extract_pending(self) -> int:
|
||||
@@ -614,6 +641,16 @@ class Store:
|
||||
"SELECT company_key, COUNT(*) AS n FROM roles "
|
||||
"WHERE closed_at IS NULL GROUP BY company_key")}
|
||||
|
||||
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
|
||||
"""Per subscription id, its open roles counted by `country`."""
|
||||
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
|
||||
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
||||
GROUP BY subscription_id, country""")
|
||||
out: dict[int, dict[str, int]] = {}
|
||||
for r in rows:
|
||||
out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"]
|
||||
return out
|
||||
|
||||
def company_boards(self) -> dict[str, str]:
|
||||
"""`company_key -> ats host`, read off each company's own
|
||||
subscription. `Subscription.company` is the display name it and the
|
||||
|
||||
Reference in new issue
Block a user