Find roles by search, not only by company: Jobindex and Adzuna searches

The ats boards only read companies someone listed, and the seed list is
US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs
saved queries on boards that span companies, pulled in the same task as the
boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU,
free key, built to its documented API). Jobnet and Jooble were checked and
left out.

`country` is now filled from the board's location line by a lookup, with EU
for Europe-wide roles, so country gates work without extract. `app companies
prune --in` unsubscribes boards with no open role in a region. The starter
agents and searches are Copenhagen and Europe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 19:55:50 +02:00
1 parent bfab68c878
commit 047f7cd8ad
17 files changed
+1293 -72

No files matched your search

+124 -1
View File
@@ -3,7 +3,8 @@
app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle]
app companies seed|add|check
app companies seed|add|check|prune
app searches add|list|remove
app db init
`app job` queues the work on a queue of its own and drains it here, so a
@@ -72,6 +73,32 @@ def build_parser() -> argparse.ArgumentParser:
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
add.add_argument("name")
companies_sub.add_parser("check", help="report boards that do not answer")
prune = companies_sub.add_parser(
"prune", help="unsubscribe every board with no open role in the region")
prune.add_argument("--in", dest="region", default="europe",
help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)")
prune.add_argument("--apply", action="store_true",
help="unsubscribe and close their roles; without it, only list them")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
search_add = searches_sub.add_parser(
"add", help="run a search once, then save it for every pull")
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
search_add.add_argument("--q", required=True,
help="the query, in the provider's own syntax")
search_add.add_argument("--area", type=int, default=0,
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
search_add.add_argument("--country", default="",
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
search_add.add_argument("--pages", type=int, default=0,
help="result pages per pull (default 5, at most 10)")
searches_sub.add_parser("list", help="every saved search and its open roles")
search_remove = searches_sub.add_parser(
"remove", help="stop running a search; its roles stay until they go stale")
search_remove.add_argument("name")
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
@@ -91,6 +118,8 @@ def main(argv: list[str] | None = None) -> int:
return run_worker(args)
if args.command == "companies":
return run_companies(args)
if args.command == "searches":
return run_searches(args)
return init_db(args)
except (FileNotFoundError, config_mod.ConfigError) as exc:
print(exc, file=sys.stderr)
@@ -205,6 +234,8 @@ def run_companies(args: argparse.Namespace) -> int:
return 0
if args.action == "add":
return add_company(store, args.board, args.name)
if args.action == "prune":
return prune_companies(store, args.region, apply=args.apply)
return check_companies(store)
@@ -293,6 +324,98 @@ def check_companies(store: Store) -> int:
return 1 if dead else 0
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
"""`app companies prune --in DK,EU`: every subscribed board none of whose
open roles is in the region, by `country`, which this fills from `place`
first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles,
since a board no longer pulled would otherwise keep them open for good.
`app companies seed` never brings one back: it skips a company that exists.
A board whose open roles all name no country is left alone: it says
nothing about where that company hires."""
from app.models import EUROPE
filled = store.backfill_country()
countries = store.open_countries_by_subscription()
wanted = {c.strip().upper() for c in region.split(",") if c.strip()}
if "EUROPE" in wanted:
wanted = (wanted - {"EUROPE"}) | set(EUROPE)
pruned = [(sub, sum(countries[sub.id].values()))
for sub in store.subscriptions(source="ats", fetchable=True)
if set(countries.get(sub.id, {})) - {""}
and not wanted.intersection(countries[sub.id])]
closed = 0
for sub, count in sorted(pruned, key=lambda p: -p[1]):
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
if apply:
store.unsubscribe(sub.id)
closed += store.close_subscription(sub.id)
kept = len(store.subscriptions(source="ats", fetchable=True))
print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply,
"roles_closed": closed, "boards_left": kept}))
return 0
def run_searches(args: argparse.Namespace) -> int:
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "add":
return add_search(config, store, args)
if args.action == "remove":
sub = store.subscription_for("search", args.name)
if sub is None:
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
return 1
store.unsubscribe(sub.id)
print(json.dumps({"removed": sub.label}))
return 0
countries = store.open_countries_by_subscription()
for sub in store.subscriptions(source="search"):
removed = "" if sub.subscribed and sub.fetch else " (removed)"
open_roles = sum(countries.get(sub.id, {}).values())
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
return 0
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
--area 1`: run the search once and refuse if it fails or finds nothing, so
a typo in a query is found now rather than by an empty tab later."""
from app.jobs import search_target
from plugins.search import run_search
params: dict[str, Any] = {"q": args.q}
if args.provider == "jobindex" and args.area:
params["area"] = args.area
if args.provider == "adzuna":
params["country"] = args.country.lower()
if args.where:
params["where"] = args.where
if args.pages:
params["pages"] = args.pages
target = search_target(args.provider, args.name)
try:
rows, complete = run_search(target, params, dict(config.sources.get("search") or {}))
except Exception as exc:
print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr)
return 1
if not rows:
print("searches add: the search found nothing; not saved", file=sys.stderr)
return 1
try:
store.add_subscription("search", target, label=args.name, board=args.provider,
params=params)
except ValueError as exc:
print(f"searches add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"search": args.name, "target": target, "roles": len(rows),
"complete": complete}, ensure_ascii=False))
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from app.storage import LATEST, Store
+15 -1
View File
@@ -17,7 +17,7 @@ from __future__ import annotations
import os
from pathlib import Path
from typing import Any
from typing import Any, Literal
import yaml
from pydantic import Field, ValidationError, model_validator
@@ -72,6 +72,17 @@ class AgentSeed(Section):
enabled: bool = True
class SearchSeed(Section):
"""A saved search as `config.yaml` lists it: one `search` subscription,
imported once like the agents. `params` are the provider's own, see
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
`where` and `pages` for adzuna."""
name: str
provider: Literal["jobindex", "adzuna"]
params: dict[str, Any] = Field(default_factory=dict)
class Config(Section):
"""The whole file."""
@@ -101,6 +112,9 @@ class Config(Section):
embed: EmbedConfig = Field(default_factory=EmbedConfig)
#: Starter agents, imported once into an empty table.
agents: list[AgentSeed] = Field(default_factory=list)
#: Starter searches, imported once into an empty set of `search`
#: subscriptions.
searches: list[SearchSeed] = Field(default_factory=list)
#: One block per plugin: `enabled`, and whatever that plugin reads.
sources: dict[str, Any] = Field(default_factory=dict)
+3 -2
View File
@@ -1,7 +1,7 @@
"""The work roledesk queues. Importing this package registers every job, every
task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
pull.py read every configured plugin and store the roles it yields
pull.py read every board and search and store the roles they yield
extract.py title and description into the gate columns, on Haiku
embed.py one vector per open role, for fit
score.py every open role scored for every enabled search agent
@@ -14,6 +14,7 @@ from __future__ import annotations
from typing import Any
from app.jobs import cascade, embed, extract, pull, score
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
#: What `JobQueue(bootstrap=...)` and `pipeline.worker --bootstrap` are given.
@@ -32,4 +33,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score",
"seed_agents"]
"search_target", "seed_agents", "seed_searches"]
+114 -46
View File
@@ -1,28 +1,34 @@
"""`pull`: fetch every active `ats` board and store the roles it lists.
"""`pull`: fetch every active `ats` board and `search`, and store the roles
they list.
A task, not a single job: ten boards per `pull_batch` job on the `http`
resource pool, because a board is one HTTP round trip and that pool is sized
for exactly this. The final job's `settle` then closes every board a
A task, not a single job: ten boards or searches per `pull_batch` job on the
`http` resource pool, because a board is one HTTP round trip and that pool is
sized for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which the `ats` plugin itself stamps per
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
than from what each `pull_batch` job happens to return. A batch job that
raises partway through still leaves the boards it had already finished
stamped, so one bad board in a batch of ten does not cost the other nine
their closure.
Closure is read from `self.state`, which each plugin itself stamps per target
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
under `search:`), rather than from what each `pull_batch` job happens to
return. A batch job that raises partway through still leaves the boards it had
already finished stamped, so one bad board in a batch of ten does not cost the
other nine their closure.
A search closes the same way when its results ran out before its page limit,
since it then listed everything that matches. One that hit the limit did not,
so its roles close only once no search has seen them for `STALE_DAYS`.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.models import company_key
from app.storage import Store
log = logging.getLogger(__name__)
@@ -34,49 +40,74 @@ BATCH_SIZE = 10
SETTLE = "app.jobs.pull:close_boards"
#: The plugins a pull reads, by the `source` their subscriptions carry.
SOURCES = ("ats", "search")
#: How long a search role stays open once no search lists it any more.
STALE_DAYS = 21
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
marker `close_boards` will look for, before any batch runs — finders run
synchronously, in whoever starts the task, ahead of every job it queues."""
subs = store.subscriptions(source="ats", fetchable=True)
if not subs:
return []
store.state["ats:run"] = _now()
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
for i in range(0, len(subs), BATCH_SIZE)]
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
source. Stamps the run marker `close_boards` will look for, before any
batch runs — finders run synchronously, in whoever starts the task, ahead
of every job it queues."""
seed_searches(config, store)
batches: list[dict[str, Any]] = []
for source in SOURCES:
subs = store.subscriptions(source=source, fetchable=True)
if not subs:
continue
store.state[f"{source}:run"] = _now()
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
for i in range(0, len(subs), BATCH_SIZE)]
return batches
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str],
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards and `upsert_role` every item they list.
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
`targets` is keyword-only without a default, which is what makes it a
required payload key to the queue: the Jobs tab offers no button for a
batch that could only fail without one.
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
reads `options[subscription_field]`, so handing it only this batch's
entries is what keeps one job from re-fetching every other job's boards.
Builds the plugin scoped to just these targets — `Source.targets()` reads
`options[subscription_field]`, so handing it only this batch's entries is
what keeps one job from re-fetching every other job's boards.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
options = dict(config.sources.get("ats") or {})
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
for target in targets if target in subs]
source = ATSSource(options)
source.state = store.state
subs = {s.target: s for s in store.subscriptions(source=source) if s.target in targets}
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
for t in targets if t in subs]
plugin = SearchSource(options)
else:
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
for t in targets if t in subs]
plugin = ATSSource(options)
plugin.state = store.state
stored = 0
for item in source.fetch():
item.source = "ats"
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
for item in plugin.fetch():
item.source = source
if source == "search":
target = str(item.raw.get("search", ""))
else:
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
store.upsert_role(item, subs.get(target))
stored += 1
return {"roles": stored}
@@ -91,20 +122,57 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed.
whole batch job never got to it — is never closed. A search closes by its
list only when that list was complete, and otherwise by `STALE_DAYS`.
"""
run_marker = store.state.get("ats:run", "")
answered = failed = closed = 0
for sub in store.subscriptions(source="ats", fetchable=True):
stamp = store.state.get(f"ats:ok:{sub.target}", "")
if stamp and stamp == run_marker:
answered += 1
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
closed += store.mark_closed("ats", seen_ids, sub.id)
else:
failed += 1
answered = searched = failed = closed = 0
for source in SOURCES:
run_marker = store.state.get(f"{source}:run", "")
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
failed += 1
continue
if source == "search":
searched += 1
if not store.state.get(f"search:complete:{sub.target}", ""):
continue
else:
answered += 1
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
closed += store.mark_closed(source, seen_ids, sub.id)
if searched:
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
"closed": closed}
def search_target(provider: str, name: str) -> str:
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
The query itself lives in `params`, so editing it keeps the target."""
return f"{provider}:{company_key(name)}"
def seed_searches(config: Config, store: Store) -> list[str]:
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
and only while there are none: the same rule as `seed_agents`."""
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
return []
added = []
for seed in config.searches:
try:
store.add_subscription("search", search_target(seed.provider, seed.name),
label=seed.name, board=seed.provider, params=seed.params)
except ValueError as exc:
log.warning("searches: %s", exc)
continue
added.append(seed.name)
if added:
store.state[SEARCHES_SEEDED] = store.now()
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
return added
def pull_task() -> Task:
+2 -1
View File
@@ -45,9 +45,10 @@ def create_app(config: Config | None = None) -> FastAPI:
# auth add their own tables to it.
store = Store(config.database)
jobs.seed_agents(config, store)
jobs.seed_searches(config, store)
app = FastAPI(title="roledesk", docs_url="/api/docs",
summary="ATS boards in, roles scored per search agent, "
summary="ATS boards and job searches in, roles scored per search agent, "
"applications tracked.")
app.state.config = config
app.state.store = store
+129 -1
View File
@@ -34,7 +34,7 @@ _GENDER = re.compile(
#: board and "celonis" on another share a key.
_LEGAL = re.compile(
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|aps))+\.?$",
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
re.IGNORECASE,
)
@@ -64,6 +64,134 @@ def company_key(company: str) -> str:
return _fold(name).replace(" ", "-")
# ---------------------------------------------------------------- geography
#: The EU, the EEA, the UK and Switzerland: what "a European role" means to a
#: gate on `country`. `EU` is ISO's reserved code, used here for a role open
#: anywhere in Europe ("Remote - Europe", "EMEA") that names no one country.
EUROPE: tuple[str, ...] = (
"AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
"IS", "LI", "NO", "GB", "CH", "EU",
)
#: Folded names a board writes in its location line, to an ISO code: every
#: European country in English and its own language, the cities the boards
#: name most, and the non-European places that fill the seed boards, so those
#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
_PLACES: dict[str, str] = {
**dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
**dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
**dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
"ghent"), "BE"),
**dict.fromkeys(("bulgaria", "sofia"), "BG"),
**dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
**dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
**dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
# Folding keeps ø and æ, which are letters of their own rather than an o
# or an a with a mark, so København is listed beside Kobenhavn.
**dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
"århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
"lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
"bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
"birkerød"), "DK"),
**dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
**dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
**dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
"bordeaux", "lille"), "FR"),
**dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
"frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
"dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
"heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
**dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
**dict.fromkeys(("hungary", "budapest"), "HU"),
**dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
**dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
"torino", "bologna", "florence"), "IT"),
**dict.fromkeys(("latvia", "riga"), "LV"),
**dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
**dict.fromkeys(("luxembourg",), "LU"),
**dict.fromkeys(("malta", "valletta"), "MT"),
**dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
"rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
"delft"), "NL"),
**dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
"gdansk", "poznan", "lodz"), "PL"),
**dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
**dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
"RO"),
**dict.fromkeys(("slovakia", "bratislava"), "SK"),
**dict.fromkeys(("slovenia", "ljubljana"), "SI"),
**dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
"malaga", "bilbao"), "ES"),
**dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
"uppsala"), "SE"),
**dict.fromkeys(("iceland", "reykjavik"), "IS"),
**dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
**dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
"wales", "northern ireland", "london", "manchester", "edinburgh",
"glasgow", "bristol", "cambridge uk", "oxford", "leeds",
"birmingham uk", "belfast"), "GB"),
**dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
"basel", "lausanne", "bern", "zug"), "CH"),
**dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
"nyc", "san francisco", "seattle", "boston", "chicago", "austin",
"los angeles", "palo alto", "mountain view", "san mateo",
"foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
"costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
"houston", "phoenix", "tempe", "portland", "salt lake city",
"pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
"nashville", "minneapolis"), "US"),
**dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
"waterloo"), "CA"),
**dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
"chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
**dict.fromkeys(("singapore",), "SG"),
**dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
**dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
**dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
**dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
**dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
**dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
**dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
}
#: Keyed the way a place is folded before it is looked up, so a name can be
#: written as it is spelt.
_PLACES = {_fold(name): code for name, code in _PLACES.items()}
#: The longest name first, so "new york city" wins over "york" and "northern
#: ireland" over "ireland".
_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
#: US state codes after a comma, the way US boards write a city: "Boston, MA".
#: Read only when no name matched, since "DE" and "IN" are countries too.
_US_STATE = re.compile(
r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
r"\b")
def country_of(place: str) -> str:
"""The one country a board's location line names, as an ISO code, or ''
when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
is EU, and "London, EMEA" is GB: a country beats the region it is in."""
folded = f" {_fold(place or '')} "
found: set[str] = set()
for name in _PLACE_NAMES:
needle = f" {name} "
if needle in folded:
found.add(_PLACES[name])
folded = folded.replace(needle, " ")
if not found and _US_STATE.search(place or ""):
# Only when nothing else named a country: "Berlin, DE" is Germany.
found.add("US")
if len(found) > 1:
found.discard("EU")
return found.pop() if len(found) == 1 else ""
# --------------------------------------------------------------- vocabulary
Remote = Literal["", "remote", "hybrid", "onsite"]
+40 -3
View File
@@ -22,8 +22,8 @@ import numpy as np
from desk.models import Item
from app.models import (
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, place_key,
title_key,
Company, Gate, Role, ScoredRole, Subscription, Weights, company_key, country_of,
place_key, title_key,
)
from app.storage import schema
@@ -212,7 +212,8 @@ class Store:
`first_seen` and `created_at` are written on insert only; `last_seen` on
every call, and a role seen again is open again. The gate columns come
from `item.raw` under their own names, with `company` falling back to
the subscription's; an empty value never overwrites a filled column.
the subscription's and `country` to what `place` names; an empty value
never overwrites a filled column.
"""
if not item.source:
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
@@ -241,6 +242,7 @@ class Store:
"created_at": stamp,
}
row |= {name: str(raw.get(name) or "") for name in _RAW_TEXT}
row["country"] = row["country"] or country_of(place)
row |= {name: _int_or_none(raw.get(name)) for name in _RAW_INT}
keep_filled = ("company", "company_key", "place_key", "remote", "description",
@@ -341,6 +343,31 @@ class Store:
AND source_id NOT IN (SELECT value FROM json_each(?))""",
(self.now(), source, subscription_id, json.dumps(list(seen_ids))))
def close_unseen(self, source: str, before: str) -> int:
"""Close every open role of `source` last seen before `before`. For a
source whose fetch is never a complete list, a search, where a role
dropping out of one run's results is not the role closing."""
return self._write(
"""UPDATE roles SET closed_at = ?
WHERE source = ? AND closed_at IS NULL AND last_seen < ?""",
(self.now(), source, before))
def close_subscription(self, subscription_id: int) -> int:
"""Close every open role of one subscription, for a board that is no
longer fetched: nothing would ever close them otherwise."""
return self._write(
"UPDATE roles SET closed_at = ? WHERE subscription_id = ? AND closed_at IS NULL",
(self.now(), subscription_id))
def backfill_country(self) -> int:
"""Fill `country` from `place` on every role that has none, the way
`upsert_role` does for a new one. Returns how many it filled."""
rows = self._rows("SELECT uid, place FROM roles WHERE country = '' AND place != ''")
filled = [(code, r["uid"]) for r in rows if (code := country_of(r["place"]))]
with self.transaction() as conn:
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", filled)
return len(filled)
# -------------------------------------------------------------- extract
def extract_pending(self) -> int:
@@ -614,6 +641,16 @@ class Store:
"SELECT company_key, COUNT(*) AS n FROM roles "
"WHERE closed_at IS NULL GROUP BY company_key")}
def open_countries_by_subscription(self) -> dict[int, dict[str, int]]:
"""Per subscription id, its open roles counted by `country`."""
rows = self._rows("""SELECT subscription_id, country, COUNT(*) AS n FROM roles
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
GROUP BY subscription_id, country""")
out: dict[int, dict[str, int]] = {}
for r in rows:
out.setdefault(r["subscription_id"], {})[r["country"]] = r["n"]
return out
def company_boards(self) -> dict[str, str]:
"""`company_key -> ats host`, read off each company's own
subscription. `Subscription.company` is the display name it and the