A require gate on an empty column gated the role ("country unknown"), and extract, the stage that fills the column, is bounded to the roles that rank, so 2,281 of 15,556 open roles could never come back: every board that writes "Remote", "Hybrid" or "Distributed" as its location. An empty column now neither passes nor fails a gate; the role ranks, extract reads it, the rescore decides.
`any` is a third gate mode: the any gates are one group a role must match one of, so a ranking can say "in Denmark, or remote", which require alone could not. The group is gated only when some member had evidence against it.
`remote_of(place)` fills `remote` from the location line the way `country_of` fills `country`, and the startup backfill does both, since Greenhouse writes "Hybrid" as the location and gives no flag.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
447 lines
18 KiB
Python
447 lines
18 KiB
Python
"""The shapes roledesk hands round, and the keys that decide two rows are one role.
|
|
|
|
One set of models, as in newsdesk: the row the table returns and the object
|
|
the API serves are the same declaration. `Model.from_row` ignores columns a
|
|
model does not declare, so a new column is never a 500 somewhere else.
|
|
|
|
The three keys cluster roles on `(company_key, title_key, place_key)`. The same
|
|
title at two companies is two jobs, and the same ad on two boards is one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
from typing import Annotated, Any, Literal, get_args
|
|
|
|
from pydantic import BeforeValidator, Field
|
|
|
|
from desk.models import JsonDict, Model, ModelTaskResult
|
|
|
|
# ------------------------------------------------------------------- keys
|
|
|
|
#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x),
|
|
#: (w/m/div), (all genders), (gn). Matched with or without the parentheses.
|
|
_GENDER = re.compile(
|
|
r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)"
|
|
r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)"
|
|
r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
#: Legal forms cut from the end of a company name, so "Celonis SE" on one
|
|
#: board and "celonis" on another share a key.
|
|
_LEGAL = re.compile(
|
|
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
|
|
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _fold(text: str) -> str:
|
|
"""Casefolded, accents stripped, punctuation to spaces, whitespace collapsed."""
|
|
text = unicodedata.normalize("NFKD", text.casefold())
|
|
text = "".join(c for c in text if not unicodedata.combining(c))
|
|
text = re.sub(r"[\W_]+", " ", text)
|
|
return " ".join(text.split())
|
|
|
|
|
|
def title_key(title: str) -> str:
|
|
"""`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`."""
|
|
return _fold(_GENDER.sub(" ", title or ""))
|
|
|
|
|
|
def place_key(place: str) -> str:
|
|
"""`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`."""
|
|
return _fold(place or "")
|
|
|
|
|
|
def company_key(company: str) -> str:
|
|
"""A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the
|
|
primary key of `companies`."""
|
|
name = _LEGAL.sub("", (company or "").strip())
|
|
return _fold(name).replace(" ", "-")
|
|
|
|
|
|
# ---------------------------------------------------------------- geography
|
|
|
|
#: Folded names a board writes in its location line, to an ISO code: every
|
|
#: European country in English and its own language, the cities the boards
|
|
#: name most, and the non-European places that fill the seed boards, so those
|
|
#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
|
|
_PLACES: dict[str, str] = {
|
|
**dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
|
|
**dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
|
|
**dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
|
|
"ghent"), "BE"),
|
|
**dict.fromkeys(("bulgaria", "sofia"), "BG"),
|
|
**dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
|
|
**dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
|
|
**dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
|
|
# Folding keeps ø and æ, which are letters of their own rather than an o
|
|
# or an a with a mark, so København is listed beside Kobenhavn.
|
|
**dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
|
|
"århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
|
|
"lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
|
|
"bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
|
|
"birkerød"), "DK"),
|
|
**dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
|
|
**dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
|
|
**dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
|
|
"bordeaux", "lille"), "FR"),
|
|
**dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
|
|
"frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
|
|
"dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
|
|
"heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
|
|
**dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
|
|
**dict.fromkeys(("hungary", "budapest"), "HU"),
|
|
**dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
|
|
**dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
|
|
"torino", "bologna", "florence"), "IT"),
|
|
**dict.fromkeys(("latvia", "riga"), "LV"),
|
|
**dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
|
|
**dict.fromkeys(("luxembourg",), "LU"),
|
|
**dict.fromkeys(("malta", "valletta"), "MT"),
|
|
**dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
|
|
"rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
|
"delft"), "NL"),
|
|
**dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
|
|
"gdansk", "poznan", "lodz"), "PL"),
|
|
**dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
|
|
**dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
|
|
"RO"),
|
|
**dict.fromkeys(("slovakia", "bratislava"), "SK"),
|
|
**dict.fromkeys(("slovenia", "ljubljana"), "SI"),
|
|
**dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
|
|
"malaga", "bilbao"), "ES"),
|
|
**dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
|
|
"uppsala"), "SE"),
|
|
**dict.fromkeys(("iceland", "reykjavik"), "IS"),
|
|
**dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
|
|
**dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
|
|
"wales", "northern ireland", "london", "manchester", "edinburgh",
|
|
"glasgow", "bristol", "cambridge uk", "oxford", "leeds",
|
|
"birmingham uk", "belfast"), "GB"),
|
|
**dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
|
|
"basel", "lausanne", "bern", "zug"), "CH"),
|
|
**dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
|
|
"nyc", "san francisco", "seattle", "boston", "chicago", "austin",
|
|
"los angeles", "palo alto", "mountain view", "san mateo",
|
|
"foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
|
|
"costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
|
|
"houston", "phoenix", "tempe", "portland", "salt lake city",
|
|
"pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
|
|
"nashville", "minneapolis"), "US"),
|
|
**dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
|
|
"waterloo"), "CA"),
|
|
**dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
|
|
"chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
|
**dict.fromkeys(("singapore",), "SG"),
|
|
**dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
|
|
**dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
|
|
**dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
|
**dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
|
|
**dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
|
**dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
|
|
**dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
|
|
}
|
|
|
|
#: Keyed the way a place is folded before it is looked up, so a name can be
|
|
#: written as it is spelt.
|
|
_PLACES = {_fold(name): code for name, code in _PLACES.items()}
|
|
|
|
#: The longest name first, so "new york city" wins over "york" and "northern
|
|
#: ireland" over "ireland".
|
|
_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
|
|
|
|
#: US state codes after a comma, the way US boards write a city: "Boston, MA".
|
|
#: Read only when no name matched, since "DE" and "IN" are countries too.
|
|
_US_STATE = re.compile(
|
|
r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
|
|
r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
|
|
r"\b")
|
|
|
|
|
|
def country_of(place: str) -> str:
|
|
"""The one country a board's location line names, as an ISO code, or ''
|
|
when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
|
|
US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
|
|
is EU, and "London, EMEA" is GB: a country beats the region it is in."""
|
|
folded = f" {_fold(place or '')} "
|
|
found: set[str] = set()
|
|
for name in _PLACE_NAMES:
|
|
needle = f" {name} "
|
|
if needle in folded:
|
|
found.add(_PLACES[name])
|
|
folded = folded.replace(needle, " ")
|
|
if not found and _US_STATE.search(place or ""):
|
|
# Only when nothing else named a country: "Berlin, DE" is Germany.
|
|
found.add("US")
|
|
if len(found) > 1:
|
|
found.discard("EU")
|
|
return found.pop() if len(found) == 1 else ""
|
|
|
|
|
|
# --------------------------------------------------------------- vocabulary
|
|
|
|
Remote = Literal["", "remote", "hybrid", "onsite"]
|
|
REMOTE: tuple[str, ...] = get_args(Remote)
|
|
|
|
#: Words in a location line that say how the role is worked, for a board that
|
|
#: gives no remote flag of its own: Greenhouse writes "Remote", "Hybrid" or
|
|
#: "Distributed" as the location and nothing else.
|
|
_REMOTE_WORDS: tuple[tuple[str, str], ...] = (
|
|
("hybrid", "hybrid"),
|
|
("remote", "remote"), ("distributed", "remote"), ("home based", "remote"),
|
|
("home-based", "remote"), ("work from home", "remote"), ("anywhere", "remote"),
|
|
("worldwide", "remote"), ("telecommute", "remote"),
|
|
("on-site", "onsite"), ("onsite", "onsite"), ("on site", "onsite"),
|
|
("in-office", "onsite"), ("in office", "onsite"),
|
|
)
|
|
|
|
|
|
def remote_of(place: str) -> str:
|
|
"""How a board's location line says the role is worked, as one of
|
|
`REMOTE`, or '' when it does not say. "Hybrid" wins over "remote" in the
|
|
same line ("Remote or hybrid, Berlin" is hybrid: some days in an office),
|
|
and a line that names only a place is ''."""
|
|
folded = _fold(place or "")
|
|
for word, mode in _REMOTE_WORDS:
|
|
if word in folded:
|
|
return mode
|
|
return ""
|
|
|
|
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
|
|
"director", "exec"]
|
|
SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
|
|
|
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
|
Permit = Literal["", "sponsors", "no_sponsorship"]
|
|
PERMITS: tuple[str, ...] = get_args(Permit)
|
|
|
|
Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer",
|
|
"rejected", "withdrawn"]
|
|
STATUSES: tuple[str, ...] = get_args(Status)
|
|
|
|
#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score).
|
|
AFFINITY = range(-1, 3)
|
|
|
|
|
|
def _json(value: Any, empty: str) -> Any:
|
|
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
|
|
value that is already parsed passes, and one that does not parse is the
|
|
empty value rather than a 500."""
|
|
if not isinstance(value, str):
|
|
return value
|
|
try:
|
|
return json.loads(value or empty)
|
|
except ValueError:
|
|
return json.loads(empty)
|
|
|
|
|
|
#: A column holding a JSON array of strings, such as `roles.languages`.
|
|
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
|
|
|
|
|
|
# ------------------------------------------------------------------ models
|
|
|
|
class Role(Model):
|
|
"""A row of `roles`: one posting on one board."""
|
|
|
|
uid: str
|
|
source: str
|
|
source_id: str
|
|
subscription_id: int | None = None
|
|
company_key: str = ""
|
|
company: str = ""
|
|
title: str = ""
|
|
title_key: str = ""
|
|
place: str = ""
|
|
place_key: str = ""
|
|
country: str = ""
|
|
remote: Remote = ""
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
currency: str = ""
|
|
seniority: str = ""
|
|
function: str = ""
|
|
permit: Permit = ""
|
|
#: ISO 639-1 codes the posting says are required, lowercase.
|
|
languages: JsonStrings = []
|
|
description: str = ""
|
|
feed_text: str = ""
|
|
original_url: str = ""
|
|
published_at: str
|
|
first_seen: str
|
|
last_seen: str
|
|
closed_at: str | None = None
|
|
raw: JsonDict = {}
|
|
extracted_at: str | None = None
|
|
created_at: str
|
|
#: The vote of the agent the role was read for (see `role_votes`): None
|
|
#: unjudged, else -1 pass, 0 neutral, 1 like.
|
|
vote: Literal[-1, 0, 1] | None = None
|
|
voted_at: str | None = None
|
|
#: NULL unread, else when the feed opened it or judged it. A vote stamps
|
|
#: this too, see `Store.set_vote`: voting is reading.
|
|
read_at: str | None = None
|
|
summary: str = ""
|
|
summarised_at: str | None = None
|
|
#: The posting page as plain text, for a role whose description is a teaser.
|
|
article_text: str = ""
|
|
article_at: str | None = None
|
|
#: The open vocabulary `summarise` read off the posting, sorted. Not a
|
|
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
|
|
tags: list[str] = []
|
|
|
|
|
|
class ExtractResult(ModelTaskResult):
|
|
"""What one run of `extract` did. See app/jobs/extract.py."""
|
|
|
|
#: Roles the model answered for, whether or not it found anything to fill.
|
|
extracted: int = 0
|
|
#: Columns written, across those roles. A role answered with nothing the
|
|
#: posting states adds to `extracted` and not to this.
|
|
filled: int = 0
|
|
|
|
@property
|
|
def done(self) -> int:
|
|
return self.extracted
|
|
|
|
|
|
class ScoredRole(Role):
|
|
"""A role as `/api/roles` serves it: with one agent's score and gate
|
|
reason when an agent was asked for, and without otherwise."""
|
|
|
|
score: float | None = None
|
|
gated_reason: str = ""
|
|
|
|
|
|
class Company(Model):
|
|
"""A row of `companies`, with `affinity` as the agent it was read for
|
|
has it (see `agent_companies`); 0 when no agent was asked for."""
|
|
|
|
key: str
|
|
name: str
|
|
domain: str = ""
|
|
affinity: int = 0
|
|
status: str = "researching"
|
|
careers_url: str = ""
|
|
notes: str = ""
|
|
#: The host the careers site runs on, readable or not; "" when unknown.
|
|
#: A company with an `ats` and no board subscription is one the app
|
|
#: cannot read yet. See `app.jobs.discover`.
|
|
ats: str = ""
|
|
country: str = ""
|
|
city: str = ""
|
|
created_at: str
|
|
updated_at: str
|
|
|
|
|
|
class Subscription(Model):
|
|
"""A row of `subscriptions`. `company` and `board` are the two labels
|
|
newsdesk calls outlet and section."""
|
|
|
|
id: int
|
|
source: str
|
|
target: str
|
|
label: str
|
|
company: str = ""
|
|
board: str = ""
|
|
params: JsonDict = {}
|
|
subscribed: int = 1
|
|
fetch: int = 1
|
|
show: int = 1
|
|
added_at: str
|
|
target_id: str = ""
|
|
|
|
|
|
class Application(Model):
|
|
"""A row of `applications`. `id` empty means not saved yet; the store mints
|
|
one. `role_uid` is None for a role that was never posted."""
|
|
|
|
id: str = ""
|
|
role_uid: str | None = None
|
|
company_key: str = ""
|
|
role_title: str = ""
|
|
url: str = ""
|
|
status: Status = "saved"
|
|
source: str = ""
|
|
applied_at: str | None = None
|
|
next_action: str = ""
|
|
next_date: str | None = None
|
|
contact: str = ""
|
|
notes: str = ""
|
|
created_at: str = ""
|
|
updated_at: str = ""
|
|
|
|
|
|
# ------------------------------------------------------------------ agents
|
|
|
|
#: The role columns a gate can name. `company` matches on the company key;
|
|
#: `place` is a case-insensitive substring, like the place filter; the rest are
|
|
#: case-insensitive equality, except `salary_min` (at least the value) and
|
|
#: `languages` (every language the role requires is one of the values).
|
|
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
|
|
"currency", "permit", "languages", "company"]
|
|
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
|
|
|
|
GateMode = Literal["require", "any", "prefer"]
|
|
|
|
|
|
class Gate(Model):
|
|
"""One condition of an agent. `require` gates a role that fails it (the
|
|
row is kept with the reason); the `any` gates together are one group a
|
|
role must match at least one of ("in Denmark, or remote"); `prefer` adds
|
|
`weight` to its score when it matches. A gate rejects on evidence only: a
|
|
role whose column is empty is not rejected, since `extract` fills the
|
|
column later for the roles that rank, and only for those."""
|
|
|
|
field: GateField
|
|
values: list[str | int | float] = Field(min_length=1)
|
|
mode: GateMode = "require"
|
|
weight: float = Field(1.0, ge=0)
|
|
|
|
|
|
class Weights(Model):
|
|
"""How much each score term counts. Summed with the prefer gates' weights
|
|
into the denominator, so a score stays in 0..1 whatever the numbers."""
|
|
|
|
fit: float = Field(0.6, ge=0)
|
|
recency: float = Field(0.2, ge=0)
|
|
affinity: float = Field(0.2, ge=0)
|
|
#: Cosine to the liked-minus-passed direction of the agent's votes.
|
|
taste: float = Field(0.2, ge=0)
|
|
|
|
|
|
class Agent(Model):
|
|
"""A row of `agents`: a named search with its own intent, gates and
|
|
weights. The cached intent vector is a column this does not declare."""
|
|
|
|
id: int
|
|
name: str
|
|
intent: str = ""
|
|
gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
|
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
|
|
enabled: bool = True
|
|
created_at: str
|
|
|
|
|
|
class Candidate(Model):
|
|
"""What scoring reads about one open role for one agent: when it was
|
|
first seen, its company's affinity for that agent, and per gate `1`
|
|
match, `0` no match, `-1` the column is empty."""
|
|
|
|
uid: str
|
|
first_seen: str
|
|
affinity: int = 0
|
|
gates: list[int] = []
|
|
|
|
|
|
class ApplicationEvent(Model):
|
|
id: int
|
|
application_id: str
|
|
status: Status
|
|
at: str
|