The per-agent migration becomes v6 behind main's v5. A vote still stamps the role's read_at, tag counts hide the agent's hidden companies, and /api/filters takes the agent like /api/roles. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
423 lines
17 KiB
Python
423 lines
17 KiB
Python
"""The shapes roledesk hands round, and the keys that decide two rows are one role.
|
|
|
|
One set of models, as in newsdesk: the row the table returns and the object
|
|
the API serves are the same declaration. `Model.from_row` ignores columns a
|
|
model does not declare, so a new column is never a 500 somewhere else.
|
|
|
|
The three keys cluster roles on `(company_key, title_key, place_key)`. The same
|
|
title at two companies is two jobs, and the same ad on two boards is one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
from typing import Annotated, Any, Literal, get_args
|
|
|
|
from pydantic import BeforeValidator, Field
|
|
|
|
from desk.models import JsonDict, Model, ModelTaskResult
|
|
|
|
# ------------------------------------------------------------------- keys
|
|
|
|
#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x),
|
|
#: (w/m/div), (all genders), (gn). Matched with or without the parentheses.
|
|
_GENDER = re.compile(
|
|
r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)"
|
|
r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)"
|
|
r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
#: Legal forms cut from the end of a company name, so "Celonis SE" on one
|
|
#: board and "celonis" on another share a key.
|
|
_LEGAL = re.compile(
|
|
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
|
|
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _fold(text: str) -> str:
|
|
"""Casefolded, accents stripped, punctuation to spaces, whitespace collapsed."""
|
|
text = unicodedata.normalize("NFKD", text.casefold())
|
|
text = "".join(c for c in text if not unicodedata.combining(c))
|
|
text = re.sub(r"[\W_]+", " ", text)
|
|
return " ".join(text.split())
|
|
|
|
|
|
def title_key(title: str) -> str:
|
|
"""`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`."""
|
|
return _fold(_GENDER.sub(" ", title or ""))
|
|
|
|
|
|
def place_key(place: str) -> str:
|
|
"""`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`."""
|
|
return _fold(place or "")
|
|
|
|
|
|
def company_key(company: str) -> str:
|
|
"""A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the
|
|
primary key of `companies`."""
|
|
name = _LEGAL.sub("", (company or "").strip())
|
|
return _fold(name).replace(" ", "-")
|
|
|
|
|
|
# ---------------------------------------------------------------- geography
|
|
|
|
#: The EU, the EEA, the UK and Switzerland: what "a European role" means to a
|
|
#: gate on `country`. `EU` is ISO's reserved code, used here for a role open
|
|
#: anywhere in Europe ("Remote - Europe", "EMEA") that names no one country.
|
|
EUROPE: tuple[str, ...] = (
|
|
"AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
|
|
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
|
|
"IS", "LI", "NO", "GB", "CH", "EU",
|
|
)
|
|
|
|
#: Folded names a board writes in its location line, to an ISO code: every
|
|
#: European country in English and its own language, the cities the boards
|
|
#: name most, and the non-European places that fill the seed boards, so those
|
|
#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
|
|
_PLACES: dict[str, str] = {
|
|
**dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
|
|
**dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
|
|
**dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
|
|
"ghent"), "BE"),
|
|
**dict.fromkeys(("bulgaria", "sofia"), "BG"),
|
|
**dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
|
|
**dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
|
|
**dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
|
|
# Folding keeps ø and æ, which are letters of their own rather than an o
|
|
# or an a with a mark, so København is listed beside Kobenhavn.
|
|
**dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
|
|
"århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
|
|
"lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
|
|
"bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
|
|
"birkerød"), "DK"),
|
|
**dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
|
|
**dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
|
|
**dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
|
|
"bordeaux", "lille"), "FR"),
|
|
**dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
|
|
"frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
|
|
"dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
|
|
"heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
|
|
**dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
|
|
**dict.fromkeys(("hungary", "budapest"), "HU"),
|
|
**dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
|
|
**dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
|
|
"torino", "bologna", "florence"), "IT"),
|
|
**dict.fromkeys(("latvia", "riga"), "LV"),
|
|
**dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
|
|
**dict.fromkeys(("luxembourg",), "LU"),
|
|
**dict.fromkeys(("malta", "valletta"), "MT"),
|
|
**dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
|
|
"rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
|
"delft"), "NL"),
|
|
**dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
|
|
"gdansk", "poznan", "lodz"), "PL"),
|
|
**dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
|
|
**dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
|
|
"RO"),
|
|
**dict.fromkeys(("slovakia", "bratislava"), "SK"),
|
|
**dict.fromkeys(("slovenia", "ljubljana"), "SI"),
|
|
**dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
|
|
"malaga", "bilbao"), "ES"),
|
|
**dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
|
|
"uppsala"), "SE"),
|
|
**dict.fromkeys(("iceland", "reykjavik"), "IS"),
|
|
**dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
|
|
**dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
|
|
"wales", "northern ireland", "london", "manchester", "edinburgh",
|
|
"glasgow", "bristol", "cambridge uk", "oxford", "leeds",
|
|
"birmingham uk", "belfast"), "GB"),
|
|
**dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
|
|
"basel", "lausanne", "bern", "zug"), "CH"),
|
|
**dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
|
|
"nyc", "san francisco", "seattle", "boston", "chicago", "austin",
|
|
"los angeles", "palo alto", "mountain view", "san mateo",
|
|
"foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
|
|
"costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
|
|
"houston", "phoenix", "tempe", "portland", "salt lake city",
|
|
"pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
|
|
"nashville", "minneapolis"), "US"),
|
|
**dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
|
|
"waterloo"), "CA"),
|
|
**dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
|
|
"chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
|
**dict.fromkeys(("singapore",), "SG"),
|
|
**dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
|
|
**dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
|
|
**dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
|
**dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
|
|
**dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
|
**dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
|
|
**dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
|
|
}
|
|
|
|
#: Keyed the way a place is folded before it is looked up, so a name can be
|
|
#: written as it is spelt.
|
|
_PLACES = {_fold(name): code for name, code in _PLACES.items()}
|
|
|
|
#: The longest name first, so "new york city" wins over "york" and "northern
|
|
#: ireland" over "ireland".
|
|
_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
|
|
|
|
#: US state codes after a comma, the way US boards write a city: "Boston, MA".
|
|
#: Read only when no name matched, since "DE" and "IN" are countries too.
|
|
_US_STATE = re.compile(
|
|
r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
|
|
r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
|
|
r"\b")
|
|
|
|
|
|
def country_of(place: str) -> str:
|
|
"""The one country a board's location line names, as an ISO code, or ''
|
|
when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
|
|
US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
|
|
is EU, and "London, EMEA" is GB: a country beats the region it is in."""
|
|
folded = f" {_fold(place or '')} "
|
|
found: set[str] = set()
|
|
for name in _PLACE_NAMES:
|
|
needle = f" {name} "
|
|
if needle in folded:
|
|
found.add(_PLACES[name])
|
|
folded = folded.replace(needle, " ")
|
|
if not found and _US_STATE.search(place or ""):
|
|
# Only when nothing else named a country: "Berlin, DE" is Germany.
|
|
found.add("US")
|
|
if len(found) > 1:
|
|
found.discard("EU")
|
|
return found.pop() if len(found) == 1 else ""
|
|
|
|
|
|
# --------------------------------------------------------------- vocabulary
|
|
|
|
Remote = Literal["", "remote", "hybrid", "onsite"]
|
|
REMOTE: tuple[str, ...] = get_args(Remote)
|
|
|
|
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
|
|
"director", "exec"]
|
|
SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
|
|
|
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
|
Permit = Literal["", "sponsors", "no_sponsorship"]
|
|
PERMITS: tuple[str, ...] = get_args(Permit)
|
|
|
|
Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer",
|
|
"rejected", "withdrawn"]
|
|
STATUSES: tuple[str, ...] = get_args(Status)
|
|
|
|
#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score).
|
|
AFFINITY = range(-1, 3)
|
|
|
|
|
|
def _json(value: Any, empty: str) -> Any:
|
|
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
|
|
value that is already parsed passes, and one that does not parse is the
|
|
empty value rather than a 500."""
|
|
if not isinstance(value, str):
|
|
return value
|
|
try:
|
|
return json.loads(value or empty)
|
|
except ValueError:
|
|
return json.loads(empty)
|
|
|
|
|
|
#: A column holding a JSON array of strings, such as `roles.languages`.
|
|
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
|
|
|
|
|
|
# ------------------------------------------------------------------ models
|
|
|
|
class Role(Model):
|
|
"""A row of `roles`: one posting on one board."""
|
|
|
|
uid: str
|
|
source: str
|
|
source_id: str
|
|
subscription_id: int | None = None
|
|
company_key: str = ""
|
|
company: str = ""
|
|
title: str = ""
|
|
title_key: str = ""
|
|
place: str = ""
|
|
place_key: str = ""
|
|
country: str = ""
|
|
remote: Remote = ""
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
currency: str = ""
|
|
seniority: str = ""
|
|
function: str = ""
|
|
permit: Permit = ""
|
|
#: ISO 639-1 codes the posting says are required, lowercase.
|
|
languages: JsonStrings = []
|
|
description: str = ""
|
|
feed_text: str = ""
|
|
original_url: str = ""
|
|
published_at: str
|
|
first_seen: str
|
|
last_seen: str
|
|
closed_at: str | None = None
|
|
raw: JsonDict = {}
|
|
extracted_at: str | None = None
|
|
created_at: str
|
|
#: The vote of the agent the role was read for (see `role_votes`): None
|
|
#: unjudged, else -1 pass, 0 neutral, 1 like.
|
|
vote: Literal[-1, 0, 1] | None = None
|
|
voted_at: str | None = None
|
|
#: NULL unread, else when the feed opened it or judged it. A vote stamps
|
|
#: this too, see `Store.set_vote`: voting is reading.
|
|
read_at: str | None = None
|
|
summary: str = ""
|
|
summarised_at: str | None = None
|
|
#: The posting page as plain text, for a role whose description is a teaser.
|
|
article_text: str = ""
|
|
article_at: str | None = None
|
|
#: The open vocabulary `summarise` read off the posting, sorted. Not a
|
|
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
|
|
tags: list[str] = []
|
|
|
|
|
|
class ExtractResult(ModelTaskResult):
|
|
"""What one run of `extract` did. See app/jobs/extract.py."""
|
|
|
|
#: Roles the model answered for, whether or not it found anything to fill.
|
|
extracted: int = 0
|
|
#: Columns written, across those roles. A role answered with nothing the
|
|
#: posting states adds to `extracted` and not to this.
|
|
filled: int = 0
|
|
|
|
@property
|
|
def done(self) -> int:
|
|
return self.extracted
|
|
|
|
|
|
class ScoredRole(Role):
|
|
"""A role as `/api/roles` serves it: with one agent's score and gate
|
|
reason when an agent was asked for, and without otherwise."""
|
|
|
|
score: float | None = None
|
|
gated_reason: str = ""
|
|
|
|
|
|
class Company(Model):
|
|
"""A row of `companies`, with `affinity` as the agent it was read for
|
|
has it (see `agent_companies`); 0 when no agent was asked for."""
|
|
|
|
key: str
|
|
name: str
|
|
domain: str = ""
|
|
affinity: int = 0
|
|
status: str = "researching"
|
|
careers_url: str = ""
|
|
notes: str = ""
|
|
created_at: str
|
|
updated_at: str
|
|
|
|
|
|
class Subscription(Model):
|
|
"""A row of `subscriptions`. `company` and `board` are the two labels
|
|
newsdesk calls outlet and section."""
|
|
|
|
id: int
|
|
source: str
|
|
target: str
|
|
label: str
|
|
company: str = ""
|
|
board: str = ""
|
|
params: JsonDict = {}
|
|
subscribed: int = 1
|
|
fetch: int = 1
|
|
show: int = 1
|
|
added_at: str
|
|
target_id: str = ""
|
|
|
|
|
|
class Application(Model):
|
|
"""A row of `applications`. `id` empty means not saved yet; the store mints
|
|
one. `role_uid` is None for a role that was never posted."""
|
|
|
|
id: str = ""
|
|
role_uid: str | None = None
|
|
company_key: str = ""
|
|
role_title: str = ""
|
|
url: str = ""
|
|
status: Status = "saved"
|
|
source: str = ""
|
|
applied_at: str | None = None
|
|
next_action: str = ""
|
|
next_date: str | None = None
|
|
contact: str = ""
|
|
notes: str = ""
|
|
created_at: str = ""
|
|
updated_at: str = ""
|
|
|
|
|
|
# ------------------------------------------------------------------ agents
|
|
|
|
#: The role columns a gate can name. `company` matches on the company key;
|
|
#: `place` is a case-insensitive substring, like the place filter; the rest are
|
|
#: case-insensitive equality, except `salary_min` (at least the value) and
|
|
#: `languages` (every language the role requires is one of the values).
|
|
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
|
|
"currency", "permit", "languages", "company"]
|
|
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
|
|
|
|
GateMode = Literal["require", "prefer"]
|
|
|
|
|
|
class Gate(Model):
|
|
"""One condition of an agent. `require` gates a role that fails it (the
|
|
row is kept with the reason); `prefer` adds `weight` to its score when it
|
|
matches."""
|
|
|
|
field: GateField
|
|
values: list[str | int | float] = Field(min_length=1)
|
|
mode: GateMode = "require"
|
|
weight: float = Field(1.0, ge=0)
|
|
|
|
|
|
class Weights(Model):
|
|
"""How much each score term counts. Summed with the prefer gates' weights
|
|
into the denominator, so a score stays in 0..1 whatever the numbers."""
|
|
|
|
fit: float = Field(0.6, ge=0)
|
|
recency: float = Field(0.2, ge=0)
|
|
affinity: float = Field(0.2, ge=0)
|
|
#: Cosine to the liked-minus-passed direction of the agent's votes.
|
|
taste: float = Field(0.2, ge=0)
|
|
|
|
|
|
class Agent(Model):
|
|
"""A row of `agents`: a named search with its own intent, gates and
|
|
weights. The cached intent vector is a column this does not declare."""
|
|
|
|
id: int
|
|
name: str
|
|
intent: str = ""
|
|
gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
|
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
|
|
enabled: bool = True
|
|
created_at: str
|
|
|
|
|
|
class Candidate(Model):
|
|
"""What scoring reads about one open role for one agent: when it was
|
|
first seen, its company's affinity for that agent, and per gate `1`
|
|
match, `0` no match, `-1` the column is empty."""
|
|
|
|
uid: str
|
|
first_seen: str
|
|
affinity: int = 0
|
|
gates: list[int] = []
|
|
|
|
|
|
class ApplicationEvent(Model):
|
|
id: int
|
|
application_id: str
|
|
status: Status
|
|
at: str
|