Files
jobboard/app/models.py
T
WolfandClaude Fable 5.1 fb83ae5099 Gates reject on evidence only, any gates are one group, remote is read off the location line
A require gate on an empty column gated the role ("country unknown"), and extract, the stage that fills the column, is bounded to the roles that rank, so 2,281 of 15,556 open roles could never come back: every board that writes "Remote", "Hybrid" or "Distributed" as its location. An empty column now neither passes nor fails a gate; the role ranks, extract reads it, the rescore decides.

`any` is a third gate mode: the any gates are one group a role must match one of, so a ranking can say "in Denmark, or remote", which require alone could not. The group is gated only when some member had evidence against it.

`remote_of(place)` fills `remote` from the location line the way `country_of` fills `country`, and the startup backfill does both, since Greenhouse writes "Hybrid" as the location and gives no flag.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-28 19:30:06 +02:00

447 lines
18 KiB
Python

"""The shapes roledesk hands round, and the keys that decide two rows are one role.
One set of models, as in newsdesk: the row the table returns and the object
the API serves are the same declaration. `Model.from_row` ignores columns a
model does not declare, so a new column is never a 500 somewhere else.
The three keys cluster roles on `(company_key, title_key, place_key)`. The same
title at two companies is two jobs, and the same ad on two boards is one.
"""
from __future__ import annotations
import json
import re
import unicodedata
from typing import Annotated, Any, Literal, get_args
from pydantic import BeforeValidator, Field
from desk.models import JsonDict, Model, ModelTaskResult
# ------------------------------------------------------------------- keys
#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x),
#: (w/m/div), (all genders), (gn). Matched with or without the parentheses.
_GENDER = re.compile(
r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)"
r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)"
r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b",
re.IGNORECASE,
)
#: Legal forms cut from the end of a company name, so "Celonis SE" on one
#: board and "celonis" on another share a key.
_LEGAL = re.compile(
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
re.IGNORECASE,
)
def _fold(text: str) -> str:
"""Casefolded, accents stripped, punctuation to spaces, whitespace collapsed."""
text = unicodedata.normalize("NFKD", text.casefold())
text = "".join(c for c in text if not unicodedata.combining(c))
text = re.sub(r"[\W_]+", " ", text)
return " ".join(text.split())
def title_key(title: str) -> str:
"""`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`."""
return _fold(_GENDER.sub(" ", title or ""))
def place_key(place: str) -> str:
"""`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`."""
return _fold(place or "")
def company_key(company: str) -> str:
"""A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the
primary key of `companies`."""
name = _LEGAL.sub("", (company or "").strip())
return _fold(name).replace(" ", "-")
# ---------------------------------------------------------------- geography
#: Folded names a board writes in its location line, to an ISO code: every
#: European country in English and its own language, the cities the boards
#: name most, and the non-European places that fill the seed boards, so those
#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
_PLACES: dict[str, str] = {
**dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
**dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
**dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
"ghent"), "BE"),
**dict.fromkeys(("bulgaria", "sofia"), "BG"),
**dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
**dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
**dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
# Folding keeps ø and æ, which are letters of their own rather than an o
# or an a with a mark, so København is listed beside Kobenhavn.
**dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
"århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
"lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
"bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
"birkerød"), "DK"),
**dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
**dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
**dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
"bordeaux", "lille"), "FR"),
**dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
"frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
"dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
"heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
**dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
**dict.fromkeys(("hungary", "budapest"), "HU"),
**dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
**dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
"torino", "bologna", "florence"), "IT"),
**dict.fromkeys(("latvia", "riga"), "LV"),
**dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
**dict.fromkeys(("luxembourg",), "LU"),
**dict.fromkeys(("malta", "valletta"), "MT"),
**dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
"rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
"delft"), "NL"),
**dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
"gdansk", "poznan", "lodz"), "PL"),
**dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
**dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
"RO"),
**dict.fromkeys(("slovakia", "bratislava"), "SK"),
**dict.fromkeys(("slovenia", "ljubljana"), "SI"),
**dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
"malaga", "bilbao"), "ES"),
**dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
"uppsala"), "SE"),
**dict.fromkeys(("iceland", "reykjavik"), "IS"),
**dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
**dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
"wales", "northern ireland", "london", "manchester", "edinburgh",
"glasgow", "bristol", "cambridge uk", "oxford", "leeds",
"birmingham uk", "belfast"), "GB"),
**dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
"basel", "lausanne", "bern", "zug"), "CH"),
**dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
"nyc", "san francisco", "seattle", "boston", "chicago", "austin",
"los angeles", "palo alto", "mountain view", "san mateo",
"foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
"costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
"houston", "phoenix", "tempe", "portland", "salt lake city",
"pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
"nashville", "minneapolis"), "US"),
**dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
"waterloo"), "CA"),
**dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
"chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
**dict.fromkeys(("singapore",), "SG"),
**dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
**dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
**dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
**dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
**dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
**dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
**dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
}
#: Keyed the way a place is folded before it is looked up, so a name can be
#: written as it is spelt.
_PLACES = {_fold(name): code for name, code in _PLACES.items()}
#: The longest name first, so "new york city" wins over "york" and "northern
#: ireland" over "ireland".
_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
#: US state codes after a comma, the way US boards write a city: "Boston, MA".
#: Read only when no name matched, since "DE" and "IN" are countries too.
_US_STATE = re.compile(
r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
r"\b")
def country_of(place: str) -> str:
"""The one country a board's location line names, as an ISO code, or ''
when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
is EU, and "London, EMEA" is GB: a country beats the region it is in."""
folded = f" {_fold(place or '')} "
found: set[str] = set()
for name in _PLACE_NAMES:
needle = f" {name} "
if needle in folded:
found.add(_PLACES[name])
folded = folded.replace(needle, " ")
if not found and _US_STATE.search(place or ""):
# Only when nothing else named a country: "Berlin, DE" is Germany.
found.add("US")
if len(found) > 1:
found.discard("EU")
return found.pop() if len(found) == 1 else ""
# --------------------------------------------------------------- vocabulary
Remote = Literal["", "remote", "hybrid", "onsite"]
REMOTE: tuple[str, ...] = get_args(Remote)
#: Words in a location line that say how the role is worked, for a board that
#: gives no remote flag of its own: Greenhouse writes "Remote", "Hybrid" or
#: "Distributed" as the location and nothing else.
_REMOTE_WORDS: tuple[tuple[str, str], ...] = (
("hybrid", "hybrid"),
("remote", "remote"), ("distributed", "remote"), ("home based", "remote"),
("home-based", "remote"), ("work from home", "remote"), ("anywhere", "remote"),
("worldwide", "remote"), ("telecommute", "remote"),
("on-site", "onsite"), ("onsite", "onsite"), ("on site", "onsite"),
("in-office", "onsite"), ("in office", "onsite"),
)
def remote_of(place: str) -> str:
"""How a board's location line says the role is worked, as one of
`REMOTE`, or '' when it does not say. "Hybrid" wins over "remote" in the
same line ("Remote or hybrid, Berlin" is hybrid: some days in an office),
and a line that names only a place is ''."""
folded = _fold(place or "")
for word, mode in _REMOTE_WORDS:
if word in folded:
return mode
return ""
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
"director", "exec"]
SENIORITIES: tuple[str, ...] = get_args(Seniority)
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
Permit = Literal["", "sponsors", "no_sponsorship"]
PERMITS: tuple[str, ...] = get_args(Permit)
Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer",
"rejected", "withdrawn"]
STATUSES: tuple[str, ...] = get_args(Status)
#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score).
AFFINITY = range(-1, 3)
def _json(value: Any, empty: str) -> Any:
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
value that is already parsed passes, and one that does not parse is the
empty value rather than a 500."""
if not isinstance(value, str):
return value
try:
return json.loads(value or empty)
except ValueError:
return json.loads(empty)
#: A column holding a JSON array of strings, such as `roles.languages`.
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
# ------------------------------------------------------------------ models
class Role(Model):
"""A row of `roles`: one posting on one board."""
uid: str
source: str
source_id: str
subscription_id: int | None = None
company_key: str = ""
company: str = ""
title: str = ""
title_key: str = ""
place: str = ""
place_key: str = ""
country: str = ""
remote: Remote = ""
salary_min: int | None = None
salary_max: int | None = None
currency: str = ""
seniority: str = ""
function: str = ""
permit: Permit = ""
#: ISO 639-1 codes the posting says are required, lowercase.
languages: JsonStrings = []
description: str = ""
feed_text: str = ""
original_url: str = ""
published_at: str
first_seen: str
last_seen: str
closed_at: str | None = None
raw: JsonDict = {}
extracted_at: str | None = None
created_at: str
#: The vote of the agent the role was read for (see `role_votes`): None
#: unjudged, else -1 pass, 0 neutral, 1 like.
vote: Literal[-1, 0, 1] | None = None
voted_at: str | None = None
#: NULL unread, else when the feed opened it or judged it. A vote stamps
#: this too, see `Store.set_vote`: voting is reading.
read_at: str | None = None
summary: str = ""
summarised_at: str | None = None
#: The posting page as plain text, for a role whose description is a teaser.
article_text: str = ""
article_at: str | None = None
#: The open vocabulary `summarise` read off the posting, sorted. Not a
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
tags: list[str] = []
class ExtractResult(ModelTaskResult):
"""What one run of `extract` did. See app/jobs/extract.py."""
#: Roles the model answered for, whether or not it found anything to fill.
extracted: int = 0
#: Columns written, across those roles. A role answered with nothing the
#: posting states adds to `extracted` and not to this.
filled: int = 0
@property
def done(self) -> int:
return self.extracted
class ScoredRole(Role):
"""A role as `/api/roles` serves it: with one agent's score and gate
reason when an agent was asked for, and without otherwise."""
score: float | None = None
gated_reason: str = ""
class Company(Model):
"""A row of `companies`, with `affinity` as the agent it was read for
has it (see `agent_companies`); 0 when no agent was asked for."""
key: str
name: str
domain: str = ""
affinity: int = 0
status: str = "researching"
careers_url: str = ""
notes: str = ""
#: The host the careers site runs on, readable or not; "" when unknown.
#: A company with an `ats` and no board subscription is one the app
#: cannot read yet. See `app.jobs.discover`.
ats: str = ""
country: str = ""
city: str = ""
created_at: str
updated_at: str
class Subscription(Model):
"""A row of `subscriptions`. `company` and `board` are the two labels
newsdesk calls outlet and section."""
id: int
source: str
target: str
label: str
company: str = ""
board: str = ""
params: JsonDict = {}
subscribed: int = 1
fetch: int = 1
show: int = 1
added_at: str
target_id: str = ""
class Application(Model):
"""A row of `applications`. `id` empty means not saved yet; the store mints
one. `role_uid` is None for a role that was never posted."""
id: str = ""
role_uid: str | None = None
company_key: str = ""
role_title: str = ""
url: str = ""
status: Status = "saved"
source: str = ""
applied_at: str | None = None
next_action: str = ""
next_date: str | None = None
contact: str = ""
notes: str = ""
created_at: str = ""
updated_at: str = ""
# ------------------------------------------------------------------ agents
#: The role columns a gate can name. `company` matches on the company key;
#: `place` is a case-insensitive substring, like the place filter; the rest are
#: case-insensitive equality, except `salary_min` (at least the value) and
#: `languages` (every language the role requires is one of the values).
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company"]
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
GateMode = Literal["require", "any", "prefer"]
class Gate(Model):
"""One condition of an agent. `require` gates a role that fails it (the
row is kept with the reason); the `any` gates together are one group a
role must match at least one of ("in Denmark, or remote"); `prefer` adds
`weight` to its score when it matches. A gate rejects on evidence only: a
role whose column is empty is not rejected, since `extract` fills the
column later for the roles that rank, and only for those."""
field: GateField
values: list[str | int | float] = Field(min_length=1)
mode: GateMode = "require"
weight: float = Field(1.0, ge=0)
class Weights(Model):
"""How much each score term counts. Summed with the prefer gates' weights
into the denominator, so a score stays in 0..1 whatever the numbers."""
fit: float = Field(0.6, ge=0)
recency: float = Field(0.2, ge=0)
affinity: float = Field(0.2, ge=0)
#: Cosine to the liked-minus-passed direction of the agent's votes.
taste: float = Field(0.2, ge=0)
class Agent(Model):
"""A row of `agents`: a named search with its own intent, gates and
weights. The cached intent vector is a column this does not declare."""
id: int
name: str
intent: str = ""
gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = []
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
enabled: bool = True
created_at: str
class Candidate(Model):
"""What scoring reads about one open role for one agent: when it was
first seen, its company's affinity for that agent, and per gate `1`
match, `0` no match, `-1` the column is empty."""
uid: str
first_seen: str
affinity: int = 0
gates: list[int] = []
class ApplicationEvent(Model):
id: int
application_id: str
status: Status
at: str