Files
jobboard/app/models.py
T

1152 lines
43 KiB
Python

"""The shapes roledesk hands round, and the keys that decide two rows are one role.
One set of models, as in newsdesk: the row the table returns and the object
the API serves are the same declaration. `Model.from_row` ignores columns a
model does not declare, so a new column is never a 500 somewhere else.
The three keys cluster roles on `(company_key, title_key, place_key)`. The same
title at two companies is two jobs, and the same ad on two boards is one.
"""
from __future__ import annotations
import json
import re
import sys
import unicodedata
from functools import lru_cache
from pathlib import Path
from typing import Annotated, Any, Literal, get_args
from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator
from desk.models import JsonDict, Model, ModelTaskResult
# ------------------------------------------------------------------- keys
#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x),
#: (w/m/div), (all genders), (gn). Matched with or without the parentheses.
_GENDER = re.compile(
r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)"
r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)"
r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b",
re.IGNORECASE,
)
#: Legal forms cut from the end of a company name, so "Celonis SE" on one
#: board and "celonis" on another share a key.
_LEGAL = re.compile(
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
re.IGNORECASE,
)
def _fold(text: str) -> str:
"""Casefolded, accents stripped, punctuation to spaces, whitespace collapsed."""
text = unicodedata.normalize("NFKD", text.casefold())
text = "".join(c for c in text if not unicodedata.combining(c))
text = re.sub(r"[\W_]+", " ", text)
return " ".join(text.split())
def title_key(title: str) -> str:
"""`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`."""
return _fold(_GENDER.sub(" ", title or ""))
def place_key(place: str) -> str:
"""`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`."""
return _fold(place or "")
def company_key(company: str) -> str:
"""A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the
primary key of `companies`."""
name = _LEGAL.sub("", (company or "").strip())
return _fold(name).replace(" ", "-")
def company_fold(company: str) -> str:
"""What `company_aliases` keys on: `company_key`'s cut of the legal form,
then the place fold, so "Orsted A/S" and "Ørsted" are both `orsted`. The
key itself keeps `ø`, since it is a primary key."""
return _place_fold(_LEGAL.sub("", (company or "").strip()))
# ---------------------------------------------------------------- geography
#: Letters NFKD leaves whole because they are letters of their own rather
#: than a base letter with a mark, spelt the way boards write them in ASCII.
_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe",
"þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"})
def _place_fold(text: str) -> str:
"""`_fold` for the place lookup only: "Wrocław", "Łódź" and "København"
meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is,
since it also builds `company_key`, the primary key of `companies`."""
return _fold((text or "").casefold().translate(_LETTERS))
#: Every ISO 3166-1 country by its English name, plus the names boards also
#: write for it: its own language's for a European one, and the usual short
#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one.
_COUNTRY_NAMES = """
AD Andorra
AE United Arab Emirates | UAE | U.A.E. | Emirates
AF Afghanistan
AG Antigua and Barbuda
AI Anguilla
AL Albania | Shqipëria | Shqipëri
AM Armenia | Hayastan
AO Angola
AQ Antarctica
AR Argentina
AS American Samoa
AT Austria | Österreich
AU Australia
AW Aruba
AX Åland Islands | Åland
AZ Azerbaijan | Azərbaycan
BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina
BB Barbados
BD Bangladesh
BE Belgium | België | Belgique | Belgien
BF Burkina Faso
BG Bulgaria | България | Balgariya
BH Bahrain
BI Burundi
BJ Benin
BL Saint Barthélemy
BM Bermuda
BN Brunei | Brunei Darussalam
BO Bolivia
BQ Caribbean Netherlands | Bonaire
BR Brazil | Brasil
BS Bahamas
BT Bhutan
BV Bouvet Island
BW Botswana
BY Belarus | Беларусь | Belarus'
BZ Belize
CA Canada
CC Cocos Islands | Cocos (Keeling) Islands
CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa
CF Central African Republic
CG Republic of the Congo | Congo-Brazzaville
CH Switzerland | Schweiz | Suisse | Svizzera | Svizra
CI Côte d'Ivoire | Ivory Coast
CK Cook Islands
CL Chile
CM Cameroon
CN China | People's Republic of China
CO Colombia
CR Costa Rica
CU Cuba
CV Cabo Verde | Cape Verde
CW Curaçao
CX Christmas Island
CY Cyprus | Κύπρος | Kypros | Kibris
CZ Czechia | Czech Republic | Česko | Česká republika
DE Germany | Deutschland
DJ Djibouti
DK Denmark | Danmark
DM Dominica
DO Dominican Republic
DZ Algeria
EC Ecuador
EE Estonia | Eesti
EG Egypt
EH Western Sahara
ER Eritrea
ES Spain | España | Espanya
ET Ethiopia
FI Finland | Suomi
FJ Fiji
FK Falkland Islands
FM Micronesia
FO Faroe Islands | Føroyar | Færøerne
FR France
GA Gabon
GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland
GD Grenada
GE Georgia | Sakartvelo
GF French Guiana
GG Guernsey
GH Ghana
GI Gibraltar
GL Greenland
GM Gambia
GN Guinea
GP Guadeloupe
GQ Equatorial Guinea
GR Greece | Ελλάδα | Hellas | Ellada
GS South Georgia and the South Sandwich Islands
GT Guatemala
GU Guam
GW Guinea-Bissau
GY Guyana
HK Hong Kong
HM Heard Island and McDonald Islands
HN Honduras
HR Croatia | Hrvatska
HT Haiti
HU Hungary | Magyarország
ID Indonesia
IE Ireland | Éire
IL Israel
IM Isle of Man
IN India
IO British Indian Ocean Territory
IQ Iraq
IR Iran
IS Iceland | Ísland
IT Italy | Italia
JE Jersey
JM Jamaica
JO Jordan
JP Japan
KE Kenya
KG Kyrgyzstan
KH Cambodia
KI Kiribati
KM Comoros
KN Saint Kitts and Nevis
KP North Korea
KR South Korea | Korea | Republic of Korea
KW Kuwait
KY Cayman Islands
KZ Kazakhstan
LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic
LB Lebanon
LC Saint Lucia
LI Liechtenstein
LK Sri Lanka
LR Liberia
LS Lesotho
LT Lithuania | Lietuva
LU Luxembourg | Lëtzebuerg | Luxemburg
LV Latvia | Latvija
LY Libya
MA Morocco
MC Monaco
MD Moldova | Republic of Moldova
ME Montenegro | Crna Gora
MF Saint Martin
MG Madagascar
MH Marshall Islands
MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија
ML Mali
MM Myanmar | Burma
MN Mongolia
MO Macao | Macau
MP Northern Mariana Islands
MQ Martinique
MR Mauritania
MS Montserrat
MT Malta
MU Mauritius
MV Maldives
MW Malawi
MX Mexico | México
MY Malaysia
MZ Mozambique
NA Namibia
NC New Caledonia
NE Niger
NF Norfolk Island
NG Nigeria
NI Nicaragua
NL Netherlands | The Netherlands | Nederland | Holland
NO Norway | Norge | Noreg
NP Nepal
NR Nauru
NU Niue
NZ New Zealand | Aotearoa
OM Oman
PA Panama | Panamá
PE Peru | Perú
PF French Polynesia
PG Papua New Guinea
PH Philippines
PK Pakistan
PL Poland | Polska
PM Saint Pierre and Miquelon
PN Pitcairn Islands
PR Puerto Rico
PS Palestine
PT Portugal
PW Palau
PY Paraguay
QA Qatar
RE Réunion
RO Romania | România
RS Serbia | Srbija | Србија
RU Russia | Russian Federation | Россия | Rossiya
RW Rwanda
SA Saudi Arabia | KSA
SB Solomon Islands
SC Seychelles
SD Sudan
SE Sweden | Sverige
SG Singapore
SH Saint Helena
SI Slovenia | Slovenija
SJ Svalbard and Jan Mayen | Svalbard
SK Slovakia | Slovensko
SL Sierra Leone
SM San Marino
SN Senegal
SO Somalia
SR Suriname
SS South Sudan
ST São Tomé and Príncipe
SV El Salvador
SX Sint Maarten
SY Syria
SZ Eswatini | Swaziland
TC Turks and Caicos Islands
TD Chad
TF French Southern Territories
TG Togo
TH Thailand
TJ Tajikistan
TK Tokelau
TL Timor-Leste | East Timor
TM Turkmenistan
TN Tunisia
TO Tonga
TR Turkey | Türkiye
TT Trinidad and Tobago
TV Tuvalu
TW Taiwan
TZ Tanzania
UA Ukraine | Україна | Ukraina
UG Uganda
UM United States Minor Outlying Islands
US United States | United States of America | USA | U.S.A. | US | U.S.
UY Uruguay
UZ Uzbekistan
VA Vatican City | Holy See
VC Saint Vincent and the Grenadines
VE Venezuela
VG British Virgin Islands
VI U.S. Virgin Islands | US Virgin Islands
VN Vietnam | Viet Nam
VU Vanuatu
WF Wallis and Futuna
WS Samoa
XK Kosovo | Kosova
YE Yemen
YT Mayotte
ZA South Africa
ZM Zambia
ZW Zimbabwe
"""
#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK.
_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line)
#: The countries `EU` stands for when a line names several: the UN
#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and
#: the Council of Europe's members in the Caucasus and Turkey, which
#: European employers list beside their other offices.
_EUROPE = frozenset((
"AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
"DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE",
"IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL",
"NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA",
"XK",
))
_EUROPE_OR_EU = _EUROPE | {"EU"}
#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them.
_REGIONS = ("europe", "emea", "eu", "european union", "eea")
#: The cities boards name most, and a few regions below a country. A US
#: state or a Canadian province in the same part of the line beats one of
#: these ("London, Ontario" is Canada); a country name does not lose to it.
_CITIES: dict[str, str] = {
**dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck"), "AT"),
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege",
"luik"), "BE"),
**dict.fromkeys(("sofia",), "BG"),
**dict.fromkeys(("zagreb",), "HR"),
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
**dict.fromkeys(("prague", "praha", "brno"), "CZ"),
**dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg",
"region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby",
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
**dict.fromkeys(("tallinn", "tartu"), "EE"),
**dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"),
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
"lille", "rennes", "amiens"), "FR"),
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
"stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg",
"nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund",
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck"), "DE"),
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
**dict.fromkeys(("budapest",), "HU"),
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
"florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples",
"napoli", "venice", "venezia", "genoa", "genova", "bolzano"), "IT"),
**dict.fromkeys(("riga",), "LV"),
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
**dict.fromkeys(("luxembourg", "luxemburg"), "LU"),
**dict.fromkeys(("valletta",), "MT"),
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
"delft"), "NL"),
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
"PL"),
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara"), "RO"),
**dict.fromkeys(("belgrade", "beograd"), "RS"),
**dict.fromkeys(("bratislava",), "SK"),
**dict.fromkeys(("ljubljana",), "SI"),
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
"granada", "santiago de compostela"), "ES"),
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras",
"linkoping", "orebro", "norrkoping", "jonkoping"), "SE"),
**dict.fromkeys(("reykjavik",), "IS"),
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
"oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff",
"guildford", "truro", "swansea", "blackpool", "coventry", "brighton",
"weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley",
"slough", "southampton", "swindon", "liverpool", "redhill",
"milton keynes"), "GB"),
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
"sankt gallen", "st gallen", "lucerne", "luzern"), "CH"),
**dict.fromkeys(("kyiv", "kiev"), "UA"),
**dict.fromkeys(("istanbul", "ankara"), "TR"),
**dict.fromkeys(("tirana",), "AL"),
**dict.fromkeys(("yerevan",), "AM"),
**dict.fromkeys(("tbilisi",), "GE"),
**dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle",
"boston", "chicago", "austin", "los angeles", "palo alto",
"mountain view", "san mateo", "foster city", "sunnyvale", "san jose",
"menlo park", "redwood city", "costa mesa", "denver", "atlanta",
"washington dc", "washington d c", "miami", "dallas", "houston",
"phoenix", "tempe", "portland", "salt lake city", "pittsburgh",
"philadelphia", "detroit", "san diego", "raleigh", "nashville",
"minneapolis", "cambridge ma", "bay area", "arlington"), "US"),
**dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"),
"CA"),
**dict.fromkeys(("buenos aires",), "AR"),
**dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"),
**dict.fromkeys(("santiago", "santiago de chile"), "CL"),
**dict.fromkeys(("bogota",), "CO"),
**dict.fromkeys(("lima", "miraflores"), "PE"),
**dict.fromkeys(("montevideo",), "UY"),
**dict.fromkeys(("managua",), "NI"),
**dict.fromkeys(("santo domingo",), "DO"),
**dict.fromkeys(("san salvador",), "SV"),
**dict.fromkeys(("panama city",), "PA"),
**dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"),
**dict.fromkeys(("dubai", "abu dhabi"), "AE"),
**dict.fromkeys(("riyadh",), "SA"),
**dict.fromkeys(("doha",), "QA"),
**dict.fromkeys(("manama",), "BH"),
**dict.fromkeys(("salmiya",), "KW"),
**dict.fromkeys(("muscat",), "OM"),
**dict.fromkeys(("cairo", "maadi"), "EG"),
**dict.fromkeys(("casablanca",), "MA"),
**dict.fromkeys(("dakar",), "SN"),
**dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
**dict.fromkeys(("almaty",), "KZ"),
**dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"),
**dict.fromkeys(("dhaka",), "BD"),
**dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai",
"gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
**dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"),
**dict.fromkeys(("singapore",), "SG"),
**dict.fromkeys(("manila", "taguig"), "PH"),
**dict.fromkeys(("bangkok",), "TH"),
**dict.fromkeys(("phnom penh",), "KH"),
**dict.fromkeys(("taipei",), "TW"),
**dict.fromkeys(("tokyo", "osaka"), "JP"),
**dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"),
**dict.fromkeys(("seoul",), "KR"),
**dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales",
"queensland"), "AU"),
}
#: US states and Canadian provinces, by name and by the code boards write
#: after a comma ("Boston, MA", "Toronto, ON").
_SUBDIVISIONS = """
US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO
US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA
US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY
US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN
US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH
US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND
US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI
US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT
US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY
CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB
CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL
CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT
CA Nunavut NU
"""
def _subdivisions() -> tuple[dict[str, str], dict[str, str]]:
names: dict[str, str] = {}
codes: dict[str, str] = {}
for line in _SUBDIVISIONS.split("\n"):
if line:
country, rest = line.split(" ", 1)
for entry in rest.split("|"):
name, code = entry.strip().rsplit(" ", 1)
names[_place_fold(name)] = country
codes[code] = country
return names, codes
_STATE_NAMES, _STATE_CODES = _subdivisions()
def _names() -> dict[str, tuple[str, str]]:
"""Every folded name to `(code, kind)`; kind is `country`, `city` or
`state`. A name that is a city and a state ("New York") is the city, so
it does not beat another city in its line. A name that is a country and
a state ("Georgia") is the country here and read as the state only when
the line names a place in the US too."""
names: dict[str, tuple[str, str]] = {}
for name, code in _STATE_NAMES.items():
names[name] = (code, "state")
for name, code in _CITIES.items():
names[_place_fold(name)] = (code, "city")
for line in _COUNTRY_NAMES.split("\n"):
if line:
code, rest = line.split(" ", 1)
for name in rest.split("|"):
names[_place_fold(name)] = (code, "country")
for name in _REGIONS:
names[name] = ("EU", "country")
return names
_NAMES = _names()
#: A state name that is also a country's, read as the state only when the
#: line names a place in that state's country too.
_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES
if _NAMES[name][1] == "country"}
#: The longest name in words, so a lookup tries "new york city" before "new
#: york" and "northern ireland" before "ireland".
_LONGEST = max(len(name.split()) for name in _NAMES)
#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin".
_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b")
#: A country code in parentheses, "Leicester (GB)", read only when no name matched.
_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)")
#: Where one location in a line ends and the next begins. A state beats a
#: city only within one: "London; Ontario" is two places.
_SEGMENT = re.compile(r"[;|/\n]")
def _hits(folded: str) -> list[tuple[str, str, str]]:
"""The names in a folded line, longest first at each word, as
`(name, code, kind)`."""
words = folded.split()
hits = []
i = 0
while i < len(words):
for n in range(min(_LONGEST, len(words) - i), 0, -1):
name = " ".join(words[i:i + n])
if name in _NAMES:
hits.append((name, *_NAMES[name]))
i += n
break
else:
i += 1
return hits
def country_of(place: str) -> str:
"""The one country a board's location line names, as an ISO code; `EU`
when it names several and all of them are in Europe; '' when it names
none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston,
MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or
New York" and "Remote, EU or US" are ''.
Within one location (the line split at `;`, `|` and `/`) the widest unit
named wins: a country beats a state or a province, which beats a city.
So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A
country also beats the region it is in: "London, EMEA" is GB.
A state code that is also a country's ("DE", "CO", "MA"), and "Georgia",
count as the state only when the line names a place in the US too:
"Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather
than Delaware. A country code in parentheses, "Leicester (GB)", is read
only when no name matched."""
segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")]
named = {code for _, hits in segments for _, code, _ in hits}
found: set[str] = set()
for text, hits in segments:
shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named}
countries = {code for name, code, kind in hits
if kind == "country" and code != "EU" and name not in shared}
states = {code for _, code, kind in hits if kind == "state"}
states |= {_SHARED_NAMES[name] for name in shared}
states |= {country for code in _STATE_CODE.findall(text)
if (country := _STATE_CODES.get(code))
and (code not in _ISO or country in named)}
found |= (countries or states or {code for _, code, _ in hits if code != "EU"})
found |= {code for _, code, _ in hits if code == "EU"}
if not found:
found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO}
if len(found) == 1:
return found.pop()
if found and found <= _EUROPE_OR_EU:
countries = found - {"EU"}
return countries.pop() if "EU" in found and len(countries) == 1 else "EU"
return ""
# --------------------------------------------------------------- vocabulary
Remote = Literal["", "remote", "hybrid", "onsite"]
REMOTE: tuple[str, ...] = get_args(Remote)
#: Words in a location line that say how the role is worked, for a board that
#: gives no remote flag of its own: Greenhouse writes "Remote", "Hybrid" or
#: "Distributed" as the location and nothing else.
_REMOTE_WORDS: tuple[tuple[str, str], ...] = (
("hybrid", "hybrid"),
("remote", "remote"), ("distributed", "remote"), ("home based", "remote"),
("home-based", "remote"), ("work from home", "remote"), ("anywhere", "remote"),
("worldwide", "remote"), ("telecommute", "remote"),
("on-site", "onsite"), ("onsite", "onsite"), ("on site", "onsite"),
("in-office", "onsite"), ("in office", "onsite"),
)
def remote_of(place: str) -> str:
"""How a board's location line says the role is worked, as one of
`REMOTE`, or '' when it does not say. "Hybrid" wins over "remote" in the
same line ("Remote or hybrid, Berlin" is hybrid: some days in an office),
and a line that names only a place is ''."""
folded = _fold(place or "")
for word, mode in _REMOTE_WORDS:
if word in folded:
return mode
return ""
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
"director", "exec"]
SENIORITIES: tuple[str, ...] = get_args(Seniority)
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
Permit = Literal["", "sponsors", "no_sponsorship"]
PERMITS: tuple[str, ...] = get_args(Permit)
Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer",
"rejected", "withdrawn"]
STATUSES: tuple[str, ...] = get_args(Status)
#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips the `any` gates).
AFFINITY = range(-1, 3)
#: Why a company is on the list checked by hand. `unread` is a host the app
#: could read and has no parser for yet, `robots` a site whose robots.txt
#: disallows it, `blocked` one that answers a program with a challenge, `key`
#: one whose API answers only with a key lifted from its own script, `group` a
#: company that sits inside its owner's board, `gone` one whose site is dead,
#: and `added` one put there by hand or by an import and not looked at. ''
#: is none of these: the company is read, or is not a matter for the list.
HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"]
HAND_REASONS: tuple[str, ...] = get_args(HandReason)
# -------------------------------------------------------------- posting dates
@lru_cache(maxsize=1)
def _declared_posted() -> dict[str, str]:
"""`fields.posted` of every reader that declares `READS`, by name. Read
once: the readers are loaded when `plugins.readers` is imported. `plugins`
is a top-level folder beside `app`, reached as `app/jobs/extract.py`
reaches it."""
root = str(Path(__file__).resolve().parents[1])
if root not in sys.path:
sys.path.insert(0, root)
from plugins.readers import READERS
return {name: module.READS.fields.posted for name, module in READERS.items()
if getattr(module, "READS", None) is not None}
def trusted_posted(host: str) -> bool:
"""Whether `host` states a date that is safe to read as when a role was
posted: its reader declares `fields.posted="trusted"`. Which field each
host reads, and when it was checked, is in the reader module. A role
without a trusted date keeps `first_seen` as its posted date."""
return _declared_posted().get(host) == "trusted"
#: A `published_at` before this is a parsing slip or a placeholder, not a
#: real posting date: no board here existed before 2015.
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
def posted_host(raw: dict[str, Any]) -> str:
"""The host a role's dates came from, read the way `ats` and `search`
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
`lever`, ...), `provider` for a cross-company search (`jobindex`,
`adzuna`). '' when neither plugin wrote one."""
return str(raw.get("board") or raw.get("provider") or "")
def posted_at(published_at: str, first_seen: str, host: str) -> str:
"""The date a role counts as posted: the earlier of `first_seen` (when
the app itself first stored it) and `published_at` (what the board
claims), but only when `trusted_posted(host)` and
`published_at` is neither absurdly old nor later than `first_seen` — a
role cannot be posted after the app already saw it, trusted host or not,
which also catches a `published_at` the board reports in the future
(`first_seen` is never later than the moment this runs). Both dates are
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
is exact and needs no parsing. A role with no trusted date, or with one
that fails the check, keeps `first_seen`."""
if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen:
return published_at
return first_seen
def _json(value: Any, empty: str) -> Any:
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
value that is already parsed passes, and one that does not parse is the
empty value rather than a 500."""
if not isinstance(value, str):
return value
try:
return json.loads(value or empty)
except ValueError:
return json.loads(empty)
#: A column holding a JSON array of strings, such as `roles.languages`.
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
# ------------------------------------------------------------------ models
class RoleCopy(Model):
"""One open copy of a kept role, as the reader lists it under "Also
posted": where it is listed (`host`, the search provider or the board's
ATS) and the link that opens it there."""
uid: str
host: str = ""
place: str = ""
original_url: str = ""
posted_at: str = ""
class RolePlace(Model):
"""One place of a role (`role_places`): the host's `text`, and the place
of the gazetteer it resolves to, `place_id` None (and `name` and `kind`
empty) while it resolves to none. `url` is the link of a copy folded into
the role, '' for the role's own places."""
position: int = 0
text: str = ""
place_id: int | None = None
name: str = ""
kind: str = ""
url: str = ""
class Role(Model):
"""A row of `roles`: one posting on one board."""
uid: str
source: str
source_id: str
subscription_id: int | None = None
company_key: str = ""
company: str = ""
title: str = ""
title_key: str = ""
place: str = ""
place_key: str = ""
country: str = ""
remote: Remote = ""
salary_min: int | None = None
salary_max: int | None = None
currency: str = ""
#: The pay a board stated, as one human-readable line, whether or not it
#: could be normalised into `salary_min`/`salary_max`/`currency`
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
salary_text: str = ""
#: The level shown and filtered on: the first filled of `seniority_model`,
#: `seniority_stated` and `seniority_title`.
seniority: str = ""
seniority_model: str = ""
seniority_stated: str = ""
seniority_title: str = ""
function: str = ""
permit: Permit = ""
#: ISO 639-1 codes the posting says are required, lowercase.
languages: JsonStrings = []
description: str = ""
feed_text: str = ""
#: The short text the host's list carries, until `description` arrives.
teaser: str = ""
original_url: str = ""
published_at: str
#: The date this role counts as posted: see `posted_at`. Used for
#: `recency` and the feed's newest order; `first_seen` stays what it
#: always was, when the app itself first stored the role.
posted_at: str
first_seen: str
last_seen: str
closed_at: str | None = None
raw: JsonDict = {}
extracted_at: str | None = None
created_at: str
#: The vote of the agent the role was read for (see `role_votes`): None
#: unjudged, else -1 pass, 0 neutral, 1 like.
vote: Literal[-1, 0, 1] | None = None
voted_at: str | None = None
#: NULL unread, else when the feed opened it or judged it. A vote stamps
#: this too, see `Store.set_vote`: voting is reading.
read_at: str | None = None
summary: str = ""
summarised_at: str | None = None
#: The posting page as plain text, for a role whose description is a teaser.
article_text: str = ""
article_at: str | None = None
#: The open vocabulary `summarise` read off the posting, sorted. Not a
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
tags: list[str] = []
#: The uid of the kept role when this one is a copy of the same posting
#: (`app.dupes`, `Store.rebuild_dupes`), else None. A copy is left out of
#: the feed and the counts but still opens by uid.
dup_of: str | None = None
#: The open copies of this role, oldest first, when it is the kept one;
#: empty otherwise. Not a column: filled by `Store.role`.
copies: list[RoleCopy] = []
#: Every place of the role, in the host's order. Not a column: filled by
#: `Store.role` from `role_places`.
places: list[RolePlace] = []
@computed_field # type: ignore[prop-decorator]
@property
def salary_state(self) -> Literal["normalised", "stated", ""]:
"""Not a column: 'normalised' when `salary_min` or `salary_max` is
set, 'stated' when only `salary_text` is, '' otherwise. Gates and
scoring read `salary_min` alone and never this (docs/decisions.md);
it is what the feed's Pay filter and badge read instead
(`Store.PAY`, `static/roles.js`)."""
if self.salary_min is not None or self.salary_max is not None:
return "normalised"
if self.salary_text:
return "stated"
return ""
class ExtractResult(ModelTaskResult):
"""What one run of `extract` did. See app/jobs/extract.py."""
#: Roles the model answered for, whether or not it found anything to fill.
extracted: int = 0
#: Columns written, across those roles. A role answered with nothing the
#: posting states adds to `extracted` and not to this.
filled: int = 0
@property
def done(self) -> int:
return self.extracted
class ScoredRole(Role):
"""A role as `/api/roles/{uid}` serves it: with one agent's score and
gate reason when an agent was asked for, and without otherwise."""
score: float | None = None
gated_reason: str = ""
#: The agent's prompt that gave the role its fit; "" when fit did not
#: count (gated, no vector, no prompt with text).
prompt: str = ""
#: How many characters `RoleRow.excerpt` keeps at most: the feed row's own
#: cap, built once here rather than by the client, so a list response never
#: carries more of a role's text than a row draws.
EXCERPT_LIMIT = 200
def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str:
"""`text` (a role's summary, or the opening of its description) with its
whitespace collapsed, cut to `limit` characters at a word boundary with
an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a
`SUBSTR` of the source column rather than the whole of it, so the feed's
list never carries what only the reader needs."""
collapsed = " ".join((text or "").split())
if len(collapsed) <= limit:
return collapsed
cut = collapsed[:limit]
space = cut.rfind(" ")
trimmed = cut[:space] if space > limit * 0.6 else cut
return trimmed.rstrip(" ,.;:" + chr(0x2013) + "-") + "…"
class RoleRow(Model):
"""One row of the feed: what `GET /api/roles` serves per role, the
fields a row draws and nothing else — never `description`, `feed_text`,
`raw` or `article_text`, the four columns docs/decisions.md measures at
about 5.5 KB a role between them. The reader fetches the rest by uid when
it opens a role (`GET /api/roles/{uid}`, the full `Role`).
`excerpt` is built on the server (see `excerpt`, above): the summary when
there is one, else the description's opening, cut to `EXCERPT_LIMIT`
characters. The pay fields are what the row's badge reads: the three
normalised columns, the line the board stated, and `salary_state`,
derived as on `Role`. `original_url` is one short string, kept so the
`o` key opens a selected row's posting without opening the reader.
`seniority` is the title's level until extract reads the posting, ''
when neither says. `copies` counts the open copies the feed hides behind
this row."""
uid: str
company_key: str = ""
company: str = ""
title: str = ""
place: str = ""
country: str = ""
remote: Remote = ""
function: str = ""
seniority: str = ""
salary_min: int | None = None
salary_max: int | None = None
currency: str = ""
salary_text: str = ""
original_url: str = ""
posted_at: str = ""
score: float | None = None
gated_reason: str = ""
prompt: str = ""
vote: Literal[-1, 0, 1] | None = None
voted_at: str | None = None
read_at: str | None = None
tags: list[str] = []
excerpt: str = ""
#: How many open roles are copies of this one (`Role.copies`), 0 when none.
copies: int = 0
@computed_field # type: ignore[prop-decorator]
@property
def salary_state(self) -> str:
"""As `Role.salary_state`: 'normalised', 'stated' or ''."""
if self.salary_min is not None or self.salary_max is not None:
return "normalised"
return "stated" if self.salary_text else ""
class Company(Model):
"""A row of `companies`, with `affinity` as the agent it was read for
has it (see `agent_companies`); 0 when no agent was asked for."""
key: str
name: str
domain: str = ""
affinity: int = 0
status: str = "researching"
careers_url: str = ""
notes: str = ""
#: The host the careers site runs on, readable or not; "" when unknown.
#: A company with an `ats` and no board subscription is one the app
#: cannot read yet. See `app.jobs.discover`.
ats: str = ""
country: str = ""
city: str = ""
#: The name `ats` knows the company by, the `acme` of acme.teamtailor.com;
#: "" where discovery found none. Kept for a host no parser reads yet.
ats_slug: str = ""
#: The Watchlist's rating, global and not an agent's: None not yet rated,
#: 1 worth checking by hand, 0 not.
watch: int | None = None
#: When the careers page was last opened from the Watchlist; "" never.
careers_opened_at: str = ""
#: Why the company is read by hand, one of `HAND_REASONS`; "" when it is
#: not on that list.
hand_reason: str = ""
created_at: str
updated_at: str
class Subscription(Model):
"""A row of `subscriptions`. `company` and `board` are the two labels
newsdesk calls outlet and section."""
id: int
source: str
target: str
label: str
company: str = ""
board: str = ""
params: JsonDict = {}
subscribed: int = 1
fetch: int = 1
show: int = 1
added_at: str
target_id: str = ""
class Application(Model):
"""A row of `applications`. `id` empty means not saved yet; the store mints
one. `role_uid` is None for a role that was never posted."""
id: str = ""
role_uid: str | None = None
company_key: str = ""
role_title: str = ""
url: str = ""
status: Status = "saved"
source: str = ""
applied_at: str | None = None
next_action: str = ""
next_date: str | None = None
contact: str = ""
notes: str = ""
created_at: str = ""
updated_at: str = ""
# ------------------------------------------------------------------ agents
#: The role columns a gate can name. `company` matches on the company key;
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; the rest are
#: case-insensitive equality, except `salary_min` (at least the value) and
#: `languages` (every language the role requires is one of the values).
GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min",
"currency", "permit", "languages", "company"]
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
GateMode = Literal["require", "any", "prefer"]
class Gate(Model):
"""One condition of an agent. `require` gates a role that fails it (the
row is kept with the reason); the `any` gates together are one group a
role must match at least one of ("in Denmark, or remote"); `prefer` adds
`weight` to its score when it matches. A gate rejects on evidence only: a
role whose column is empty is not rejected, since `extract` fills the
column later for the roles that rank, and only for those."""
field: GateField
values: list[str | int | float] = Field(min_length=1)
mode: GateMode = "require"
weight: float = Field(1.0, ge=0)
class Weights(Model):
"""How much each score term counts. Summed with the prefer gates' weights
into the denominator, so a score stays in 0..1 whatever the numbers."""
fit: float = Field(0.6, ge=0)
recency: float = Field(0.2, ge=0)
affinity: float = Field(0.2, ge=0)
#: Cosine to the liked-minus-passed direction of the agent's votes.
taste: float = Field(0.2, ge=0)
class Limits(Model):
"""How far down an agent's ranking the two model stages read. Each pair
is a top N, the backstop, and a minimum score, the real stop: a role
is read when it is among the agent's best `extract_top` ungated scores
and not below `extract_min`, and summarised likewise. Fit is a
calibrated level (`score.scale`), so a bar means the same in every run.
A fresh role with the worst fit scores about 0.27 on the default
weights, a perfect one about 0.87. Measured on 16,841 open roles for the
"Copenhagen engineering" starter agent (4,156 ungated, no votes), 0.55
admits about 300 roles to extract and 0.65 about 35 to summarise, the
long answer; the top N catches an agent whose bar admits far more. An
agent keeps the limits stored with it; only a key its JSON lacks
follows these defaults."""
extract_top: int = Field(500, ge=0)
extract_min: float = Field(0.55, ge=0, le=1)
summarise_top: int = Field(30, ge=0)
summarise_min: float = Field(0.65, ge=0, le=1)
class Prompt(Model):
"""One of an agent's descriptions of what it looks for. Each is embedded
on its own and a role's fit is its best prompt's (see `app.jobs.score`),
so one agent can hunt several kinds of role without blending them. A
prompt with no text has no fit to give."""
name: str
text: str = ""
@field_validator("name")
@classmethod
def _named(cls, name: str) -> str:
name = name.strip()
if not name:
raise ValueError("a prompt needs a name")
return name
def check_prompts(prompts: list[Prompt]) -> list[Prompt]:
"""The prompts as they are, or ValueError when two share a name
regardless of case."""
seen: set[str] = set()
for prompt in prompts:
if prompt.name.casefold() in seen:
raise ValueError(f"two prompts are named {prompt.name!r}")
seen.add(prompt.name.casefold())
return prompts
#: An agent's prompts, in order, their names unique regardless of case.
Prompts = Annotated[list[Prompt], AfterValidator(check_prompts)]
class Agent(Model):
"""A row of `agents`: a named search with its own prompts, gates, weights
and limits. The prompts are rows of `agent_prompts`, read alongside; their
cached vectors are columns this does not declare."""
id: int
name: str
prompts: Prompts = []
gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = []
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
limits: Annotated[Limits, BeforeValidator(lambda v: _json(v, "{}"))] = Limits()
enabled: bool = True
created_at: str
class Candidate(Model):
"""What scoring reads about one open role for one agent: its posted
date (for `recency`), its company's affinity for that agent, and per
gate `1` match, `0` no match, `-1` the column is empty."""
uid: str
posted_at: str
affinity: int = 0
gates: list[int] = []
class ApplicationEvent(Model):
id: int
application_id: str
status: Status
at: str