The CV is cut into chunks inside e5's window, embedded like a prompt and cached per agent under a hash of the model, chunking and text. A role's cv_fit (scores.cv_fit, migration 31) is its best chunk cosine, scaled with its own calibration band: fit's clipped 88 of agent 2's ungated roles to 1. It is no term of the Agent match score. judge_cv_top (default 50) sits beside judge_top and judge_min on the agents page and card. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1608 lines
66 KiB
Python
1608 lines
66 KiB
Python
"""The shapes roledesk hands round, and the keys that decide two rows are one role.
|
|
|
|
One set of models, as in newsdesk: the row the table returns and the object
|
|
the API serves are the same declaration. `Model.from_row` ignores columns a
|
|
model does not declare, so a new column is never a 500 somewhere else.
|
|
|
|
The three keys cluster roles on `(company_key, title_key, place_key)`. The same
|
|
title at two companies is two jobs, and the same ad on two boards is one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import unicodedata
|
|
from datetime import date
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from typing import Annotated, Any, Literal, get_args
|
|
|
|
from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator
|
|
|
|
from desk.models import JsonDict, Model, ModelTaskResult
|
|
|
|
# ------------------------------------------------------------------- keys
|
|
|
|
#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x),
|
|
#: (w/m/div), (all genders), (gn). Matched with or without the parentheses.
|
|
_GENDER = re.compile(
|
|
r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)"
|
|
r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)"
|
|
r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
#: Legal forms cut from the end of a company name, so "Celonis SE" on one
|
|
#: board and "celonis" on another share a key.
|
|
_LEGAL = re.compile(
|
|
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
|
|
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _fold(text: str) -> str:
|
|
"""Casefolded, accents stripped, punctuation to spaces, whitespace collapsed."""
|
|
text = unicodedata.normalize("NFKD", text.casefold())
|
|
text = "".join(c for c in text if not unicodedata.combining(c))
|
|
text = re.sub(r"[\W_]+", " ", text)
|
|
return " ".join(text.split())
|
|
|
|
|
|
def title_key(title: str) -> str:
|
|
"""`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`."""
|
|
return _fold(_GENDER.sub(" ", title or ""))
|
|
|
|
|
|
def place_key(place: str) -> str:
|
|
"""`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`."""
|
|
return _fold(place or "")
|
|
|
|
|
|
def company_key(company: str) -> str:
|
|
"""A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the
|
|
primary key of `companies`."""
|
|
name = _LEGAL.sub("", (company or "").strip())
|
|
return _fold(name).replace(" ", "-")
|
|
|
|
|
|
def company_fold(company: str) -> str:
|
|
"""What `company_aliases` keys on: `company_key`'s cut of the legal form,
|
|
then the place fold, so "Orsted A/S" and "Ørsted" are both `orsted`. The
|
|
key itself keeps `ø`, since it is a primary key."""
|
|
return _place_fold(_LEGAL.sub("", (company or "").strip()))
|
|
|
|
|
|
# ---------------------------------------------------------------- geography
|
|
|
|
#: Letters NFKD leaves whole because they are letters of their own rather
|
|
#: than a base letter with a mark, spelt the way boards write them in ASCII.
|
|
_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe",
|
|
"þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"})
|
|
|
|
|
|
def _place_fold(text: str) -> str:
|
|
"""`_fold` for the place lookup only: "Wrocław", "Łódź" and "København"
|
|
meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is,
|
|
since it also builds `company_key`, the primary key of `companies`."""
|
|
return _fold((text or "").casefold().translate(_LETTERS))
|
|
|
|
|
|
#: Every ISO 3166-1 country by its English name, plus the names boards also
|
|
#: write for it: its own language's for a European one, and the usual short
|
|
#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one.
|
|
_COUNTRY_NAMES = """
|
|
AD Andorra
|
|
AE United Arab Emirates | UAE | U.A.E. | Emirates
|
|
AF Afghanistan
|
|
AG Antigua and Barbuda
|
|
AI Anguilla
|
|
AL Albania | Shqipëria | Shqipëri
|
|
AM Armenia | Hayastan
|
|
AO Angola
|
|
AQ Antarctica
|
|
AR Argentina
|
|
AS American Samoa
|
|
AT Austria | Österreich
|
|
AU Australia
|
|
AW Aruba
|
|
AX Åland Islands | Åland
|
|
AZ Azerbaijan | Azərbaycan
|
|
BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina
|
|
BB Barbados
|
|
BD Bangladesh
|
|
BE Belgium | België | Belgique | Belgien
|
|
BF Burkina Faso
|
|
BG Bulgaria | България | Balgariya
|
|
BH Bahrain
|
|
BI Burundi
|
|
BJ Benin
|
|
BL Saint Barthélemy
|
|
BM Bermuda
|
|
BN Brunei | Brunei Darussalam
|
|
BO Bolivia
|
|
BQ Caribbean Netherlands | Bonaire
|
|
BR Brazil | Brasil
|
|
BS Bahamas
|
|
BT Bhutan
|
|
BV Bouvet Island
|
|
BW Botswana
|
|
BY Belarus | Беларусь | Belarus'
|
|
BZ Belize
|
|
CA Canada
|
|
CC Cocos Islands | Cocos (Keeling) Islands
|
|
CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa
|
|
CF Central African Republic
|
|
CG Republic of the Congo | Congo-Brazzaville
|
|
CH Switzerland | Schweiz | Suisse | Svizzera | Svizra | CHE
|
|
CI Côte d'Ivoire | Ivory Coast
|
|
CK Cook Islands
|
|
CL Chile
|
|
CM Cameroon
|
|
CN China | People's Republic of China
|
|
CO Colombia
|
|
CR Costa Rica
|
|
CU Cuba
|
|
CV Cabo Verde | Cape Verde
|
|
CW Curaçao
|
|
CX Christmas Island
|
|
CY Cyprus | Κύπρος | Kypros | Kibris
|
|
CZ Czechia | Czech Republic | Česko | Česká republika
|
|
DE Germany | Deutschland | DEU | Deutschlandweit
|
|
DJ Djibouti
|
|
DK Denmark | Danmark | Dänemark
|
|
DM Dominica
|
|
DO Dominican Republic
|
|
DZ Algeria
|
|
EC Ecuador
|
|
EE Estonia | Eesti
|
|
EG Egypt
|
|
EH Western Sahara
|
|
ER Eritrea
|
|
ES Spain | España | Espanya | ESP | Spanien | Espagne
|
|
ET Ethiopia
|
|
FI Finland | Suomi
|
|
FJ Fiji
|
|
FK Falkland Islands
|
|
FM Micronesia
|
|
FO Faroe Islands | Føroyar | Færøerne
|
|
FR France | FRA
|
|
GA Gabon
|
|
GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland | GBR
|
|
GD Grenada
|
|
GE Georgia | Sakartvelo
|
|
GF French Guiana
|
|
GG Guernsey
|
|
GH Ghana
|
|
GI Gibraltar
|
|
GL Greenland
|
|
GM Gambia
|
|
GN Guinea
|
|
GP Guadeloupe
|
|
GQ Equatorial Guinea
|
|
GR Greece | Ελλάδα | Hellas | Ellada
|
|
GS South Georgia and the South Sandwich Islands
|
|
GT Guatemala
|
|
GU Guam
|
|
GW Guinea-Bissau
|
|
GY Guyana
|
|
HK Hong Kong | 香港
|
|
HM Heard Island and McDonald Islands
|
|
HN Honduras
|
|
HR Croatia | Hrvatska
|
|
HT Haiti
|
|
HU Hungary | Magyarország
|
|
ID Indonesia
|
|
IE Ireland | Éire
|
|
IL Israel
|
|
IM Isle of Man
|
|
IN India
|
|
IO British Indian Ocean Territory
|
|
IQ Iraq
|
|
IR Iran
|
|
IS Iceland | Ísland
|
|
IT Italy | Italia
|
|
JE Jersey
|
|
JM Jamaica
|
|
JO Jordan
|
|
JP Japan | JPN
|
|
KE Kenya
|
|
KG Kyrgyzstan
|
|
KH Cambodia
|
|
KI Kiribati
|
|
KM Comoros
|
|
KN Saint Kitts and Nevis
|
|
KP North Korea
|
|
KR South Korea | Korea | Republic of Korea
|
|
KW Kuwait
|
|
KY Cayman Islands
|
|
KZ Kazakhstan
|
|
LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic
|
|
LB Lebanon
|
|
LC Saint Lucia
|
|
LI Liechtenstein
|
|
LK Sri Lanka
|
|
LR Liberia
|
|
LS Lesotho
|
|
LT Lithuania | Lietuva
|
|
LU Luxembourg | Lëtzebuerg | Luxemburg
|
|
LV Latvia | Latvija
|
|
LY Libya
|
|
MA Morocco
|
|
MC Monaco
|
|
MD Moldova | Republic of Moldova
|
|
ME Montenegro | Crna Gora
|
|
MF Saint Martin
|
|
MG Madagascar
|
|
MH Marshall Islands
|
|
MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија
|
|
ML Mali
|
|
MM Myanmar | Burma
|
|
MN Mongolia
|
|
MO Macao | Macau
|
|
MP Northern Mariana Islands
|
|
MQ Martinique
|
|
MR Mauritania
|
|
MS Montserrat
|
|
MT Malta
|
|
MU Mauritius
|
|
MV Maldives
|
|
MW Malawi
|
|
MX Mexico | México | MEX
|
|
MY Malaysia | MYS
|
|
MZ Mozambique
|
|
NA Namibia
|
|
NC New Caledonia
|
|
NE Niger
|
|
NF Norfolk Island
|
|
NG Nigeria
|
|
NI Nicaragua
|
|
NL Netherlands | The Netherlands | Nederland | Holland | NLD | Niederlande
|
|
NO Norway | Norge | Noreg
|
|
NP Nepal
|
|
NR Nauru
|
|
NU Niue
|
|
NZ New Zealand | Aotearoa
|
|
OM Oman
|
|
PA Panama | Panamá
|
|
PE Peru | Perú
|
|
PF French Polynesia
|
|
PG Papua New Guinea
|
|
PH Philippines
|
|
PK Pakistan
|
|
PL Poland | Polska
|
|
PM Saint Pierre and Miquelon
|
|
PN Pitcairn Islands
|
|
PR Puerto Rico
|
|
PS Palestine
|
|
PT Portugal
|
|
PW Palau
|
|
PY Paraguay
|
|
QA Qatar
|
|
RE Réunion
|
|
RO Romania | România
|
|
RS Serbia | Srbija | Србија
|
|
RU Russia | Russian Federation | Россия | Rossiya
|
|
RW Rwanda
|
|
SA Saudi Arabia | KSA | SAU
|
|
SB Solomon Islands
|
|
SC Seychelles
|
|
SD Sudan
|
|
SE Sweden | Sverige | Suède
|
|
SG Singapore
|
|
SH Saint Helena
|
|
SI Slovenia | Slovenija
|
|
SJ Svalbard and Jan Mayen | Svalbard
|
|
SK Slovakia | Slovensko
|
|
SL Sierra Leone
|
|
SM San Marino
|
|
SN Senegal
|
|
SO Somalia
|
|
SR Suriname
|
|
SS South Sudan
|
|
ST São Tomé and Príncipe
|
|
SV El Salvador
|
|
SX Sint Maarten
|
|
SY Syria
|
|
SZ Eswatini | Swaziland
|
|
TC Turks and Caicos Islands
|
|
TD Chad
|
|
TF French Southern Territories
|
|
TG Togo
|
|
TH Thailand | THA
|
|
TJ Tajikistan
|
|
TK Tokelau
|
|
TL Timor-Leste | East Timor
|
|
TM Turkmenistan
|
|
TN Tunisia
|
|
TO Tonga
|
|
TR Turkey | Türkiye
|
|
TT Trinidad and Tobago
|
|
TV Tuvalu
|
|
TW Taiwan
|
|
TZ Tanzania
|
|
UA Ukraine | Україна | Ukraina
|
|
UG Uganda
|
|
UM United States Minor Outlying Islands
|
|
US United States | United States of America | USA | U.S.A. | US | U.S. | Americas | AMER | North America
|
|
UY Uruguay
|
|
UZ Uzbekistan
|
|
VA Vatican City | Holy See
|
|
VC Saint Vincent and the Grenadines
|
|
VE Venezuela
|
|
VG British Virgin Islands
|
|
VI U.S. Virgin Islands | US Virgin Islands
|
|
VN Vietnam | Viet Nam
|
|
VU Vanuatu
|
|
WF Wallis and Futuna
|
|
WS Samoa
|
|
XK Kosovo | Kosova
|
|
YE Yemen
|
|
YT Mayotte
|
|
ZA South Africa
|
|
ZM Zambia
|
|
ZW Zimbabwe
|
|
"""
|
|
|
|
#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK.
|
|
_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line)
|
|
|
|
#: The countries `EU` stands for when a line names several: the UN
|
|
#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and
|
|
#: the Council of Europe's members in the Caucasus and Turkey, which
|
|
#: European employers list beside their other offices.
|
|
_EUROPE = frozenset((
|
|
"AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
|
|
"DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE",
|
|
"IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL",
|
|
"NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA",
|
|
"XK",
|
|
))
|
|
_EUROPE_OR_EU = _EUROPE | {"EU"}
|
|
|
|
#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them.
|
|
_REGIONS = ("europe", "emea", "eu", "european union", "eea")
|
|
|
|
#: The cities boards name most, and a few regions below a country. A US
|
|
#: state or a Canadian province in the same part of the line beats one of
|
|
#: these ("London, Ontario" is Canada); a country name does not lose to it.
|
|
_CITIES: dict[str, str] = {
|
|
**dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck", "villach",
|
|
"klagenfurt"), "AT"),
|
|
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege",
|
|
"luik"), "BE"),
|
|
**dict.fromkeys(("sofia",), "BG"),
|
|
**dict.fromkeys(("zagreb",), "HR"),
|
|
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
|
|
**dict.fromkeys(("prague", "praha", "brno"), "CZ"),
|
|
**dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg",
|
|
"region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby",
|
|
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", "brondby",
|
|
"glostrup", "soborg", "malov", "birkerod", "hornbaek"), "DK"),
|
|
**dict.fromkeys(("tallinn", "tartu"), "EE"),
|
|
**dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"),
|
|
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", "lille",
|
|
"rennes", "amiens", "rouen", "aix en provence", "grenoble", "annecy", "metz",
|
|
"strasbourg", "antibes", "avignon", "bourg en bresse", "brest", "caen",
|
|
"chambery", "colmar", "montpellier", "nancy", "nice", "nimes",
|
|
"saint etienne", "saint nazaire", "saint ouen l aumone", "valence", "angers",
|
|
"bayonne", "beauvais", "boulogne sur mer", "challans", "cholet",
|
|
"clermont ferrand", "dunkerque", "la roche sur yon", "la rochelle",
|
|
"lamballe", "lannion", "laval", "le havre", "le mans", "lens",
|
|
"les sables d olonne", "limoges", "lorient", "loudeac", "morlaix", "mulhouse",
|
|
"orleans", "orvault", "paimpol", "pornic", "quimper", "reims", "reze",
|
|
"saint brieuc", "saint herblain", "saint malo", "tours", "troyes", "vannes",
|
|
"vertou"), "FR"),
|
|
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
|
|
"stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg",
|
|
"nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund",
|
|
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
|
|
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
|
|
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", "oldenburg",
|
|
"emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck",
|
|
"bayreuth", "bad homburg", "homburg", "ettlingen", "bremerhaven",
|
|
"breidenbach", "bielefeld", "saarbrucken", "hildesheim", "moers", "kiel",
|
|
"ulm", "regensburg", "freiburg", "obersulm", "seevetal", "neukirchen vluyn",
|
|
"flensburg", "ingolstadt", "kempten", "konstanz", "magdeburg", "weimar",
|
|
"chemnitz", "erfurt", "freiberg", "gustrow", "kaiserslautern", "mohnesee",
|
|
"neubrandenburg", "neumarkt", "neumunster", "paderborn", "pforzheim",
|
|
"aachen", "amberg", "annaberg buchholz", "backnang", "bad kissingen",
|
|
"bad liebenzell", "baden baden", "bamberg", "besigheim",
|
|
"bietigheim bissingen", "bocholt", "brackenheim", "bretten", "bruchsal",
|
|
"cloppenburg", "cottbus", "crailsheim", "dachau", "darmstadt",
|
|
"donaueschingen", "donauworth", "ebersberg", "eppingen", "erding", "erlangen",
|
|
"ettenheim", "feuchtwangen", "filderstadt", "florsheim", "flohrsheim",
|
|
"freising", "friedrichshafen", "fulda", "furth", "geislingen", "gera",
|
|
"germering", "goppingen", "gundelsheim", "gunzburg", "hagenow", "halle saale",
|
|
"heidenheim", "heilbronn", "herrsching", "herzogenaurach", "holzkirchen",
|
|
"holzminden", "hoxter", "idar oberstein", "jena", "kaisheim", "kierspe",
|
|
"kirchheim", "kitzingen", "koblenz", "kolbermoor", "konigsbrunn", "kunzelsau",
|
|
"landau", "landsberg", "landshut", "lauf an der pegnitz", "lindern", "lingen",
|
|
"lorrach", "ludwigsburg", "ludwigshafen", "malchow", "marburg", "meissen",
|
|
"memmingen", "meppen", "metzingen", "monchengladbach", "moosburg", "mosbach",
|
|
"muhldorf", "mulheim", "murnau", "neuburg an der donau", "neustadt",
|
|
"nordlingen", "northeim", "oberhausen", "offenburg", "ohringen", "passau",
|
|
"peine", "petershausen", "pfaffenhofen", "pfullendorf", "plau am see",
|
|
"prien am chiemsee", "rastatt", "reutlingen", "riesa", "robel", "roth",
|
|
"rothenburg ob der tauber", "rottweil", "salzgitter", "sankt georgen",
|
|
"schorndorf", "schwabisch gmund", "schwabisch hall", "schweinfurt",
|
|
"starnberg", "stockach", "tegernsee", "templin", "trier", "tuttlingen",
|
|
"uberlingen", "uedem", "vaihingen", "villingen schwenningen",
|
|
"waren an der muritz", "wasserburg", "weilheim", "wernigerode", "wismar",
|
|
"wittenberg", "wittstock", "wolfenbuttel", "wolfratshausen", "wolfsburg",
|
|
"zwickau", "euskirchen", "tangstedt", "erkrath", "sigmaringen", "grimmen",
|
|
"ribnitz damgarten", "stralsund", "warstein", "ansbach", "balingen", "coburg",
|
|
"duisburg", "grossostheim", "merzig", "saarlouis", "schwerin", "singen",
|
|
"troisdorf", "bad bevensen", "dannenberg", "luchow", "nagold", "salzwedel",
|
|
"uelzen", "boblingen", "giessen", "hagen", "hechingen", "leverkusen",
|
|
"solingen", "wetzlar"), "DE"),
|
|
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
|
|
**dict.fromkeys(("budapest", "cegled"), "HU"),
|
|
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
|
|
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", "florence",
|
|
"verona", "cagliari", "ascoli piceno", "ancarano", "naples", "napoli",
|
|
"venice", "venezia", "genoa", "genova", "bolzano", "padua", "padova",
|
|
"brindisi", "matera", "pistoia", "taranto", "asti", "campobasso", "cassino",
|
|
"chieti", "civitavecchia", "cosenza", "crema", "fiumicino", "frosinone",
|
|
"grosseto", "guidonia montecelio", "l aquila", "ladispoli", "lamezia terme",
|
|
"latina", "livorno", "lucca", "massa carrara", "pescara", "pisa", "ravenna",
|
|
"rimini", "sanremo", "teramo", "terni", "trieste", "vibo valentia", "forli",
|
|
"cesena", "emilia romagna", "tuscany", "lazio"), "IT"),
|
|
**dict.fromkeys(("riga",), "LV"),
|
|
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
|
|
**dict.fromkeys(("luxembourg", "luxemburg"), "LU"),
|
|
**dict.fromkeys(("valletta",), "MT"),
|
|
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
|
"delft", "amersfoort", "deventer", "maastricht", "bleiswijk", "almere",
|
|
"enschede"), "NL"),
|
|
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz",
|
|
"biskupiec", "sochaczew", "teresin"), "PL"),
|
|
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
|
|
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara",
|
|
"brasov"), "RO"),
|
|
**dict.fromkeys(("belgrade", "beograd"), "RS"),
|
|
**dict.fromkeys(("bratislava",), "SK"),
|
|
**dict.fromkeys(("ljubljana",), "SI"),
|
|
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", "granada",
|
|
"santiago de compostela", "pontevedra", "tarragona"), "ES"),
|
|
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras",
|
|
"linkoping", "orebro", "norrkoping", "jonkoping", "almhult", "halmstad",
|
|
"solna"), "SE"),
|
|
**dict.fromkeys(("reykjavik",), "IS"),
|
|
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
|
|
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
|
|
"oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff",
|
|
"guildford", "truro", "swansea", "blackpool", "coventry", "brighton",
|
|
"weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley",
|
|
"slough", "southampton", "swindon", "liverpool", "redhill", "milton keynes",
|
|
"birmingham", "cambridge", "fermanagh", "tyrone"), "GB"),
|
|
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
|
|
"sankt gallen", "st gallen", "lucerne", "luzern", "landquart"), "CH"),
|
|
**dict.fromkeys(("kyiv", "kiev"), "UA"),
|
|
**dict.fromkeys(("istanbul", "ankara"), "TR"),
|
|
**dict.fromkeys(("tirana",), "AL"),
|
|
**dict.fromkeys(("yerevan",), "AM"),
|
|
**dict.fromkeys(("tbilisi",), "GE"),
|
|
**dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle",
|
|
"boston", "chicago", "austin", "los angeles", "palo alto", "mountain view",
|
|
"san mateo", "foster city", "sunnyvale", "san jose", "menlo park",
|
|
"redwood city", "costa mesa", "denver", "atlanta", "washington dc",
|
|
"washington d c", "miami", "dallas", "houston", "phoenix", "tempe",
|
|
"portland", "salt lake city", "pittsburgh", "philadelphia", "detroit",
|
|
"san diego", "raleigh", "nashville", "minneapolis", "cambridge ma",
|
|
"bay area", "arlington", "leominster", "andover", "el segundo", "columbus",
|
|
"tysons corner", "tysons", "lowell", "mclean", "morrisville", "southlake",
|
|
"st louis", "aventura", "brooklyn", "charlotte", "corte madera",
|
|
"morgan hill", "newport beach", "las vegas", "san antonio", "tampa",
|
|
"indianapolis", "honolulu", "memphis", "glenview", "shakopee", "lexington",
|
|
"wilmington", "murrieta"), "US"),
|
|
**dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo",
|
|
"edmonton", "winnipeg"), "CA"),
|
|
**dict.fromkeys(("buenos aires",), "AR"),
|
|
**dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos", "curitiba"), "BR"),
|
|
**dict.fromkeys(("santiago", "santiago de chile"), "CL"),
|
|
**dict.fromkeys(("bogota",), "CO"),
|
|
**dict.fromkeys(("lima", "miraflores"), "PE"),
|
|
**dict.fromkeys(("montevideo",), "UY"),
|
|
**dict.fromkeys(("managua",), "NI"),
|
|
**dict.fromkeys(("santo domingo",), "DO"),
|
|
**dict.fromkeys(("san salvador",), "SV"),
|
|
**dict.fromkeys(("panama city",), "PA"),
|
|
**dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara", "cdmx"), "MX"),
|
|
**dict.fromkeys(("dubai", "abu dhabi"), "AE"),
|
|
**dict.fromkeys(("riyadh",), "SA"),
|
|
**dict.fromkeys(("doha",), "QA"),
|
|
**dict.fromkeys(("manama",), "BH"),
|
|
**dict.fromkeys(("salmiya",), "KW"),
|
|
**dict.fromkeys(("muscat",), "OM"),
|
|
**dict.fromkeys(("cairo", "maadi"), "EG"),
|
|
**dict.fromkeys(("casablanca",), "MA"),
|
|
**dict.fromkeys(("dakar",), "SN"),
|
|
**dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa", "netanya"), "IL"),
|
|
**dict.fromkeys(("almaty",), "KZ"),
|
|
**dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"),
|
|
**dict.fromkeys(("dhaka",), "BD"),
|
|
**dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", "gurgaon",
|
|
"gurugram", "noida", "new delhi", "delhi", "ahmedabad"), "IN"),
|
|
**dict.fromkeys(("kuala lumpur", "bangsar south", "melaka", "malacca", "kulim", "penang",
|
|
"ipoh"), "MY"),
|
|
**dict.fromkeys(("singapore",), "SG"),
|
|
**dict.fromkeys(("manila", "taguig", "cavite", "general trias", "muntinlupa"), "PH"),
|
|
**dict.fromkeys(("bangkok", "samut prakan", "samut prakarn"), "TH"),
|
|
**dict.fromkeys(("phnom penh",), "KH"),
|
|
**dict.fromkeys(("taipei", "hsinchu", "kaohsiung", "tainan"), "TW"),
|
|
**dict.fromkeys(("tokyo", "osaka", "nagoya"), "JP"),
|
|
**dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou", "wuxi", "xi an",
|
|
"qingdao", "chongqing", "nanjing", "tianjin", "wuhan", "xiamen"), "CN"),
|
|
**dict.fromkeys(("seoul", "seongnam", "busan", "daegu", "daejeon", "gwangju"), "KR"),
|
|
**dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales", "queensland",
|
|
"canberra"), "AU"),
|
|
**dict.fromkeys(("jakarta", "batam"), "ID"),
|
|
**dict.fromkeys(("hanoi", "ho chi minh"), "VN"),
|
|
**dict.fromkeys(("tunis",), "TN"),
|
|
**dict.fromkeys(("ebene",), "MU"),
|
|
**dict.fromkeys(("cape town", "johannesburg"), "ZA"),
|
|
**dict.fromkeys(("auckland",), "NZ"),
|
|
}
|
|
|
|
#: US states and Canadian provinces, by name and by the code boards write
|
|
#: after a comma ("Boston, MA", "Toronto, ON").
|
|
_SUBDIVISIONS = """
|
|
US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO
|
|
US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA
|
|
US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY
|
|
US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN
|
|
US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH
|
|
US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND
|
|
US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI
|
|
US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT
|
|
US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY
|
|
CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB
|
|
CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL
|
|
CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT
|
|
CA Nunavut NU
|
|
"""
|
|
|
|
|
|
def _subdivisions() -> tuple[dict[str, str], dict[str, str]]:
|
|
names: dict[str, str] = {}
|
|
codes: dict[str, str] = {}
|
|
for line in _SUBDIVISIONS.split("\n"):
|
|
if line:
|
|
country, rest = line.split(" ", 1)
|
|
for entry in rest.split("|"):
|
|
name, code = entry.strip().rsplit(" ", 1)
|
|
names[_place_fold(name)] = country
|
|
codes[code] = country
|
|
return names, codes
|
|
|
|
|
|
_STATE_NAMES, _STATE_CODES = _subdivisions()
|
|
|
|
|
|
def _names() -> dict[str, tuple[str, str]]:
|
|
"""Every folded name to `(code, kind)`; kind is `country`, `city` or
|
|
`state`. A name that is a city and a state ("New York") is the city, so
|
|
it does not beat another city in its line. A name that is a country and
|
|
a state ("Georgia") is the country here and read as the state only when
|
|
the line names a place in the US too."""
|
|
names: dict[str, tuple[str, str]] = {}
|
|
for name, code in _STATE_NAMES.items():
|
|
names[name] = (code, "state")
|
|
for name, code in _CITIES.items():
|
|
names[_place_fold(name)] = (code, "city")
|
|
for line in _COUNTRY_NAMES.split("\n"):
|
|
if line:
|
|
code, rest = line.split(" ", 1)
|
|
for name in rest.split("|"):
|
|
names[_place_fold(name)] = (code, "country")
|
|
for name in _REGIONS:
|
|
names[name] = ("EU", "country")
|
|
return names
|
|
|
|
|
|
_NAMES = _names()
|
|
|
|
#: A state name that is also a country's, read as the state only when the
|
|
#: line names a place in that state's country too.
|
|
_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES
|
|
if _NAMES[name][1] == "country"}
|
|
|
|
#: The longest name in words, so a lookup tries "new york city" before "new
|
|
#: york" and "northern ireland" before "ireland".
|
|
_LONGEST = max(len(name.split()) for name in _NAMES)
|
|
|
|
#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin".
|
|
_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b")
|
|
|
|
#: A country code in parentheses, "Leicester (GB)", read only when no name matched.
|
|
_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)")
|
|
|
|
#: Where one location in a line ends and the next begins. A state beats a
|
|
#: city only within one: "London; Ontario" is two places.
|
|
_SEGMENT = re.compile(r"[;|/\n]")
|
|
|
|
|
|
def _hits(folded: str) -> list[tuple[str, str, str]]:
|
|
"""The names in a folded line, longest first at each word, as
|
|
`(name, code, kind)`."""
|
|
words = folded.split()
|
|
hits = []
|
|
i = 0
|
|
while i < len(words):
|
|
for n in range(min(_LONGEST, len(words) - i), 0, -1):
|
|
name = " ".join(words[i:i + n])
|
|
if name in _NAMES:
|
|
hits.append((name, *_NAMES[name]))
|
|
i += n
|
|
break
|
|
else:
|
|
i += 1
|
|
return hits
|
|
|
|
|
|
def country_of(place: str) -> str:
|
|
"""The one country a board's location line names, as an ISO code; `EU`
|
|
when it names several and all of them are in Europe; '' when it names
|
|
none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston,
|
|
MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or
|
|
New York" and "Remote, EU or US" are ''.
|
|
|
|
Within one location (the line split at `;`, `|` and `/`) the widest unit
|
|
named wins: a country beats a state or a province, which beats a city.
|
|
So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A
|
|
country also beats the region it is in: "London, EMEA" is GB.
|
|
|
|
A state code that is also a country's ("DE", "CO", "MA"), and "Georgia",
|
|
count as the state only when the line names a place in the US too:
|
|
"Berlin, DE" is DE, "Bogota, CO" is CO, "Ellwangen, DE" is '' rather
|
|
than Delaware. A country code in parentheses, "Leicester (GB)", is read
|
|
only when no name matched."""
|
|
segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")]
|
|
named = {code for _, hits in segments for _, code, _ in hits}
|
|
found: set[str] = set()
|
|
for text, hits in segments:
|
|
shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named}
|
|
countries = {code for name, code, kind in hits
|
|
if kind == "country" and code != "EU" and name not in shared}
|
|
states = {code for _, code, kind in hits if kind == "state"}
|
|
states |= {_SHARED_NAMES[name] for name in shared}
|
|
states |= {country for code in _STATE_CODE.findall(text)
|
|
if (country := _STATE_CODES.get(code))
|
|
and (code not in _ISO or country in named)}
|
|
found |= (countries or states or {code for _, code, _ in hits if code != "EU"})
|
|
found |= {code for _, code, _ in hits if code == "EU"}
|
|
if not found:
|
|
found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO}
|
|
if len(found) == 1:
|
|
return found.pop()
|
|
if found and found <= _EUROPE_OR_EU:
|
|
countries = found - {"EU"}
|
|
return countries.pop() if "EU" in found and len(countries) == 1 else "EU"
|
|
return ""
|
|
|
|
|
|
# --------------------------------------------------------------- vocabulary
|
|
|
|
Remote = Literal["", "remote", "hybrid", "onsite"]
|
|
REMOTE: tuple[str, ...] = get_args(Remote)
|
|
|
|
#: Words in a location line that say how the role is worked, for a board that
|
|
#: gives no remote flag of its own: Greenhouse writes "Remote", "Hybrid" or
|
|
#: "Distributed" as the location and nothing else.
|
|
_REMOTE_WORDS: tuple[tuple[str, str], ...] = (
|
|
("hybrid", "hybrid"),
|
|
("remote", "remote"), ("distributed", "remote"), ("home based", "remote"),
|
|
("home-based", "remote"), ("work from home", "remote"), ("anywhere", "remote"),
|
|
("worldwide", "remote"), ("telecommute", "remote"),
|
|
("on-site", "onsite"), ("onsite", "onsite"), ("on site", "onsite"),
|
|
("in-office", "onsite"), ("in office", "onsite"),
|
|
)
|
|
|
|
|
|
def remote_of(place: str) -> str:
|
|
"""How a board's location line says the role is worked, as one of
|
|
`REMOTE`, or '' when it does not say. "Hybrid" wins over "remote" in the
|
|
same line ("Remote or hybrid, Berlin" is hybrid: some days in an office),
|
|
and a line that names only a place is ''."""
|
|
folded = _fold(place or "")
|
|
for word, mode in _REMOTE_WORDS:
|
|
if word in folded:
|
|
return mode
|
|
return ""
|
|
|
|
#: A role's level, '' when nothing says. From `staff` up a level sits on one
|
|
#: of two tracks, individual contributor or people lead, and each rung pairs
|
|
#: the two as big tech firms do: Staff with Manager, Principal with Senior
|
|
#: Manager, Distinguished with Director. `lead` is a team lead (Team Lead,
|
|
#: Supervisor, Teamleiter); a "Tech Lead" or "Marketing Lead" is `staff`.
|
|
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
|
|
"principal", "senior_manager", "distinguished", "director", "exec"]
|
|
SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
|
#: The levels a seniority filter may name: `SENIORITIES` without the unknown.
|
|
LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level)
|
|
|
|
#: The rungs of the ladder from the bottom, each named after its lowest
|
|
#: individual level; a range filter names two of them.
|
|
RUNGS: tuple[str, ...] = ("intern", "junior", "mid", "senior", "staff", "principal",
|
|
"distinguished", "exec")
|
|
#: The tracks a filter may keep: `ic` (individual contributor) or `people`.
|
|
TRACKS: tuple[str, ...] = ("ic", "people")
|
|
#: Each level's rung (an index into `RUNGS`) and track; the rungs below
|
|
#: `staff` have no track and pass either.
|
|
LADDER: dict[str, tuple[int, str]] = {
|
|
"intern": (0, ""), "junior": (1, ""), "mid": (2, ""), "senior": (3, ""),
|
|
"staff": (4, "ic"), "lead": (4, "people"), "manager": (4, "people"),
|
|
"principal": (5, "ic"), "senior_manager": (5, "people"),
|
|
"distinguished": (6, "ic"), "director": (6, "people"),
|
|
"exec": (7, "people"),
|
|
}
|
|
|
|
|
|
def levels_between(low: str = "", high: str = "", track: str = "") -> tuple[str, ...]:
|
|
"""The levels from rung `low` to rung `high` (names in `RUNGS`, '' for
|
|
the open end) on `track` (`TRACKS`, '' for both), in `LEVELS` order. A
|
|
level with no track passes any track. An unknown name is a ValueError."""
|
|
for name, value, allowed in (("low", low, RUNGS), ("high", high, RUNGS),
|
|
("track", track, TRACKS)):
|
|
if value and value not in allowed:
|
|
raise ValueError(f"{name} is one of {', '.join(allowed)}, not {value!r}")
|
|
bottom = RUNGS.index(low) if low else 0
|
|
top = RUNGS.index(high) if high else len(RUNGS) - 1
|
|
return tuple(level for level in LEVELS
|
|
if bottom <= LADDER[level][0] <= top
|
|
and (not track or LADDER[level][1] in ("", track)))
|
|
|
|
#: The kind of contract a posting states, as `app.prep` maps a host's word.
|
|
#: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the
|
|
#: readers' own tests, since a plugin imports nothing from `app`.
|
|
Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship",
|
|
"freelance"]
|
|
EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e)
|
|
|
|
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
|
Permit = Literal["", "sponsors", "no_sponsorship"]
|
|
PERMITS: tuple[str, ...] = get_args(Permit)
|
|
|
|
Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer",
|
|
"rejected", "withdrawn"]
|
|
STATUSES: tuple[str, ...] = get_args(Status)
|
|
|
|
#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips the `any` gates).
|
|
AFFINITY = range(-1, 3)
|
|
|
|
#: Why a company is on the list checked by hand. `unread` is a host the app
|
|
#: could read and has no parser for yet, `robots` a site whose robots.txt
|
|
#: disallows it, `blocked` one that answers a program with a challenge, `key`
|
|
#: one whose API answers only with a key lifted from its own script, `group` a
|
|
#: company that sits inside its owner's board, `gone` one whose site is dead,
|
|
#: and `added` one put there by hand or by an import and not looked at. ''
|
|
#: is none of these: the company is read, or is not a matter for the list.
|
|
HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"]
|
|
HAND_REASONS: tuple[str, ...] = get_args(HandReason)
|
|
|
|
#: How well somebody speaks a language, lowest first: what an agent's CV
|
|
#: states and what a posting asks for. The order is the comparison.
|
|
LanguageLevel = Literal["basic", "conversational", "professional", "fluent", "native"]
|
|
LANGUAGE_LEVELS: tuple[str, ...] = get_args(LanguageLevel)
|
|
|
|
#: Whether a posting requires a language or calls it a plus.
|
|
LanguageKind = Literal["required", "plus"]
|
|
|
|
#: English names of the languages a posting in reach names, by ISO 639-1
|
|
#: code, for `language_flag`'s note. A code not here is shown as it is.
|
|
LANGUAGE_NAMES: dict[str, str] = {
|
|
"ar": "Arabic", "bg": "Bulgarian", "cs": "Czech", "da": "Danish", "de": "German",
|
|
"el": "Greek", "en": "English", "es": "Spanish", "et": "Estonian", "fi": "Finnish",
|
|
"fo": "Faroese", "fr": "French", "hi": "Hindi", "hr": "Croatian", "hu": "Hungarian",
|
|
"is": "Icelandic", "it": "Italian", "ja": "Japanese", "kl": "Greenlandic",
|
|
"ko": "Korean", "lt": "Lithuanian", "lv": "Latvian", "nb": "Norwegian",
|
|
"nl": "Dutch", "nn": "Norwegian", "no": "Norwegian", "pl": "Polish",
|
|
"pt": "Portuguese", "ro": "Romanian", "ru": "Russian", "sk": "Slovak",
|
|
"sl": "Slovenian", "sr": "Serbian", "sv": "Swedish", "tr": "Turkish",
|
|
"uk": "Ukrainian", "zh": "Chinese",
|
|
}
|
|
|
|
_LANGUAGE_CODE = re.compile(r"^[a-z]{2}$")
|
|
|
|
|
|
def language_code(value: Any) -> str:
|
|
"""An ISO 639-1 code, lowercased, or '' when `value` is not one."""
|
|
code = str(value or "").strip().lower()
|
|
return code if _LANGUAGE_CODE.match(code) else ""
|
|
|
|
|
|
# -------------------------------------------------------------- posting dates
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _declared_posted() -> dict[str, str]:
|
|
"""`fields.posted` of every reader that declares `READS`, by name. Read
|
|
once: the readers are loaded when `plugins.readers` is imported. `plugins`
|
|
is a top-level folder beside `app`, reached as `app/jobs/extract.py`
|
|
reaches it."""
|
|
root = str(Path(__file__).resolve().parents[1])
|
|
if root not in sys.path:
|
|
sys.path.insert(0, root)
|
|
from plugins.readers import READERS
|
|
|
|
return {name: module.READS.fields.posted for name, module in READERS.items()
|
|
if getattr(module, "READS", None) is not None}
|
|
|
|
|
|
def trusted_posted(host: str) -> bool:
|
|
"""Whether `host` states a date that is safe to read as when a role was
|
|
posted: its reader declares `fields.posted="trusted"`. Which field each
|
|
host reads, and when it was checked, is in the reader module. A role
|
|
without a trusted date keeps `first_seen` as its posted date."""
|
|
return _declared_posted().get(host) == "trusted"
|
|
|
|
|
|
#: A `published_at` before this is a parsing slip or a placeholder, not a
|
|
#: real posting date: no board here existed before 2015.
|
|
_POSTED_FLOOR = "2015-01-01T00:00:00+00:00"
|
|
|
|
|
|
def host_word(value: Any) -> str:
|
|
"""A host's word as `roles.*_host` stores it: stripped text, a bool as
|
|
'true' or 'false', '' for nothing. What `app.prep` maps."""
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, bool):
|
|
return "true" if value else "false"
|
|
return str(value).strip()
|
|
|
|
|
|
def posted_host(raw: dict[str, Any]) -> str:
|
|
"""The host a role's dates came from, read the way `ats` and `search`
|
|
both put it in `Item.raw`: `board` for an ATS board (`greenhouse`,
|
|
`lever`, ...), `provider` for a cross-company search (`jobindex`,
|
|
`adzuna`). '' when neither plugin wrote one."""
|
|
return str(raw.get("board") or raw.get("provider") or "")
|
|
|
|
|
|
def posted_at(published_at: str, first_seen: str, host: str) -> str:
|
|
"""The date a role counts as posted: the earlier of `first_seen` (when
|
|
the app itself first stored it) and `published_at` (what the board
|
|
claims), but only when `trusted_posted(host)` and
|
|
`published_at` is neither absurdly old nor later than `first_seen` — a
|
|
role cannot be posted after the app already saw it, trusted host or not,
|
|
which also catches a `published_at` the board reports in the future
|
|
(`first_seen` is never later than the moment this runs). Both dates are
|
|
`Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings
|
|
is exact and needs no parsing. A role with no trusted date, or with one
|
|
that fails the check, keeps `first_seen`."""
|
|
if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen:
|
|
return published_at
|
|
return first_seen
|
|
|
|
|
|
def iso_day(value: Any) -> str:
|
|
"""A date as `YYYY-MM-DD` from an ISO date or timestamp ("2026-10-18",
|
|
"2026-10-18T00:00:00"), or '' when `value` holds no real date."""
|
|
text = str(value or "").strip()[:10]
|
|
try:
|
|
return date.fromisoformat(text).isoformat() if len(text) == 10 else ""
|
|
except ValueError:
|
|
return ""
|
|
|
|
|
|
def _json(value: Any, empty: str) -> Any:
|
|
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
|
|
value that is already parsed passes, and one that does not parse is the
|
|
empty value rather than a 500."""
|
|
if not isinstance(value, str):
|
|
return value
|
|
try:
|
|
return json.loads(value or empty)
|
|
except ValueError:
|
|
return json.loads(empty)
|
|
|
|
|
|
#: A column holding a JSON array of strings, such as `roles.languages`.
|
|
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
|
|
|
|
|
|
def _json_list(value: Any) -> list[Any]:
|
|
"""A nullable JSON array column as a list: NULL, a parse failure and
|
|
anything but a list are []."""
|
|
found = _json(value, "[]") if value is not None else []
|
|
return found if isinstance(found, list) else []
|
|
|
|
|
|
class CvLanguage(Model):
|
|
"""A language an agent's CV states, with how well: `language` an ISO
|
|
639-1 code, `level` one of `LANGUAGE_LEVELS`."""
|
|
|
|
language: str
|
|
level: LanguageLevel
|
|
|
|
@field_validator("language", mode="before")
|
|
@classmethod
|
|
def _code(cls, value: Any) -> str:
|
|
code = language_code(value)
|
|
if not code:
|
|
raise ValueError(f"a language is an ISO 639-1 code such as 'en', not {value!r}")
|
|
return code
|
|
|
|
|
|
def cv_languages(value: Any) -> list[CvLanguage]:
|
|
"""An agent's languages as stored and sent: one per code, the first
|
|
kept, in the order given."""
|
|
seen: set[str] = set()
|
|
out: list[CvLanguage] = []
|
|
for item in _json_list(value):
|
|
found = item if isinstance(item, CvLanguage) else CvLanguage.model_validate(item)
|
|
if found.language not in seen:
|
|
seen.add(found.language)
|
|
out.append(found)
|
|
return out
|
|
|
|
|
|
#: An agent's languages, normalised by `cv_languages`.
|
|
CvLanguages = Annotated[list[CvLanguage], BeforeValidator(cv_languages)]
|
|
|
|
|
|
class RoleLanguage(Model):
|
|
"""A language a posting names (`roles.languages_required`, read by
|
|
`extract`): its ISO 639-1 code, the level asked for ('' when the posting
|
|
does not say), and whether it is required or a plus."""
|
|
|
|
language: str
|
|
level: Literal["", "basic", "conversational", "professional", "fluent", "native"] = ""
|
|
kind: LanguageKind = "required"
|
|
|
|
|
|
def role_languages(value: Any) -> list[RoleLanguage]:
|
|
out = []
|
|
for item in _json_list(value):
|
|
try:
|
|
out.append(item if isinstance(item, RoleLanguage)
|
|
else RoleLanguage.model_validate(item))
|
|
except ValueError:
|
|
continue
|
|
return out
|
|
|
|
|
|
#: `roles.languages_required`: NULL (never read) and [] are both no language.
|
|
RoleLanguages = Annotated[list[RoleLanguage], BeforeValidator(role_languages)]
|
|
|
|
LangFlag = Literal["missing", "below", "plus", ""]
|
|
|
|
|
|
def _language_name(code: str) -> str:
|
|
return LANGUAGE_NAMES.get(code, code)
|
|
|
|
|
|
def _asked(language: RoleLanguage) -> str:
|
|
name = _language_name(language.language)
|
|
return f"{name} ({language.level})" if language.level else name
|
|
|
|
|
|
def language_flag(required: list[RoleLanguage],
|
|
mine: list[CvLanguage]) -> tuple[LangFlag, str]:
|
|
"""`(flag, note)` for a posting's languages against an agent's.
|
|
`missing` when a required language is not one of the agent's, `below`
|
|
when the level it states is above the agent's, else `plus` when a plus
|
|
language is one of the agent's, else ''. The note says why, as
|
|
"requires Danish (fluent); yours: none". No flag when the agent states
|
|
no languages: there is nothing to compare with."""
|
|
if not mine:
|
|
return "", ""
|
|
levels = {m.language: m.level for m in mine}
|
|
rank = LANGUAGE_LEVELS.index
|
|
needed = [r for r in required if r.kind == "required"]
|
|
missing = [r for r in needed if r.language not in levels]
|
|
if missing:
|
|
return "missing", "; ".join(f"requires {_asked(r)}; yours: none" for r in missing)
|
|
below = [r for r in needed if r.level and rank(r.level) > rank(levels[r.language])]
|
|
if below:
|
|
return "below", "; ".join(f"requires {_asked(r)}; yours: {levels[r.language]}"
|
|
for r in below)
|
|
plus = [r for r in required if r.kind == "plus" and r.language in levels]
|
|
if plus:
|
|
return "plus", "; ".join(f"{_asked(r)} is a plus; yours: {levels[r.language]}"
|
|
for r in plus)
|
|
return "", ""
|
|
|
|
|
|
#: What career counts as at most when a posting breaks one of the agent's
|
|
#: must-haves: the top of the rubric's 0-39 band for a dead end. The model
|
|
#: reports whether each must-have is met; this cap is applied in code, when a
|
|
#: role is served (`cv_score`, `Store.judgement`, `store._cv_sql`).
|
|
BROKEN_CAP = 39
|
|
|
|
#: A must-have's verdict, as the judge answers it.
|
|
MUST_VERDICTS = ("met", "broken", "unclear")
|
|
|
|
_BULLET = re.compile(r"^\s*(?:[-*\u2022]+|\(?\d+[.)])\s+")
|
|
|
|
|
|
def must_lines(text: str) -> list[str]:
|
|
"""The agent's must-haves text read as one must-have per line: a leading
|
|
bullet (`-`, `*`, `•`) or number (`1.`, `2)`) is dropped, so are empty
|
|
lines."""
|
|
found = []
|
|
for line in text.splitlines():
|
|
line = " ".join(_BULLET.sub("", line, count=1).split())
|
|
if line:
|
|
found.append(line)
|
|
return found
|
|
|
|
|
|
def capped_career(career: int, broken: int = 0) -> int:
|
|
"""`career` as it counts: at most `BROKEN_CAP` when any must-have is
|
|
broken."""
|
|
return min(career, BROKEN_CAP) if broken else career
|
|
|
|
|
|
def cv_score(skills: int, experience: int, career: int,
|
|
weights: "Weights | None" = None, broken: int = 0) -> float | None:
|
|
"""The one number of a CV match, 0 to 100: the three numbers of the
|
|
judge averaged by the agent's `cv_skills`, `cv_experience` and
|
|
`cv_career` weights (30, 25 and 30 by default), career as
|
|
`capped_career` has it. None when all three weights are 0."""
|
|
w = weights or Weights()
|
|
total = w.cv_skills + w.cv_experience + w.cv_career
|
|
if not total:
|
|
return None
|
|
career = capped_career(career, broken)
|
|
return round((w.cv_skills * skills + w.cv_experience * experience
|
|
+ w.cv_career * career) / total, 1)
|
|
|
|
|
|
class Must(Model):
|
|
"""One must-have of an agent as the judge read it against a role: its
|
|
`id` (`M1`, by line), the line's `text` and the `verdict`."""
|
|
|
|
id: str
|
|
text: str = ""
|
|
verdict: Literal["met", "broken", "unclear"] = "unclear"
|
|
|
|
|
|
class Judgement(Model):
|
|
"""What `judge` read of one role for one agent (`role_judgements`): the
|
|
three dimensions 0 to 100 as the model gave them, `cv_score`
|
|
(`cv_score`) and 1 to 3 strengths and gaps grounded in the posting.
|
|
`musts` is the verdict on each must-have and `broken` how many are
|
|
broken; any makes `career_capped` the lower of `career` and
|
|
`BROKEN_CAP`. `cv_score` and `career_capped` are not read from the
|
|
table: `Store.judgement` sets them from the agent's weights; `cv_score`
|
|
is None when all three are 0."""
|
|
|
|
skills: int
|
|
experience: int
|
|
career: int
|
|
career_capped: int | None = None
|
|
broken: int = 0
|
|
musts: Annotated[list[Must], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
|
cv_score: float | None = None
|
|
strengths: JsonStrings = []
|
|
gaps: JsonStrings = []
|
|
judged_at: str
|
|
|
|
|
|
# ------------------------------------------------------------------ models
|
|
|
|
class RoleCopy(Model):
|
|
"""One open copy of a kept role, as the reader lists it under "Also
|
|
posted": where it is listed (`host`, the search provider or the board's
|
|
ATS) and the link that opens it there."""
|
|
|
|
uid: str
|
|
host: str = ""
|
|
place: str = ""
|
|
original_url: str = ""
|
|
posting_url: str = ""
|
|
posted_at: str = ""
|
|
|
|
|
|
class RolePlace(Model):
|
|
"""One place of a role (`role_places`): the host's `text`, and the place
|
|
of the gazetteer it resolves to, `place_id` None (and `name` and `kind`
|
|
empty) while it resolves to none. `url` is the link of a copy folded into
|
|
the role, '' for the role's own places."""
|
|
|
|
position: int = 0
|
|
text: str = ""
|
|
place_id: int | None = None
|
|
name: str = ""
|
|
kind: str = ""
|
|
url: str = ""
|
|
|
|
|
|
class Role(Model):
|
|
"""A row of `roles`: one posting on one board."""
|
|
|
|
uid: str
|
|
source: str
|
|
source_id: str
|
|
subscription_id: int | None = None
|
|
company_key: str = ""
|
|
company: str = ""
|
|
title: str = ""
|
|
title_key: str = ""
|
|
place: str = ""
|
|
place_key: str = ""
|
|
country: str = ""
|
|
remote: Remote = ""
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
currency: str = ""
|
|
#: The pay a board stated, as one human-readable line, whether or not it
|
|
#: could be normalised into `salary_min`/`salary_max`/`currency`
|
|
#: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md).
|
|
salary_text: str = ""
|
|
#: The level shown and filtered on: the first filled of `seniority_model`,
|
|
#: `seniority_stated` and `seniority_title`.
|
|
seniority: str = ""
|
|
seniority_model: str = ""
|
|
seniority_stated: str = ""
|
|
seniority_title: str = ""
|
|
#: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`.
|
|
employment: str = ""
|
|
function: str = ""
|
|
permit: Permit = ""
|
|
#: ISO 639-1 codes the posting says are required, lowercase: what the
|
|
#: `languages` gate reads.
|
|
languages: JsonStrings = []
|
|
#: Every language the posting names, with the level it asks for and
|
|
#: whether it is required or a plus (`extract`); [] until read.
|
|
languages_required: RoleLanguages = []
|
|
#: The last day to apply, `YYYY-MM-DD`: the host's own date, else the
|
|
#: one `extract` read; None when neither says (`app.prep`).
|
|
deadline: str | None = None
|
|
description: str = ""
|
|
feed_text: str = ""
|
|
#: The short text the host's list carries, until `description` arrives.
|
|
teaser: str = ""
|
|
original_url: str = ""
|
|
#: Where a Jobindex ad's own page leads, found when the text is fetched
|
|
#: (`app.jobs.posting`); '' until then and for every other host.
|
|
posting_url: str = ""
|
|
published_at: str
|
|
#: The date this role counts as posted: see `posted_at`. Used for
|
|
#: `recency` and the feed's newest order; `first_seen` stays what it
|
|
#: always was, when the app itself first stored the role.
|
|
posted_at: str
|
|
first_seen: str
|
|
last_seen: str
|
|
closed_at: str | None = None
|
|
raw: JsonDict = {}
|
|
extracted_at: str | None = None
|
|
created_at: str
|
|
#: The vote of the agent the role was read for (see `role_votes`): None
|
|
#: unjudged, else -1 pass, 0 neutral, 1 like.
|
|
vote: Literal[-1, 0, 1] | None = None
|
|
voted_at: str | None = None
|
|
#: NULL unread, else when the feed opened it or judged it. A vote stamps
|
|
#: this too, see `Store.set_vote`: voting is reading.
|
|
read_at: str | None = None
|
|
summary: str = ""
|
|
summarised_at: str | None = None
|
|
#: The posting page as plain text, for a role whose description is a teaser.
|
|
article_text: str = ""
|
|
article_at: str | None = None
|
|
#: The open vocabulary `summarise` read off the posting, sorted. Not a
|
|
#: column: loaded from `role_tags` alongside the row, see `Store.role`.
|
|
tags: list[str] = []
|
|
#: The uid of the kept role when this one is a copy of the same posting
|
|
#: (`app.dupes`, `Store.rebuild_dupes`), else None. A copy is left out of
|
|
#: the feed and the counts but still opens by uid.
|
|
dup_of: str | None = None
|
|
#: The open copies of this role, oldest first, when it is the kept one;
|
|
#: empty otherwise. Not a column: filled by `Store.role`.
|
|
copies: list[RoleCopy] = []
|
|
#: Every place of the role, in the host's order. Not a column: filled by
|
|
#: `Store.role` from `role_places`.
|
|
places: list[RolePlace] = []
|
|
#: Where a remote hire may live, as a label: "anywhere", "Europe",
|
|
#: "United States, Canada". '' when the role is not remote or its area is
|
|
#: not stated. Not a column: filled by `Store` from `role_remote`.
|
|
remote_where: str = ""
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def salary_state(self) -> Literal["normalised", "stated", ""]:
|
|
"""Not a column: 'normalised' when `salary_min` or `salary_max` is
|
|
set, 'stated' when only `salary_text` is, '' otherwise. Gates and
|
|
scoring read `salary_min` alone and never this (docs/decisions.md);
|
|
it is what the feed's Pay filter and badge read instead
|
|
(`Store.PAY`, `static/roles.js`)."""
|
|
if self.salary_min is not None or self.salary_max is not None:
|
|
return "normalised"
|
|
if self.salary_text:
|
|
return "stated"
|
|
return ""
|
|
|
|
|
|
class ExtractResult(ModelTaskResult):
|
|
"""What one run of `extract` did. See app/jobs/extract.py."""
|
|
|
|
#: Roles the model answered for, whether or not it found anything to fill.
|
|
extracted: int = 0
|
|
#: Columns written, across those roles. A role answered with nothing the
|
|
#: posting states adds to `extracted` and not to this.
|
|
filled: int = 0
|
|
|
|
@property
|
|
def done(self) -> int:
|
|
return self.extracted
|
|
|
|
|
|
class ScoredRole(Role):
|
|
"""A role as `/api/roles/{uid}` serves it: with one agent's score and
|
|
gate reason when an agent was asked for, and without otherwise."""
|
|
|
|
score: float | None = None
|
|
gated_reason: str = ""
|
|
#: The agent's prompt that gave the role its fit; "" when fit did not
|
|
#: count (gated, no vector, no prompt with text).
|
|
prompt: str = ""
|
|
#: The agent's `judge` reading of the role against its CV, None until
|
|
#: judged; `cv_score` is its total.
|
|
judgement: Judgement | None = None
|
|
cv_score: float | None = None
|
|
#: How close the role is to the agent's CV, 0..1 (`scores.cv_fit`), None
|
|
#: without a CV, a vector or a score, and for a gated role.
|
|
cv_fit: float | None = None
|
|
#: The posting's languages against the agent's (`language_flag`).
|
|
lang_flag: LangFlag = ""
|
|
lang_note: str = ""
|
|
|
|
|
|
#: How many characters `RoleRow.excerpt` keeps at most: the feed row's own
|
|
#: cap, built once here rather than by the client, so a list response never
|
|
#: carries more of a role's text than a row draws.
|
|
EXCERPT_LIMIT = 200
|
|
|
|
#: The marks `html_text` starts a heading and a list item with.
|
|
_LINE_MARK = re.compile("^(## |\u2022 )", re.M)
|
|
|
|
|
|
def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str:
|
|
"""`text` (a role's summary, or the opening of its description) with its
|
|
whitespace collapsed, cut to `limit` characters at a word boundary with
|
|
an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a
|
|
`SUBSTR` of the source column rather than the whole of it, so the feed's
|
|
list never carries what only the reader needs. A description's heading
|
|
and bullet marks (`plugins.readers._shared.html_text`) are left out."""
|
|
collapsed = " ".join(_LINE_MARK.sub("", text or "").split())
|
|
if len(collapsed) <= limit:
|
|
return collapsed
|
|
cut = collapsed[:limit]
|
|
space = cut.rfind(" ")
|
|
trimmed = cut[:space] if space > limit * 0.6 else cut
|
|
return trimmed.rstrip(" ,.;:" + chr(0x2013) + "-") + "…"
|
|
|
|
|
|
class RoleRow(Model):
|
|
"""One row of the feed: what `GET /api/roles` serves per role, the
|
|
fields a row draws and nothing else — never `description`, `feed_text`,
|
|
`raw` or `article_text`, the four columns docs/decisions.md measures at
|
|
about 5.5 KB a role between them. The reader fetches the rest by uid when
|
|
it opens a role (`GET /api/roles/{uid}`, the full `Role`).
|
|
|
|
`excerpt` is built on the server (see `excerpt`, above): the summary when
|
|
there is one, else the description's opening, cut to `EXCERPT_LIMIT`
|
|
characters. The pay fields are what the row's badge reads: the three
|
|
normalised columns, the line the board stated, and `salary_state`,
|
|
derived as on `Role`. `original_url` and `posting_url` are short strings,
|
|
kept so the `o` key opens a selected row's posting without opening the
|
|
reader.
|
|
`seniority` is the title's level until extract reads the posting, ''
|
|
when neither says. `copies` counts the open copies the feed hides behind
|
|
this row."""
|
|
|
|
uid: str
|
|
company_key: str = ""
|
|
company: str = ""
|
|
title: str = ""
|
|
place: str = ""
|
|
country: str = ""
|
|
remote: Remote = ""
|
|
function: str = ""
|
|
seniority: str = ""
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
currency: str = ""
|
|
salary_text: str = ""
|
|
original_url: str = ""
|
|
posting_url: str = ""
|
|
posted_at: str = ""
|
|
score: float | None = None
|
|
gated_reason: str = ""
|
|
prompt: str = ""
|
|
vote: Literal[-1, 0, 1] | None = None
|
|
voted_at: str | None = None
|
|
read_at: str | None = None
|
|
tags: list[str] = []
|
|
excerpt: str = ""
|
|
#: How many open roles are copies of this one (`Role.copies`), 0 when none.
|
|
copies: int = 0
|
|
#: As `Role.remote_where`: the area a remote hire may live in, '' when
|
|
#: the role is not remote or does not say.
|
|
remote_where: str = ""
|
|
#: As `Role.deadline`: the last day to apply, None when nothing says.
|
|
deadline: str | None = None
|
|
#: As `ScoredRole`: the posting's languages against the agent's, and
|
|
#: the total of the agent's `judge` reading, None until judged.
|
|
lang_flag: LangFlag = ""
|
|
lang_note: str = ""
|
|
cv_score: float | None = None
|
|
#: As `ScoredRole.cv_fit`: how close the role is to the agent's CV, 0..1.
|
|
cv_fit: float | None = None
|
|
|
|
@computed_field # type: ignore[prop-decorator]
|
|
@property
|
|
def salary_state(self) -> str:
|
|
"""As `Role.salary_state`: 'normalised', 'stated' or ''."""
|
|
if self.salary_min is not None or self.salary_max is not None:
|
|
return "normalised"
|
|
return "stated" if self.salary_text else ""
|
|
|
|
|
|
class Company(Model):
|
|
"""A row of `companies`, with `affinity` as the agent it was read for
|
|
has it (see `agent_companies`); 0 when no agent was asked for."""
|
|
|
|
key: str
|
|
name: str
|
|
domain: str = ""
|
|
affinity: int = 0
|
|
status: str = "researching"
|
|
careers_url: str = ""
|
|
notes: str = ""
|
|
#: The host the careers site runs on, readable or not; "" when unknown.
|
|
#: A company with an `ats` and no board subscription is one the app
|
|
#: cannot read yet. See `app.jobs.discover`.
|
|
ats: str = ""
|
|
country: str = ""
|
|
city: str = ""
|
|
#: The name `ats` knows the company by, the `acme` of acme.teamtailor.com;
|
|
#: "" where discovery found none. Kept for a host no parser reads yet.
|
|
ats_slug: str = ""
|
|
#: The Watchlist's rating, global and not an agent's: None not yet rated,
|
|
#: 1 worth checking by hand, 0 not.
|
|
watch: int | None = None
|
|
#: When the careers page was last opened from the Watchlist; "" never.
|
|
careers_opened_at: str = ""
|
|
#: Why the company is read by hand, one of `HAND_REASONS`; "" when it is
|
|
#: not on that list.
|
|
hand_reason: str = ""
|
|
#: The two fetch overrides (`app/jobs/reach.py`), global rather than an
|
|
#: agent's, since fetching is shared. `all_countries` 1: every big board
|
|
#: of the company is read with the agents' wide reach, Europe today,
|
|
#: rather than their places. `all_roles` 1: no role-type narrowing, no
|
|
#: fetch terms today, on any of its boards.
|
|
all_countries: int = 0
|
|
all_roles: int = 0
|
|
created_at: str
|
|
updated_at: str
|
|
|
|
|
|
class Subscription(Model):
|
|
"""A row of `subscriptions`. `company` and `board` are the two labels
|
|
newsdesk calls outlet and section."""
|
|
|
|
id: int
|
|
source: str
|
|
target: str
|
|
label: str
|
|
company: str = ""
|
|
board: str = ""
|
|
params: JsonDict = {}
|
|
subscribed: int = 1
|
|
fetch: int = 1
|
|
show: int = 1
|
|
added_at: str
|
|
target_id: str = ""
|
|
|
|
|
|
class Application(Model):
|
|
"""A row of `applications`. `id` empty means not saved yet; the store mints
|
|
one. `role_uid` is None for a role that was never posted."""
|
|
|
|
id: str = ""
|
|
role_uid: str | None = None
|
|
company_key: str = ""
|
|
role_title: str = ""
|
|
url: str = ""
|
|
status: Status = "saved"
|
|
source: str = ""
|
|
applied_at: str | None = None
|
|
next_action: str = ""
|
|
next_date: str | None = None
|
|
contact: str = ""
|
|
notes: str = ""
|
|
created_at: str = ""
|
|
updated_at: str = ""
|
|
|
|
|
|
# ------------------------------------------------------------------ agents
|
|
|
|
#: The role columns a gate can name. `company` matches on the company key;
|
|
#: `place` takes the Place filter's tokens (`app.places`: `country:DK`,
|
|
#: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; `title`
|
|
#: matches a value as whole words anywhere in the title, folded as `title_key`
|
|
#: folds it, so `account executive` finds "Enterprise Account Executive, UK"
|
|
#: and `data engineer` does not find "Data Engineering Manager"; `remote_from`
|
|
#: takes place tokens (`country:DK`, `country:EU`, `anywhere`) and is met by a
|
|
#: remote role whose area overlaps one, as a `remote:` chip is; the rest are
|
|
#: case-insensitive equality, except `salary_min` (at least the value) and
|
|
#: `languages` (every language the role requires is one of the values).
|
|
GateField = Literal["place", "country", "remote", "remote_from", "seniority", "function",
|
|
"salary_min", "currency", "permit", "languages", "company", "title"]
|
|
GATE_FIELDS: tuple[str, ...] = get_args(GateField)
|
|
|
|
GateMode = Literal["require", "any", "prefer", "exclude"]
|
|
|
|
|
|
class Gate(Model):
|
|
"""One condition of an agent. `require` gates a role that fails it (the
|
|
row is kept with the reason); the `any` gates together are one group a
|
|
role must match at least one of ("in Denmark, or remote"); `prefer` adds
|
|
`weight` to its score when it matches; `exclude` gates a role that matches
|
|
any of its values ("title account executive"), a favourite's too. A gate
|
|
rejects on evidence only: a
|
|
role whose column is empty is not rejected, since `extract` fills the
|
|
column later for the roles that rank, and only for those."""
|
|
|
|
field: GateField
|
|
values: list[str | int | float] = Field(min_length=1)
|
|
mode: GateMode = "require"
|
|
weight: float = Field(1.0, ge=0)
|
|
|
|
|
|
class Weights(Model):
|
|
"""How much each score term counts. Summed with the prefer gates' weights
|
|
into the denominator, so a score stays in 0..1 whatever the numbers."""
|
|
|
|
fit: float = Field(0.6, ge=0)
|
|
recency: float = Field(0.2, ge=0)
|
|
affinity: float = Field(0.2, ge=0)
|
|
#: Cosine to the liked-minus-passed direction of the agent's votes.
|
|
taste: float = Field(0.2, ge=0)
|
|
#: How much each of the judge's three numbers counts in `cv_score`. Not
|
|
#: a score term: they are read when a role is served, so changing them
|
|
#: asks the model for nothing. 0 leaves a number out, all three 0 means
|
|
#: no `cv_score`.
|
|
cv_skills: float = Field(30, ge=0, le=1000, allow_inf_nan=False)
|
|
cv_experience: float = Field(25, ge=0, le=1000, allow_inf_nan=False)
|
|
cv_career: float = Field(30, ge=0, le=1000, allow_inf_nan=False)
|
|
|
|
|
|
class Limits(Model):
|
|
"""How far down an agent's ranking the two model stages read. Each pair
|
|
is a top N, the backstop, and a minimum score, the real stop: a role
|
|
is read when it is among the agent's best `extract_top` ungated scores
|
|
and not below `extract_min`, and summarised likewise. Fit is a
|
|
calibrated level (`score.scale`), so a bar means the same in every run.
|
|
A fresh role with the worst fit scores about 0.27 on the default
|
|
weights, a perfect one about 0.87. Measured on 16,841 open roles for the
|
|
"Copenhagen engineering" starter agent (4,156 ungated, no votes), 0.55
|
|
admits about 300 roles to extract and 0.65 about 35 to summarise, the
|
|
long answer; the top N catches an agent whose bar admits far more. An
|
|
agent keeps the limits stored with it; only a key its JSON lacks
|
|
follows these defaults."""
|
|
|
|
extract_top: int = Field(500, ge=0)
|
|
extract_min: float = Field(0.55, ge=0, le=1)
|
|
summarise_top: int = Field(30, ge=0)
|
|
summarise_min: float = Field(0.65, ge=0, le=1)
|
|
#: How deep `judge` reads an agent with a CV: a long answer per role,
|
|
#: so between summarise's 30 and extract's 500.
|
|
judge_top: int = Field(150, ge=0)
|
|
judge_min: float = Field(0.55, ge=0, le=1)
|
|
#: And the agent's ungated roles closest to its CV (`scores.cv_fit`, see
|
|
#: `app.jobs.score`) that the pair above leaves out, as many as this:
|
|
#: a role the prompts rank low can still be one the CV fits.
|
|
judge_cv_top: int = Field(50, ge=0)
|
|
|
|
|
|
class Prompt(Model):
|
|
"""One of an agent's descriptions of what it looks for. Each is embedded
|
|
on its own and a role's fit is its best prompt's (see `app.jobs.score`),
|
|
so one agent can hunt several kinds of role without blending them. A
|
|
prompt with no text has no fit to give."""
|
|
|
|
name: str
|
|
text: str = ""
|
|
|
|
@field_validator("name")
|
|
@classmethod
|
|
def _named(cls, name: str) -> str:
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("a prompt needs a name")
|
|
return name
|
|
|
|
|
|
def check_prompts(prompts: list[Prompt]) -> list[Prompt]:
|
|
"""The prompts as they are, or ValueError when two share a name
|
|
regardless of case."""
|
|
seen: set[str] = set()
|
|
for prompt in prompts:
|
|
if prompt.name.casefold() in seen:
|
|
raise ValueError(f"two prompts are named {prompt.name!r}")
|
|
seen.add(prompt.name.casefold())
|
|
return prompts
|
|
|
|
|
|
#: An agent's prompts, in order, their names unique regardless of case.
|
|
Prompts = Annotated[list[Prompt], AfterValidator(check_prompts)]
|
|
|
|
|
|
def fetch_terms(value: Any) -> list[str]:
|
|
"""Fetch terms as stored and sent: a JSON list or a list, each stripped,
|
|
empty ones dropped, one of each regardless of case, in the order given."""
|
|
items = _json(value, "[]")
|
|
seen: set[str] = set()
|
|
out: list[str] = []
|
|
for item in items if isinstance(items, list) else []:
|
|
term = " ".join(str(item).split())
|
|
if term and term.casefold() not in seen:
|
|
seen.add(term.casefold())
|
|
out.append(term)
|
|
return out
|
|
|
|
|
|
#: An agent's fetch terms, normalised by `fetch_terms`.
|
|
FetchTerms = Annotated[list[str], BeforeValidator(fetch_terms)]
|
|
|
|
|
|
class Agent(Model):
|
|
"""A row of `agents`: a named search with its own prompts, gates, weights
|
|
and limits. The prompts are rows of `agent_prompts`, read alongside; their
|
|
cached vectors are columns this does not declare.
|
|
|
|
`fetch_terms` are words a role must match for a big board read with the
|
|
wide reach to send it (`app/jobs/reach.py`): the pull's terms are the
|
|
union over the enabled agents, and an agent with none means no board is
|
|
narrowed by words. They decide what is fetched, not what ranks.
|
|
|
|
`cv_text`, `cv_languages`, `like_text`, `avoid_text` and `must_text` are
|
|
what `judge` reads a role against: the CV, the languages it states with
|
|
a level, what the person likes and does not like (the work, the
|
|
workplace, situations; both feed the career score), and their
|
|
must-haves and deal-breakers (set through the API only, not on the
|
|
form; soft, read by the model only; a hard rule is a gate). An agent without
|
|
a CV is not judged, and one without languages flags none
|
|
(`language_flag`)."""
|
|
|
|
id: int
|
|
name: str
|
|
prompts: Prompts = []
|
|
gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
|
weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights()
|
|
limits: Annotated[Limits, BeforeValidator(lambda v: _json(v, "{}"))] = Limits()
|
|
fetch_terms: FetchTerms = []
|
|
cv_text: str = ""
|
|
cv_languages: CvLanguages = []
|
|
like_text: str = ""
|
|
avoid_text: str = ""
|
|
must_text: str = ""
|
|
enabled: bool = True
|
|
created_at: str
|
|
|
|
|
|
class Candidate(Model):
|
|
"""What scoring reads about one open role for one agent: its posted
|
|
date (for `recency`), its company's affinity for that agent, and per
|
|
gate `1` match, `0` no match, `-1` the column is empty. An `exclude`
|
|
gate's match is the position of the first value it matched, from 1, so
|
|
the reason can name it."""
|
|
|
|
uid: str
|
|
posted_at: str
|
|
affinity: int = 0
|
|
gates: list[int] = []
|
|
|
|
|
|
class ApplicationEvent(Model):
|
|
id: int
|
|
application_id: str
|
|
status: Status
|
|
at: str
|