"""The shapes roledesk hands round, and the keys that decide two rows are one role. One set of models, as in newsdesk: the row the table returns and the object the API serves are the same declaration. `Model.from_row` ignores columns a model does not declare, so a new column is never a 500 somewhere else. The three keys cluster roles on `(company_key, title_key, place_key)`. The same title at two companies is two jobs, and the same ad on two boards is one. """ from __future__ import annotations import json import re import sys import unicodedata from datetime import date from functools import lru_cache from pathlib import Path from typing import Annotated, Any, Literal, get_args from pydantic import AfterValidator, BeforeValidator, Field, computed_field, field_validator from desk.models import JsonDict, Model, ModelTaskResult # ------------------------------------------------------------------- keys #: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x), #: (w/m/div), (all genders), (gn). Matched with or without the parentheses. _GENDER = re.compile( r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)" r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)" r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b", re.IGNORECASE, ) #: Legal forms cut from the end of a company name, so "Celonis SE" on one #: board and "celonis" on another share a key. _LEGAL = re.compile( r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|" r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$", re.IGNORECASE, ) def _fold(text: str) -> str: """Casefolded, accents stripped, punctuation to spaces, whitespace collapsed.""" text = unicodedata.normalize("NFKD", text.casefold()) text = "".join(c for c in text if not unicodedata.combining(c)) text = re.sub(r"[\W_]+", " ", text) return " ".join(text.split()) def title_key(title: str) -> str: """`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`.""" return _fold(_GENDER.sub(" ", title or "")) def place_key(place: str) -> str: """`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`.""" return _fold(place or "") def company_key(company: str) -> str: """A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the primary key of `companies`.""" name = _LEGAL.sub("", (company or "").strip()) return _fold(name).replace(" ", "-") def company_fold(company: str) -> str: """What `company_aliases` keys on: `company_key`'s cut of the legal form, then the place fold, so "Orsted A/S" and "Ørsted" are both `orsted`. The key itself keeps `ø`, since it is a primary key.""" return _place_fold(_LEGAL.sub("", (company or "").strip())) # ---------------------------------------------------------------- geography #: Letters NFKD leaves whole because they are letters of their own rather #: than a base letter with a mark, spelt the way boards write them in ASCII. _LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe", "þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"}) def _place_fold(text: str) -> str: """`_fold` for the place lookup only: "Wrocław", "Łódź" and "København" meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is, since it also builds `company_key`, the primary key of `companies`.""" return _fold((text or "").casefold().translate(_LETTERS)) #: Every ISO 3166-1 country by its English name, plus the names boards also #: write for it: its own language's for a European one, and the usual short #: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one. _COUNTRY_NAMES = """ AD Andorra AE United Arab Emirates | UAE | U.A.E. | Emirates AF Afghanistan AG Antigua and Barbuda AI Anguilla AL Albania | Shqipëria | Shqipëri AM Armenia | Hayastan AO Angola AQ Antarctica AR Argentina AS American Samoa AT Austria | Österreich AU Australia AW Aruba AX Åland Islands | Åland AZ Azerbaijan | Azərbaycan BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina BB Barbados BD Bangladesh BE Belgium | België | Belgique | Belgien BF Burkina Faso BG Bulgaria | България | Balgariya BH Bahrain BI Burundi BJ Benin BL Saint Barthélemy BM Bermuda BN Brunei | Brunei Darussalam BO Bolivia BQ Caribbean Netherlands | Bonaire BR Brazil | Brasil BS Bahamas BT Bhutan BV Bouvet Island BW Botswana BY Belarus | Беларусь | Belarus' BZ Belize CA Canada CC Cocos Islands | Cocos (Keeling) Islands CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa CF Central African Republic CG Republic of the Congo | Congo-Brazzaville CH Switzerland | Schweiz | Suisse | Svizzera | Svizra | CHE CI Côte d'Ivoire | Ivory Coast CK Cook Islands CL Chile CM Cameroon CN China | People's Republic of China CO Colombia CR Costa Rica CU Cuba CV Cabo Verde | Cape Verde CW Curaçao CX Christmas Island CY Cyprus | Κύπρος | Kypros | Kibris CZ Czechia | Czech Republic | Česko | Česká republika DE Germany | Deutschland | DEU | Deutschlandweit DJ Djibouti DK Denmark | Danmark | Dänemark DM Dominica DO Dominican Republic DZ Algeria EC Ecuador EE Estonia | Eesti EG Egypt EH Western Sahara ER Eritrea ES Spain | España | Espanya | ESP | Spanien | Espagne ET Ethiopia FI Finland | Suomi FJ Fiji FK Falkland Islands FM Micronesia FO Faroe Islands | Føroyar | Færøerne FR France | FRA GA Gabon GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland | GBR GD Grenada GE Georgia | Sakartvelo GF French Guiana GG Guernsey GH Ghana GI Gibraltar GL Greenland GM Gambia GN Guinea GP Guadeloupe GQ Equatorial Guinea GR Greece | Ελλάδα | Hellas | Ellada GS South Georgia and the South Sandwich Islands GT Guatemala GU Guam GW Guinea-Bissau GY Guyana HK Hong Kong | 香港 HM Heard Island and McDonald Islands HN Honduras HR Croatia | Hrvatska HT Haiti HU Hungary | Magyarország ID Indonesia IE Ireland | Éire IL Israel IM Isle of Man IN India IO British Indian Ocean Territory IQ Iraq IR Iran IS Iceland | Ísland IT Italy | Italia JE Jersey JM Jamaica JO Jordan JP Japan | JPN KE Kenya KG Kyrgyzstan KH Cambodia KI Kiribati KM Comoros KN Saint Kitts and Nevis KP North Korea KR South Korea | Korea | Republic of Korea KW Kuwait KY Cayman Islands KZ Kazakhstan LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic LB Lebanon LC Saint Lucia LI Liechtenstein LK Sri Lanka LR Liberia LS Lesotho LT Lithuania | Lietuva LU Luxembourg | Lëtzebuerg | Luxemburg LV Latvia | Latvija LY Libya MA Morocco MC Monaco MD Moldova | Republic of Moldova ME Montenegro | Crna Gora MF Saint Martin MG Madagascar MH Marshall Islands MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија ML Mali MM Myanmar | Burma MN Mongolia MO Macao | Macau MP Northern Mariana Islands MQ Martinique MR Mauritania MS Montserrat MT Malta MU Mauritius MV Maldives MW Malawi MX Mexico | México | MEX MY Malaysia | MYS MZ Mozambique NA Namibia NC New Caledonia NE Niger NF Norfolk Island NG Nigeria NI Nicaragua NL Netherlands | The Netherlands | Nederland | Holland | NLD | Niederlande NO Norway | Norge | Noreg NP Nepal NR Nauru NU Niue NZ New Zealand | Aotearoa OM Oman PA Panama | Panamá PE Peru | Perú PF French Polynesia PG Papua New Guinea PH Philippines PK Pakistan PL Poland | Polska PM Saint Pierre and Miquelon PN Pitcairn Islands PR Puerto Rico PS Palestine PT Portugal PW Palau PY Paraguay QA Qatar RE Réunion RO Romania | România RS Serbia | Srbija | Србија RU Russia | Russian Federation | Россия | Rossiya RW Rwanda SA Saudi Arabia | KSA | SAU SB Solomon Islands SC Seychelles SD Sudan SE Sweden | Sverige | Suède SG Singapore SH Saint Helena SI Slovenia | Slovenija SJ Svalbard and Jan Mayen | Svalbard SK Slovakia | Slovensko SL Sierra Leone SM San Marino SN Senegal SO Somalia SR Suriname SS South Sudan ST São Tomé and Príncipe SV El Salvador SX Sint Maarten SY Syria SZ Eswatini | Swaziland TC Turks and Caicos Islands TD Chad TF French Southern Territories TG Togo TH Thailand | THA TJ Tajikistan TK Tokelau TL Timor-Leste | East Timor TM Turkmenistan TN Tunisia TO Tonga TR Turkey | Türkiye TT Trinidad and Tobago TV Tuvalu TW Taiwan TZ Tanzania UA Ukraine | Україна | Ukraina UG Uganda UM United States Minor Outlying Islands US United States | United States of America | USA | U.S.A. | US | U.S. | Americas | AMER | North America UY Uruguay UZ Uzbekistan VA Vatican City | Holy See VC Saint Vincent and the Grenadines VE Venezuela VG British Virgin Islands VI U.S. Virgin Islands | US Virgin Islands VN Vietnam | Viet Nam VU Vanuatu WF Wallis and Futuna WS Samoa XK Kosovo | Kosova YE Yemen YT Mayotte ZA South Africa ZM Zambia ZW Zimbabwe """ #: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK. _ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line) #: The countries `EU` stands for when a line names several: the UN #: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and #: the Council of Europe's members in the Caucasus and Turkey, which #: European employers list beside their other offices. _EUROPE = frozenset(( "AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE", "DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE", "IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL", "NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA", "XK", )) _EUROPE_OR_EU = _EUROPE | {"EU"} #: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them. _REGIONS = ("europe", "emea", "eu", "european union", "eea") #: The cities boards name most, and a few regions below a country. A US #: state or a Canadian province in the same part of the line beats one of #: these ("London, Ontario" is Canada); a country name does not lose to it. _CITIES: dict[str, str] = { **dict.fromkeys(("vienna", "wien", "graz", "linz", "salzburg", "innsbruck", "villach", "klagenfurt"), "AT"), **dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent", "bruges", "brugge", "liege", "luik"), "BE"), **dict.fromkeys(("sofia",), "BG"), **dict.fromkeys(("zagreb",), "HR"), **dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"), **dict.fromkeys(("prague", "praha", "brno"), "CZ"), **dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg", "region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm", "brondby", "glostrup", "soborg", "malov", "birkerod", "hornbaek"), "DK"), **dict.fromkeys(("tallinn", "tartu"), "EE"), **dict.fromkeys(("helsinki", "espoo", "tampere", "vantaa"), "FI"), **dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", "lille", "rennes", "amiens", "rouen", "aix en provence", "grenoble", "annecy", "metz", "strasbourg", "antibes", "avignon", "bourg en bresse", "brest", "caen", "chambery", "colmar", "montpellier", "nancy", "nice", "nimes", "saint etienne", "saint nazaire", "saint ouen l aumone", "valence", "angers", "bayonne", "beauvais", "boulogne sur mer", "challans", "cholet", "clermont ferrand", "dunkerque", "la roche sur yon", "la rochelle", "lamballe", "lannion", "laval", "le havre", "le mans", "lens", "les sables d olonne", "limoges", "lorient", "loudeac", "morlaix", "mulhouse", "orleans", "orvault", "paimpol", "pornic", "quimper", "reims", "reze", "saint brieuc", "saint herblain", "saint malo", "tours", "troyes", "vannes", "vertou"), "FR"), **dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund", "mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen", "bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg", "kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck", "oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen", "mainz", "lubeck", "bayreuth", "bad homburg", "homburg", "ettlingen", "bremerhaven", "breidenbach", "bielefeld", "saarbrucken", "hildesheim", "moers", "kiel", "ulm", "regensburg", "freiburg", "obersulm", "seevetal", "neukirchen vluyn", "flensburg", "ingolstadt", "kempten", "konstanz", "magdeburg", "weimar", "chemnitz", "erfurt", "freiberg", "gustrow", "kaiserslautern", "mohnesee", "neubrandenburg", "neumarkt", "neumunster", "paderborn", "pforzheim", "aachen", "amberg", "annaberg buchholz", "backnang", "bad kissingen", "bad liebenzell", "baden baden", "bamberg", "besigheim", "bietigheim bissingen", "bocholt", "brackenheim", "bretten", "bruchsal", "cloppenburg", "cottbus", "crailsheim", "dachau", "darmstadt", "donaueschingen", "donauworth", "ebersberg", "eppingen", "erding", "erlangen", "ettenheim", "feuchtwangen", "filderstadt", "florsheim", "flohrsheim", "freising", "friedrichshafen", "fulda", "furth", "geislingen", "gera", "germering", "goppingen", "gundelsheim", "gunzburg", "hagenow", "halle saale", "heidenheim", "heilbronn", "herrsching", "herzogenaurach", "holzkirchen", "holzminden", "hoxter", "idar oberstein", "jena", "kaisheim", "kierspe", "kirchheim", "kitzingen", "koblenz", "kolbermoor", "konigsbrunn", "kunzelsau", "landau", "landsberg", "landshut", "lauf an der pegnitz", "lindern", "lingen", "lorrach", "ludwigsburg", "ludwigshafen", "malchow", "marburg", "meissen", "memmingen", "meppen", "metzingen", "monchengladbach", "moosburg", "mosbach", "muhldorf", "mulheim", "murnau", "neuburg an der donau", "neustadt", "nordlingen", "northeim", "oberhausen", "offenburg", "ohringen", "passau", "peine", "petershausen", "pfaffenhofen", "pfullendorf", "plau am see", "prien am chiemsee", "rastatt", "reutlingen", "riesa", "robel", "roth", "rothenburg ob der tauber", "rottweil", "salzgitter", "sankt georgen", "schorndorf", "schwabisch gmund", "schwabisch hall", "schweinfurt", "starnberg", "stockach", "tegernsee", "templin", "trier", "tuttlingen", "uberlingen", "uedem", "vaihingen", "villingen schwenningen", "waren an der muritz", "wasserburg", "weilheim", "wernigerode", "wismar", "wittenberg", "wittstock", "wolfenbuttel", "wolfratshausen", "wolfsburg", "zwickau", "euskirchen", "tangstedt", "erkrath", "sigmaringen", "grimmen", "ribnitz damgarten", "stralsund", "warstein", "ansbach", "balingen", "coburg", "duisburg", "grossostheim", "merzig", "saarlouis", "schwerin", "singen", "troisdorf", "bad bevensen", "dannenberg", "luchow", "nagold", "salzwedel", "uelzen", "boblingen", "giessen", "hagen", "hechingen", "leverkusen", "solingen", "wetzlar"), "DE"), **dict.fromkeys(("athens", "thessaloniki"), "GR"), **dict.fromkeys(("budapest", "cegled"), "HU"), **dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"), **dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna", "florence", "verona", "cagliari", "ascoli piceno", "ancarano", "naples", "napoli", "venice", "venezia", "genoa", "genova", "bolzano", "padua", "padova", "brindisi", "matera", "pistoia", "taranto", "asti", "campobasso", "cassino", "chieti", "civitavecchia", "cosenza", "crema", "fiumicino", "frosinone", "grosseto", "guidonia montecelio", "l aquila", "ladispoli", "lamezia terme", "latina", "livorno", "lucca", "massa carrara", "pescara", "pisa", "ravenna", "rimini", "sanremo", "teramo", "terni", "trieste", "vibo valentia", "forli", "cesena", "emilia romagna", "tuscany", "lazio"), "IT"), **dict.fromkeys(("riga",), "LV"), **dict.fromkeys(("vilnius", "kaunas"), "LT"), **dict.fromkeys(("luxembourg", "luxemburg"), "LU"), **dict.fromkeys(("valletta",), "MT"), **dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", "delft", "amersfoort", "deventer", "maastricht", "bleiswijk", "almere", "enschede"), "NL"), **dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz", "biskupiec", "sochaczew", "teresin"), "PL"), **dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"), **dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi", "timisoara", "brasov"), "RO"), **dict.fromkeys(("belgrade", "beograd"), "RS"), **dict.fromkeys(("bratislava",), "SK"), **dict.fromkeys(("ljubljana",), "SI"), **dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao", "granada", "santiago de compostela", "pontevedra", "tarragona"), "ES"), **dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala", "vasteras", "linkoping", "orebro", "norrkoping", "jonkoping", "almhult", "halmstad", "solna"), "SE"), **dict.fromkeys(("reykjavik",), "IS"), **dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"), **dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk", "oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff", "guildford", "truro", "swansea", "blackpool", "coventry", "brighton", "weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley", "slough", "southampton", "swindon", "liverpool", "redhill", "milton keynes", "birmingham", "cambridge", "fermanagh", "tyrone"), "GB"), **dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug", "sankt gallen", "st gallen", "lucerne", "luzern", "landquart"), "CH"), **dict.fromkeys(("kyiv", "kiev"), "UA"), **dict.fromkeys(("istanbul", "ankara"), "TR"), **dict.fromkeys(("tirana",), "AL"), **dict.fromkeys(("yerevan",), "AM"), **dict.fromkeys(("tbilisi",), "GE"), **dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle", "boston", "chicago", "austin", "los angeles", "palo alto", "mountain view", "san mateo", "foster city", "sunnyvale", "san jose", "menlo park", "redwood city", "costa mesa", "denver", "atlanta", "washington dc", "washington d c", "miami", "dallas", "houston", "phoenix", "tempe", "portland", "salt lake city", "pittsburgh", "philadelphia", "detroit", "san diego", "raleigh", "nashville", "minneapolis", "cambridge ma", "bay area", "arlington", "leominster", "andover", "el segundo", "columbus", "tysons corner", "tysons", "lowell", "mclean", "morrisville", "southlake", "st louis", "aventura", "brooklyn", "charlotte", "corte madera", "morgan hill", "newport beach", "las vegas", "san antonio", "tampa", "indianapolis", "honolulu", "memphis", "glenview", "shakopee", "lexington", "wilmington", "murrieta"), "US"), **dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo", "edmonton", "winnipeg"), "CA"), **dict.fromkeys(("buenos aires",), "AR"), **dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos", "curitiba"), "BR"), **dict.fromkeys(("santiago", "santiago de chile"), "CL"), **dict.fromkeys(("bogota",), "CO"), **dict.fromkeys(("lima", "miraflores"), "PE"), **dict.fromkeys(("montevideo",), "UY"), **dict.fromkeys(("managua",), "NI"), **dict.fromkeys(("santo domingo",), "DO"), **dict.fromkeys(("san salvador",), "SV"), **dict.fromkeys(("panama city",), "PA"), **dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara", "cdmx"), "MX"), **dict.fromkeys(("dubai", "abu dhabi"), "AE"), **dict.fromkeys(("riyadh",), "SA"), **dict.fromkeys(("doha",), "QA"), **dict.fromkeys(("manama",), "BH"), **dict.fromkeys(("salmiya",), "KW"), **dict.fromkeys(("muscat",), "OM"), **dict.fromkeys(("cairo", "maadi"), "EG"), **dict.fromkeys(("casablanca",), "MA"), **dict.fromkeys(("dakar",), "SN"), **dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa", "netanya"), "IL"), **dict.fromkeys(("almaty",), "KZ"), **dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"), **dict.fromkeys(("dhaka",), "BD"), **dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi", "ahmedabad"), "IN"), **dict.fromkeys(("kuala lumpur", "bangsar south", "melaka", "malacca", "kulim", "penang", "ipoh"), "MY"), **dict.fromkeys(("singapore",), "SG"), **dict.fromkeys(("manila", "taguig", "cavite", "general trias", "muntinlupa"), "PH"), **dict.fromkeys(("bangkok", "samut prakan", "samut prakarn"), "TH"), **dict.fromkeys(("phnom penh",), "KH"), **dict.fromkeys(("taipei", "hsinchu", "kaohsiung", "tainan"), "TW"), **dict.fromkeys(("tokyo", "osaka", "nagoya"), "JP"), **dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou", "wuxi", "xi an", "qingdao", "chongqing", "nanjing", "tianjin", "wuhan", "xiamen"), "CN"), **dict.fromkeys(("seoul", "seongnam", "busan", "daegu", "daejeon", "gwangju"), "KR"), **dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales", "queensland", "canberra"), "AU"), **dict.fromkeys(("jakarta", "batam"), "ID"), **dict.fromkeys(("hanoi", "ho chi minh"), "VN"), **dict.fromkeys(("tunis",), "TN"), **dict.fromkeys(("ebene",), "MU"), **dict.fromkeys(("cape town", "johannesburg"), "ZA"), **dict.fromkeys(("auckland",), "NZ"), } #: US states and Canadian provinces, by name and by the code boards write #: after a comma ("Boston, MA", "Toronto, ON"). _SUBDIVISIONS = """ US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT CA Nunavut NU """ def _subdivisions() -> tuple[dict[str, str], dict[str, str]]: names: dict[str, str] = {} codes: dict[str, str] = {} for line in _SUBDIVISIONS.split("\n"): if line: country, rest = line.split(" ", 1) for entry in rest.split("|"): name, code = entry.strip().rsplit(" ", 1) names[_place_fold(name)] = country codes[code] = country return names, codes _STATE_NAMES, _STATE_CODES = _subdivisions() def _names() -> dict[str, tuple[str, str]]: """Every folded name to `(code, kind)`; kind is `country`, `city` or `state`. A name that is a city and a state ("New York") is the city, so it does not beat another city in its line. A name that is a country and a state ("Georgia") is the country here and read as the state only when the line names a place in the US too.""" names: dict[str, tuple[str, str]] = {} for name, code in _STATE_NAMES.items(): names[name] = (code, "state") for name, code in _CITIES.items(): names[_place_fold(name)] = (code, "city") for line in _COUNTRY_NAMES.split("\n"): if line: code, rest = line.split(" ", 1) for name in rest.split("|"): names[_place_fold(name)] = (code, "country") for name in _REGIONS: names[name] = ("EU", "country") return names _NAMES = _names() #: A state name that is also a country's, read as the state only when the #: line names a place in that state's country too. _SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES if _NAMES[name][1] == "country"} #: The longest name in words, so a lookup tries "new york city" before "new #: york" and "northern ireland" before "ireland". _LONGEST = max(len(name.split()) for name in _NAMES) #: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin". _STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b") #: A country code in parentheses, "Leicester (GB)", read only when no name matched. _PAREN_CODE = re.compile(r"\(([A-Z]{2})\)") #: Where one location in a line ends and the next begins. A state beats a #: city only within one: "London; Ontario" is two places. _SEGMENT = re.compile(r"[;|/\n]") def _hits(folded: str) -> list[tuple[str, str, str]]: """The names in a folded line, longest first at each word, as `(name, code, kind)`.""" words = folded.split() hits = [] i = 0 while i < len(words): for n in range(min(_LONGEST, len(words) - i), 0, -1): name = " ".join(words[i:i + n]) if name in _NAMES: hits.append((name, *_NAMES[name])) i += n break else: i += 1 return hits def country_of(place: str) -> str: """The one country a board's location line names, as an ISO code; `EU` when it names several and all of them are in Europe; '' when it names none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston, MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or New York" and "Remote, EU or US" are ''. Within one location (the line split at `;`, `|` and `/`) the widest unit named wins: a country beats a state or a province, which beats a city. So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A country also beats the region it is in: "London, EMEA" is GB. A state code that is also a country's ("DE", "CO", "MA"), and "Georgia", count as the state only when the line names a place in the US too: "Berlin, DE" is DE, "Bogota, CO" is CO, "Ellwangen, DE" is '' rather than Delaware. A country code in parentheses, "Leicester (GB)", is read only when no name matched.""" segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")] named = {code for _, hits in segments for _, code, _ in hits} found: set[str] = set() for text, hits in segments: shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named} countries = {code for name, code, kind in hits if kind == "country" and code != "EU" and name not in shared} states = {code for _, code, kind in hits if kind == "state"} states |= {_SHARED_NAMES[name] for name in shared} states |= {country for code in _STATE_CODE.findall(text) if (country := _STATE_CODES.get(code)) and (code not in _ISO or country in named)} found |= (countries or states or {code for _, code, _ in hits if code != "EU"}) found |= {code for _, code, _ in hits if code == "EU"} if not found: found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO} if len(found) == 1: return found.pop() if found and found <= _EUROPE_OR_EU: countries = found - {"EU"} return countries.pop() if "EU" in found and len(countries) == 1 else "EU" return "" # --------------------------------------------------------------- vocabulary Remote = Literal["", "remote", "hybrid", "onsite"] REMOTE: tuple[str, ...] = get_args(Remote) #: Words in a location line that say how the role is worked, for a board that #: gives no remote flag of its own: Greenhouse writes "Remote", "Hybrid" or #: "Distributed" as the location and nothing else. _REMOTE_WORDS: tuple[tuple[str, str], ...] = ( ("hybrid", "hybrid"), ("remote", "remote"), ("distributed", "remote"), ("home based", "remote"), ("home-based", "remote"), ("work from home", "remote"), ("anywhere", "remote"), ("worldwide", "remote"), ("telecommute", "remote"), ("on-site", "onsite"), ("onsite", "onsite"), ("on site", "onsite"), ("in-office", "onsite"), ("in office", "onsite"), ) def remote_of(place: str) -> str: """How a board's location line says the role is worked, as one of `REMOTE`, or '' when it does not say. "Hybrid" wins over "remote" in the same line ("Remote or hybrid, Berlin" is hybrid: some days in an office), and a line that names only a place is ''.""" folded = _fold(place or "") for word, mode in _REMOTE_WORDS: if word in folded: return mode return "" #: A role's level, '' when nothing says. From `staff` up a level sits on one #: of two tracks, individual contributor or people lead, and each rung pairs #: the two as big tech firms do: Staff with Manager, Principal with Senior #: Manager, Distinguished with Director. `lead` is a team lead (Team Lead, #: Supervisor, Teamleiter); a "Tech Lead" or "Marketing Lead" is `staff`. Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager", "principal", "senior_manager", "distinguished", "director", "exec"] SENIORITIES: tuple[str, ...] = get_args(Seniority) #: The levels a seniority filter may name: `SENIORITIES` without the unknown. LEVELS: tuple[str, ...] = tuple(level for level in SENIORITIES if level) #: The rungs of the ladder from the bottom, each named after its lowest #: individual level; a range filter names two of them. RUNGS: tuple[str, ...] = ("intern", "junior", "mid", "senior", "staff", "principal", "distinguished", "exec") #: The tracks a filter may keep: `ic` (individual contributor) or `people`. TRACKS: tuple[str, ...] = ("ic", "people") #: Each level's rung (an index into `RUNGS`) and track; the rungs below #: `staff` have no track and pass either. LADDER: dict[str, tuple[int, str]] = { "intern": (0, ""), "junior": (1, ""), "mid": (2, ""), "senior": (3, ""), "staff": (4, "ic"), "lead": (4, "people"), "manager": (4, "people"), "principal": (5, "ic"), "senior_manager": (5, "people"), "distinguished": (6, "ic"), "director": (6, "people"), "exec": (7, "people"), } def levels_between(low: str = "", high: str = "", track: str = "") -> tuple[str, ...]: """The levels from rung `low` to rung `high` (names in `RUNGS`, '' for the open end) on `track` (`TRACKS`, '' for both), in `LEVELS` order. A level with no track passes any track. An unknown name is a ValueError.""" for name, value, allowed in (("low", low, RUNGS), ("high", high, RUNGS), ("track", track, TRACKS)): if value and value not in allowed: raise ValueError(f"{name} is one of {', '.join(allowed)}, not {value!r}") bottom = RUNGS.index(low) if low else 0 top = RUNGS.index(high) if high else len(RUNGS) - 1 return tuple(level for level in LEVELS if bottom <= LADDER[level][0] <= top and (not track or LADDER[level][1] in ("", track))) #: The kind of contract a posting states, as `app.prep` maps a host's word. #: '' is "it does not say". `plugins/readers/_shared.py` keeps a copy for the #: readers' own tests, since a plugin imports nothing from `app`. Employment = Literal["", "full_time", "part_time", "contract", "temporary", "internship", "freelance"] EMPLOYMENTS: tuple[str, ...] = tuple(e for e in get_args(Employment) if e) #: Whether a posting says it sponsors a work permit. '' is "it does not say". Permit = Literal["", "sponsors", "no_sponsorship"] PERMITS: tuple[str, ...] = get_args(Permit) Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer", "rejected", "withdrawn"] STATUSES: tuple[str, ...] = get_args(Status) #: -1 mute, 0 neutral, 1 interested, 2 favourite (skips the `any` gates). AFFINITY = range(-1, 3) #: Why a company is on the list checked by hand. `unread` is a host the app #: could read and has no parser for yet, `robots` a site whose robots.txt #: disallows it, `blocked` one that answers a program with a challenge, `key` #: one whose API answers only with a key lifted from its own script, `group` a #: company that sits inside its owner's board, `gone` one whose site is dead, #: and `added` one put there by hand or by an import and not looked at. '' #: is none of these: the company is read, or is not a matter for the list. HandReason = Literal["", "unread", "robots", "blocked", "key", "group", "gone", "added"] HAND_REASONS: tuple[str, ...] = get_args(HandReason) #: How well somebody speaks a language, lowest first: what an agent's CV #: states and what a posting asks for. The order is the comparison. LanguageLevel = Literal["basic", "conversational", "professional", "fluent", "native"] LANGUAGE_LEVELS: tuple[str, ...] = get_args(LanguageLevel) #: Whether a posting requires a language or calls it a plus. LanguageKind = Literal["required", "plus"] #: English names of the languages a posting in reach names, by ISO 639-1 #: code, for `language_flag`'s note. A code not here is shown as it is. LANGUAGE_NAMES: dict[str, str] = { "ar": "Arabic", "bg": "Bulgarian", "cs": "Czech", "da": "Danish", "de": "German", "el": "Greek", "en": "English", "es": "Spanish", "et": "Estonian", "fi": "Finnish", "fo": "Faroese", "fr": "French", "hi": "Hindi", "hr": "Croatian", "hu": "Hungarian", "is": "Icelandic", "it": "Italian", "ja": "Japanese", "kl": "Greenlandic", "ko": "Korean", "lt": "Lithuanian", "lv": "Latvian", "nb": "Norwegian", "nl": "Dutch", "nn": "Norwegian", "no": "Norwegian", "pl": "Polish", "pt": "Portuguese", "ro": "Romanian", "ru": "Russian", "sk": "Slovak", "sl": "Slovenian", "sr": "Serbian", "sv": "Swedish", "tr": "Turkish", "uk": "Ukrainian", "zh": "Chinese", } _LANGUAGE_CODE = re.compile(r"^[a-z]{2}$") def language_code(value: Any) -> str: """An ISO 639-1 code, lowercased, or '' when `value` is not one.""" code = str(value or "").strip().lower() return code if _LANGUAGE_CODE.match(code) else "" # -------------------------------------------------------------- posting dates @lru_cache(maxsize=1) def _declared_posted() -> dict[str, str]: """`fields.posted` of every reader that declares `READS`, by name. Read once: the readers are loaded when `plugins.readers` is imported. `plugins` is a top-level folder beside `app`, reached as `app/jobs/extract.py` reaches it.""" root = str(Path(__file__).resolve().parents[1]) if root not in sys.path: sys.path.insert(0, root) from plugins.readers import READERS return {name: module.READS.fields.posted for name, module in READERS.items() if getattr(module, "READS", None) is not None} def trusted_posted(host: str) -> bool: """Whether `host` states a date that is safe to read as when a role was posted: its reader declares `fields.posted="trusted"`. Which field each host reads, and when it was checked, is in the reader module. A role without a trusted date keeps `first_seen` as its posted date.""" return _declared_posted().get(host) == "trusted" #: A `published_at` before this is a parsing slip or a placeholder, not a #: real posting date: no board here existed before 2015. _POSTED_FLOOR = "2015-01-01T00:00:00+00:00" def host_word(value: Any) -> str: """A host's word as `roles.*_host` stores it: stripped text, a bool as 'true' or 'false', '' for nothing. What `app.prep` maps.""" if value is None: return "" if isinstance(value, bool): return "true" if value else "false" return str(value).strip() def posted_host(raw: dict[str, Any]) -> str: """The host a role's dates came from, read the way `ats` and `search` both put it in `Item.raw`: `board` for an ATS board (`greenhouse`, `lever`, ...), `provider` for a cross-company search (`jobindex`, `adzuna`). '' when neither plugin wrote one.""" return str(raw.get("board") or raw.get("provider") or "") def posted_at(published_at: str, first_seen: str, host: str) -> str: """The date a role counts as posted: the earlier of `first_seen` (when the app itself first stored it) and `published_at` (what the board claims), but only when `trusted_posted(host)` and `published_at` is neither absurdly old nor later than `first_seen` — a role cannot be posted after the app already saw it, trusted host or not, which also catches a `published_at` the board reports in the future (`first_seen` is never later than the moment this runs). Both dates are `Store`'s own UTC, zero-padded `_iso` text, so comparing them as strings is exact and needs no parsing. A role with no trusted date, or with one that fails the check, keeps `first_seen`.""" if trusted_posted(host) and _POSTED_FLOOR <= published_at <= first_seen: return published_at return first_seen def iso_day(value: Any) -> str: """A date as `YYYY-MM-DD` from an ISO date or timestamp ("2026-10-18", "2026-10-18T00:00:00"), or '' when `value` holds no real date.""" text = str(value or "").strip()[:10] try: return date.fromisoformat(text).isoformat() if len(text) == 10 else "" except ValueError: return "" def _json(value: Any, empty: str) -> Any: """A JSON text column as the Python value, tolerant like `JsonDict`: a value that is already parsed passes, and one that does not parse is the empty value rather than a 500.""" if not isinstance(value, str): return value try: return json.loads(value or empty) except ValueError: return json.loads(empty) #: A column holding a JSON array of strings, such as `roles.languages`. JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))] def _json_list(value: Any) -> list[Any]: """A nullable JSON array column as a list: NULL, a parse failure and anything but a list are [].""" found = _json(value, "[]") if value is not None else [] return found if isinstance(found, list) else [] class CvLanguage(Model): """A language an agent's CV states, with how well: `language` an ISO 639-1 code, `level` one of `LANGUAGE_LEVELS`.""" language: str level: LanguageLevel @field_validator("language", mode="before") @classmethod def _code(cls, value: Any) -> str: code = language_code(value) if not code: raise ValueError(f"a language is an ISO 639-1 code such as 'en', not {value!r}") return code def cv_languages(value: Any) -> list[CvLanguage]: """An agent's languages as stored and sent: one per code, the first kept, in the order given.""" seen: set[str] = set() out: list[CvLanguage] = [] for item in _json_list(value): found = item if isinstance(item, CvLanguage) else CvLanguage.model_validate(item) if found.language not in seen: seen.add(found.language) out.append(found) return out #: An agent's languages, normalised by `cv_languages`. CvLanguages = Annotated[list[CvLanguage], BeforeValidator(cv_languages)] class RoleLanguage(Model): """A language a posting names (`roles.languages_required`, read by `extract`): its ISO 639-1 code, the level asked for ('' when the posting does not say), and whether it is required or a plus.""" language: str level: Literal["", "basic", "conversational", "professional", "fluent", "native"] = "" kind: LanguageKind = "required" def role_languages(value: Any) -> list[RoleLanguage]: out = [] for item in _json_list(value): try: out.append(item if isinstance(item, RoleLanguage) else RoleLanguage.model_validate(item)) except ValueError: continue return out #: `roles.languages_required`: NULL (never read) and [] are both no language. RoleLanguages = Annotated[list[RoleLanguage], BeforeValidator(role_languages)] LangFlag = Literal["missing", "below", "plus", ""] def _language_name(code: str) -> str: return LANGUAGE_NAMES.get(code, code) def _asked(language: RoleLanguage) -> str: name = _language_name(language.language) return f"{name} ({language.level})" if language.level else name def language_flag(required: list[RoleLanguage], mine: list[CvLanguage]) -> tuple[LangFlag, str]: """`(flag, note)` for a posting's languages against an agent's. `missing` when a required language is not one of the agent's, `below` when the level it states is above the agent's, else `plus` when a plus language is one of the agent's, else ''. The note says why, as "requires Danish (fluent); yours: none". No flag when the agent states no languages: there is nothing to compare with.""" if not mine: return "", "" levels = {m.language: m.level for m in mine} rank = LANGUAGE_LEVELS.index needed = [r for r in required if r.kind == "required"] missing = [r for r in needed if r.language not in levels] if missing: return "missing", "; ".join(f"requires {_asked(r)}; yours: none" for r in missing) below = [r for r in needed if r.level and rank(r.level) > rank(levels[r.language])] if below: return "below", "; ".join(f"requires {_asked(r)}; yours: {levels[r.language]}" for r in below) plus = [r for r in required if r.kind == "plus" and r.language in levels] if plus: return "plus", "; ".join(f"{_asked(r)} is a plus; yours: {levels[r.language]}" for r in plus) return "", "" #: What career counts as at most when a posting breaks one of the agent's #: must-haves: the top of the rubric's 0-39 band for a dead end. The model #: reports whether each must-have is met; this cap is applied in code, when a #: role is served (`cv_score`, `Store.judgement`, `store._cv_sql`). BROKEN_CAP = 39 #: A must-have's verdict, as the judge answers it. MUST_VERDICTS = ("met", "broken", "unclear") _BULLET = re.compile(r"^\s*(?:[-*\u2022]+|\(?\d+[.)])\s+") def must_lines(text: str) -> list[str]: """The agent's must-haves text read as one must-have per line: a leading bullet (`-`, `*`, `•`) or number (`1.`, `2)`) is dropped, so are empty lines.""" found = [] for line in text.splitlines(): line = " ".join(_BULLET.sub("", line, count=1).split()) if line: found.append(line) return found def capped_career(career: int, broken: int = 0) -> int: """`career` as it counts: at most `BROKEN_CAP` when any must-have is broken.""" return min(career, BROKEN_CAP) if broken else career def cv_score(skills: int, experience: int, career: int, weights: "Weights | None" = None, broken: int = 0) -> float | None: """The one number of a CV match, 0 to 100: the three numbers of the judge averaged by the agent's `cv_skills`, `cv_experience` and `cv_career` weights (30, 25 and 30 by default), career as `capped_career` has it. None when all three weights are 0.""" w = weights or Weights() total = w.cv_skills + w.cv_experience + w.cv_career if not total: return None career = capped_career(career, broken) return round((w.cv_skills * skills + w.cv_experience * experience + w.cv_career * career) / total, 1) class Must(Model): """One must-have of an agent as the judge read it against a role: its `id` (`M1`, by line), the line's `text` and the `verdict`.""" id: str text: str = "" verdict: Literal["met", "broken", "unclear"] = "unclear" class Judgement(Model): """What `judge` read of one role for one agent (`role_judgements`): the three dimensions 0 to 100 as the model gave them, `cv_score` (`cv_score`) and 1 to 3 strengths and gaps grounded in the posting. `musts` is the verdict on each must-have and `broken` how many are broken; any makes `career_capped` the lower of `career` and `BROKEN_CAP`. `cv_score` and `career_capped` are not read from the table: `Store.judgement` sets them from the agent's weights; `cv_score` is None when all three are 0.""" skills: int experience: int career: int career_capped: int | None = None broken: int = 0 musts: Annotated[list[Must], BeforeValidator(lambda v: _json(v, "[]"))] = [] cv_score: float | None = None strengths: JsonStrings = [] gaps: JsonStrings = [] judged_at: str # ------------------------------------------------------------------ models class RoleCopy(Model): """One open copy of a kept role, as the reader lists it under "Also posted": where it is listed (`host`, the search provider or the board's ATS) and the link that opens it there.""" uid: str host: str = "" place: str = "" original_url: str = "" posting_url: str = "" posted_at: str = "" class RolePlace(Model): """One place of a role (`role_places`): the host's `text`, and the place of the gazetteer it resolves to, `place_id` None (and `name` and `kind` empty) while it resolves to none. `url` is the link of a copy folded into the role, '' for the role's own places.""" position: int = 0 text: str = "" place_id: int | None = None name: str = "" kind: str = "" url: str = "" class Role(Model): """A row of `roles`: one posting on one board.""" uid: str source: str source_id: str subscription_id: int | None = None company_key: str = "" company: str = "" title: str = "" title_key: str = "" place: str = "" place_key: str = "" country: str = "" remote: Remote = "" salary_min: int | None = None salary_max: int | None = None currency: str = "" #: The pay a board stated, as one human-readable line, whether or not it #: could be normalised into `salary_min`/`salary_max`/`currency` #: (`plugins.ats`'s per-host `_*_salary_text`; docs/decisions.md). salary_text: str = "" #: The level shown and filtered on: the first filled of `seniority_model`, #: `seniority_stated` and `seniority_title`. seniority: str = "" seniority_model: str = "" seniority_stated: str = "" seniority_title: str = "" #: The contract kind, one of `EMPLOYMENTS` or ''; mapped by `app.prep`. employment: str = "" function: str = "" permit: Permit = "" #: ISO 639-1 codes the posting says are required, lowercase: what the #: `languages` gate reads. languages: JsonStrings = [] #: Every language the posting names, with the level it asks for and #: whether it is required or a plus (`extract`); [] until read. languages_required: RoleLanguages = [] #: The last day to apply, `YYYY-MM-DD`: the host's own date, else the #: one `extract` read; None when neither says (`app.prep`). deadline: str | None = None description: str = "" feed_text: str = "" #: The short text the host's list carries, until `description` arrives. teaser: str = "" original_url: str = "" #: Where a Jobindex ad's own page leads, found when the text is fetched #: (`app.jobs.posting`); '' until then and for every other host. posting_url: str = "" published_at: str #: The date this role counts as posted: see `posted_at`. Used for #: `recency` and the feed's newest order; `first_seen` stays what it #: always was, when the app itself first stored the role. posted_at: str first_seen: str last_seen: str closed_at: str | None = None raw: JsonDict = {} extracted_at: str | None = None created_at: str #: The vote of the agent the role was read for (see `role_votes`): None #: unjudged, else -1 pass, 0 neutral, 1 like. vote: Literal[-1, 0, 1] | None = None voted_at: str | None = None #: NULL unread, else when the feed opened it or judged it. A vote stamps #: this too, see `Store.set_vote`: voting is reading. read_at: str | None = None summary: str = "" summarised_at: str | None = None #: The posting page as plain text, for a role whose description is a teaser. article_text: str = "" article_at: str | None = None #: The open vocabulary `summarise` read off the posting, sorted. Not a #: column: loaded from `role_tags` alongside the row, see `Store.role`. tags: list[str] = [] #: The uid of the kept role when this one is a copy of the same posting #: (`app.dupes`, `Store.rebuild_dupes`), else None. A copy is left out of #: the feed and the counts but still opens by uid. dup_of: str | None = None #: The open copies of this role, oldest first, when it is the kept one; #: empty otherwise. Not a column: filled by `Store.role`. copies: list[RoleCopy] = [] #: Every place of the role, the nearest to the viewing agent first #: (`places.Nearness`), else in the host's order. Not a column: filled #: by `Store.role` from `role_places`. places: list[RolePlace] = [] #: The text of the first of `places`, what the row and the reader lead #: with; '' for a role with no place. lead_place: str = "" #: The link of the folded copy whose place leads, which the posting #: opens at; '' when the role's own place leads, and the role's own #: link stands. lead_url: str = "" #: Where a remote hire may live, as a label: "anywhere", "Europe", #: "United States, Canada". '' when the role is not remote or its area is #: not stated. Not a column: filled by `Store` from `role_remote`. remote_where: str = "" @computed_field # type: ignore[prop-decorator] @property def salary_state(self) -> Literal["normalised", "stated", ""]: """Not a column: 'normalised' when `salary_min` or `salary_max` is set, 'stated' when only `salary_text` is, '' otherwise. Gates and scoring read `salary_min` alone and never this (docs/decisions.md); it is what the feed's Pay filter and badge read instead (`Store.PAY`, `static/roles.js`).""" if self.salary_min is not None or self.salary_max is not None: return "normalised" if self.salary_text: return "stated" return "" class ExtractResult(ModelTaskResult): """What one run of `extract` did. See app/jobs/extract.py.""" #: Roles the model answered for, whether or not it found anything to fill. extracted: int = 0 #: Columns written, across those roles. A role answered with nothing the #: posting states adds to `extracted` and not to this. filled: int = 0 @property def done(self) -> int: return self.extracted class ScoredRole(Role): """A role as `/api/roles/{uid}` serves it: with one agent's score and gate reason when an agent was asked for, and without otherwise.""" score: float | None = None gated_reason: str = "" #: The agent's prompt that gave the role its fit; "" when fit did not #: count (gated, no vector, no prompt with text). prompt: str = "" #: The agent's `judge` reading of the role against its CV, None until #: judged; `cv_score` is its total. judgement: Judgement | None = None cv_score: float | None = None #: How close the role is to the agent's CV, 0..1 (`scores.cv_fit`), None #: without a CV, a vector or a score, and for a gated role. cv_fit: float | None = None #: The posting's languages against the agent's (`language_flag`). lang_flag: LangFlag = "" lang_note: str = "" #: How many characters `RoleRow.excerpt` keeps at most: the feed row's own #: cap, built once here rather than by the client, so a list response never #: carries more of a role's text than a row draws. EXCERPT_LIMIT = 200 #: The marks `html_text` starts a heading and a list item with. _LINE_MARK = re.compile("^(## |\u2022 )", re.M) def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str: """`text` (a role's summary, or the opening of its description) with its whitespace collapsed, cut to `limit` characters at a word boundary with an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a `SUBSTR` of the source column rather than the whole of it, so the feed's list never carries what only the reader needs. A description's heading and bullet marks (`plugins.readers._shared.html_text`) are left out.""" collapsed = " ".join(_LINE_MARK.sub("", text or "").split()) if len(collapsed) <= limit: return collapsed cut = collapsed[:limit] space = cut.rfind(" ") trimmed = cut[:space] if space > limit * 0.6 else cut return trimmed.rstrip(" ,.;:" + chr(0x2013) + "-") + "…" class RoleRow(Model): """One row of the feed: what `GET /api/roles` serves per role, the fields a row draws and nothing else — never `description`, `feed_text`, `raw` or `article_text`, the four columns docs/decisions.md measures at about 5.5 KB a role between them. The reader fetches the rest by uid when it opens a role (`GET /api/roles/{uid}`, the full `Role`). `excerpt` is built on the server (see `excerpt`, above): the summary when there is one, else the description's opening, cut to `EXCERPT_LIMIT` characters. The pay fields are what the row's badge reads: the three normalised columns, the line the board stated, and `salary_state`, derived as on `Role`. `original_url` and `posting_url` are short strings, kept so the `o` key opens a selected row's posting without opening the reader. `seniority` is the title's level until extract reads the posting, '' when neither says. `copies` counts the open copies the feed hides behind this row.""" uid: str company_key: str = "" company: str = "" title: str = "" place: str = "" country: str = "" remote: Remote = "" function: str = "" seniority: str = "" salary_min: int | None = None salary_max: int | None = None currency: str = "" salary_text: str = "" original_url: str = "" posting_url: str = "" posted_at: str = "" score: float | None = None gated_reason: str = "" prompt: str = "" vote: Literal[-1, 0, 1] | None = None voted_at: str | None = None read_at: str | None = None tags: list[str] = [] excerpt: str = "" #: How many open roles are copies of this one (`Role.copies`), 0 when none. copies: int = 0 #: The distinct place texts, the nearest to the agent first; `place` is #: the first of them and "+N" for the others. `lead_place` and #: `lead_url` as on `Role`. place_texts: list[str] = [] lead_place: str = "" lead_url: str = "" #: As `Role.remote_where`: the area a remote hire may live in, '' when #: the role is not remote or does not say. remote_where: str = "" #: As `Role.deadline`: the last day to apply, None when nothing says. deadline: str | None = None #: As `ScoredRole`: the posting's languages against the agent's, and #: the total of the agent's `judge` reading, None until judged. lang_flag: LangFlag = "" lang_note: str = "" cv_score: float | None = None #: As `ScoredRole.cv_fit`: how close the role is to the agent's CV, 0..1. cv_fit: float | None = None @computed_field # type: ignore[prop-decorator] @property def salary_state(self) -> str: """As `Role.salary_state`: 'normalised', 'stated' or ''.""" if self.salary_min is not None or self.salary_max is not None: return "normalised" return "stated" if self.salary_text else "" class Company(Model): """A row of `companies`, with `affinity` as the agent it was read for has it (see `agent_companies`); 0 when no agent was asked for.""" key: str name: str domain: str = "" affinity: int = 0 status: str = "researching" careers_url: str = "" notes: str = "" #: The host the careers site runs on, readable or not; "" when unknown. #: A company with an `ats` and no board subscription is one the app #: cannot read yet. See `app.jobs.discover`. ats: str = "" country: str = "" city: str = "" #: The name `ats` knows the company by, the `acme` of acme.teamtailor.com; #: "" where discovery found none. Kept for a host no parser reads yet. ats_slug: str = "" #: The Watchlist's rating, global and not an agent's: None not yet rated, #: 1 worth checking by hand, 0 not. watch: int | None = None #: When the careers page was last opened from the Watchlist; "" never. careers_opened_at: str = "" #: Why the company is read by hand, one of `HAND_REASONS`; "" when it is #: not on that list. hand_reason: str = "" #: The two fetch overrides (`app/jobs/reach.py`), global rather than an #: agent's, since fetching is shared. `all_countries` 1: every big board #: of the company is read with the agents' wide reach, Europe today, #: rather than their places. `all_roles` 1: no role-type narrowing, no #: fetch terms today, on any of its boards. all_countries: int = 0 all_roles: int = 0 created_at: str updated_at: str class Subscription(Model): """A row of `subscriptions`. `company` and `board` are the two labels newsdesk calls outlet and section.""" id: int source: str target: str label: str company: str = "" board: str = "" params: JsonDict = {} subscribed: int = 1 fetch: int = 1 show: int = 1 added_at: str target_id: str = "" class Application(Model): """A row of `applications`. `id` empty means not saved yet; the store mints one. `role_uid` is None for a role that was never posted.""" id: str = "" role_uid: str | None = None company_key: str = "" role_title: str = "" url: str = "" status: Status = "saved" source: str = "" applied_at: str | None = None next_action: str = "" next_date: str | None = None contact: str = "" notes: str = "" created_at: str = "" updated_at: str = "" # ------------------------------------------------------------------ agents #: The role columns a gate can name. `company` matches on the company key; #: `place` takes the Place filter's tokens (`app.places`: `country:DK`, #: `region:skane`, `place:malmo`, bare text a `place:`) on whole words; `title` #: matches a value as whole words anywhere in the title, folded as `title_key` #: folds it, so `account executive` finds "Enterprise Account Executive, UK" #: and `data engineer` does not find "Data Engineering Manager"; `remote_from` #: takes place tokens (`country:DK`, `country:EU`, `anywhere`) and is met by a #: remote role whose area overlaps one, as a `remote:` chip is; the rest are #: case-insensitive equality, except `salary_min` (at least the value) and #: `languages` (every language the role requires is one of the values). GateField = Literal["place", "country", "remote", "remote_from", "seniority", "function", "salary_min", "currency", "permit", "languages", "company", "title"] GATE_FIELDS: tuple[str, ...] = get_args(GateField) GateMode = Literal["require", "any", "exclude"] class Gate(Model): """One condition of an agent's fetch gate. `require` gates a role that fails it (the row is kept with the reason); the `any` gates together are one group a role must match at least one of ("in Denmark, or remote"); `exclude` gates a role that matches any of its values ("title account executive"). A company set to All countries skips the `any` group and one set to All roles skips the `exclude` gates on `title` (`score.failed_gates`). A gate rejects on evidence only: a role whose column is empty is not rejected, since `extract` fills the column later for the roles in the cut, and only for those.""" field: GateField values: list[str | int | float] = Field(min_length=1) mode: GateMode = "require" class Weights(Model): """How much each term of the final score counts. The score is the weighted sum over the sum of the weights, so it stays in 0..1 whatever the numbers.""" fit: float = Field(0.6, ge=0) recency: float = Field(0.2, ge=0) affinity: float = Field(0.2, ge=0) #: Cosine to the liked-minus-passed direction of the agent's votes. taste: float = Field(0.2, ge=0) #: How much each of the judge's three numbers counts in `cv_score`. Not #: a score term: they are read when a role is served, so changing them #: asks the model for nothing. 0 leaves a number out, all three 0 means #: no `cv_score`. cv_skills: float = Field(30, ge=0, le=1000, allow_inf_nan=False) cv_experience: float = Field(25, ge=0, le=1000, allow_inf_nan=False) cv_career: float = Field(30, ge=0, le=1000, allow_inf_nan=False) class Limits(Model): """The agent's cut and the depth of summarise. The cut is the roles that get fetch text, extract and judge (`app.jobs.score`): the best `cut_fit_top` by fit level at or above `cut_fit_min`, then the best `cut_cv_top` by CV level at or above `cut_cv_min` that the first list leaves out, then every role liked or saved. Both levels are calibrated (`score.scale`) with the company's affinity as a small bonus, so a minimum means the same in every run. For a broad agent the count stops a list, for a narrow one the minimum. Summarise reads the best `summarise_top` by final score, and every role liked or saved.""" cut_fit_top: int = Field(500, ge=0) cut_fit_min: float = Field(0.6, ge=0, le=1) cut_cv_top: int = Field(150, ge=0) cut_cv_min: float = Field(0.6, ge=0, le=1) summarise_top: int = Field(50, ge=0) class Prompt(Model): """One of an agent's descriptions of what it looks for. Each is embedded on its own and a role's fit is its best prompt's (see `app.jobs.score`), so one agent can hunt several kinds of role without blending them. A prompt with no text has no fit to give.""" name: str text: str = "" @field_validator("name") @classmethod def _named(cls, name: str) -> str: name = name.strip() if not name: raise ValueError("a prompt needs a name") return name def check_prompts(prompts: list[Prompt]) -> list[Prompt]: """The prompts as they are, or ValueError when two share a name regardless of case.""" seen: set[str] = set() for prompt in prompts: if prompt.name.casefold() in seen: raise ValueError(f"two prompts are named {prompt.name!r}") seen.add(prompt.name.casefold()) return prompts #: An agent's prompts, in order, their names unique regardless of case. Prompts = Annotated[list[Prompt], AfterValidator(check_prompts)] def fetch_terms(value: Any) -> list[str]: """Fetch terms as stored and sent: a JSON list or a list, each stripped, empty ones dropped, one of each regardless of case, in the order given.""" items = _json(value, "[]") seen: set[str] = set() out: list[str] = [] for item in items if isinstance(items, list) else []: term = " ".join(str(item).split()) if term and term.casefold() not in seen: seen.add(term.casefold()) out.append(term) return out #: An agent's fetch terms, normalised by `fetch_terms`. FetchTerms = Annotated[list[str], BeforeValidator(fetch_terms)] class Agent(Model): """A row of `agents`: a named search with its own prompts, gates, weights and limits. The prompts are rows of `agent_prompts`, read alongside; their cached vectors are columns this does not declare. `fetch_terms` are words a role must match for a big board read with the wide reach to send it (`app/jobs/reach.py`): the pull's terms are the union over the enabled agents, and an agent with none means no board is narrowed by words. They decide what is fetched, not what ranks. `cv_text`, `cv_languages`, `like_text`, `avoid_text` and `must_text` are what `judge` reads a role against: the CV, the languages it states with a level, what the person likes and does not like (the work, the workplace, situations; both feed the career score), and their must-haves and deal-breakers (set through the API only, not on the form; soft, read by the model only; a hard rule is a gate). An agent without a CV is not judged, and one without languages flags none (`language_flag`).""" id: int name: str prompts: Prompts = [] gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = [] weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights() limits: Annotated[Limits, BeforeValidator(lambda v: _json(v, "{}"))] = Limits() fetch_terms: FetchTerms = [] cv_text: str = "" cv_languages: CvLanguages = [] like_text: str = "" avoid_text: str = "" must_text: str = "" enabled: bool = True created_at: str class SavedFilter(Model): """A row of `saved_filters`: the feed's filter settings saved under a name for one agent. `query` is the feed's query string as `roles.js` writes it, without the agent. There is no editing: a change is a new one.""" id: int name: str query: str created_at: str class Candidate(Model): """What scoring reads about one open role for one agent: its posted date (for `recency`), its company's affinity for that agent and the company's two overrides of the fetch gate, and per gate `1` match, `0` no match, `-1` the column is empty. An `exclude` gate's match is the position of the first value it matched, from 1, so the reason can name it.""" uid: str posted_at: str affinity: int = 0 #: `companies.all_countries`: the `any` group (the place part) is skipped. all_countries: bool = False #: `companies.all_roles`: the `exclude` gates on `title` are skipped. all_roles: bool = False gates: list[int] = [] class ApplicationEvent(Model): id: int application_id: str status: Status at: str