"""The shapes roledesk hands round, and the keys that decide two rows are one role. One set of models, as in newsdesk: the row the table returns and the object the API serves are the same declaration. `Model.from_row` ignores columns a model does not declare, so a new column is never a 500 somewhere else. The three keys cluster roles on `(company_key, title_key, place_key)`. The same title at two companies is two jobs, and the same ad on two boards is one. """ from __future__ import annotations import json import re import unicodedata from typing import Annotated, Any, Literal, get_args from pydantic import BeforeValidator, Field from desk.models import JsonDict, Model, ModelTaskResult # ------------------------------------------------------------------- keys #: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x), #: (w/m/div), (all genders), (gn). Matched with or without the parentheses. _GENDER = re.compile( r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)" r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)" r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b", re.IGNORECASE, ) #: Legal forms cut from the end of a company name, so "Celonis SE" on one #: board and "celonis" on another share a key. _LEGAL = re.compile( r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|" r"co|bv|b\.v|sa|sas|srl|oy|ab|as|a/s|aps))+\.?$", re.IGNORECASE, ) def _fold(text: str) -> str: """Casefolded, accents stripped, punctuation to spaces, whitespace collapsed.""" text = unicodedata.normalize("NFKD", text.casefold()) text = "".join(c for c in text if not unicodedata.combining(c)) text = re.sub(r"[\W_]+", " ", text) return " ".join(text.split()) def title_key(title: str) -> str: """`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`.""" return _fold(_GENDER.sub(" ", title or "")) def place_key(place: str) -> str: """`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`.""" return _fold(place or "") def company_key(company: str) -> str: """A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the primary key of `companies`.""" name = _LEGAL.sub("", (company or "").strip()) return _fold(name).replace(" ", "-") # ---------------------------------------------------------------- geography #: The EU, the EEA, the UK and Switzerland: what "a European role" means to a #: gate on `country`. `EU` is ISO's reserved code, used here for a role open #: anywhere in Europe ("Remote - Europe", "EMEA") that names no one country. EUROPE: tuple[str, ...] = ( "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS", "LI", "NO", "GB", "CH", "EU", ) #: Folded names a board writes in its location line, to an ISO code: every #: European country in English and its own language, the cities the boards #: name most, and the non-European places that fill the seed boards, so those #: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest. _PLACES: dict[str, str] = { **dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"), **dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"), **dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp", "ghent"), "BE"), **dict.fromkeys(("bulgaria", "sofia"), "BG"), **dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"), **dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"), **dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"), # Folding keeps ø and æ, which are letters of their own rather than an o # or an a with a mark, so København is listed beside Kobenhavn. **dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus", "århus", "odense", "aalborg", "region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup", "bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv", "birkerød"), "DK"), **dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"), **dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"), **dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux", "lille"), "FR"), **dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"), **dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"), **dict.fromkeys(("hungary", "budapest"), "HU"), **dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"), **dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin", "torino", "bologna", "florence"), "IT"), **dict.fromkeys(("latvia", "riga"), "LV"), **dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"), **dict.fromkeys(("luxembourg",), "LU"), **dict.fromkeys(("malta", "valletta"), "MT"), **dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven", "delft"), "NL"), **dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"), "PL"), **dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"), **dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"), **dict.fromkeys(("slovakia", "bratislava"), "SK"), **dict.fromkeys(("slovenia", "ljubljana"), "SI"), **dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville", "malaga", "bilbao"), "ES"), **dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"), **dict.fromkeys(("iceland", "reykjavik"), "IS"), **dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"), **dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland", "wales", "northern ireland", "london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk", "oxford", "leeds", "birmingham uk", "belfast"), "GB"), **dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug"), "CH"), **dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city", "nyc", "san francisco", "seattle", "boston", "chicago", "austin", "los angeles", "palo alto", "mountain view", "san mateo", "foster city", "sunnyvale", "san jose", "menlo park", "redwood city", "costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas", "houston", "phoenix", "tempe", "portland", "salt lake city", "pittsburgh", "philadelphia", "detroit", "san diego", "raleigh", "nashville", "minneapolis"), "US"), **dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"), "CA"), **dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"), **dict.fromkeys(("singapore",), "SG"), **dict.fromkeys(("japan", "tokyo", "osaka"), "JP"), **dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"), **dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"), **dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"), **dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"), **dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"), **dict.fromkeys(("south korea", "korea", "seoul"), "KR"), } #: Keyed the way a place is folded before it is looked up, so a name can be #: written as it is spelt. _PLACES = {_fold(name): code for name, code in _PLACES.items()} #: The longest name first, so "new york city" wins over "york" and "northern #: ireland" over "ireland". _PLACE_NAMES = sorted(_PLACES, key=len, reverse=True) #: US state codes after a comma, the way US boards write a city: "Boston, MA". #: Read only when no name matched, since "DE" and "IN" are countries too. _US_STATE = re.compile( r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|" r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)" r"\b") def country_of(place: str) -> str: """The one country a board's location line names, as an ISO code, or '' when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe" is EU, and "London, EMEA" is GB: a country beats the region it is in.""" folded = f" {_fold(place or '')} " found: set[str] = set() for name in _PLACE_NAMES: needle = f" {name} " if needle in folded: found.add(_PLACES[name]) folded = folded.replace(needle, " ") if not found and _US_STATE.search(place or ""): # Only when nothing else named a country: "Berlin, DE" is Germany. found.add("US") if len(found) > 1: found.discard("EU") return found.pop() if len(found) == 1 else "" # --------------------------------------------------------------- vocabulary Remote = Literal["", "remote", "hybrid", "onsite"] REMOTE: tuple[str, ...] = get_args(Remote) Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager", "director", "exec"] SENIORITIES: tuple[str, ...] = get_args(Seniority) #: Whether a posting says it sponsors a work permit. '' is "it does not say". Permit = Literal["", "sponsors", "no_sponsorship"] PERMITS: tuple[str, ...] = get_args(Permit) Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer", "rejected", "withdrawn"] STATUSES: tuple[str, ...] = get_args(Status) #: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score). AFFINITY = range(-1, 3) def _json(value: Any, empty: str) -> Any: """A JSON text column as the Python value, tolerant like `JsonDict`: a value that is already parsed passes, and one that does not parse is the empty value rather than a 500.""" if not isinstance(value, str): return value try: return json.loads(value or empty) except ValueError: return json.loads(empty) #: A column holding a JSON array of strings, such as `roles.languages`. JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))] # ------------------------------------------------------------------ models class Role(Model): """A row of `roles`: one posting on one board.""" uid: str source: str source_id: str subscription_id: int | None = None company_key: str = "" company: str = "" title: str = "" title_key: str = "" place: str = "" place_key: str = "" country: str = "" remote: Remote = "" salary_min: int | None = None salary_max: int | None = None currency: str = "" seniority: str = "" function: str = "" permit: Permit = "" #: ISO 639-1 codes the posting says are required, lowercase. languages: JsonStrings = [] description: str = "" feed_text: str = "" original_url: str = "" published_at: str first_seen: str last_seen: str closed_at: str | None = None raw: JsonDict = {} extracted_at: str | None = None created_at: str #: None unjudged, else -1 pass, 0 neutral, 1 like. vote: Literal[-1, 0, 1] | None = None voted_at: str | None = None #: NULL unread, else when the feed opened it or judged it. A vote stamps #: this too, see `Store.set_vote`: voting is reading. read_at: str | None = None summary: str = "" summarised_at: str | None = None #: The posting page as plain text, for a role whose description is a teaser. article_text: str = "" article_at: str | None = None #: The open vocabulary `summarise` read off the posting, sorted. Not a #: column: loaded from `role_tags` alongside the row, see `Store.role`. tags: list[str] = [] class ExtractResult(ModelTaskResult): """What one run of `extract` did. See app/jobs/extract.py.""" #: Roles the model answered for, whether or not it found anything to fill. extracted: int = 0 #: Columns written, across those roles. A role answered with nothing the #: posting states adds to `extracted` and not to this. filled: int = 0 @property def done(self) -> int: return self.extracted class ScoredRole(Role): """A role as `/api/roles` serves it: with one agent's score and gate reason when an agent was asked for, and without otherwise.""" score: float | None = None gated_reason: str = "" class Company(Model): key: str name: str domain: str = "" affinity: int = 0 status: str = "researching" careers_url: str = "" notes: str = "" created_at: str updated_at: str class Subscription(Model): """A row of `subscriptions`. `company` and `board` are the two labels newsdesk calls outlet and section.""" id: int source: str target: str label: str company: str = "" board: str = "" params: JsonDict = {} subscribed: int = 1 fetch: int = 1 show: int = 1 added_at: str target_id: str = "" class Application(Model): """A row of `applications`. `id` empty means not saved yet; the store mints one. `role_uid` is None for a role that was never posted.""" id: str = "" role_uid: str | None = None company_key: str = "" role_title: str = "" url: str = "" status: Status = "saved" source: str = "" applied_at: str | None = None next_action: str = "" next_date: str | None = None contact: str = "" notes: str = "" created_at: str = "" updated_at: str = "" # ------------------------------------------------------------------ agents #: The role columns a gate can name. `company` matches on the company key; #: `place` is a case-insensitive substring, like the place filter; the rest are #: case-insensitive equality, except `salary_min` (at least the value) and #: `languages` (every language the role requires is one of the values). GateField = Literal["place", "country", "remote", "seniority", "function", "salary_min", "currency", "permit", "languages", "company"] GATE_FIELDS: tuple[str, ...] = get_args(GateField) GateMode = Literal["require", "prefer"] class Gate(Model): """One condition of an agent. `require` gates a role that fails it (the row is kept with the reason); `prefer` adds `weight` to its score when it matches.""" field: GateField values: list[str | int | float] = Field(min_length=1) mode: GateMode = "require" weight: float = Field(1.0, ge=0) class Weights(Model): """How much each score term counts. Summed with the prefer gates' weights into the denominator, so a score stays in 0..1 whatever the numbers.""" fit: float = Field(0.6, ge=0) recency: float = Field(0.2, ge=0) affinity: float = Field(0.2, ge=0) class Agent(Model): """A row of `agents`: a named search with its own intent, gates and weights. The cached intent vector is a column this does not declare.""" id: int name: str intent: str = "" gates: Annotated[list[Gate], BeforeValidator(lambda v: _json(v, "[]"))] = [] weights: Annotated[Weights, BeforeValidator(lambda v: _json(v, "{}"))] = Weights() enabled: bool = True created_at: str class Candidate(Model): """What scoring reads about one open role for one agent: when it was first seen, its company's affinity, and per gate `1` match, `0` no match, `-1` the column is empty.""" uid: str first_seen: str affinity: int = 0 gates: list[int] = [] class ApplicationEvent(Model): id: int application_id: str status: Status at: str