roledesk job extract reads open, unanswered roles 20 to a call and fills place, country, remote, seniority, salary, currency, function, permit and languages, only into columns the board left empty. Schema v2 adds roles.permit, roles.languages and roles.extract_tries. Off by default. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
210 lines
6.1 KiB
Python
210 lines
6.1 KiB
Python
"""The shapes roledesk hands round, and the keys that decide two rows are one role.
|
|
|
|
One set of models, as in newsdesk: the row the table returns and the object
|
|
the API serves are the same declaration. `Model.from_row` ignores columns a
|
|
model does not declare, so a new column is never a 500 somewhere else.
|
|
|
|
The three keys cluster roles on `(company_key, title_key, place_key)`. The same
|
|
title at two companies is two jobs, and the same ad on two boards is one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
from typing import Annotated, Any, Literal, get_args
|
|
|
|
from pydantic import BeforeValidator
|
|
|
|
from desk.models import JsonDict, Model, ModelTaskResult
|
|
|
|
# ------------------------------------------------------------------- keys
|
|
|
|
#: Gender markers German and Austrian ads append to a title: (m/w/d), (f/m/x),
|
|
#: (w/m/div), (all genders), (gn). Matched with or without the parentheses.
|
|
_GENDER = re.compile(
|
|
r"\(\s*(?:all\s+genders?|gn\*?|[mwfdx]|div(?:ers)?)"
|
|
r"(?:\s*[/|,]\s*(?:[mwfdx]|div(?:ers)?))*\s*\)"
|
|
r"|\b[mwfd]\s*/\s*[mwfd]\s*/\s*(?:[mwfdx]|div(?:ers)?)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
#: Legal forms cut from the end of a company name, so "Celonis SE" on one
|
|
#: board and "celonis" on another share a key.
|
|
_LEGAL = re.compile(
|
|
r"(?:\s+(?:gmbh(?:\s*&\s*co\.?\s*kg)?|ag|se|kg|ug|inc|ltd|llc|plc|corp|"
|
|
r"co|bv|b\.v|sa|sas|srl|oy|ab|as|aps))+\.?$",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _fold(text: str) -> str:
|
|
"""Casefolded, accents stripped, punctuation to spaces, whitespace collapsed."""
|
|
text = unicodedata.normalize("NFKD", text.casefold())
|
|
text = "".join(c for c in text if not unicodedata.combining(c))
|
|
text = re.sub(r"[\W_]+", " ", text)
|
|
return " ".join(text.split())
|
|
|
|
|
|
def title_key(title: str) -> str:
|
|
"""`Senior Backend Engineer (m/w/d)` -> `senior backend engineer`."""
|
|
return _fold(_GENDER.sub(" ", title or ""))
|
|
|
|
|
|
def place_key(place: str) -> str:
|
|
"""`Berlin, Germany` -> `berlin germany`; `Zürich` -> `zurich`."""
|
|
return _fold(place or "")
|
|
|
|
|
|
def company_key(company: str) -> str:
|
|
"""A slug: `Trade Republic Bank GmbH` -> `trade-republic-bank`. Also the
|
|
primary key of `companies`."""
|
|
name = _LEGAL.sub("", (company or "").strip())
|
|
return _fold(name).replace(" ", "-")
|
|
|
|
|
|
# --------------------------------------------------------------- vocabulary
|
|
|
|
Remote = Literal["", "remote", "hybrid", "onsite"]
|
|
REMOTE: tuple[str, ...] = get_args(Remote)
|
|
|
|
Seniority = Literal["", "intern", "junior", "mid", "senior", "staff", "lead", "manager",
|
|
"director", "exec"]
|
|
SENIORITIES: tuple[str, ...] = get_args(Seniority)
|
|
|
|
#: Whether a posting says it sponsors a work permit. '' is "it does not say".
|
|
Permit = Literal["", "sponsors", "no_sponsorship"]
|
|
PERMITS: tuple[str, ...] = get_args(Permit)
|
|
|
|
Status = Literal["saved", "applied", "screen", "technical", "onsite", "offer",
|
|
"rejected", "withdrawn"]
|
|
STATUSES: tuple[str, ...] = get_args(Status)
|
|
|
|
#: -1 mute, 0 neutral, 1 interested, 2 favourite (skips gates, floors the score).
|
|
AFFINITY = range(-1, 3)
|
|
|
|
|
|
def _json(value: Any, empty: str) -> Any:
|
|
"""A JSON text column as the Python value, tolerant like `JsonDict`: a
|
|
value that is already parsed passes, and one that does not parse is the
|
|
empty value rather than a 500."""
|
|
if not isinstance(value, str):
|
|
return value
|
|
try:
|
|
return json.loads(value or empty)
|
|
except ValueError:
|
|
return json.loads(empty)
|
|
|
|
|
|
#: A column holding a JSON array of strings, such as `roles.languages`.
|
|
JsonStrings = Annotated[list[str], BeforeValidator(lambda v: _json(v, "[]"))]
|
|
|
|
|
|
# ------------------------------------------------------------------ models
|
|
|
|
class Role(Model):
|
|
"""A row of `roles`: one posting on one board."""
|
|
|
|
uid: str
|
|
source: str
|
|
source_id: str
|
|
subscription_id: int | None = None
|
|
company_key: str = ""
|
|
company: str = ""
|
|
title: str = ""
|
|
title_key: str = ""
|
|
place: str = ""
|
|
place_key: str = ""
|
|
country: str = ""
|
|
remote: Remote = ""
|
|
salary_min: int | None = None
|
|
salary_max: int | None = None
|
|
currency: str = ""
|
|
seniority: str = ""
|
|
function: str = ""
|
|
permit: Permit = ""
|
|
#: ISO 639-1 codes the posting says are required, lowercase.
|
|
languages: JsonStrings = []
|
|
description: str = ""
|
|
feed_text: str = ""
|
|
original_url: str = ""
|
|
published_at: str
|
|
first_seen: str
|
|
last_seen: str
|
|
closed_at: str | None = None
|
|
raw: JsonDict = {}
|
|
extracted_at: str | None = None
|
|
created_at: str
|
|
|
|
|
|
class ExtractResult(ModelTaskResult):
|
|
"""What one run of `extract` did. See roledesk/jobs/extract.py."""
|
|
|
|
#: Roles the model answered for, whether or not it found anything to fill.
|
|
extracted: int = 0
|
|
#: Columns written, across those roles. A role answered with nothing the
|
|
#: posting states adds to `extracted` and not to this.
|
|
filled: int = 0
|
|
|
|
@property
|
|
def done(self) -> int:
|
|
return self.extracted
|
|
|
|
|
|
class Company(Model):
|
|
key: str
|
|
name: str
|
|
domain: str = ""
|
|
affinity: int = 0
|
|
status: str = "researching"
|
|
careers_url: str = ""
|
|
notes: str = ""
|
|
created_at: str
|
|
updated_at: str
|
|
|
|
|
|
class Subscription(Model):
|
|
"""A row of `subscriptions`. `company` and `board` are the two labels
|
|
newsdesk calls outlet and section."""
|
|
|
|
id: int
|
|
source: str
|
|
target: str
|
|
label: str
|
|
company: str = ""
|
|
board: str = ""
|
|
params: JsonDict = {}
|
|
subscribed: int = 1
|
|
fetch: int = 1
|
|
show: int = 1
|
|
added_at: str
|
|
target_id: str = ""
|
|
|
|
|
|
class Application(Model):
|
|
"""A row of `applications`. `id` empty means not saved yet; the store mints
|
|
one. `role_uid` is None for a role that was never posted."""
|
|
|
|
id: str = ""
|
|
role_uid: str | None = None
|
|
company_key: str = ""
|
|
role_title: str = ""
|
|
url: str = ""
|
|
status: Status = "saved"
|
|
source: str = ""
|
|
applied_at: str | None = None
|
|
next_action: str = ""
|
|
next_date: str | None = None
|
|
contact: str = ""
|
|
notes: str = ""
|
|
created_at: str = ""
|
|
updated_at: str = ""
|
|
|
|
|
|
class ApplicationEvent(Model):
|
|
id: int
|
|
application_id: str
|
|
status: Status
|
|
at: str
|