Merge branch 'r2-backend'

# Conflicts:
#	app/storage/schema.py
This commit is contained in:
Wolf committed 2026-09-29 15:22:03 +02:00
commit 562fa3ab09
15 files changed
+924 -157

No files matched your search

+7 -3
View File
@@ -47,9 +47,13 @@ from app.storage import Store
log = logging.getLogger(__name__)
#: As many boards in the air per job as there are `http` workers by default
#: (see `config.yaml`), so one pull keeps the pool busy without one job
#: holding all 190-odd boards' worth of retries against a single attempt.
#: Targets per `pull_batch` job. A batch reads its boards one after another,
#: so the parallelism is the number of batches the `http` pool runs at once
#: (8 workers by default, see `config.yaml`), not this number. On the live
#: database of 2026-09-29, 175 boards and 4 searches make 19 batches, enough
#: to keep eight workers busy, while a batch that fails or is killed costs at
#: most ten targets' progress. A board's request is bounded by
#: `plugins.ats.DEADLINE` plus one read timeout.
BATCH_SIZE = 10
SETTLE = "app.jobs.pull:close_boards"
+522 -105
View File
@@ -66,121 +66,538 @@ def company_key(company: str) -> str:
# ---------------------------------------------------------------- geography
#: Folded names a board writes in its location line, to an ISO code: every
#: European country in English and its own language, the cities the boards
#: name most, and the non-European places that fill the seed boards, so those
#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
_PLACES: dict[str, str] = {
**dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
**dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
**dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
"ghent"), "BE"),
**dict.fromkeys(("bulgaria", "sofia"), "BG"),
**dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
**dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
**dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
# Folding keeps ø and æ, which are letters of their own rather than an o
# or an a with a mark, so København is listed beside Kobenhavn.
**dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
"århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
"lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
"bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
"birkerød"), "DK"),
**dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
**dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
**dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
"bordeaux", "lille"), "FR"),
**dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
"frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
"dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
"heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
**dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
**dict.fromkeys(("hungary", "budapest"), "HU"),
**dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
**dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
"torino", "bologna", "florence"), "IT"),
**dict.fromkeys(("latvia", "riga"), "LV"),
**dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
**dict.fromkeys(("luxembourg",), "LU"),
**dict.fromkeys(("malta", "valletta"), "MT"),
**dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
"rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
#: Letters NFKD leaves whole because they are letters of their own rather
#: than a base letter with a mark, spelt the way boards write them in ASCII.
_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe",
"þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"})
def _place_fold(text: str) -> str:
"""`_fold` for the place lookup only: "Wrocław", "Łódź" and "København"
meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is,
since it also builds `company_key`, the primary key of `companies`."""
return _fold((text or "").casefold().translate(_LETTERS))
#: Every ISO 3166-1 country by its English name, plus the names boards also
#: write for it: its own language's for a European one, and the usual short
#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one.
_COUNTRY_NAMES = """
AD Andorra
AE United Arab Emirates | UAE | U.A.E. | Emirates
AF Afghanistan
AG Antigua and Barbuda
AI Anguilla
AL Albania | Shqipëria | Shqipëri
AM Armenia | Hayastan
AO Angola
AQ Antarctica
AR Argentina
AS American Samoa
AT Austria | Österreich
AU Australia
AW Aruba
AX Åland Islands | Åland
AZ Azerbaijan | Azərbaycan
BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina
BB Barbados
BD Bangladesh
BE Belgium | België | Belgique | Belgien
BF Burkina Faso
BG Bulgaria | България | Balgariya
BH Bahrain
BI Burundi
BJ Benin
BL Saint Barthélemy
BM Bermuda
BN Brunei | Brunei Darussalam
BO Bolivia
BQ Caribbean Netherlands | Bonaire
BR Brazil | Brasil
BS Bahamas
BT Bhutan
BV Bouvet Island
BW Botswana
BY Belarus | Беларусь | Belarus'
BZ Belize
CA Canada
CC Cocos Islands | Cocos (Keeling) Islands
CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa
CF Central African Republic
CG Republic of the Congo | Congo-Brazzaville
CH Switzerland | Schweiz | Suisse | Svizzera | Svizra
CI Côte d'Ivoire | Ivory Coast
CK Cook Islands
CL Chile
CM Cameroon
CN China | People's Republic of China
CO Colombia
CR Costa Rica
CU Cuba
CV Cabo Verde | Cape Verde
CW Curaçao
CX Christmas Island
CY Cyprus | Κύπρος | Kypros | Kibris
CZ Czechia | Czech Republic | Česko | Česká republika
DE Germany | Deutschland
DJ Djibouti
DK Denmark | Danmark
DM Dominica
DO Dominican Republic
DZ Algeria
EC Ecuador
EE Estonia | Eesti
EG Egypt
EH Western Sahara
ER Eritrea
ES Spain | España | Espanya
ET Ethiopia
FI Finland | Suomi
FJ Fiji
FK Falkland Islands
FM Micronesia
FO Faroe Islands | Føroyar | Færøerne
FR France
GA Gabon
GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland
GD Grenada
GE Georgia | Sakartvelo
GF French Guiana
GG Guernsey
GH Ghana
GI Gibraltar
GL Greenland
GM Gambia
GN Guinea
GP Guadeloupe
GQ Equatorial Guinea
GR Greece | Ελλάδα | Hellas | Ellada
GS South Georgia and the South Sandwich Islands
GT Guatemala
GU Guam
GW Guinea-Bissau
GY Guyana
HK Hong Kong
HM Heard Island and McDonald Islands
HN Honduras
HR Croatia | Hrvatska
HT Haiti
HU Hungary | Magyarország
ID Indonesia
IE Ireland | Éire
IL Israel
IM Isle of Man
IN India
IO British Indian Ocean Territory
IQ Iraq
IR Iran
IS Iceland | Ísland
IT Italy | Italia
JE Jersey
JM Jamaica
JO Jordan
JP Japan
KE Kenya
KG Kyrgyzstan
KH Cambodia
KI Kiribati
KM Comoros
KN Saint Kitts and Nevis
KP North Korea
KR South Korea | Korea | Republic of Korea
KW Kuwait
KY Cayman Islands
KZ Kazakhstan
LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic
LB Lebanon
LC Saint Lucia
LI Liechtenstein
LK Sri Lanka
LR Liberia
LS Lesotho
LT Lithuania | Lietuva
LU Luxembourg | Lëtzebuerg | Luxemburg
LV Latvia | Latvija
LY Libya
MA Morocco
MC Monaco
MD Moldova | Republic of Moldova
ME Montenegro | Crna Gora
MF Saint Martin
MG Madagascar
MH Marshall Islands
MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија
ML Mali
MM Myanmar | Burma
MN Mongolia
MO Macao | Macau
MP Northern Mariana Islands
MQ Martinique
MR Mauritania
MS Montserrat
MT Malta
MU Mauritius
MV Maldives
MW Malawi
MX Mexico | México
MY Malaysia
MZ Mozambique
NA Namibia
NC New Caledonia
NE Niger
NF Norfolk Island
NG Nigeria
NI Nicaragua
NL Netherlands | The Netherlands | Nederland | Holland
NO Norway | Norge | Noreg
NP Nepal
NR Nauru
NU Niue
NZ New Zealand | Aotearoa
OM Oman
PA Panama | Panamá
PE Peru | Perú
PF French Polynesia
PG Papua New Guinea
PH Philippines
PK Pakistan
PL Poland | Polska
PM Saint Pierre and Miquelon
PN Pitcairn Islands
PR Puerto Rico
PS Palestine
PT Portugal
PW Palau
PY Paraguay
QA Qatar
RE Réunion
RO Romania | România
RS Serbia | Srbija | Србија
RU Russia | Russian Federation | Россия | Rossiya
RW Rwanda
SA Saudi Arabia | KSA
SB Solomon Islands
SC Seychelles
SD Sudan
SE Sweden | Sverige
SG Singapore
SH Saint Helena
SI Slovenia | Slovenija
SJ Svalbard and Jan Mayen | Svalbard
SK Slovakia | Slovensko
SL Sierra Leone
SM San Marino
SN Senegal
SO Somalia
SR Suriname
SS South Sudan
ST São Tomé and Príncipe
SV El Salvador
SX Sint Maarten
SY Syria
SZ Eswatini | Swaziland
TC Turks and Caicos Islands
TD Chad
TF French Southern Territories
TG Togo
TH Thailand
TJ Tajikistan
TK Tokelau
TL Timor-Leste | East Timor
TM Turkmenistan
TN Tunisia
TO Tonga
TR Turkey | Türkiye
TT Trinidad and Tobago
TV Tuvalu
TW Taiwan
TZ Tanzania
UA Ukraine | Україна | Ukraina
UG Uganda
UM United States Minor Outlying Islands
US United States | United States of America | USA | U.S.A. | US | U.S.
UY Uruguay
UZ Uzbekistan
VA Vatican City | Holy See
VC Saint Vincent and the Grenadines
VE Venezuela
VG British Virgin Islands
VI U.S. Virgin Islands | US Virgin Islands
VN Vietnam | Viet Nam
VU Vanuatu
WF Wallis and Futuna
WS Samoa
XK Kosovo | Kosova
YE Yemen
YT Mayotte
ZA South Africa
ZM Zambia
ZW Zimbabwe
"""
#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK.
_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line)
#: The countries `EU` stands for when a line names several: the UN
#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and
#: the Council of Europe's members in the Caucasus and Turkey, which
#: European employers list beside their other offices.
_EUROPE = frozenset((
"AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
"DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE",
"IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL",
"NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA",
"XK",
))
_EUROPE_OR_EU = _EUROPE | {"EU"}
#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them.
_REGIONS = ("europe", "emea", "eu", "european union", "eea")
#: The cities boards name most, and a few regions below a country. A US
#: state or a Canadian province in the same part of the line beats one of
#: these ("London, Ontario" is Canada); a country name does not lose to it.
_CITIES: dict[str, str] = {
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
**dict.fromkeys(("sofia",), "BG"),
**dict.fromkeys(("zagreb",), "HR"),
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
**dict.fromkeys(("prague", "praha", "brno"), "CZ"),
**dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg",
"region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby",
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
**dict.fromkeys(("tallinn", "tartu"), "EE"),
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
"lille", "rennes", "amiens"), "FR"),
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
"stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg",
"nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund",
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
**dict.fromkeys(("budapest",), "HU"),
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
**dict.fromkeys(("riga",), "LV"),
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
**dict.fromkeys(("valletta",), "MT"),
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
"delft"), "NL"),
**dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
"gdansk", "poznan", "lodz"), "PL"),
**dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
**dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
"RO"),
**dict.fromkeys(("slovakia", "bratislava"), "SK"),
**dict.fromkeys(("slovenia", "ljubljana"), "SI"),
**dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
"malaga", "bilbao"), "ES"),
**dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
"uppsala"), "SE"),
**dict.fromkeys(("iceland", "reykjavik"), "IS"),
**dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
**dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
"wales", "northern ireland", "london", "manchester", "edinburgh",
"glasgow", "bristol", "cambridge uk", "oxford", "leeds",
"birmingham uk", "belfast"), "GB"),
**dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
"basel", "lausanne", "bern", "zug"), "CH"),
**dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
"nyc", "san francisco", "seattle", "boston", "chicago", "austin",
"los angeles", "palo alto", "mountain view", "san mateo",
"foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
"costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
"houston", "phoenix", "tempe", "portland", "salt lake city",
"pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
"nashville", "minneapolis"), "US"),
**dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
"waterloo"), "CA"),
**dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
"chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
"PL"),
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
**dict.fromkeys(("belgrade", "beograd"), "RS"),
**dict.fromkeys(("bratislava",), "SK"),
**dict.fromkeys(("ljubljana",), "SI"),
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
"granada", "santiago de compostela"), "ES"),
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
**dict.fromkeys(("reykjavik",), "IS"),
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
"oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff",
"guildford", "truro", "swansea", "blackpool", "coventry", "brighton",
"weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley",
"slough", "southampton", "swindon", "liverpool", "redhill",
"milton keynes"), "GB"),
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
"sankt gallen", "st gallen"), "CH"),
**dict.fromkeys(("kyiv", "kiev"), "UA"),
**dict.fromkeys(("istanbul", "ankara"), "TR"),
**dict.fromkeys(("tirana",), "AL"),
**dict.fromkeys(("yerevan",), "AM"),
**dict.fromkeys(("tbilisi",), "GE"),
**dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle",
"boston", "chicago", "austin", "los angeles", "palo alto",
"mountain view", "san mateo", "foster city", "sunnyvale", "san jose",
"menlo park", "redwood city", "costa mesa", "denver", "atlanta",
"washington dc", "washington d c", "miami", "dallas", "houston",
"phoenix", "tempe", "portland", "salt lake city", "pittsburgh",
"philadelphia", "detroit", "san diego", "raleigh", "nashville",
"minneapolis", "cambridge ma", "bay area", "arlington"), "US"),
**dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"),
"CA"),
**dict.fromkeys(("buenos aires",), "AR"),
**dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"),
**dict.fromkeys(("santiago", "santiago de chile"), "CL"),
**dict.fromkeys(("bogota",), "CO"),
**dict.fromkeys(("lima", "miraflores"), "PE"),
**dict.fromkeys(("montevideo",), "UY"),
**dict.fromkeys(("managua",), "NI"),
**dict.fromkeys(("santo domingo",), "DO"),
**dict.fromkeys(("san salvador",), "SV"),
**dict.fromkeys(("panama city",), "PA"),
**dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"),
**dict.fromkeys(("dubai", "abu dhabi"), "AE"),
**dict.fromkeys(("riyadh",), "SA"),
**dict.fromkeys(("doha",), "QA"),
**dict.fromkeys(("manama",), "BH"),
**dict.fromkeys(("salmiya",), "KW"),
**dict.fromkeys(("muscat",), "OM"),
**dict.fromkeys(("cairo", "maadi"), "EG"),
**dict.fromkeys(("casablanca",), "MA"),
**dict.fromkeys(("dakar",), "SN"),
**dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
**dict.fromkeys(("almaty",), "KZ"),
**dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"),
**dict.fromkeys(("dhaka",), "BD"),
**dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai",
"gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
**dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"),
**dict.fromkeys(("singapore",), "SG"),
**dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
**dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
**dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
**dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
**dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
**dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
**dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
**dict.fromkeys(("manila", "taguig"), "PH"),
**dict.fromkeys(("bangkok",), "TH"),
**dict.fromkeys(("phnom penh",), "KH"),
**dict.fromkeys(("taipei",), "TW"),
**dict.fromkeys(("tokyo", "osaka"), "JP"),
**dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"),
**dict.fromkeys(("seoul",), "KR"),
**dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales",
"queensland"), "AU"),
}
#: Keyed the way a place is folded before it is looked up, so a name can be
#: written as it is spelt.
_PLACES = {_fold(name): code for name, code in _PLACES.items()}
#: US states and Canadian provinces, by name and by the code boards write
#: after a comma ("Boston, MA", "Toronto, ON").
_SUBDIVISIONS = """
US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO
US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA
US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY
US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN
US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH
US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND
US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI
US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT
US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY
CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB
CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL
CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT
CA Nunavut NU
"""
#: The longest name first, so "new york city" wins over "york" and "northern
#: ireland" over "ireland".
_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
#: US state codes after a comma, the way US boards write a city: "Boston, MA".
#: Read only when no name matched, since "DE" and "IN" are countries too.
_US_STATE = re.compile(
r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
r"\b")
def _subdivisions() -> tuple[dict[str, str], dict[str, str]]:
names: dict[str, str] = {}
codes: dict[str, str] = {}
for line in _SUBDIVISIONS.split("\n"):
if line:
country, rest = line.split(" ", 1)
for entry in rest.split("|"):
name, code = entry.strip().rsplit(" ", 1)
names[_place_fold(name)] = country
codes[code] = country
return names, codes
_STATE_NAMES, _STATE_CODES = _subdivisions()
def _names() -> dict[str, tuple[str, str]]:
"""Every folded name to `(code, kind)`; kind is `country`, `city` or
`state`. A name that is a city and a state ("New York") is the city, so
it does not beat another city in its line. A name that is a country and
a state ("Georgia") is the country here and read as the state only when
the line names a place in the US too."""
names: dict[str, tuple[str, str]] = {}
for name, code in _STATE_NAMES.items():
names[name] = (code, "state")
for name, code in _CITIES.items():
names[_place_fold(name)] = (code, "city")
for line in _COUNTRY_NAMES.split("\n"):
if line:
code, rest = line.split(" ", 1)
for name in rest.split("|"):
names[_place_fold(name)] = (code, "country")
for name in _REGIONS:
names[name] = ("EU", "country")
return names
_NAMES = _names()
#: A state name that is also a country's, read as the state only when the
#: line names a place in that state's country too.
_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES
if _NAMES[name][1] == "country"}
#: The longest name in words, so a lookup tries "new york city" before "new
#: york" and "northern ireland" before "ireland".
_LONGEST = max(len(name.split()) for name in _NAMES)
#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin".
_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b")
#: A country code in parentheses, "Leicester (GB)", read only when no name matched.
_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)")
#: Where one location in a line ends and the next begins. A state beats a
#: city only within one: "London; Ontario" is two places.
_SEGMENT = re.compile(r"[;|/\n]")
def _hits(folded: str) -> list[tuple[str, str, str]]:
"""The names in a folded line, longest first at each word, as
`(name, code, kind)`."""
words = folded.split()
hits = []
i = 0
while i < len(words):
for n in range(min(_LONGEST, len(words) - i), 0, -1):
name = " ".join(words[i:i + n])
if name in _NAMES:
hits.append((name, *_NAMES[name]))
i += n
break
else:
i += 1
return hits
def country_of(place: str) -> str:
"""The one country a board's location line names, as an ISO code, or ''
when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
is EU, and "London, EMEA" is GB: a country beats the region it is in."""
folded = f" {_fold(place or '')} "
"""The one country a board's location line names, as an ISO code; `EU`
when it names several and all of them are in Europe; '' when it names
none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston,
MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or
New York" and "Remote, EU or US" are ''.
Within one location (the line split at `;`, `|` and `/`) the widest unit
named wins: a country beats a state or a province, which beats a city.
So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A
country also beats the region it is in: "London, EMEA" is GB.
A state code that is also a country's ("DE", "CO", "MA"), and "Georgia",
count as the state only when the line names a place in the US too:
"Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather
than Delaware. A country code in parentheses, "Leicester (GB)", is read
only when no name matched."""
segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")]
named = {code for _, hits in segments for _, code, _ in hits}
found: set[str] = set()
for name in _PLACE_NAMES:
needle = f" {name} "
if needle in folded:
found.add(_PLACES[name])
folded = folded.replace(needle, " ")
if not found and _US_STATE.search(place or ""):
# Only when nothing else named a country: "Berlin, DE" is Germany.
found.add("US")
if len(found) > 1:
found.discard("EU")
return found.pop() if len(found) == 1 else ""
for text, hits in segments:
shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named}
countries = {code for name, code, kind in hits
if kind == "country" and code != "EU" and name not in shared}
states = {code for _, code, kind in hits if kind == "state"}
states |= {_SHARED_NAMES[name] for name in shared}
states |= {country for code in _STATE_CODE.findall(text)
if (country := _STATE_CODES.get(code))
and (code not in _ISO or country in named)}
found |= (countries or states or {code for _, code, _ in hits if code != "EU"})
found |= {code for _, code, _ in hits if code == "EU"}
if not found:
found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO}
if len(found) == 1:
return found.pop()
if found and found <= _EUROPE_OR_EU:
countries = found - {"EU"}
return countries.pop() if "EU" in found and len(countries) == 1 else "EU"
return ""
# --------------------------------------------------------------- vocabulary
+5 -3
View File
@@ -289,14 +289,16 @@
render();
}
// Start the `score` job and wait for it, so "scored" means the Roles tab
// already orders by what was saved. `said` is what happened first.
// Ask for a `score` and wait for the job the server answers with, so
// "scored" means the Roles tab already orders by what was saved. While a
// score is running that is a follow-up queued after it, since the running
// one read the agent before the save. `said` is what happened first.
async function rescore(status, agent, said) {
const runs = () => link("/settings/sources#runs", "follow it under Runs");
status.replaceChildren(`${said}. Scoring every open role; `, runs(), ".");
let job;
try {
job = await api("/api/jobs", { method: "POST", body: { kind: "score", payload: {} } });
job = await api("/api/score", { method: "POST" });
} catch (err) {
status.textContent = `${said}, but scoring did not start.`;
return;
+6 -3
View File
@@ -71,9 +71,12 @@
: "own site";
sub.appendChild(badge);
sub.append(" · not readable yet");
} else if (company.from_search) {
// Nobody followed it: its roles came from a search, and it was judged.
sub.append("from job boards");
} else {
// Every row comes from the seed, Follow or discover: none of them
// found where this company posts its roles.
// Followed by the seed, Follow or discover, and none of them found
// where this company posts its roles.
sub.append("board not found");
}
if (company.careers_url && !company.board) {
@@ -214,7 +217,7 @@
case "hidden": return company.affinity === -1;
case "on": return company.on === true;
case "off": return company.on === false;
case "unreadable": return company.subscription_id == null;
case "unreadable": return company.subscription_id == null && !company.from_search;
default: return true;
}
}
+6 -4
View File
@@ -834,12 +834,14 @@
}
/* The company's ★ ▲ ⊘, the one place a company is judged outside the
Companies page. A company seen only through a search has no row of its
own to judge. A change dims or undims its rows in the feed. */
Companies page. A company seen only through a search has no row until
its first change creates one, so the list is reloaded then. A change
dims or undims its rows in the feed. */
function renderAffinity(slot, r) {
const company = companies.get(r.company_key);
if (!company) { slot.replaceChildren(); return; }
slot.replaceChildren(affinityControl(r.company_key, company.affinity, () => {
if (!r.company_key) { slot.replaceChildren(); return; }
slot.replaceChildren(affinityControl(r.company_key, company ? company.affinity : 0, async () => {
if (!company) await loadCompanies(state.filter.agent).catch(() => {});
renderAffinity(slot, r);
renderFeed();
}, state.filter.agent));
+32 -1
View File
@@ -17,7 +17,7 @@ import logging
import sqlite3
from collections.abc import Callable
from app.models import posted_at, posted_host
from app.models import country_of, posted_at, posted_host
log = logging.getLogger(__name__)
@@ -386,6 +386,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
"could not be normalised into salary_min/salary_max/currency", """
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
"""),
(13, "roles.country recomputed from place for every not-yet-extracted role whose "
"board gave none, country lookup fix", lambda conn: _recompute_country(conn)),
)
#: What a file created from `SCHEMA` today is already at.
@@ -434,6 +436,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None:
len(updates), len(rows))
def _recompute_country(conn: sqlite3.Connection) -> None:
"""Migration 13's step. `app.models.country_of` used to read a US state
code over a country's ("Bogota, CO" was US), return '' for a line naming
several European countries, and miss most countries outside Europe;
`Store.backfill_from_place` fills only an empty column, so a wrong code
would stay. Recomputed from `place` for every role `extract` has not
read (its answer only ever filled an empty column, but a role it read is
left as it is) and whose board sent no `country` of its own in `raw`
(the `search` plugin's Jobindex and Adzuna rows do), which is what
`Store.upsert_role` would write for that role today."""
rows = conn.execute(
"SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall()
updates = []
for uid, place, raw, old in rows:
try:
given = json.loads(raw or "{}").get("country") or ""
except (TypeError, ValueError, AttributeError):
given = ""
if given:
continue
new = country_of(place or "")
if new != old:
updates.append((new, uid))
if updates:
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates)
log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles",
len(updates), len(rows))
def user_version(conn: sqlite3.Connection) -> int:
return int(conn.execute("PRAGMA user_version").fetchone()[0])
+15 -2
View File
@@ -1094,11 +1094,24 @@ class Store:
"""One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
favourite. 0 deletes the row, since no row means neutral. Returns the
company with this agent's affinity. Raises ValueError for another
value and KeyError for an unknown company or agent."""
value and KeyError for an unknown company or agent.
A key with roles and no company row, an employer known only from a
search, gets a row here holding only its key and the name its roles
give it, the most frequent one: nothing else can be judged without
one. It has no board, `ats`, `careers_url` or `domain`, which is how
the Companies list tells it from a followed company."""
_check_affinity(affinity)
with self.transaction() as conn:
self._known(conn, "companies", "key", key)
self._known(conn, "agents", "id", agent_id)
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
named = conn.execute(
"""SELECT company FROM roles WHERE company_key=? AND company<>''
GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC
LIMIT 1""", (key,)).fetchone()
if named is None:
raise KeyError(key)
self._insert_company(conn, key, named[0], {})
if affinity == 0:
conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?",
(agent_id, key))
+1 -1
View File
@@ -42,7 +42,7 @@
<summary>Advanced</summary>
<label>Gates
<textarea id="a-gates" class="code" rows="6" spellcheck="false">[]</textarea>
<span class="hint">Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: <code>[{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}]</code> is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role open anywhere in Europe. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.</span></label>
<span class="hint">Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: <code>[{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}]</code> is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role in Europe that names no single country. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.</span></label>
</details>
<div class="agent-actions">
<button type="submit">Save and score</button>
+70 -10
View File
@@ -9,6 +9,7 @@ This is roledesk's own API, over the same store. Every refusal is
from __future__ import annotations
import sqlite3
from typing import Annotated, Any, Literal
from fastapi import APIRouter, HTTPException, Query, Response
@@ -270,7 +271,10 @@ class CompanyOut(BaseModel):
no `board` and an `ats` is one the app cannot read yet; `careers_url` is
where a person can. `failing_since` and `failing_reason` say since when
the board has failed on every fetch and why it last did, both empty
while it answers."""
while it answers. `from_search` marks a row that exists only because a
search listed its roles and somebody judged it (`Store.set_affinity`):
no board, not even a detached one, and no `ats`, `careers_url` or
`domain`, which every way of following a company writes one of."""
key: str
name: str
@@ -285,6 +289,7 @@ class CompanyOut(BaseModel):
country: str = ""
failing_since: str = ""
failing_reason: str = ""
from_search: bool = False
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
@@ -307,6 +312,7 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
"""Every company, with `affinity` as `agent` (the default agent when
absent) has it; 0 throughout when there is no agent."""
boards = _company_boards(store)
ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")}
counts = store.open_role_counts()
failing = jobs.pull.failing_boards(store)
out = []
@@ -318,7 +324,9 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
open_roles=counts.get(c.key, 0), ats=c.ats,
ats_slug=c.ats_slug, careers_url=c.careers_url,
country=c.country,
failing_since=since, failing_reason=reason))
failing_since=since, failing_reason=reason,
from_search=not (c.key in ever_boarded or c.ats
or c.careers_url or c.domain)))
return out
@@ -352,18 +360,70 @@ def detach_company_board(key: str, store: StoreDep) -> dict[str, Any]:
return {"closed": closed}
def _score_after_change(queue: Any) -> Job | None:
"""The `score` job whose result will include a change just written.
A `score` that is queued and has not started will read the change, so
it is the answer: the plain one, keyed `score`, or a follow-up. Else a
plain request, which the queue dedupes to a live job keyed `score`
because the kind is `solo`. When that job is already running it read
the old state, so it gets one follow-up that waits for it (`depends_on`,
`after="done"`), deduped on `score:after:<id>`: ten requests during one
run are one extra pass. A pull's `score` stage is a plain `score` job
too, queued when its stage starts, so a request during a pull joins it.
The retry is for a race in `pipeline.table.Jobs.enqueue`: when the
INSERT collides with a live job that finishes before the SELECT after
it, the SELECT finds nothing and the `IntegrityError` is re-raised. That
job has finished by then, so a second try succeeds."""
for attempt in range(3):
try:
queued = queue.jobs.last("score", states=("queued",))
if queued is not None and (queued.dedup_key == "score"
or queued.dedup_key.startswith("score:after:")):
return queued
job = queue.enqueue("score")
if job is not None and job.state == "running":
job = queue.enqueue("score", depends_on=[job.id], after="done",
dedup_key=f"score:after:{job.id}")
return job
except sqlite3.IntegrityError:
if attempt == 2:
raise
return None
@router.post("/score")
def request_score(queue: QueueDep) -> Job:
"""Score every agent's roles after a change: now, or right after the
`score` that is already running (see `_score_after_change`). Returns the
job to wait for; the Search agents page waits on it after a save."""
job = _score_after_change(queue)
if job is None:
raise HTTPException(409, "Scoring would not start.")
return job
class AffinityBody(BaseModel):
affinity: int
#: Whose affinity; the default agent when absent.
agent: int | None = None
class AffinityOut(Company):
#: The `score` job whose result will include this change; None when none
#: could be queued.
score_job: int | None = None
@router.post("/companies/{key}/affinity")
def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) -> Company:
def set_affinity(key: str, body: AffinityBody, store: StoreDep,
queue: QueueDep) -> AffinityOut:
"""-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the
default agent. A change here is not seen in Best match until `score` has
run: the queue dedupes to a live job of that kind, so several clicks are
one run."""
default agent. A key with roles and no company row gets one first (see
`Store.set_affinity`); 404 for a key with neither. A change here is not
seen in Best match until `score` has run, so it queues one
(`_score_after_change`) and answers with its id."""
if body.affinity not in AFFINITY:
raise HTTPException(422, "affinity is -1, 0, 1 or 2")
owner = _agent_or_default(store, body.agent)
@@ -371,8 +431,8 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep)
company = store.set_affinity(owner.id, key, body.affinity)
except KeyError:
raise HTTPException(404, f"no such company: {key}") from None
queue.enqueue("score")
return company
job = _score_after_change(queue)
return AffinityOut(**company.model_dump(), score_job=job.id if job else None)
# ----------------------------------------------------------------- sources
@@ -592,8 +652,8 @@ class AgentBody(BaseModel):
@router.post("/agents")
def save_agent(body: AgentBody, store: StoreDep) -> Agent:
"""Scores are not recomputed here: that is the `score` task, which the
page starts after a save."""
"""Scores are not recomputed here: the page asks `POST /api/score` after
a save and waits on the job it answers with."""
try:
return store.save_agent(body.name, body.intent, body.gates, body.weights,
body.enabled, agent_id=body.id, limits=body.limits)