Merge branch 'r2-backend'
# Conflicts: # app/storage/schema.py
This commit is contained in:
commit
562fa3ab09
15 files changed
+924
-157
No files matched your search
+7
-3
@@ -47,9 +47,13 @@ from app.storage import Store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: As many boards in the air per job as there are `http` workers by default
|
||||
#: (see `config.yaml`), so one pull keeps the pool busy without one job
|
||||
#: holding all 190-odd boards' worth of retries against a single attempt.
|
||||
#: Targets per `pull_batch` job. A batch reads its boards one after another,
|
||||
#: so the parallelism is the number of batches the `http` pool runs at once
|
||||
#: (8 workers by default, see `config.yaml`), not this number. On the live
|
||||
#: database of 2026-09-29, 175 boards and 4 searches make 19 batches, enough
|
||||
#: to keep eight workers busy, while a batch that fails or is killed costs at
|
||||
#: most ten targets' progress. A board's request is bounded by
|
||||
#: `plugins.ats.DEADLINE` plus one read timeout.
|
||||
BATCH_SIZE = 10
|
||||
|
||||
SETTLE = "app.jobs.pull:close_boards"
|
||||
|
||||
+522
-105
@@ -66,121 +66,538 @@ def company_key(company: str) -> str:
|
||||
|
||||
# ---------------------------------------------------------------- geography
|
||||
|
||||
#: Folded names a board writes in its location line, to an ISO code: every
|
||||
#: European country in English and its own language, the cities the boards
|
||||
#: name most, and the non-European places that fill the seed boards, so those
|
||||
#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
|
||||
_PLACES: dict[str, str] = {
|
||||
**dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
|
||||
**dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
|
||||
**dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
|
||||
"ghent"), "BE"),
|
||||
**dict.fromkeys(("bulgaria", "sofia"), "BG"),
|
||||
**dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
|
||||
**dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
|
||||
**dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
|
||||
# Folding keeps ø and æ, which are letters of their own rather than an o
|
||||
# or an a with a mark, so København is listed beside Kobenhavn.
|
||||
**dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
|
||||
"århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
|
||||
"lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
|
||||
"bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
|
||||
"birkerød"), "DK"),
|
||||
**dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
|
||||
**dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
|
||||
**dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
|
||||
"bordeaux", "lille"), "FR"),
|
||||
**dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
|
||||
"frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
|
||||
"dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
|
||||
"heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
|
||||
**dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
|
||||
**dict.fromkeys(("hungary", "budapest"), "HU"),
|
||||
**dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
|
||||
**dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
|
||||
"torino", "bologna", "florence"), "IT"),
|
||||
**dict.fromkeys(("latvia", "riga"), "LV"),
|
||||
**dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
|
||||
**dict.fromkeys(("luxembourg",), "LU"),
|
||||
**dict.fromkeys(("malta", "valletta"), "MT"),
|
||||
**dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
|
||||
"rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
||||
#: Letters NFKD leaves whole because they are letters of their own rather
|
||||
#: than a base letter with a mark, spelt the way boards write them in ASCII.
|
||||
_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe",
|
||||
"þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"})
|
||||
|
||||
|
||||
def _place_fold(text: str) -> str:
|
||||
"""`_fold` for the place lookup only: "Wrocław", "Łódź" and "København"
|
||||
meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is,
|
||||
since it also builds `company_key`, the primary key of `companies`."""
|
||||
return _fold((text or "").casefold().translate(_LETTERS))
|
||||
|
||||
|
||||
#: Every ISO 3166-1 country by its English name, plus the names boards also
|
||||
#: write for it: its own language's for a European one, and the usual short
|
||||
#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one.
|
||||
_COUNTRY_NAMES = """
|
||||
AD Andorra
|
||||
AE United Arab Emirates | UAE | U.A.E. | Emirates
|
||||
AF Afghanistan
|
||||
AG Antigua and Barbuda
|
||||
AI Anguilla
|
||||
AL Albania | Shqipëria | Shqipëri
|
||||
AM Armenia | Hayastan
|
||||
AO Angola
|
||||
AQ Antarctica
|
||||
AR Argentina
|
||||
AS American Samoa
|
||||
AT Austria | Österreich
|
||||
AU Australia
|
||||
AW Aruba
|
||||
AX Åland Islands | Åland
|
||||
AZ Azerbaijan | Azərbaycan
|
||||
BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina
|
||||
BB Barbados
|
||||
BD Bangladesh
|
||||
BE Belgium | België | Belgique | Belgien
|
||||
BF Burkina Faso
|
||||
BG Bulgaria | България | Balgariya
|
||||
BH Bahrain
|
||||
BI Burundi
|
||||
BJ Benin
|
||||
BL Saint Barthélemy
|
||||
BM Bermuda
|
||||
BN Brunei | Brunei Darussalam
|
||||
BO Bolivia
|
||||
BQ Caribbean Netherlands | Bonaire
|
||||
BR Brazil | Brasil
|
||||
BS Bahamas
|
||||
BT Bhutan
|
||||
BV Bouvet Island
|
||||
BW Botswana
|
||||
BY Belarus | Беларусь | Belarus'
|
||||
BZ Belize
|
||||
CA Canada
|
||||
CC Cocos Islands | Cocos (Keeling) Islands
|
||||
CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa
|
||||
CF Central African Republic
|
||||
CG Republic of the Congo | Congo-Brazzaville
|
||||
CH Switzerland | Schweiz | Suisse | Svizzera | Svizra
|
||||
CI Côte d'Ivoire | Ivory Coast
|
||||
CK Cook Islands
|
||||
CL Chile
|
||||
CM Cameroon
|
||||
CN China | People's Republic of China
|
||||
CO Colombia
|
||||
CR Costa Rica
|
||||
CU Cuba
|
||||
CV Cabo Verde | Cape Verde
|
||||
CW Curaçao
|
||||
CX Christmas Island
|
||||
CY Cyprus | Κύπρος | Kypros | Kibris
|
||||
CZ Czechia | Czech Republic | Česko | Česká republika
|
||||
DE Germany | Deutschland
|
||||
DJ Djibouti
|
||||
DK Denmark | Danmark
|
||||
DM Dominica
|
||||
DO Dominican Republic
|
||||
DZ Algeria
|
||||
EC Ecuador
|
||||
EE Estonia | Eesti
|
||||
EG Egypt
|
||||
EH Western Sahara
|
||||
ER Eritrea
|
||||
ES Spain | España | Espanya
|
||||
ET Ethiopia
|
||||
FI Finland | Suomi
|
||||
FJ Fiji
|
||||
FK Falkland Islands
|
||||
FM Micronesia
|
||||
FO Faroe Islands | Føroyar | Færøerne
|
||||
FR France
|
||||
GA Gabon
|
||||
GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland
|
||||
GD Grenada
|
||||
GE Georgia | Sakartvelo
|
||||
GF French Guiana
|
||||
GG Guernsey
|
||||
GH Ghana
|
||||
GI Gibraltar
|
||||
GL Greenland
|
||||
GM Gambia
|
||||
GN Guinea
|
||||
GP Guadeloupe
|
||||
GQ Equatorial Guinea
|
||||
GR Greece | Ελλάδα | Hellas | Ellada
|
||||
GS South Georgia and the South Sandwich Islands
|
||||
GT Guatemala
|
||||
GU Guam
|
||||
GW Guinea-Bissau
|
||||
GY Guyana
|
||||
HK Hong Kong
|
||||
HM Heard Island and McDonald Islands
|
||||
HN Honduras
|
||||
HR Croatia | Hrvatska
|
||||
HT Haiti
|
||||
HU Hungary | Magyarország
|
||||
ID Indonesia
|
||||
IE Ireland | Éire
|
||||
IL Israel
|
||||
IM Isle of Man
|
||||
IN India
|
||||
IO British Indian Ocean Territory
|
||||
IQ Iraq
|
||||
IR Iran
|
||||
IS Iceland | Ísland
|
||||
IT Italy | Italia
|
||||
JE Jersey
|
||||
JM Jamaica
|
||||
JO Jordan
|
||||
JP Japan
|
||||
KE Kenya
|
||||
KG Kyrgyzstan
|
||||
KH Cambodia
|
||||
KI Kiribati
|
||||
KM Comoros
|
||||
KN Saint Kitts and Nevis
|
||||
KP North Korea
|
||||
KR South Korea | Korea | Republic of Korea
|
||||
KW Kuwait
|
||||
KY Cayman Islands
|
||||
KZ Kazakhstan
|
||||
LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic
|
||||
LB Lebanon
|
||||
LC Saint Lucia
|
||||
LI Liechtenstein
|
||||
LK Sri Lanka
|
||||
LR Liberia
|
||||
LS Lesotho
|
||||
LT Lithuania | Lietuva
|
||||
LU Luxembourg | Lëtzebuerg | Luxemburg
|
||||
LV Latvia | Latvija
|
||||
LY Libya
|
||||
MA Morocco
|
||||
MC Monaco
|
||||
MD Moldova | Republic of Moldova
|
||||
ME Montenegro | Crna Gora
|
||||
MF Saint Martin
|
||||
MG Madagascar
|
||||
MH Marshall Islands
|
||||
MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија
|
||||
ML Mali
|
||||
MM Myanmar | Burma
|
||||
MN Mongolia
|
||||
MO Macao | Macau
|
||||
MP Northern Mariana Islands
|
||||
MQ Martinique
|
||||
MR Mauritania
|
||||
MS Montserrat
|
||||
MT Malta
|
||||
MU Mauritius
|
||||
MV Maldives
|
||||
MW Malawi
|
||||
MX Mexico | México
|
||||
MY Malaysia
|
||||
MZ Mozambique
|
||||
NA Namibia
|
||||
NC New Caledonia
|
||||
NE Niger
|
||||
NF Norfolk Island
|
||||
NG Nigeria
|
||||
NI Nicaragua
|
||||
NL Netherlands | The Netherlands | Nederland | Holland
|
||||
NO Norway | Norge | Noreg
|
||||
NP Nepal
|
||||
NR Nauru
|
||||
NU Niue
|
||||
NZ New Zealand | Aotearoa
|
||||
OM Oman
|
||||
PA Panama | Panamá
|
||||
PE Peru | Perú
|
||||
PF French Polynesia
|
||||
PG Papua New Guinea
|
||||
PH Philippines
|
||||
PK Pakistan
|
||||
PL Poland | Polska
|
||||
PM Saint Pierre and Miquelon
|
||||
PN Pitcairn Islands
|
||||
PR Puerto Rico
|
||||
PS Palestine
|
||||
PT Portugal
|
||||
PW Palau
|
||||
PY Paraguay
|
||||
QA Qatar
|
||||
RE Réunion
|
||||
RO Romania | România
|
||||
RS Serbia | Srbija | Србија
|
||||
RU Russia | Russian Federation | Россия | Rossiya
|
||||
RW Rwanda
|
||||
SA Saudi Arabia | KSA
|
||||
SB Solomon Islands
|
||||
SC Seychelles
|
||||
SD Sudan
|
||||
SE Sweden | Sverige
|
||||
SG Singapore
|
||||
SH Saint Helena
|
||||
SI Slovenia | Slovenija
|
||||
SJ Svalbard and Jan Mayen | Svalbard
|
||||
SK Slovakia | Slovensko
|
||||
SL Sierra Leone
|
||||
SM San Marino
|
||||
SN Senegal
|
||||
SO Somalia
|
||||
SR Suriname
|
||||
SS South Sudan
|
||||
ST São Tomé and Príncipe
|
||||
SV El Salvador
|
||||
SX Sint Maarten
|
||||
SY Syria
|
||||
SZ Eswatini | Swaziland
|
||||
TC Turks and Caicos Islands
|
||||
TD Chad
|
||||
TF French Southern Territories
|
||||
TG Togo
|
||||
TH Thailand
|
||||
TJ Tajikistan
|
||||
TK Tokelau
|
||||
TL Timor-Leste | East Timor
|
||||
TM Turkmenistan
|
||||
TN Tunisia
|
||||
TO Tonga
|
||||
TR Turkey | Türkiye
|
||||
TT Trinidad and Tobago
|
||||
TV Tuvalu
|
||||
TW Taiwan
|
||||
TZ Tanzania
|
||||
UA Ukraine | Україна | Ukraina
|
||||
UG Uganda
|
||||
UM United States Minor Outlying Islands
|
||||
US United States | United States of America | USA | U.S.A. | US | U.S.
|
||||
UY Uruguay
|
||||
UZ Uzbekistan
|
||||
VA Vatican City | Holy See
|
||||
VC Saint Vincent and the Grenadines
|
||||
VE Venezuela
|
||||
VG British Virgin Islands
|
||||
VI U.S. Virgin Islands | US Virgin Islands
|
||||
VN Vietnam | Viet Nam
|
||||
VU Vanuatu
|
||||
WF Wallis and Futuna
|
||||
WS Samoa
|
||||
XK Kosovo | Kosova
|
||||
YE Yemen
|
||||
YT Mayotte
|
||||
ZA South Africa
|
||||
ZM Zambia
|
||||
ZW Zimbabwe
|
||||
"""
|
||||
|
||||
#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK.
|
||||
_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line)
|
||||
|
||||
#: The countries `EU` stands for when a line names several: the UN
|
||||
#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and
|
||||
#: the Council of Europe's members in the Caucasus and Turkey, which
|
||||
#: European employers list beside their other offices.
|
||||
_EUROPE = frozenset((
|
||||
"AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
|
||||
"DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE",
|
||||
"IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL",
|
||||
"NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA",
|
||||
"XK",
|
||||
))
|
||||
_EUROPE_OR_EU = _EUROPE | {"EU"}
|
||||
|
||||
#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them.
|
||||
_REGIONS = ("europe", "emea", "eu", "european union", "eea")
|
||||
|
||||
#: The cities boards name most, and a few regions below a country. A US
|
||||
#: state or a Canadian province in the same part of the line beats one of
|
||||
#: these ("London, Ontario" is Canada); a country name does not lose to it.
|
||||
_CITIES: dict[str, str] = {
|
||||
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
|
||||
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
|
||||
**dict.fromkeys(("sofia",), "BG"),
|
||||
**dict.fromkeys(("zagreb",), "HR"),
|
||||
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
|
||||
**dict.fromkeys(("prague", "praha", "brno"), "CZ"),
|
||||
**dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg",
|
||||
"region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby",
|
||||
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
|
||||
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
|
||||
**dict.fromkeys(("tallinn", "tartu"), "EE"),
|
||||
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
|
||||
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
|
||||
"lille", "rennes", "amiens"), "FR"),
|
||||
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
|
||||
"stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg",
|
||||
"nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund",
|
||||
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
|
||||
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
|
||||
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
|
||||
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
|
||||
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
|
||||
**dict.fromkeys(("budapest",), "HU"),
|
||||
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
|
||||
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
|
||||
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
|
||||
**dict.fromkeys(("riga",), "LV"),
|
||||
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
|
||||
**dict.fromkeys(("valletta",), "MT"),
|
||||
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
||||
"delft"), "NL"),
|
||||
**dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
|
||||
"gdansk", "poznan", "lodz"), "PL"),
|
||||
**dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
|
||||
**dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
|
||||
"RO"),
|
||||
**dict.fromkeys(("slovakia", "bratislava"), "SK"),
|
||||
**dict.fromkeys(("slovenia", "ljubljana"), "SI"),
|
||||
**dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
|
||||
"malaga", "bilbao"), "ES"),
|
||||
**dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
|
||||
"uppsala"), "SE"),
|
||||
**dict.fromkeys(("iceland", "reykjavik"), "IS"),
|
||||
**dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
|
||||
**dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
|
||||
"wales", "northern ireland", "london", "manchester", "edinburgh",
|
||||
"glasgow", "bristol", "cambridge uk", "oxford", "leeds",
|
||||
"birmingham uk", "belfast"), "GB"),
|
||||
**dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
|
||||
"basel", "lausanne", "bern", "zug"), "CH"),
|
||||
**dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
|
||||
"nyc", "san francisco", "seattle", "boston", "chicago", "austin",
|
||||
"los angeles", "palo alto", "mountain view", "san mateo",
|
||||
"foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
|
||||
"costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
|
||||
"houston", "phoenix", "tempe", "portland", "salt lake city",
|
||||
"pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
|
||||
"nashville", "minneapolis"), "US"),
|
||||
**dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
|
||||
"waterloo"), "CA"),
|
||||
**dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
|
||||
"chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
||||
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
|
||||
"PL"),
|
||||
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
|
||||
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
|
||||
**dict.fromkeys(("belgrade", "beograd"), "RS"),
|
||||
**dict.fromkeys(("bratislava",), "SK"),
|
||||
**dict.fromkeys(("ljubljana",), "SI"),
|
||||
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
|
||||
"granada", "santiago de compostela"), "ES"),
|
||||
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
|
||||
**dict.fromkeys(("reykjavik",), "IS"),
|
||||
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
|
||||
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
|
||||
"oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff",
|
||||
"guildford", "truro", "swansea", "blackpool", "coventry", "brighton",
|
||||
"weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley",
|
||||
"slough", "southampton", "swindon", "liverpool", "redhill",
|
||||
"milton keynes"), "GB"),
|
||||
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
|
||||
"sankt gallen", "st gallen"), "CH"),
|
||||
**dict.fromkeys(("kyiv", "kiev"), "UA"),
|
||||
**dict.fromkeys(("istanbul", "ankara"), "TR"),
|
||||
**dict.fromkeys(("tirana",), "AL"),
|
||||
**dict.fromkeys(("yerevan",), "AM"),
|
||||
**dict.fromkeys(("tbilisi",), "GE"),
|
||||
**dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle",
|
||||
"boston", "chicago", "austin", "los angeles", "palo alto",
|
||||
"mountain view", "san mateo", "foster city", "sunnyvale", "san jose",
|
||||
"menlo park", "redwood city", "costa mesa", "denver", "atlanta",
|
||||
"washington dc", "washington d c", "miami", "dallas", "houston",
|
||||
"phoenix", "tempe", "portland", "salt lake city", "pittsburgh",
|
||||
"philadelphia", "detroit", "san diego", "raleigh", "nashville",
|
||||
"minneapolis", "cambridge ma", "bay area", "arlington"), "US"),
|
||||
**dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"),
|
||||
"CA"),
|
||||
**dict.fromkeys(("buenos aires",), "AR"),
|
||||
**dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"),
|
||||
**dict.fromkeys(("santiago", "santiago de chile"), "CL"),
|
||||
**dict.fromkeys(("bogota",), "CO"),
|
||||
**dict.fromkeys(("lima", "miraflores"), "PE"),
|
||||
**dict.fromkeys(("montevideo",), "UY"),
|
||||
**dict.fromkeys(("managua",), "NI"),
|
||||
**dict.fromkeys(("santo domingo",), "DO"),
|
||||
**dict.fromkeys(("san salvador",), "SV"),
|
||||
**dict.fromkeys(("panama city",), "PA"),
|
||||
**dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
||||
**dict.fromkeys(("dubai", "abu dhabi"), "AE"),
|
||||
**dict.fromkeys(("riyadh",), "SA"),
|
||||
**dict.fromkeys(("doha",), "QA"),
|
||||
**dict.fromkeys(("manama",), "BH"),
|
||||
**dict.fromkeys(("salmiya",), "KW"),
|
||||
**dict.fromkeys(("muscat",), "OM"),
|
||||
**dict.fromkeys(("cairo", "maadi"), "EG"),
|
||||
**dict.fromkeys(("casablanca",), "MA"),
|
||||
**dict.fromkeys(("dakar",), "SN"),
|
||||
**dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
||||
**dict.fromkeys(("almaty",), "KZ"),
|
||||
**dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"),
|
||||
**dict.fromkeys(("dhaka",), "BD"),
|
||||
**dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai",
|
||||
"gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
||||
**dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"),
|
||||
**dict.fromkeys(("singapore",), "SG"),
|
||||
**dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
|
||||
**dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
|
||||
**dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
||||
**dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
|
||||
**dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
||||
**dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
|
||||
**dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
|
||||
**dict.fromkeys(("manila", "taguig"), "PH"),
|
||||
**dict.fromkeys(("bangkok",), "TH"),
|
||||
**dict.fromkeys(("phnom penh",), "KH"),
|
||||
**dict.fromkeys(("taipei",), "TW"),
|
||||
**dict.fromkeys(("tokyo", "osaka"), "JP"),
|
||||
**dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"),
|
||||
**dict.fromkeys(("seoul",), "KR"),
|
||||
**dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales",
|
||||
"queensland"), "AU"),
|
||||
}
|
||||
|
||||
#: Keyed the way a place is folded before it is looked up, so a name can be
|
||||
#: written as it is spelt.
|
||||
_PLACES = {_fold(name): code for name, code in _PLACES.items()}
|
||||
#: US states and Canadian provinces, by name and by the code boards write
|
||||
#: after a comma ("Boston, MA", "Toronto, ON").
|
||||
_SUBDIVISIONS = """
|
||||
US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO
|
||||
US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA
|
||||
US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY
|
||||
US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN
|
||||
US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH
|
||||
US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND
|
||||
US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI
|
||||
US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT
|
||||
US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY
|
||||
CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB
|
||||
CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL
|
||||
CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT
|
||||
CA Nunavut NU
|
||||
"""
|
||||
|
||||
#: The longest name first, so "new york city" wins over "york" and "northern
|
||||
#: ireland" over "ireland".
|
||||
_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
|
||||
|
||||
#: US state codes after a comma, the way US boards write a city: "Boston, MA".
|
||||
#: Read only when no name matched, since "DE" and "IN" are countries too.
|
||||
_US_STATE = re.compile(
|
||||
r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
|
||||
r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
|
||||
r"\b")
|
||||
def _subdivisions() -> tuple[dict[str, str], dict[str, str]]:
|
||||
names: dict[str, str] = {}
|
||||
codes: dict[str, str] = {}
|
||||
for line in _SUBDIVISIONS.split("\n"):
|
||||
if line:
|
||||
country, rest = line.split(" ", 1)
|
||||
for entry in rest.split("|"):
|
||||
name, code = entry.strip().rsplit(" ", 1)
|
||||
names[_place_fold(name)] = country
|
||||
codes[code] = country
|
||||
return names, codes
|
||||
|
||||
|
||||
_STATE_NAMES, _STATE_CODES = _subdivisions()
|
||||
|
||||
|
||||
def _names() -> dict[str, tuple[str, str]]:
|
||||
"""Every folded name to `(code, kind)`; kind is `country`, `city` or
|
||||
`state`. A name that is a city and a state ("New York") is the city, so
|
||||
it does not beat another city in its line. A name that is a country and
|
||||
a state ("Georgia") is the country here and read as the state only when
|
||||
the line names a place in the US too."""
|
||||
names: dict[str, tuple[str, str]] = {}
|
||||
for name, code in _STATE_NAMES.items():
|
||||
names[name] = (code, "state")
|
||||
for name, code in _CITIES.items():
|
||||
names[_place_fold(name)] = (code, "city")
|
||||
for line in _COUNTRY_NAMES.split("\n"):
|
||||
if line:
|
||||
code, rest = line.split(" ", 1)
|
||||
for name in rest.split("|"):
|
||||
names[_place_fold(name)] = (code, "country")
|
||||
for name in _REGIONS:
|
||||
names[name] = ("EU", "country")
|
||||
return names
|
||||
|
||||
|
||||
_NAMES = _names()
|
||||
|
||||
#: A state name that is also a country's, read as the state only when the
|
||||
#: line names a place in that state's country too.
|
||||
_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES
|
||||
if _NAMES[name][1] == "country"}
|
||||
|
||||
#: The longest name in words, so a lookup tries "new york city" before "new
|
||||
#: york" and "northern ireland" before "ireland".
|
||||
_LONGEST = max(len(name.split()) for name in _NAMES)
|
||||
|
||||
#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin".
|
||||
_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b")
|
||||
|
||||
#: A country code in parentheses, "Leicester (GB)", read only when no name matched.
|
||||
_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)")
|
||||
|
||||
#: Where one location in a line ends and the next begins. A state beats a
|
||||
#: city only within one: "London; Ontario" is two places.
|
||||
_SEGMENT = re.compile(r"[;|/\n]")
|
||||
|
||||
|
||||
def _hits(folded: str) -> list[tuple[str, str, str]]:
|
||||
"""The names in a folded line, longest first at each word, as
|
||||
`(name, code, kind)`."""
|
||||
words = folded.split()
|
||||
hits = []
|
||||
i = 0
|
||||
while i < len(words):
|
||||
for n in range(min(_LONGEST, len(words) - i), 0, -1):
|
||||
name = " ".join(words[i:i + n])
|
||||
if name in _NAMES:
|
||||
hits.append((name, *_NAMES[name]))
|
||||
i += n
|
||||
break
|
||||
else:
|
||||
i += 1
|
||||
return hits
|
||||
|
||||
|
||||
def country_of(place: str) -> str:
|
||||
"""The one country a board's location line names, as an ISO code, or ''
|
||||
when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
|
||||
US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
|
||||
is EU, and "London, EMEA" is GB: a country beats the region it is in."""
|
||||
folded = f" {_fold(place or '')} "
|
||||
"""The one country a board's location line names, as an ISO code; `EU`
|
||||
when it names several and all of them are in Europe; '' when it names
|
||||
none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston,
|
||||
MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or
|
||||
New York" and "Remote, EU or US" are ''.
|
||||
|
||||
Within one location (the line split at `;`, `|` and `/`) the widest unit
|
||||
named wins: a country beats a state or a province, which beats a city.
|
||||
So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A
|
||||
country also beats the region it is in: "London, EMEA" is GB.
|
||||
|
||||
A state code that is also a country's ("DE", "CO", "MA"), and "Georgia",
|
||||
count as the state only when the line names a place in the US too:
|
||||
"Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather
|
||||
than Delaware. A country code in parentheses, "Leicester (GB)", is read
|
||||
only when no name matched."""
|
||||
segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")]
|
||||
named = {code for _, hits in segments for _, code, _ in hits}
|
||||
found: set[str] = set()
|
||||
for name in _PLACE_NAMES:
|
||||
needle = f" {name} "
|
||||
if needle in folded:
|
||||
found.add(_PLACES[name])
|
||||
folded = folded.replace(needle, " ")
|
||||
if not found and _US_STATE.search(place or ""):
|
||||
# Only when nothing else named a country: "Berlin, DE" is Germany.
|
||||
found.add("US")
|
||||
if len(found) > 1:
|
||||
found.discard("EU")
|
||||
return found.pop() if len(found) == 1 else ""
|
||||
for text, hits in segments:
|
||||
shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named}
|
||||
countries = {code for name, code, kind in hits
|
||||
if kind == "country" and code != "EU" and name not in shared}
|
||||
states = {code for _, code, kind in hits if kind == "state"}
|
||||
states |= {_SHARED_NAMES[name] for name in shared}
|
||||
states |= {country for code in _STATE_CODE.findall(text)
|
||||
if (country := _STATE_CODES.get(code))
|
||||
and (code not in _ISO or country in named)}
|
||||
found |= (countries or states or {code for _, code, _ in hits if code != "EU"})
|
||||
found |= {code for _, code, _ in hits if code == "EU"}
|
||||
if not found:
|
||||
found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO}
|
||||
if len(found) == 1:
|
||||
return found.pop()
|
||||
if found and found <= _EUROPE_OR_EU:
|
||||
countries = found - {"EU"}
|
||||
return countries.pop() if "EU" in found and len(countries) == 1 else "EU"
|
||||
return ""
|
||||
|
||||
|
||||
# --------------------------------------------------------------- vocabulary
|
||||
|
||||
@@ -289,14 +289,16 @@
|
||||
render();
|
||||
}
|
||||
|
||||
// Start the `score` job and wait for it, so "scored" means the Roles tab
|
||||
// already orders by what was saved. `said` is what happened first.
|
||||
// Ask for a `score` and wait for the job the server answers with, so
|
||||
// "scored" means the Roles tab already orders by what was saved. While a
|
||||
// score is running that is a follow-up queued after it, since the running
|
||||
// one read the agent before the save. `said` is what happened first.
|
||||
async function rescore(status, agent, said) {
|
||||
const runs = () => link("/settings/sources#runs", "follow it under Runs");
|
||||
status.replaceChildren(`${said}. Scoring every open role; `, runs(), ".");
|
||||
let job;
|
||||
try {
|
||||
job = await api("/api/jobs", { method: "POST", body: { kind: "score", payload: {} } });
|
||||
job = await api("/api/score", { method: "POST" });
|
||||
} catch (err) {
|
||||
status.textContent = `${said}, but scoring did not start.`;
|
||||
return;
|
||||
|
||||
@@ -71,9 +71,12 @@
|
||||
: "own site";
|
||||
sub.appendChild(badge);
|
||||
sub.append(" · not readable yet");
|
||||
} else if (company.from_search) {
|
||||
// Nobody followed it: its roles came from a search, and it was judged.
|
||||
sub.append("from job boards");
|
||||
} else {
|
||||
// Every row comes from the seed, Follow or discover: none of them
|
||||
// found where this company posts its roles.
|
||||
// Followed by the seed, Follow or discover, and none of them found
|
||||
// where this company posts its roles.
|
||||
sub.append("board not found");
|
||||
}
|
||||
if (company.careers_url && !company.board) {
|
||||
@@ -214,7 +217,7 @@
|
||||
case "hidden": return company.affinity === -1;
|
||||
case "on": return company.on === true;
|
||||
case "off": return company.on === false;
|
||||
case "unreadable": return company.subscription_id == null;
|
||||
case "unreadable": return company.subscription_id == null && !company.from_search;
|
||||
default: return true;
|
||||
}
|
||||
}
|
||||
|
||||
+6
-4
@@ -834,12 +834,14 @@
|
||||
}
|
||||
|
||||
/* The company's ★ ▲ ⊘, the one place a company is judged outside the
|
||||
Companies page. A company seen only through a search has no row of its
|
||||
own to judge. A change dims or undims its rows in the feed. */
|
||||
Companies page. A company seen only through a search has no row until
|
||||
its first change creates one, so the list is reloaded then. A change
|
||||
dims or undims its rows in the feed. */
|
||||
function renderAffinity(slot, r) {
|
||||
const company = companies.get(r.company_key);
|
||||
if (!company) { slot.replaceChildren(); return; }
|
||||
slot.replaceChildren(affinityControl(r.company_key, company.affinity, () => {
|
||||
if (!r.company_key) { slot.replaceChildren(); return; }
|
||||
slot.replaceChildren(affinityControl(r.company_key, company ? company.affinity : 0, async () => {
|
||||
if (!company) await loadCompanies(state.filter.agent).catch(() => {});
|
||||
renderAffinity(slot, r);
|
||||
renderFeed();
|
||||
}, state.filter.agent));
|
||||
|
||||
+32
-1
@@ -17,7 +17,7 @@ import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app.models import posted_at, posted_host
|
||||
from app.models import country_of, posted_at, posted_host
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -386,6 +386,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
||||
"could not be normalised into salary_min/salary_max/currency", """
|
||||
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
(13, "roles.country recomputed from place for every not-yet-extracted role whose "
|
||||
"board gave none, country lookup fix", lambda conn: _recompute_country(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -434,6 +436,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None:
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def _recompute_country(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 13's step. `app.models.country_of` used to read a US state
|
||||
code over a country's ("Bogota, CO" was US), return '' for a line naming
|
||||
several European countries, and miss most countries outside Europe;
|
||||
`Store.backfill_from_place` fills only an empty column, so a wrong code
|
||||
would stay. Recomputed from `place` for every role `extract` has not
|
||||
read (its answer only ever filled an empty column, but a role it read is
|
||||
left as it is) and whose board sent no `country` of its own in `raw`
|
||||
(the `search` plugin's Jobindex and Adzuna rows do), which is what
|
||||
`Store.upsert_role` would write for that role today."""
|
||||
rows = conn.execute(
|
||||
"SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall()
|
||||
updates = []
|
||||
for uid, place, raw, old in rows:
|
||||
try:
|
||||
given = json.loads(raw or "{}").get("country") or ""
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
given = ""
|
||||
if given:
|
||||
continue
|
||||
new = country_of(place or "")
|
||||
if new != old:
|
||||
updates.append((new, uid))
|
||||
if updates:
|
||||
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates)
|
||||
log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles",
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+15
-2
@@ -1094,11 +1094,24 @@ class Store:
|
||||
"""One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
|
||||
favourite. 0 deletes the row, since no row means neutral. Returns the
|
||||
company with this agent's affinity. Raises ValueError for another
|
||||
value and KeyError for an unknown company or agent."""
|
||||
value and KeyError for an unknown company or agent.
|
||||
|
||||
A key with roles and no company row, an employer known only from a
|
||||
search, gets a row here holding only its key and the name its roles
|
||||
give it, the most frequent one: nothing else can be judged without
|
||||
one. It has no board, `ats`, `careers_url` or `domain`, which is how
|
||||
the Companies list tells it from a followed company."""
|
||||
_check_affinity(affinity)
|
||||
with self.transaction() as conn:
|
||||
self._known(conn, "companies", "key", key)
|
||||
self._known(conn, "agents", "id", agent_id)
|
||||
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
|
||||
named = conn.execute(
|
||||
"""SELECT company FROM roles WHERE company_key=? AND company<>''
|
||||
GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC
|
||||
LIMIT 1""", (key,)).fetchone()
|
||||
if named is None:
|
||||
raise KeyError(key)
|
||||
self._insert_company(conn, key, named[0], {})
|
||||
if affinity == 0:
|
||||
conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?",
|
||||
(agent_id, key))
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
<summary>Advanced</summary>
|
||||
<label>Gates
|
||||
<textarea id="a-gates" class="code" rows="6" spellcheck="false">[]</textarea>
|
||||
<span class="hint">Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: <code>[{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}]</code> is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role open anywhere in Europe. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.</span></label>
|
||||
<span class="hint">Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: <code>[{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}]</code> is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role in Europe that names no single country. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.</span></label>
|
||||
</details>
|
||||
<div class="agent-actions">
|
||||
<button type="submit">Save and score</button>
|
||||
|
||||
+70
-10
@@ -9,6 +9,7 @@ This is roledesk's own API, over the same store. Every refusal is
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
@@ -270,7 +271,10 @@ class CompanyOut(BaseModel):
|
||||
no `board` and an `ats` is one the app cannot read yet; `careers_url` is
|
||||
where a person can. `failing_since` and `failing_reason` say since when
|
||||
the board has failed on every fetch and why it last did, both empty
|
||||
while it answers."""
|
||||
while it answers. `from_search` marks a row that exists only because a
|
||||
search listed its roles and somebody judged it (`Store.set_affinity`):
|
||||
no board, not even a detached one, and no `ats`, `careers_url` or
|
||||
`domain`, which every way of following a company writes one of."""
|
||||
|
||||
key: str
|
||||
name: str
|
||||
@@ -285,6 +289,7 @@ class CompanyOut(BaseModel):
|
||||
country: str = ""
|
||||
failing_since: str = ""
|
||||
failing_reason: str = ""
|
||||
from_search: bool = False
|
||||
|
||||
|
||||
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
|
||||
@@ -307,6 +312,7 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
|
||||
"""Every company, with `affinity` as `agent` (the default agent when
|
||||
absent) has it; 0 throughout when there is no agent."""
|
||||
boards = _company_boards(store)
|
||||
ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")}
|
||||
counts = store.open_role_counts()
|
||||
failing = jobs.pull.failing_boards(store)
|
||||
out = []
|
||||
@@ -318,7 +324,9 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
|
||||
open_roles=counts.get(c.key, 0), ats=c.ats,
|
||||
ats_slug=c.ats_slug, careers_url=c.careers_url,
|
||||
country=c.country,
|
||||
failing_since=since, failing_reason=reason))
|
||||
failing_since=since, failing_reason=reason,
|
||||
from_search=not (c.key in ever_boarded or c.ats
|
||||
or c.careers_url or c.domain)))
|
||||
return out
|
||||
|
||||
|
||||
@@ -352,18 +360,70 @@ def detach_company_board(key: str, store: StoreDep) -> dict[str, Any]:
|
||||
return {"closed": closed}
|
||||
|
||||
|
||||
def _score_after_change(queue: Any) -> Job | None:
|
||||
"""The `score` job whose result will include a change just written.
|
||||
|
||||
A `score` that is queued and has not started will read the change, so
|
||||
it is the answer: the plain one, keyed `score`, or a follow-up. Else a
|
||||
plain request, which the queue dedupes to a live job keyed `score`
|
||||
because the kind is `solo`. When that job is already running it read
|
||||
the old state, so it gets one follow-up that waits for it (`depends_on`,
|
||||
`after="done"`), deduped on `score:after:<id>`: ten requests during one
|
||||
run are one extra pass. A pull's `score` stage is a plain `score` job
|
||||
too, queued when its stage starts, so a request during a pull joins it.
|
||||
|
||||
The retry is for a race in `pipeline.table.Jobs.enqueue`: when the
|
||||
INSERT collides with a live job that finishes before the SELECT after
|
||||
it, the SELECT finds nothing and the `IntegrityError` is re-raised. That
|
||||
job has finished by then, so a second try succeeds."""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
queued = queue.jobs.last("score", states=("queued",))
|
||||
if queued is not None and (queued.dedup_key == "score"
|
||||
or queued.dedup_key.startswith("score:after:")):
|
||||
return queued
|
||||
job = queue.enqueue("score")
|
||||
if job is not None and job.state == "running":
|
||||
job = queue.enqueue("score", depends_on=[job.id], after="done",
|
||||
dedup_key=f"score:after:{job.id}")
|
||||
return job
|
||||
except sqlite3.IntegrityError:
|
||||
if attempt == 2:
|
||||
raise
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/score")
|
||||
def request_score(queue: QueueDep) -> Job:
|
||||
"""Score every agent's roles after a change: now, or right after the
|
||||
`score` that is already running (see `_score_after_change`). Returns the
|
||||
job to wait for; the Search agents page waits on it after a save."""
|
||||
job = _score_after_change(queue)
|
||||
if job is None:
|
||||
raise HTTPException(409, "Scoring would not start.")
|
||||
return job
|
||||
|
||||
|
||||
class AffinityBody(BaseModel):
|
||||
affinity: int
|
||||
#: Whose affinity; the default agent when absent.
|
||||
agent: int | None = None
|
||||
|
||||
|
||||
class AffinityOut(Company):
|
||||
#: The `score` job whose result will include this change; None when none
|
||||
#: could be queued.
|
||||
score_job: int | None = None
|
||||
|
||||
|
||||
@router.post("/companies/{key}/affinity")
|
||||
def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) -> Company:
|
||||
def set_affinity(key: str, body: AffinityBody, store: StoreDep,
|
||||
queue: QueueDep) -> AffinityOut:
|
||||
"""-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the
|
||||
default agent. A change here is not seen in Best match until `score` has
|
||||
run: the queue dedupes to a live job of that kind, so several clicks are
|
||||
one run."""
|
||||
default agent. A key with roles and no company row gets one first (see
|
||||
`Store.set_affinity`); 404 for a key with neither. A change here is not
|
||||
seen in Best match until `score` has run, so it queues one
|
||||
(`_score_after_change`) and answers with its id."""
|
||||
if body.affinity not in AFFINITY:
|
||||
raise HTTPException(422, "affinity is -1, 0, 1 or 2")
|
||||
owner = _agent_or_default(store, body.agent)
|
||||
@@ -371,8 +431,8 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep)
|
||||
company = store.set_affinity(owner.id, key, body.affinity)
|
||||
except KeyError:
|
||||
raise HTTPException(404, f"no such company: {key}") from None
|
||||
queue.enqueue("score")
|
||||
return company
|
||||
job = _score_after_change(queue)
|
||||
return AffinityOut(**company.model_dump(), score_job=job.id if job else None)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- sources
|
||||
@@ -592,8 +652,8 @@ class AgentBody(BaseModel):
|
||||
|
||||
@router.post("/agents")
|
||||
def save_agent(body: AgentBody, store: StoreDep) -> Agent:
|
||||
"""Scores are not recomputed here: that is the `score` task, which the
|
||||
page starts after a save."""
|
||||
"""Scores are not recomputed here: the page asks `POST /api/score` after
|
||||
a save and waits on the job it answers with."""
|
||||
try:
|
||||
return store.save_agent(body.name, body.intent, body.gates, body.weights,
|
||||
body.enabled, agent_id=body.id, limits=body.limits)
|
||||
|
||||
Reference in new issue
Block a user