Merge branch 'r2-backend'
# Conflicts: # app/storage/schema.py
This commit is contained in:
commit
562fa3ab09
15 files changed
+924
-157
No files matched your search
+7
-3
@@ -47,9 +47,13 @@ from app.storage import Store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: As many boards in the air per job as there are `http` workers by default
|
||||
#: (see `config.yaml`), so one pull keeps the pool busy without one job
|
||||
#: holding all 190-odd boards' worth of retries against a single attempt.
|
||||
#: Targets per `pull_batch` job. A batch reads its boards one after another,
|
||||
#: so the parallelism is the number of batches the `http` pool runs at once
|
||||
#: (8 workers by default, see `config.yaml`), not this number. On the live
|
||||
#: database of 2026-09-29, 175 boards and 4 searches make 19 batches, enough
|
||||
#: to keep eight workers busy, while a batch that fails or is killed costs at
|
||||
#: most ten targets' progress. A board's request is bounded by
|
||||
#: `plugins.ats.DEADLINE` plus one read timeout.
|
||||
BATCH_SIZE = 10
|
||||
|
||||
SETTLE = "app.jobs.pull:close_boards"
|
||||
|
||||
+522
-105
@@ -66,121 +66,538 @@ def company_key(company: str) -> str:
|
||||
|
||||
# ---------------------------------------------------------------- geography
|
||||
|
||||
#: Folded names a board writes in its location line, to an ISO code: every
|
||||
#: European country in English and its own language, the cities the boards
|
||||
#: name most, and the non-European places that fill the seed boards, so those
|
||||
#: resolve instead of staying unknown. Not a gazetteer; `extract` is the rest.
|
||||
_PLACES: dict[str, str] = {
|
||||
**dict.fromkeys(("europe", "emea", "eu", "european union", "eea"), "EU"),
|
||||
**dict.fromkeys(("austria", "osterreich", "vienna", "wien", "graz", "linz"), "AT"),
|
||||
**dict.fromkeys(("belgium", "belgie", "belgique", "brussels", "bruxelles", "antwerp",
|
||||
"ghent"), "BE"),
|
||||
**dict.fromkeys(("bulgaria", "sofia"), "BG"),
|
||||
**dict.fromkeys(("croatia", "hrvatska", "zagreb"), "HR"),
|
||||
**dict.fromkeys(("cyprus", "limassol", "nicosia"), "CY"),
|
||||
**dict.fromkeys(("czech republic", "czechia", "prague", "praha", "brno"), "CZ"),
|
||||
# Folding keeps ø and æ, which are letters of their own rather than an o
|
||||
# or an a with a mark, so København is listed beside Kobenhavn.
|
||||
**dict.fromkeys(("denmark", "danmark", "copenhagen", "københavn", "kobenhavn", "aarhus",
|
||||
"århus", "odense", "aalborg", "region hovedstaden", "frederiksberg",
|
||||
"lyngby", "kgs lyngby", "kongens lyngby", "ballerup", "hellerup",
|
||||
"bagsværd", "hørsholm", "brøndby", "glostrup", "søborg", "måløv",
|
||||
"birkerød"), "DK"),
|
||||
**dict.fromkeys(("estonia", "eesti", "tallinn", "tartu"), "EE"),
|
||||
**dict.fromkeys(("finland", "suomi", "helsinki", "espoo", "tampere"), "FI"),
|
||||
**dict.fromkeys(("france", "paris", "lyon", "marseille", "toulouse", "nantes",
|
||||
"bordeaux", "lille"), "FR"),
|
||||
**dict.fromkeys(("germany", "deutschland", "berlin", "munich", "munchen", "hamburg",
|
||||
"frankfurt", "cologne", "koln", "stuttgart", "dusseldorf", "leipzig",
|
||||
"dresden", "hannover", "nuremberg", "nurnberg", "karlsruhe",
|
||||
"heidelberg", "bonn", "essen", "dortmund", "mannheim"), "DE"),
|
||||
**dict.fromkeys(("greece", "athens", "thessaloniki"), "GR"),
|
||||
**dict.fromkeys(("hungary", "budapest"), "HU"),
|
||||
**dict.fromkeys(("ireland", "dublin", "cork", "galway", "limerick"), "IE"),
|
||||
**dict.fromkeys(("italy", "italia", "milan", "milano", "rome", "roma", "turin",
|
||||
"torino", "bologna", "florence"), "IT"),
|
||||
**dict.fromkeys(("latvia", "riga"), "LV"),
|
||||
**dict.fromkeys(("lithuania", "vilnius", "kaunas"), "LT"),
|
||||
**dict.fromkeys(("luxembourg",), "LU"),
|
||||
**dict.fromkeys(("malta", "valletta"), "MT"),
|
||||
**dict.fromkeys(("netherlands", "the netherlands", "nederland", "holland", "amsterdam",
|
||||
"rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
||||
#: Letters NFKD leaves whole because they are letters of their own rather
|
||||
#: than a base letter with a mark, spelt the way boards write them in ASCII.
|
||||
_LETTERS = str.maketrans({"ł": "l", "đ": "d", "ð": "d", "ø": "o", "æ": "ae", "œ": "oe",
|
||||
"þ": "th", "\u0131": "i", "ħ": "h", "ŀ": "l"})
|
||||
|
||||
|
||||
def _place_fold(text: str) -> str:
|
||||
"""`_fold` for the place lookup only: "Wrocław", "Łódź" and "København"
|
||||
meet "wroclaw", "lodz" and "kobenhavn". `_fold` itself stays as it is,
|
||||
since it also builds `company_key`, the primary key of `companies`."""
|
||||
return _fold((text or "").casefold().translate(_LETTERS))
|
||||
|
||||
|
||||
#: Every ISO 3166-1 country by its English name, plus the names boards also
|
||||
#: write for it: its own language's for a European one, and the usual short
|
||||
#: forms. Kosovo is XK, the code the EU uses; ISO has not assigned one.
|
||||
_COUNTRY_NAMES = """
|
||||
AD Andorra
|
||||
AE United Arab Emirates | UAE | U.A.E. | Emirates
|
||||
AF Afghanistan
|
||||
AG Antigua and Barbuda
|
||||
AI Anguilla
|
||||
AL Albania | Shqipëria | Shqipëri
|
||||
AM Armenia | Hayastan
|
||||
AO Angola
|
||||
AQ Antarctica
|
||||
AR Argentina
|
||||
AS American Samoa
|
||||
AT Austria | Österreich
|
||||
AU Australia
|
||||
AW Aruba
|
||||
AX Åland Islands | Åland
|
||||
AZ Azerbaijan | Azərbaycan
|
||||
BA Bosnia and Herzegovina | Bosnia & Herzegovina | Bosnia | Bosna i Hercegovina
|
||||
BB Barbados
|
||||
BD Bangladesh
|
||||
BE Belgium | België | Belgique | Belgien
|
||||
BF Burkina Faso
|
||||
BG Bulgaria | България | Balgariya
|
||||
BH Bahrain
|
||||
BI Burundi
|
||||
BJ Benin
|
||||
BL Saint Barthélemy
|
||||
BM Bermuda
|
||||
BN Brunei | Brunei Darussalam
|
||||
BO Bolivia
|
||||
BQ Caribbean Netherlands | Bonaire
|
||||
BR Brazil | Brasil
|
||||
BS Bahamas
|
||||
BT Bhutan
|
||||
BV Bouvet Island
|
||||
BW Botswana
|
||||
BY Belarus | Беларусь | Belarus'
|
||||
BZ Belize
|
||||
CA Canada
|
||||
CC Cocos Islands | Cocos (Keeling) Islands
|
||||
CD Democratic Republic of the Congo | DR Congo | DRC | Congo-Kinshasa
|
||||
CF Central African Republic
|
||||
CG Republic of the Congo | Congo-Brazzaville
|
||||
CH Switzerland | Schweiz | Suisse | Svizzera | Svizra
|
||||
CI Côte d'Ivoire | Ivory Coast
|
||||
CK Cook Islands
|
||||
CL Chile
|
||||
CM Cameroon
|
||||
CN China | People's Republic of China
|
||||
CO Colombia
|
||||
CR Costa Rica
|
||||
CU Cuba
|
||||
CV Cabo Verde | Cape Verde
|
||||
CW Curaçao
|
||||
CX Christmas Island
|
||||
CY Cyprus | Κύπρος | Kypros | Kibris
|
||||
CZ Czechia | Czech Republic | Česko | Česká republika
|
||||
DE Germany | Deutschland
|
||||
DJ Djibouti
|
||||
DK Denmark | Danmark
|
||||
DM Dominica
|
||||
DO Dominican Republic
|
||||
DZ Algeria
|
||||
EC Ecuador
|
||||
EE Estonia | Eesti
|
||||
EG Egypt
|
||||
EH Western Sahara
|
||||
ER Eritrea
|
||||
ES Spain | España | Espanya
|
||||
ET Ethiopia
|
||||
FI Finland | Suomi
|
||||
FJ Fiji
|
||||
FK Falkland Islands
|
||||
FM Micronesia
|
||||
FO Faroe Islands | Føroyar | Færøerne
|
||||
FR France
|
||||
GA Gabon
|
||||
GB United Kingdom | UK | U.K. | Great Britain | Britain | England | Scotland | Wales | Northern Ireland
|
||||
GD Grenada
|
||||
GE Georgia | Sakartvelo
|
||||
GF French Guiana
|
||||
GG Guernsey
|
||||
GH Ghana
|
||||
GI Gibraltar
|
||||
GL Greenland
|
||||
GM Gambia
|
||||
GN Guinea
|
||||
GP Guadeloupe
|
||||
GQ Equatorial Guinea
|
||||
GR Greece | Ελλάδα | Hellas | Ellada
|
||||
GS South Georgia and the South Sandwich Islands
|
||||
GT Guatemala
|
||||
GU Guam
|
||||
GW Guinea-Bissau
|
||||
GY Guyana
|
||||
HK Hong Kong
|
||||
HM Heard Island and McDonald Islands
|
||||
HN Honduras
|
||||
HR Croatia | Hrvatska
|
||||
HT Haiti
|
||||
HU Hungary | Magyarország
|
||||
ID Indonesia
|
||||
IE Ireland | Éire
|
||||
IL Israel
|
||||
IM Isle of Man
|
||||
IN India
|
||||
IO British Indian Ocean Territory
|
||||
IQ Iraq
|
||||
IR Iran
|
||||
IS Iceland | Ísland
|
||||
IT Italy | Italia
|
||||
JE Jersey
|
||||
JM Jamaica
|
||||
JO Jordan
|
||||
JP Japan
|
||||
KE Kenya
|
||||
KG Kyrgyzstan
|
||||
KH Cambodia
|
||||
KI Kiribati
|
||||
KM Comoros
|
||||
KN Saint Kitts and Nevis
|
||||
KP North Korea
|
||||
KR South Korea | Korea | Republic of Korea
|
||||
KW Kuwait
|
||||
KY Cayman Islands
|
||||
KZ Kazakhstan
|
||||
LA Laos | Lao People's Democratic Republic | Lao Peoples Democratic Republic
|
||||
LB Lebanon
|
||||
LC Saint Lucia
|
||||
LI Liechtenstein
|
||||
LK Sri Lanka
|
||||
LR Liberia
|
||||
LS Lesotho
|
||||
LT Lithuania | Lietuva
|
||||
LU Luxembourg | Lëtzebuerg | Luxemburg
|
||||
LV Latvia | Latvija
|
||||
LY Libya
|
||||
MA Morocco
|
||||
MC Monaco
|
||||
MD Moldova | Republic of Moldova
|
||||
ME Montenegro | Crna Gora
|
||||
MF Saint Martin
|
||||
MG Madagascar
|
||||
MH Marshall Islands
|
||||
MK North Macedonia | Macedonia | Severna Makedonija | Северна Македонија
|
||||
ML Mali
|
||||
MM Myanmar | Burma
|
||||
MN Mongolia
|
||||
MO Macao | Macau
|
||||
MP Northern Mariana Islands
|
||||
MQ Martinique
|
||||
MR Mauritania
|
||||
MS Montserrat
|
||||
MT Malta
|
||||
MU Mauritius
|
||||
MV Maldives
|
||||
MW Malawi
|
||||
MX Mexico | México
|
||||
MY Malaysia
|
||||
MZ Mozambique
|
||||
NA Namibia
|
||||
NC New Caledonia
|
||||
NE Niger
|
||||
NF Norfolk Island
|
||||
NG Nigeria
|
||||
NI Nicaragua
|
||||
NL Netherlands | The Netherlands | Nederland | Holland
|
||||
NO Norway | Norge | Noreg
|
||||
NP Nepal
|
||||
NR Nauru
|
||||
NU Niue
|
||||
NZ New Zealand | Aotearoa
|
||||
OM Oman
|
||||
PA Panama | Panamá
|
||||
PE Peru | Perú
|
||||
PF French Polynesia
|
||||
PG Papua New Guinea
|
||||
PH Philippines
|
||||
PK Pakistan
|
||||
PL Poland | Polska
|
||||
PM Saint Pierre and Miquelon
|
||||
PN Pitcairn Islands
|
||||
PR Puerto Rico
|
||||
PS Palestine
|
||||
PT Portugal
|
||||
PW Palau
|
||||
PY Paraguay
|
||||
QA Qatar
|
||||
RE Réunion
|
||||
RO Romania | România
|
||||
RS Serbia | Srbija | Србија
|
||||
RU Russia | Russian Federation | Россия | Rossiya
|
||||
RW Rwanda
|
||||
SA Saudi Arabia | KSA
|
||||
SB Solomon Islands
|
||||
SC Seychelles
|
||||
SD Sudan
|
||||
SE Sweden | Sverige
|
||||
SG Singapore
|
||||
SH Saint Helena
|
||||
SI Slovenia | Slovenija
|
||||
SJ Svalbard and Jan Mayen | Svalbard
|
||||
SK Slovakia | Slovensko
|
||||
SL Sierra Leone
|
||||
SM San Marino
|
||||
SN Senegal
|
||||
SO Somalia
|
||||
SR Suriname
|
||||
SS South Sudan
|
||||
ST São Tomé and Príncipe
|
||||
SV El Salvador
|
||||
SX Sint Maarten
|
||||
SY Syria
|
||||
SZ Eswatini | Swaziland
|
||||
TC Turks and Caicos Islands
|
||||
TD Chad
|
||||
TF French Southern Territories
|
||||
TG Togo
|
||||
TH Thailand
|
||||
TJ Tajikistan
|
||||
TK Tokelau
|
||||
TL Timor-Leste | East Timor
|
||||
TM Turkmenistan
|
||||
TN Tunisia
|
||||
TO Tonga
|
||||
TR Turkey | Türkiye
|
||||
TT Trinidad and Tobago
|
||||
TV Tuvalu
|
||||
TW Taiwan
|
||||
TZ Tanzania
|
||||
UA Ukraine | Україна | Ukraina
|
||||
UG Uganda
|
||||
UM United States Minor Outlying Islands
|
||||
US United States | United States of America | USA | U.S.A. | US | U.S.
|
||||
UY Uruguay
|
||||
UZ Uzbekistan
|
||||
VA Vatican City | Holy See
|
||||
VC Saint Vincent and the Grenadines
|
||||
VE Venezuela
|
||||
VG British Virgin Islands
|
||||
VI U.S. Virgin Islands | US Virgin Islands
|
||||
VN Vietnam | Viet Nam
|
||||
VU Vanuatu
|
||||
WF Wallis and Futuna
|
||||
WS Samoa
|
||||
XK Kosovo | Kosova
|
||||
YE Yemen
|
||||
YT Mayotte
|
||||
ZA South Africa
|
||||
ZM Zambia
|
||||
ZW Zimbabwe
|
||||
"""
|
||||
|
||||
#: Every code `_COUNTRY_NAMES` has, which is every ISO code plus XK.
|
||||
_ISO = frozenset(line.split()[0] for line in _COUNTRY_NAMES.split("\n") if line)
|
||||
|
||||
#: The countries `EU` stands for when a line names several: the UN
|
||||
#: geoscheme's Europe, plus Cyprus, an EU member it files under Asia, and
|
||||
#: the Council of Europe's members in the Caucasus and Turkey, which
|
||||
#: European employers list beside their other offices.
|
||||
_EUROPE = frozenset((
|
||||
"AD", "AL", "AM", "AT", "AX", "AZ", "BA", "BE", "BG", "BY", "CH", "CY", "CZ", "DE",
|
||||
"DK", "EE", "ES", "FI", "FO", "FR", "GB", "GE", "GG", "GI", "GR", "HR", "HU", "IE",
|
||||
"IM", "IS", "IT", "JE", "LI", "LT", "LU", "LV", "MC", "MD", "ME", "MK", "MT", "NL",
|
||||
"NO", "PL", "PT", "RO", "RS", "RU", "SE", "SI", "SJ", "SK", "SM", "TR", "UA", "VA",
|
||||
"XK",
|
||||
))
|
||||
_EUROPE_OR_EU = _EUROPE | {"EU"}
|
||||
|
||||
#: Names for Europe as a whole. `EU`, ISO's reserved code, stands for them.
|
||||
_REGIONS = ("europe", "emea", "eu", "european union", "eea")
|
||||
|
||||
#: The cities boards name most, and a few regions below a country. A US
|
||||
#: state or a Canadian province in the same part of the line beats one of
|
||||
#: these ("London, Ontario" is Canada); a country name does not lose to it.
|
||||
_CITIES: dict[str, str] = {
|
||||
**dict.fromkeys(("vienna", "wien", "graz", "linz"), "AT"),
|
||||
**dict.fromkeys(("brussels", "bruxelles", "antwerp", "ghent"), "BE"),
|
||||
**dict.fromkeys(("sofia",), "BG"),
|
||||
**dict.fromkeys(("zagreb",), "HR"),
|
||||
**dict.fromkeys(("limassol", "nicosia", "paphos"), "CY"),
|
||||
**dict.fromkeys(("prague", "praha", "brno"), "CZ"),
|
||||
**dict.fromkeys(("copenhagen", "kobenhavn", "aarhus", "arhus", "odense", "aalborg",
|
||||
"region hovedstaden", "frederiksberg", "lyngby", "kgs lyngby",
|
||||
"kongens lyngby", "ballerup", "hellerup", "bagsvaerd", "horsholm",
|
||||
"brondby", "glostrup", "soborg", "malov", "birkerod"), "DK"),
|
||||
**dict.fromkeys(("tallinn", "tartu"), "EE"),
|
||||
**dict.fromkeys(("helsinki", "espoo", "tampere"), "FI"),
|
||||
**dict.fromkeys(("paris", "lyon", "marseille", "toulouse", "nantes", "bordeaux",
|
||||
"lille", "rennes", "amiens"), "FR"),
|
||||
**dict.fromkeys(("berlin", "munich", "munchen", "hamburg", "frankfurt", "cologne", "koln",
|
||||
"stuttgart", "dusseldorf", "leipzig", "dresden", "hannover", "nuremberg",
|
||||
"nurnberg", "karlsruhe", "heidelberg", "bonn", "essen", "dortmund",
|
||||
"mannheim", "gottingen", "riederich", "rosenheim", "rostock", "bremen",
|
||||
"bochum", "augsburg", "potsdam", "munster", "deggendorf", "luneburg",
|
||||
"kassel", "braunschweig", "wurzburg", "remscheid", "osnabruck",
|
||||
"oldenburg", "emmendingen", "wuppertal", "wiesbaden", "tubingen"), "DE"),
|
||||
**dict.fromkeys(("athens", "thessaloniki"), "GR"),
|
||||
**dict.fromkeys(("budapest",), "HU"),
|
||||
**dict.fromkeys(("dublin", "cork", "galway", "limerick"), "IE"),
|
||||
**dict.fromkeys(("milan", "milano", "rome", "roma", "turin", "torino", "bologna",
|
||||
"florence", "verona", "cagliari", "ascoli piceno", "ancarano"), "IT"),
|
||||
**dict.fromkeys(("riga",), "LV"),
|
||||
**dict.fromkeys(("vilnius", "kaunas"), "LT"),
|
||||
**dict.fromkeys(("valletta",), "MT"),
|
||||
**dict.fromkeys(("amsterdam", "rotterdam", "utrecht", "the hague", "den haag", "eindhoven",
|
||||
"delft"), "NL"),
|
||||
**dict.fromkeys(("poland", "polska", "warsaw", "warszawa", "krakow", "wroclaw",
|
||||
"gdansk", "poznan", "lodz"), "PL"),
|
||||
**dict.fromkeys(("portugal", "lisbon", "lisboa", "porto", "braga"), "PT"),
|
||||
**dict.fromkeys(("romania", "bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"),
|
||||
"RO"),
|
||||
**dict.fromkeys(("slovakia", "bratislava"), "SK"),
|
||||
**dict.fromkeys(("slovenia", "ljubljana"), "SI"),
|
||||
**dict.fromkeys(("spain", "espana", "madrid", "barcelona", "valencia", "seville",
|
||||
"malaga", "bilbao"), "ES"),
|
||||
**dict.fromkeys(("sweden", "sverige", "stockholm", "gothenburg", "goteborg", "malmo",
|
||||
"uppsala"), "SE"),
|
||||
**dict.fromkeys(("iceland", "reykjavik"), "IS"),
|
||||
**dict.fromkeys(("norway", "norge", "oslo", "bergen", "trondheim"), "NO"),
|
||||
**dict.fromkeys(("united kingdom", "uk", "great britain", "england", "scotland",
|
||||
"wales", "northern ireland", "london", "manchester", "edinburgh",
|
||||
"glasgow", "bristol", "cambridge uk", "oxford", "leeds",
|
||||
"birmingham uk", "belfast"), "GB"),
|
||||
**dict.fromkeys(("switzerland", "schweiz", "suisse", "zurich", "geneva", "geneve",
|
||||
"basel", "lausanne", "bern", "zug"), "CH"),
|
||||
**dict.fromkeys(("united states", "usa", "us", "u s", "new york", "new york city",
|
||||
"nyc", "san francisco", "seattle", "boston", "chicago", "austin",
|
||||
"los angeles", "palo alto", "mountain view", "san mateo",
|
||||
"foster city", "sunnyvale", "san jose", "menlo park", "redwood city",
|
||||
"costa mesa", "denver", "atlanta", "washington dc", "miami", "dallas",
|
||||
"houston", "phoenix", "tempe", "portland", "salt lake city",
|
||||
"pittsburgh", "philadelphia", "detroit", "san diego", "raleigh",
|
||||
"nashville", "minneapolis"), "US"),
|
||||
**dict.fromkeys(("canada", "toronto", "vancouver", "montreal", "ottawa", "calgary",
|
||||
"waterloo"), "CA"),
|
||||
**dict.fromkeys(("india", "bengaluru", "bangalore", "hyderabad", "pune", "mumbai",
|
||||
"chennai", "gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
||||
**dict.fromkeys(("warsaw", "warszawa", "krakow", "wroclaw", "gdansk", "poznan", "lodz"),
|
||||
"PL"),
|
||||
**dict.fromkeys(("lisbon", "lisboa", "porto", "braga"), "PT"),
|
||||
**dict.fromkeys(("bucharest", "bucuresti", "cluj", "cluj napoca", "iasi"), "RO"),
|
||||
**dict.fromkeys(("belgrade", "beograd"), "RS"),
|
||||
**dict.fromkeys(("bratislava",), "SK"),
|
||||
**dict.fromkeys(("ljubljana",), "SI"),
|
||||
**dict.fromkeys(("madrid", "barcelona", "valencia", "seville", "malaga", "bilbao",
|
||||
"granada", "santiago de compostela"), "ES"),
|
||||
**dict.fromkeys(("stockholm", "gothenburg", "goteborg", "malmo", "uppsala"), "SE"),
|
||||
**dict.fromkeys(("reykjavik",), "IS"),
|
||||
**dict.fromkeys(("oslo", "bergen", "trondheim"), "NO"),
|
||||
**dict.fromkeys(("london", "manchester", "edinburgh", "glasgow", "bristol", "cambridge uk",
|
||||
"oxford", "leeds", "birmingham uk", "belfast", "leicester", "cardiff",
|
||||
"guildford", "truro", "swansea", "blackpool", "coventry", "brighton",
|
||||
"weybridge", "surrey", "sussex", "east anglia", "carlisle", "hinckley",
|
||||
"slough", "southampton", "swindon", "liverpool", "redhill",
|
||||
"milton keynes"), "GB"),
|
||||
**dict.fromkeys(("zurich", "geneva", "geneve", "basel", "lausanne", "bern", "zug",
|
||||
"sankt gallen", "st gallen"), "CH"),
|
||||
**dict.fromkeys(("kyiv", "kiev"), "UA"),
|
||||
**dict.fromkeys(("istanbul", "ankara"), "TR"),
|
||||
**dict.fromkeys(("tirana",), "AL"),
|
||||
**dict.fromkeys(("yerevan",), "AM"),
|
||||
**dict.fromkeys(("tbilisi",), "GE"),
|
||||
**dict.fromkeys(("new york", "new york city", "nyc", "san francisco", "sf", "seattle",
|
||||
"boston", "chicago", "austin", "los angeles", "palo alto",
|
||||
"mountain view", "san mateo", "foster city", "sunnyvale", "san jose",
|
||||
"menlo park", "redwood city", "costa mesa", "denver", "atlanta",
|
||||
"washington dc", "washington d c", "miami", "dallas", "houston",
|
||||
"phoenix", "tempe", "portland", "salt lake city", "pittsburgh",
|
||||
"philadelphia", "detroit", "san diego", "raleigh", "nashville",
|
||||
"minneapolis", "cambridge ma", "bay area", "arlington"), "US"),
|
||||
**dict.fromkeys(("toronto", "vancouver", "montreal", "ottawa", "calgary", "waterloo"),
|
||||
"CA"),
|
||||
**dict.fromkeys(("buenos aires",), "AR"),
|
||||
**dict.fromkeys(("sao paulo", "rio de janeiro", "sao jose dos campos"), "BR"),
|
||||
**dict.fromkeys(("santiago", "santiago de chile"), "CL"),
|
||||
**dict.fromkeys(("bogota",), "CO"),
|
||||
**dict.fromkeys(("lima", "miraflores"), "PE"),
|
||||
**dict.fromkeys(("montevideo",), "UY"),
|
||||
**dict.fromkeys(("managua",), "NI"),
|
||||
**dict.fromkeys(("santo domingo",), "DO"),
|
||||
**dict.fromkeys(("san salvador",), "SV"),
|
||||
**dict.fromkeys(("panama city",), "PA"),
|
||||
**dict.fromkeys(("mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
||||
**dict.fromkeys(("dubai", "abu dhabi"), "AE"),
|
||||
**dict.fromkeys(("riyadh",), "SA"),
|
||||
**dict.fromkeys(("doha",), "QA"),
|
||||
**dict.fromkeys(("manama",), "BH"),
|
||||
**dict.fromkeys(("salmiya",), "KW"),
|
||||
**dict.fromkeys(("muscat",), "OM"),
|
||||
**dict.fromkeys(("cairo", "maadi"), "EG"),
|
||||
**dict.fromkeys(("casablanca",), "MA"),
|
||||
**dict.fromkeys(("dakar",), "SN"),
|
||||
**dict.fromkeys(("tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
||||
**dict.fromkeys(("almaty",), "KZ"),
|
||||
**dict.fromkeys(("karachi", "lahore", "islamabad"), "PK"),
|
||||
**dict.fromkeys(("dhaka",), "BD"),
|
||||
**dict.fromkeys(("bengaluru", "bangalore", "hyderabad", "pune", "mumbai", "chennai",
|
||||
"gurgaon", "gurugram", "noida", "new delhi", "delhi"), "IN"),
|
||||
**dict.fromkeys(("kuala lumpur", "bangsar south"), "MY"),
|
||||
**dict.fromkeys(("singapore",), "SG"),
|
||||
**dict.fromkeys(("japan", "tokyo", "osaka"), "JP"),
|
||||
**dict.fromkeys(("australia", "sydney", "melbourne", "brisbane", "perth"), "AU"),
|
||||
**dict.fromkeys(("israel", "tel aviv", "tel aviv yafo", "jerusalem", "haifa"), "IL"),
|
||||
**dict.fromkeys(("brazil", "brasil", "sao paulo"), "BR"),
|
||||
**dict.fromkeys(("mexico", "mexico city", "ciudad de mexico", "guadalajara"), "MX"),
|
||||
**dict.fromkeys(("china", "shanghai", "beijing", "shenzhen", "hong kong"), "CN"),
|
||||
**dict.fromkeys(("south korea", "korea", "seoul"), "KR"),
|
||||
**dict.fromkeys(("manila", "taguig"), "PH"),
|
||||
**dict.fromkeys(("bangkok",), "TH"),
|
||||
**dict.fromkeys(("phnom penh",), "KH"),
|
||||
**dict.fromkeys(("taipei",), "TW"),
|
||||
**dict.fromkeys(("tokyo", "osaka"), "JP"),
|
||||
**dict.fromkeys(("shanghai", "beijing", "shenzhen", "chengdu", "guangzhou"), "CN"),
|
||||
**dict.fromkeys(("seoul",), "KR"),
|
||||
**dict.fromkeys(("sydney", "melbourne", "brisbane", "perth", "new south wales",
|
||||
"queensland"), "AU"),
|
||||
}
|
||||
|
||||
#: Keyed the way a place is folded before it is looked up, so a name can be
|
||||
#: written as it is spelt.
|
||||
_PLACES = {_fold(name): code for name, code in _PLACES.items()}
|
||||
#: US states and Canadian provinces, by name and by the code boards write
|
||||
#: after a comma ("Boston, MA", "Toronto, ON").
|
||||
_SUBDIVISIONS = """
|
||||
US Alabama AL | Alaska AK | Arizona AZ | Arkansas AR | California CA | Colorado CO
|
||||
US Connecticut CT | Delaware DE | District of Columbia DC | Florida FL | Georgia GA
|
||||
US Hawaii HI | Idaho ID | Illinois IL | Indiana IN | Iowa IA | Kansas KS | Kentucky KY
|
||||
US Louisiana LA | Maine ME | Maryland MD | Massachusetts MA | Michigan MI | Minnesota MN
|
||||
US Mississippi MS | Missouri MO | Montana MT | Nebraska NE | Nevada NV | New Hampshire NH
|
||||
US New Jersey NJ | New Mexico NM | New York NY | North Carolina NC | North Dakota ND
|
||||
US Ohio OH | Oklahoma OK | Oregon OR | Pennsylvania PA | Rhode Island RI
|
||||
US South Carolina SC | South Dakota SD | Tennessee TN | Texas TX | Utah UT | Vermont VT
|
||||
US Virginia VA | Washington WA | West Virginia WV | Wisconsin WI | Wyoming WY
|
||||
CA Ontario ON | Quebec QC | British Columbia BC | Alberta AB | Manitoba MB
|
||||
CA Saskatchewan SK | Nova Scotia NS | New Brunswick NB | Newfoundland and Labrador NL
|
||||
CA Newfoundland NL | Prince Edward Island PE | Yukon YT | Northwest Territories NT
|
||||
CA Nunavut NU
|
||||
"""
|
||||
|
||||
#: The longest name first, so "new york city" wins over "york" and "northern
|
||||
#: ireland" over "ireland".
|
||||
_PLACE_NAMES = sorted(_PLACES, key=len, reverse=True)
|
||||
|
||||
#: US state codes after a comma, the way US boards write a city: "Boston, MA".
|
||||
#: Read only when no name matched, since "DE" and "IN" are countries too.
|
||||
_US_STATE = re.compile(
|
||||
r",\s*(?:AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|"
|
||||
r"MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY)"
|
||||
r"\b")
|
||||
def _subdivisions() -> tuple[dict[str, str], dict[str, str]]:
|
||||
names: dict[str, str] = {}
|
||||
codes: dict[str, str] = {}
|
||||
for line in _SUBDIVISIONS.split("\n"):
|
||||
if line:
|
||||
country, rest = line.split(" ", 1)
|
||||
for entry in rest.split("|"):
|
||||
name, code = entry.strip().rsplit(" ", 1)
|
||||
names[_place_fold(name)] = country
|
||||
codes[code] = country
|
||||
return names, codes
|
||||
|
||||
|
||||
_STATE_NAMES, _STATE_CODES = _subdivisions()
|
||||
|
||||
|
||||
def _names() -> dict[str, tuple[str, str]]:
|
||||
"""Every folded name to `(code, kind)`; kind is `country`, `city` or
|
||||
`state`. A name that is a city and a state ("New York") is the city, so
|
||||
it does not beat another city in its line. A name that is a country and
|
||||
a state ("Georgia") is the country here and read as the state only when
|
||||
the line names a place in the US too."""
|
||||
names: dict[str, tuple[str, str]] = {}
|
||||
for name, code in _STATE_NAMES.items():
|
||||
names[name] = (code, "state")
|
||||
for name, code in _CITIES.items():
|
||||
names[_place_fold(name)] = (code, "city")
|
||||
for line in _COUNTRY_NAMES.split("\n"):
|
||||
if line:
|
||||
code, rest = line.split(" ", 1)
|
||||
for name in rest.split("|"):
|
||||
names[_place_fold(name)] = (code, "country")
|
||||
for name in _REGIONS:
|
||||
names[name] = ("EU", "country")
|
||||
return names
|
||||
|
||||
|
||||
_NAMES = _names()
|
||||
|
||||
#: A state name that is also a country's, read as the state only when the
|
||||
#: line names a place in that state's country too.
|
||||
_SHARED_NAMES = {name: _STATE_NAMES[name] for name in _STATE_NAMES
|
||||
if _NAMES[name][1] == "country"}
|
||||
|
||||
#: The longest name in words, so a lookup tries "new york city" before "new
|
||||
#: york" and "northern ireland" before "ireland".
|
||||
_LONGEST = max(len(name.split()) for name in _NAMES)
|
||||
|
||||
#: A state or province code after a comma or a dash: "Boston, MA", "US-CA-Dublin".
|
||||
_STATE_CODE = re.compile(r"(?:,|-|\u2013)\s*([A-Z]{2})\b")
|
||||
|
||||
#: A country code in parentheses, "Leicester (GB)", read only when no name matched.
|
||||
_PAREN_CODE = re.compile(r"\(([A-Z]{2})\)")
|
||||
|
||||
#: Where one location in a line ends and the next begins. A state beats a
|
||||
#: city only within one: "London; Ontario" is two places.
|
||||
_SEGMENT = re.compile(r"[;|/\n]")
|
||||
|
||||
|
||||
def _hits(folded: str) -> list[tuple[str, str, str]]:
|
||||
"""The names in a folded line, longest first at each word, as
|
||||
`(name, code, kind)`."""
|
||||
words = folded.split()
|
||||
hits = []
|
||||
i = 0
|
||||
while i < len(words):
|
||||
for n in range(min(_LONGEST, len(words) - i), 0, -1):
|
||||
name = " ".join(words[i:i + n])
|
||||
if name in _NAMES:
|
||||
hits.append((name, *_NAMES[name]))
|
||||
i += n
|
||||
break
|
||||
else:
|
||||
i += 1
|
||||
return hits
|
||||
|
||||
|
||||
def country_of(place: str) -> str:
|
||||
"""The one country a board's location line names, as an ISO code, or ''
|
||||
when it names none or several: "Berlin, Germany" is DE, "Boston, MA" is
|
||||
US, "London | Remote" is GB, "London or New York" is ''. "Remote - Europe"
|
||||
is EU, and "London, EMEA" is GB: a country beats the region it is in."""
|
||||
folded = f" {_fold(place or '')} "
|
||||
"""The one country a board's location line names, as an ISO code; `EU`
|
||||
when it names several and all of them are in Europe; '' when it names
|
||||
none, or countries not all in Europe. "Berlin, Germany" is DE, "Boston,
|
||||
MA" is US, "London | Remote" is GB, "Dublin, London" is EU, "London or
|
||||
New York" and "Remote, EU or US" are ''.
|
||||
|
||||
Within one location (the line split at `;`, `|` and `/`) the widest unit
|
||||
named wins: a country beats a state or a province, which beats a city.
|
||||
So "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A
|
||||
country also beats the region it is in: "London, EMEA" is GB.
|
||||
|
||||
A state code that is also a country's ("DE", "CO", "MA"), and "Georgia",
|
||||
count as the state only when the line names a place in the US too:
|
||||
"Berlin, DE" is DE, "Bogota, CO" is CO, "Paderborn, DE" is '' rather
|
||||
than Delaware. A country code in parentheses, "Leicester (GB)", is read
|
||||
only when no name matched."""
|
||||
segments = [(text, _hits(_place_fold(text))) for text in _SEGMENT.split(place or "")]
|
||||
named = {code for _, hits in segments for _, code, _ in hits}
|
||||
found: set[str] = set()
|
||||
for name in _PLACE_NAMES:
|
||||
needle = f" {name} "
|
||||
if needle in folded:
|
||||
found.add(_PLACES[name])
|
||||
folded = folded.replace(needle, " ")
|
||||
if not found and _US_STATE.search(place or ""):
|
||||
# Only when nothing else named a country: "Berlin, DE" is Germany.
|
||||
found.add("US")
|
||||
if len(found) > 1:
|
||||
found.discard("EU")
|
||||
return found.pop() if len(found) == 1 else ""
|
||||
for text, hits in segments:
|
||||
shared = {name for name, _, _ in hits if _SHARED_NAMES.get(name) in named}
|
||||
countries = {code for name, code, kind in hits
|
||||
if kind == "country" and code != "EU" and name not in shared}
|
||||
states = {code for _, code, kind in hits if kind == "state"}
|
||||
states |= {_SHARED_NAMES[name] for name in shared}
|
||||
states |= {country for code in _STATE_CODE.findall(text)
|
||||
if (country := _STATE_CODES.get(code))
|
||||
and (code not in _ISO or country in named)}
|
||||
found |= (countries or states or {code for _, code, _ in hits if code != "EU"})
|
||||
found |= {code for _, code, _ in hits if code == "EU"}
|
||||
if not found:
|
||||
found = {code for code in _PAREN_CODE.findall(place or "") if code in _ISO}
|
||||
if len(found) == 1:
|
||||
return found.pop()
|
||||
if found and found <= _EUROPE_OR_EU:
|
||||
countries = found - {"EU"}
|
||||
return countries.pop() if "EU" in found and len(countries) == 1 else "EU"
|
||||
return ""
|
||||
|
||||
|
||||
# --------------------------------------------------------------- vocabulary
|
||||
|
||||
@@ -289,14 +289,16 @@
|
||||
render();
|
||||
}
|
||||
|
||||
// Start the `score` job and wait for it, so "scored" means the Roles tab
|
||||
// already orders by what was saved. `said` is what happened first.
|
||||
// Ask for a `score` and wait for the job the server answers with, so
|
||||
// "scored" means the Roles tab already orders by what was saved. While a
|
||||
// score is running that is a follow-up queued after it, since the running
|
||||
// one read the agent before the save. `said` is what happened first.
|
||||
async function rescore(status, agent, said) {
|
||||
const runs = () => link("/settings/sources#runs", "follow it under Runs");
|
||||
status.replaceChildren(`${said}. Scoring every open role; `, runs(), ".");
|
||||
let job;
|
||||
try {
|
||||
job = await api("/api/jobs", { method: "POST", body: { kind: "score", payload: {} } });
|
||||
job = await api("/api/score", { method: "POST" });
|
||||
} catch (err) {
|
||||
status.textContent = `${said}, but scoring did not start.`;
|
||||
return;
|
||||
|
||||
@@ -71,9 +71,12 @@
|
||||
: "own site";
|
||||
sub.appendChild(badge);
|
||||
sub.append(" · not readable yet");
|
||||
} else if (company.from_search) {
|
||||
// Nobody followed it: its roles came from a search, and it was judged.
|
||||
sub.append("from job boards");
|
||||
} else {
|
||||
// Every row comes from the seed, Follow or discover: none of them
|
||||
// found where this company posts its roles.
|
||||
// Followed by the seed, Follow or discover, and none of them found
|
||||
// where this company posts its roles.
|
||||
sub.append("board not found");
|
||||
}
|
||||
if (company.careers_url && !company.board) {
|
||||
@@ -214,7 +217,7 @@
|
||||
case "hidden": return company.affinity === -1;
|
||||
case "on": return company.on === true;
|
||||
case "off": return company.on === false;
|
||||
case "unreadable": return company.subscription_id == null;
|
||||
case "unreadable": return company.subscription_id == null && !company.from_search;
|
||||
default: return true;
|
||||
}
|
||||
}
|
||||
|
||||
+6
-4
@@ -834,12 +834,14 @@
|
||||
}
|
||||
|
||||
/* The company's ★ ▲ ⊘, the one place a company is judged outside the
|
||||
Companies page. A company seen only through a search has no row of its
|
||||
own to judge. A change dims or undims its rows in the feed. */
|
||||
Companies page. A company seen only through a search has no row until
|
||||
its first change creates one, so the list is reloaded then. A change
|
||||
dims or undims its rows in the feed. */
|
||||
function renderAffinity(slot, r) {
|
||||
const company = companies.get(r.company_key);
|
||||
if (!company) { slot.replaceChildren(); return; }
|
||||
slot.replaceChildren(affinityControl(r.company_key, company.affinity, () => {
|
||||
if (!r.company_key) { slot.replaceChildren(); return; }
|
||||
slot.replaceChildren(affinityControl(r.company_key, company ? company.affinity : 0, async () => {
|
||||
if (!company) await loadCompanies(state.filter.agent).catch(() => {});
|
||||
renderAffinity(slot, r);
|
||||
renderFeed();
|
||||
}, state.filter.agent));
|
||||
|
||||
+32
-1
@@ -17,7 +17,7 @@ import logging
|
||||
import sqlite3
|
||||
from collections.abc import Callable
|
||||
|
||||
from app.models import posted_at, posted_host
|
||||
from app.models import country_of, posted_at, posted_host
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -386,6 +386,8 @@ ALTER TABLE companies ADD COLUMN ats_slug TEXT NOT NULL DEFAULT '';
|
||||
"could not be normalised into salary_min/salary_max/currency", """
|
||||
ALTER TABLE roles ADD COLUMN salary_text TEXT NOT NULL DEFAULT '';
|
||||
"""),
|
||||
(13, "roles.country recomputed from place for every not-yet-extracted role whose "
|
||||
"board gave none, country lookup fix", lambda conn: _recompute_country(conn)),
|
||||
)
|
||||
|
||||
#: What a file created from `SCHEMA` today is already at.
|
||||
@@ -434,6 +436,35 @@ def _recompute_function(conn: sqlite3.Connection) -> None:
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def _recompute_country(conn: sqlite3.Connection) -> None:
|
||||
"""Migration 13's step. `app.models.country_of` used to read a US state
|
||||
code over a country's ("Bogota, CO" was US), return '' for a line naming
|
||||
several European countries, and miss most countries outside Europe;
|
||||
`Store.backfill_from_place` fills only an empty column, so a wrong code
|
||||
would stay. Recomputed from `place` for every role `extract` has not
|
||||
read (its answer only ever filled an empty column, but a role it read is
|
||||
left as it is) and whose board sent no `country` of its own in `raw`
|
||||
(the `search` plugin's Jobindex and Adzuna rows do), which is what
|
||||
`Store.upsert_role` would write for that role today."""
|
||||
rows = conn.execute(
|
||||
"SELECT uid, place, raw, country FROM roles WHERE extracted_at IS NULL").fetchall()
|
||||
updates = []
|
||||
for uid, place, raw, old in rows:
|
||||
try:
|
||||
given = json.loads(raw or "{}").get("country") or ""
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
given = ""
|
||||
if given:
|
||||
continue
|
||||
new = country_of(place or "")
|
||||
if new != old:
|
||||
updates.append((new, uid))
|
||||
if updates:
|
||||
conn.executemany("UPDATE roles SET country = ? WHERE uid = ?", updates)
|
||||
log.info("migration 13: recomputed country for %d of %d not-yet-extracted roles",
|
||||
len(updates), len(rows))
|
||||
|
||||
|
||||
def user_version(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
+15
-2
@@ -1094,11 +1094,24 @@ class Store:
|
||||
"""One agent's stance on a company: -1 hidden, 0 neutral, 1 liked, 2
|
||||
favourite. 0 deletes the row, since no row means neutral. Returns the
|
||||
company with this agent's affinity. Raises ValueError for another
|
||||
value and KeyError for an unknown company or agent."""
|
||||
value and KeyError for an unknown company or agent.
|
||||
|
||||
A key with roles and no company row, an employer known only from a
|
||||
search, gets a row here holding only its key and the name its roles
|
||||
give it, the most frequent one: nothing else can be judged without
|
||||
one. It has no board, `ats`, `careers_url` or `domain`, which is how
|
||||
the Companies list tells it from a followed company."""
|
||||
_check_affinity(affinity)
|
||||
with self.transaction() as conn:
|
||||
self._known(conn, "companies", "key", key)
|
||||
self._known(conn, "agents", "id", agent_id)
|
||||
if conn.execute("SELECT 1 FROM companies WHERE key=?", (key,)).fetchone() is None:
|
||||
named = conn.execute(
|
||||
"""SELECT company FROM roles WHERE company_key=? AND company<>''
|
||||
GROUP BY company ORDER BY count(*) DESC, max(last_seen) DESC
|
||||
LIMIT 1""", (key,)).fetchone()
|
||||
if named is None:
|
||||
raise KeyError(key)
|
||||
self._insert_company(conn, key, named[0], {})
|
||||
if affinity == 0:
|
||||
conn.execute("DELETE FROM agent_companies WHERE agent_id=? AND company_key=?",
|
||||
(agent_id, key))
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
<summary>Advanced</summary>
|
||||
<label>Gates
|
||||
<textarea id="a-gates" class="code" rows="6" spellcheck="false">[]</textarea>
|
||||
<span class="hint">Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: <code>[{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}]</code> is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role open anywhere in Europe. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.</span></label>
|
||||
<span class="hint">Rules a role must meet (require), must meet one of (any), or that lift it (prefer), as JSON: <code>[{"field": "country", "values": ["DK"], "mode": "any"}, {"field": "remote", "values": ["remote"], "mode": "any"}, {"field": "seniority", "values": ["senior", "staff"], "mode": "prefer", "weight": 0.05}]</code> is "in Denmark, or remote". Fields: place, country, remote, seniority, function, salary_min, currency, permit, languages, company. EU is a role in Europe that names no single country. A rule only rejects a role whose column is filled; an empty one ranks until extract reads the posting.</span></label>
|
||||
</details>
|
||||
<div class="agent-actions">
|
||||
<button type="submit">Save and score</button>
|
||||
|
||||
+70
-10
@@ -9,6 +9,7 @@ This is roledesk's own API, over the same store. Every refusal is
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
@@ -270,7 +271,10 @@ class CompanyOut(BaseModel):
|
||||
no `board` and an `ats` is one the app cannot read yet; `careers_url` is
|
||||
where a person can. `failing_since` and `failing_reason` say since when
|
||||
the board has failed on every fetch and why it last did, both empty
|
||||
while it answers."""
|
||||
while it answers. `from_search` marks a row that exists only because a
|
||||
search listed its roles and somebody judged it (`Store.set_affinity`):
|
||||
no board, not even a detached one, and no `ats`, `careers_url` or
|
||||
`domain`, which every way of following a company writes one of."""
|
||||
|
||||
key: str
|
||||
name: str
|
||||
@@ -285,6 +289,7 @@ class CompanyOut(BaseModel):
|
||||
country: str = ""
|
||||
failing_since: str = ""
|
||||
failing_reason: str = ""
|
||||
from_search: bool = False
|
||||
|
||||
|
||||
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
|
||||
@@ -307,6 +312,7 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
|
||||
"""Every company, with `affinity` as `agent` (the default agent when
|
||||
absent) has it; 0 throughout when there is no agent."""
|
||||
boards = _company_boards(store)
|
||||
ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")}
|
||||
counts = store.open_role_counts()
|
||||
failing = jobs.pull.failing_boards(store)
|
||||
out = []
|
||||
@@ -318,7 +324,9 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
|
||||
open_roles=counts.get(c.key, 0), ats=c.ats,
|
||||
ats_slug=c.ats_slug, careers_url=c.careers_url,
|
||||
country=c.country,
|
||||
failing_since=since, failing_reason=reason))
|
||||
failing_since=since, failing_reason=reason,
|
||||
from_search=not (c.key in ever_boarded or c.ats
|
||||
or c.careers_url or c.domain)))
|
||||
return out
|
||||
|
||||
|
||||
@@ -352,18 +360,70 @@ def detach_company_board(key: str, store: StoreDep) -> dict[str, Any]:
|
||||
return {"closed": closed}
|
||||
|
||||
|
||||
def _score_after_change(queue: Any) -> Job | None:
|
||||
"""The `score` job whose result will include a change just written.
|
||||
|
||||
A `score` that is queued and has not started will read the change, so
|
||||
it is the answer: the plain one, keyed `score`, or a follow-up. Else a
|
||||
plain request, which the queue dedupes to a live job keyed `score`
|
||||
because the kind is `solo`. When that job is already running it read
|
||||
the old state, so it gets one follow-up that waits for it (`depends_on`,
|
||||
`after="done"`), deduped on `score:after:<id>`: ten requests during one
|
||||
run are one extra pass. A pull's `score` stage is a plain `score` job
|
||||
too, queued when its stage starts, so a request during a pull joins it.
|
||||
|
||||
The retry is for a race in `pipeline.table.Jobs.enqueue`: when the
|
||||
INSERT collides with a live job that finishes before the SELECT after
|
||||
it, the SELECT finds nothing and the `IntegrityError` is re-raised. That
|
||||
job has finished by then, so a second try succeeds."""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
queued = queue.jobs.last("score", states=("queued",))
|
||||
if queued is not None and (queued.dedup_key == "score"
|
||||
or queued.dedup_key.startswith("score:after:")):
|
||||
return queued
|
||||
job = queue.enqueue("score")
|
||||
if job is not None and job.state == "running":
|
||||
job = queue.enqueue("score", depends_on=[job.id], after="done",
|
||||
dedup_key=f"score:after:{job.id}")
|
||||
return job
|
||||
except sqlite3.IntegrityError:
|
||||
if attempt == 2:
|
||||
raise
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/score")
|
||||
def request_score(queue: QueueDep) -> Job:
|
||||
"""Score every agent's roles after a change: now, or right after the
|
||||
`score` that is already running (see `_score_after_change`). Returns the
|
||||
job to wait for; the Search agents page waits on it after a save."""
|
||||
job = _score_after_change(queue)
|
||||
if job is None:
|
||||
raise HTTPException(409, "Scoring would not start.")
|
||||
return job
|
||||
|
||||
|
||||
class AffinityBody(BaseModel):
|
||||
affinity: int
|
||||
#: Whose affinity; the default agent when absent.
|
||||
agent: int | None = None
|
||||
|
||||
|
||||
class AffinityOut(Company):
|
||||
#: The `score` job whose result will include this change; None when none
|
||||
#: could be queued.
|
||||
score_job: int | None = None
|
||||
|
||||
|
||||
@router.post("/companies/{key}/affinity")
|
||||
def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep) -> Company:
|
||||
def set_affinity(key: str, body: AffinityBody, store: StoreDep,
|
||||
queue: QueueDep) -> AffinityOut:
|
||||
"""-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the
|
||||
default agent. A change here is not seen in Best match until `score` has
|
||||
run: the queue dedupes to a live job of that kind, so several clicks are
|
||||
one run."""
|
||||
default agent. A key with roles and no company row gets one first (see
|
||||
`Store.set_affinity`); 404 for a key with neither. A change here is not
|
||||
seen in Best match until `score` has run, so it queues one
|
||||
(`_score_after_change`) and answers with its id."""
|
||||
if body.affinity not in AFFINITY:
|
||||
raise HTTPException(422, "affinity is -1, 0, 1 or 2")
|
||||
owner = _agent_or_default(store, body.agent)
|
||||
@@ -371,8 +431,8 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep, queue: QueueDep)
|
||||
company = store.set_affinity(owner.id, key, body.affinity)
|
||||
except KeyError:
|
||||
raise HTTPException(404, f"no such company: {key}") from None
|
||||
queue.enqueue("score")
|
||||
return company
|
||||
job = _score_after_change(queue)
|
||||
return AffinityOut(**company.model_dump(), score_job=job.id if job else None)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- sources
|
||||
@@ -592,8 +652,8 @@ class AgentBody(BaseModel):
|
||||
|
||||
@router.post("/agents")
|
||||
def save_agent(body: AgentBody, store: StoreDep) -> Agent:
|
||||
"""Scores are not recomputed here: that is the `score` task, which the
|
||||
page starts after a save."""
|
||||
"""Scores are not recomputed here: the page asks `POST /api/score` after
|
||||
a save and waits on the job it answers with."""
|
||||
try:
|
||||
return store.save_agent(body.name, body.intent, body.gates, body.weights,
|
||||
body.enabled, agent_id=body.id, limits=body.limits)
|
||||
|
||||
+6
-2
@@ -34,7 +34,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**`extract` reads the board's location too.** The model gets title, the board's `place` text, and the first 3000 characters of the description. Greenhouse's location line ("Berlin, Germany") is often the only place a country is stated, and it is part of the posting, so the "only what the posting states" contract still holds.
|
||||
|
||||
**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the folded location line to an ISO code from a table of European countries and cities, plus the non-European places that fill the seed boards, and returns nothing when a line names two countries. It resolves about 90% of the seed boards' roles for free, where `extract` would be a paid call per role for the same answer. `EU`, ISO's reserved code, stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it. A US state after a comma counts only when no name matched, because DE and IN are countries too.
|
||||
**`country` is filled from `place` by a lookup, before `extract`.** `country_of` maps the location line to an ISO code from every ISO country by its English name (and its own-language name for a European one) and the cities the boards name most, where `extract` would be a paid call per role for the same answer. On the live data (2026-09-29) 7.0% of open roles resolve to nothing, down from 16.8%, and most of those write only "Hybrid", "Remote" or a region such as "Americas". The line is folded by `_place_fold`, which also maps ł, đ, ø and æ since NFKD leaves them whole; `_fold` stays as it is because it builds `company_key`. Within one location (the line split at `;`, `|` and `/`) the widest unit named wins: a country beats a US state or a Canadian province, which beats a city, so "London, Ontario" is CA and "Tilburg, NB, Netherlands" is NL. A state code that is also a country's (DE, CO, MA, IN) counts as the state only when the line also names a US place, so "Bogota, CO" is CO and "Paderborn, DE" is ''. A line naming several countries is `EU` when all of them are European (the UN's Europe, plus Cyprus, Turkey and the Council of Europe's Caucasus members) and '' otherwise: "Dublin, London" is EU, "Remote, EU or US" is ''. `EU`, ISO's reserved code, also stands for "Remote - Europe" and "EMEA", so a Europe gate is a list of codes that includes it; a country beats the region it is in ("London, EMEA" is GB). `backfill_from_place` only fills an empty column, so a fix that changes answers comes with a migration: v12 recomputes `country` for every role `extract` has not read and whose board sent no `country` in `raw`, which changed 1,705 open roles, 48 of them from one code to another or to '' (25 Hong Kong roles moved from CN to HK).
|
||||
|
||||
**`permit` and `languages` are columns, and so is `extract_tries`.** `permit` answers the plan's open item 1 (visa sponsorship as a gate field) while nothing depends on it yet. newsdesk keeps give-up counts in `raw`; here `upsert_role` rewrites `raw` on every pull, which would reset the count, so the counter is its own column.
|
||||
|
||||
@@ -60,6 +60,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**A company row carries `ats`, `ats_slug`, `country` and `city`.** `ats` is what discovery found whether or not the app reads it; a company with an `ats` and no board subscription is one it cannot read yet. `ats_slug` is the name that host knows the company by (the `acme` of `acme.teamtailor.com`), so the parser written later has its targets without a second discovery run; with a board it is the board's slug. `country` and `city` come from the seed for now and are the first of the company facts the feed will filter by; funding, size and industry come from a source that has them, not from a column filled by hand.
|
||||
|
||||
**An employer known only from a search gets a company row when it is first judged, and the Companies list derives that it is one.** Jobindex and Adzuna roles carry a `company_key` with no row in `companies` (67 keys with open roles on the live data of 2026-09-29), so they could not be hidden or liked. `Store.set_affinity` now creates a row for a key that has roles, named by the name its roles give most often, with nothing else set, and the reader draws the control for any role with a `company_key`. Such a row has no board, `ats` or `careers_url`, which the Companies list read as "board not found" and put under "Not readable yet". The rule: a company with no `ats` subscription (not even a detached one) and no `ats`, `careers_url` or `domain` is from job boards (`from_search` on `/api/companies`), labelled so and left out of that filter. Derived rather than a column, because every way of following a company writes one of those (the seed and discover a domain, Follow and `set-board` a board, which detaching hides but keeps), and a search-only company that later gets a domain or a board is followed from then on without a flag to flip.
|
||||
|
||||
**Jobindex and Adzuna, not Jobnet or Jooble.** Jobindex, Denmark's largest board, has an RSS twin for every search page, with phrase and required-word syntax and a region id; no key. Adzuna has a documented API with a free key for the UK and most of the EU, but no Denmark. Jobnet's search endpoint answers 401 outside its own site's session, and Jooble gives a key only on a manual application and answered a Cloudflare challenge. LinkedIn and Indeed have no public search API.
|
||||
|
||||
**A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. An Adzuna search listed everything when it read as many distinct results as the `count` it states; a Jobindex search, whose feed states no count, when a page within its page limit came back with fewer than 20 items, counted before the parser drops any, so one malformed item does not pass for the end. Such a search closes like a board; the rest close once no search has listed them for 21 days. A Jobindex answer with no RSS channel raises instead of reading as an empty, complete search. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists.
|
||||
@@ -80,6 +82,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**The body cap is 96 MB, with one retry for Greenhouse.** openings.go's 8 MB cap was written against arbitrary URLs, while `ats` only fetches a fixed allow-list of seven ATS hosts, and the real run found `?content=true` boards between 9 and 41 MB. A board that still exceeds 96 MB retries once without `?content=true`, trading the description for the listing.
|
||||
|
||||
**A board request is streamed, with a 30 s read timeout and a 60 s deadline.** `_get` counts the body as it arrives and refuses it at `MAX_BYTES`, where reading `response.content` first held the whole body in memory before the check. httpx's timeouts are per phase, so a board that keeps sending a byte now and then never timed out; the deadline covers the whole request, redirects and body included, and is checked between chunks. openings.go's 10 s read timeout failed `greenhouse:mongodb` and `greenhouse:feedzai` in a pull of 2026-09-28. Measured on 2026-09-29, three runs each of those two and the five Greenhouse boards with the most open roles, with and without `pay_transparency=true` (84 requests, every one a CloudFront miss): a body of up to 10.4 MB arrives in under a second once it starts, and the first byte waited up to 16.6 s, with the three slowest at 16.6, 14.4 and 10.9 s. The flag made no measurable difference; the slow first byte is the origin's, with or without it. A read may wait 30 s, about twice the worst seen, and the deadline is 60 s, which leaves a large board room on a slower link.
|
||||
|
||||
**Workable's list endpoint has no URL and no text.** The parser was first written to Workable's published job object, since every slug tried in the build answered an empty list; the first populated board (Hugging Face, 2026-09-28) showed the v3 shape instead: `location` an object, `department` a list, a `shortcode` and no URL, and no description. The job page is `apply.workable.com/{slug}/j/{shortcode}/`, and `description` stays empty as for SmartRecruiters rather than costing one request per role.
|
||||
|
||||
**Workable and SmartRecruiters are read to the end, and a board closes roles only when it read the total it states.** Workable answers 10 roles a page with a `nextPage` token and SmartRecruiters at most 100 a page by `offset`, and both parsers read page one only, so the settle took page one for the whole board and closed what fell off it: Delivery Hero stated 980 roles and 100 were open. `_paged` follows the pages 0.5 s apart, up to 100 pages, and the plugin stamps `ats:complete` only when the distinct ids read equal `total` or `totalFound`. The count is taken on the raw results, so a row the parser drops does not block closure, and on distinct ids, so a role repeated by a shifted offset does not fake completeness. A 429 after page one ends the read: the roles read are stored and nothing closes, because a partial list is no evidence that the rest closed. A 429 on page one is a failure like any other. The other five hosts answer with the whole board and are complete whenever they parse.
|
||||
@@ -164,7 +168,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**"Hidden" is the word the UI uses for affinity -1; the code keeps `mute` and `hide_muted`.** Renaming the columns would touch every reader of `agents` and `companies` for no behavioural gain, so only the toggle's title, the filter label and the dimmed row say "hidden" in the UI.
|
||||
|
||||
**A company affinity change enqueues `score`.** `POST /api/companies/{key}/affinity` queues the `score` job right after the update, deduped to a live run of the same kind so ten clicks are one pass. Otherwise an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows at all until then, so they are missing from that order entirely.
|
||||
**A company affinity change or an agent save queues a `score` that will see it.** `POST /api/companies/{key}/affinity` and `POST /api/score`, which the Search agents page calls after a save, answer with the id of that job, and the page waits on it. `score` is `solo`, so the queue dedupes a plain request into the live job keyed `score`; a queued one has not read anything yet and sees the change, but a running one read the old state, and deduping into it reported "Saved and scored" for a score that did not include the save. So a request during a run queues one follow-up with `depends_on` the running job, `after="done"` and the key `score:after:<id>`, which makes ten clicks during one run one extra pass, and a queued follow-up is the answer until it starts. Without a score after it, an affinity change is invisible in Best match until the next 6-hourly pull, and an unmuted company's roles have no score rows until then. `pipeline`'s `Jobs.enqueue` has a race: when its INSERT collides with a live job that finishes before the SELECT after it, it re-raises the `IntegrityError` (`tests/test_web.py`'s affinity test failed on it about one run in ten). The app retries, which succeeds because that job has finished; the fix belongs upstream in `../boilerplate`.
|
||||
|
||||
**A search role's posting text is fetched on demand, not in the pull.** `POST /api/roles/{uid}/text` scrapes `original_url` only when the read page asks for it, because following Jobindex's redirect for every role during a pull would count as a click in Jobindex's own statistics; one click for a posting somebody actually opened is fair.
|
||||
|
||||
|
||||
+50
-19
@@ -5,7 +5,7 @@ unauthenticated JSON (Personio: XML) — first-party, no scraping, and a role
|
||||
that drops off the board is *closed*, which is the cheapest staleness check
|
||||
there is.
|
||||
|
||||
The field map, the redirect allow-list and the 10 s timeout are ported from
|
||||
The field map and the redirect allow-list are ported from
|
||||
`openings.go` and `cmd/seedgen/main.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
||||
`classify`'s nine functions and their keywords. Workable, SmartRecruiters,
|
||||
@@ -47,8 +47,7 @@ from desk.sources import (
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: Matches openings.go: a board answers in 10s or it is skipped. openings.go's
|
||||
#: 8 MB body cap does not: it was written against arbitrary URLs, and every
|
||||
#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every
|
||||
#: URL fetched here is `?content=true` against a fixed allow-list of seven ATS
|
||||
#: API hosts. The real run of 2026-09-23 hit it on the biggest employers —
|
||||
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
|
||||
@@ -56,9 +55,26 @@ log = logging.getLogger(__name__)
|
||||
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
|
||||
#: (Greenhouse only, see `_fetch_rows`) without `?content=true`, so at least
|
||||
#: the titles land.
|
||||
TIMEOUT = 10.0
|
||||
MAX_BYTES = 96 << 20
|
||||
|
||||
#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai`
|
||||
#: failed a pull of 2026-09-28 with a read timeout. Measured live on
|
||||
#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse
|
||||
#: boards with the most open roles (databricks, stripe, anthropic, feverup,
|
||||
#: datadog), with and without `pay_transparency=true`, cached and with a
|
||||
#: cache-busting parameter (84 requests; CloudFront answered every one
|
||||
#: "Miss", so all reached the origin): the body arrives in under 1 s once it
|
||||
#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the
|
||||
#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic,
|
||||
#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise.
|
||||
#: The flag made no difference that three runs could tell apart from that
|
||||
#: noise. So a read may wait 30 s, about twice the worst first byte seen,
|
||||
#: and a whole request, redirects and body included, 60 s (see `_get`),
|
||||
#: about four times the slowest seen, which still leaves a 96 MB body room
|
||||
#: on a link far slower than the one measured.
|
||||
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
|
||||
DEADLINE = 60.0
|
||||
|
||||
#: host -> (feed URL template, careers URL template). `{slug}` is filled in;
|
||||
#: recruitee and personio put the slug in the hostname instead of the path.
|
||||
_BOARDS: dict[str, tuple[str, str]] = {
|
||||
@@ -141,29 +157,44 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
"""Fetch one board feed: https only, the host's own allow-list, at most
|
||||
three redirect hops each checked the same way, and the body cut off at
|
||||
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none."""
|
||||
sends `payload` as JSON, `{}` when there is none.
|
||||
|
||||
The body is streamed and counted as it arrives, so an oversized one is
|
||||
refused at `MAX_BYTES` rather than after it has been read whole, and
|
||||
the whole request, redirects included, must end within `DEADLINE`: the
|
||||
per-read `TIMEOUTS` alone let a board that trickles bytes run on for
|
||||
good. The deadline is checked between chunks, so it can be overrun by
|
||||
at most one read timeout."""
|
||||
allow = _allowed_host(host, slug)
|
||||
deadline = time.monotonic() + DEADLINE
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
parsed = httpx.URL(current)
|
||||
if parsed.scheme != "https" or not allow(parsed.host):
|
||||
raise _Refused(f"{current} is not an allowed host for {host}")
|
||||
response = client.request(
|
||||
method, current, timeout=TIMEOUT, follow_redirects=False,
|
||||
with client.stream(
|
||||
method, current, timeout=TIMEOUTS, follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
|
||||
json=(payload or {}) if method == "POST" else None,
|
||||
)
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
if not location:
|
||||
raise ValueError("redirect with no location")
|
||||
current = str(httpx.URL(current).join(location))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
body = response.content
|
||||
if len(body) > MAX_BYTES:
|
||||
raise _TooLarge(f"response too large ({len(body)} bytes)")
|
||||
return body
|
||||
) as response:
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
if not location:
|
||||
raise ValueError("redirect with no location")
|
||||
current = str(httpx.URL(current).join(location))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
for chunk in response.iter_bytes():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)")
|
||||
if time.monotonic() > deadline:
|
||||
raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s",
|
||||
request=response.request)
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
raise ValueError("too many redirects")
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -333,6 +334,72 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def test_get_stops_reading_at_max_bytes_rather_than_after_the_body(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
sent: list[int] = []
|
||||
|
||||
def chunks() -> Iterator[bytes]:
|
||||
for i in range(100):
|
||||
sent.append(i)
|
||||
yield b"x" * 10
|
||||
|
||||
monkeypatch.setattr(ats, "MAX_BYTES", 25)
|
||||
with _client(lambda request: httpx.Response(200, content=chunks())) as client, \
|
||||
pytest.raises(ats._TooLarge):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
assert len(sent) == 3 # refused on the third chunk, not the hundredth
|
||||
|
||||
|
||||
def test_get_gives_up_at_the_deadline_however_steadily_bytes_arrive(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
clock = iter(range(0, 1000, 10))
|
||||
monkeypatch.setattr(ats.time, "monotonic", lambda: float(next(clock)))
|
||||
monkeypatch.setattr(ats, "DEADLINE", 25.0)
|
||||
body = (b"x" for _ in range(100))
|
||||
with _client(lambda request: httpx.Response(200, content=body)) as client, \
|
||||
pytest.raises(httpx.ReadTimeout, match="within 25 s"):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/acme", "lever", "acme")
|
||||
|
||||
|
||||
def test_get_follows_an_allowed_redirect_and_refuses_a_foreign_one() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/v0/postings/old":
|
||||
return httpx.Response(301, headers={"location": "/v0/postings/acme"})
|
||||
if request.url.path == "/v0/postings/away":
|
||||
return httpx.Response(302, headers={"location": "https://evil.example/"})
|
||||
return httpx.Response(200, content=b"[]")
|
||||
|
||||
with _client(handler) as client:
|
||||
assert ats._get(client, "https://api.lever.co/v0/postings/old", "lever", "acme") == b"[]"
|
||||
with pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.lever.co/v0/postings/away", "lever", "acme")
|
||||
|
||||
|
||||
def test_paging_reads_through_the_streamed_get(no_delay: None) -> None:
|
||||
"""`_paged` over the real `_get`: the POST body carries the cursor, and a
|
||||
429 on a later page still reaches it as an `HTTPStatusError`."""
|
||||
bodies: list[Any] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
payload = json.loads(request.content or b"{}")
|
||||
bodies.append(payload)
|
||||
if payload.get("token") == "stop":
|
||||
return httpx.Response(429)
|
||||
page = json.loads(_page("workable-2.json" if payload else "workable.json"))
|
||||
if payload:
|
||||
page["total"], page["nextPage"] = 4, "stop"
|
||||
return httpx.Response(200, json=page)
|
||||
|
||||
with _client(handler) as client:
|
||||
rows, complete = ats._fetch_rows(client, "workable:exotec")
|
||||
assert bodies == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}, {"token": "stop"}]
|
||||
assert (len(rows), complete) == (3, False)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- paging
|
||||
|
||||
def _page(name: str) -> bytes:
|
||||
|
||||
@@ -50,6 +50,33 @@ def store(tmp_path: Path) -> Store:
|
||||
("London or New York", ""),
|
||||
("Remote", ""),
|
||||
("", ""),
|
||||
("Wrocław", "PL"), # ł and đ do not decompose under NFKD
|
||||
("Łódź, Poland", "PL"),
|
||||
("Bogota, CO", "CO"), # a state code that is a country's too
|
||||
("Paderborn, DE", ""), # ... is no US state without a US place
|
||||
("Sliema, MT", ""),
|
||||
("Casablanca, MA", "MA"),
|
||||
("Cambridge, MA", "US"),
|
||||
("Chicago, IL", "US"),
|
||||
("Madrid, MD, Spain", "ES"),
|
||||
("Dublin, London", "EU"), # several European countries
|
||||
("Anywhere in France, Belgium, Spain", "EU"),
|
||||
("Amsterdam, Netherlands; London, United Kingdom; Remote - Europe", "EU"),
|
||||
("Remote, EU or US", ""), # a region and a country outside it
|
||||
("London, Ontario", "CA"), # a province beats the city before it
|
||||
("Bellevue, Washington", "US"),
|
||||
("Tilburg, NB, Netherlands", "NL"), # a country beats a province code
|
||||
("Atlanta, Georgia", "US"),
|
||||
("Tbilisi, Georgia", "GE"),
|
||||
("Malaysia", "MY"),
|
||||
("Belgrade, Serbia", "RS"),
|
||||
("Dubai", "AE"),
|
||||
("United Arab Emirates", "AE"),
|
||||
("Buenos Aires, , Argentina", "AR"),
|
||||
("Leicester (GB)", "GB"), # a code in parentheses when nothing else
|
||||
("Cagliari (CA)", "IT"), # ... and not over a name
|
||||
("Hybrid", ""),
|
||||
("Distributed", ""),
|
||||
])
|
||||
def test_country_of(place: str, code: str) -> None:
|
||||
assert country_of(place) == code
|
||||
|
||||
+28
-1
@@ -66,7 +66,7 @@ def test_schema_creates_on_an_empty_file(tmp_path: Path) -> None:
|
||||
"application_events", "agents", "scores", "state"} <= tables
|
||||
assert "idx_roles_cluster" in indexes
|
||||
assert store.migrations_applied == []
|
||||
assert LATEST == 12
|
||||
assert LATEST == 13
|
||||
# A second open finds it current and applies nothing.
|
||||
assert Store(path).migrations_applied == []
|
||||
|
||||
@@ -260,6 +260,33 @@ def test_v10_backfills_posted_at_by_the_same_rule_upsert_uses(
|
||||
assert after_untrusted.posted_at == before_untrusted.posted_at == before_untrusted.first_seen
|
||||
|
||||
|
||||
def test_v13_recomputes_country_from_place_unless_the_board_or_extract_set_it(
|
||||
tmp_path: Path, clock: Clock) -> None:
|
||||
path = tmp_path / "v11.db"
|
||||
store = Store(path)
|
||||
store.clock = clock
|
||||
wrong = posting("1", place="Bogota, CO") # was read as US
|
||||
several = posting("2", place="Dublin, London") # was ''
|
||||
given = posting("3", place="Bogota, CO", country="US") # the board said so
|
||||
extracted = posting("4", place="Bogota, CO")
|
||||
for item in (wrong, several, given, extracted):
|
||||
store.upsert_role(item)
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("UPDATE roles SET country = 'US' WHERE place = 'Bogota, CO'")
|
||||
conn.execute("UPDATE roles SET country = '' WHERE uid = ?", (several.uid,))
|
||||
conn.execute("UPDATE roles SET extracted_at = '2026-09-23' WHERE uid = ?",
|
||||
(extracted.uid,))
|
||||
conn.execute("PRAGMA user_version = 12")
|
||||
|
||||
migrated = Store(path)
|
||||
|
||||
assert migrated.migrations_applied == list(range(13, LATEST + 1))
|
||||
countries = {uid: role.country for uid in (wrong.uid, several.uid, given.uid, extracted.uid)
|
||||
if (role := migrated.role(uid)) is not None}
|
||||
assert countries == {wrong.uid: "CO", several.uid: "EU", given.uid: "US",
|
||||
extracted.uid: "US"}
|
||||
|
||||
|
||||
def test_state_is_a_mapping(store: Store) -> None:
|
||||
store.state["ats:last"] = "2026-09-23"
|
||||
assert store.state["ats:last"] == "2026-09-23"
|
||||
|
||||
+82
-3
@@ -5,6 +5,7 @@ worker beyond starting and stopping it with the app."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -18,6 +19,7 @@ from app.config import Config, ServerConfig
|
||||
from app.main import create_app
|
||||
from app.storage import Store
|
||||
from app.web import MOVED, SETTINGS, TABS
|
||||
from app.web.api import _score_after_change
|
||||
from app.web.pages import SECTIONS
|
||||
from auth import AuthConfig
|
||||
from desk.models import Item
|
||||
@@ -411,7 +413,7 @@ def test_companies_lists_board_and_open_role_count(
|
||||
sub = store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
||||
store.add_company("Acme", key="acme", ats="greenhouse", country="DK")
|
||||
store.upsert_role(posting("1", company="Acme"), sub)
|
||||
store.add_company("Other", key="other") # no board found
|
||||
store.add_company("Other", key="other", domain="other.example") # no board found
|
||||
store.add_company("Novo", key="novo", ats="successfactors", # followed, not readable
|
||||
careers_url="https://novo.example/careers")
|
||||
|
||||
@@ -421,13 +423,47 @@ def test_companies_lists_board_and_open_role_count(
|
||||
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
|
||||
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
|
||||
"ats": "greenhouse", "ats_slug": "", "careers_url": "",
|
||||
"country": "DK", "failing_since": "", "failing_reason": ""}
|
||||
"country": "DK", "failing_since": "", "failing_reason": "",
|
||||
"from_search": False}
|
||||
[other] = [c for c in res.json() if c["key"] == "other"]
|
||||
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
|
||||
"", None, None, "")
|
||||
[novo] = [c for c in res.json() if c["key"] == "novo"]
|
||||
assert (novo["board"], novo["subscription_id"], novo["ats"], novo["careers_url"]) == (
|
||||
"", None, "successfactors", "https://novo.example/careers")
|
||||
assert not any(c["from_search"] for c in res.json())
|
||||
|
||||
|
||||
def test_judging_a_company_known_only_from_a_search_creates_its_row(
|
||||
client: TestClient, store: Store) -> None:
|
||||
"""Jobindex and Adzuna employers have roles and no company row. The first
|
||||
affinity change makes a minimal one, named by its roles, which the
|
||||
Companies list marks `from_search`, unlike a followed company without a
|
||||
board, and a company whose board was detached."""
|
||||
store.save_agent("A")
|
||||
search = store.add_subscription("search", "jobindex:cph", label="cph", board="jobindex")
|
||||
for i, name in enumerate(("Novo Nordisk A/S", "Novo Nordisk A/S", "NOVO NORDISK A/S")):
|
||||
item = posting(f"n{i}", company=name)
|
||||
item.source = "search"
|
||||
store.upsert_role(item, search)
|
||||
key = "novo-nordisk"
|
||||
assert store.company(key) is None
|
||||
|
||||
res = client.post(f"/api/companies/{key}/affinity", json={"affinity": -1})
|
||||
assert res.status_code == 200, res.text
|
||||
assert (res.json()["name"], res.json()["affinity"]) == ("Novo Nordisk A/S", -1)
|
||||
assert client.post("/api/companies/nobody/affinity", json={"affinity": 1}).status_code == 404
|
||||
|
||||
store.add_company("Pleo", key="pleo", domain="pleo.io") # followed, no board found
|
||||
sub = store.add_subscription("ats", "lever:gone", company="Gone", board="lever")
|
||||
store.add_company("Gone", key="gone")
|
||||
store.set_company_board("gone", "") # detached
|
||||
assert not store.subscription(sub.id).show # type: ignore[union-attr]
|
||||
rows = {c["key"]: c for c in client.get("/api/companies").json()}
|
||||
assert rows["gone"]["board"] == ""
|
||||
assert rows[key]["from_search"] and rows[key]["open_roles"] == 3
|
||||
assert (rows[key]["board"], rows[key]["ats"], rows[key]["careers_url"]) == ("", "", "")
|
||||
assert not rows["pleo"]["from_search"] and not rows["gone"]["from_search"]
|
||||
|
||||
|
||||
def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,
|
||||
@@ -468,7 +504,7 @@ def test_set_affinity_validates_updates_and_enqueues_a_score(
|
||||
assert res.json()["affinity"] == 2
|
||||
assert store.company("acme", agent.id).affinity == 2 # type: ignore[union-attr]
|
||||
assert store.company("acme", other.id).affinity == 0 # type: ignore[union-attr]
|
||||
assert queue.jobs.last("score", states=("queued", "running", "done")) is not None
|
||||
assert queue.jobs.get(res.json()["score_job"]).kind == "score"
|
||||
|
||||
res = client.post("/api/companies/acme/affinity", json={"affinity": -1, "agent": other.id})
|
||||
assert res.status_code == 200 and res.json()["affinity"] == -1
|
||||
@@ -482,6 +518,49 @@ def test_set_affinity_validates_updates_and_enqueues_a_score(
|
||||
assert client.get("/api/companies", params={"agent": 999}).status_code == 404
|
||||
|
||||
|
||||
def test_a_score_asked_for_during_a_run_is_one_follow_up_after_it(queue: Any) -> None:
|
||||
"""No worker runs here (no client), so the test moves the jobs itself."""
|
||||
first = _score_after_change(queue)
|
||||
assert first is not None and first.dedup_key == "score"
|
||||
assert _score_after_change(queue).id == first.id # queued: it will read the change
|
||||
|
||||
assert queue.jobs.claim({"cpu": 1}).id == first.id
|
||||
during = {_score_after_change(queue).id for _ in range(10)}
|
||||
assert len(during) == 1 and first.id not in during
|
||||
follow = queue.jobs.get(during.pop())
|
||||
assert (follow.depends_on, follow.after, follow.dedup_key) == (
|
||||
[first.id], "done", f"score:after:{first.id}")
|
||||
|
||||
queue.jobs.finish(first.id, "done")
|
||||
assert _score_after_change(queue).id == follow.id # queued, not started: still the one
|
||||
|
||||
assert queue.jobs.claim({"cpu": 1}).id == follow.id
|
||||
after = _score_after_change(queue)
|
||||
assert after.id not in (first.id, follow.id) and after.dedup_key == "score"
|
||||
|
||||
|
||||
def test_score_after_change_retries_the_queues_dedup_race(
|
||||
queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`Jobs.enqueue` re-raises the UNIQUE failure when the live job it
|
||||
collided with finishes before it reads it back."""
|
||||
real, calls = queue.enqueue, []
|
||||
|
||||
def racy(*args: Any, **kwargs: Any) -> Any:
|
||||
calls.append(args)
|
||||
if len(calls) == 1:
|
||||
raise sqlite3.IntegrityError("UNIQUE constraint failed: jobs.dedup_key")
|
||||
return real(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(queue, "enqueue", racy)
|
||||
assert _score_after_change(queue) is not None and len(calls) == 2
|
||||
|
||||
|
||||
def test_post_score_answers_with_the_job_to_wait_for(client: TestClient, queue: Any) -> None:
|
||||
res = client.post("/api/score")
|
||||
assert res.status_code == 200
|
||||
assert queue.jobs.get(res.json()["id"]).kind == "score"
|
||||
|
||||
|
||||
# ----------------------------------------------------------- applications
|
||||
|
||||
def test_save_creates_an_application_with_a_saved_event(
|
||||
|
||||
Reference in new issue
Block a user