Compare commits
70
Commits
97c3046f53
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8d90054b09 | ||
|
|
32d95857c7 | ||
|
|
6e51134477 | ||
|
|
6513e2ab95 | ||
|
|
14793f4707 | ||
|
|
244e57ac29 | ||
|
|
47889ebf66 | ||
|
|
a1f281a23a | ||
|
|
5fe5e00735 | ||
|
|
83e965dfeb | ||
|
|
920457d326 | ||
|
|
2262cda8f1 | ||
|
|
1913101061 | ||
|
|
849adb197d | ||
|
|
136047023e | ||
|
|
911dfe8584 | ||
|
|
8b875b377b | ||
|
|
c6730062fd | ||
|
|
49929372c7 | ||
|
|
f141cffc3c | ||
|
|
02ea191b53 | ||
|
|
1976f6a353 | ||
|
|
2159c7925f | ||
|
|
0c4f3c9bbe | ||
|
|
38c31c830b | ||
|
|
4a9bcb2b1c | ||
|
|
e61185e4b9 | ||
|
|
e8157d9509 | ||
|
|
b4774f1da9 | ||
|
|
8f774b5147 | ||
|
|
d2f18cb3c7 | ||
|
|
62cf1c738c | ||
|
|
c3c527dc6c | ||
|
|
c4c17fb019 | ||
|
|
6b12d10bee | ||
|
|
9da9bd7942 | ||
|
|
7f37270eb3 | ||
|
|
af1efd4fd6 | ||
|
|
937573e048 | ||
|
|
408c5ed73e | ||
|
|
8c3c51dd9f | ||
|
|
9907816cc0 | ||
|
|
80b8416579 | ||
|
|
56a0bb8558 | ||
|
|
9b120a3b94 | ||
|
|
c3bc2cd7f3 | ||
|
|
85aed0807b | ||
|
|
19fb73caa0 | ||
|
|
0b92f19f4d | ||
|
|
b64d77d224 | ||
|
|
1b24de6980 | ||
|
|
caed82d416 | ||
|
|
92ad8c2c1b | ||
|
|
8be59a7ee4 | ||
|
|
2c2b9b6bee | ||
|
|
7ac63dea7c | ||
|
|
3388b97cf4 | ||
|
|
b6d0a54f39 | ||
|
|
1127d06f03 | ||
|
|
d80f26a930 | ||
|
|
1f444aa077 | ||
|
|
1b5a38b61c | ||
|
|
6d82287402 | ||
|
|
463e53ac32 | ||
|
|
b6e27fcb26 | ||
|
|
744d14160d | ||
|
|
6cad9713ac | ||
|
|
a0cb8c6a71 | ||
|
|
9b70f64a20 | ||
|
|
8f37f51254 |
No files matched your search
@@ -12,3 +12,5 @@
|
|||||||
# PowerShell reads LF happily; what it cannot read is a non-ASCII byte, which
|
# PowerShell reads LF happily; what it cannot read is a non-ASCII byte, which
|
||||||
# is a different rule and lives in `tests/test_packaging.py`.
|
# is a different rule and lives in `tests/test_packaging.py`.
|
||||||
deploy/** text eol=lf
|
deploy/** text eol=lf
|
||||||
|
# cmd.exe misparses LF-only batch files, so the double-click launcher stays CRLF.
|
||||||
|
*.cmd text eol=crlf
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<component name="ProjectRunConfigurationManager">
|
|
||||||
<configuration default="false" name="pull" type="ShConfigurationType">
|
|
||||||
<option name="SCRIPT_TEXT" value="wsl.exe -d claudebox --cd /home/claude/cboxcode/jobboard -- env PYTHONUNBUFFERED=1 /home/claude/.local/bin/uv run app job pull" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
|
|
||||||
<option name="SCRIPT_PATH" value="" />
|
|
||||||
<option name="SCRIPT_OPTIONS" value="" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
|
|
||||||
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
|
|
||||||
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
|
|
||||||
<option name="INTERPRETER_PATH" value="" />
|
|
||||||
<option name="INTERPRETER_OPTIONS" value="" />
|
|
||||||
<option name="EXECUTE_IN_TERMINAL" value="true" />
|
|
||||||
<option name="EXECUTE_SCRIPT_FILE" value="false" />
|
|
||||||
<envs />
|
|
||||||
<method v="2" />
|
|
||||||
</configuration>
|
|
||||||
</component>
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<component name="ProjectRunConfigurationManager">
|
|
||||||
<configuration default="false" name="pytest" type="ShConfigurationType">
|
|
||||||
<option name="SCRIPT_TEXT" value="wsl.exe -d claudebox --cd /home/claude/cboxcode/jobboard -- env PYTHONUNBUFFERED=1 /home/claude/.local/bin/uv run pytest -q" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
|
|
||||||
<option name="SCRIPT_PATH" value="" />
|
|
||||||
<option name="SCRIPT_OPTIONS" value="" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
|
|
||||||
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
|
|
||||||
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
|
|
||||||
<option name="INTERPRETER_PATH" value="" />
|
|
||||||
<option name="INTERPRETER_OPTIONS" value="" />
|
|
||||||
<option name="EXECUTE_IN_TERMINAL" value="true" />
|
|
||||||
<option name="EXECUTE_SCRIPT_FILE" value="false" />
|
|
||||||
<envs />
|
|
||||||
<method v="2" />
|
|
||||||
</configuration>
|
|
||||||
</component>
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<component name="ProjectRunConfigurationManager">
|
|
||||||
<configuration default="false" name="seed companies" type="ShConfigurationType">
|
|
||||||
<option name="SCRIPT_TEXT" value="wsl.exe -d claudebox --cd /home/claude/cboxcode/jobboard -- env PYTHONUNBUFFERED=1 /home/claude/.local/bin/uv run app companies seed" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
|
|
||||||
<option name="SCRIPT_PATH" value="" />
|
|
||||||
<option name="SCRIPT_OPTIONS" value="" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
|
|
||||||
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
|
|
||||||
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
|
|
||||||
<option name="INTERPRETER_PATH" value="" />
|
|
||||||
<option name="INTERPRETER_OPTIONS" value="" />
|
|
||||||
<option name="EXECUTE_IN_TERMINAL" value="true" />
|
|
||||||
<option name="EXECUTE_SCRIPT_FILE" value="false" />
|
|
||||||
<envs />
|
|
||||||
<method v="2" />
|
|
||||||
</configuration>
|
|
||||||
</component>
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<component name="ProjectRunConfigurationManager">
|
|
||||||
<configuration default="false" name="serve" type="ShConfigurationType">
|
|
||||||
<option name="SCRIPT_TEXT" value="wsl.exe -d claudebox --cd /home/claude/cboxcode/jobboard -- env PYTHONUNBUFFERED=1 /home/claude/.local/bin/uv run app serve --reload" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
|
|
||||||
<option name="SCRIPT_PATH" value="" />
|
|
||||||
<option name="SCRIPT_OPTIONS" value="" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
|
|
||||||
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
|
|
||||||
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
|
|
||||||
<option name="INTERPRETER_PATH" value="" />
|
|
||||||
<option name="INTERPRETER_OPTIONS" value="" />
|
|
||||||
<option name="EXECUTE_IN_TERMINAL" value="true" />
|
|
||||||
<option name="EXECUTE_SCRIPT_FILE" value="false" />
|
|
||||||
<envs />
|
|
||||||
<method v="2" />
|
|
||||||
</configuration>
|
|
||||||
</component>
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<component name="ProjectRunConfigurationManager">
|
|
||||||
<configuration default="false" name="worker" type="ShConfigurationType">
|
|
||||||
<option name="SCRIPT_TEXT" value="wsl.exe -d claudebox --cd /home/claude/cboxcode/jobboard -- env PYTHONUNBUFFERED=1 /home/claude/.local/bin/uv run app worker" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
|
|
||||||
<option name="SCRIPT_PATH" value="" />
|
|
||||||
<option name="SCRIPT_OPTIONS" value="" />
|
|
||||||
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
|
|
||||||
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$" />
|
|
||||||
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
|
|
||||||
<option name="INTERPRETER_PATH" value="" />
|
|
||||||
<option name="INTERPRETER_OPTIONS" value="" />
|
|
||||||
<option name="EXECUTE_IN_TERMINAL" value="true" />
|
|
||||||
<option name="EXECUTE_SCRIPT_FILE" value="false" />
|
|
||||||
<envs />
|
|
||||||
<method v="2" />
|
|
||||||
</configuration>
|
|
||||||
</component>
|
|
||||||
@@ -31,11 +31,11 @@ The package is `app`, following the boilerplate: the deploy kit, `APP_CONFIG`, `
|
|||||||
|
|
||||||
# Rule:
|
# Rule:
|
||||||
|
|
||||||
`pipeline/`, `auth/` and `deploy/` are downstream copies of `../boilerplate`, `desk/` of `../news`. They are refreshed with `git archive`, and a fix lands upstream first, otherwise the next refresh reverts it.
|
`pipeline/`, `auth/`, `llm/` and `deploy/` are downstream copies of `../boilerplate`, taken whole with `git archive`; the generic modules of `desk/` (`modeltask`, `notices`, `config`, `tools/*`) are copies of `../news`'s, taken file by file; `desk/sources/*` and `desk/models.py` are roledesk's own. A fix to a copy lands upstream first, otherwise the next refresh reverts it. The drift check is `diff -rq --strip-trailing-cr -x __pycache__ <folder> ../boilerplate/<folder>`.
|
||||||
|
|
||||||
# Rule:
|
# Rule:
|
||||||
|
|
||||||
Phone and UI patterns come from `../news` (`newsdesk/web/templates/index.html`, `newsdesk/web/static/app.css`) and `../health` (`webapp/`), which went through the phone fixes already; the boilerplate has none. Copy their fragments, do not write new ones.
|
Phone and UI patterns come from `../news` (`frontend/src/app.css`, a React page since October 2026) and `../health` (`webapp/static/app.css`), which went through the phone fixes already; the boilerplate's are in its `frontend/src/app.css` and its Kit tab. roledesk's page is Jinja templates and plain JS: copy their CSS fragments and layout rules, do not write new ones, and do not port components.
|
||||||
|
|
||||||
# Rule:
|
# Rule:
|
||||||
|
|
||||||
@@ -43,4 +43,4 @@ A larger rework is planned first, as a doc in `docs/` committed on main with wor
|
|||||||
|
|
||||||
# Rule:
|
# Rule:
|
||||||
|
|
||||||
`docs/architecture.md` is the owner's document: the target design of the flow from board to feed and its key decisions, with no description of today's code. What is built is tested against it. Change it only when the owner makes a key decision; the way there goes into a plan, the detailed reasoning into `decisions.md`.
|
`documentation/` holds the owner's ADRs (`ADR Fetch prep and score.md`, `ADR Plugins.md`): the target design and its key decisions, with no description of today's code. Each stands alone and may refer only to the other ADRs, never to `docs/` or the code's files. What is built is tested against them. Change them only when the owner makes a key decision; the way there goes into a plan in `docs/`, the detailed reasoning into `decisions.md`.
|
||||||
@@ -13,11 +13,11 @@ uv run app job pull # read every board and search, map and fold, embe
|
|||||||
uv run app serve # http://127.0.0.1:8790
|
uv run app serve # http://127.0.0.1:8790
|
||||||
```
|
```
|
||||||
|
|
||||||
`uv run app worker` drains the queue without the web app, and `uv run pytest` runs the app's tests and the copied modules' own. `node --test tests/js/*.test.mjs` runs the chip input's. `config.yaml` is the committed default, with one starter agent and four Copenhagen searches on Jobindex; `config.example.yaml` documents the keys, among them the `extract:` and `summarise:` blocks, which set the default for the two model stages; the switch under Settings, Sources, Runs overrides it without a restart, and `app stages` does the same from the command line.
|
`uv run app worker` drains the queue without the web app, and `uv run pytest` runs the app's tests and the copied modules' own. `node --test tests/js/*.test.mjs` runs the tests of the chip input and the remote chips. `config.yaml` is the committed default, with one starter agent and four Copenhagen searches on Jobindex; `config.example.yaml` documents the keys. Its `extract:`, `summarise:` and `judge:` blocks set the default for the three model stages, and the `fetch_text:` block that of the stage reading role pages. The switches under Settings, Sources, Runs override them without a restart, and `app stages` does the same from the command line.
|
||||||
|
|
||||||
## Sources: job boards and company boards
|
## Sources: job boards and company boards
|
||||||
|
|
||||||
The Sources sub-tab does all of this; the same actions exist on the command line:
|
The Sources page under Settings does all of this; the same actions exist on the command line:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv run app searches add jobindex "Copenhagen golang" --q "golang go-udvikler" --area 1
|
uv run app searches add jobindex "Copenhagen golang" --q "golang go-udvikler" --area 1
|
||||||
@@ -40,11 +40,11 @@ uv run app agents list # the agents and their pr
|
|||||||
uv run app prep unmapped # the host words no table maps; prep unlinked lists the place texts no place resolves
|
uv run app prep unmapped # the host words no table maps; prep unlinked lists the place texts no place resolves
|
||||||
```
|
```
|
||||||
|
|
||||||
The seed is a hand-picked list of European tech employers, not a list of boards: a company whose careers site runs on something the app cannot read (its own pages, a host with no module, a site whose robots.txt disallows reading) is still a row, with its careers page and the reason, shown on the Watchlist tab for checking by hand and under Companies as "not readable yet", so the gap is visible and one module at a time closes it. A row where nothing was found reads "board not found"; the "Not readable yet" filter shows both. Each row has a Set board control (Change once it has one) to attach, replace or detach its board by hand. Replacing or detaching closes the old board's open roles, and asks first when there are open roles to lose; a board is read once before it is kept, and one that lists no roles yet is kept too. Setting a board queues a pull right after, so its roles are read by the one already running or the very next one, rather than waiting for the schedule. `companies discover` reads a company's site for a link into one of the boards it knows, probes the hosts with a cheap per-slug endpoint only when no page names any host, and with `--add` sets the board of a new or existing company, or records the host it found when nothing readable was. A probe hit is a guess by name: check the roles it lists before trusting it for a common name.
|
The seed is a hand-picked list of European tech employers, not a list of boards. A company whose careers site the app cannot read is still a row, with its careers page and the reason, shown on the Watchlist tab and under Companies as "not readable yet". A row's Set board control (Change once it has a board) attaches, replaces or detaches its board, which is read once before it is kept; replacing or detaching closes the old board's open roles, and so does switching a board or a search off. `companies discover` reads a company's site for a link into one of the hosts the app knows, probes the hosts only when no page names one, and with `--add` sets the board of a new or existing company. A probe hit is a guess by name: check the roles it lists before trusting it for a common name.
|
||||||
|
|
||||||
`app stages` shows or switches extract, summarise, judge and fetch text, `app seniority` shows per host how often the stated level equals the model's, `app dupes` recomputes which open roles are copies of one posting, and `app prep` maps the host words and folds the copies without a pull.
|
`app stages` shows or switches extract, summarise, judge and fetch text, `app seniority` shows per host how often the stated level equals the model's, `app dupes` recomputes which open roles are copies of one posting, and `app prep` maps the host words and folds the copies without a pull.
|
||||||
|
|
||||||
A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. A Platsbanken query is free text and `--region` the id of a län (`CaRE_1nn_cSU` is Skåne). An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. A freehire query is free text over about 50 hosts whose words are OR over title, company, text and place, so it is noisy: `--country` takes comma-separated ISO codes (`dk,se`) and `--remote` a work mode. A Jobnet query is a loose match without phrases and `--region` one of its five regions (`HovedstadenOgBornholm`, `Midtjylland`, `Syddanmark`, `OevrigeSjaelland`, `Nordjylland`). An Akademikernes Jobbank query is OR words, `--region` its `amt` id (2 is Storkøbenhavn) and `--remote` `helt` or `delvist`. A freehire role whose employer is a board roledesk reads is a copy of that board's role and is hidden behind it. `searches add` runs the search once and saves it only if it finds something. Switching a board or search off closes its open roles, and asks first when there are open roles to lose; switching it back on, or saving a search, queues a pull right after, so the ones it still lists reopen without waiting for the schedule. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys.
|
A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. A Platsbanken query is free text and `--region` the id of a län (`CaRE_1nn_cSU` is Skåne). An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. A freehire query is free text over about 50 hosts whose words are OR over title, company, text and place, so it is noisy: `--country` takes comma-separated ISO codes (`dk,se`) and `--remote` a work mode. A Jobnet query requires every word and has no phrases, so a search is one term; `--region` is one of its five regions (`HovedstadenOgBornholm`, `Midtjylland`, `Syddanmark`, `OevrigeSjaelland`, `Nordjylland`). An Akademikernes Jobbank query is OR words, `--region` its `amt` id (2 is Storkøbenhavn) and `--remote` `helt` or `delvist`. `searches add` runs the search once and saves it only if it finds something. The `embeddings` extra is for the embeddings behind prompt closeness and CV closeness, `agent` serves extract, summarise and judge, and `passkeys` is for the Account tab's passkeys.
|
||||||
|
|
||||||
## The tabs
|
## The tabs
|
||||||
|
|
||||||
@@ -52,7 +52,7 @@ Three tabs, and a cog for settings.
|
|||||||
|
|
||||||
| | |
|
| | |
|
||||||
|---|---|
|
|---|---|
|
||||||
| **Roles** | newsdesk's feed: a filter rail (Viewing as, Show read, Agent match or Newest, first seen, tags, and under more filters company and place chips, seniority from, to and track, function, remote, pay and votes), the open roles with company, place, age, tags and a summary, votes (▲ like ● neither way ▼ pass) and a Save button. A title opens the reader beside the list; on a phone it takes the screen. `j`/`k` move, `Enter` opens, `/` searches |
|
| **Roles** | newsdesk's feed: a filter rail (Viewing as, saved filters, Show read, the order (What I'm looking for, What fits me or Newest), Posted, tags, and under more filters company and place chips, seniority from, to and track, function, remote, pay, votes, Closing soon and Show gated), and the open roles that pass the agent's fetch gate, with company, place, age, tags and a summary, votes (▲ like ● neither way ▼ pass) and a Save button (★). What fits me needs a CV; an agent without one is shown What I'm looking for. A title opens the reader beside the list; on a phone it takes the screen. `j`/`k` move, `Enter` opens, `/` searches |
|
||||||
| **Applications** | the saved roles and their status, each change recorded as an event |
|
| **Applications** | the saved roles and their status, each change recorded as an event |
|
||||||
| **Watchlist** | the companies whose board the app cannot read, for going through their careers pages by hand: Yes or No marks whether one is worth checking (a filter shows the unrated, the yes, the no or all), the careers page opens in a new tab and notes when, and a missing or wrong link is set in the row. A company whose board is failing is listed too. Separate from the affinity a search agent has |
|
| **Watchlist** | the companies whose board the app cannot read, for going through their careers pages by hand: Yes or No marks whether one is worth checking (a filter shows the unrated, the yes, the no or all), the careers page opens in a new tab and notes when, and a missing or wrong link is set in the row. A company whose board is failing is listed too. Separate from the affinity a search agent has |
|
||||||
|
|
||||||
@@ -60,19 +60,20 @@ The cog opens `/settings`: a rail of pages on the left, the page beside it, stic
|
|||||||
|
|
||||||
| | |
|
| | |
|
||||||
|---|---|
|
|---|---|
|
||||||
| **Search agents** | a card per agent, newest first: name, an On/Off switch, its prompts, its rules in words, and its weights. Edit turns the card into the form in place (name, prompts as rows, a Where block for the location rules, the other require/any/exclude rules as rows, fetch terms, weights and limits); New agent puts a blank card on top. Saving or switching on rescores; deleting one takes its votes, liked companies and scores with it |
|
| **Search agents** | a card per agent, newest first: name, an On/Off switch, its prompts, its fetch gate in words, its weights, its analytics filter and its summarise size. Edit turns the card into the form in place; New agent puts a blank card on top. The form is name, prompts, the fetch gate in four parts (World, Location, Role exclusion, Role fetch), CV, languages, likes and dislikes, the analytics filter, the want feed score weights, the CV match weights and the summarise size; other rules are under Advanced. Saving or switching on rescores; deleting one takes its votes, liked companies and scores with it |
|
||||||
| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex, Adzuna, Platsbanken, freehire, Jobnet or Akademikernes Jobbank: new, try, save, edit, remove), company boards (every company, its board or the host it cannot be read from, its open roles and its affinity for the picked agent, what the pull asks its board for, as a reach line such as "follows agents: DK, SE, remote · all roles", the board's size with a Recheck, the All countries and All roles overrides, following a company, setting or changing a company's board), runs (Run the pull now, the switches of extract, summarise, judge and fetch text, what is running, the schedule, and the Problems log) and unmapped (the place texts no place resolves and the host words no table maps, to copy) |
|
| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex, Adzuna, Platsbanken, freehire, Jobnet or Akademikernes Jobbank: new, try, save, edit, remove), company boards (every company, its board or the host it cannot be read from, its open roles and its affinity for the picked agent, what a pull asks its board for and what its read cannot do, the board's size with a Recheck, the All locations and All roles switches, following a company, setting or changing a company's board), runs (Run the pull now, the switches of extract, summarise, judge and fetch text, what is running, the schedule, and the Problems log) and unmapped (the place texts no place resolves and the host words no table maps, to copy) |
|
||||||
| **Account** | the password, sessions and passkeys. Open on loopback, locked on anything else |
|
| **Account** | the password, sessions and passkeys. Open on loopback, locked on anything else |
|
||||||
|
|
||||||
Likes, hidden companies and votes belong to a search agent, not to the account: two agents rank the same roles differently, and one may hide a company the other favours. An agent also learns from votes: its `taste` weight scores a role by how close it is to the roles the agent liked minus the ones it passed.
|
Likes, hidden companies and votes belong to a search agent, not to the account: two agents rank the same roles differently, and one may hide a company the other favours. An agent also learns from votes: its job like weight (`taste` in the config) scores a role by how close it is to the roles the agent liked, away from the ones it passed.
|
||||||
|
|
||||||
The reader, `/roles/{uid}`, shows the full posting: the same vote and Save controls, the company's own affinity, the tags and the summary when there is one. Opening a role marks it read, and so does a vote. A role whose text is a teaser or missing (a search role, or a board role whose text fetch text has not read yet) gets a Fetch button that reads the posting page once and stores the text; for a Jobindex role it follows the "Se jobbet" link to the posting itself. A Summarise button asks for a short summary of that one role; the switch under Settings, Sources, Runs (default from `config.yaml`'s `summarise:` block) turns summarising on for the pull itself, bounded to the top of each agent's feed (its `summarise_top`) and the roles liked or saved, so the cost stays flat.
|
The reader, `/roles/{uid}`, shows the full posting: the same vote and Save controls, the company's own affinity, the tags and the summary when there is one. Opening a role marks it read, and so does a vote. A role whose text is a teaser or missing (a search role, or a board role whose page fetch text has not read yet) gets a Fetch button that reads the posting page once and stores the text; for a Jobindex role it follows the "Se jobbet" link to the posting itself. A Summarise button asks for a short summary of that one role. The switch under Settings, Sources, Runs (default from `config.yaml`'s `summarise:` block) turns summarising on for the pull itself, which reads the top of both orders (each agent's `summarise_top`, 30 to start) and the roles liked or saved, so the cost stays flat.
|
||||||
|
|
||||||
## Docs
|
## Docs
|
||||||
|
|
||||||
|
- [`documentation/ADR Fetch prep and score.md`](<documentation/ADR Fetch prep and score.md>) and [`documentation/ADR Plugins.md`](<documentation/ADR Plugins.md>): the owner's target design, for the ranking from the fetch gate to the feed and for what a source plugin delivers.
|
||||||
- [`docs/decisions.md`](docs/decisions.md): what was decided for roledesk and why.
|
- [`docs/decisions.md`](docs/decisions.md): what was decided for roledesk and why.
|
||||||
- [`docs/open.md`](docs/open.md): what is left to do, for whoever picks this up next.
|
- [`docs/open.md`](docs/open.md): what is left to do, for whoever picks this up next.
|
||||||
- [`plugins/README.md`](plugins/README.md): every reader, its limits and the Watchlist; [`docs/filter-survey.md`](docs/filter-survey.md): which source-side filters each host takes.
|
- [`plugins/README.md`](plugins/README.md): every reader, its endpoints and what each states; [`docs/filter-survey.md`](docs/filter-survey.md): the filter parameters each host takes at the source.
|
||||||
- `docs/plan-*.md`: plans in progress, folded into `decisions.md` and deleted when their work lands.
|
- `docs/plan-*.md`: plans in progress, folded into `decisions.md` and deleted when their work lands.
|
||||||
- [`desk/README.md`](desk/README.md), [`pipeline/README.md`](pipeline/README.md), [`auth/README.md`](auth/README.md), [`deploy/README.md`](deploy/README.md): the copied modules, where each comes from and how to refresh it.
|
- [`desk/README.md`](desk/README.md), [`pipeline/README.md`](pipeline/README.md), [`auth/README.md`](auth/README.md), [`deploy/README.md`](deploy/README.md): the copied modules, where each comes from and how to refresh it.
|
||||||
- [`docs/hosting.md`](docs/hosting.md), [`docs/release.md`](docs/release.md), [`docs/server.md`](docs/server.md), [`docs/password-protection.md`](docs/password-protection.md), [`docs/llm-backends.md`](docs/llm-backends.md): the boilerplate's deployment docs.
|
- [`docs/Install Server.md`](<docs/Install Server.md>) and [`docs/Install WSL.md`](<docs/Install WSL.md>): the server and the Windows machine the release runs on, copies of the boilerplate's files; the release steps are in [`deploy/README.md`](deploy/README.md).
|
||||||
+1
-2
@@ -4,7 +4,7 @@ https://elevenlabs.io/careers/6b84bd1f-05cb-4b54-b0db-ff64c84217ae/deployment-st
|
|||||||
|
|
||||||
https://jobs.ashbyhq.com/cohere/2d256112-b336-4539-8133-a0bf7f6698f0?locationId=f15fd0a7-62c8-4e26-a06d-f396a4e21ffb
|
https://jobs.ashbyhq.com/cohere/2d256112-b336-4539-8133-a0bf7f6698f0?locationId=f15fd0a7-62c8-4e26-a06d-f396a4e21ffb
|
||||||
|
|
||||||
https://jobs.ashbyhq.com/mistral.ai/2c1ff089-db74-467c-b17d-608713e43699?departmentId=cdcd26c0-0c1e-4aff-91f7-5f6306b726b3
|
|
||||||
|
|
||||||
https://jobs.ashbyhq.com/Sierra/44120ba5-4cf7-48a1-86af-7981c41c179e
|
https://jobs.ashbyhq.com/Sierra/44120ba5-4cf7-48a1-86af-7981c41c179e
|
||||||
|
|
||||||
@@ -274,7 +274,6 @@ I need a mailbox and an app password from you.
|
|||||||
|
|
||||||
- Ashby marks hybrid roles as remote.
|
- Ashby marks hybrid roles as remote.
|
||||||
- Lever keeps only the intro paragraph.
|
- Lever keeps only the intro paragraph.
|
||||||
- Roles with a place but no country fail the gates (Zeiss "Oberkochen; Jena").
|
|
||||||
- The top search box doesn't match `Orsted` or `Skane` without the special letters.
|
- The top search box doesn't match `Orsted` or `Skane` without the special letters.
|
||||||
|
|
||||||
**6. Housekeeping.**
|
**6. Housekeeping.**
|
||||||
|
|||||||
+2
-1
@@ -1,12 +1,13 @@
|
|||||||
"""The command line.
|
"""The command line.
|
||||||
|
|
||||||
app serve [--host H] [--port P] [--reload]
|
app serve [--host H] [--port P] [--reload]
|
||||||
app job pull|extract|embed|score [--payload JSON]
|
app job pull|fetch_text|prep|embed|score|extract|judge|summarise [--payload JSON]
|
||||||
app worker [--workers http=8,cpu=1] [--until-idle]
|
app worker [--workers http=8,cpu=1] [--until-idle]
|
||||||
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
|
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
|
||||||
app searches add|list|remove
|
app searches add|list|remove
|
||||||
app agents list|prompt
|
app agents list|prompt
|
||||||
app stages [extract|summarise|judge|fetch_text on|off]
|
app stages [extract|summarise|judge|fetch_text on|off]
|
||||||
|
app seniority
|
||||||
app dupes
|
app dupes
|
||||||
app prep [unmapped|unlinked] [--remap]
|
app prep [unmapped|unlinked] [--remap]
|
||||||
app db init
|
app db init
|
||||||
|
|||||||
+3
-3
@@ -55,8 +55,8 @@ class ExtractConfig(ModelTaskConfig):
|
|||||||
class SummariseConfig(ModelTaskConfig):
|
class SummariseConfig(ModelTaskConfig):
|
||||||
"""About 60 words on a role, for the list and the read page. See
|
"""About 60 words on a role, for the list and the read page. See
|
||||||
app/jobs/summarise.py. Off by default like extract; on, a pull summarises
|
app/jobs/summarise.py. Off by default like extract; on, a pull summarises
|
||||||
the best `summarise_top` final scores of each enabled agent (its
|
the top `summarise_top` (its `limits`) of each enabled agent's want order
|
||||||
`limits`), so the cost is bounded whatever the backlog, and every role
|
and of its match order, so the cost is bounded whatever the backlog, and every role
|
||||||
that is saved or liked, whatever its rank. The read page's Summarise button asks for one
|
that is saved or liked, whatever its rank. The read page's Summarise button asks for one
|
||||||
role whether this is on or not."""
|
role whether this is on or not."""
|
||||||
|
|
||||||
@@ -70,7 +70,7 @@ class SummariseConfig(ModelTaskConfig):
|
|||||||
class JudgeConfig(ModelTaskConfig):
|
class JudgeConfig(ModelTaskConfig):
|
||||||
"""Each search agent's cut read against its CV, as three numbers and a
|
"""Each search agent's cut read against its CV, as three numbers and a
|
||||||
total. See app/jobs/judge.py. Off by default like extract; on, a pull
|
total. See app/jobs/judge.py. Off by default like extract; on, a pull
|
||||||
judges the cut of each enabled agent with a CV and a prompt, and only
|
judges the cut of each enabled agent with a CV, and only
|
||||||
the roles whose judgement is missing or was read from another CV or
|
the roles whose judgement is missing or was read from another CV or
|
||||||
text."""
|
text."""
|
||||||
|
|
||||||
|
|||||||
+23
-4
@@ -82,6 +82,8 @@ class Copy:
|
|||||||
text: str = ""
|
text: str = ""
|
||||||
#: The link that opens this row on its host.
|
#: The link that opens this row on its host.
|
||||||
url: str = ""
|
url: str = ""
|
||||||
|
#: Whether the row is a remote role (`roles.remote` is 'remote').
|
||||||
|
remote: bool = False
|
||||||
|
|
||||||
|
|
||||||
def _country(c: Copy) -> str:
|
def _country(c: Copy) -> str:
|
||||||
@@ -109,6 +111,12 @@ def same_posting(a: Copy, b: Copy) -> bool:
|
|||||||
return not ta or not tb or bool(ta & tb)
|
return not ta or not tb or bool(ta & tb)
|
||||||
|
|
||||||
|
|
||||||
|
def _same_country(a: Copy, b: Copy) -> bool:
|
||||||
|
"""Whether two rows can be in one country: equal, or either unknown."""
|
||||||
|
ca, cb = _country(a), _country(b)
|
||||||
|
return not ca or not cb or ca == cb
|
||||||
|
|
||||||
|
|
||||||
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
||||||
"""The sort key that picks the kept copy of a group, smallest first: a
|
"""The sort key that picks the kept copy of a group, smallest first: a
|
||||||
board over a search provider, then one with a description, then the
|
board over a search provider, then one with a description, then the
|
||||||
@@ -191,9 +199,19 @@ def fold(rows: Iterable[Copy]) -> Folded:
|
|||||||
|
|
||||||
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
Within a `(company_key, title_key)` the kept rows of `assign` are grouped
|
||||||
again, in `keep_order`: a row joins the first group whose members are all
|
again, in `keep_order`: a row joins the first group whose members are all
|
||||||
of its host and `similar` to it. Only a row with a link can be folded, as
|
of its host, in its country (equal, or either unknown, as `same_posting`
|
||||||
its link is what the kept role keeps. Every row of a merged group points
|
reads it), `similar` to it and remote as it is or not. Only a row with
|
||||||
at its first row, the rows that pointed at a folded one included."""
|
a link can be folded, as its link is what the kept role keeps. Every row
|
||||||
|
of a merged group points at its first row, the rows that pointed at a
|
||||||
|
folded one included.
|
||||||
|
|
||||||
|
One country, as ADR Fetch prep and score 1.4 has it: a job posted once
|
||||||
|
per office in several countries is several roles, one per country, which
|
||||||
|
keeps a different opening with an alike text (a federal sales role in
|
||||||
|
the US and one in Australia) from merging. A remote listing and an office listing stay two roles: the kept role
|
||||||
|
takes a copy's places and not its remote, so the fetch gate, which reads
|
||||||
|
a remote role by its area, would reject the remote listing with an
|
||||||
|
office role elsewhere, and the office one with a remote role elsewhere."""
|
||||||
rows = list(rows)
|
rows = list(rows)
|
||||||
result = assign(rows)
|
result = assign(rows)
|
||||||
by_uid = {r.uid: r for r in rows}
|
by_uid = {r.uid: r for r in rows}
|
||||||
@@ -207,7 +225,8 @@ def fold(rows: Iterable[Copy]) -> Folded:
|
|||||||
groups: list[list[Copy]] = []
|
groups: list[list[Copy]] = []
|
||||||
for row in sorted(part, key=keep_order):
|
for row in sorted(part, key=keep_order):
|
||||||
for group in groups:
|
for group in groups:
|
||||||
if all(m.host == row.host and similar(m.text, row.text) for m in group):
|
if all(m.host == row.host and m.remote == row.remote
|
||||||
|
and _same_country(m, row) and similar(m.text, row.text) for m in group):
|
||||||
group.append(row)
|
group.append(row)
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
|
|||||||
+13
-7
@@ -3,14 +3,20 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
|||||||
|
|
||||||
pull.py read every board and search and store the roles they yield
|
pull.py read every board and search and store the roles they yield
|
||||||
prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold
|
prep.py the `prep` stage: `app.prep` over what a pull stored, then the fold
|
||||||
|
embed.py one vector per open role, for prompt closeness and CV closeness
|
||||||
|
score.py every open role scored for every enabled search agent, and each
|
||||||
|
agent's cut
|
||||||
|
fetch_text.py the roles in the cut that came without text, read on their own
|
||||||
|
pages, with no model
|
||||||
extract.py title and description into the gate columns, on Haiku
|
extract.py title and description into the gate columns, on Haiku
|
||||||
embed.py one vector per open role, for fit
|
judge.py each agent's cut against its CV: skills, experience, career
|
||||||
score.py every open role scored for every enabled search agent
|
summarise.py about 60 words on the top of both orders and the roles liked or
|
||||||
summarise.py about 60 words on each of the best roles, on Haiku
|
saved, on Haiku
|
||||||
judge.py each agent's best roles against its CV: skills, experience, career
|
cascade.py pull → prep → embed → score → fetch_text → prep → embed → score
|
||||||
cascade.py pull → prep → embed → score → ... → summarise → judge, what `TASKS`
|
→ extract → score → judge → summarise, what `TASKS` offers, the
|
||||||
offers, the `pull` job a schedule fires, and that schedule
|
`pull` and `fetch_text` jobs that start it, and the default schedule
|
||||||
stages.py whether extract, summarise and judge run: the switch, else config.yaml
|
stages.py whether extract, summarise, judge and fetch text run: the switch,
|
||||||
|
else config.yaml
|
||||||
sources.py what a pull reads, changed by hand: boards and searches
|
sources.py what a pull reads, changed by hand: boards and searches
|
||||||
added, switched on or off
|
added, switched on or off
|
||||||
discover.py which ATS a company's careers site runs on, from its pages
|
discover.py which ATS a company's careers site runs on, from its pages
|
||||||
|
|||||||
+3
-2
@@ -22,8 +22,9 @@ it filled none, `step` passes over all three. Extract reads the cut; the
|
|||||||
score after it applies the fetch gate and the cut again with the columns
|
score after it applies the fetch gate and the cut again with the columns
|
||||||
extract filled, so a role can drop out or enter (one that enters is
|
extract filled, so a role can drop out or enter (one that enters is
|
||||||
extracted on the next pull). Judge reads each agent's cut as recomputed,
|
extracted on the next pull). Judge reads each agent's cut as recomputed,
|
||||||
against its CV. Summarise runs last, on the top of each agent's feed by
|
against its CV. Summarise runs last, on the top of each agent's feed in
|
||||||
final score and the roles liked or saved: it is no part of any score.
|
the want order and in the match order, and the roles liked or saved: it is
|
||||||
|
no part of any score.
|
||||||
|
|
||||||
A batch that failed, or a stage somebody killed, does not stop the chain: the
|
A batch that failed, or a stage somebody killed, does not stop the chain: the
|
||||||
work comes round on the next pull, and a rescore of what did arrive is worth
|
work comes round on the next pull, and a rescore of what did arrive is worth
|
||||||
|
|||||||
+2
-1
@@ -9,7 +9,8 @@ refused batch, and failing with a notice when no model is reachable are
|
|||||||
|
|
||||||
`extract` is also the afterburner for the level and the remote area. A role
|
`extract` is also the afterburner for the level and the remote area. A role
|
||||||
is waiting while it is open, has not been given up on, has never been
|
is waiting while it is open, has not been given up on, has never been
|
||||||
answered (`extracted_at IS NULL`), has no model level (`seniority_model`
|
answered or was answered before a better text came (`extracted_at IS
|
||||||
|
NULL`, which `Store.upsert_role` sets again), has no model level (`seniority_model`
|
||||||
empty), was never asked where a remote hire may live (`remote_where_model`
|
empty), was never asked where a remote hire may live (`remote_where_model`
|
||||||
NULL) or for its languages and deadline (`languages_required` NULL), and is
|
NULL) or for its languages and deadline (`languages_required` NULL), and is
|
||||||
in the cut of an enabled agent (`scores.cut`, see `app.jobs.score`). That
|
in the cut of an enabled agent (`scores.cut`, see `app.jobs.score`). That
|
||||||
|
|||||||
+55
-91
@@ -2,21 +2,19 @@
|
|||||||
|
|
||||||
A `ModelTask` beside `summarise`, after the `/rank` of
|
A `ModelTask` beside `summarise`, after the `/rank` of
|
||||||
github.com/MadsLorentzen/ai-job-search: the model reads a posting against a
|
github.com/MadsLorentzen/ai-job-search: the model reads a posting against a
|
||||||
written profile and scores skills, experience and career 0 to 100, with one
|
written profile and scores skills, experience and career 0 to 10, with one
|
||||||
to three strengths and gaps. The total is `app.models.cv_score`, the three
|
to three strengths and gaps. A model's rating is an estimate, so the scale is
|
||||||
numbers under the agent's `cv_*` weights, worked out when a role is served and
|
coarse. The total is `app.models.cv_score`, the three numbers under the
|
||||||
not stored, so the weights are no part of `input_key`; it is what the feed's
|
agent's `cv_*` weights times 10, so 0 to 100, worked out when a role is served
|
||||||
CV match sorts by. The model also reports a verdict on each of the
|
and not stored, so the weights are no part of `input_key`; it is what the
|
||||||
agent's must-haves (one per line of its must text, numbered M1..Mn in the
|
feed's CV match sorts by. Culture and location are left out: the first is
|
||||||
prompt); it does not apply them. The rule is code's: a "broken" one makes
|
guessed from ad text, the second is what the fetch gate does.
|
||||||
career count as at most `app.models.BROKEN_CAP`, when the role is served. Culture and location are left out: the first
|
|
||||||
is guessed from ad text, the second is what the fetch gate does.
|
|
||||||
|
|
||||||
The work is pairs, `(agent, role)`: an enabled agent with a CV and a prompt,
|
The work is pairs, `(agent, role)`: an enabled agent with a CV, with or
|
||||||
and an open role in its cut (`scores.cut`, see `app.jobs.score`: the best by
|
without a prompt, and an open role in its cut (`scores.cut`, see `app.jobs.score`: the best by
|
||||||
fit, the best by CV closeness that those leave out, and the liked and saved)
|
fit, the best by CV closeness that those leave out, and the liked and saved)
|
||||||
with some text, that has no judgement yet, or one read from another CV,
|
with some text, that has no judgement yet, or one read from another CV,
|
||||||
prompts, like, avoid or must text or role text than today's (`input_key`).
|
prompts, languages, like or avoid text or role text than today's (`input_key`).
|
||||||
The cut is the only selection: the pull judges it after extract and the
|
The cut is the only selection: the pull judges it after extract and the
|
||||||
rescore that follows, so it reads the cut as recomputed. A pair is named by the composite key `"<agent_id>:<uid>"`,
|
rescore that follows, so it reads the cut as recomputed. A pair is named by the composite key `"<agent_id>:<uid>"`,
|
||||||
which is what `desk.modeltask` hands round as a uid, so `desk/` is untouched.
|
which is what `desk.modeltask` hands round as a uid, so `desk/` is untouched.
|
||||||
@@ -49,9 +47,7 @@ from pipeline import NULL, RunContext, Task, job
|
|||||||
from pipeline.task import Batch
|
from pipeline.task import Batch
|
||||||
from app.config import Config
|
from app.config import Config
|
||||||
from app.jobs import stages
|
from app.jobs import stages
|
||||||
from app.models import (
|
from app.models import LANGUAGE_LEVELS, Agent, CvLanguage, _language_name, cv_languages
|
||||||
LANGUAGE_LEVELS, MUST_VERDICTS, Agent, CvLanguage, cv_languages, must_lines,
|
|
||||||
)
|
|
||||||
from app.storage import Store
|
from app.storage import Store
|
||||||
|
|
||||||
#: How much of a posting goes in one item: summarise's 8000, since the
|
#: How much of a posting goes in one item: summarise's 8000, since the
|
||||||
@@ -63,42 +59,39 @@ EXCERPT = 8000
|
|||||||
#: 2: the verdict on each numbered must-have (`musts`), the cap in code.
|
#: 2: the verdict on each numbered must-have (`musts`), the cap in code.
|
||||||
#: 3: what the person likes (`like_text`) beside what they do not, and the
|
#: 3: what the person likes (`like_text`) beside what they do not, and the
|
||||||
#: career rubric reading both.
|
#: career rubric reading both.
|
||||||
FORMAT = 3
|
#: 4: the three numbers 0 to 10 rather than 0 to 100.
|
||||||
|
#: 5: the person's languages, read into skills.
|
||||||
|
#: 6: no must-haves: the profile is the CV, languages, likes and dislikes.
|
||||||
|
FORMAT = 6
|
||||||
|
|
||||||
#: How many strengths and gaps are kept, and how long each may be.
|
#: How many strengths and gaps are kept, and how long each may be.
|
||||||
MOST, LONGEST = 3, 300
|
MOST, LONGEST = 3, 300
|
||||||
|
|
||||||
SYSTEM = """You judge job postings for one person: how well each fits their CV and what they want.
|
SYSTEM = """You judge job postings for one person: how well each fits their CV and what they want.
|
||||||
|
|
||||||
Below the rules is the person: their CV, what they want, what they like and
|
Below the rules is the person: their CV, the languages they speak and how
|
||||||
what they do not like (the work, the workplace, situations), and their
|
well, what they want, what they like and what they do not like (the work,
|
||||||
must-haves and deal-breakers when they gave any. Each item is one posting with a key, its title, company, place
|
the workplace, situations). Each item is one posting with a key, its title,
|
||||||
and text. Answer one judgement per item.
|
company, place and text. Answer one judgement per item.
|
||||||
|
|
||||||
Score three dimensions, each 0 to 100:
|
Score three dimensions, each a whole number from 0 to 10:
|
||||||
- skills: 80-100 the core requirements are the person's primary skills; 60-79
|
- skills: 8-10 the core requirements are the person's primary skills; 6-7
|
||||||
most requirements match, with 1-2 gaps that are learnable; 40-59 a partial
|
most requirements match, with 1-2 gaps that are learnable; 4-5 a partial
|
||||||
match, significant upskilling needed; 0-39 a fundamental mismatch.
|
match, significant upskilling needed; 0-3 a fundamental mismatch. A
|
||||||
|
language the posting requires counts as a requirement: one the person
|
||||||
|
speaks below the level asked, or not at all when their languages are
|
||||||
|
listed, lowers skills and is a gap. A language the posting does not
|
||||||
|
mention changes nothing.
|
||||||
- experience: match on the function and nature of the work performed, not the
|
- experience: match on the function and nature of the work performed, not the
|
||||||
literal job title, and on its level. 80-100 direct experience in the same
|
literal job title, and on its level. 8-10 direct experience in the same
|
||||||
domain and role type; 60-79 related, with transferable skills clear; 40-59
|
domain and role type; 6-7 related, with transferable skills clear; 4-5
|
||||||
adjacent, the person would need to make the case; 0-39 unrelated.
|
adjacent, the person would need to make the case; 0-3 unrelated.
|
||||||
- career: against what the person wants (the prompts), what they like and
|
- career: against what the person wants (the prompts), what they like and
|
||||||
what they do not like. 80-100 strongly
|
what they do not like. 8-10 strongly aligned with their direction, with
|
||||||
aligned with their direction, with clear growth; 60-79 a good role, partly
|
clear growth; 6-7 a good role, partly aligned; 4-5 a decent job that does
|
||||||
aligned; 40-59 a decent job that does not build toward their goals; 0-39 a
|
not build toward their goals; 0-3 a dead end or a step backwards. Judge
|
||||||
dead end or a step backwards. Judge whether the tasks and the setting would
|
whether the tasks and the setting would energise them, not only whether
|
||||||
energise them, not only whether they could do them.
|
they could do them.
|
||||||
|
|
||||||
Their must-haves and deal-breakers are numbered M1, M2 and so on. Give a
|
|
||||||
verdict on every one of them, in `musts`, for each item:
|
|
||||||
- "met": the posting text says it is met.
|
|
||||||
- "broken": only when the posting text clearly contradicts it.
|
|
||||||
- "unclear": the text does not say. Never infer from silence: a posting that
|
|
||||||
is silent on a must-have is "unclear", not "broken".
|
|
||||||
Judge a must-have by its own words, not by how well the role fits otherwise.
|
|
||||||
Answer musts as [] when there are none. The verdicts do not change the three
|
|
||||||
scores, which follow their rubrics; name a broken must-have as a gap.
|
|
||||||
|
|
||||||
Then 1 to 3 strengths and 1 to 3 gaps, each one short sentence grounded in
|
Then 1 to 3 strengths and 1 to 3 gaps, each one short sentence grounded in
|
||||||
the posting text. State the gaps honestly: a prestigious role that fits
|
the posting text. State the gaps honestly: a prestigious role that fits
|
||||||
@@ -123,21 +116,8 @@ SCHEMA: dict[str, Any] = {
|
|||||||
"career": {"type": "integer"},
|
"career": {"type": "integer"},
|
||||||
"strengths": {"type": "array", "items": {"type": "string"}},
|
"strengths": {"type": "array", "items": {"type": "string"}},
|
||||||
"gaps": {"type": "array", "items": {"type": "string"}},
|
"gaps": {"type": "array", "items": {"type": "string"}},
|
||||||
"musts": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"id": {"type": "string"},
|
|
||||||
"verdict": {"type": "string", "enum": list(MUST_VERDICTS)},
|
|
||||||
},
|
},
|
||||||
"required": ["id", "verdict"],
|
"required": ["key", "skills", "experience", "career", "strengths", "gaps"],
|
||||||
"additionalProperties": False,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["key", "skills", "experience", "career", "strengths", "gaps",
|
|
||||||
"musts"],
|
|
||||||
"additionalProperties": False,
|
"additionalProperties": False,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -178,18 +158,19 @@ def _text(row: Any) -> str:
|
|||||||
|
|
||||||
def input_key(agent: Agent, text: str) -> str:
|
def input_key(agent: Agent, text: str) -> str:
|
||||||
"""A hash of everything a judgement reads: the CV, the prompts, the
|
"""A hash of everything a judgement reads: the CV, the prompts, the
|
||||||
like, avoid and must texts, and the role's text as sent. A judgement stored
|
languages, the like and avoid texts, and the role's text as sent. A judgement
|
||||||
under another key is read again."""
|
stored under another key is read again."""
|
||||||
blob = json.dumps([FORMAT, agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
blob = json.dumps([FORMAT, agent.cv_text, [[p.name, p.text] for p in agent.prompts],
|
||||||
agent.like_text, agent.avoid_text, agent.must_text, text], ensure_ascii=False)
|
[[l.language, l.level] for l in agent.cv_languages],
|
||||||
|
agent.like_text, agent.avoid_text, text], ensure_ascii=False)
|
||||||
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
return hashlib.sha1(blob.encode()).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
def _judged(agent: Agent) -> bool:
|
def _judged(agent: Agent) -> bool:
|
||||||
"""Whether `judge` reads for this agent at all: enabled, with a CV and a
|
"""Whether `judge` reads for this agent at all: enabled, with a CV. An
|
||||||
prompt with text (without one nothing of its cut is in by fit)."""
|
agent without a prompt is judged too: its cut is the CV list and the
|
||||||
return agent.enabled and bool(agent.cv_text.strip()) \
|
kept, and the profile goes without `<wants>` (`system`)."""
|
||||||
and any(p.text.strip() for p in agent.prompts)
|
return agent.enabled and bool(agent.cv_text.strip())
|
||||||
|
|
||||||
|
|
||||||
def waiting(store: Store, keys: Sequence[str] | None = None,
|
def waiting(store: Store, keys: Sequence[str] | None = None,
|
||||||
@@ -227,6 +208,9 @@ def pending(store: Store) -> int:
|
|||||||
def system(agent: Agent) -> str:
|
def system(agent: Agent) -> str:
|
||||||
"""The rubric, and the person as this agent describes them."""
|
"""The rubric, and the person as this agent describes them."""
|
||||||
parts = [SYSTEM, "", "The person.", "", f"<cv>\n{agent.cv_text.strip()}\n</cv>"]
|
parts = [SYSTEM, "", "The person.", "", f"<cv>\n{agent.cv_text.strip()}\n</cv>"]
|
||||||
|
if agent.cv_languages:
|
||||||
|
spoken = ", ".join(f"{_language_name(l.language)} {l.level}" for l in agent.cv_languages)
|
||||||
|
parts.append(f"\n<languages title=\"the languages they speak, and how well\">\n{spoken}\n</languages>")
|
||||||
wants = "\n".join(f"- {p.name}: {' '.join(p.text.split())}"
|
wants = "\n".join(f"- {p.name}: {' '.join(p.text.split())}"
|
||||||
for p in agent.prompts if p.text.strip())
|
for p in agent.prompts if p.text.strip())
|
||||||
if wants:
|
if wants:
|
||||||
@@ -237,9 +221,6 @@ def system(agent: Agent) -> str:
|
|||||||
if agent.avoid_text.strip():
|
if agent.avoid_text.strip():
|
||||||
parts.append("\n<dislikes title=\"what they do not like\">\n"
|
parts.append("\n<dislikes title=\"what they do not like\">\n"
|
||||||
f"{agent.avoid_text.strip()}\n</dislikes>")
|
f"{agent.avoid_text.strip()}\n</dislikes>")
|
||||||
if musts := must_lines(agent.must_text):
|
|
||||||
lines = "\n".join(f"M{n}: {line}" for n, line in enumerate(musts, start=1))
|
|
||||||
parts.append(f"\n<must title=\"must-haves and deal-breakers\">\n{lines}\n</must>")
|
|
||||||
return "\n".join(parts)
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
@@ -269,10 +250,16 @@ def build_batch(config: Config, store: Store, limit: int = 5,
|
|||||||
|
|
||||||
# -------------------------------------------------------------------- answer
|
# -------------------------------------------------------------------- answer
|
||||||
|
|
||||||
|
#: The top of the judge's scale.
|
||||||
|
TOP = 10
|
||||||
|
|
||||||
|
|
||||||
def _mark(value: Any) -> int | None:
|
def _mark(value: Any) -> int | None:
|
||||||
|
"""One of the three numbers, rounded and held to 0..`TOP`; None for
|
||||||
|
anything not a number."""
|
||||||
if isinstance(value, bool) or not isinstance(value, int | float):
|
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||||
return None
|
return None
|
||||||
return max(0, min(100, round(value)))
|
return max(0, min(TOP, round(value)))
|
||||||
|
|
||||||
|
|
||||||
def _lines(value: Any) -> list[str]:
|
def _lines(value: Any) -> list[str]:
|
||||||
@@ -281,25 +268,6 @@ def _lines(value: Any) -> list[str]:
|
|||||||
return [line for line in found if line][:MOST]
|
return [line for line in found if line][:MOST]
|
||||||
|
|
||||||
|
|
||||||
def _musts(agent: Agent | None, value: Any) -> list[dict[str, str]]:
|
|
||||||
"""The verdicts on the agent's must-haves as they are stored: one
|
|
||||||
`{id, text, verdict}` per line of its must text, in order. A must-have
|
|
||||||
the model left out, or gave a verdict outside "met", "broken", "unclear",
|
|
||||||
is "unclear": nothing is broken by silence. An id that is not one of the
|
|
||||||
agent's is ignored, the first answer for an id counts."""
|
|
||||||
said: dict[str, str] = {}
|
|
||||||
for one in value if isinstance(value, list) else []:
|
|
||||||
if isinstance(one, dict):
|
|
||||||
said.setdefault(str(one.get("id") or "").strip().upper(),
|
|
||||||
str(one.get("verdict") or "").strip().lower())
|
|
||||||
found = []
|
|
||||||
for n, line in enumerate(must_lines(agent.must_text if agent else ""), start=1):
|
|
||||||
verdict = said.get(f"M{n}", "")
|
|
||||||
found.append({"id": f"M{n}", "text": line,
|
|
||||||
"verdict": verdict if verdict in MUST_VERDICTS else "unclear"})
|
|
||||||
return found
|
|
||||||
|
|
||||||
|
|
||||||
def apply_batch(store: Store, answer: dict[str, Any],
|
def apply_batch(store: Store, answer: dict[str, Any],
|
||||||
batch: dict[str, Any] | None = None,
|
batch: dict[str, Any] | None = None,
|
||||||
config: Config | None = None) -> JudgeResult:
|
config: Config | None = None) -> JudgeResult:
|
||||||
@@ -308,7 +276,6 @@ def apply_batch(store: Store, answer: dict[str, Any],
|
|||||||
all three numbers; the pair stays waiting."""
|
all three numbers; the pair stays waiting."""
|
||||||
keys: dict[str, str] = dict((batch or {}).get("keys") or {})
|
keys: dict[str, str] = dict((batch or {}).get("keys") or {})
|
||||||
stats = JudgeResult()
|
stats = JudgeResult()
|
||||||
agents: dict[int, Agent | None] = {}
|
|
||||||
for one in answer.get("judgements") or []:
|
for one in answer.get("judgements") or []:
|
||||||
key = str(one.get("key") or "")
|
key = str(one.get("key") or "")
|
||||||
pair = pair_of(key)
|
pair = pair_of(key)
|
||||||
@@ -317,11 +284,8 @@ def apply_batch(store: Store, answer: dict[str, Any],
|
|||||||
stats.rejected.append(key or "?")
|
stats.rejected.append(key or "?")
|
||||||
continue
|
continue
|
||||||
skills, experience, career = (int(m) for m in marks if m is not None)
|
skills, experience, career = (int(m) for m in marks if m is not None)
|
||||||
if pair[0] not in agents:
|
|
||||||
agents[pair[0]] = store.agent(pair[0])
|
|
||||||
if not store.set_judgement(pair[0], pair[1], keys.pop(key), skills, experience, career,
|
if not store.set_judgement(pair[0], pair[1], keys.pop(key), skills, experience, career,
|
||||||
_lines(one.get("strengths")), _lines(one.get("gaps")),
|
_lines(one.get("strengths")), _lines(one.get("gaps"))):
|
||||||
_musts(agents[pair[0]], one.get("musts"))):
|
|
||||||
stats.rejected.append(key)
|
stats.rejected.append(key)
|
||||||
continue
|
continue
|
||||||
stats.judged += 1
|
stats.judged += 1
|
||||||
|
|||||||
+17
-3
@@ -10,7 +10,10 @@ both; nothing here decides how a board is read, so the words never disagree
|
|||||||
with the pull as long as they read the keys the pull writes.
|
with the pull as long as they read the keys the pull writes.
|
||||||
|
|
||||||
A word with a number carries it after a colon ("without text: 12"), and the
|
A word with a number carries it after a colon ("without text: 12"), and the
|
||||||
filter matches the phrase before the colon (`WORDS`).
|
filter matches the phrase before the colon (`WORDS`). One word is outside
|
||||||
|
`WORDS`: the listings the last read dropped for want of an id, a title or a
|
||||||
|
link ("3 listings dropped: no title or link", `_runner.valid`), shown on the
|
||||||
|
board's row and not offered by the filter.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -49,7 +52,7 @@ WORDS: dict[str, str] = {
|
|||||||
|
|
||||||
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
||||||
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
||||||
"narrowed", "detailfailed")
|
"narrowed", "detailfailed", "dropped")
|
||||||
|
|
||||||
|
|
||||||
class Texts(NamedTuple):
|
class Texts(NamedTuple):
|
||||||
@@ -84,7 +87,8 @@ def texts(store: Store) -> dict[int, Texts]:
|
|||||||
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
||||||
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
||||||
line: str = "", counts: Texts | None = None) -> list[str]:
|
line: str = "", counts: Texts | None = None) -> list[str]:
|
||||||
"""The limits of one board, in the order of `WORDS`, those that apply.
|
"""The limits of one board, in the order of `WORDS`, those that apply, the
|
||||||
|
rows the last read dropped after "read incomplete".
|
||||||
|
|
||||||
`state` holds the board's keys (`state_of`), `size` its kept size,
|
`state` holds the board's keys (`state_of`), `size` its kept size,
|
||||||
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
||||||
@@ -105,6 +109,8 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
|
|||||||
out.append("failing")
|
out.append("failing")
|
||||||
if state.get(key("ok")) and not state.get(key("complete")):
|
if state.get(key("ok")) and not state.get(key("complete")):
|
||||||
out.append("read incomplete")
|
out.append("read incomplete")
|
||||||
|
if dropped := _count(state.get(key("dropped"))):
|
||||||
|
out.append(f"{dropped} listing{'' if dropped == 1 else 's'} dropped: no title or link")
|
||||||
if state.get(key("swept")) == "1" or key("sweep") in state:
|
if state.get(key("swept")) == "1" or key("sweep") in state:
|
||||||
out.append("swept")
|
out.append("swept")
|
||||||
if state.get(key("filtered")) == "1":
|
if state.get(key("filtered")) == "1":
|
||||||
@@ -130,6 +136,14 @@ def words(module: ModuleType, state: Mapping[str, str], target: str,
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _count(value: str | None) -> int:
|
||||||
|
"""A number kept in state, 0 for none or one that does not parse."""
|
||||||
|
try:
|
||||||
|
return max(0, int(value or 0))
|
||||||
|
except ValueError:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
||||||
"""Whether the reach leaves the board unread by place: its reader cannot
|
"""Whether the reach leaves the board unread by place: its reader cannot
|
||||||
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
||||||
|
|||||||
+44
-32
@@ -37,16 +37,23 @@ not swept clears them.
|
|||||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||||
the roles matching a word, which says nothing of the roles it did not list,
|
the roles matching a word, which says nothing of the roles it did not list,
|
||||||
so it closes by list nothing: a role of such a board closes once the board
|
so it closes by list nothing: a role of such a board closes once the board
|
||||||
has not listed it for `STALE_DAYS`, the rule searches have.
|
has not listed it for `STALE_DAYS`, the rule searches have. The same age
|
||||||
|
rule is the floor under every board (ADR Fetch prep and score 1.3): an
|
||||||
|
incomplete read (a ceiling, a page cap without a sweep) and a filtered read,
|
||||||
|
which spares the roles acted on and the remote ones it could not ask for,
|
||||||
|
close by age what no read has listed for `STALE_DAYS` (`_close_aged`), so no
|
||||||
|
role stays open for good.
|
||||||
|
|
||||||
A search closes the same way when it listed everything that matches (see
|
A search never closes by its list, even one stamped complete because it
|
||||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
read as many results as its provider stated: a search is never a company's
|
||||||
roles only once no search has seen them for `STALE_DAYS`. A board or a
|
whole list, so a role that drops out of its results may still be open. Its
|
||||||
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
|
roles close once no search has listed them for `STALE_DAYS`
|
||||||
`search:failing:<target>`) closes all of its roles, and its next answer
|
(`Store.close_unseen`). A board or a search that has failed on every fetch
|
||||||
reopens what it lists. A target that raises in its batch after its read, a
|
for `STALE_DAYS` (`ats:failing:<target>`, `search:failing:<target>`) closes
|
||||||
row that does not store, is recorded as failing the same way and the batch
|
all of its roles, and its next answer reopens what it lists. A target that
|
||||||
goes on with the next one (`run_batch`).
|
raises in its batch after its read, a row that does not store, is recorded
|
||||||
|
as failing the same way and the batch goes on with the next one
|
||||||
|
(`run_batch`).
|
||||||
|
|
||||||
The settle says what needs a look in one notice (`settled`): the targets
|
The settle says what needs a look in one notice (`settled`): the targets
|
||||||
that did not answer or were read incomplete, and the boards that closed more
|
that did not answer or were read incomplete, and the boards that closed more
|
||||||
@@ -302,11 +309,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
had already parsed stamped — see the module docstring. A board whose
|
had already parsed stamped — see the module docstring. A board whose
|
||||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||||
whole batch job never got to it — is never closed by its list, and a board
|
whole batch job never got to it — is never closed by its list, and a board
|
||||||
that answered incompletely is not either. A search closes by its list only
|
that answered incompletely is not either. A search never closes by its
|
||||||
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
|
list, complete or not: its roles close once no search has listed them
|
||||||
search that has failed on every fetch since `STALE_DAYS` ago closes all
|
for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A
|
||||||
its roles (`failing_boards`). How a read that answered closes is
|
board or a search that has failed on every fetch since `STALE_DAYS` ago
|
||||||
`_close_read`'s.
|
closes all its roles (`failing_boards`). How a board's read that
|
||||||
|
answered closes is `_close_read`'s.
|
||||||
|
|
||||||
The stamps are compared with this run's own marker, the one its batches
|
The stamps are compared with this run's own marker, the one its batches
|
||||||
carried, never with the stored `pull:run`, which a later run may have
|
carried, never with the stored `pull:run`, which a later run may have
|
||||||
@@ -350,18 +358,22 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
unanswered.append(sub.target)
|
unanswered.append(sub.target)
|
||||||
since = failing.get(sub.target, ("", ""))[0]
|
since = failing.get(sub.target, ("", ""))[0]
|
||||||
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
||||||
else:
|
elif source == "search":
|
||||||
if source == "search":
|
# Never a company's whole list, complete or not: a search's
|
||||||
|
# roles close by age alone (`close_unseen`, after the loop).
|
||||||
searched += 1
|
searched += 1
|
||||||
|
continue
|
||||||
else:
|
else:
|
||||||
answered += 1
|
answered += 1
|
||||||
read = _close_read(store, source, sub, stale)
|
read = _close_read(store, source, sub, stale)
|
||||||
if read is None:
|
if read is None:
|
||||||
if source == "ats":
|
|
||||||
incomplete += 1
|
incomplete += 1
|
||||||
if not store.state.get(f"ats:sweep:{sub.target}"):
|
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||||
partial.append(sub.target)
|
partial.append(sub.target)
|
||||||
continue
|
# An incomplete read closes nothing by its list; what no
|
||||||
|
# read has listed for STALE_DAYS still goes.
|
||||||
|
read = _close_aged(store, source, sub.id, json.loads(
|
||||||
|
store.state.get(f"{source}:ids:{sub.target}", "[]")), stale)
|
||||||
shut = read
|
shut = read
|
||||||
closed += shut
|
closed += shut
|
||||||
before = open_before.get(sub.id, 0)
|
before = open_before.get(sub.id, 0)
|
||||||
@@ -380,8 +392,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
|
|
||||||
|
|
||||||
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||||
"""Close what one target's read of this run says has closed, and return
|
"""Close what one board's read of this run says has closed, and return
|
||||||
how many; None when the read was incomplete, which closes nothing.
|
how many; None when the read was incomplete, which closes nothing. A
|
||||||
|
search's read never comes here (see `close_boards`).
|
||||||
|
|
||||||
A read narrowed by terms closes only the roles it has not listed for
|
A read narrowed by terms closes only the roles it has not listed for
|
||||||
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
||||||
@@ -389,8 +402,9 @@ def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
|||||||
clears that record and closes what it did not list, all of it unfiltered
|
clears that record and closes what it did not list, all of it unfiltered
|
||||||
(`Store.mark_closed`), filtered sparing the roles acted on
|
(`Store.mark_closed`), filtered sparing the roles acted on
|
||||||
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
||||||
(`remote_unasked`) spares the remote ones as well, and closes those by
|
(`remote_unasked`) spares the remote ones as well. What a filtered read
|
||||||
age instead."""
|
spares closes by age (`_close_aged`), as on an incomplete read
|
||||||
|
(`close_boards`)."""
|
||||||
state, target = store.state, sub.target
|
state, target = store.state, sub.target
|
||||||
if not state.get(f"{source}:complete:{target}", ""):
|
if not state.get(f"{source}:complete:{target}", ""):
|
||||||
return None
|
return None
|
||||||
@@ -399,8 +413,7 @@ def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
|||||||
return _close_aged(store, source, sub.id, seen_ids, stale)
|
return _close_aged(store, source, sub.id, seen_ids, stale)
|
||||||
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
||||||
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
||||||
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
aged = _close_aged(store, source, sub.id, seen_ids, stale) if is_filtered else 0
|
||||||
if spare_remote else 0)
|
|
||||||
if state.get(f"{source}:swept:{target}", ""):
|
if state.get(f"{source}:swept:{target}", ""):
|
||||||
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
||||||
spare_remote=spare_remote)
|
spare_remote=spare_remote)
|
||||||
@@ -479,20 +492,19 @@ def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription
|
|||||||
|
|
||||||
|
|
||||||
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
||||||
stale: str, *, remote_only: bool = False) -> int:
|
stale: str) -> int:
|
||||||
"""Close the open roles of one board that its read did not list and that
|
"""Close the open roles of one board that its read did not list and that
|
||||||
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
||||||
writes on every listing): the rule for what a read cannot speak for. A
|
writes on every listing): the rule for what a read cannot speak for, a
|
||||||
read narrowed by terms closes all such roles, as a search does; with
|
read narrowed by terms, an incomplete one, or what a filtered read
|
||||||
`remote_only`, a filtered read that could not ask for remote closes only
|
spares. A role acted on is not spared here: three weeks unlisted is the
|
||||||
its remote roles so, sparing those acted on as its list does."""
|
one evidence a filtered or incomplete read gives."""
|
||||||
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
return conn.execute(
|
return conn.execute(
|
||||||
f"""UPDATE roles SET closed_at = ?
|
"""UPDATE roles SET closed_at = ?
|
||||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
AND last_seen < ?
|
AND last_seen < ?
|
||||||
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
||||||
(store.now(), source, subscription_id, stale,
|
(store.now(), source, subscription_id, stale,
|
||||||
json.dumps(list(seen_ids)))).rowcount
|
json.dumps(list(seen_ids)))).rowcount
|
||||||
|
|
||||||
|
|||||||
+48
-17
@@ -10,7 +10,7 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
|
|||||||
empty, or holds any other member (a function, a remote value other than
|
empty, or holds any other member (a function, a remote value other than
|
||||||
`remote`, a place the gazetteer does not know), can take a role from
|
`remote`, a place the gazetteer does not know), can take a role from
|
||||||
anywhere, so the base reach is then everywhere.
|
anywhere, so the base reach is then everywhere.
|
||||||
- **Wide reach, for the companies set to All countries**
|
- **Wide reach, for the companies set to All locations**
|
||||||
(`companies.all_countries`, global): the union over the enabled agents of
|
(`companies.all_countries`, global): the union over the enabled agents of
|
||||||
their `require` reach, the intersection of each one's `require` gates on
|
their `require` reach, the intersection of each one's `require` gates on
|
||||||
country and place, Europe today, expanded to its countries through the
|
country and place, Europe today, expanded to its countries through the
|
||||||
@@ -272,7 +272,7 @@ class Plan:
|
|||||||
def why(self, sub: Subscription) -> tuple[Reach, str]:
|
def why(self, sub: Subscription) -> tuple[Reach, str]:
|
||||||
"""The reach `sub` is read with, and why: `override`, `small`,
|
"""The reach `sub` is read with, and why: `override`, `small`,
|
||||||
`unknown` (a board never read whole), `countries` (its company is
|
`unknown` (a board never read whole), `countries` (its company is
|
||||||
set to All countries) or `agents`."""
|
set to All locations) or `agents`."""
|
||||||
if OVERRIDE_KEYS & set(sub.params or {}):
|
if OVERRIDE_KEYS & set(sub.params or {}):
|
||||||
return EVERYWHERE, "override"
|
return EVERYWHERE, "override"
|
||||||
size = self.sizes.get(sub.target)
|
size = self.sizes.get(sub.target)
|
||||||
@@ -283,9 +283,18 @@ class Plan:
|
|||||||
return self._big(sub)
|
return self._big(sub)
|
||||||
|
|
||||||
def _big(self, sub: Subscription) -> tuple[Reach, str]:
|
def _big(self, sub: Subscription) -> tuple[Reach, str]:
|
||||||
"""The reach of `sub` as a board of `SMALL` roles or more."""
|
"""The reach of `sub` as a board of `SMALL` roles or more. A host
|
||||||
|
that cannot filter on remote while the agents want remote roles is
|
||||||
|
asked for the wide reach, the agents' world (ADR Fetch prep and
|
||||||
|
score 1.2): asked for "DK, SE" alone it would never list a remote
|
||||||
|
role filed under Germany, and the gate sorts the rest out. Only
|
||||||
|
where the host takes the world in one read, a list of countries or
|
||||||
|
their ids: on a host that takes one country a request, Europe is
|
||||||
|
the whole board, so the location read stays and remote is not asked."""
|
||||||
if company_key(sub.company or sub.label) in self.all_countries:
|
if company_key(sub.company or sub.label) in self.all_countries:
|
||||||
return self.wide, "countries"
|
return self.wide, "countries"
|
||||||
|
if self.base.remote and not _filters_remote(sub) and _takes_world(sub, self.wide):
|
||||||
|
return self.wide, "world"
|
||||||
return self.base, "agents"
|
return self.base, "agents"
|
||||||
|
|
||||||
def narrows_roles(self, sub: Subscription) -> bool:
|
def narrows_roles(self, sub: Subscription) -> bool:
|
||||||
@@ -296,7 +305,7 @@ class Plan:
|
|||||||
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
|
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
|
||||||
"""The `reach` a board's batch hands `ask_for`; None for none. With
|
"""The `reach` a board's batch hands `ask_for`; None for none. With
|
||||||
fetch terms, and unless its company is set to All roles, a big board
|
fetch terms, and unless its company is set to All roles, a big board
|
||||||
carries them, and `wide` when its company is set to All countries, for
|
carries them, and `wide` when its company is set to All locations, for
|
||||||
`ask_for` to decide which side gives way over `MAX_SERIES` (see the
|
`ask_for` to decide which side gives way over `MAX_SERIES` (see the
|
||||||
module docstring). A board with no size carries
|
module docstring). A board with no size carries
|
||||||
`unsized` and `small`, and the reach of a big board unless its params
|
`unsized` and `small`, and the reach of a big board unless its params
|
||||||
@@ -305,15 +314,34 @@ class Plan:
|
|||||||
if why == "unknown":
|
if why == "unknown":
|
||||||
found, why = self._big(sub)
|
found, why = self._big(sub)
|
||||||
out = found.params()
|
out = found.params()
|
||||||
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
|
if self.terms and why in ("countries", "agents", "world") and self.narrows_roles(sub):
|
||||||
out = (out or {}) | {"terms": list(self.terms)}
|
out = (out or {}) | {"terms": list(self.terms)}
|
||||||
if why == "countries":
|
if why in ("countries", "world"):
|
||||||
out["wide"] = True
|
out["wide"] = True
|
||||||
if sub.target not in self.sizes:
|
if sub.target not in self.sizes:
|
||||||
out = (out or {}) | {"unsized": True, "small": SMALL}
|
out = (out or {}) | {"unsized": True, "small": SMALL}
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _filters_remote(sub: Subscription) -> bool:
|
||||||
|
"""Whether `sub`'s host declares a remote filter at all. A host that
|
||||||
|
declares one by ids may still lack the facet on a site, which the
|
||||||
|
runner finds out when it asks (`_runner.ask_for`)."""
|
||||||
|
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||||
|
return module is None or module.READS.filters.remote != "none"
|
||||||
|
|
||||||
|
|
||||||
|
def _takes_world(sub: Subscription, wide: Reach) -> bool:
|
||||||
|
"""Whether `sub`'s host can be asked for the wide reach in one read: a
|
||||||
|
country filter that takes a list (`many`, `ids`), or one value a request
|
||||||
|
with the countries within `MAX_SERIES`; never everywhere."""
|
||||||
|
module = readers.HOSTS.get(sub.target.partition(":")[0])
|
||||||
|
if module is None or wide.everywhere or not wide.countries:
|
||||||
|
return False
|
||||||
|
reach = module.READS.filters.country
|
||||||
|
return reach in ("many", "ids") or (reach == "one" and len(wide.countries) <= _runner.MAX_SERIES)
|
||||||
|
|
||||||
|
|
||||||
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
||||||
"""Each board's size as kept (`ats:size:<target>`); a board never sized
|
"""Each board's size as kept (`ats:size:<target>`); a board never sized
|
||||||
has none."""
|
has none."""
|
||||||
@@ -360,7 +388,7 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
|
|||||||
base = wide = EVERYWHERE
|
base = wide = EVERYWHERE
|
||||||
subs = list(store.subscriptions(source="ats") if subs is None else subs)
|
subs = list(store.subscriptions(source="ats") if subs is None else subs)
|
||||||
rows = store.companies()
|
rows = store.companies()
|
||||||
# All countries never reads less than the board would without it: the
|
# All locations never reads less than the board would without it: the
|
||||||
# base's remote, which a `require` of Europe does not name, is kept.
|
# base's remote, which a `require` of Europe does not name, is kept.
|
||||||
return Plan(base, wide | base, sizes(store, subs),
|
return Plan(base, wide | base, sizes(store, subs),
|
||||||
all_countries=frozenset(c.key for c in rows if c.all_countries),
|
all_countries=frozenset(c.key for c in rows if c.all_countries),
|
||||||
@@ -380,8 +408,8 @@ def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str
|
|||||||
|
|
||||||
#: Why a board is read whole, as the Companies list says it.
|
#: Why a board is read whole, as the Companies list says it.
|
||||||
_WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
|
_WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
|
||||||
"override": "set here", "countries": "all countries (everywhere)",
|
"override": "set here", "countries": "all locations (everywhere)",
|
||||||
"agents": "an agent takes any place"}
|
"world": "the agents' world (everywhere)", "agents": "an agent takes any place"}
|
||||||
|
|
||||||
|
|
||||||
def _codes(codes: Iterable[str]) -> list[str]:
|
def _codes(codes: Iterable[str]) -> list[str]:
|
||||||
@@ -452,8 +480,8 @@ def line(found: Plan, sub: Subscription) -> str:
|
|||||||
"follows agents: DK, SE, remote · follows agents' terms: engineer,
|
"follows agents: DK, SE, remote · follows agents' terms: engineer,
|
||||||
product", "follows agents: DK, SE, remote · all roles" (a company set to
|
product", "follows agents: DK, SE, remote · all roles" (a company set to
|
||||||
All roles, or a reader without a query, or terms over `MAX_SERIES`),
|
All roles, or a reader without a query, or terms over `MAX_SERIES`),
|
||||||
"all countries (Europe) · follows agents' terms: engineer, product,
|
"all locations (Europe) · follows agents' terms: engineer, product,
|
||||||
data", "all countries (Europe) · all roles", "set here: DE · all roles",
|
data", "all locations (Europe) · all roles", "set here: DE · all roles",
|
||||||
"all places, too many countries to ask · follows agents' terms:
|
"all places, too many countries to ask · follows agents' terms:
|
||||||
engineer", or, for a board read whole, why: "under 300 roles: read
|
engineer", or, for a board read whole, why: "under 300 roles: read
|
||||||
whole", "Greenhouse cannot filter: read whole". A board with no size yet
|
whole", "Greenhouse cannot filter: read whole". A board with no size yet
|
||||||
@@ -485,23 +513,26 @@ def line(found: Plan, sub: Subscription) -> str:
|
|||||||
else "")
|
else "")
|
||||||
if why == "countries":
|
if why == "countries":
|
||||||
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
||||||
return f"all countries ({named}){unasked} · {roles}"
|
return f"all locations ({named}){unasked} · {roles}"
|
||||||
|
if why == "world":
|
||||||
|
named = found.wide_names or f"{len(found.wide.countries)} countries"
|
||||||
|
return f"follows agents' world ({named}){unasked} · {roles}"
|
||||||
if why == "override":
|
if why == "override":
|
||||||
return f"set here: {_places(ask)}{unasked} · {roles}"
|
return f"set here: {_places(ask)}{unasked} · {roles}"
|
||||||
return f"follows agents: {_places(ask)}{unasked} · {roles}"
|
return f"follows agents: {_places(ask)}{unasked} · {roles}"
|
||||||
if ask.terms:
|
if ask.terms:
|
||||||
if given.everywhere:
|
if given.everywhere:
|
||||||
what = "all countries (everywhere)" if why == "countries" else _WHOLE[why]
|
what = "all locations (everywhere)" if why == "countries" else _WHOLE[why]
|
||||||
return f"{what} · {roles}"
|
return f"{what} · {roles}"
|
||||||
return f"all places, too many countries to ask · {roles}"
|
return f"all places, too many countries to ask · {roles}"
|
||||||
if why in ("agents", "countries") and given.everywhere:
|
if why in ("agents", "countries", "world") and given.everywhere:
|
||||||
return f"{_WHOLE[why]}: read whole"
|
return f"{_WHOLE[why]}: read whole"
|
||||||
if why in ("agents", "countries", "override") and not module.READS.filters.declared():
|
if why in ("agents", "countries", "world", "override") and not module.READS.filters.declared():
|
||||||
return f"{module.LABEL} cannot filter: read whole"
|
return f"{module.LABEL} cannot filter: read whole"
|
||||||
reads = module.READS.filters
|
reads = module.READS.filters
|
||||||
if why in ("agents", "countries") and reads.country == "one" \
|
if why in ("agents", "countries", "world") and reads.country == "one" \
|
||||||
and len(given.countries) > _runner.MAX_SERIES:
|
and len(given.countries) > _runner.MAX_SERIES:
|
||||||
return "too many countries to ask: read whole"
|
return "too many countries to ask: read whole"
|
||||||
if why in ("agents", "countries"):
|
if why in ("agents", "countries", "world"):
|
||||||
return f"{module.LABEL} cannot ask for these: read whole"
|
return f"{module.LABEL} cannot ask for these: read whole"
|
||||||
return f"{_WHOLE[why]}: read whole"
|
return f"{_WHOLE[why]}: read whole"
|
||||||
+19
-9
@@ -9,16 +9,19 @@ good for a given search. Each agent first applies its fetch gate:
|
|||||||
lifts
|
lifts
|
||||||
any gates one group, the place part: a role that matches none of
|
any gates one group, the place part: a role that matches none of
|
||||||
them, and has the column filled for at least one, is
|
them, and has the column filled for at least one, is
|
||||||
gated ("none of country DK / remote remote"); skipped for
|
gated ("none of country DK / remote remote"), unless its
|
||||||
a company set to All countries
|
location is unknown; skipped for a company set to All
|
||||||
|
locations
|
||||||
exclude gates a role that matches one of the values keeps its row, with
|
exclude gates a role that matches one of the values keeps its row, with
|
||||||
score 0 and the value ("title account executive"); those
|
score 0 and the value ("title account executive"); those
|
||||||
on `title` are skipped for a company set to All roles
|
on `title` are skipped for a company set to All roles
|
||||||
|
|
||||||
A gate rejects on evidence, never on absence: a role whose column is empty
|
A gate rejects on evidence, never on absence: a role whose column is empty
|
||||||
is not gated by it. `extract` is what fills an empty column, and it reads
|
is not gated by it, and a place the gazetteer could not resolve, or one
|
||||||
only the roles in the cut, so gating on an empty column would keep those
|
that holds the agent's places (Øresund for Copenhagen), is no evidence.
|
||||||
roles out of extract's reach for good. A hidden company (`affinity = -1`)
|
`extract` is what fills an empty column, and it reads only the roles in
|
||||||
|
the cut, so gating on an empty column would keep those roles out of
|
||||||
|
extract's reach for good. A hidden company (`affinity = -1`)
|
||||||
is not scored at all; a favourite passes the gate like any other company.
|
is not scored at all; a favourite passes the gate like any other company.
|
||||||
|
|
||||||
The final score, the feed's order, is the weighted sum of four terms over
|
The final score, the feed's order, is the weighted sum of four terms over
|
||||||
@@ -42,8 +45,8 @@ all there is, when it is gated as "no vector". An agent with no votes scores
|
|||||||
every role without `taste`; the direction is newsdesk's
|
every role without `taste`; the direction is newsdesk's
|
||||||
`centroid(liked) - centroid(disliked)`. An agent with no prompt that has
|
`centroid(liked) - centroid(disliked)`. An agent with no prompt that has
|
||||||
text has no `fit` either, so nothing of it enters the cut by fit and its
|
text has no `fit` either, so nothing of it enters the cut by fit and its
|
||||||
scores do not decide what summarise reads (`Store._summarise_top`); a run
|
want order does not decide what summarise reads (`Store._summarise_top`); a
|
||||||
names such agents in one notice.
|
run names such agents in one notice.
|
||||||
|
|
||||||
Beside the score, `fit` is stored as it is, and for an agent with a CV,
|
Beside the score, `fit` is stored as it is, and for an agent with a CV,
|
||||||
`cv_fit`: how close the role is to the CV, the best cosine between the role
|
`cv_fit`: how close the role is to the CV, the best cosine between the role
|
||||||
@@ -181,7 +184,11 @@ def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
|
|||||||
it fails, every exclude gate it matches, and the `any` group when it
|
it fails, every exclude gate it matches, and the `any` group when it
|
||||||
matches none of them. A gate with an empty column (state -1) neither
|
matches none of them. A gate with an empty column (state -1) neither
|
||||||
passes nor fails, so the group fails only when some member had evidence.
|
passes nor fails, so the group fails only when some member had evidence.
|
||||||
A company set to All countries skips the `any` group, the place part,
|
A role whose location is unknown, every `country` and `place` member
|
||||||
|
empty, passes the group whatever its remote says: a hybrid role with no
|
||||||
|
place may still be in one of the agent's towns, and extract reads it
|
||||||
|
only in the cut. A group of remote alone still fails a hybrid role.
|
||||||
|
A company set to All locations skips the `any` group, the place part,
|
||||||
and one set to All roles the exclude gates on `title`, the titles to
|
and one set to All roles the exclude gates on `title`, the titles to
|
||||||
leave out; the require gates, the region, apply to every company."""
|
leave out; the require gates, the region, apply to every company."""
|
||||||
states = list(zip(gates, candidate.gates, strict=True))
|
states = list(zip(gates, candidate.gates, strict=True))
|
||||||
@@ -190,7 +197,10 @@ def failed_gates(candidate: Candidate, gates: Sequence[Gate]) -> list[str]:
|
|||||||
if g.mode == "exclude" and state > 0
|
if g.mode == "exclude" and state > 0
|
||||||
and not (candidate.all_roles and g.field == "title")]
|
and not (candidate.all_roles and g.field == "title")]
|
||||||
group = [(g, state) for g, state in states if g.mode == "any"]
|
group = [(g, state) for g, state in states if g.mode == "any"]
|
||||||
if not candidate.all_countries and group and not any(state == 1 for _, state in group) \
|
located = [state for g, state in group if g.field in ("country", "place")]
|
||||||
|
unknown = bool(located) and all(state == -1 for state in located)
|
||||||
|
if not candidate.all_countries and group and not unknown \
|
||||||
|
and not any(state == 1 for _, state in group) \
|
||||||
and any(state == 0 for _, state in group):
|
and any(state == 0 for _, state in group):
|
||||||
reasons.append("none of " + ", ".join(_wanted(g) for g, _ in group))
|
reasons.append("none of " + ", ".join(_wanted(g) for g, _ in group))
|
||||||
return reasons
|
return reasons
|
||||||
|
|||||||
@@ -7,8 +7,10 @@ answer), and running it is `desk.modeltask`'s.
|
|||||||
|
|
||||||
A role is waiting while it has no summary, has some text, is open and no copy,
|
A role is waiting while it has no summary, has some text, is open and no copy,
|
||||||
and is either in the top of an enabled agent's feed, the best `summarise_top`
|
and is either in the top of an enabled agent's feed, the best `summarise_top`
|
||||||
final scores among the roles its fetch gate passes (its `limits`), or one the
|
(its `limits`, 30 by default) by the want feed score and the best
|
||||||
owner cares about whatever its rank: saved (it has an application, the star)
|
`summarise_top` by the match order (CV match, then CV closeness) among the
|
||||||
|
roles its fetch gate passes, so at most twice that many, or one the owner
|
||||||
|
cares about whatever its rank: saved (it has an application, the star)
|
||||||
or voted up by an enabled agent. Summarise is no part of any score and runs
|
or voted up by an enabled agent. Summarise is no part of any score and runs
|
||||||
last in the pull, once the final scores are set. The limit keeps the cost
|
last in the pull, once the final scores are set. The limit keeps the cost
|
||||||
flat: a pull summarises the roles somebody will look at, not the 30k that are
|
flat: a pull summarises the roles somebody will look at, not the 30k that are
|
||||||
|
|||||||
+47
-4
@@ -4,8 +4,9 @@
|
|||||||
uv run uvicorn app.main:app --reload
|
uv run uvicorn app.main:app --reload
|
||||||
|
|
||||||
`create_app()` is the whole of the wiring, and the order in it is the point:
|
`create_app()` is the whole of the wiring, and the order in it is the point:
|
||||||
config and store, the door, the queue, the pages. Everything before the queue
|
config and store, the door, the queue, the pages, and around all of them the
|
||||||
is something the queue needs; everything after it is something that enqueues.
|
hardening headers. Everything before the queue is something the queue needs;
|
||||||
|
everything after it is something that enqueues.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -17,7 +18,9 @@ from fastapi import FastAPI, Request
|
|||||||
from fastapi.exceptions import RequestValidationError
|
from fastapi.exceptions import RequestValidationError
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
from starlette.datastructures import MutableHeaders
|
||||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||||
|
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||||
|
|
||||||
import auth
|
import auth
|
||||||
from app import jobs
|
from app import jobs
|
||||||
@@ -33,6 +36,39 @@ log = logging.getLogger(__name__)
|
|||||||
#: data gets `{"detail": ...}`; a browser that asked for a page gets one.
|
#: data gets `{"detail": ...}`; a browser that asked for a page gets one.
|
||||||
API_PREFIXES = ("/api", "/ws", "/_jobs", "/_auth", "/static", "/health")
|
API_PREFIXES = ("/api", "/ws", "/_jobs", "/_auth", "/static", "/health")
|
||||||
|
|
||||||
|
#: On every HTTP answer, the door's refusals and the error pages included. No
|
||||||
|
#: CSP yet: the templates carry inline scripts and styles, which would need a
|
||||||
|
#: nonce or a hash each.
|
||||||
|
HARDENING = {
|
||||||
|
"X-Content-Type-Options": "nosniff",
|
||||||
|
"X-Frame-Options": "DENY",
|
||||||
|
"Referrer-Policy": "same-origin",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Hardening:
|
||||||
|
"""`HARDENING` on every response that does not set the header itself.
|
||||||
|
|
||||||
|
Pure ASGI rather than `BaseHTTPMiddleware`, so a streamed answer and the
|
||||||
|
socket pass through untouched."""
|
||||||
|
|
||||||
|
def __init__(self, app: ASGIApp) -> None:
|
||||||
|
self.app = app
|
||||||
|
|
||||||
|
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||||
|
if scope["type"] != "http":
|
||||||
|
await self.app(scope, receive, send)
|
||||||
|
return
|
||||||
|
|
||||||
|
async def harden(message: Message) -> None:
|
||||||
|
if message["type"] == "http.response.start":
|
||||||
|
headers = MutableHeaders(scope=message)
|
||||||
|
for name, value in HARDENING.items():
|
||||||
|
headers.setdefault(name, value)
|
||||||
|
await send(message)
|
||||||
|
|
||||||
|
await self.app(scope, receive, harden)
|
||||||
|
|
||||||
|
|
||||||
def create_app(config: Config | None = None) -> FastAPI:
|
def create_app(config: Config | None = None) -> FastAPI:
|
||||||
"""Build the application. A test passes its own config; nobody else does."""
|
"""Build the application. A test passes its own config; nobody else does."""
|
||||||
@@ -59,8 +95,12 @@ def create_app(config: Config | None = None) -> FastAPI:
|
|||||||
|
|
||||||
# The door, first, because it is the door. The middleware covers every
|
# The door, first, because it is the door. The middleware covers every
|
||||||
# route whenever it was added, `/ws/jobs` included; written first, nobody
|
# route whenever it was added, `/ws/jobs` included; written first, nobody
|
||||||
# has to work that out. `auto` locks unless the bind address is loopback.
|
# has to work that out. `auto` locks unless the bind address is loopback,
|
||||||
# Auth's tables are the `auth_*` ones in the same file.
|
# and an app it leaves open answers 421 to a request that names any host
|
||||||
|
# but a loopback one or comes through a proxy. Auth's tables are the
|
||||||
|
# `auth_*` ones in the same file, `auth_tokens` for the API tokens among
|
||||||
|
# them. No `session_only=`: no route here hands out or replaces the
|
||||||
|
# database as a file.
|
||||||
auth.connect(app, auth.AuthStore(config.database), config.auth,
|
auth.connect(app, auth.AuthStore(config.database), config.auth,
|
||||||
host=config.server.host, open=("/static/",),
|
host=config.server.host, open=("/static/",),
|
||||||
stylesheet="/static/app.css")
|
stylesheet="/static/app.css")
|
||||||
@@ -84,6 +124,9 @@ def create_app(config: Config | None = None) -> FastAPI:
|
|||||||
--wait` waits on this, so it has to prove the database answers."""
|
--wait` waits on this, so it has to prove the database answers."""
|
||||||
return {"ok": True, "open_roles": sum(store.open_role_counts().values())}
|
return {"ok": True, "open_roles": sum(store.open_role_counts().values())}
|
||||||
|
|
||||||
|
# Last, so outermost: around the door and the error pages.
|
||||||
|
app.add_middleware(Hardening)
|
||||||
|
|
||||||
@app.exception_handler(StarletteHTTPException)
|
@app.exception_handler(StarletteHTTPException)
|
||||||
async def errors(request: Request, exc: StarletteHTTPException) -> Any:
|
async def errors(request: Request, exc: StarletteHTTPException) -> Any:
|
||||||
"""JSON for a caller, a page for a browser. One shape for every API
|
"""JSON for a caller, a page for a browser. One shape for every API
|
||||||
|
|||||||
+25
-71
@@ -1008,76 +1008,30 @@ def language_flag(required: list[RoleLanguage],
|
|||||||
return "", ""
|
return "", ""
|
||||||
|
|
||||||
|
|
||||||
#: What career counts as at most when a posting breaks one of the agent's
|
|
||||||
#: must-haves: the top of the rubric's 0-39 band for a dead end. The model
|
|
||||||
#: reports whether each must-have is met; this cap is applied in code, when a
|
|
||||||
#: role is served (`cv_score`, `Store.judgement`, `store._cv_sql`).
|
|
||||||
BROKEN_CAP = 39
|
|
||||||
|
|
||||||
#: A must-have's verdict, as the judge answers it.
|
|
||||||
MUST_VERDICTS = ("met", "broken", "unclear")
|
|
||||||
|
|
||||||
_BULLET = re.compile(r"^\s*(?:[-*\u2022]+|\(?\d+[.)])\s+")
|
|
||||||
|
|
||||||
|
|
||||||
def must_lines(text: str) -> list[str]:
|
|
||||||
"""The agent's must-haves text read as one must-have per line: a leading
|
|
||||||
bullet (`-`, `*`, `•`) or number (`1.`, `2)`) is dropped, so are empty
|
|
||||||
lines."""
|
|
||||||
found = []
|
|
||||||
for line in text.splitlines():
|
|
||||||
line = " ".join(_BULLET.sub("", line, count=1).split())
|
|
||||||
if line:
|
|
||||||
found.append(line)
|
|
||||||
return found
|
|
||||||
|
|
||||||
|
|
||||||
def capped_career(career: int, broken: int = 0) -> int:
|
|
||||||
"""`career` as it counts: at most `BROKEN_CAP` when any must-have is
|
|
||||||
broken."""
|
|
||||||
return min(career, BROKEN_CAP) if broken else career
|
|
||||||
|
|
||||||
|
|
||||||
def cv_score(skills: int, experience: int, career: int,
|
def cv_score(skills: int, experience: int, career: int,
|
||||||
weights: Weights | None = None, broken: int = 0) -> float | None:
|
weights: Weights | None = None) -> float | None:
|
||||||
"""The one number of a CV match, 0 to 100: the three numbers of the
|
"""The one number of a CV match, 0 to 100 for sorting and display: the
|
||||||
judge averaged by the agent's `cv_skills`, `cv_experience` and
|
judge's three numbers, each 0 to 10, averaged by the agent's
|
||||||
`cv_career` weights (30, 25 and 30 by default), career as
|
`cv_skills`, `cv_experience` and `cv_career` weights (30, 25 and 30 by
|
||||||
`capped_career` has it. None when all three weights are 0."""
|
default), times 10. None when all three weights are 0."""
|
||||||
w = weights or Weights()
|
w = weights or Weights()
|
||||||
total = w.cv_skills + w.cv_experience + w.cv_career
|
total = w.cv_skills + w.cv_experience + w.cv_career
|
||||||
if not total:
|
if not total:
|
||||||
return None
|
return None
|
||||||
career = capped_career(career, broken)
|
return round(10 * (w.cv_skills * skills + w.cv_experience * experience
|
||||||
return round((w.cv_skills * skills + w.cv_experience * experience
|
|
||||||
+ w.cv_career * career) / total, 1)
|
+ w.cv_career * career) / total, 1)
|
||||||
|
|
||||||
|
|
||||||
class Must(Model):
|
|
||||||
"""One must-have of an agent as the judge read it against a role: its
|
|
||||||
`id` (`M1`, by line), the line's `text` and the `verdict`."""
|
|
||||||
|
|
||||||
id: str
|
|
||||||
text: str = ""
|
|
||||||
verdict: Literal["met", "broken", "unclear"] = "unclear"
|
|
||||||
|
|
||||||
|
|
||||||
class Judgement(Model):
|
class Judgement(Model):
|
||||||
"""What `judge` read of one role for one agent (`role_judgements`): the
|
"""What `judge` read of one role for one agent (`role_judgements`): the
|
||||||
three dimensions 0 to 100 as the model gave them, `cv_score`
|
three dimensions 0 to 10 as the model gave them, `cv_score`
|
||||||
(`cv_score`) and 1 to 3 strengths and gaps grounded in the posting.
|
(`cv_score`, 0 to 100) and 1 to 3 strengths and gaps grounded in the
|
||||||
`musts` is the verdict on each must-have and `broken` how many are
|
posting. `cv_score` is not read from the table: `Store.judgement` sets
|
||||||
broken; any makes `career_capped` the lower of `career` and
|
it from the agent's weights; it is None when all three are 0."""
|
||||||
`BROKEN_CAP`. `cv_score` and `career_capped` are not read from the
|
|
||||||
table: `Store.judgement` sets them from the agent's weights; `cv_score`
|
|
||||||
is None when all three are 0."""
|
|
||||||
|
|
||||||
skills: int
|
skills: int
|
||||||
experience: int
|
experience: int
|
||||||
career: int
|
career: int
|
||||||
career_capped: int | None = None
|
|
||||||
broken: int = 0
|
|
||||||
musts: Annotated[list[Must], BeforeValidator(lambda v: _json(v, "[]"))] = []
|
|
||||||
cv_score: float | None = None
|
cv_score: float | None = None
|
||||||
strengths: JsonStrings = []
|
strengths: JsonStrings = []
|
||||||
gaps: JsonStrings = []
|
gaps: JsonStrings = []
|
||||||
@@ -1387,7 +1341,8 @@ class Company(Model):
|
|||||||
#: not on that list.
|
#: not on that list.
|
||||||
hand_reason: str = ""
|
hand_reason: str = ""
|
||||||
#: The two fetch overrides (`app/jobs/reach.py`), global rather than an
|
#: The two fetch overrides (`app/jobs/reach.py`), global rather than an
|
||||||
#: agent's, since fetching is shared. `all_countries` 1: every big board
|
#: agent's, since fetching is shared. `all_countries` 1 (All locations,
|
||||||
|
#: as the switch is called; the column keeps its name): every big board
|
||||||
#: of the company is read with the agents' wide reach, Europe today,
|
#: of the company is read with the agents' wide reach, Europe today,
|
||||||
#: rather than their places. `all_roles` 1: no role-type narrowing, no
|
#: rather than their places. `all_roles` 1: no role-type narrowing, no
|
||||||
#: fetch terms today, on any of its boards.
|
#: fetch terms today, on any of its boards.
|
||||||
@@ -1459,7 +1414,7 @@ class Gate(Model):
|
|||||||
fails it (the row is kept with the reason); the `any` gates together are
|
fails it (the row is kept with the reason); the `any` gates together are
|
||||||
one group a role must match at least one of ("in Denmark, or remote");
|
one group a role must match at least one of ("in Denmark, or remote");
|
||||||
`exclude` gates a role that matches any of its values ("title account
|
`exclude` gates a role that matches any of its values ("title account
|
||||||
executive"). A company set to All countries skips the `any` group and one
|
executive"). A company set to All locations skips the `any` group and one
|
||||||
set to All roles skips the `exclude` gates on `title` (`score.failed_gates`).
|
set to All roles skips the `exclude` gates on `title` (`score.failed_gates`).
|
||||||
A gate rejects on evidence only: a role whose column is empty is not
|
A gate rejects on evidence only: a role whose column is empty is not
|
||||||
rejected, since `extract` fills the column later for the roles in the
|
rejected, since `extract` fills the column later for the roles in the
|
||||||
@@ -1497,14 +1452,16 @@ class Limits(Model):
|
|||||||
leaves out, then every role liked or saved. Both levels are calibrated
|
leaves out, then every role liked or saved. Both levels are calibrated
|
||||||
(`score.scale`) with the company's affinity as a small bonus, so a
|
(`score.scale`) with the company's affinity as a small bonus, so a
|
||||||
minimum means the same in every run. For a broad agent the count stops a
|
minimum means the same in every run. For a broad agent the count stops a
|
||||||
list, for a narrow one the minimum. Summarise reads the best
|
list, for a narrow one the minimum. Summarise reads the top
|
||||||
`summarise_top` by final score, and every role liked or saved."""
|
`summarise_top` of the want feed score (the score order) and the top
|
||||||
|
`summarise_top` of the match order (CV match, then CV closeness), so at
|
||||||
|
most twice as many, and every role liked or saved."""
|
||||||
|
|
||||||
cut_fit_top: int = Field(500, ge=0)
|
cut_fit_top: int = Field(500, ge=0)
|
||||||
cut_fit_min: float = Field(0.6, ge=0, le=1)
|
cut_fit_min: float = Field(0.6, ge=0, le=1)
|
||||||
cut_cv_top: int = Field(150, ge=0)
|
cut_cv_top: int = Field(150, ge=0)
|
||||||
cut_cv_min: float = Field(0.6, ge=0, le=1)
|
cut_cv_min: float = Field(0.6, ge=0, le=1)
|
||||||
summarise_top: int = Field(50, ge=0)
|
summarise_top: int = Field(30, ge=0)
|
||||||
|
|
||||||
|
|
||||||
class Prompt(Model):
|
class Prompt(Model):
|
||||||
@@ -1568,14 +1525,11 @@ class Agent(Model):
|
|||||||
are the union over the enabled agents, and an agent with none means no
|
are the union over the enabled agents, and an agent with none means no
|
||||||
board is narrowed by words. They decide what is fetched, not what ranks.
|
board is narrowed by words. They decide what is fetched, not what ranks.
|
||||||
|
|
||||||
`cv_text`, `cv_languages`, `like_text`, `avoid_text` and `must_text` are
|
`cv_text`, `cv_languages`, `like_text` and `avoid_text` are what `judge`
|
||||||
what `judge` reads a role against: the CV, the languages it states with
|
reads a role against: the CV, the languages it states with a level, and
|
||||||
a level, what the person likes and does not like (the work, the
|
what the person likes and does not like (the work, the workplace,
|
||||||
workplace, situations; both feed the career score), and their
|
situations; both feed the career score). An agent without a CV is not
|
||||||
must-haves and deal-breakers (set through the API only, not on the
|
judged, and one without languages flags none (`language_flag`)."""
|
||||||
form; soft, read by the model only; a hard rule is a gate). An agent without
|
|
||||||
a CV is not judged, and one without languages flags none
|
|
||||||
(`language_flag`)."""
|
|
||||||
|
|
||||||
id: int
|
id: int
|
||||||
name: str
|
name: str
|
||||||
@@ -1588,7 +1542,6 @@ class Agent(Model):
|
|||||||
cv_languages: CvLanguages = []
|
cv_languages: CvLanguages = []
|
||||||
like_text: str = ""
|
like_text: str = ""
|
||||||
avoid_text: str = ""
|
avoid_text: str = ""
|
||||||
must_text: str = ""
|
|
||||||
enabled: bool = True
|
enabled: bool = True
|
||||||
created_at: str
|
created_at: str
|
||||||
|
|
||||||
@@ -1616,7 +1569,8 @@ class Candidate(Model):
|
|||||||
uid: str
|
uid: str
|
||||||
posted_at: str
|
posted_at: str
|
||||||
affinity: int = 0
|
affinity: int = 0
|
||||||
#: `companies.all_countries`: the `any` group (the place part) is skipped.
|
#: `companies.all_countries`, All locations: the `any` group (the place
|
||||||
|
#: part) is skipped.
|
||||||
all_countries: bool = False
|
all_countries: bool = False
|
||||||
#: `companies.all_roles`: the `exclude` gates on `title` are skipped.
|
#: `companies.all_roles`: the `exclude` gates on `title` are skipped.
|
||||||
all_roles: bool = False
|
all_roles: bool = False
|
||||||
|
|||||||
+7
-4
@@ -283,7 +283,8 @@ _SKANE = Region(
|
|||||||
|
|
||||||
_HOVEDSTADEN = Region(
|
_HOVEDSTADEN = Region(
|
||||||
"hovedstaden", "Capital Region of Denmark",
|
"hovedstaden", "Capital Region of Denmark",
|
||||||
_split("Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden"),
|
_split("""Capital Region of Denmark, Capital Region Denmark, Region Hovedstaden, Hovedstaden,
|
||||||
|
Storkøbenhavn, Nordsjælland"""),
|
||||||
_split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup,
|
_split("""København, Copenhagen, Frederiksberg, Albertslund, Allerød, Ballerup,
|
||||||
Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte,
|
Bornholm, Brøndby, Dragør, Egedal, Fredensborg, Frederikssund, Furesø, Gentofte,
|
||||||
Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre,
|
Gladsaxe, Glostrup, Gribskov, Halsnæs, Helsingør, Herlev, Hillerød, Hvidovre,
|
||||||
@@ -303,7 +304,8 @@ _HOVEDSTADEN = Region(
|
|||||||
# The bare word is left out of `names` since "New Zealand" would match it.
|
# The bare word is left out of `names` since "New Zealand" would match it.
|
||||||
_ZEALAND = Region(
|
_ZEALAND = Region(
|
||||||
"zealand", "Zealand",
|
"zealand", "Zealand",
|
||||||
_split("Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland"),
|
_split("""Region Zealand, Region Sjælland, Zealand Denmark, Sjælland, Sjaelland,
|
||||||
|
Østsjælland, Vestsjælland, Sydsjælland"""),
|
||||||
_split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved,
|
_split("""Faxe, Greve, Guldborgsund, Holbæk, Kalundborg, Køge, Lejre, Lolland, Næstved,
|
||||||
Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg,
|
Odsherred, Ringsted, Roskilde, Slagelse, Solrød, Sorø, Stevns, Vordingborg,
|
||||||
Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing,
|
Nykøbing Falster, Nykøbing F, Nykøbing Sjælland, Nakskov, Maribo, Sakskøbing,
|
||||||
@@ -316,7 +318,7 @@ _JUTLAND = Region(
|
|||||||
"jutland", "Jutland",
|
"jutland", "Jutland",
|
||||||
_split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland,
|
_split("""Jutland, Jylland, Central Jutland, Midtjylland, Region Midtjylland,
|
||||||
North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland,
|
North Jutland, Nordjylland, Region Nordjylland, South Jutland, Sønderjylland,
|
||||||
East Jutland, West Jutland, Vestjylland, Østjylland"""),
|
East Jutland, West Jutland, Vestjylland, Østjylland, Sydøstjylland, Sydvestjylland"""),
|
||||||
_split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning,
|
_split("""Aarhus, Aalborg, Esbjerg, Randers, Kolding, Horsens, Vejle, Silkeborg, Herning,
|
||||||
Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov,
|
Viborg, Billund, Fredericia, Holstebro, Skive, Ringkøbing, Videbæk, Lem, Risskov,
|
||||||
Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå,
|
Åbyhøj, Brabrand, Viby J, Aabenraa, Haderslev, Sønderborg, Grenaa, Grenå,
|
||||||
@@ -882,7 +884,8 @@ _PLACELESS = frozenset((
|
|||||||
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
|
"remote", "hybrid", "anywhere", "worldwide", "global", "distributed", "home", "based",
|
||||||
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
|
"from", "work", "working", "wfh", "office", "offices", "onsite", "on", "site", "in",
|
||||||
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
|
"multiple", "several", "various", "locations", "location", "flexible", "or", "and", "any",
|
||||||
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position"))
|
"first", "field", "travel", "tbd", "n", "a", "other", "job", "jobs", "position",
|
||||||
|
"udlandet", "ovrige"))
|
||||||
|
|
||||||
|
|
||||||
def placeless(text: str) -> bool:
|
def placeless(text: str) -> bool:
|
||||||
|
|||||||
+6
-2
@@ -37,8 +37,9 @@ from app.storage import Store
|
|||||||
#: Raised when the rules in this package change the way a stored word maps,
|
#: Raised when the rules in this package change the way a stored word maps,
|
||||||
#: so every open role is mapped again as it is after a table changes. 3: the
|
#: so every open role is mapped again as it is after a table changes. 3: the
|
||||||
#: remote area, and remote as the model's before the host's word. 4: the
|
#: remote area, and remote as the model's before the host's word. 4: the
|
||||||
#: deadline, the host's before the model's.
|
#: deadline, the host's before the model's. 5: `remote_text`, the title or
|
||||||
VERSION = 4
|
#: text saying remote while no field does.
|
||||||
|
VERSION = 5
|
||||||
|
|
||||||
#: The state key the last run's `fingerprint` is kept under.
|
#: The state key the last run's `fingerprint` is kept under.
|
||||||
STATE = "prep:tables"
|
STATE = "prep:tables"
|
||||||
@@ -61,6 +62,7 @@ def fingerprint(tables: Tables = TABLES) -> str:
|
|||||||
"""A hash of every host table, the shared remote words, the gazetteer the
|
"""A hash of every host table, the shared remote words, the gazetteer the
|
||||||
code describes and `VERSION`."""
|
code describes and `VERSION`."""
|
||||||
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
blob = json.dumps({"version": VERSION, "tables": tables, "remote": words.REMOTE_WORDS,
|
||||||
|
"remote_text": [words.REMOTE_TEXT, words.REMOTE_TEXT_NOT],
|
||||||
"places": repr(places.seed())}, sort_keys=True)
|
"places": repr(places.seed())}, sort_keys=True)
|
||||||
return hashlib.sha1(blob.encode()).hexdigest()
|
return hashlib.sha1(blob.encode()).hexdigest()
|
||||||
|
|
||||||
@@ -114,6 +116,7 @@ def map_role(row: Mapping[str, Any], gaz: places.Gazetteer, folds: Mapping[str,
|
|||||||
"seniority_title": titled,
|
"seniority_title": titled,
|
||||||
"employment": words.employment(host, row["employment_host"], tables),
|
"employment": words.employment(host, row["employment_host"], tables),
|
||||||
"remote": remote,
|
"remote": remote,
|
||||||
|
"remote_text": int(words.remote_in_text(row["title"], row["text"] or "")),
|
||||||
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
"remote_area": list(area(gaz, texts, place_ids, _model_names(row["remote_where_model"]))
|
||||||
if remote == "remote" else ()),
|
if remote == "remote" else ()),
|
||||||
"country": country(gaz, place_ids, row["country_host"]),
|
"country": country(gaz, place_ids, row["country_host"]),
|
||||||
@@ -125,6 +128,7 @@ def _changed(row: Mapping[str, Any], found: Mapping[str, Any]) -> bool:
|
|||||||
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
level = row["seniority_model"] or found["seniority_stated"] or found["seniority_title"]
|
||||||
return bool(row["prep_due"]) or level != row["seniority"] \
|
return bool(row["prep_due"]) or level != row["seniority"] \
|
||||||
or found["remote"] != row["remote"] \
|
or found["remote"] != row["remote"] \
|
||||||
|
or found["remote_text"] != row["remote_text"] \
|
||||||
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
or found["remote_area"] != json.loads(row["remote_area"] or "[]") \
|
||||||
or (found["country"] and found["country"] != row["country"]) \
|
or (found["country"] and found["country"] != row["country"]) \
|
||||||
or found["deadline"] != row["deadline"] \
|
or found["deadline"] != row["deadline"] \
|
||||||
|
|||||||
@@ -58,6 +58,39 @@ _NOT_REMOTE_WORK = re.compile(r"\bremote[\s-]+(?:sensing|control\w*|monitoring)\
|
|||||||
re.IGNORECASE)
|
re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
#: Phrases in a title or text that say the role is worked remotely, and the
|
||||||
|
#: phrases that take one back. Both are read on the lowercased text with its
|
||||||
|
#: hyphens and runs of spaces squashed to one space. A match marks the role as
|
||||||
|
#: possibly remote (`remote_in_text`); it decides nothing by itself.
|
||||||
|
REMOTE_TEXT: tuple[str, ...] = (
|
||||||
|
"fully remote", "100% remote", "100 % remote", "remote first", "remote work",
|
||||||
|
"remote working", "work remotely", "working remotely", "remote position", "remote role",
|
||||||
|
"remote job", "remote team", "remote friendly", "remote possible", "remote option",
|
||||||
|
"remote ok", "work from home", "working from home", "work from anywhere", "home based",
|
||||||
|
"home office", "homeoffice", "hjemmearbejde", "fjernarbejde", "distansarbete",
|
||||||
|
)
|
||||||
|
REMOTE_TEXT_NOT: tuple[str, ...] = (
|
||||||
|
"no remote", "not remote", "not a remote", "non remote", "not be remote", "cannot be remote",
|
||||||
|
"no home office", "kein homeoffice", "not available remotely", "not open to remote",
|
||||||
|
"remote work is not", "remote working is not", "is not a remote", "remote sensing",
|
||||||
|
"remote control", "remote monitoring",
|
||||||
|
)
|
||||||
|
|
||||||
|
_REMOTE_TEXT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT))
|
||||||
|
_REMOTE_TEXT_NOT = re.compile("|".join(rf"\b{re.escape(p)}\b" for p in REMOTE_TEXT_NOT))
|
||||||
|
_SQUASHED = re.compile(r"[\s-]+")
|
||||||
|
|
||||||
|
|
||||||
|
def remote_in_text(title: str, text: str) -> bool:
|
||||||
|
"""Whether the title or the text says the role is worked remotely: a
|
||||||
|
phrase of `REMOTE_TEXT` left once those of `REMOTE_TEXT_NOT` are taken
|
||||||
|
out. Cheap evidence, no model: a role marked so passes a location gate
|
||||||
|
on remote until extract has read it (ADR Fetch prep and score 1.1), so a
|
||||||
|
false mark costs a cut slot at most, and a missed one nothing new."""
|
||||||
|
folded = _SQUASHED.sub(" ", f"{title or ''} {text or ''}".lower())
|
||||||
|
return bool(_REMOTE_TEXT.search(_REMOTE_TEXT_NOT.sub(" ", folded)))
|
||||||
|
|
||||||
|
|
||||||
def _line_word(text: str) -> str:
|
def _line_word(text: str) -> str:
|
||||||
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
"""`remote_of`'s reading of a line, else its Danish and Swedish words."""
|
||||||
if found := remote_of(text):
|
if found := remote_of(text):
|
||||||
|
|||||||
+78
-107
@@ -1,19 +1,23 @@
|
|||||||
/* The Search agents page: what Agent match on the Roles tab is measured
|
/* The Search agents page: what the order What I'm looking for on the Roles
|
||||||
* against, one agent at a time.
|
* tab is measured against, one agent at a time.
|
||||||
*
|
*
|
||||||
* Every agent from `/api/agents` is a card, newest first: its name, its
|
* Every agent from `/api/agents` is a card, newest first: its name, its
|
||||||
* switch, its prompts, its gates in words, its weights, and a link to the
|
* switch, its prompts, its gates in words, its weights, and a link to the
|
||||||
* Roles tab in its order. The prompts are the body, since fit is measured
|
* Roles tab in its order. The prompts are the body, since prompt closeness is
|
||||||
* against the one a role fits best; the weights only share out the terms.
|
* measured against the one a role is closest to; the weights only share out the terms.
|
||||||
* The form edits the prompts as rows, and the fetch gate in three parts: the
|
* The form edits the prompts as rows, and the fetch gate in four parts, in the
|
||||||
* region (the scope select), the place (a short list of lines a role passes by
|
* ADR's order: the world (the scope select), the location (a short list of
|
||||||
* matching any one of: countries, towns, remote) and the role (the fetch terms
|
* lines a role passes by matching any one of: countries, towns, remote), the
|
||||||
* and the titles to leave out), over a row per remaining rule under Advanced;
|
* role exclusion (the titles to leave out) and the role fetch (the fetch
|
||||||
* a gate the three parts cannot say lands in a row, so a load and a save lose
|
* terms), and nothing else: seniority, pay and the like are filters on Roles.
|
||||||
* nothing. A gate is a field, values and a mode (must, one of, not), with no
|
* A gate is a field, values and a mode (must, one of, not), with no weight.
|
||||||
* weight. After it come the feed order (the four weights of the final score and
|
* A stored gate the four parts cannot say, which only the config or the API
|
||||||
* the CV match weights), the cut (the best N by fit and the best M by CV
|
* writes, is named in a note under the fetch gate and sent back as it came,
|
||||||
* closeness, each with a minimum level) and the size of summarise. Edit turns a card
|
* so a load and a save lose nothing. After the fetch gate come the analytics
|
||||||
|
* filter (the best N by prompt closeness and the best M by CV closeness, each
|
||||||
|
* with a minimum level), the want feed score weights (prompt closeness,
|
||||||
|
* newness, company like, job like), the CV match weights and the size of
|
||||||
|
* summarise. Edit turns a card
|
||||||
* into the form from the page's `<template>`, in place, and New agent puts a
|
* into the form from the page's `<template>`, in place, and New agent puts a
|
||||||
* blank card on top in the same state. One card edits at a time, so the
|
* blank card on top in the same state. One card edits at a time, so the
|
||||||
* form's ids stay unique. Votes, liked companies and scores are the agent's,
|
* form's ids stay unique. Votes, liked companies and scores are the agent's,
|
||||||
@@ -28,10 +32,7 @@
|
|||||||
* text (typed, or read from a .md/.txt file by the browser), the languages it
|
* text (typed, or read from a .md/.txt file by the browser), the languages it
|
||||||
* states as rows of a code and a level, and the two soft texts, what the
|
* states as rows of a code and a level, and the two soft texts, what the
|
||||||
* person likes and what they do not like (the work, the workplace,
|
* person likes and what they do not like (the work, the workplace,
|
||||||
* situations), which the judge reads for the career score. The must-have text
|
* situations), which the judge reads for the career score.
|
||||||
* is no longer on the form: a save leaves `must_text` out, so the stored one
|
|
||||||
* is kept (it is set through the API, and still caps career at 39 in code
|
|
||||||
* when the judge reports one broken).
|
|
||||||
* Every save sends the CV, languages and the two texts, and the card's switch
|
* Every save sends the CV, languages and the two texts, and the card's switch
|
||||||
* re-posts the row as the server gave it, so neither drops them.
|
* re-posts the row as the server gave it, so neither drops them.
|
||||||
*
|
*
|
||||||
@@ -47,13 +48,14 @@
|
|||||||
const DEFAULT_WEIGHTS = { fit: 0.6, recency: 0.2, affinity: 0.2, taste: 0.2,
|
const DEFAULT_WEIGHTS = { fit: 0.6, recency: 0.2, affinity: 0.2, taste: 0.2,
|
||||||
cv_skills: 30, cv_experience: 25, cv_career: 30 };
|
cv_skills: 30, cv_experience: 25, cv_career: 30 };
|
||||||
// What the models read after a pull, per agent (`app.models.Limits`): the
|
// What the models read after a pull, per agent (`app.models.Limits`): the
|
||||||
// cut is the best `cut_fit_top` roles by fit at `cut_fit_min` or more, then
|
// analytics filter is the best `cut_fit_top` roles by prompt closeness at
|
||||||
// the best `cut_cv_top` by CV closeness at `cut_cv_min` or more that the
|
// `cut_fit_min` or more, then the best `cut_cv_top` by CV closeness at
|
||||||
// first list leaves out, and every role liked or saved. Fetch text, extract
|
// `cut_cv_min` or more that the first list leaves out, and every role liked
|
||||||
// and judge read the cut; summarise reads the top `summarise_top` of the
|
// or saved: the cut. Fetch text, extract and judge read the cut; summarise
|
||||||
// feed and the roles liked or saved.
|
// reads the top `summarise_top` of each feed order, want and match, and the
|
||||||
|
// roles liked or saved.
|
||||||
const DEFAULT_LIMITS = { cut_fit_top: 500, cut_fit_min: 0.6, cut_cv_top: 150, cut_cv_min: 0.6,
|
const DEFAULT_LIMITS = { cut_fit_top: 500, cut_fit_min: 0.6, cut_cv_top: 150, cut_cv_min: 0.6,
|
||||||
summarise_top: 50 };
|
summarise_top: 30 };
|
||||||
// The levels of a language, the order of the server's ladder, and how big a
|
// The levels of a language, the order of the server's ladder, and how big a
|
||||||
// CV file the browser reads into the box.
|
// CV file the browser reads into the box.
|
||||||
const LEVELS = ["basic", "conversational", "professional", "fluent", "native"];
|
const LEVELS = ["basic", "conversational", "professional", "fluent", "native"];
|
||||||
@@ -61,9 +63,8 @@
|
|||||||
const NEW = "new";
|
const NEW = "new";
|
||||||
// Europe as the starter agents spell it (static/remote.js).
|
// Europe as the starter agents spell it (static/remote.js).
|
||||||
const EUROPE = rh.EUROPE;
|
const EUROPE = rh.EUROPE;
|
||||||
const FIELDS = ["place", "country", "remote", "remote_from", "seniority", "function", "salary_min",
|
// A gate's mode in the words of the note on the gates the form keeps.
|
||||||
"currency", "permit", "languages", "company", "title"];
|
const MODES = { require: "must", any: "one of", exclude: "not" };
|
||||||
const MODES = [["require", "must"], ["any", "one of"], ["exclude", "not"]];
|
|
||||||
|
|
||||||
/** @type {object[]} every agent, as the server lists them */
|
/** @type {object[]} every agent, as the server lists them */
|
||||||
let agents = [];
|
let agents = [];
|
||||||
@@ -233,7 +234,7 @@
|
|||||||
|
|
||||||
function weightsLine(row) {
|
function weightsLine(row) {
|
||||||
const w = Object.assign({}, DEFAULT_WEIGHTS, row.weights);
|
const w = Object.assign({}, DEFAULT_WEIGHTS, row.weights);
|
||||||
return `fit ${w.fit} · newness ${w.recency} · companies ${w.affinity} · liked ${w.taste}`
|
return `prompt closeness ${w.fit} · newness ${w.recency} · company like ${w.affinity} · job like ${w.taste}`
|
||||||
+ ` · CV match: skills ${w.cv_skills}, experience ${w.cv_experience}, career ${w.cv_career}`;
|
+ ` · CV match: skills ${w.cv_skills}, experience ${w.cv_experience}, career ${w.cv_career}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -263,8 +264,8 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The region and place in words, one sentence led by the region, then the lines a
|
// The region and place in words, one sentence led by the region, then the lines a
|
||||||
// role may match, with the one-of rows among them since they are the same
|
// role may match, with any other one-of gates among them since they are the
|
||||||
// group: "Within Europe: keeps roles anywhere in Denmark, in Malmö, Lund or
|
// same group: "Within Europe: keeps roles anywhere in Denmark, in Malmö, Lund or
|
||||||
// Helsingborg, or remote, and leaves out the rest." The lines add up, which
|
// Helsingborg, or remote, and leaves out the rest." The lines add up, which
|
||||||
// the first wording ("Roles in ...") left the owner to guess.
|
// the first wording ("Roles in ...") left the owner to guess.
|
||||||
function whereSentence(where, others) {
|
function whereSentence(where, others) {
|
||||||
@@ -299,18 +300,18 @@
|
|||||||
return sentences.join(" ");
|
return sentences.join(" ");
|
||||||
}
|
}
|
||||||
|
|
||||||
// What the models read after a pull: the cut, then the stages that read
|
// What the models read after a pull: the analytics filter's cut, then the
|
||||||
// it, then summarise's own size. A stage that is off says so, with a link to
|
// stages that read it, then summarise's own size. A stage that is off says so, with a link to
|
||||||
// its switch under Runs: the numbers are only what it would read once it is
|
// its switch under Runs: the numbers are only what it would read once it is
|
||||||
// on. The stage names are those of `/api/stages`.
|
// on. The stage names are those of `/api/stages`.
|
||||||
function limitsLine(row) {
|
function limitsLine(row) {
|
||||||
const l = Object.assign({}, DEFAULT_LIMITS, row.limits);
|
const l = Object.assign({}, DEFAULT_LIMITS, row.limits);
|
||||||
const p = el("p", "agent-limits");
|
const p = el("p", "agent-limits");
|
||||||
const lists = [];
|
const lists = [];
|
||||||
if (l.cut_fit_top) lists.push(`the best ${l.cut_fit_top} by fit at ${l.cut_fit_min} or more`);
|
if (l.cut_fit_top) lists.push(`the best ${l.cut_fit_top} by prompt closeness at ${l.cut_fit_min} or more`);
|
||||||
if (l.cut_cv_top) lists.push(`the best ${l.cut_cv_top} by closeness to the CV at ${l.cut_cv_min} or more`);
|
if (l.cut_cv_top) lists.push(`the best ${l.cut_cv_top} by CV closeness at ${l.cut_cv_min} or more`);
|
||||||
const cut = lists.length ? `${lists.join(", then ")}, and liked and saved roles` : "liked and saved roles only";
|
const cut = lists.length ? `${lists.join(", then ")}, and liked and saved roles` : "liked and saved roles only";
|
||||||
p.append(`The cut: ${cut}.`);
|
p.append(`Analytics filter: ${cut}.`);
|
||||||
const off = (name, would) => {
|
const off = (name, would) => {
|
||||||
const a = link("/settings/sources#stages", `${name} is off`);
|
const a = link("/settings/sources#stages", `${name} is off`);
|
||||||
a.title = "Switch it on under Runs.";
|
a.title = "Switch it on under Runs.";
|
||||||
@@ -323,7 +324,7 @@
|
|||||||
p.append(` ${said[0].toUpperCase()}${said.slice(1)} ${on.length > 1 ? "read" : "reads"} it`);
|
p.append(` ${said[0].toUpperCase()}${said.slice(1)} ${on.length > 1 ? "read" : "reads"} it`);
|
||||||
}
|
}
|
||||||
for (const [key, name] of reads) if (stageOn[key] === false) off(name, "reads the cut");
|
for (const [key, name] of reads) if (stageOn[key] === false) off(name, "reads the cut");
|
||||||
const summarised = `the top ${l.summarise_top} of the feed and liked and saved roles`;
|
const summarised = `the top ${l.summarise_top} of each feed order and liked and saved roles`;
|
||||||
if (stageOn.summarise === false) off("summarise", `summarises ${summarised}`);
|
if (stageOn.summarise === false) off("summarise", `summarises ${summarised}`);
|
||||||
else p.append(` · summarises ${summarised}`);
|
else p.append(` · summarises ${summarised}`);
|
||||||
return p;
|
return p;
|
||||||
@@ -345,7 +346,7 @@
|
|||||||
|
|
||||||
const prompts = promptsOf(row).filter((p) => p.text.trim());
|
const prompts = promptsOf(row).filter((p) => p.text.trim());
|
||||||
if (!prompts.length) {
|
if (!prompts.length) {
|
||||||
div.appendChild(el("p", "agent-intent empty", "Nothing described yet, so fit has nothing to measure against. Edit says what this agent looks for."));
|
div.appendChild(el("p", "agent-intent empty", "Nothing described yet, so prompt closeness has nothing to measure against. Edit says what this agent looks for."));
|
||||||
}
|
}
|
||||||
for (const prompt of prompts) {
|
for (const prompt of prompts) {
|
||||||
// A lone prompt named after its agent, as a migrated intent is, says its name once.
|
// A lone prompt named after its agent, as a migrated intent is, says its name once.
|
||||||
@@ -381,7 +382,7 @@
|
|||||||
// ---------------------------------------------------------------- the form
|
// ---------------------------------------------------------------- the form
|
||||||
|
|
||||||
// Tell the form something changed that no field typed: a chip or a row
|
// Tell the form something changed that no field typed: a chip or a row
|
||||||
// added or removed. It marks the form dirty and redraws the gates view.
|
// added or removed. It marks the form dirty and redraws the gates' sentence.
|
||||||
function changed(node) {
|
function changed(node) {
|
||||||
node.dispatchEvent(new Event("input", { bubbles: true }));
|
node.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
}
|
}
|
||||||
@@ -556,9 +557,9 @@
|
|||||||
return box;
|
return box;
|
||||||
}
|
}
|
||||||
|
|
||||||
// A language as a row, after the rule rows: a code, a level, Remove. The
|
// A language as a row: a code, a level, Remove. The code is two letters,
|
||||||
// code is two letters, which the server turns into the language; its name
|
// which the server turns into the language; its name is the row's title
|
||||||
// is the row's title where the browser knows it.
|
// where the browser knows it.
|
||||||
function languageRow(language, remove) {
|
function languageRow(language, remove) {
|
||||||
const row = el("div", "rule-row");
|
const row = el("div", "rule-row");
|
||||||
const code = el("input");
|
const code = el("input");
|
||||||
@@ -584,32 +585,32 @@
|
|||||||
return { node: row, read: () => ({ language: code.value.trim().toLowerCase(), level: level.value }) };
|
return { node: row, read: () => ({ language: code.value.trim().toLowerCase(), level: level.value }) };
|
||||||
}
|
}
|
||||||
|
|
||||||
// A gate as a row: field, mode, values and Remove. Values untouched since the load are sent as they came, so a
|
// The gates the fetch gate's four parts say, in the order the starter
|
||||||
// value with a comma in it survives a save.
|
// agents have them: the world, the location lines and the titles left out,
|
||||||
function ruleRow(gate, remove) {
|
// then `kept`, the stored gates the parts cannot say, as they came. `parts`
|
||||||
const row = el("div", "rule-row");
|
// holds what the form reads: `scope` (europe, countries, everywhere) and
|
||||||
const field = select(FIELDS.map((f) => [f, f.replace("_", " ")]), gate.field, "Field");
|
// its `scoped` countries, the `within` countries, the `towns`, `remote`
|
||||||
const mode = select(MODES, gate.mode, "Mode");
|
// (none, scope, from) and its `from` areas, and the `titles`.
|
||||||
const values = el("input");
|
function formGates(parts, kept = []) {
|
||||||
values.type = "text";
|
const out = [];
|
||||||
values.required = true;
|
const { scope, scoped, within, towns: lines, remote, from, titles } = parts;
|
||||||
values.spellcheck = false;
|
if (scope === "europe") out.push({ field: "country", values: rh.europeWith(within), mode: "require" });
|
||||||
values.placeholder = "senior, staff";
|
if (scope === "countries" && scoped.length) out.push({ field: "country", values: scoped, mode: "require" });
|
||||||
values.value = gate.values.join(", ");
|
if (within.length) out.push({ field: "country", values: within, mode: "any" });
|
||||||
values.setAttribute("aria-label", "Values, separated by commas");
|
if (remote === "scope") out.push({ field: "remote", values: ["remote"], mode: "any" });
|
||||||
const loaded = values.value;
|
if (remote === "from" && from.length) out.push({ field: "remote_from", values: from, mode: "any" });
|
||||||
const drop = linky("Remove");
|
if (lines.length) out.push({ field: "place", values: lines, mode: "any" });
|
||||||
drop.addEventListener("click", () => remove(row));
|
if (titles.length) out.push({ field: "title", values: titles, mode: "exclude" });
|
||||||
row.append(field, mode, values, drop);
|
return out.concat(kept);
|
||||||
|
|
||||||
function read() {
|
|
||||||
const parts = values.value === loaded
|
|
||||||
? gate.values
|
|
||||||
: values.value.split(",").map((v) => v.trim()).filter(Boolean)
|
|
||||||
.map((v) => (field.value === "salary_min" ? Number(v) : v));
|
|
||||||
return { field: field.value, values: parts, mode: mode.value };
|
|
||||||
}
|
}
|
||||||
return { node: row, read };
|
|
||||||
|
// The note on the gates the form keeps: "The form cannot edit these rules,
|
||||||
|
// and Save keeps them as stored: must: with seniority senior; not: at Acme."
|
||||||
|
function keptNote(kept) {
|
||||||
|
if (!kept.length) return "";
|
||||||
|
const said = kept.map((g) => `${MODES[g.mode] || g.mode}: ${phrase(g)}`).join("; ");
|
||||||
|
return `The form cannot edit ${kept.length > 1 ? "these rules" : "this rule"}, and Save keeps `
|
||||||
|
+ `${kept.length > 1 ? "them" : "it"} as stored: ${said}.`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function form(row) {
|
function form(row) {
|
||||||
@@ -710,51 +711,18 @@
|
|||||||
remoteFrom.node.hidden = remoteMode.value !== "from";
|
remoteFrom.node.hidden = remoteMode.value !== "from";
|
||||||
};
|
};
|
||||||
offer();
|
offer();
|
||||||
$("details.advanced", f).open = rest.length > 0;
|
const kept = $("#a-kept", f);
|
||||||
|
kept.textContent = keptNote(rest);
|
||||||
|
kept.hidden = !rest.length;
|
||||||
|
|
||||||
const rules = [];
|
// The gates Save sends: the four parts', then the kept ones. The sentence
|
||||||
const dropRule = (node) => {
|
// under them says them all, as the card will, the kept ones included.
|
||||||
rules.splice(rules.findIndex((r) => r.node === node), 1);
|
const gates = () => formGates({
|
||||||
node.remove();
|
scope: scope.value, scoped: scoped.values(), within: countries.values(), towns: places.values(),
|
||||||
changed(f);
|
remote: remoteMode.value, from: remoteFrom.values(), titles: titles.values(),
|
||||||
};
|
}, rest);
|
||||||
const addRule = (gate) => {
|
|
||||||
const entry = ruleRow(gate, dropRule);
|
|
||||||
rules.push(entry);
|
|
||||||
$("#a-rules", f).appendChild(entry.node);
|
|
||||||
return entry;
|
|
||||||
};
|
|
||||||
rest.forEach(addRule);
|
|
||||||
$("#a-add-rule", f).addEventListener("click", () => {
|
|
||||||
$("select", addRule({ field: "seniority", values: [], mode: "require" }).node).focus();
|
|
||||||
changed(f);
|
|
||||||
});
|
|
||||||
|
|
||||||
// The gates the fetch gate's three parts say, in the order the starter
|
|
||||||
// agents have them: the region, the place lines, and the titles left out.
|
|
||||||
function whereGates() {
|
|
||||||
const out = [];
|
|
||||||
const within = countries.values();
|
|
||||||
const towns = places.values();
|
|
||||||
const from = remoteFrom.values();
|
|
||||||
if (scope.value === "europe") out.push({ field: "country", values: rh.europeWith(within), mode: "require" });
|
|
||||||
if (scope.value === "countries" && scoped.values().length) {
|
|
||||||
out.push({ field: "country", values: scoped.values(), mode: "require" });
|
|
||||||
}
|
|
||||||
if (within.length) out.push({ field: "country", values: within, mode: "any" });
|
|
||||||
if (remoteMode.value === "scope") out.push({ field: "remote", values: ["remote"], mode: "any" });
|
|
||||||
if (remoteMode.value === "from" && from.length) out.push({ field: "remote_from", values: from, mode: "any" });
|
|
||||||
if (towns.length) out.push({ field: "place", values: towns, mode: "any" });
|
|
||||||
const left = titles.values();
|
|
||||||
if (left.length) out.push({ field: "title", values: left, mode: "exclude" });
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
// The gates Save sends: the fetch gate's, then the rows. The sentence
|
|
||||||
// under it says them all, as the card will, rows included.
|
|
||||||
const gates = () => whereGates().concat(rules.map((r) => r.read()));
|
|
||||||
const view = () => {
|
const view = () => {
|
||||||
offer();
|
offer();
|
||||||
$("#a-gates", f).value = JSON.stringify(gates(), null, 2);
|
|
||||||
$("#a-where-sum", f).textContent = rulesLine(gates()) || "Passes every role.";
|
$("#a-where-sum", f).textContent = rulesLine(gates()) || "Passes every role.";
|
||||||
};
|
};
|
||||||
view();
|
view();
|
||||||
@@ -978,6 +946,9 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// For tests/js/agents.test.mjs: what Save sends and how it is said.
|
||||||
|
window.agentGates = { formGates, keptNote, rulesLine };
|
||||||
|
|
||||||
document.addEventListener("DOMContentLoaded", async () => {
|
document.addEventListener("DOMContentLoaded", async () => {
|
||||||
if (!$("#agents-list")) return;
|
if (!$("#agents-list")) return;
|
||||||
$("#a-new").addEventListener("click", () => open(null));
|
$("#a-new").addEventListener("click", () => open(null));
|
||||||
|
|||||||
+10
-22
@@ -579,10 +579,12 @@ kbd {
|
|||||||
font-size: .8rem; padding: .18rem .3rem;
|
font-size: .8rem; padding: .18rem .3rem;
|
||||||
border: 1px solid var(--line);
|
border: 1px solid var(--line);
|
||||||
}
|
}
|
||||||
/* The three order labels share one line: Agent match is the longest and
|
/* The three order labels share one row. What I'm looking for is longer than
|
||||||
would otherwise wrap in a 240px rail. Cells grow to the width, each by its
|
a 240px rail has room for on one line, so a cell may wrap its words; the
|
||||||
own text. */
|
cells split the width by their text, the short ones keeping theirs. */
|
||||||
#sort button { white-space: nowrap; }
|
#sort button { min-width: 0; line-height: 1.2; }
|
||||||
|
#sort button:nth-child(2), #sort button:nth-child(3) { flex-grow: 0; white-space: nowrap; }
|
||||||
|
#sort button:first-child { flex-basis: 0; flex-grow: 1; }
|
||||||
/* Square everything first; only the ends are ends. Every cell but the first
|
/* Square everything first; only the ends are ends. Every cell but the first
|
||||||
eats its neighbour's border, so the joins are one rule wide rather than two. */
|
eats its neighbour's border, so the joins are one rule wide rather than two. */
|
||||||
.seg > * { border-radius: 0; }
|
.seg > * { border-radius: 0; }
|
||||||
@@ -976,14 +978,6 @@ body[data-reader="open"] .company-strip { display: none; }
|
|||||||
.reader-cv .cv-total { margin: 0 0 .45rem; font-variant-numeric: tabular-nums; }
|
.reader-cv .cv-total { margin: 0 0 .45rem; font-variant-numeric: tabular-nums; }
|
||||||
.reader-cv .cv-total b { font-size: 1.25rem; }
|
.reader-cv .cv-total b { font-size: 1.25rem; }
|
||||||
.reader-cv .cv-total span { display: inline-block; margin-left: .4rem; color: var(--dim); font-size: .88rem; }
|
.reader-cv .cv-total span { display: inline-block; margin-left: .4rem; color: var(--dim); font-size: .88rem; }
|
||||||
/* The must-haves, one line each with its verdict's mark. */
|
|
||||||
.cv-musts { list-style: none; margin: 0 0 .55rem; padding: 0; font-size: .92rem; line-height: 1.4; }
|
|
||||||
.cv-musts li { display: flex; gap: .45rem; }
|
|
||||||
.cv-musts li + li { margin-top: .15rem; }
|
|
||||||
.cv-musts .mark { flex: none; width: 1rem; text-align: center; font-weight: 700; }
|
|
||||||
.cv-musts .must-met .mark { color: var(--good); }
|
|
||||||
.cv-musts .must-broken .mark { color: var(--error); }
|
|
||||||
.cv-musts .must-unclear .mark { color: var(--faint); }
|
|
||||||
.cv-lists { display: grid; grid-template-columns: 1fr 1fr; gap: .5rem 1.2rem; }
|
.cv-lists { display: grid; grid-template-columns: 1fr 1fr; gap: .5rem 1.2rem; }
|
||||||
.cv-lists ul { margin: 0; padding-left: 1.1rem; font-size: .92rem; line-height: 1.45; }
|
.cv-lists ul { margin: 0; padding-left: 1.1rem; font-size: .92rem; line-height: 1.45; }
|
||||||
.cv-lists li + li { margin-top: .2rem; }
|
.cv-lists li + li { margin-top: .2rem; }
|
||||||
@@ -1210,7 +1204,7 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
|||||||
.affinity button[data-value="-1"][aria-pressed="true"] {
|
.affinity button[data-value="-1"][aria-pressed="true"] {
|
||||||
color: var(--accent-fg); background: var(--error); border-color: var(--error);
|
color: var(--accent-fg); background: var(--error); border-color: var(--error);
|
||||||
}
|
}
|
||||||
/* The two fetch overrides, All countries and All roles, global and apart
|
/* The two fetch overrides, All locations and All roles, global and apart
|
||||||
from the stances: green, the colour the app gives what widens (`--good`). */
|
from the stances: green, the colour the app gives what widens (`--good`). */
|
||||||
.affinity button[data-value="countries"][aria-pressed="true"],
|
.affinity button[data-value="countries"][aria-pressed="true"],
|
||||||
.affinity button[data-value="roles"][aria-pressed="true"] {
|
.affinity button[data-value="roles"][aria-pressed="true"] {
|
||||||
@@ -1306,8 +1300,6 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
|||||||
background: var(--bg); border: 1px solid var(--line);
|
background: var(--bg); border: 1px solid var(--line);
|
||||||
border-radius: .3rem; padding: .35rem .5rem; width: 100%;
|
border-radius: .3rem; padding: .35rem .5rem; width: 100%;
|
||||||
}
|
}
|
||||||
/* Prose in a textarea is read in the page face; JSON is in the mono one. */
|
|
||||||
.agent-form textarea.code { font-family: var(--mono); font-size: .85rem; }
|
|
||||||
.agent-form .hint { color: var(--dim); font-size: .8rem; overflow-wrap: anywhere; }
|
.agent-form .hint { color: var(--dim); font-size: .8rem; overflow-wrap: anywhere; }
|
||||||
.weights { display: flex; gap: .75rem; flex-wrap: wrap; border: 1px solid var(--line); border-radius: .3rem; }
|
.weights { display: flex; gap: .75rem; flex-wrap: wrap; border: 1px solid var(--line); border-radius: .3rem; }
|
||||||
/* A label that wraps to two lines pushes its box down; the boxes stay in line. */
|
/* A label that wraps to two lines pushes its box down; the boxes stay in line. */
|
||||||
@@ -1322,14 +1314,13 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
|||||||
}
|
}
|
||||||
/* The fieldsets that hold rows: a column, the Add button under the rows. */
|
/* The fieldsets that hold rows: a column, the Add button under the rows. */
|
||||||
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
|
.agent-form fieldset.prompts, .agent-form fieldset.where, .agent-form fieldset.titles,
|
||||||
.agent-form fieldset.terms,
|
.agent-form fieldset.terms {
|
||||||
.agent-form fieldset.rules {
|
|
||||||
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
|
display: flex; flex-direction: column; align-items: stretch; gap: .5rem; min-width: 0;
|
||||||
border: 1px solid var(--line); border-radius: .3rem;
|
border: 1px solid var(--line); border-radius: .3rem;
|
||||||
}
|
}
|
||||||
.agent-form fieldset > button { align-self: flex-start; }
|
.agent-form fieldset > button { align-self: flex-start; }
|
||||||
/* A prompt or a rule as a row of a list, after `.crow`: the name and Remove
|
/* A prompt or a language as a row of a list, after `.crow`: the name and Remove
|
||||||
on line one, the text spanning the row under them. A rule row wraps like
|
on line one, the text spanning the row under them. A language row wraps like
|
||||||
`.crow .board-edit`, so a phone takes it in two lines rather than scrolling. */
|
`.crow .board-edit`, so a phone takes it in two lines rather than scrolling. */
|
||||||
.prompt-row {
|
.prompt-row {
|
||||||
display: grid; grid-template-columns: 1fr auto; align-items: center;
|
display: grid; grid-template-columns: 1fr auto; align-items: center;
|
||||||
@@ -1359,9 +1350,6 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
|||||||
.agent-form .chips input[type="text"] { flex: 1 1 6rem; width: auto; min-width: 0; }
|
.agent-form .chips input[type="text"] { flex: 1 1 6rem; width: auto; min-width: 0; }
|
||||||
.agent-form .chips select { flex: 0 1 12rem; min-width: 0; max-width: 100%; }
|
.agent-form .chips select { flex: 0 1 12rem; min-width: 0; max-width: 100%; }
|
||||||
.chips .tag span { color: var(--faint); }
|
.chips .tag span { color: var(--faint); }
|
||||||
.agent-form details summary { cursor: pointer; color: var(--dim); font-size: .9rem; }
|
|
||||||
.agent-form details[open] summary { margin-bottom: .5rem; }
|
|
||||||
.agent-form .advanced-body { display: flex; flex-direction: column; gap: .6rem; }
|
|
||||||
|
|
||||||
/* ================================================================ */
|
/* ================================================================ */
|
||||||
/* companies and sources — the Sources page's sections, */
|
/* companies and sources — the Sources page's sections, */
|
||||||
|
|||||||
+16
-16
@@ -16,7 +16,7 @@
|
|||||||
* Beside `api` on `window`, the few helpers more than one page script uses:
|
* Beside `api` on `window`, the few helpers more than one page script uses:
|
||||||
* `count`, `cell`, `debounce`, `safeUrl`, and the company list with its
|
* `count`, `cell`, `debounce`, `safeUrl`, and the company list with its
|
||||||
* affinity control (`loadCompanies`, `companies`, `affinityControl`,
|
* affinity control (`loadCompanies`, `companies`, `affinityControl`,
|
||||||
* `setAffinity`) and the two fetch overrides beside it, All countries and
|
* `setAffinity`) and the two fetch overrides beside it, All locations and
|
||||||
* All roles (`overridesControl`). One
|
* All roles (`overridesControl`). One
|
||||||
* copy each, since two copies of a helper is one of them
|
* copy each, since two copies of a helper is one of them
|
||||||
* being fixed.
|
* being fixed.
|
||||||
@@ -189,26 +189,26 @@
|
|||||||
/* The two fetch overrides, one more pair of toggles in the affinity
|
/* The two fetch overrides, one more pair of toggles in the affinity
|
||||||
buttons' fragment beside them, in a group of their own since they are
|
buttons' fragment beside them, in a group of their own since they are
|
||||||
global rather than an agent's stance (app/jobs/reach.py). The fetch gate
|
global rather than an agent's stance (app/jobs/reach.py). The fetch gate
|
||||||
has three parts, region, place and role, and a company follows every
|
has four parts, world, location, role exclusion and role fetch, and a
|
||||||
agent's on all three unless it is set to:
|
company follows every agent's unless it is set to:
|
||||||
- All countries: the place part is skipped. The company's roles are
|
- All locations: the location gate is skipped. The company's roles are
|
||||||
fetched and kept from the whole region (Europe), not only the agents'
|
fetched and kept from the whole world gate (Europe, say), not only the
|
||||||
places, for every agent. The region itself still holds.
|
agents' locations, for every agent. The world gate itself still holds.
|
||||||
- All roles: the role part is skipped: no fetch terms and no titles to
|
- All roles: role exclusion and role fetch are skipped: no titles to
|
||||||
leave out.
|
leave out and no fetch terms.
|
||||||
Turning either on or off queues a rescore on the server, as the other
|
Turning either on or off queues a rescore on the server, as the other
|
||||||
switches that change what passes do, and a pull reads the board anew.
|
switches that change what passes do, and a pull reads the board anew.
|
||||||
`company` is the shared list's row; `onChange` runs after the server
|
`company` is the shared list's row; `onChange` runs after the server
|
||||||
accepted a switch, with the company it answered. */
|
accepted a switch, with the company it answered. */
|
||||||
const OVERRIDES = [
|
const OVERRIDES = [
|
||||||
{ value: "countries", field: "all_countries", path: "all-countries", text: "All countries",
|
{ value: "countries", field: "all_countries", path: "all-countries", text: "All locations",
|
||||||
on: "All countries is on: this company's roles are fetched and kept from the whole "
|
on: "All locations is on: this company's roles are fetched and kept from the whole "
|
||||||
+ "region (Europe), not only the agents' places, for every agent. The region still "
|
+ "world gate (Europe, say), not only the agents' locations, for every agent. The "
|
||||||
+ "holds. Turning it off puts the agents' places back and rescores.",
|
+ "world gate still holds. Turning it off puts the agents' locations back and rescores.",
|
||||||
off: "All countries: fetch and keep this company's roles from the whole region "
|
off: "All locations: fetch and keep this company's roles from the whole world gate "
|
||||||
+ "(Europe) instead of only the agents' places, for every agent. Turning it on "
|
+ "(Europe, say) instead of only the agents' locations, for every agent. Turning it "
|
||||||
+ "rescores.",
|
+ "on rescores.",
|
||||||
said: (on) => (on ? "all countries in the region" : "the agents' places") },
|
said: (on) => (on ? "all locations in the world gate" : "the agents' locations") },
|
||||||
{ value: "roles", field: "all_roles", path: "all-roles", text: "All roles",
|
{ value: "roles", field: "all_roles", path: "all-roles", text: "All roles",
|
||||||
on: "All roles is on: this company's roles are fetched with no fetch terms and kept "
|
on: "All roles is on: this company's roles are fetched with no fetch terms and kept "
|
||||||
+ "whatever their title, for every agent. Turning it off puts the agents' fetch "
|
+ "whatever their title, for every agent. Turning it off puts the agents' fetch "
|
||||||
|
|||||||
+11
-7
@@ -5,7 +5,7 @@
|
|||||||
* joined with each company's board subscription — its id and whether it is
|
* joined with each company's board subscription — its id and whether it is
|
||||||
* on — so nothing here fetches `/api/sources` as well. The affinity buttons
|
* on — so nothing here fetches `/api/sources` as well. The affinity buttons
|
||||||
* are `affinityControl`, the same ones the Roles tab and the read page draw,
|
* are `affinityControl`, the same ones the Roles tab and the read page draw,
|
||||||
* and beside them `overridesControl`, the two fetch overrides All countries
|
* and beside them `overridesControl`, the two fetch overrides All locations
|
||||||
* and All roles, which are global and drawn with or without an agent.
|
* and All roles, which are global and drawn with or without an agent.
|
||||||
*
|
*
|
||||||
* Liked and hidden are per search agent, so the `#c-agent` select above the
|
* Liked and hidden are per search agent, so the `#c-agent` select above the
|
||||||
@@ -25,10 +25,10 @@
|
|||||||
* from app/jobs/reach.py), so the row shows whether the company follows the
|
* from app/jobs/reach.py), so the row shows whether the company follows the
|
||||||
* agents' filters or overrides them. Its editor has a second line that sets
|
* agents' filters or overrides them. Its editor has a second line that sets
|
||||||
* the board's own reach: the agents', or its own countries and remote,
|
* the board's own reach: the agents', or its own countries and remote,
|
||||||
* posted to `/api/sources/{id}/reach`. Fetching from the whole region (All
|
* posted to `/api/sources/{id}/reach`. Fetching from the whole world gate (All
|
||||||
* countries) and no fetch terms or titles to leave out (All roles) are the
|
* locations) and no fetch terms or titles to leave out (All roles) are the
|
||||||
* company's two overrides instead, and the board's own countries win over
|
* company's two overrides instead, and the board's own countries win over
|
||||||
* All countries.
|
* All locations.
|
||||||
*
|
*
|
||||||
* Beside that line stands the board's size, set when it was added and
|
* Beside that line stands the board's size, set when it was added and
|
||||||
* changed only by a Recheck ("999 roles · checked 30 Sep"). Recheck, beside
|
* changed only by a Recheck ("999 roles · checked 30 Sep"). Recheck, beside
|
||||||
@@ -326,8 +326,8 @@
|
|||||||
note.className = "muted small";
|
note.className = "muted small";
|
||||||
note.textContent = [
|
note.textContent = [
|
||||||
company.all_countries
|
company.all_countries
|
||||||
? "All countries is on: roles are fetched and kept from the whole region, and countries set here win over it."
|
? "All locations is on: roles are fetched and kept from the whole world gate, and countries set here win over it."
|
||||||
: "All countries, beside the stars, fetches and keeps this company's roles from the whole region (Europe), not only the agents' places.",
|
: "All locations, beside the stars, fetches and keeps this company's roles from the whole world gate (Europe, say), not only the agents' locations.",
|
||||||
company.all_roles
|
company.all_roles
|
||||||
? "All roles is on: no fetch terms and no titles to leave out."
|
? "All roles is on: no fetch terms and no titles to leave out."
|
||||||
: "All roles drops the fetch terms and the titles to leave out.",
|
: "All roles drops the fetch terms and the titles to leave out.",
|
||||||
@@ -420,9 +420,13 @@
|
|||||||
const row = document.createElement("div");
|
const row = document.createElement("div");
|
||||||
row.className = "crow";
|
row.className = "crow";
|
||||||
if (company.affinity === -1) row.dataset.hidden = "true";
|
if (company.affinity === -1) row.dataset.hidden = "true";
|
||||||
|
// By key, as the feed's own company links are, so the Roles page opens
|
||||||
|
// the company view with its strip and gate note. A name would be a
|
||||||
|
// substring there, which also lists other companies whose names hold it
|
||||||
|
// and misses roles spelt otherwise.
|
||||||
const link = document.createElement("a");
|
const link = document.createElement("a");
|
||||||
link.className = "name";
|
link.className = "name";
|
||||||
link.href = `/?company=${encodeURIComponent(company.name)}`;
|
link.href = `/?company_keys=${encodeURIComponent(company.key)}`;
|
||||||
link.title = "This company's open roles";
|
link.title = "This company's open roles";
|
||||||
link.textContent = company.name;
|
link.textContent = company.name;
|
||||||
row.appendChild(link);
|
row.appendChild(link);
|
||||||
|
|||||||
+12
-10
@@ -1,7 +1,8 @@
|
|||||||
/* Where a remote role may be worked from, as the pages say it: the words of
|
/* Where a remote role may be worked from, as the pages say it: the words of
|
||||||
* the badge, the picks of the agent's "Remote, open to someone in" chips, and
|
* the badge, the picks of the agent's "Remote, open to someone in" chips, and
|
||||||
* how an agent's gates share out between the fetch gate's blocks and Other rules. No
|
* how an agent's gates share out between the fetch gate's blocks and the rest
|
||||||
* DOM in it, so `tests/js/remote.test.mjs` can walk it without a browser.
|
* the form keeps as stored. No DOM in it, so `tests/js/remote.test.mjs` can
|
||||||
|
* walk it without a browser.
|
||||||
*
|
*
|
||||||
* The area is a place token in three spellings, one per layer: `remote:<place
|
* The area is a place token in three spellings, one per layer: `remote:<place
|
||||||
* token>` on the Roles rail and in the address, `<place token>` in a
|
* token>` on the Roles rail and in the address, `<place token>` in a
|
||||||
@@ -61,18 +62,19 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Share out an agent's gates between the fetch gate's blocks (region,
|
/** Share out an agent's gates between the fetch gate's blocks (region,
|
||||||
place, titles to leave out) and Other rules. Each line takes the first
|
place, titles to leave out) and the rest, which the form cannot edit
|
||||||
gate it can say whole; the rest are rows. A gate has a field, values and
|
and keeps as stored. Each line takes the first gate it can say whole.
|
||||||
a mode (require, any, exclude) and no weight; a gate with a mode the
|
A gate has a field, values and a mode (require, any, exclude) and no
|
||||||
form does not know, such as an old prefer, stays a row.
|
weight; a gate with a mode the form does not know, such as an old
|
||||||
|
prefer, is in the rest.
|
||||||
|
|
||||||
The scope line is the first plain require gate on country: `europe`
|
The scope line is the first plain require gate on country: `europe`
|
||||||
when it is exactly Europe and the countries line (`europeWith`),
|
when it is exactly Europe and the countries line (`europeWith`),
|
||||||
`countries` with its codes as chips otherwise, `everywhere` when there
|
`countries` with its codes as chips otherwise, `everywhere` when there
|
||||||
is none. Any other require gate on country is a row. The remote line is
|
is none. Any other require gate on country is in the rest. The remote
|
||||||
`scope` for the one-of on `remote: [remote]` (remote, within the
|
line is `scope` for the one-of on `remote: [remote]` (remote, within
|
||||||
scope), `from` for a one-of on `remote_from`, else `none`; when both
|
the scope), `from` for a one-of on `remote_from`, else `none`; when
|
||||||
are there `remote_from` wins and the plain one stays a row. */
|
both are there `remote_from` wins and the plain one is in the rest. */
|
||||||
function distribute(gates) {
|
function distribute(gates) {
|
||||||
const where = { scope: null, countries: null, places: null, remote: { mode: "none", values: [] }, titles: null };
|
const where = { scope: null, countries: null, places: null, remote: { mode: "none", values: [] }, titles: null };
|
||||||
const rest = [];
|
const rest = [];
|
||||||
|
|||||||
+288
-160
@@ -10,10 +10,11 @@
|
|||||||
*
|
*
|
||||||
* A row is newsdesk's `.row`: the score gutter and number (hidden under
|
* A row is newsdesk's `.row`: the score gutter and number (hidden under
|
||||||
* Newest, since the score is not what put the row there, and the CV
|
* Newest, since the score is not what put the row there, and the CV
|
||||||
* match under CV match, a dash for a role not judged yet), the title,
|
* match under What fits me, a dash for a role not judged yet), the title,
|
||||||
* company · place · badges · age · tags, the summary or the description's
|
* company · place · badges · age · tags, the summary or the description's
|
||||||
* opening, and ▼ ● ▲ ★ from judge.js. A passed role and a hidden company's
|
* opening, and ▼ ● ▲ ★ from judge.js. A passed role is dimmed, never
|
||||||
* are dimmed, never dropped, so a stray tap stays visible to be undone.
|
* dropped, so a stray tap stays visible to be undone. A hidden company's roles
|
||||||
|
* are never in the feed.
|
||||||
*
|
*
|
||||||
* Opening a role pushes `/roles/{uid}`; closing goes back. Stepping with j/k
|
* Opening a role pushes `/roles/{uid}`; closing goes back. Stepping with j/k
|
||||||
* while it is open replaces that entry rather than stacking one per role, so
|
* while it is open replaces that entry rather than stacking one per role, so
|
||||||
@@ -29,17 +30,22 @@
|
|||||||
* and each company affinity. One is always picked while any is enabled: the
|
* and each company affinity. One is always picked while any is enabled: the
|
||||||
* one in `?agent=`, else the one this browser viewed last (localStorage),
|
* one in `?agent=`, else the one this browser viewed last (localStorage),
|
||||||
* else the enabled one with the lowest id, which is the server's default too.
|
* else the enabled one with the lowest id, which is the server's default too.
|
||||||
|
* The feed's first request carries the first two before the agents are in,
|
||||||
|
* and waits for them when there is neither, since the server gates nothing
|
||||||
|
* for a request that names no agent (`start`).
|
||||||
* Picking another reloads the feed and the counts under the reader, which
|
* Picking another reloads the feed and the counts under the reader, which
|
||||||
* stays open and is fetched again for that agent's vote.
|
* stays open and is fetched again for that agent's vote.
|
||||||
*
|
*
|
||||||
* Order is Agent match (`order=score`, the agent's final score moved by its
|
* Order is What I'm looking for (`order=score`, the agent's want feed score
|
||||||
* votes), Newest (`order=newest`) or CV match (`order=cv`, the roles the
|
* moved by its votes), Newest (`order=newest`) or What fits me (`order=cv`,
|
||||||
* agent's `judge` read against its CV first, by that total), the same three
|
* the roles the agent's `judge` read against its CV first, by that total),
|
||||||
* whichever agent is viewed, and all carry the agent's votes and hidden
|
* the same three whichever agent is viewed, and all carry the agent's votes.
|
||||||
* companies. CV match needs a CV: an agent without one is shown Agent
|
* What fits me needs a CV: an agent without one is shown What I'm looking
|
||||||
* match, as an agentless page is shown Newest. Every order keeps to the
|
* for, as an agentless page is shown Newest. Every order keeps to the
|
||||||
* roles the agent's fetch gate passes, which a note under Viewing as says;
|
* roles the agent's fetch gate passes, which a note under Viewing as says,
|
||||||
* Show gated (`gated=true`) adds the ones it rejects, with the reason.
|
* and for a phone, whose rail is shut, the end of the list and the empty
|
||||||
|
* state; Show gated (`gated=true`) adds the ones it rejects, with the
|
||||||
|
* reason. The reader says when the gate rejects the role open.
|
||||||
*
|
*
|
||||||
* Saved filters belong to the viewing agent (`/api/agents/{id}/filters`):
|
* Saved filters belong to the viewing agent (`/api/agents/{id}/filters`):
|
||||||
* pills in the rail, newsdesk's `#active` row, each the query string
|
* pills in the rail, newsdesk's `#active` row, each the query string
|
||||||
@@ -75,14 +81,14 @@
|
|||||||
* screen with that company as its one company chip (`?company_keys=`); a
|
* screen with that company as its one company chip (`?company_keys=`); a
|
||||||
* plain click filters in place and pushes. With one company filtered a strip
|
* plain click filters in place and pushes. With one company filtered a strip
|
||||||
* over the feed names it, counts its roles, links its careers page and
|
* over the feed names it, counts its roles, links its careers page and
|
||||||
* carries its ⊘ ▲ ★ for the viewing agent. `?company=`, `?company_keys=` and
|
* carries its ⊘ ▲ ★ for the viewing agent. `?company_keys=` and `?agent=`
|
||||||
* `?agent=` are how the Companies and Search agents pages link in.
|
* are how Settings › Sources and Search agents link in.
|
||||||
*
|
*
|
||||||
* Company and Place in the rail are chip inputs (chips.js) on the suggest
|
* Company and Place in the rail are chip inputs (chips.js) on the suggest
|
||||||
* routes: a chip is a company key or a place token (`country:DK`,
|
* routes: a chip is a company key or a place token (`country:DK`,
|
||||||
* `region:skane`), several joined by OR. `?company=` and `?place=`, the
|
* `region:skane`), several joined by OR. `?company=` and `?place=`, the
|
||||||
* substrings of the links from other pages, still filter and show as a chip
|
* substrings of older links and saved filters, still filter and show as a
|
||||||
* of their own to take off; nothing in the rail types a new one. Seniority
|
* chip of their own to take off; nothing in the rail types a new one. Seniority
|
||||||
* (a From and a To rung and a Track, `level_from`, `level_to` and `track`)
|
* (a From and a To rung and a Track, `level_from`, `level_to` and `track`)
|
||||||
* and Pay carry the counts of `/api/roles/facets`, asked after each reload
|
* and Pay carry the counts of `/api/roles/facets`, asked after each reload
|
||||||
* with the feed's own parameters, so each says what a click would add. The
|
* with the feed's own parameters, so each says what a click would add. The
|
||||||
@@ -90,8 +96,11 @@
|
|||||||
* an old `?seniority=` is ignored.
|
* an old `?seniority=` is ignored.
|
||||||
*
|
*
|
||||||
* The list is paged: 60 rows, then 60 more each time the end of the list comes
|
* The list is paged: 60 rows, then 60 more each time the end of the list comes
|
||||||
* within 600px of the screen, with the offset the rows already shown. Back to
|
* within 600px of the screen. The server sorts afresh on every request and a
|
||||||
* this page from another tab puts the same number of rows and the scroll
|
* vote or a read moves a role in its list or out of it, so the next page
|
||||||
|
* starts a row earlier for each and a row already on screen is dropped
|
||||||
|
* (`pageAsk`, `pageMerge`): none is skipped and none is shown twice. Back
|
||||||
|
* to this page from another tab puts the same number of rows and the scroll
|
||||||
* position back, from sessionStorage.
|
* position back, from sessionStorage.
|
||||||
*/
|
*/
|
||||||
(function () {
|
(function () {
|
||||||
@@ -128,10 +137,11 @@
|
|||||||
};
|
};
|
||||||
const TRACKS = { "": "Both", ic: "Individual", people: "People" };
|
const TRACKS = { "": "Both", ic: "Individual", people: "People" };
|
||||||
|
|
||||||
// What the list is in the order of. Agent match is the viewing agent's
|
// What the list is in the order of. What I'm looking for is the viewing
|
||||||
// score, moved by its votes; Newest is first seen; CV match is
|
// agent's want feed score, moved by its votes; Newest is the posting date;
|
||||||
// the agent's `judge` total, the roles not judged after by score.
|
// What fits me is the agent's `judge` total, the roles not judged after by
|
||||||
const SORTS = { score: "Agent match", cv: "CV match", time: "Newest" };
|
// CV closeness.
|
||||||
|
const SORTS = { score: "What I\u2019m looking for", cv: "What fits me", time: "Newest" };
|
||||||
// The route's `order` for each.
|
// The route's `order` for each.
|
||||||
const ORDERS = { score: "score", time: "newest", cv: "cv" };
|
const ORDERS = { score: "score", time: "newest", cv: "cv" };
|
||||||
// How many days ahead a deadline earns a badge: today to today + 4, the
|
// How many days ahead a deadline earns a badge: today to today + 4, the
|
||||||
@@ -154,15 +164,16 @@
|
|||||||
|
|
||||||
// Everything `clearFilters` has to put back, in one place, so a new filter
|
// Everything `clearFilters` has to put back, in one place, so a new filter
|
||||||
// added here cannot be forgotten there. `company` and `place` are the
|
// added here cannot be forgotten there. `company` and `place` are the
|
||||||
// substrings of links from other pages; `companyKeys` and `places` are the
|
// substrings of older links and saved filters; `companyKeys` and `places`
|
||||||
// rail's chips. `levelFrom`, `levelTo` and `track` are the seniority range;
|
// are the rail's chips. `levelFrom`, `levelTo` and `track` are the seniority
|
||||||
// `seniorityUnknown` and `payUnknown` are the "and roles with no level / no
|
// range; `seniorityUnknown` and `payUnknown` are the "and roles with no
|
||||||
// pay" boxes, asked only while a range, a track or a pay option is picked.
|
// level / no pay" boxes, asked only while a range, a track or a pay option
|
||||||
|
// is picked.
|
||||||
function blankFilter() {
|
function blankFilter() {
|
||||||
return {
|
return {
|
||||||
sort: "score", agent: null, period: "all", tags: new Set(), showRead: true,
|
sort: "score", agent: null, period: "all", tags: new Set(), showRead: true,
|
||||||
company: "", place: "", function: "", remote: "", judged: "", pay: "",
|
company: "", place: "", function: "", remote: "", judged: "", pay: "",
|
||||||
showMuted: false, gated: false, companyKeys: [], places: [],
|
gated: false, companyKeys: [], places: [],
|
||||||
levelFrom: "", levelTo: "", track: "", seniorityUnknown: true, payUnknown: false,
|
levelFrom: "", levelTo: "", track: "", seniorityUnknown: true, payUnknown: false,
|
||||||
closing: false,
|
closing: false,
|
||||||
};
|
};
|
||||||
@@ -174,7 +185,7 @@
|
|||||||
counted. */
|
counted. */
|
||||||
function filtersOn(f = state.filter, { more = false } = {}) {
|
function filtersOn(f = state.filter, { more = false } = {}) {
|
||||||
const folded = [f.company.trim(), f.place.trim(), f.function, f.remote, f.pay, f.judged,
|
const folded = [f.company.trim(), f.place.trim(), f.function, f.remote, f.pay, f.judged,
|
||||||
f.showMuted, f.closing, f.agent !== null && f.gated].filter(Boolean).length
|
f.closing, f.agent !== null && f.gated].filter(Boolean).length
|
||||||
+ f.companyKeys.length + f.places.length
|
+ f.companyKeys.length + f.places.length
|
||||||
+ [f.levelFrom, f.levelTo, f.track].filter(Boolean).length;
|
+ [f.levelFrom, f.levelTo, f.track].filter(Boolean).length;
|
||||||
if (more) return folded;
|
if (more) return folded;
|
||||||
@@ -191,6 +202,10 @@
|
|||||||
tags: [], // [{tag, count, unread}] from /api/filters
|
tags: [], // [{tag, count, unread}] from /api/filters
|
||||||
items: [], // the rows on screen, in order
|
items: [], // the rows on screen, in order
|
||||||
total: 0, // how many roles the filters match in all
|
total: 0, // how many roles the filters match in all
|
||||||
|
offset: 0, // how far into the server's list the pages have read
|
||||||
|
more: false, // the server's list goes on past `offset`
|
||||||
|
moved: 0, // votes and reads no page has been asked after yet (`pageAsk`)
|
||||||
|
movedOnce: 0, // those one page has been asked after
|
||||||
loaded: false, // the first page has landed
|
loaded: false, // the first page has landed
|
||||||
loading: false, // a page is in flight, so the pager holds off
|
loading: false, // a page is in flight, so the pager holds off
|
||||||
selected: 0, // which row the keyboard is on
|
selected: 0, // which row the keyboard is on
|
||||||
@@ -225,13 +240,14 @@
|
|||||||
/** The viewing agent, once the agents are in. */
|
/** The viewing agent, once the agents are in. */
|
||||||
const viewed = () => state.agents.find((a) => a.id === state.filter.agent) || null;
|
const viewed = () => state.agents.find((a) => a.id === state.filter.agent) || null;
|
||||||
|
|
||||||
/** Whether CV match has a CV to go by: the viewed agent has one, or
|
/** Whether What fits me has a CV to go by: the viewed agent has one, or
|
||||||
the agents are not in yet, when the address is taken at its word. */
|
the agents are not in yet, when the address is taken at its word. */
|
||||||
const hasCv = () => !state.agentsLoaded || Boolean(viewed()?.cv_text?.trim());
|
const hasCv = () => !state.agentsLoaded || Boolean(viewed()?.cv_text?.trim());
|
||||||
|
|
||||||
/** The order the list is in. Agent match needs an agent to score by: with
|
/** The order the list is in. What I'm looking for needs an agent to score
|
||||||
none enabled the server answers newest first, and so does this. Best
|
by: with none enabled the server answers newest first, and so does this.
|
||||||
CV match needs that agent's CV too, and without one is Agent match. */
|
What fits me needs that agent's CV too, and without one is What I'm
|
||||||
|
looking for. */
|
||||||
function sortShown() {
|
function sortShown() {
|
||||||
const { sort, agent } = state.filter;
|
const { sort, agent } = state.filter;
|
||||||
if (sort === "time" || agent === null) return "time";
|
if (sort === "time" || agent === null) return "time";
|
||||||
@@ -249,7 +265,6 @@
|
|||||||
for (const key of ["function", "remote", "judged", "pay"]) if (f[key]) p.set(key, f[key]);
|
for (const key of ["function", "remote", "judged", "pay"]) if (f[key]) p.set(key, f[key]);
|
||||||
const days = PERIODS[f.period].days;
|
const days = PERIODS[f.period].days;
|
||||||
if (days) p.set("days", String(days));
|
if (days) p.set("days", String(days));
|
||||||
if (f.showMuted) p.set("show_muted", "true");
|
|
||||||
if (!f.showRead) p.set("include_read", "false");
|
if (!f.showRead) p.set("include_read", "false");
|
||||||
f.tags.forEach((t) => p.append("tags", t));
|
f.tags.forEach((t) => p.append("tags", t));
|
||||||
f.companyKeys.forEach((key) => p.append("company_keys", key));
|
f.companyKeys.forEach((key) => p.append("company_keys", key));
|
||||||
@@ -286,7 +301,6 @@
|
|||||||
f[key] = q.get(key) || "";
|
f[key] = q.get(key) || "";
|
||||||
}
|
}
|
||||||
f.period = PERIOD_OF_DAYS[q.get("days")] || "all";
|
f.period = PERIOD_OF_DAYS[q.get("days")] || "all";
|
||||||
f.showMuted = q.get("show_muted") === "true";
|
|
||||||
f.showRead = q.get("include_read") !== "false";
|
f.showRead = q.get("include_read") !== "false";
|
||||||
f.gated = q.get("gated") === "true" || q.get("gated") === "1";
|
f.gated = q.get("gated") === "true" || q.get("gated") === "1";
|
||||||
f.tags = new Set(q.getAll("tags"));
|
f.tags = new Set(q.getAll("tags"));
|
||||||
@@ -387,7 +401,6 @@
|
|||||||
written into the entry landed on, so the entry says what is shown. */
|
written into the entry landed on, so the entry says what is shown. */
|
||||||
function restore() {
|
function restore() {
|
||||||
const before = addressFor(pathFor(null));
|
const before = addressFor(pathFor(null));
|
||||||
const muted = state.filter.showMuted;
|
|
||||||
readUrl(location.search, { view: false });
|
readUrl(location.search, { view: false });
|
||||||
$("#search").value = state.q;
|
$("#search").value = state.q;
|
||||||
writeUrl();
|
writeUrl();
|
||||||
@@ -397,25 +410,79 @@
|
|||||||
labelPlaces();
|
labelPlaces();
|
||||||
if (addressFor(pathFor(null)) === before) return;
|
if (addressFor(pathFor(null)) === before) return;
|
||||||
reload();
|
reload();
|
||||||
if (state.filter.showMuted !== muted) loadFilters().catch(() => {});
|
|
||||||
renderRail();
|
renderRail();
|
||||||
renderActive();
|
renderActive();
|
||||||
}
|
}
|
||||||
|
|
||||||
// -------------------------------------------------------------- loading
|
// -------------------------------------------------------------- loading
|
||||||
|
|
||||||
|
/** The most rows one request may ask for (`limit` on /api/roles). */
|
||||||
|
const MOST = 1000;
|
||||||
|
|
||||||
|
/* Where the next page starts and how many rows it asks for. The server
|
||||||
|
sorts afresh on every request, and a vote moves a role in its order
|
||||||
|
(`_VOTED_SCORE`) or out of the list (Judged), as a read does with Show
|
||||||
|
read off. Each row that leaves the part already read lets a row from
|
||||||
|
below up into it, which a page asked from `offset` would never show, so
|
||||||
|
the page starts one row earlier for each such change and is that much
|
||||||
|
longer; the rows it repeats are dropped (`pageMerge`). `moved` holds
|
||||||
|
each change for two pages, the one asked after it and the next
|
||||||
|
(`state.moved` and `state.movedOnce`), since a vote still in flight
|
||||||
|
when a page is asked may land after the server has read that page. */
|
||||||
|
function pageAsk(offset, moved, page = PAGE) {
|
||||||
|
const back = Math.min(offset, moved, MOST - page);
|
||||||
|
return { offset: offset - back, limit: page + back };
|
||||||
|
}
|
||||||
|
|
||||||
|
/* An answer onto the rows on screen: the rows not on screen yet after
|
||||||
|
them, in the server's order, so the role a pass sank is not shown a
|
||||||
|
second time; how far into the server's list the pages have now read;
|
||||||
|
and whether it goes on. A short page is its end whatever `total` says,
|
||||||
|
since the count and the page are two queries. */
|
||||||
|
function pageMerge(items, ask, answer) {
|
||||||
|
const shown = new Set(items.map((r) => r.uid));
|
||||||
|
const offset = ask.offset + answer.items.length;
|
||||||
|
return {
|
||||||
|
items: items.concat(answer.items.filter((r) => !shown.has(r.uid))),
|
||||||
|
offset,
|
||||||
|
more: answer.items.length >= ask.limit && offset < answer.total,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// For the tests, which page through a list sorted afresh without a DOM.
|
||||||
|
window.feedPaging = { pageAsk, pageMerge };
|
||||||
|
|
||||||
let feedRequest = 0;
|
let feedRequest = 0;
|
||||||
|
|
||||||
|
/* The feed has nothing to show, so the empty state offers a retry rather
|
||||||
|
than leaving the feed blank for good. */
|
||||||
|
function feedFailed() {
|
||||||
|
state.feedError = true;
|
||||||
|
state.loaded = false;
|
||||||
|
renderFeed();
|
||||||
|
}
|
||||||
|
|
||||||
/* A reload starts the list again; an append asks for the rows after the
|
/* A reload starts the list again; an append asks for the rows after the
|
||||||
ones shown. `keep` asks for that many in one go, which is how the list is
|
ones read (`pageAsk`). `keep` asks for that many in one go, which is how
|
||||||
put back as it was. The latch is let go only by the newest request: a
|
the list is put back as it was. The latch is let go only by the newest
|
||||||
reload overtaking an append must not let a second append start with an
|
request: a reload overtaking an append must not let a second append
|
||||||
offset into the list being replaced. */
|
start with an offset into the list being replaced.
|
||||||
|
|
||||||
|
Nothing is asked while no agent is named and the agents have not said
|
||||||
|
whether there is one, since the server would list every open role,
|
||||||
|
ungated (`start`); the empty state offers the retry instead. */
|
||||||
async function loadFeed({ append = false, keep = 0 } = {}) {
|
async function loadFeed({ append = false, keep = 0 } = {}) {
|
||||||
if (append && (state.loading || state.items.length >= state.total)) return;
|
if (state.filter.agent === null && !state.agentsLoaded) {
|
||||||
|
if (!append) feedFailed();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (append && (state.loading || !state.more)) return;
|
||||||
|
const ask = append ? pageAsk(state.offset, state.moved + state.movedOnce)
|
||||||
|
: { offset: 0, limit: Math.min(MOST, Math.max(PAGE, keep)) };
|
||||||
const params = feedParams();
|
const params = feedParams();
|
||||||
params.set("offset", String(append ? state.items.length : 0));
|
params.set("offset", String(ask.offset));
|
||||||
params.set("limit", String(Math.min(1000, Math.max(PAGE, keep))));
|
params.set("limit", String(ask.limit));
|
||||||
|
const moved = state.moved;
|
||||||
const request = ++feedRequest;
|
const request = ++feedRequest;
|
||||||
state.loading = true;
|
state.loading = true;
|
||||||
let data;
|
let data;
|
||||||
@@ -423,23 +490,25 @@
|
|||||||
data = await api(`/api/roles?${params}`);
|
data = await api(`/api/roles?${params}`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// An append failure leaves what is already on screen; a first load or
|
// An append failure leaves what is already on screen; a first load or
|
||||||
// a reload has nothing to show, so the empty state offers a retry
|
// a reload has nothing to show (`feedFailed`).
|
||||||
// rather than leaving the feed blank for good.
|
if (!append && request === feedRequest) feedFailed();
|
||||||
if (!append && request === feedRequest) {
|
|
||||||
state.feedError = true;
|
|
||||||
state.loaded = false;
|
|
||||||
renderFeed();
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
} finally {
|
} finally {
|
||||||
if (request === feedRequest) state.loading = false;
|
if (request === feedRequest) state.loading = false;
|
||||||
}
|
}
|
||||||
if (request !== feedRequest) return;
|
if (request !== feedRequest) return;
|
||||||
state.feedError = false;
|
state.feedError = false;
|
||||||
|
// The changes made before this page was asked are counted once more, for
|
||||||
|
// the next; those made while it was in flight wait for theirs.
|
||||||
|
state.movedOnce = moved;
|
||||||
|
state.moved -= moved;
|
||||||
|
|
||||||
const held = append ? state.items[state.selected]?.uid : state.readingUid;
|
const held = append ? state.items[state.selected]?.uid : state.readingUid;
|
||||||
|
const page = pageMerge(append ? state.items : [], ask, data);
|
||||||
state.total = data.total;
|
state.total = data.total;
|
||||||
state.items = append ? state.items.concat(data.items) : data.items;
|
state.items = page.items;
|
||||||
|
state.offset = page.offset;
|
||||||
|
state.more = page.more;
|
||||||
state.loaded = true;
|
state.loaded = true;
|
||||||
const at = held ? state.items.findIndex((r) => r.uid === held) : -1;
|
const at = held ? state.items.findIndex((r) => r.uid === held) : -1;
|
||||||
state.selected = at >= 0 ? at : append ? state.selected : 0;
|
state.selected = at >= 0 ? at : append ? state.selected : 0;
|
||||||
@@ -514,12 +583,11 @@
|
|||||||
if (state.readingUid) refreshReader(state.readingUid);
|
if (state.readingUid) refreshReader(state.readingUid);
|
||||||
}
|
}
|
||||||
|
|
||||||
// The tag counts. Asked with `show_muted` and the agent only, the filters
|
// The tag counts. Asked with the agent only, the filter the route takes, so
|
||||||
// the route takes, so picking a tag does not renumber the others.
|
// picking a tag does not renumber the others.
|
||||||
let filtersRequest = 0;
|
let filtersRequest = 0;
|
||||||
async function loadFilters() {
|
async function loadFilters() {
|
||||||
const params = new URLSearchParams();
|
const params = new URLSearchParams();
|
||||||
if (state.filter.showMuted) params.set("show_muted", "true");
|
|
||||||
if (state.filter.agent !== null) params.set("agent", String(state.filter.agent));
|
if (state.filter.agent !== null) params.set("agent", String(state.filter.agent));
|
||||||
const request = ++filtersRequest;
|
const request = ++filtersRequest;
|
||||||
const data = await api(`/api/filters?${params}`);
|
const data = await api(`/api/filters?${params}`);
|
||||||
@@ -547,7 +615,7 @@
|
|||||||
it: pushed, so back takes the filter off and brings the role back, or
|
it: pushed, so back takes the filter off and brings the role back, or
|
||||||
replacing the entry for text being typed (`replace`). A filter that
|
replacing the entry for text being typed (`replace`). A filter that
|
||||||
changes nothing only closes the reader. */
|
changes nothing only closes the reader. */
|
||||||
function setFilter(patch = {}, { counts = false, replace = false } = {}) {
|
function setFilter(patch = {}, { replace = false } = {}) {
|
||||||
Object.assign(state.filter, patch);
|
Object.assign(state.filter, patch);
|
||||||
if (state.readingUid && addressFor(pathFor(null)) === pathFor(null) + location.search) {
|
if (state.readingUid && addressFor(pathFor(null)) === pathFor(null) + location.search) {
|
||||||
closeReader();
|
closeReader();
|
||||||
@@ -556,7 +624,6 @@
|
|||||||
route();
|
route();
|
||||||
}
|
}
|
||||||
reload();
|
reload();
|
||||||
if (counts) loadFilters().catch(() => {});
|
|
||||||
renderRail();
|
renderRail();
|
||||||
renderActive();
|
renderActive();
|
||||||
}
|
}
|
||||||
@@ -568,7 +635,7 @@
|
|||||||
// Text half typed into a chip field goes too, and with it its list.
|
// Text half typed into a chip field goes too, and with it its list.
|
||||||
companyChips?.clear();
|
companyChips?.clear();
|
||||||
placeChips?.clear();
|
placeChips?.clear();
|
||||||
setFilter({}, { counts: true });
|
setFilter();
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The other order over the same filter: the entry is replaced, since an
|
/* The other order over the same filter: the entry is replaced, since an
|
||||||
@@ -593,11 +660,9 @@
|
|||||||
|
|
||||||
/** Where a company's name links: the list on screen, every filter and the
|
/** Where a company's name links: the list on screen, every filter and the
|
||||||
search kept, with this company as its one company filter, in place of
|
search kept, with this company as its one company filter, in place of
|
||||||
any company chips and of `?company=`. A company this agent hid turns
|
any company chips and of `?company=`. */
|
||||||
on Show hidden companies, or its own name would lead to an empty list. */
|
|
||||||
function companyHref(key) {
|
function companyHref(key) {
|
||||||
const f = Object.assign({}, state.filter, { companyKeys: [key], company: "" });
|
const f = Object.assign({}, state.filter, { companyKeys: [key], company: "" });
|
||||||
if (companies.get(key)?.affinity === -1) f.showMuted = true;
|
|
||||||
return addressFor(pathFor(null), f);
|
return addressFor(pathFor(null), f);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -627,10 +692,9 @@
|
|||||||
const link = e.currentTarget;
|
const link = e.currentTarget;
|
||||||
link.href = companyHref(link.dataset.company);
|
link.href = companyHref(link.dataset.company);
|
||||||
companyNames.set(link.dataset.company, link.textContent.trim());
|
companyNames.set(link.dataset.company, link.textContent.trim());
|
||||||
const counts = state.filter.showMuted;
|
|
||||||
readUrl(new URL(link.href).search, { view: false });
|
readUrl(new URL(link.href).search, { view: false });
|
||||||
$("#search").value = state.q;
|
$("#search").value = state.q;
|
||||||
setFilter({}, { counts: counts !== state.filter.showMuted });
|
setFilter();
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The strip's way back to every company: back, when the entry under this
|
/* The strip's way back to every company: back, when the entry under this
|
||||||
@@ -669,8 +733,8 @@
|
|||||||
|
|
||||||
renderAgent();
|
renderAgent();
|
||||||
|
|
||||||
// The order, the same three whatever is viewed. Agent match needs an
|
// The order, the same three whatever is viewed. What I'm looking for needs
|
||||||
// agent to score by, and CV match that agent's CV, so without one
|
// an agent to score by, and What fits me that agent's CV, so without one
|
||||||
// each is greyed and says why.
|
// each is greyed and says why.
|
||||||
const none = state.agentsLoaded && !state.agents.length;
|
const none = state.agentsLoaded && !state.agents.length;
|
||||||
const why = {
|
const why = {
|
||||||
@@ -714,7 +778,6 @@
|
|||||||
setValue("f-remote", f.remote);
|
setValue("f-remote", f.remote);
|
||||||
setValue("f-pay", f.pay);
|
setValue("f-pay", f.pay);
|
||||||
setValue("f-judged", f.judged);
|
setValue("f-judged", f.judged);
|
||||||
setValue("f-show-muted", f.showMuted);
|
|
||||||
setValue("f-closing", f.closing);
|
setValue("f-closing", f.closing);
|
||||||
setValue("f-gated", f.gated);
|
setValue("f-gated", f.gated);
|
||||||
renderFacets();
|
renderFacets();
|
||||||
@@ -746,7 +809,7 @@
|
|||||||
input.set(want);
|
input.set(want);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* `?company=` or `?place=`, the substring a link from another page
|
/* `?company=` or `?place=`, the substring an older link or saved filter
|
||||||
brought: a plain chip saying what it matches, which takes it off and
|
brought: a plain chip saying what it matches, which takes it off and
|
||||||
pushes, as a filter does. */
|
pushes, as a filter does. */
|
||||||
function renderText(id, text, noun, key) {
|
function renderText(id, text, noun, key) {
|
||||||
@@ -852,7 +915,7 @@
|
|||||||
gate.hidden = state.filter.agent === null;
|
gate.hidden = state.filter.agent === null;
|
||||||
gate.textContent = state.filter.gated
|
gate.textContent = state.filter.gated
|
||||||
? "Filtered by this agent's fetch gate, and Show gated adds the roles it rejects."
|
? "Filtered by this agent's fetch gate, and Show gated adds the roles it rejects."
|
||||||
: "Already filtered by this agent's fetch gate: region, places and role.";
|
: "Already filtered by this agent's fetch gate: world, location and role exclusion. Hidden companies are never shown.";
|
||||||
if (agents.length) {
|
if (agents.length) {
|
||||||
// Rebuilt only when the agents changed, since the rail is drawn again on
|
// Rebuilt only when the agents changed, since the rail is drawn again on
|
||||||
// every filter and a rebuild shuts a select that is open.
|
// every filter and a rebuild shuts a select that is open.
|
||||||
@@ -950,7 +1013,6 @@
|
|||||||
async function applySaved(id) {
|
async function applySaved(id) {
|
||||||
const saved = state.saved.find((x) => x.id === id);
|
const saved = state.saved.find((x) => x.id === id);
|
||||||
if (!saved) return;
|
if (!saved) return;
|
||||||
const muted = state.filter.showMuted;
|
|
||||||
const query = new URLSearchParams(saved.query);
|
const query = new URLSearchParams(saved.query);
|
||||||
readUrl(`?${saved.query}`, { view: false });
|
readUrl(`?${saved.query}`, { view: false });
|
||||||
const sort = query.get("sort");
|
const sort = query.get("sort");
|
||||||
@@ -961,7 +1023,7 @@
|
|||||||
await labelPlaces({ write: false });
|
await labelPlaces({ write: false });
|
||||||
if (filtersOn(state.filter, { more: true })) $("#f-more").open = true;
|
if (filtersOn(state.filter, { more: true })) $("#f-more").open = true;
|
||||||
if (overlayRail() && state.railOpen) toggleRail();
|
if (overlayRail() && state.railOpen) toggleRail();
|
||||||
setFilter({}, { counts: muted !== state.filter.showMuted });
|
setFilter();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function deleteSaved(id) {
|
async function deleteSaved(id) {
|
||||||
@@ -1236,7 +1298,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* The gutter's number, of 100, and what it says: the agent's score, or
|
/* The gutter's number, of 100, and what it says: the agent's score, or
|
||||||
under CV match the CV match. A role `judge` has not read yet has an
|
under What fits me the CV match. A role `judge` has not read yet has an
|
||||||
empty bar and, faint, how close it is to the CV (`cv_fit`), which is
|
empty bar and, faint, how close it is to the CV (`cv_fit`), which is
|
||||||
what orders it there, else a dash. */
|
what orders it there, else a dash. */
|
||||||
function gutterOf(role) {
|
function gutterOf(role) {
|
||||||
@@ -1300,12 +1362,10 @@
|
|||||||
el.querySelector(".company")?.addEventListener("click", companyClick);
|
el.querySelector(".company")?.addEventListener("click", companyClick);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* A passed role and a hidden company's are dimmed, never dropped: the pass
|
/* A passed role is dimmed, never dropped: the pass has to stay visible to be
|
||||||
has to stay visible to be undone. */
|
undone. */
|
||||||
function paintRow(el, role) {
|
function paintRow(el, role) {
|
||||||
const company = companies.get(role.company_key);
|
|
||||||
el.dataset.read = String(Boolean(role.read_at));
|
el.dataset.read = String(Boolean(role.read_at));
|
||||||
el.classList.toggle("muted", Boolean(company && company.affinity === -1));
|
|
||||||
el.classList.toggle("passed", role.vote === -1);
|
el.classList.toggle("passed", role.vote === -1);
|
||||||
el.classList.toggle("gated", Boolean(role.gated_reason));
|
el.classList.toggle("gated", Boolean(role.gated_reason));
|
||||||
}
|
}
|
||||||
@@ -1350,14 +1410,25 @@
|
|||||||
empty.innerHTML = `<h2>Could not load the feed</h2>
|
empty.innerHTML = `<h2>Could not load the feed</h2>
|
||||||
<p>The server did not answer. <button type="button" class="linky" data-retry
|
<p>The server did not answer. <button type="button" class="linky" data-retry
|
||||||
>Try again</button>.</p>`;
|
>Try again</button>.</p>`;
|
||||||
|
// Without the agents the feed may not know whose it is, so they are
|
||||||
|
// asked again with it.
|
||||||
const retry = empty.querySelector("[data-retry]");
|
const retry = empty.querySelector("[data-retry]");
|
||||||
if (retry) retry.onclick = () => loadFeed();
|
if (retry) retry.onclick = () => (state.agentsLoaded ? loadFeed() : start());
|
||||||
} else {
|
} else {
|
||||||
|
// An agent's list is what its fetch gate passes, which a phone,
|
||||||
|
// whose rail is shut, learns here and at the end of the list.
|
||||||
const filtered = state.q || filtersOn();
|
const filtered = state.q || filtersOn();
|
||||||
|
const gate = gateKept() ? `${esc(gateOwner())} fetch gate` : "";
|
||||||
empty.innerHTML = filtered
|
empty.innerHTML = filtered
|
||||||
? `<h2>Nothing matches</h2>
|
? `<h2>Nothing matches</h2>
|
||||||
<p>No open role fits these filters.
|
<p>No open role ${gate ? `that passes ${gate} ` : ""}fits these filters.
|
||||||
<button type="button" class="linky" data-clear>Clear them</button> and start again.</p>`
|
<button type="button" class="linky" data-clear>Clear them</button> and start again.</p>`
|
||||||
|
: gate
|
||||||
|
? `<h2>No roles yet</h2>
|
||||||
|
<p>No open role passes ${gate} yet. <a href="/settings/sources#runs">Runs</a>
|
||||||
|
starts a pull; <a href="/settings/sources#job-boards">Job boards</a> and
|
||||||
|
<a href="/settings/sources#company-boards">Company boards</a> decide what it
|
||||||
|
brings in, <a href="/settings/agents">Search agents</a> what the gate passes.</p>`
|
||||||
: `<h2>No roles yet</h2>
|
: `<h2>No roles yet</h2>
|
||||||
<p>Nothing has been pulled yet. <a href="/settings/sources#runs">Runs</a> starts one;
|
<p>Nothing has been pulled yet. <a href="/settings/sources#runs">Runs</a> starts one;
|
||||||
<a href="/settings/sources#job-boards">Job boards</a> and
|
<a href="/settings/sources#job-boards">Job boards</a> and
|
||||||
@@ -1368,13 +1439,38 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The end of the list says which end it is: a list that has run out and
|
// The end of the list says which end it is: a list that has run out and
|
||||||
// one that is still loading look the same otherwise.
|
// one that is still loading look the same otherwise. Whether it has run
|
||||||
|
// out is the server's to say (`pageMerge`), since the rows on screen
|
||||||
|
// keep the ones a vote or a read has since taken out of its list.
|
||||||
const end = $("#feed-end");
|
const end = $("#feed-end");
|
||||||
const n = state.items.length;
|
const n = state.items.length;
|
||||||
end.hidden = !state.loaded || !n;
|
end.hidden = !state.loaded || !n;
|
||||||
end.textContent = n < state.total
|
end.textContent = state.more ? "Loading more…" : `${listSaid(n)} — that is all of them.`;
|
||||||
? "Loading more…"
|
}
|
||||||
: `${number.format(n)} ${n === 1 ? "role" : "roles"} — that is all of them.`;
|
|
||||||
|
/** Whether the list keeps to the viewing agent's fetch gate: there is an
|
||||||
|
agent, and Show gated is off. */
|
||||||
|
const gateKept = () => state.filter.agent !== null && !state.filter.gated;
|
||||||
|
|
||||||
|
/** Whose fetch gate, for a sentence: the viewing agent's name, or "this
|
||||||
|
agent's" until the agents are in. */
|
||||||
|
function gateOwner() {
|
||||||
|
const agent = viewed();
|
||||||
|
return agent ? `${agent.name}'s` : "this agent's";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The rows on screen in words, for the end of the list: how many and,
|
||||||
|
with an agent viewing, that they are what its fetch gate passes (and
|
||||||
|
the filters keep), or with Show gated that they hold what it rejects. */
|
||||||
|
function listSaid(n) {
|
||||||
|
const roles = `${number.format(n)} ${n === 1 ? "role" : "roles"}`;
|
||||||
|
if (state.filter.agent === null) return roles;
|
||||||
|
const gate = `${gateOwner()} fetch gate`;
|
||||||
|
const filtered = Boolean(state.q) || filtersOn() > (state.filter.gated ? 1 : 0);
|
||||||
|
if (state.filter.gated) {
|
||||||
|
return `${roles}${filtered ? " in these filters" : ""}, the ones ${gate} rejects included`;
|
||||||
|
}
|
||||||
|
return `${roles} ${n === 1 ? "passes" : "pass"} ${gate}${filtered ? " and these filters" : ""}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* With exactly one company filtered, which is where its name leads, a
|
/* With exactly one company filtered, which is where its name leads, a
|
||||||
@@ -1432,43 +1528,48 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Which parts of the viewing agent's fetch gate apply to a company's
|
/* Which parts of the viewing agent's fetch gate apply to a company's
|
||||||
roles: All countries lifts the place part (the Where lines), All roles
|
roles: All locations lifts the location gate, All roles the role
|
||||||
the role part (fetch terms, titles to leave out); the region always
|
exclusion and the fetch terms; the world gate always holds. A favourite
|
||||||
holds. A favourite only ranks higher, so it is not named here. Nothing
|
only ranks higher, so it is not named here. A hidden company has no
|
||||||
with no agent, when no fetch gate applies. */
|
roles in the feed at all. Nothing with no agent, when no fetch gate
|
||||||
|
applies. */
|
||||||
function gateNote(company) {
|
function gateNote(company) {
|
||||||
if (state.filter.agent === null) return "";
|
if (state.filter.agent === null) return "";
|
||||||
const countries = Boolean(company?.all_countries);
|
if (company?.affinity === -1) return "Hidden for this agent: its roles are never shown.";
|
||||||
|
const locations = Boolean(company?.all_countries);
|
||||||
const roles = Boolean(company?.all_roles);
|
const roles = Boolean(company?.all_roles);
|
||||||
if (countries && roles) {
|
if (locations && roles) {
|
||||||
return "All countries and All roles: every role in the agent's region, "
|
return "All locations and All roles: every role in the agent's world, "
|
||||||
+ "not filtered by place or by role.";
|
+ "not filtered by location, role exclusion or fetch terms.";
|
||||||
}
|
}
|
||||||
if (countries) {
|
if (locations) {
|
||||||
return "All countries: every role in the agent's region, not filtered by place. "
|
return "All locations: every role in the agent's world, not filtered by location. "
|
||||||
+ "The agent's fetch terms and titles to leave out still apply.";
|
+ "The agent's role exclusion still applies.";
|
||||||
}
|
}
|
||||||
if (roles) {
|
if (roles) {
|
||||||
return "All roles: not filtered by role, no fetch terms and no titles left out. "
|
return "All roles: no titles left out and no fetch terms. "
|
||||||
+ "The agent's places still apply.";
|
+ "The agent's locations still apply.";
|
||||||
}
|
}
|
||||||
return "The roles this agent's fetch gate passes: its region, places and role. "
|
return "The roles this agent's fetch gate passes: its world, locations and role exclusion. "
|
||||||
+ "All countries lifts the places for this company, All roles the role.";
|
+ "All locations lifts the locations for this company, All roles the role exclusion.";
|
||||||
}
|
}
|
||||||
|
|
||||||
/* A company seen only through a search has no row until its first change
|
/* A company seen only through a search has no row until its first change
|
||||||
creates one, so the list is reloaded then. A change dims or undims its
|
creates one, so the list is reloaded then. Hiding takes the company's
|
||||||
rows at once; where they rank waits on the `score` it queued, whose id
|
rows out at once, since a hidden company is never in the feed; where the
|
||||||
the affinity route answers with (`updated.score_job`), and the feed
|
others rank waits on the `score` it queued, whose id the affinity route
|
||||||
reloads when that has finished. */
|
answers with (`updated.score_job`), and the feed reloads when that has
|
||||||
|
finished. A list the company filter has come off meanwhile has its rows
|
||||||
|
asked again in the new order, as `renderAffinity` does. */
|
||||||
async function affinityChanged(key, company, updated) {
|
async function affinityChanged(key, company, updated) {
|
||||||
if (!company) await loadCompanies(state.filter.agent).catch(() => {});
|
if (!company) await loadCompanies(state.filter.agent).catch(() => {});
|
||||||
renderFeed();
|
if (updated?.affinity === -1) reload(); else renderFeed();
|
||||||
const status = () => $("#company-strip .strip-status");
|
const status = () => $("#company-strip .strip-status");
|
||||||
if (status()) status().textContent = "Scoring again; the list reloads when that is done.";
|
if (status()) status().textContent = "Scoring again; the list reloads when that is done.";
|
||||||
await scored(updated?.score_job);
|
await scored(updated?.score_job);
|
||||||
if (status()) status().textContent = "";
|
if (status()) status().textContent = "";
|
||||||
if (state.filter.companyKeys.includes(key)) reload();
|
if (state.filter.companyKeys.includes(key)) reload();
|
||||||
|
else loadFeed({ keep: state.items.length });
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Wait for the `score` job `job`, the way agents.js's `rescore` waits:
|
/* Wait for the `score` job `job`, the way agents.js's `rescore` waits:
|
||||||
@@ -1568,6 +1669,8 @@
|
|||||||
async function markRead(uid) {
|
async function markRead(uid) {
|
||||||
const item = state.items.find((r) => r.uid === uid);
|
const item = state.items.find((r) => r.uid === uid);
|
||||||
if (item && item.read_at) return;
|
if (item && item.read_at) return;
|
||||||
|
// With Show read off the server's list loses it, see `pageAsk`.
|
||||||
|
if (!state.filter.showRead) state.moved += 1;
|
||||||
if (item) { item.read_at = new Date().toISOString(); renderFeed(); }
|
if (item) { item.read_at = new Date().toISOString(); renderFeed(); }
|
||||||
try {
|
try {
|
||||||
const role = await api(`/api/roles/${encodeURIComponent(uid)}/read`, { method: "POST" });
|
const role = await api(`/api/roles/${encodeURIComponent(uid)}/read`, { method: "POST" });
|
||||||
@@ -1641,33 +1744,20 @@
|
|||||||
try { return languageNames?.of(code) || code; } catch (err) { return code; }
|
try { return languageNames?.of(code) || code; } catch (err) { return code; }
|
||||||
}
|
}
|
||||||
|
|
||||||
const MUST_MARKS = { met: ["✓", "met"], broken: ["✗", "broken"], unclear: ["?", "unclear"] };
|
|
||||||
|
|
||||||
/* The agent's `judge` reading against its CV: the total and its three
|
/* The agent's `judge` reading against its CV: the total and its three
|
||||||
numbers, career as the server counted it (capped at 39 when a must-have
|
numbers, each of 10, then the strengths and gaps it named. A box as the
|
||||||
is broken, with what the model said), the must-haves with a verdict
|
summary is, since it is the same kind of thing: a model's reading, not
|
||||||
each, then the strengths and gaps it named. A box as the summary is,
|
the posting. */
|
||||||
since it is the same kind of thing: a model's reading, not the posting. */
|
|
||||||
function cvMatch(r) {
|
function cvMatch(r) {
|
||||||
const j = r.judgement;
|
const j = r.judgement;
|
||||||
if (!j) return "";
|
if (!j) return "";
|
||||||
const list = (items, none) => (items?.length
|
const list = (items, none) => (items?.length
|
||||||
? `<ul>${items.map((x) => `<li>${esc(x)}</li>`).join("")}</ul>`
|
? `<ul>${items.map((x) => `<li>${esc(x)}</li>`).join("")}</ul>`
|
||||||
: `<p class="muted small">${none}</p>`);
|
: `<p class="muted small">${none}</p>`);
|
||||||
const counted = j.career_capped ?? j.career;
|
|
||||||
const career = counted < j.career
|
|
||||||
? `career ${counted} (capped: breaks a must-have; model said ${j.career})` : `career ${j.career}`;
|
|
||||||
const musts = Array.isArray(j.musts) && j.musts.length
|
|
||||||
? `<ul class="cv-musts">${j.musts.map((m) => {
|
|
||||||
const [mark, word] = MUST_MARKS[m.verdict] || MUST_MARKS.unclear;
|
|
||||||
return `<li class="must-${word}"><span class="mark" role="img"
|
|
||||||
aria-label="${word}" title="${word}">${mark}</span><span>${esc(m.text)}</span></li>`;
|
|
||||||
}).join("")}</ul>` : "";
|
|
||||||
return `<section class="reader-summary reader-cv">
|
return `<section class="reader-summary reader-cv">
|
||||||
<h2 class="reader-label">CV match</h2>
|
<h2 class="reader-label">CV match</h2>
|
||||||
<p class="cv-total"><b>${Math.round(j.cv_score)}</b> of 100
|
<p class="cv-total"><b>${Math.round(j.cv_score)}</b> of 100
|
||||||
<span>skills ${j.skills} · experience ${j.experience} · ${career}</span></p>
|
<span>skills ${j.skills}/10 · experience ${j.experience}/10 · career ${j.career}/10</span></p>
|
||||||
${musts}
|
|
||||||
<div class="cv-lists">
|
<div class="cv-lists">
|
||||||
<div><h3 class="reader-label">Strengths</h3>${list(j.strengths, "None named.")}</div>
|
<div><h3 class="reader-label">Strengths</h3>${list(j.strengths, "None named.")}</div>
|
||||||
<div><h3 class="reader-label">Gaps</h3>${list(j.gaps, "None named.")}</div>
|
<div><h3 class="reader-label">Gaps</h3>${list(j.gaps, "None named.")}</div>
|
||||||
@@ -1738,6 +1828,10 @@
|
|||||||
const waiting = state.summarising === state.readingUid;
|
const waiting = state.summarising === state.readingUid;
|
||||||
const teaser = loaded && !r.article_text && out && (r.description || "").length < TEASER;
|
const teaser = loaded && !r.article_text && out && (r.description || "").length < TEASER;
|
||||||
|
|
||||||
|
// Under the title, in the copy note's style: that the viewing agent's
|
||||||
|
// fetch gate rejects the role and why (`gated_reason`), since a link
|
||||||
|
// from elsewhere, a saved role on Applications say, opens one the feed
|
||||||
|
// never lists.
|
||||||
el.innerHTML = `
|
el.innerHTML = `
|
||||||
<div class="reader-bar">
|
<div class="reader-bar">
|
||||||
<span class="judge"></span>
|
<span class="judge"></span>
|
||||||
@@ -1761,6 +1855,9 @@
|
|||||||
<h1>${esc(r.title || "")}</h1>
|
<h1>${esc(r.title || "")}</h1>
|
||||||
${loaded && r.dup_of ? `<p class="reader-note reader-copy">A copy of
|
${loaded && r.dup_of ? `<p class="reader-note reader-copy">A copy of
|
||||||
<a href="${esc(pathFor(r.dup_of))}">a posting</a> the feed lists once.</p>` : ""}
|
<a href="${esc(pathFor(r.dup_of))}">a posting</a> the feed lists once.</p>` : ""}
|
||||||
|
${loaded && state.filter.agent !== null && r.gated_reason
|
||||||
|
? `<p class="reader-note reader-copy">Rejected by ${esc(gateOwner())} fetch gate:
|
||||||
|
${esc(r.gated_reason)}. The feed lists it only with Show gated.</p>` : ""}
|
||||||
${pay ? `<p class="reader-salary">${esc(pay.range)}</p>
|
${pay ? `<p class="reader-salary">${esc(pay.range)}</p>
|
||||||
${pay.detail ? `<p class="muted small">${esc(pay.detail)}</p>` : ""}
|
${pay.detail ? `<p class="muted small">${esc(pay.detail)}</p>` : ""}
|
||||||
${pay.note ? `<p class="muted small">${esc(pay.note)}</p>` : ""}` : ""}
|
${pay.note ? `<p class="muted small">${esc(pay.note)}</p>` : ""}` : ""}
|
||||||
@@ -1812,15 +1909,22 @@
|
|||||||
/* The company's ⊘ ▲ ★, the one place a company is judged outside the
|
/* The company's ⊘ ▲ ★, the one place a company is judged outside the
|
||||||
Companies page, and its two fetch overrides beside them. A company seen
|
Companies page, and its two fetch overrides beside them. A company seen
|
||||||
only through a search has no row until its first change creates one, so
|
only through a search has no row until its first change creates one, so
|
||||||
the list is reloaded then, and the overrides, which need a row, wait for it. A
|
the list is reloaded then, and the overrides, which need a row, wait for it.
|
||||||
change dims or undims its rows in the feed. */
|
Hiding reloads the feed, which no longer holds the company's roles. Any
|
||||||
|
other stance moves every role of the company once the `score` it queued
|
||||||
|
has run, up into the rows already read as well as down out of them,
|
||||||
|
which no page asked from an offset makes up for (`pageAsk`); so the rows
|
||||||
|
on screen are asked again then, as many, in the new order. */
|
||||||
function renderAffinity(slot, r) {
|
function renderAffinity(slot, r) {
|
||||||
const company = companies.get(r.company_key);
|
const company = companies.get(r.company_key);
|
||||||
if (!r.company_key) { slot.replaceChildren(); return; }
|
if (!r.company_key) { slot.replaceChildren(); return; }
|
||||||
slot.replaceChildren(affinityControl(r.company_key, company ? company.affinity : 0, async () => {
|
slot.replaceChildren(affinityControl(r.company_key, company ? company.affinity : 0, async (updated) => {
|
||||||
if (!company) await loadCompanies(state.filter.agent).catch(() => {});
|
if (!company) await loadCompanies(state.filter.agent).catch(() => {});
|
||||||
renderAffinity(slot, r);
|
renderAffinity(slot, r);
|
||||||
|
if (updated?.affinity === -1) { reload(); return; }
|
||||||
renderFeed();
|
renderFeed();
|
||||||
|
await scored(updated?.score_job);
|
||||||
|
loadFeed({ keep: state.items.length });
|
||||||
}, state.filter.agent));
|
}, state.filter.agent));
|
||||||
if (company) {
|
if (company) {
|
||||||
slot.append(" ", overridesControl(r.company_key, company,
|
slot.append(" ", overridesControl(r.company_key, company,
|
||||||
@@ -2109,8 +2213,6 @@
|
|||||||
});
|
});
|
||||||
$("#f-seniority-unknown").addEventListener("change",
|
$("#f-seniority-unknown").addEventListener("change",
|
||||||
(e) => setFilter({ seniorityUnknown: e.target.checked }));
|
(e) => setFilter({ seniorityUnknown: e.target.checked }));
|
||||||
$("#f-show-muted").addEventListener("change",
|
|
||||||
(e) => setFilter({ showMuted: e.target.checked }, { counts: true }));
|
|
||||||
$("#f-gated").addEventListener("change", (e) => setFilter({ gated: e.target.checked }));
|
$("#f-gated").addEventListener("change", (e) => setFilter({ gated: e.target.checked }));
|
||||||
$("#f-closing").addEventListener("change", (e) => setFilter({ closing: e.target.checked }));
|
$("#f-closing").addEventListener("change", (e) => setFilter({ closing: e.target.checked }));
|
||||||
|
|
||||||
@@ -2132,6 +2234,16 @@
|
|||||||
new IntersectionObserver((entries) => {
|
new IntersectionObserver((entries) => {
|
||||||
if (entries.some((e) => e.isIntersecting)) loadFeed({ append: true });
|
if (entries.some((e) => e.isIntersecting)) loadFeed({ append: true });
|
||||||
}, { rootMargin: "600px" }).observe($("#feed-end"));
|
}, { rootMargin: "600px" }).observe($("#feed-end"));
|
||||||
|
// ▼ ● ▲ and ★, which likes, on a row or in the reader move the role in
|
||||||
|
// the server's order, or out of its list under Judged or with Show read
|
||||||
|
// off, which the next page makes up for (`pageAsk`). The keys press the
|
||||||
|
// same buttons. Counted at the click, failed or undone ones too, since a
|
||||||
|
// page that starts a row too early only repeats one.
|
||||||
|
for (const pane of [$("#roles-list"), $("#reader")]) {
|
||||||
|
pane.addEventListener("click", (e) => {
|
||||||
|
if (e.target.closest(".judge button")) state.moved += 1;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Kept on the way out, whichever way that is, so back finds it.
|
// Kept on the way out, whichever way that is, so back finds it.
|
||||||
window.addEventListener("pagehide", keep);
|
window.addEventListener("pagehide", keep);
|
||||||
@@ -2151,6 +2263,60 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Whose feed it is, then the feed. The server keeps to a fetch gate only
|
||||||
|
for an agent it is told of, and without one lists every open role, so
|
||||||
|
the first request carries the agent whenever one is known before the
|
||||||
|
agents are in: the address's, else the one this browser viewed last
|
||||||
|
(`storedAgent`). `chooseAgent` checks it against the enabled ones when
|
||||||
|
they land, and the feed reloads if it picks another. With neither, the
|
||||||
|
feed waits for the agents: with none enabled the list of every open
|
||||||
|
role is the right one, and when they cannot be had the feed's error
|
||||||
|
state offers to try again (`loadFeed`), rather than every open role
|
||||||
|
passing for the agent's list.
|
||||||
|
|
||||||
|
The rest fills in when it arrives and the feed does not wait on it: the
|
||||||
|
companies, the tag counts, the saved filters. Each one drawn before is
|
||||||
|
greyed or missing, which is what "no agent" (renderAgent), an unjoined
|
||||||
|
company (paintRow) and the tag row being empty already mean, so nothing
|
||||||
|
here is a new state to handle. `keep` and `y` put the list back as it
|
||||||
|
was (`kept`). Run again by the error state's retry when the agents did
|
||||||
|
not come. */
|
||||||
|
async function start({ keep = 0, y = null } = {}) {
|
||||||
|
if (state.filter.agent === null) state.filter.agent = storedAgent();
|
||||||
|
const agentsIn = loadAgents().then(() => { chooseAgent(); return true; }, () => false);
|
||||||
|
const trackedIn = loadTracked().catch(() => {});
|
||||||
|
// What the reader's fetch and then the feed's first request go out with,
|
||||||
|
// so the agents landing later know whether either is to be asked again.
|
||||||
|
const routed = state.filter.agent;
|
||||||
|
route();
|
||||||
|
if (state.filter.agent === null) await agentsIn;
|
||||||
|
const asked = state.filter.agent;
|
||||||
|
const sortAsked = sortShown();
|
||||||
|
const enhance = (async () => {
|
||||||
|
const [ready] = await Promise.all([agentsIn, trackedIn]);
|
||||||
|
if (!ready) return;
|
||||||
|
await loadCompanies(state.filter.agent).catch(() => {});
|
||||||
|
renderRail();
|
||||||
|
renderActive();
|
||||||
|
loadFilters().catch(() => {});
|
||||||
|
loadSaved().catch(() => {});
|
||||||
|
// Reload for the score and votes when the agent chosen is not the one
|
||||||
|
// asked for (a remembered one disabled since), and either way bring
|
||||||
|
// the reader's affinity control and judge buttons up to date, since
|
||||||
|
// `companies` was still empty when it first painted. What fits me
|
||||||
|
// asked of an agent with no CV is What I'm looking for.
|
||||||
|
if (state.filter.agent !== asked || sortShown() !== sortAsked) reload();
|
||||||
|
else renderFeed();
|
||||||
|
if (state.readingUid) {
|
||||||
|
if (state.filter.agent !== routed) refreshReader(state.readingUid);
|
||||||
|
else renderReader();
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
await loadFeed({ keep });
|
||||||
|
if (y !== null && !state.readingUid) window.scrollTo(0, y);
|
||||||
|
await enhance;
|
||||||
|
}
|
||||||
|
|
||||||
document.addEventListener("DOMContentLoaded", async () => {
|
document.addEventListener("DOMContentLoaded", async () => {
|
||||||
readUrl();
|
readUrl();
|
||||||
const was = kept();
|
const was = kept();
|
||||||
@@ -2161,44 +2327,6 @@
|
|||||||
// Named before the first request, which a token the server does not
|
// Named before the first request, which a token the server does not
|
||||||
// know would turn into a 422 rather than a list.
|
// know would turn into a 422 rather than a list.
|
||||||
await labelPlaces();
|
await labelPlaces();
|
||||||
|
await start(was ? { keep: was.count, y: was.y || 0 } : {});
|
||||||
// The feed is the page; agents and companies are enhancements that fill
|
|
||||||
// in when they arrive, not something it waits on — a slow or failed
|
|
||||||
// /api/agents or /api/companies must not leave it blank. Each one drawn
|
|
||||||
// here is greyed or missing until `enhance` lands, which is what "no
|
|
||||||
// agent" (renderAgent), an unjoined company (paintRow) and the tag row
|
|
||||||
// being empty already mean, so nothing here is a new state to handle.
|
|
||||||
const before = state.filter.agent;
|
|
||||||
const sortBefore = sortShown();
|
|
||||||
const enhance = (async () => {
|
|
||||||
try {
|
|
||||||
await Promise.all([loadTracked(), loadAgents()]);
|
|
||||||
chooseAgent();
|
|
||||||
await loadCompanies(state.filter.agent);
|
|
||||||
} catch (err) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
renderRail();
|
|
||||||
renderActive();
|
|
||||||
loadFilters().catch(() => {});
|
|
||||||
loadSaved().catch(() => {});
|
|
||||||
// The feed's first request went out with whatever agent `readUrl` had
|
|
||||||
// (often none): reload for the score and votes once the real one is
|
|
||||||
// known, and either way bring the reader's affinity control and judge
|
|
||||||
// buttons up to date, since `companies` was still empty when it first
|
|
||||||
// painted.
|
|
||||||
// CV match asked of an agent with no CV is Agent match.
|
|
||||||
if (state.filter.agent !== before || sortShown() !== sortBefore) reload();
|
|
||||||
else renderFeed();
|
|
||||||
if (state.readingUid) {
|
|
||||||
if (state.filter.agent !== before) refreshReader(state.readingUid);
|
|
||||||
else renderReader();
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
|
|
||||||
route();
|
|
||||||
await loadFeed({ keep: was ? was.count : 0 });
|
|
||||||
if (was && !state.readingUid) window.scrollTo(0, was.y || 0);
|
|
||||||
await enhance;
|
|
||||||
});
|
});
|
||||||
})();
|
})();
|
||||||
+66
-13
@@ -55,6 +55,10 @@ CREATE TABLE IF NOT EXISTS roles (
|
|||||||
country TEXT NOT NULL DEFAULT '',
|
country TEXT NOT NULL DEFAULT '',
|
||||||
remote TEXT NOT NULL DEFAULT ''
|
remote TEXT NOT NULL DEFAULT ''
|
||||||
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
CHECK (remote IN ('', 'remote', 'hybrid', 'onsite')),
|
||||||
|
-- 1 when the title or text says the role is worked remotely while no
|
||||||
|
-- field does (prep, app.prep.words.remote_in_text): the location gate
|
||||||
|
-- lets such a role through until extract has read it.
|
||||||
|
remote_text INTEGER NOT NULL DEFAULT 0,
|
||||||
salary_min INTEGER,
|
salary_min INTEGER,
|
||||||
salary_max INTEGER,
|
salary_max INTEGER,
|
||||||
currency TEXT NOT NULL DEFAULT '',
|
currency TEXT NOT NULL DEFAULT '',
|
||||||
@@ -328,9 +332,8 @@ END;
|
|||||||
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
|
-- as JSON: a key it lacks means the `Limits` default. fetch_terms is a JSON
|
||||||
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
|
-- list of words that narrow what big boards send (app/jobs/reach.py); empty
|
||||||
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
|
-- means no narrowing. cv_text, cv_languages (a JSON list of {language,
|
||||||
-- level}), like_text (what they like: the work, the workplace, situations),
|
-- level}), like_text (what they like: the work, the workplace, situations)
|
||||||
-- avoid_text (what they do not like) and must_text (must-haves and
|
-- and avoid_text (what they do not like) are what judge reads a role
|
||||||
-- deal-breakers, set through the API only) are what judge reads a role
|
|
||||||
-- against; an agent with no CV is not judged.
|
-- against; an agent with no CV is not judged.
|
||||||
CREATE TABLE IF NOT EXISTS agents (
|
CREATE TABLE IF NOT EXISTS agents (
|
||||||
id INTEGER PRIMARY KEY,
|
id INTEGER PRIMARY KEY,
|
||||||
@@ -344,7 +347,6 @@ CREATE TABLE IF NOT EXISTS agents (
|
|||||||
cv_text TEXT NOT NULL DEFAULT '',
|
cv_text TEXT NOT NULL DEFAULT '',
|
||||||
cv_languages TEXT NOT NULL DEFAULT '[]',
|
cv_languages TEXT NOT NULL DEFAULT '[]',
|
||||||
avoid_text TEXT NOT NULL DEFAULT '',
|
avoid_text TEXT NOT NULL DEFAULT '',
|
||||||
must_text TEXT NOT NULL DEFAULT '',
|
|
||||||
like_text TEXT NOT NULL DEFAULT ''
|
like_text TEXT NOT NULL DEFAULT ''
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -431,13 +433,12 @@ CREATE TABLE IF NOT EXISTS agent_cv_vectors (
|
|||||||
|
|
||||||
-- One agent's judge reading of a role against its CV. Its own table, since
|
-- One agent's judge reading of a role against its CV. Its own table, since
|
||||||
-- replace_scores rewrites scores every run and a judgement outlives that
|
-- replace_scores rewrites scores every run and a judgement outlives that
|
||||||
-- and the role closing. skills, experience and career are 0 to 100,
|
-- and the role closing. skills, experience and career are 0 to 10 (0 to
|
||||||
-- cv_score their weighted total, strengths and gaps JSON lists. input_key
|
-- 100 until migration 33), cv_score their weighted total times 10 at the
|
||||||
-- is a hash of the CV, prompts, avoid text and the role's text it read: a
|
-- default weights (not read: app.models.cv_score is worked out when a role
|
||||||
-- different one is read again. musts is a JSON list of {id, text, verdict},
|
-- is served), strengths and gaps JSON lists. input_key
|
||||||
-- the model's verdict on each of the agent's must-haves (one per line of its
|
-- is a hash of the CV, prompts, languages, like and avoid texts and the
|
||||||
-- must text), broken how many are "broken": career counts as at most 39 then
|
-- role's text it read: a different one is read again. tries counts
|
||||||
-- (app.models.BROKEN_CAP), applied when the role is served. tries counts
|
|
||||||
-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A
|
-- replies that left the role without one since; at 3 it is given up until the agent is saved again. A
|
||||||
-- row with judged_at NULL holds only tries.
|
-- row with judged_at NULL holds only tries.
|
||||||
CREATE TABLE IF NOT EXISTS role_judgements (
|
CREATE TABLE IF NOT EXISTS role_judgements (
|
||||||
@@ -449,8 +450,6 @@ CREATE TABLE IF NOT EXISTS role_judgements (
|
|||||||
cv_score REAL,
|
cv_score REAL,
|
||||||
strengths TEXT NOT NULL DEFAULT '[]',
|
strengths TEXT NOT NULL DEFAULT '[]',
|
||||||
gaps TEXT NOT NULL DEFAULT '[]',
|
gaps TEXT NOT NULL DEFAULT '[]',
|
||||||
musts TEXT NOT NULL DEFAULT '[]',
|
|
||||||
broken INTEGER NOT NULL DEFAULT 0,
|
|
||||||
input_key TEXT NOT NULL DEFAULT '',
|
input_key TEXT NOT NULL DEFAULT '',
|
||||||
tries INTEGER NOT NULL DEFAULT 0,
|
tries INTEGER NOT NULL DEFAULT 0,
|
||||||
judged_at TEXT,
|
judged_at TEXT,
|
||||||
@@ -758,6 +757,15 @@ UPDATE roles SET seniority = COALESCE(NULLIF(seniority_model, ''),
|
|||||||
(32, "the cut: scores.fit and scores.cut, agents' limits reset to the cut's defaults, "
|
(32, "the cut: scores.fit and scores.cut, agents' limits reset to the cut's defaults, "
|
||||||
"prefer gates gone, saved_filters, and enrich renamed fetch text",
|
"prefer gates gone, saved_filters, and enrich renamed fetch text",
|
||||||
lambda conn: _add_cut(conn)),
|
lambda conn: _add_cut(conn)),
|
||||||
|
# A Python step: JSON limits, and a count for the log.
|
||||||
|
(33, "the judge on 0 to 10: every judgement cleared to be made again, and summarise_top "
|
||||||
|
"50 becomes 30", lambda conn: _judge_out_of_ten(conn)),
|
||||||
|
# A Python step: it drops the columns only where they are there. The
|
||||||
|
# judge reads every role again (judge.FORMAT).
|
||||||
|
(34, "agents.must_text, role_judgements.musts and broken dropped: the profile is the "
|
||||||
|
"CV, languages, likes and dislikes", lambda conn: _drop_musts(conn)),
|
||||||
|
(35, "roles.remote_text: the text says remote while no field does, read by prep for the "
|
||||||
|
"location gate", lambda conn: _add_remote_text(conn)),
|
||||||
)
|
)
|
||||||
|
|
||||||
#: What a file created from `SCHEMA` today is already at.
|
#: What a file created from `SCHEMA` today is already at.
|
||||||
@@ -1007,6 +1015,15 @@ def _add_posting_url(conn: sqlite3.Connection) -> None:
|
|||||||
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
conn.execute("ALTER TABLE roles ADD COLUMN posting_url TEXT NOT NULL DEFAULT ''")
|
||||||
|
|
||||||
|
|
||||||
|
def _add_remote_text(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Migration 35's step: `roles.remote_text`, 0 until the next prep,
|
||||||
|
which maps every open role again since its rules changed. Runs twice
|
||||||
|
without harm."""
|
||||||
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(roles)")}
|
||||||
|
if "remote_text" not in columns:
|
||||||
|
conn.execute("ALTER TABLE roles ADD COLUMN remote_text INTEGER NOT NULL DEFAULT 0")
|
||||||
|
|
||||||
|
|
||||||
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
def _add_remote_area(conn: sqlite3.Connection) -> None:
|
||||||
"""Migration 27's step: `roles.remote_model` empty and
|
"""Migration 27's step: `roles.remote_model` empty and
|
||||||
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
`roles.remote_where_model` NULL for every role, so `extract` reads the
|
||||||
@@ -1191,6 +1208,42 @@ def _add_cut(conn: sqlite3.Connection) -> None:
|
|||||||
"dropped", reset, dropped)
|
"dropped", reset, dropped)
|
||||||
|
|
||||||
|
|
||||||
|
def _judge_out_of_ten(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Migration 33's step. The judge scores 0 to 10 rather than 0 to 100
|
||||||
|
(`judge.FORMAT` 4), so every judgement is cleared: its numbers, the
|
||||||
|
verdicts, strengths and gaps, `input_key`, `judged_at` and `tries`, and no
|
||||||
|
old number is shown on the new scale. The rows stay, as rows of no
|
||||||
|
judgement, and the judge reads every pair again. An agent whose
|
||||||
|
`summarise_top` is 50, migration 32's default, takes 30, the new one.
|
||||||
|
Runs twice without harm, also after migration 34 dropped the verdicts."""
|
||||||
|
columns = {r[1] for r in conn.execute("PRAGMA table_info(role_judgements)")}
|
||||||
|
verdicts = ", musts = '[]', broken = 0" if {"musts", "broken"} <= columns else ""
|
||||||
|
cleared = conn.execute(
|
||||||
|
f"""UPDATE role_judgements
|
||||||
|
SET skills = NULL, experience = NULL, career = NULL, cv_score = NULL,
|
||||||
|
strengths = '[]', gaps = '[]'{verdicts},
|
||||||
|
input_key = '', tries = 0, judged_at = NULL""").rowcount
|
||||||
|
lowered = conn.execute(
|
||||||
|
"""UPDATE agents SET limits = json_set(limits, '$.summarise_top', 30)
|
||||||
|
WHERE json_valid(limits) AND json_type(limits) = 'object'
|
||||||
|
AND json_extract(limits, '$.summarise_top') = 50""").rowcount
|
||||||
|
log.info("migration 33: %d judgements cleared for the 0 to 10 scale, %d agents' "
|
||||||
|
"summarise_top 50 set to 30", cleared, lowered)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_musts(conn: sqlite3.Connection) -> None:
|
||||||
|
"""Migration 34's step: the must-haves are gone, the agent's text
|
||||||
|
(`agents.must_text`) and the judge's verdicts on it
|
||||||
|
(`role_judgements.musts`, `broken`), and with them the cap on career.
|
||||||
|
Runs twice without harm."""
|
||||||
|
for table, columns in (("agents", ("must_text",)),
|
||||||
|
("role_judgements", ("musts", "broken"))):
|
||||||
|
present = {r[1] for r in conn.execute(f"PRAGMA table_info({table})")}
|
||||||
|
for column in columns:
|
||||||
|
if column in present:
|
||||||
|
conn.execute(f"ALTER TABLE {table} DROP COLUMN {column}")
|
||||||
|
|
||||||
|
|
||||||
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
def _add_all_roles(conn: sqlite3.Connection) -> None:
|
||||||
"""Migration 21's step. The column; the tag for every company that is a
|
"""Migration 21's step. The column; the tag for every company that is a
|
||||||
favourite of any agent, since favourite no longer widens what a pull
|
favourite of any agent, since favourite no longer widens what a pull
|
||||||
|
|||||||
+217
-142
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from collections.abc import Callable, Iterable, Iterator, Mapping, MutableMapping, Sequence
|
from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence
|
||||||
from contextlib import closing, contextmanager
|
from contextlib import closing, contextmanager
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -26,7 +26,7 @@ from app.models import (
|
|||||||
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
AFFINITY, HAND_REASONS, LEVELS, STATUSES, Agent, Application,
|
||||||
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
|
ApplicationEvent, Candidate, Company, CvLanguage, Gate, Judgement, Limits, Prompt, Role,
|
||||||
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
|
RoleCopy, RolePlace, RoleRow, SavedFilter, ScoredRole, cv_languages as normal_languages, cv_score,
|
||||||
BROKEN_CAP, capped_career, language_flag, role_languages,
|
language_flag, role_languages,
|
||||||
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
Subscription, Weights, check_prompts, company_fold, company_key, country_of, excerpt, place_key,
|
||||||
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
|
fetch_terms as normal_terms, host_word, iso_day, posted_at, posted_host, remote_of, title_key,
|
||||||
)
|
)
|
||||||
@@ -65,7 +65,8 @@ _EXTRACT_TEXT = ("place", "country", "currency", "permit")
|
|||||||
GIVE_UP_AFTER = 3
|
GIVE_UP_AFTER = 3
|
||||||
|
|
||||||
#: A role `extract` has yet to read: still open, no copy of another
|
#: A role `extract` has yet to read: still open, no copy of another
|
||||||
#: (`rebuild_dupes`), not given up, and either never answered, without a
|
#: (`rebuild_dupes`), not given up, and either never answered (or answered
|
||||||
|
#: before a better text came, `upsert_role`), without a
|
||||||
#: model level, never asked where a remote hire may live
|
#: model level, never asked where a remote hire may live
|
||||||
#: (`remote_where_model` NULL), or never asked for its languages and deadline
|
#: (`remote_where_model` NULL), or never asked for its languages and deadline
|
||||||
#: (`languages_required` NULL). The last three make `extract` the
|
#: (`languages_required` NULL). The last three make `extract` the
|
||||||
@@ -83,6 +84,15 @@ _EXTRACT_WAITING = ("(extracted_at IS NULL OR seniority_model = '' OR remote_whe
|
|||||||
_IN_CUT = ("r.uid IN (SELECT s.uid FROM scores s JOIN agents a ON a.id = s.agent_id "
|
_IN_CUT = ("r.uid IN (SELECT s.uid FROM scores s JOIN agents a ON a.id = s.agent_id "
|
||||||
"AND a.enabled = 1 WHERE s.cut <> '')")
|
"AND a.enabled = 1 WHERE s.cut <> '')")
|
||||||
|
|
||||||
|
#: A role one agent's feed shows, as a `WHERE` on `r` taking the agent's id
|
||||||
|
#: twice: its fetch gate let the role through (`_roles_from` with
|
||||||
|
#: `fetch_gate`), and the company is not one it hid. With the open and kept
|
||||||
|
#: copy test, what the rail's counts for that agent cover.
|
||||||
|
_SHOWN = ("EXISTS (SELECT 1 FROM scores s WHERE s.uid = r.uid AND s.agent_id = ? "
|
||||||
|
"AND s.gated_reason = '') "
|
||||||
|
"AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
|
||||||
|
"WHERE agent_id = ? AND affinity = -1)")
|
||||||
|
|
||||||
#: The order within one agent's cut, over its scores as `s`: in by fit, best
|
#: The order within one agent's cut, over its scores as `s`: in by fit, best
|
||||||
#: fit first, then in by CV closeness, closest first, then kept.
|
#: fit first, then in by CV closeness, closest first, then kept.
|
||||||
_CUT_ORDER = ("CASE s.cut WHEN 'fit' THEN 0 WHEN 'cv' THEN 1 ELSE 2 END, "
|
_CUT_ORDER = ("CASE s.cut WHEN 'fit' THEN 0 WHEN 'cv' THEN 1 ELSE 2 END, "
|
||||||
@@ -130,7 +140,7 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
|
|||||||
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
|
||||||
|
|
||||||
#: The agent's `judge` readings beside every role, the same way: `j.skills`,
|
#: The agent's `judge` readings beside every role, the same way: `j.skills`,
|
||||||
#: `j.experience`, `j.career` and `j.broken` are NULL for a role not judged and with no
|
#: `j.experience` and `j.career` are NULL for a role not judged and with no
|
||||||
#: agent.
|
#: agent.
|
||||||
_WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? "
|
_WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agent_id = ? "
|
||||||
"AND j.judged_at IS NOT NULL")
|
"AND j.judged_at IS NOT NULL")
|
||||||
@@ -138,20 +148,17 @@ _WITH_JUDGEMENTS = ("LEFT JOIN role_judgements j ON j.uid = roles.uid AND j.agen
|
|||||||
|
|
||||||
|
|
||||||
def _cv_sql(weights: Weights | None) -> str:
|
def _cv_sql(weights: Weights | None) -> str:
|
||||||
"""`cv_score` as SQL for one agent's weights, unrounded (`app.models.cv_score`
|
"""`cv_score` as SQL for one agent's weights, 0 to 100 from the judge's
|
||||||
rounds it): NULL for a role not judged, and for all three weights 0. Career
|
three numbers 0 to 10, unrounded (`app.models.cv_score` rounds it): NULL
|
||||||
counts as at most `BROKEN_CAP` when the judgement holds a broken
|
for a role not judged, and for all three weights 0. The weights are
|
||||||
must-have (`j.broken`), as `app.models.capped_career` has it. The
|
validated, finite floats, so they are written in rather than bound, the
|
||||||
numbers are the model's validated, finite floats, so they are written in
|
select's placeholders being then no concern."""
|
||||||
rather than bound, the select's placeholders being then no concern."""
|
|
||||||
w = weights or Weights()
|
w = weights or Weights()
|
||||||
total = w.cv_skills + w.cv_experience + w.cv_career
|
total = w.cv_skills + w.cv_experience + w.cv_career
|
||||||
if not total:
|
if not total:
|
||||||
return "NULL"
|
return "NULL"
|
||||||
career = (f"(CASE WHEN j.broken > 0 AND j.career > {BROKEN_CAP} THEN {BROKEN_CAP} "
|
return (f"(10.0 * ({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience "
|
||||||
"ELSE j.career END)")
|
f"+ {w.cv_career!r} * j.career) / {total!r})")
|
||||||
return (f"(({w.cv_skills!r} * j.skills + {w.cv_experience!r} * j.experience "
|
|
||||||
f"+ {w.cv_career!r} * {career}) / {total!r})")
|
|
||||||
|
|
||||||
|
|
||||||
def _by_cv(expression: str) -> str:
|
def _by_cv(expression: str) -> str:
|
||||||
@@ -288,14 +295,22 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
|||||||
`params` belong to `match`; `empty` takes none.
|
`params` belong to `match`; `empty` takes none.
|
||||||
|
|
||||||
`country` and `place` match over the gazetteer, as the feed's Place
|
`country` and `place` match over the gazetteer, as the feed's Place
|
||||||
filter does (`_places_sql`): a country gate is met by a role whose
|
filter does (`_places_sql`), and also on a role place that holds one of
|
||||||
|
the gate's places (`wider`): a country gate is met by a role whose
|
||||||
`country` is one of its codes or that has a place in one of them, so
|
`country` is one of its codes or that has a place in one of them, so
|
||||||
`DK` is met by `Copenhagen; London`. Its column is empty only when the
|
`DK` is met by `Copenhagen; London`, and by Øresund or Europe alone,
|
||||||
role has neither. A `require` gate on country reads a remote role whose
|
which are no evidence the role is outside Denmark; a town in Germany
|
||||||
|
is, also beside Europe. Its column is empty only when the role has neither a country
|
||||||
|
nor a resolved place. A `require` gate on country reads a remote role whose
|
||||||
area is stated through the area alone, as the remote chips do
|
area is stated through the area alone, as the remote chips do
|
||||||
(`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)"
|
(`Gazetteer.overlap`): "Home Based - Worldwide" and "Germany (Remote)"
|
||||||
meet Europe, "Remote US" does not, and neither is empty. A place gate's
|
meet Europe, "Remote US" does not, and neither is empty. A place gate's
|
||||||
value is a place token, bare text a `place:`.
|
value is a place token, bare text a `place:`. Its column is empty when
|
||||||
|
the role has no place the gazetteer resolved, as "Storkøbenhavn" or
|
||||||
|
"Headquarters" are not: that is no evidence the role is elsewhere. Such
|
||||||
|
a role can still match, by its country or by whole words of its line
|
||||||
|
for a town the gazetteer does not know, so a place gate's `match` is
|
||||||
|
tested before its `empty` (`Store.score_candidates`).
|
||||||
|
|
||||||
`remote_from` takes place tokens without their `remote:` (`country:DK`,
|
`remote_from` takes place tokens without their `remote:` (`country:DK`,
|
||||||
`anywhere`) and matches as the remote chips do. It is empty for a role
|
`anywhere`) and matches as the remote chips do. It is empty for a role
|
||||||
@@ -313,7 +328,8 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
|||||||
return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params
|
return f"(r.remote = '' OR (r.remote = 'remote' AND NOT {_HAS_AREA}))", match, params
|
||||||
if gate.field == "country":
|
if gate.field == "country":
|
||||||
tokens = [f"country:{v}" for v in values]
|
tokens = [f"country:{v}" for v in values]
|
||||||
match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values])
|
match, params = _places_sql(tokens, gaz, table="r", codes=[v.upper() for v in values],
|
||||||
|
wider=True)
|
||||||
empty = f"(r.country = '' AND {_unplaced(gaz)})"
|
empty = f"(r.country = '' AND {_unplaced(gaz)})"
|
||||||
if gate.mode != "require":
|
if gate.mode != "require":
|
||||||
return empty, match, params
|
return empty, match, params
|
||||||
@@ -322,8 +338,8 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
|||||||
return (f"({empty} AND NOT {_HAS_AREA})",
|
return (f"({empty} AND NOT {_HAS_AREA})",
|
||||||
f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params])
|
f"(CASE WHEN {_HAS_AREA} THEN {area} ELSE {match} END)", [*more, *params])
|
||||||
if gate.field == "place":
|
if gate.field == "place":
|
||||||
match, params = _places_sql(values, gaz, table="r")
|
match, params = _places_sql(values, gaz, table="r", wider=True)
|
||||||
return "r.place = ''", match, params
|
return _unplaced(gaz), match, params
|
||||||
if gate.field == "title":
|
if gate.field == "title":
|
||||||
# Whole words of the folded title: " account executive " inside
|
# Whole words of the folded title: " account executive " inside
|
||||||
# " enterprise account executive uk ". A fold drops `%` and `_`, so a
|
# " enterprise account executive uk ". A fold drops `%` and `_`, so a
|
||||||
@@ -339,19 +355,33 @@ def _gate_sql(gate: Gate, gaz: places.Gazetteer) -> tuple[str, str, list[Any]]:
|
|||||||
return ("r.languages IN ('', '[]')",
|
return ("r.languages IN ('', '[]')",
|
||||||
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
f"NOT EXISTS (SELECT 1 FROM json_each(r.languages) "
|
||||||
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
f"WHERE LOWER(value) NOT IN ({marks}))", values)
|
||||||
|
if gate.field == "remote" and "remote" in values:
|
||||||
|
# A role whose text says remote while no field does is possibly
|
||||||
|
# remote (`remote_text`, prep): it meets a gate on remote until
|
||||||
|
# extract has read it, and is not empty for it.
|
||||||
|
return (f"({column} = '' AND r.remote_text = 0)",
|
||||||
|
f"(LOWER({column}) IN ({marks}) OR ({column} = '' AND r.remote_text = 1))", values)
|
||||||
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
return f"{column} = ''", f"LOWER({column}) IN ({marks})", values
|
||||||
|
|
||||||
|
|
||||||
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
|
def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "roles",
|
||||||
codes: Sequence[str] = (), scope: Sequence[Gate] = ()) -> tuple[str, list[Any]]:
|
codes: Sequence[str] = (), scope: Sequence[Gate] = (),
|
||||||
|
wider: bool = False) -> tuple[str, list[Any]]:
|
||||||
"""Any of the place tokens over `table`, as a join over the gazetteer: a
|
"""Any of the place tokens over `table`, as a join over the gazetteer: a
|
||||||
role with a place under one of the places the tokens name (`country:DK`
|
role with a place under one of the places the tokens name (`country:DK`
|
||||||
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
|
every place in Denmark, `region:skane` every town in Skåne, `place:malmo`
|
||||||
Malmö by any spelling), or whose `country` is one of their countries (a
|
Malmö by any spelling), or whose `country` is one of their countries (a
|
||||||
role whose board gave a country and no place). `codes` are more values of
|
role whose board gave a country and no place). `codes` are more values of
|
||||||
`country` to accept. A `place:` token the gazetteer does not know is
|
`country` to accept. `wider` also takes a role placed only at places
|
||||||
matched on whole words of the role's folded line (`places.match_sql`),
|
that hold one of them (`Gazetteer.overlap`): Øresund, Europe or Anywhere
|
||||||
so a town nobody has entered yet is still found.
|
alone meet `country:DK`, as the fetch gate reads them (`_gate_sql`); the
|
||||||
|
feed's Place filter leaves it off. A role that also names a place
|
||||||
|
elsewhere is not taken: `London; Europe` is in London, and `Aarhus;
|
||||||
|
Denmark` does not meet Copenhagen. The `country` column is not widened,
|
||||||
|
as Sweden would then meet Malmö for a role in Stockholm. A `place:`
|
||||||
|
token the gazetteer does not know is matched on whole words of the
|
||||||
|
role's folded line (`places.match_sql`), so a town nobody has entered
|
||||||
|
yet is still found.
|
||||||
|
|
||||||
A `remote:` token is a remote role whose area overlaps its places
|
A `remote:` token is a remote role whose area overlaps its places
|
||||||
(`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere.
|
(`Gazetteer.overlap`), `remote:anywhere` one whose area is Anywhere.
|
||||||
@@ -359,6 +389,7 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
|
|||||||
`scope`, the viewing agent's `require` gates on country; with none,
|
`scope`, the viewing agent's `require` gates on country; with none,
|
||||||
every remote role."""
|
every remote role."""
|
||||||
ids: set[int] = set()
|
ids: set[int] = set()
|
||||||
|
around: set[int] = set()
|
||||||
areas: set[int] = set()
|
areas: set[int] = set()
|
||||||
found = {c for c in codes if c}
|
found = {c for c in codes if c}
|
||||||
clauses: list[str] = []
|
clauses: list[str] = []
|
||||||
@@ -381,15 +412,26 @@ def _places_sql(tokens: Sequence[str], gaz: places.Gazetteer, *, table: str = "r
|
|||||||
continue
|
continue
|
||||||
under = gaz.under(named)
|
under = gaz.under(named)
|
||||||
ids |= under
|
ids |= under
|
||||||
|
if wider:
|
||||||
|
around |= gaz.overlap(named)
|
||||||
found |= gaz.codes(under)
|
found |= gaz.codes(under)
|
||||||
|
first: list[tuple[str, list[Any]]] = []
|
||||||
if ids:
|
if ids:
|
||||||
clauses.insert(0, f"{table}.uid IN (SELECT uid FROM role_places "
|
first.append((f"{table}.uid IN (SELECT uid FROM role_places "
|
||||||
"WHERE place_id IN (SELECT value FROM json_each(?)))")
|
"WHERE place_id IN (SELECT value FROM json_each(?)))",
|
||||||
params.insert(0, json.dumps(sorted(ids)))
|
[json.dumps(sorted(ids))]))
|
||||||
if found:
|
if found:
|
||||||
clauses.insert(1 if ids else 0,
|
first.append((f"{table}.country IN (SELECT value FROM json_each(?))",
|
||||||
f"{table}.country IN (SELECT value FROM json_each(?))")
|
[json.dumps(sorted(found))]))
|
||||||
params.insert(1 if ids else 0, json.dumps(sorted(found)))
|
if around - ids:
|
||||||
|
first.append((f"({table}.uid IN (SELECT uid FROM role_places "
|
||||||
|
"WHERE place_id IN (SELECT value FROM json_each(?))) "
|
||||||
|
f"AND {table}.uid NOT IN (SELECT uid FROM role_places "
|
||||||
|
"WHERE place_id IS NOT NULL "
|
||||||
|
"AND place_id NOT IN (SELECT value FROM json_each(?))))",
|
||||||
|
[json.dumps(sorted(around - ids)), json.dumps(sorted(around))]))
|
||||||
|
clauses[:0] = [sql for sql, _ in first]
|
||||||
|
params[:0] = [value for _, values in first for value in values]
|
||||||
if areas:
|
if areas:
|
||||||
remote.append(_areas_sql(areas, table))
|
remote.append(_areas_sql(areas, table))
|
||||||
for sql, values in remote:
|
for sql, values in remote:
|
||||||
@@ -508,7 +550,8 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
|||||||
|
|
||||||
A copy the host published for another place (`dupes.Folded.placed`)
|
A copy the host published for another place (`dupes.Folded.placed`)
|
||||||
leaves its places on the kept role as `role_places` rows with the copy's
|
leaves its places on the kept role as `role_places` rows with the copy's
|
||||||
link, see `_sync_copy_places`."""
|
link, see `_sync_copy_places`. Its remote stays its own, which is why a
|
||||||
|
remote listing is never folded into an office one (`dupes.fold`)."""
|
||||||
# Migration 18 runs this on a file that has no `teaser` and no
|
# Migration 18 runs this on a file that has no `teaser` and no
|
||||||
# `role_places` yet: there it groups as before and folds no places.
|
# `role_places` yet: there it groups as before and folds no places.
|
||||||
current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)"))
|
current = any(r[1] == "teaser" for r in conn.execute("PRAGMA table_info(roles)"))
|
||||||
@@ -516,7 +559,7 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
|||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
f"""SELECT uid, company_key, title_key, country, place, {_HOST} AS host,
|
||||||
description != '' AS described, posted_at,
|
description != '' AS described, posted_at,
|
||||||
SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url
|
SUBSTR({text}, 1, {dupes.TEXT_CHARS}), original_url, remote = 'remote'
|
||||||
FROM roles
|
FROM roles
|
||||||
WHERE closed_at IS NULL AND company_key != ''
|
WHERE closed_at IS NULL AND company_key != ''
|
||||||
AND (company_key, title_key) IN (
|
AND (company_key, title_key) IN (
|
||||||
@@ -527,7 +570,7 @@ def rebuild_dupes(conn: sqlite3.Connection) -> DupeCounts:
|
|||||||
folded = made(
|
folded = made(
|
||||||
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
dupes.Copy(uid=r[0], company_key=r[1], title_key=r[2], country=r[3], place=r[4],
|
||||||
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None,
|
host=r[5], has_description=bool(r[6]), posted_at=r[7] or None,
|
||||||
text=r[8] or "", url=r[9] or "")
|
text=r[8] or "", url=r[9] or "", remote=bool(r[10]))
|
||||||
for r in rows)
|
for r in rows)
|
||||||
new = folded.dup_of
|
new = folded.dup_of
|
||||||
old: dict[str, str | None] = dict.fromkeys(new)
|
old: dict[str, str | None] = dict.fromkeys(new)
|
||||||
@@ -819,8 +862,9 @@ class Store:
|
|||||||
the way a board's own structured field is (docs/decisions.md, "a
|
the way a board's own structured field is (docs/decisions.md, "a
|
||||||
guess is not a board field"). So once a role has been through
|
guess is not a board field"). So once a role has been through
|
||||||
`extract` (`extracted_at` is set), this upsert leaves `function`
|
`extract` (`extracted_at` is set), this upsert leaves `function`
|
||||||
alone for good, whatever the plugin's raw guess now says — see
|
alone, whatever the plugin's raw guess now says, until a better text
|
||||||
`set_extracted` for the other half of the rule.
|
puts the role back in extract's queue (below) — see `set_extracted`
|
||||||
|
for the other half of the rule.
|
||||||
|
|
||||||
The role's places (`role_places`) are the texts of `raw["places"]`,
|
The role's places (`role_places`) are the texts of `raw["places"]`,
|
||||||
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
else `raw["place"]` cut at `;` (`gazetteer.texts`), written unresolved
|
||||||
@@ -832,9 +876,13 @@ class Store:
|
|||||||
under the same keep-filled rule. `teaser` is kept-filled too.
|
under the same keep-filled rule. `teaser` is kept-filled too.
|
||||||
|
|
||||||
A role is embedded from its description, else its teaser, else its
|
A role is embedded from its description, else its teaser, else its
|
||||||
title (`app.jobs.embed.document`). When a pull brings a better one of
|
title (`app.jobs.embed.document`), and `extract` reads the same. When
|
||||||
these than the role had, its vector is dropped so the next `embed`
|
a pull brings a better one of these than the role had, its vector is
|
||||||
reads it.
|
dropped so the next `embed` reads it, and `extracted_at` and
|
||||||
|
`extract_tries` are reset so `extract` reads it again (if it is still
|
||||||
|
in a cut). That read overwrites the model's own columns and fills
|
||||||
|
what is still empty (`set_extracted`); until it comes, `function` is
|
||||||
|
the plugin's guess's again, as for any role not yet read.
|
||||||
"""
|
"""
|
||||||
if not item.source:
|
if not item.source:
|
||||||
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
raise ValueError(f"item {item.source_id!r} has no source; the runner sets it")
|
||||||
@@ -921,14 +969,17 @@ class Store:
|
|||||||
if before is not None:
|
if before is not None:
|
||||||
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
conn.execute("UPDATE roles SET prep_due = prep_due + 1 WHERE uid = ?",
|
||||||
(row["uid"],))
|
(row["uid"],))
|
||||||
# `embed` encodes a role once, from the best text it has. When a
|
# `embed` encodes a role once, from the best text it has, and
|
||||||
# later pull brings a better one, as a host that states the text
|
# `extract` reads it once. When a later pull brings a better one,
|
||||||
# on the role's own page does, the vector goes, so the next
|
# as fetch text does for a host that states the text on the
|
||||||
# `embed` reads it.
|
# role's own page, the vector goes and the role waits for
|
||||||
|
# extract again, so both read the new text.
|
||||||
if before is not None and _text_rank(row["description"] or before["description"],
|
if before is not None and _text_rank(row["description"] or before["description"],
|
||||||
row["teaser"] or before["teaser"]) \
|
row["teaser"] or before["teaser"]) \
|
||||||
> _text_rank(before["description"], before["teaser"]):
|
> _text_rank(before["description"], before["teaser"]):
|
||||||
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
|
conn.execute("DELETE FROM vectors WHERE uid=?", (row["uid"],))
|
||||||
|
conn.execute("UPDATE roles SET extracted_at = NULL, extract_tries = 0 "
|
||||||
|
"WHERE uid=?", (row["uid"],))
|
||||||
return before is None
|
return before is None
|
||||||
|
|
||||||
def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None:
|
def role(self, uid: str, agent_id: int | None = None) -> ScoredRole | None:
|
||||||
@@ -1134,7 +1185,7 @@ class Store:
|
|||||||
place: str | None = None, open_only: bool = True,
|
place: str | None = None, open_only: bool = True,
|
||||||
q: str | None = None, function: str | None = None,
|
q: str | None = None, function: str | None = None,
|
||||||
remote: str | None = None, days: int | None = None,
|
remote: str | None = None, days: int | None = None,
|
||||||
hide_muted: bool = False, include_read: bool = True,
|
include_read: bool = True,
|
||||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||||
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
|
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
|
||||||
@@ -1161,18 +1212,18 @@ class Store:
|
|||||||
|
|
||||||
`agent` says whose votes, hidden companies and scores are meant. Each
|
`agent` says whose votes, hidden companies and scores are meant. Each
|
||||||
row carries that agent's `vote` and `voted_at`; `judged` (a key of
|
row carries that agent's `vote` and `voted_at`; `judged` (a key of
|
||||||
`JUDGED`: liked, unjudged or passed) filters on them. `hide_muted`
|
`JUDGED`: liked, unjudged or passed) filters on them. The companies
|
||||||
excludes the companies that agent hid, before `limit` is applied, in
|
that agent hid are always left out, in every order and count: a
|
||||||
SQL rather than after the fact: a company that has published a lot,
|
hidden company's roles are never shown. That is done before `limit`,
|
||||||
all at once — a fresh board added, a re-pull that finally reached
|
in SQL rather than after the fact: a company that has published a
|
||||||
it — can otherwise fill the whole page with roles that then all turn
|
lot, all at once — a fresh board added, a re-pull that finally
|
||||||
out to be from the one company someone hid, leaving nothing to show
|
reached it — could otherwise fill the whole page with roles that then
|
||||||
for a default query that would otherwise have plenty. With no agent
|
all turn out to be from the one company someone hid. With no agent
|
||||||
every vote is None, nothing is hidden and `judged` reads as such.
|
every vote is None, nothing is hidden and `judged` reads as such.
|
||||||
|
|
||||||
`include_read` set False excludes a role `set_read` or `set_vote` has
|
`include_read` set False excludes a role `set_read` or `set_vote` has
|
||||||
stamped, the same way and for the same reason as `hide_muted`:
|
stamped, the same way and for the same reason as the hidden
|
||||||
before `limit`, not after. `tags` keeps only roles carrying at least
|
companies: before `limit`, not after. `tags` keeps only roles carrying at least
|
||||||
one of them (`tag_mode="any"`, the default) or every one
|
one of them (`tag_mode="any"`, the default) or every one
|
||||||
(`tag_mode="all"`), also before `limit`.
|
(`tag_mode="all"`), also before `limit`.
|
||||||
|
|
||||||
@@ -1201,7 +1252,7 @@ class Store:
|
|||||||
the whole page rather than one per role."""
|
the whole page rather than one per role."""
|
||||||
sql, params = self._roles_from(
|
sql, params = self._roles_from(
|
||||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
remote=remote, days=days, include_read=include_read,
|
||||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||||
fetch_gate=fetch_gate,
|
fetch_gate=fetch_gate,
|
||||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||||
@@ -1253,7 +1304,7 @@ class Store:
|
|||||||
place: str | None = None, open_only: bool = True,
|
place: str | None = None, open_only: bool = True,
|
||||||
q: str | None = None, function: str | None = None,
|
q: str | None = None, function: str | None = None,
|
||||||
remote: str | None = None, days: int | None = None,
|
remote: str | None = None, days: int | None = None,
|
||||||
hide_muted: bool = False, include_read: bool = True,
|
include_read: bool = True,
|
||||||
tags: Sequence[str] = (), tag_mode: str = "any",
|
tags: Sequence[str] = (), tag_mode: str = "any",
|
||||||
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
agent: int | None = None, by_score: bool = True, gated: bool = False,
|
||||||
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
|
fetch_gate: bool = False, judged: str | None = None, pay: str | None = None,
|
||||||
@@ -1277,7 +1328,7 @@ class Store:
|
|||||||
column itself never crosses out of SQLite for a list."""
|
column itself never crosses out of SQLite for a list."""
|
||||||
sql, params = self._roles_from(
|
sql, params = self._roles_from(
|
||||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
remote=remote, days=days, include_read=include_read,
|
||||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||||
fetch_gate=fetch_gate,
|
fetch_gate=fetch_gate,
|
||||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||||
@@ -1333,8 +1384,7 @@ class Store:
|
|||||||
def count_roles(self, company: str | None = None, place: str | None = None,
|
def count_roles(self, company: str | None = None, place: str | None = None,
|
||||||
open_only: bool = True, q: str | None = None,
|
open_only: bool = True, q: str | None = None,
|
||||||
function: str | None = None, remote: str | None = None,
|
function: str | None = None, remote: str | None = None,
|
||||||
days: int | None = None, hide_muted: bool = False,
|
days: int | None = None, include_read: bool = True, tags: Sequence[str] = (),
|
||||||
include_read: bool = True, tags: Sequence[str] = (),
|
|
||||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||||
gated: bool = False, fetch_gate: bool = False,
|
gated: bool = False, fetch_gate: bool = False,
|
||||||
judged: str | None = None,
|
judged: str | None = None,
|
||||||
@@ -1345,7 +1395,7 @@ class Store:
|
|||||||
"""How many roles `roles` would list with the same filters and no limit."""
|
"""How many roles `roles` would list with the same filters and no limit."""
|
||||||
sql, params = self._roles_from(
|
sql, params = self._roles_from(
|
||||||
company=company, place=place, open_only=open_only, q=q, function=function,
|
company=company, place=place, open_only=open_only, q=q, function=function,
|
||||||
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
|
remote=remote, days=days, include_read=include_read,
|
||||||
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
|
||||||
fetch_gate=fetch_gate,
|
fetch_gate=fetch_gate,
|
||||||
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
judged=judged, pay=pay, company_keys=company_keys, places=places,
|
||||||
@@ -1357,8 +1407,7 @@ class Store:
|
|||||||
def role_facets(self, company: str | None = None, place: str | None = None,
|
def role_facets(self, company: str | None = None, place: str | None = None,
|
||||||
open_only: bool = True, q: str | None = None,
|
open_only: bool = True, q: str | None = None,
|
||||||
function: str | None = None, remote: str | None = None,
|
function: str | None = None, remote: str | None = None,
|
||||||
days: int | None = None, hide_muted: bool = False,
|
days: int | None = None, include_read: bool = True, tags: Sequence[str] = (),
|
||||||
include_read: bool = True, tags: Sequence[str] = (),
|
|
||||||
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
tag_mode: str = "any", agent: int | None = None, by_score: bool = True,
|
||||||
gated: bool = False, fetch_gate: bool = False,
|
gated: bool = False, fetch_gate: bool = False,
|
||||||
judged: str | None = None,
|
judged: str | None = None,
|
||||||
@@ -1375,7 +1424,7 @@ class Store:
|
|||||||
and then `""`, no pay stated. A value no role has counts 0."""
|
and then `""`, no pay stated. A value no role has counts 0."""
|
||||||
shared: dict[str, Any] = {
|
shared: dict[str, Any] = {
|
||||||
"company": company, "place": place, "open_only": open_only, "q": q,
|
"company": company, "place": place, "open_only": open_only, "q": q,
|
||||||
"function": function, "remote": remote, "days": days, "hide_muted": hide_muted,
|
"function": function, "remote": remote, "days": days,
|
||||||
"include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent,
|
"include_read": include_read, "tags": tags, "tag_mode": tag_mode, "agent": agent,
|
||||||
"by_score": by_score, "gated": gated, "fetch_gate": fetch_gate, "judged": judged,
|
"by_score": by_score, "gated": gated, "fetch_gate": fetch_gate, "judged": judged,
|
||||||
"company_keys": company_keys, "places": places, "closing": closing}
|
"company_keys": company_keys, "places": places, "closing": closing}
|
||||||
@@ -1394,7 +1443,7 @@ class Store:
|
|||||||
|
|
||||||
def _roles_from(self, *, company: str | None, place: str | None, open_only: bool,
|
def _roles_from(self, *, company: str | None, place: str | None, open_only: bool,
|
||||||
q: str | None, function: str | None, remote: str | None,
|
q: str | None, function: str | None, remote: str | None,
|
||||||
days: int | None, hide_muted: bool, include_read: bool,
|
days: int | None, include_read: bool,
|
||||||
tags: Sequence[str], tag_mode: str, agent: int | None,
|
tags: Sequence[str], tag_mode: str, agent: int | None,
|
||||||
by_score: bool, gated: bool, judged: str | None,
|
by_score: bool, gated: bool, judged: str | None,
|
||||||
fetch_gate: bool = False,
|
fetch_gate: bool = False,
|
||||||
@@ -1408,7 +1457,7 @@ class Store:
|
|||||||
(see `_WITH_VOTES`) and its judgements as `j` (`_WITH_JUDGEMENTS`);
|
(see `_WITH_VOTES`) and its judgements as `j` (`_WITH_JUDGEMENTS`);
|
||||||
its scores as `s` when ordering by them or with `fetch_gate`, and
|
its scores as `s` when ordering by them or with `fetch_gate`, and
|
||||||
then a role the agent's fetch gate rejects is left out unless
|
then a role the agent's fetch gate rejects is left out unless
|
||||||
`gated`. A copy of another role
|
`gated`. A company the agent hid is always left out. A copy of another role
|
||||||
(`rebuild_dupes`) is never listed: its kept role stands for it and
|
(`rebuild_dupes`) is never listed: its kept role stands for it and
|
||||||
counts its copies. `closing` keeps the roles whose deadline is
|
counts its copies. `closing` keeps the roles whose deadline is
|
||||||
today or within `CLOSING_DAYS` after it."""
|
today or within `CLOSING_DAYS` after it."""
|
||||||
@@ -1451,7 +1500,7 @@ class Store:
|
|||||||
params += list(seniority)
|
params += list(seniority)
|
||||||
if open_only:
|
if open_only:
|
||||||
where.append("closed_at IS NULL")
|
where.append("closed_at IS NULL")
|
||||||
if hide_muted and agent is not None:
|
if agent is not None:
|
||||||
where.append("company_key NOT IN (SELECT company_key FROM agent_companies "
|
where.append("company_key NOT IN (SELECT company_key FROM agent_companies "
|
||||||
"WHERE agent_id = ? AND affinity = -1)")
|
"WHERE agent_id = ? AND affinity = -1)")
|
||||||
params.append(agent)
|
params.append(agent)
|
||||||
@@ -1499,14 +1548,18 @@ class Store:
|
|||||||
if found is not None else []
|
if found is not None else []
|
||||||
|
|
||||||
def tag_counts(self, agent_id: int | None = None,
|
def tag_counts(self, agent_id: int | None = None,
|
||||||
show_muted: bool = False) -> list[tuple[str, int, int]]:
|
fetch_gate: bool = False) -> list[tuple[str, int, int]]:
|
||||||
"""`(tag, open_count, unread_count)` over open roles, most-used tag
|
"""`(tag, open_count, unread_count)` over open roles, most-used tag
|
||||||
first and then alphabetically. What `/api/filters` serves; the
|
first and then alphabetically. What `/api/filters` serves; the
|
||||||
companies `agent_id` hid are left out unless `show_muted`, the same
|
companies `agent_id` hid are left out, as in `roles`, and with no
|
||||||
default as `roles`, and with no agent nothing is hidden."""
|
agent nothing is hidden. With `fetch_gate`, only the roles the
|
||||||
|
agent's feed shows count (`_SHOWN`)."""
|
||||||
where = "r.closed_at IS NULL AND r.dup_of IS NULL"
|
where = "r.closed_at IS NULL AND r.dup_of IS NULL"
|
||||||
params: list[Any] = []
|
params: list[Any] = []
|
||||||
if not show_muted and agent_id is not None:
|
if agent_id is not None and fetch_gate:
|
||||||
|
where += f" AND {_SHOWN}"
|
||||||
|
params += [agent_id, agent_id]
|
||||||
|
elif agent_id is not None:
|
||||||
where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
|
where += (" AND r.company_key NOT IN (SELECT company_key FROM agent_companies "
|
||||||
"WHERE agent_id = ? AND affinity = -1)")
|
"WHERE agent_id = ? AND affinity = -1)")
|
||||||
params.append(agent_id)
|
params.append(agent_id)
|
||||||
@@ -1578,8 +1631,9 @@ class Store:
|
|||||||
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
f"""SELECT uid, prep_due, source, {_HOST} AS host, title, company, company_key, place,
|
||||||
seniority_host, employment_host, remote_host, country_host,
|
seniority_host, employment_host, remote_host, country_host,
|
||||||
seniority_model, seniority_stated, seniority_title, seniority,
|
seniority_model, seniority_stated, seniority_title, seniority,
|
||||||
employment, remote, country, remote_model, remote_where_model,
|
employment, remote, remote_text, country, remote_model,
|
||||||
deadline_host, deadline_model, deadline,
|
remote_where_model, deadline_host, deadline_model, deadline,
|
||||||
|
SUBSTR(COALESCE(NULLIF(description, ''), teaser), 1, 6000) AS text,
|
||||||
json_extract(raw, '$.places') AS raw_places,
|
json_extract(raw, '$.places') AS raw_places,
|
||||||
(SELECT json_group_array(place_id) FROM (
|
(SELECT json_group_array(place_id) FROM (
|
||||||
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
SELECT place_id FROM role_remote rr WHERE rr.uid = roles.uid
|
||||||
@@ -1605,8 +1659,8 @@ class Store:
|
|||||||
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
def write_prep(self, mapped: Sequence[dict[str, Any]]) -> int:
|
||||||
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
"""Write what prep mapped, one dict per role with `uid`, `due` (the
|
||||||
`prep_due` it read), `company_key`, `seniority_stated`,
|
`prep_due` it read), `company_key`, `seniority_stated`,
|
||||||
`seniority_title`, `employment`, `remote`, `remote_area`, `country`
|
`seniority_title`, `employment`, `remote`, `remote_text`, `remote_area`,
|
||||||
and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
`country` and `deadline` (`YYYY-MM-DD`, None for none, written as it is). `seniority` is the model's level, else the stated, else
|
||||||
the title's. `remote` is written as prep composed it (the model's,
|
the title's. `remote` is written as prep composed it (the model's,
|
||||||
else the host's word, else the place line) and `remote_area`, place
|
else the host's word, else the place line) and `remote_area`, place
|
||||||
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
ids, replaces the role's `role_remote`. An empty `country` leaves the
|
||||||
@@ -1622,6 +1676,7 @@ class Store:
|
|||||||
NULLIF(:seniority_stated, ''), :seniority_title),
|
NULLIF(:seniority_stated, ''), :seniority_title),
|
||||||
employment = :employment,
|
employment = :employment,
|
||||||
remote = :remote,
|
remote = :remote,
|
||||||
|
remote_text = :remote_text,
|
||||||
country = COALESCE(NULLIF(:country, ''), country),
|
country = COALESCE(NULLIF(:country, ''), country),
|
||||||
deadline = :deadline,
|
deadline = :deadline,
|
||||||
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
prep_due = CASE WHEN prep_due = :due THEN 0 ELSE prep_due END
|
||||||
@@ -1664,29 +1719,43 @@ class Store:
|
|||||||
# -------------------------------------------------------------- extract
|
# -------------------------------------------------------------- extract
|
||||||
|
|
||||||
def _summarise_top(self) -> tuple[str, list[Any]]:
|
def _summarise_top(self) -> tuple[str, list[Any]]:
|
||||||
"""The `WHERE` fragment for a role among the best `summarise_top`
|
"""The `WHERE` fragment for a role in the top of an enabled agent's
|
||||||
final scores of an enabled agent with a prompt, open, no copy and
|
feed in either order, open, no copy, passing that agent's fetch gate
|
||||||
passing that agent's fetch gate: the top of its feed. The rank is
|
and not of a company it hid: the best `summarise_top` (its `limits`)
|
||||||
numbered per agent and the agent's limits ride out of the window
|
by the want feed score, the score order of `roles` with the vote
|
||||||
subquery beside it, so each row is held to the limit of the agent
|
applied (`_VOTED_SCORE`), and the best `summarise_top` by the match
|
||||||
that ranked it; a key the JSON lacks is the `Limits` default. The
|
order, `cv_score` under the agent's weights, then `cv_fit`, then
|
||||||
role is `r`. An agent without a prompt with text counts for nothing
|
score (`_by_cv`). So at most twice `summarise_top` per agent. An
|
||||||
here: its scores have no fit, so they say how fresh a role is, not
|
agent without a prompt with text gives only its match top (its
|
||||||
whether it is wanted."""
|
scores have no fit, so they say how fresh a role is, not whether it
|
||||||
return ("""r.uid IN (SELECT uid FROM (
|
is wanted), one without a CV only its want top. The role is `r`; no
|
||||||
SELECT s.uid, a.limits, ROW_NUMBER() OVER (
|
agent that counts is a fragment that holds for no role."""
|
||||||
PARTITION BY s.agent_id
|
arms: list[str] = []
|
||||||
ORDER BY s.score DESC, top.posted_at DESC, s.uid) AS n
|
params: list[Any] = []
|
||||||
FROM scores s
|
for agent in self.agents(enabled_only=True):
|
||||||
JOIN agents a ON a.id = s.agent_id AND a.enabled = 1
|
top = agent.limits.summarise_top
|
||||||
AND EXISTS (SELECT 1 FROM agent_prompts p
|
orders = []
|
||||||
WHERE p.agent_id = a.id
|
if any(p.text.strip() for p in agent.prompts):
|
||||||
AND TRIM(p.text, ' ' || char(9, 10, 13)) <> '')
|
orders.append(f"{_VOTED_SCORE} DESC")
|
||||||
|
if agent.cv_text.strip():
|
||||||
|
orders.append(_by_cv(_cv_sql(agent.weights)) + f"{_VOTED_SCORE} DESC")
|
||||||
|
for order in orders:
|
||||||
|
arms.append(f"""SELECT uid FROM (
|
||||||
|
SELECT s.uid FROM scores s
|
||||||
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
JOIN roles top ON top.uid = s.uid AND top.closed_at IS NULL
|
||||||
AND top.dup_of IS NULL
|
AND top.dup_of IS NULL
|
||||||
WHERE s.gated_reason = '')
|
LEFT JOIN role_votes v ON v.uid = s.uid AND v.agent_id = s.agent_id
|
||||||
WHERE n <= COALESCE(json_extract(limits, '$.summarise_top'), ?))""",
|
LEFT JOIN role_judgements j ON j.uid = s.uid AND j.agent_id = s.agent_id
|
||||||
[Limits().summarise_top])
|
AND j.judged_at IS NOT NULL
|
||||||
|
WHERE s.agent_id = ? AND s.gated_reason = ''
|
||||||
|
AND top.company_key NOT IN (
|
||||||
|
SELECT company_key FROM agent_companies
|
||||||
|
WHERE agent_id = s.agent_id AND affinity = -1)
|
||||||
|
ORDER BY {order}, top.posted_at DESC, s.uid LIMIT ?)""")
|
||||||
|
params += [agent.id, top]
|
||||||
|
if not arms:
|
||||||
|
return "0", []
|
||||||
|
return f"r.uid IN ({' UNION '.join(arms)})", params
|
||||||
|
|
||||||
def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
def _extract_scope(self, uids: Sequence[str] | None) -> tuple[str, list[Any]]:
|
||||||
"""The `WHERE` for a role `extract` may read: open, never answered,
|
"""The `WHERE` for a role `extract` may read: open, never answered,
|
||||||
@@ -1845,9 +1914,9 @@ class Store:
|
|||||||
some text to summarise. Given `uids`, those of them and nothing else,
|
some text to summarise. Given `uids`, those of them and nothing else,
|
||||||
open or not and given up or not, because somebody asked for them.
|
open or not and given up or not, because somebody asked for them.
|
||||||
Otherwise open, not given up, and in the top of an enabled agent's
|
Otherwise open, not given up, and in the top of an enabled agent's
|
||||||
feed (`_summarise_top`) or one the owner cares about whatever its
|
feed in either order (`_summarise_top`) or one the owner cares about
|
||||||
rank: saved (it has an application) or voted up by an enabled
|
whatever its rank: saved (it has an application) or voted up by an
|
||||||
agent."""
|
enabled agent."""
|
||||||
where = ["r.summary = ''",
|
where = ["r.summary = ''",
|
||||||
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
"(TRIM(r.article_text) <> '' OR TRIM(r.description) <> '')"]
|
||||||
if uids is not None:
|
if uids is not None:
|
||||||
@@ -1875,9 +1944,10 @@ class Store:
|
|||||||
return int(row["n"]) if row else 0
|
return int(row["n"]) if row else 0
|
||||||
|
|
||||||
def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
def summarise_waiting(self, limit: int, uids: Sequence[str] | None = None) -> list[Role]:
|
||||||
"""Roles `summarise` has yet to write, newest first: the best
|
"""Roles `summarise` has yet to write, newest first: the top
|
||||||
`summarise_top` final scores of every enabled agent, so the backlog
|
`summarise_top` of every enabled agent's want order and of its match
|
||||||
is bounded by the agents rather than by the number of open roles,
|
order (`_summarise_top`), so the backlog is bounded by the agents
|
||||||
|
rather than by the number of open roles,
|
||||||
and every open role that is saved or voted up by an enabled agent,
|
and every open role that is saved or voted up by an enabled agent,
|
||||||
whatever its rank; given `uids`, those still without a summary,
|
whatever its rank; given `uids`, those still without a summary,
|
||||||
whatever their rank. A batch asks again when it runs."""
|
whatever their rank. A batch asks again when it runs."""
|
||||||
@@ -1955,33 +2025,28 @@ class Store:
|
|||||||
|
|
||||||
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
|
def set_judgement(self, agent_id: int, uid: str, input_key: str, skills: int,
|
||||||
experience: int, career: int, strengths: Sequence[str],
|
experience: int, career: int, strengths: Sequence[str],
|
||||||
gaps: Sequence[str], musts: Sequence[Mapping[str, str]] = ()) -> bool:
|
gaps: Sequence[str]) -> bool:
|
||||||
"""Store `judge`'s reading of one role for one agent, replacing the
|
"""Store `judge`'s reading of one role for one agent, replacing the
|
||||||
one before: the three numbers as the model gave them, the verdict on
|
one before: the three numbers as the model gave them, `cv_score`
|
||||||
each of the agent's must-haves (`musts`, `{id, text, verdict}`) and how
|
(`app.models.cv_score`, at the default weights: the table's column is
|
||||||
many are broken, `cv_score` (`app.models.cv_score`, at the default
|
not read, see `judgement`) and its tries back at 0. False for an
|
||||||
weights and with the cap: the table's column is not read, see
|
unknown role or agent."""
|
||||||
`judgement`) and its tries back at 0. False for an unknown role or
|
|
||||||
agent."""
|
|
||||||
broken = sum(1 for m in musts if m.get("verdict") == "broken")
|
|
||||||
try:
|
try:
|
||||||
with self.transaction() as conn:
|
with self.transaction() as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""INSERT INTO role_judgements (uid, agent_id, skills, experience, career,
|
"""INSERT INTO role_judgements (uid, agent_id, skills, experience, career,
|
||||||
cv_score, strengths, gaps, musts, broken,
|
cv_score, strengths, gaps,
|
||||||
input_key, tries, judged_at)
|
input_key, tries, judged_at)
|
||||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,0,?)
|
VALUES (?,?,?,?,?,?,?,?,?,0,?)
|
||||||
ON CONFLICT(uid, agent_id) DO UPDATE SET
|
ON CONFLICT(uid, agent_id) DO UPDATE SET
|
||||||
skills=excluded.skills, experience=excluded.experience,
|
skills=excluded.skills, experience=excluded.experience,
|
||||||
career=excluded.career, cv_score=excluded.cv_score,
|
career=excluded.career, cv_score=excluded.cv_score,
|
||||||
strengths=excluded.strengths, gaps=excluded.gaps,
|
strengths=excluded.strengths, gaps=excluded.gaps,
|
||||||
musts=excluded.musts, broken=excluded.broken,
|
|
||||||
input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""",
|
input_key=excluded.input_key, tries=0, judged_at=excluded.judged_at""",
|
||||||
(uid, agent_id, skills, experience, career,
|
(uid, agent_id, skills, experience, career,
|
||||||
cv_score(skills, experience, career, broken=broken),
|
cv_score(skills, experience, career),
|
||||||
json.dumps(list(strengths), ensure_ascii=False),
|
json.dumps(list(strengths), ensure_ascii=False),
|
||||||
json.dumps(list(gaps), ensure_ascii=False),
|
json.dumps(list(gaps), ensure_ascii=False),
|
||||||
json.dumps([dict(m) for m in musts], ensure_ascii=False), broken,
|
|
||||||
input_key, self.now()))
|
input_key, self.now()))
|
||||||
except sqlite3.IntegrityError:
|
except sqlite3.IntegrityError:
|
||||||
return False
|
return False
|
||||||
@@ -2009,8 +2074,7 @@ class Store:
|
|||||||
def judgement(self, agent_id: int | None, uid: str) -> Judgement | None:
|
def judgement(self, agent_id: int | None, uid: str) -> Judgement | None:
|
||||||
"""The agent's judgement of one role, None when it has none, its
|
"""The agent's judgement of one role, None when it has none, its
|
||||||
`cv_score` the three numbers under the agent's weights as they are
|
`cv_score` the three numbers under the agent's weights as they are
|
||||||
now, not as stored, and `career_capped` the model's career as the
|
now, not as stored."""
|
||||||
broken must-haves leave it (`BROKEN_CAP`)."""
|
|
||||||
if agent_id is None:
|
if agent_id is None:
|
||||||
return None
|
return None
|
||||||
row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? "
|
row = self._row("SELECT * FROM role_judgements WHERE agent_id = ? AND uid = ? "
|
||||||
@@ -2019,9 +2083,8 @@ class Store:
|
|||||||
return None
|
return None
|
||||||
found = Judgement.from_row(row)
|
found = Judgement.from_row(row)
|
||||||
agent = self.agent(agent_id)
|
agent = self.agent(agent_id)
|
||||||
found.career_capped = capped_career(found.career, found.broken)
|
|
||||||
found.cv_score = cv_score(found.skills, found.experience, found.career,
|
found.cv_score = cv_score(found.skills, found.experience, found.career,
|
||||||
agent.weights if agent else None, found.broken)
|
agent.weights if agent else None)
|
||||||
return found
|
return found
|
||||||
|
|
||||||
def set_article_text(self, uid: str, text: str) -> Role | None:
|
def set_article_text(self, uid: str, text: str) -> Role | None:
|
||||||
@@ -2143,14 +2206,13 @@ class Store:
|
|||||||
cv_text: str | None = None,
|
cv_text: str | None = None,
|
||||||
cv_languages: Sequence[CvLanguage] | None = None,
|
cv_languages: Sequence[CvLanguage] | None = None,
|
||||||
avoid_text: str | None = None,
|
avoid_text: str | None = None,
|
||||||
must_text: str | None = None,
|
|
||||||
like_text: str | None = None) -> Agent:
|
like_text: str | None = None) -> Agent:
|
||||||
"""Create an agent, or with `agent_id` replace that one's fields and
|
"""Create an agent, or with `agent_id` replace that one's fields and
|
||||||
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
|
prompts. Without `prompts`, a non-empty `intent` is its one prompt,
|
||||||
named after the agent. Without `fetch_terms` an update keeps the
|
named after the agent. Without `fetch_terms` an update keeps the
|
||||||
stored ones and a new agent has none, so a caller that does not know
|
stored ones and a new agent has none, so a caller that does not know
|
||||||
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
|
them (the CLI's prompt edit) loses none; `cv_text`, `cv_languages`,
|
||||||
`like_text`, `avoid_text` and `must_text` are kept the same way. A prompt whose text is
|
`like_text` and `avoid_text` are kept the same way. A prompt whose text is
|
||||||
unchanged keeps its cached vector, whatever its name or place, so an
|
unchanged keeps its cached vector, whatever its name or place, so an
|
||||||
edit re-embeds only the prompts it changed. A save gives every role
|
edit re-embeds only the prompts it changed. A save gives every role
|
||||||
`judge` gave up on for the agent its tries back, since what it is
|
`judge` gave up on for the agent its tries back, since what it is
|
||||||
@@ -2171,7 +2233,6 @@ class Store:
|
|||||||
(limits or Limits()).model_dump_json(), int(enabled), terms,
|
(limits or Limits()).model_dump_json(), int(enabled), terms,
|
||||||
None if cv_text is None else cv_text.strip(), spoken,
|
None if cv_text is None else cv_text.strip(), spoken,
|
||||||
None if avoid_text is None else avoid_text.strip(),
|
None if avoid_text is None else avoid_text.strip(),
|
||||||
None if must_text is None else must_text.strip(),
|
|
||||||
None if like_text is None else like_text.strip())
|
None if like_text is None else like_text.strip())
|
||||||
try:
|
try:
|
||||||
with self.transaction() as conn:
|
with self.transaction() as conn:
|
||||||
@@ -2179,10 +2240,9 @@ class Store:
|
|||||||
agent_id = conn.execute(
|
agent_id = conn.execute(
|
||||||
"""INSERT INTO agents (name, gates, weights, limits, enabled,
|
"""INSERT INTO agents (name, gates, weights, limits, enabled,
|
||||||
fetch_terms, cv_text, cv_languages, avoid_text,
|
fetch_terms, cv_text, cv_languages, avoid_text,
|
||||||
must_text, like_text, created_at)
|
like_text, created_at)
|
||||||
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
|
VALUES (?,?,?,?,?,COALESCE(?, '[]'),COALESCE(?, ''),
|
||||||
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),
|
COALESCE(?, '[]'),COALESCE(?, ''),COALESCE(?, ''),?)""",
|
||||||
COALESCE(?, ''),?)""",
|
|
||||||
(*fields, self.now())).lastrowid
|
(*fields, self.now())).lastrowid
|
||||||
elif not conn.execute(
|
elif not conn.execute(
|
||||||
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
|
"""UPDATE agents SET name=?, gates=?, weights=?, limits=?, enabled=?,
|
||||||
@@ -2190,7 +2250,6 @@ class Store:
|
|||||||
cv_text=COALESCE(?, cv_text),
|
cv_text=COALESCE(?, cv_text),
|
||||||
cv_languages=COALESCE(?, cv_languages),
|
cv_languages=COALESCE(?, cv_languages),
|
||||||
avoid_text=COALESCE(?, avoid_text),
|
avoid_text=COALESCE(?, avoid_text),
|
||||||
must_text=COALESCE(?, must_text),
|
|
||||||
like_text=COALESCE(?, like_text)
|
like_text=COALESCE(?, like_text)
|
||||||
WHERE id=?""",
|
WHERE id=?""",
|
||||||
(*fields, agent_id)).rowcount:
|
(*fields, agent_id)).rowcount:
|
||||||
@@ -2288,7 +2347,7 @@ class Store:
|
|||||||
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
|
def score_candidates(self, agent_id: int, gates: Sequence[Gate]) -> list[Candidate]:
|
||||||
"""Every open role of a company this agent has not hidden, copies
|
"""Every open role of a company this agent has not hidden, copies
|
||||||
(`rebuild_dupes`) aside, with the agent's affinity for the company,
|
(`rebuild_dupes`) aside, with the agent's affinity for the company,
|
||||||
the company's All countries and All roles, and how the role meets
|
the company's All locations and All roles, and how the role meets
|
||||||
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
|
each gate: 1 match, 0 no match, -1 the column is empty. The gates are
|
||||||
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
|
evaluated here, in SQL, so a `require` gate is the `WHERE` it reads
|
||||||
as, written as a column so a gated role keeps its row and reason."""
|
as, written as a column so a gated role keeps its row and reason."""
|
||||||
@@ -2304,7 +2363,11 @@ class Store:
|
|||||||
empty, match, values = _gate_sql(part, gaz)
|
empty, match, values = _gate_sql(part, gaz)
|
||||||
whens.append(f"WHEN {match} THEN {k}")
|
whens.append(f"WHEN {match} THEN {k}")
|
||||||
params += values
|
params += values
|
||||||
columns.append(f"CASE WHEN {empty} THEN -1 {' '.join(whens)} ELSE 0 END AS g{n}")
|
# A place gate's match goes first: an unresolved place can still
|
||||||
|
# meet one of its values (`_gate_sql`).
|
||||||
|
case = (f"{' '.join(whens)} WHEN {empty} THEN -1" if gate.field == "place"
|
||||||
|
else f"WHEN {empty} THEN -1 {' '.join(whens)}")
|
||||||
|
columns.append(f"CASE {case} ELSE 0 END AS g{n}")
|
||||||
select = ", ".join(["r.uid", "r.posted_at",
|
select = ", ".join(["r.uid", "r.posted_at",
|
||||||
"COALESCE(ac.affinity, 0) AS affinity",
|
"COALESCE(ac.affinity, 0) AS affinity",
|
||||||
"COALESCE(c.all_countries, 0) AS all_countries",
|
"COALESCE(c.all_countries, 0) AS all_countries",
|
||||||
@@ -2467,7 +2530,7 @@ class Store:
|
|||||||
return self._company(key)
|
return self._company(key)
|
||||||
|
|
||||||
def set_all_countries(self, key: str, on: bool) -> Company:
|
def set_all_countries(self, key: str, on: bool) -> Company:
|
||||||
"""Switch a company's All countries override: its big boards are read
|
"""Switch a company's All locations override: its big boards are read
|
||||||
with the agents' wide reach, Europe today, rather than their places,
|
with the agents' wide reach, Europe today, rather than their places,
|
||||||
from the next pull (`app/jobs/reach.py`). Global, not per agent.
|
from the next pull (`app/jobs/reach.py`). Global, not per agent.
|
||||||
Raises KeyError for an unknown company."""
|
Raises KeyError for an unknown company."""
|
||||||
@@ -2536,7 +2599,7 @@ class Store:
|
|||||||
def merge_companies(self, keep: str, other: str) -> Company:
|
def merge_companies(self, keep: str, other: str) -> Company:
|
||||||
"""Fold company `other` into `keep`, in one transaction: its roles and
|
"""Fold company `other` into `keep`, in one transaction: its roles and
|
||||||
subscriptions move to `keep`, each agent's stance moves unless the
|
subscriptions move to `keep`, each agent's stance moves unless the
|
||||||
agent has one on `keep` already, the All countries and All roles
|
agent has one on `keep` already, the All locations and All roles
|
||||||
overrides carry over, its aliases and its name become aliases of
|
overrides carry over, its aliases and its name become aliases of
|
||||||
`keep`, and its row goes. Copies are not recomputed (`rebuild_dupes`).
|
`keep`, and its row goes. Copies are not recomputed (`rebuild_dupes`).
|
||||||
Raises KeyError for an unknown company and ValueError for merging one
|
Raises KeyError for an unknown company and ValueError for merging one
|
||||||
@@ -2583,25 +2646,30 @@ class Store:
|
|||||||
"SELECT company_key, COUNT(*) AS n FROM roles "
|
"SELECT company_key, COUNT(*) AS n FROM roles "
|
||||||
"WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")}
|
"WHERE closed_at IS NULL AND dup_of IS NULL GROUP BY company_key")}
|
||||||
|
|
||||||
def open_companies(self, agent_id: int | None = None
|
def open_companies(self, agent_id: int | None = None, fetch_gate: bool = False
|
||||||
) -> list[tuple[str, list[str], int, int]]:
|
) -> list[tuple[str, list[str], int, int]]:
|
||||||
"""`(company_key, names, open roles, affinity)` for every company with
|
"""`(company_key, names, open roles, affinity)` for every company with
|
||||||
an open role, what the Company field suggests from. `names` are the
|
an open role, what the Company field suggests from. `names` are the
|
||||||
company row's name and then every spelling its roles carry, commonest
|
company row's name and then every spelling its roles carry, commonest
|
||||||
first, so a search finds a company by any of them. A company
|
first, so a search finds a company by any of them. A company
|
||||||
`agent_id` hid is left out; `affinity` is that agent's, 0 without
|
`agent_id` hid is left out; `affinity` is that agent's, 0 without
|
||||||
one. Copies are not counted, as in the feed."""
|
one. Copies are not counted, as in the feed. With `fetch_gate`, only
|
||||||
|
the roles the agent's feed shows count (`_SHOWN`), and a company with
|
||||||
|
none of them is left out."""
|
||||||
|
gate, params = "", [agent_id]
|
||||||
|
if agent_id is not None and fetch_gate:
|
||||||
|
gate, params = f"AND {_SHOWN}", [agent_id, agent_id, agent_id]
|
||||||
rows = self._rows(
|
rows = self._rows(
|
||||||
"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n,
|
f"""SELECT r.company_key AS key, r.company AS name, COUNT(*) AS n,
|
||||||
c.name AS own, COALESCE(ac.affinity, 0) AS affinity
|
c.name AS own, COALESCE(ac.affinity, 0) AS affinity
|
||||||
FROM roles r
|
FROM roles r
|
||||||
LEFT JOIN companies c ON c.key = r.company_key
|
LEFT JOIN companies c ON c.key = r.company_key
|
||||||
LEFT JOIN agent_companies ac ON ac.company_key = r.company_key
|
LEFT JOIN agent_companies ac ON ac.company_key = r.company_key
|
||||||
AND ac.agent_id = ?
|
AND ac.agent_id = ?
|
||||||
WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != ''
|
WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.company_key != ''
|
||||||
AND COALESCE(ac.affinity, 0) != -1
|
AND COALESCE(ac.affinity, 0) != -1 {gate}
|
||||||
GROUP BY r.company_key, r.company
|
GROUP BY r.company_key, r.company
|
||||||
ORDER BY r.company_key, n DESC, r.company""", (agent_id,))
|
ORDER BY r.company_key, n DESC, r.company""", params)
|
||||||
found: dict[str, tuple[list[str], list[int], int]] = {}
|
found: dict[str, tuple[list[str], list[int], int]] = {}
|
||||||
for r in rows:
|
for r in rows:
|
||||||
names, counts, _ = found.setdefault(
|
names, counts, _ = found.setdefault(
|
||||||
@@ -2612,24 +2680,31 @@ class Store:
|
|||||||
return [(key, names, sum(counts), affinity)
|
return [(key, names, sum(counts), affinity)
|
||||||
for key, (names, counts, affinity) in found.items()]
|
for key, (names, counts, affinity) in found.items()]
|
||||||
|
|
||||||
def open_places(self) -> list[tuple[str, str, int]]:
|
def open_places(self, agent_id: int | None = None) -> list[tuple[str, str, int]]:
|
||||||
"""`(place, country, open roles)` for every place line an open role
|
"""`(place, country, open roles)` for every place line an open role
|
||||||
carries, what `places.suggest` reads. Copies are not counted, as in
|
carries, what `places.suggest` reads. Copies are not counted, as in
|
||||||
the feed."""
|
the feed. Given `agent_id`, only the roles that agent's feed shows
|
||||||
|
(`_SHOWN`)."""
|
||||||
|
gate, params = (f"AND {_SHOWN}", [agent_id, agent_id]) if agent_id is not None \
|
||||||
|
else ("", [])
|
||||||
return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows(
|
return [(str(r["place"]), str(r["country"]), int(r["n"])) for r in self._rows(
|
||||||
"SELECT place, country, COUNT(*) AS n FROM roles "
|
"SELECT r.place, r.country, COUNT(*) AS n FROM roles r "
|
||||||
"WHERE closed_at IS NULL AND dup_of IS NULL AND place != '' "
|
f"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.place != '' {gate} "
|
||||||
"GROUP BY place, country")]
|
"GROUP BY r.place, r.country", params)]
|
||||||
|
|
||||||
def open_remote_areas(self) -> list[tuple[tuple[int, ...], int]]:
|
def open_remote_areas(self, agent_id: int | None = None
|
||||||
|
) -> list[tuple[tuple[int, ...], int]]:
|
||||||
"""`(area, open roles)` for every remote area an open remote role
|
"""`(area, open roles)` for every remote area an open remote role
|
||||||
holds, the area as place ids in order: what `places.suggest` counts
|
holds, the area as place ids in order: what `places.suggest` counts
|
||||||
the remote tokens from. Copies are not counted, as in the feed."""
|
the remote tokens from. Copies are not counted, as in the feed.
|
||||||
|
Given `agent_id`, only the roles that agent's feed shows (`_SHOWN`)."""
|
||||||
|
gate, params = (f"AND {_SHOWN}", [agent_id, agent_id]) if agent_id is not None \
|
||||||
|
else ("", [])
|
||||||
found: dict[str, list[int]] = {}
|
found: dict[str, list[int]] = {}
|
||||||
for r in self._rows(
|
for r in self._rows(
|
||||||
"SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid "
|
"SELECT rr.uid, rr.place_id FROM role_remote rr JOIN roles r ON r.uid = rr.uid "
|
||||||
"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' "
|
f"WHERE r.closed_at IS NULL AND r.dup_of IS NULL AND r.remote = 'remote' {gate} "
|
||||||
"ORDER BY rr.uid, rr.rowid"):
|
"ORDER BY rr.uid, rr.rowid", params):
|
||||||
found.setdefault(str(r[0]), []).append(int(r[1]))
|
found.setdefault(str(r[0]), []).append(int(r[1]))
|
||||||
return list(Counter(tuple(ids) for ids in found.values()).items())
|
return list(Counter(tuple(ids) for ids in found.values()).items())
|
||||||
|
|
||||||
|
|||||||
+12
-11
@@ -46,8 +46,9 @@
|
|||||||
<h3 id="viewing-as">Viewing as <a class="linky" href="/settings/agents">agents</a></h3>
|
<h3 id="viewing-as">Viewing as <a class="linky" href="/settings/agents">agents</a></h3>
|
||||||
<select id="agent" class="rail-select agent-pick" aria-labelledby="viewing-as" hidden></select>
|
<select id="agent" class="rail-select agent-pick" aria-labelledby="viewing-as" hidden></select>
|
||||||
<p id="agent-none" class="rail-note" hidden></p>
|
<p id="agent-none" class="rail-note" hidden></p>
|
||||||
{# Every order keeps to the agent's fetch gate (docs/architecture.md,
|
{# Every order keeps to the agent's fetch gate (documentation/ADR Fetch
|
||||||
"5 Feed"), so the filters below only narrow what it passed. #}
|
prep and score.md, "5 Feed"), so the filters below only narrow what
|
||||||
|
it passed. #}
|
||||||
<p id="agent-gate" class="rail-note" hidden></p>
|
<p id="agent-gate" class="rail-note" hidden></p>
|
||||||
</section>
|
</section>
|
||||||
{# The agent's saved filters (`/api/agents/{id}/filters`), newsdesk's
|
{# The agent's saved filters (`/api/agents/{id}/filters`), newsdesk's
|
||||||
@@ -74,11 +75,13 @@
|
|||||||
</section>
|
</section>
|
||||||
<section>
|
<section>
|
||||||
<h3>Order</h3>
|
<h3>Order</h3>
|
||||||
{# Agent match is the viewing agent's final score, Newest is first
|
{# What I'm looking for is the viewing agent's want feed score, Newest
|
||||||
seen, CV match the agent's `judge` total against its CV; all carry
|
the posting date, What fits me the agent's `judge` total against its
|
||||||
that agent's votes and keep to its fetch gate. #}
|
CV; all carry that agent's votes and keep to its fetch gate. The
|
||||||
|
period counts back over the posting date too, the date a row shows
|
||||||
|
as its age. #}
|
||||||
<div class="seg" id="sort" role="group" aria-label="Order"></div>
|
<div class="seg" id="sort" role="group" aria-label="Order"></div>
|
||||||
<label class="rail-field" for="period">First seen</label>
|
<label class="rail-field" for="period">Posted</label>
|
||||||
<select id="period" class="rail-select"></select>
|
<select id="period" class="rail-select"></select>
|
||||||
</section>
|
</section>
|
||||||
<section>
|
<section>
|
||||||
@@ -92,8 +95,9 @@
|
|||||||
<div class="rail-fields">
|
<div class="rail-fields">
|
||||||
{# Company and Place are chip inputs (static/chips.js), each pick a
|
{# Company and Place are chip inputs (static/chips.js), each pick a
|
||||||
company key or a place token from the suggest routes. The chips
|
company key or a place token from the suggest routes. The chips
|
||||||
of `?company=` and `?place=`, the substrings other pages link
|
of `?company=` and `?place=`, the substrings of older links and
|
||||||
with, sit under them to be taken off; nothing types a new one. #}
|
saved filters, sit under them to be taken off; nothing types a
|
||||||
|
new one. #}
|
||||||
<div id="f-company"></div>
|
<div id="f-company"></div>
|
||||||
<div id="f-company-text" class="rail-legacy" hidden></div>
|
<div id="f-company-text" class="rail-legacy" hidden></div>
|
||||||
<div id="f-place"></div>
|
<div id="f-place"></div>
|
||||||
@@ -149,9 +153,6 @@
|
|||||||
<label class="check">
|
<label class="check">
|
||||||
<input type="checkbox" id="f-closing"> Closing soon
|
<input type="checkbox" id="f-closing"> Closing soon
|
||||||
</label>
|
</label>
|
||||||
<label class="check">
|
|
||||||
<input type="checkbox" id="f-show-muted"> Show hidden companies
|
|
||||||
</label>
|
|
||||||
{# The roles the agent's fetch gate rejects, each with the reason,
|
{# The roles the agent's fetch gate rejects, each with the reason,
|
||||||
in any order; offered only with an agent. #}
|
in any order; offered only with an agent. #}
|
||||||
<label class="check" id="show-gated" hidden
|
<label class="check" id="show-gated" hidden
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
{% block content %}
|
{% block content %}
|
||||||
<h1>Account</h1>
|
<h1>Account</h1>
|
||||||
<p class="lede">The one password, the devices signed in with it, and the passkeys that sign in without it.</p>
|
<p class="lede">The one password, the devices signed in with it, the passkeys that sign in without it, and the API tokens a script sends instead.</p>
|
||||||
|
|
||||||
<section class="card">
|
<section class="card">
|
||||||
{# As in Sources' Runs: the element and its script belong to
|
{# As in Sources' Runs: the element and its script belong to
|
||||||
|
|||||||
@@ -3,12 +3,12 @@
|
|||||||
|
|
||||||
{# A card per agent: `static/agents.js` fills the list from `/api/agents`,
|
{# A card per agent: `static/agents.js` fills the list from `/api/agents`,
|
||||||
newest first. A card shows the agent's prompts, its fetch gate in words, its
|
newest first. A card shows the agent's prompts, its fetch gate in words, its
|
||||||
switch, its weights, its cut and its summarise size; Edit turns that card into the form below, in place, and New puts
|
switch, its weights, its analytics filter and its summarise size; Edit turns
|
||||||
a blank one on top. The form is a `<template>` because only one card edits
|
that card into the form below, in place, and New puts a blank one on top. The form is a `<template>` because only one card edits
|
||||||
at a time, so its ids stay unique. #}
|
at a time, so its ids stay unique. #}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
<h1>Search agents</h1>
|
<h1>Search agents</h1>
|
||||||
<p class="lede">A search agent is what the Roles tab ranks by: describe each kind of role you want as a prompt, set the fetch gate (where, what role), and it measures every open role against the prompt each fits best. Several agents rank the same roles differently, each with its own votes and liked companies; Viewing as on Roles picks one, and its votes are that agent's.</p>
|
<p class="lede">A search agent is what the Roles tab ranks by: describe each kind of role you want as a prompt, set the fetch gate (world, location, role exclusion, role fetch), and it measures every open role's closeness to the prompt it is closest to. Several agents rank the same roles differently, each with its own votes and liked companies; Viewing as on Roles picks one, and its votes are that agent's.</p>
|
||||||
|
|
||||||
<div class="agents-head">
|
<div class="agents-head">
|
||||||
<p id="agents-count" class="count" role="status"></p>
|
<p id="agents-count" class="count" role="status"></p>
|
||||||
@@ -24,24 +24,24 @@
|
|||||||
<legend>What you are looking for</legend>
|
<legend>What you are looking for</legend>
|
||||||
<div id="a-prompts" class="prompt-rows"></div>
|
<div id="a-prompts" class="prompt-rows"></div>
|
||||||
<button type="button" id="a-add-prompt" class="secondary">Add prompt</button>
|
<button type="button" id="a-add-prompt" class="secondary">Add prompt</button>
|
||||||
<span class="hint">A prompt is one kind of role in your own words: the work, the level, the setting. A role is ranked by the prompt it fits best, so three kinds of role are three prompts, not one blended paragraph. Votes and liked companies count for the whole agent.</span>
|
<span class="hint">A prompt is one kind of role in your own words: the work, the level, the setting. A role is ranked by the prompt it is closest to, so three kinds of role are three prompts, not one blended paragraph. Votes and liked companies count for the whole agent.</span>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
<fieldset class="where">
|
<fieldset class="where">
|
||||||
<legend>The fetch gate</legend>
|
<legend>The fetch gate</legend>
|
||||||
<span class="hint">Hard rules, a yes or no per role, applied at every stage: at the host where it can filter, before anything is scored, again after extract has read the posting, and in the feed. A role that fails is never read by a model and never shown. A rule rejects on evidence only: a role whose location is not known yet passes until extract reads it. A company set to All countries skips the place part and one set to All roles skips the role part, for every agent; both are switched under Sources.</span>
|
<span class="hint">Hard rules, a yes or no per role, in four parts. A role that fails is never read by a model and never shown. A rule rejects on evidence only: a role whose location is not known yet passes until extract reads it. Two company switches under Sources override parts for every agent: All locations bypasses Location, All roles bypasses Role exclusion and Role fetch.</span>
|
||||||
<h3>Region</h3>
|
<h3>World</h3>
|
||||||
<div class="where-line">
|
<div class="where-line">
|
||||||
<span>Only roles in:</span>
|
<span>Only roles in:</span>
|
||||||
<select id="a-scope" aria-label="Only roles in">
|
<select id="a-scope" aria-label="World: only roles in">
|
||||||
<option value="europe">Europe</option>
|
<option value="europe">Europe</option>
|
||||||
<option value="countries">Chosen countries</option>
|
<option value="countries">Chosen countries</option>
|
||||||
<option value="everywhere">Everywhere</option>
|
<option value="everywhere">Everywhere</option>
|
||||||
</select>
|
</select>
|
||||||
<div id="a-scope-countries"></div>
|
<div id="a-scope-countries"></div>
|
||||||
</div>
|
</div>
|
||||||
<span class="hint">Every role must be in this, and no company switch lifts it. A company set to All countries is fetched and kept from the whole region.</span>
|
<span class="hint">The outer countries. Every role must be in them, and no company switch lifts this. All locations falls back to it.</span>
|
||||||
<h3>Place</h3>
|
<h3>Location</h3>
|
||||||
<span class="hint">A role passes when it matches at least one line: a country, a town or remote, so a line more means more roles. With every line empty a role can be anywhere in the region.</span>
|
<span class="hint">A role passes when it matches at least one line: a country, a town or remote, so a line more means more roles. With every line empty a role can be anywhere in the world gate.</span>
|
||||||
<div class="where-line">
|
<div class="where-line">
|
||||||
<span>Anywhere in</span>
|
<span>Anywhere in</span>
|
||||||
<div id="a-countries"></div>
|
<div id="a-countries"></div>
|
||||||
@@ -55,28 +55,30 @@
|
|||||||
<span>Remote:</span>
|
<span>Remote:</span>
|
||||||
<select id="a-remote" aria-label="Remote">
|
<select id="a-remote" aria-label="Remote">
|
||||||
<option value="none">no</option>
|
<option value="none">no</option>
|
||||||
<option value="scope">anywhere within the region</option>
|
<option value="scope">anywhere within the world gate</option>
|
||||||
<option value="from">only when open to someone in:</option>
|
<option value="from">only when open to someone in:</option>
|
||||||
</select>
|
</select>
|
||||||
<div id="a-remote-from"></div>
|
<div id="a-remote-from"></div>
|
||||||
</div>
|
</div>
|
||||||
<span class="hint">A remote role passes on top of the lines above. A remote role whose area is not stated yet passes until extract reads the posting.</span>
|
<span class="hint">A remote role passes on top of the lines above. A remote role whose area is not stated yet passes until extract reads the posting.</span>
|
||||||
<h3>Role</h3>
|
<h3>Role exclusion</h3>
|
||||||
<div class="where-line">
|
|
||||||
<span>Only roles matching</span>
|
|
||||||
<div id="a-terms"></div>
|
|
||||||
</div>
|
|
||||||
<span class="hint">Fetch terms, separated by a comma or Enter: <code>machine learning, data science</code> is two terms, and <code>machine learning</code> is searched as one. They are asked of the host, which then sends only roles with one of the terms anywhere in the ad, not only the title. They apply on boards of 300 or more roles whose host takes a query, and only there; smaller boards and other hosts are read whole. Empty means no narrowing.</span>
|
|
||||||
<div class="where-line">
|
<div class="where-line">
|
||||||
<span>Leave out titles with</span>
|
<span>Leave out titles with</span>
|
||||||
<div id="a-titles"></div>
|
<div id="a-titles"></div>
|
||||||
</div>
|
</div>
|
||||||
<span class="hint">Phrases, separated by a comma or Enter. A phrase matches whole words anywhere in the title, whatever the case: <code>account executive</code> takes "Enterprise Account Executive, UK", and <code>data engineer</code> leaves "Data Engineering Manager" alone. A shorter phrase takes more: <code>sales</code> takes Sales Engineer too. A host cannot be asked this, so it applies once the roles are in, to liked companies as well. A role left out keeps its row, and Show gated on Roles lists it with the phrase that took it.</span>
|
<span class="hint">Phrases, separated by a comma or Enter. A phrase matches whole words anywhere in the title, whatever the case: <code>account executive</code> takes "Enterprise Account Executive, UK", and <code>data engineer</code> leaves "Data Engineering Manager" alone. A shorter phrase takes more: <code>sales</code> takes Sales Engineer too. No host can be asked this, so it is applied to every role after the fetch, liked companies included. A role left out keeps its row, and Show gated on Roles lists it with the phrase that took it.</span>
|
||||||
|
<h3>Role fetch</h3>
|
||||||
|
<div class="where-line">
|
||||||
|
<span>Only roles matching</span>
|
||||||
|
<div id="a-terms"></div>
|
||||||
|
</div>
|
||||||
|
<span class="hint">Fetch terms, separated by a comma or Enter: <code>machine learning, data science</code> is two terms, and <code>machine learning</code> is searched as one. A host that takes a query sends only roles with one of the terms anywhere in the ad, but only for boards of 300 or more roles; smaller boards and other hosts are read whole. The terms are ignored after the fetch, since prompt closeness is the finer filter. Empty means no narrowing.</span>
|
||||||
<p id="a-where-sum" class="where-sum" role="status"></p>
|
<p id="a-where-sum" class="where-sum" role="status"></p>
|
||||||
|
<p id="a-kept" class="hint" hidden></p>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
<fieldset class="terms cv">
|
<fieldset class="terms cv">
|
||||||
<legend>Your CV</legend>
|
<legend>Your CV</legend>
|
||||||
<span class="hint">Optional. With a CV, the judge stage reads each of this agent's best roles against it and scores skills, experience and career, which CV match on Roles sorts by. A role is read at most once per change of the CV, the prompts or the texts below. The CV stays in this app and goes only to the model you set up.</span>
|
<span class="hint">Optional. With a CV, the judge stage reads each of this agent's best roles against it and rates skills, experience and career from 0 to 10, which the CV match on Roles is made of. A role is read at most once per change of the CV, the prompts or the texts below. The CV stays in this app and goes only to the model you set up.</span>
|
||||||
<label>CV
|
<label>CV
|
||||||
<textarea id="a-cv-text" rows="10" placeholder="Paste your CV or career summary here"></textarea></label>
|
<textarea id="a-cv-text" rows="10" placeholder="Paste your CV or career summary here"></textarea></label>
|
||||||
<label>or read it from a file
|
<label>or read it from a file
|
||||||
@@ -93,50 +95,36 @@
|
|||||||
<textarea id="a-like-text" rows="4" placeholder="Building a team from scratch, close work with the product side, small companies, a say in the roadmap"></textarea></label>
|
<textarea id="a-like-text" rows="4" placeholder="Building a team from scratch, close work with the product side, small companies, a say in the roadmap"></textarea></label>
|
||||||
<label>What I don't like
|
<label>What I don't like
|
||||||
<textarea id="a-avoid-text" rows="4" placeholder="Constant on-call, sales targets, large consultancies, open-plan offices"></textarea></label>
|
<textarea id="a-avoid-text" rows="4" placeholder="Constant on-call, sales targets, large consultancies, open-plan offices"></textarea></label>
|
||||||
<span class="hint">The work, the workplace, situations; read by the CV match for the career score. Rules that must hold go under the fetch gate or Other rules instead.</span>
|
<span class="hint">The work, the workplace, situations; read by the judge for the career rating. Rules that must hold go under the fetch gate instead.</span>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
|
<fieldset class="weights limits">
|
||||||
|
<legend>Analytics filter</legend>
|
||||||
|
<label>Prompt closeness: the best <input type="number" id="a-cut-fit-top" min="0" step="10" value="500"></label>
|
||||||
|
<label>each at least <input type="number" id="a-cut-fit-min" min="0" max="1" step="0.05" value="0.6"></label>
|
||||||
|
<label>CV closeness: the best <input type="number" id="a-cut-cv-top" min="0" step="10" value="150"></label>
|
||||||
|
<label>not already in, each at least <input type="number" id="a-cut-cv-min" min="0" max="1" step="0.05" value="0.6"></label>
|
||||||
|
</fieldset>
|
||||||
|
<span class="hint">Of the roles that pass the fetch gate, the analytics filter keeps the best N by prompt closeness and, of the rest, the best M by CV closeness. Liked and saved roles always pass, whatever their closeness. The roles that pass, the cut, get fetch text, extract and judge, so it sets what the models read. Closeness is a level from 0 to 1, the same in every run and free to compute; a liked company adds about 0.056 to it and a favourite about 0.111. For a broad agent the count stops a list, for a narrow one the minimum does, so roles that do not fit are not read to fill the count. An agent with no prompt has no prompt list, and one with no CV no CV list.</span>
|
||||||
<fieldset class="weights">
|
<fieldset class="weights">
|
||||||
<legend>The feed order</legend>
|
<legend>Want feed score</legend>
|
||||||
<label>Fit to the prompts <input type="number" id="a-fit" min="0" step="0.05" value="0.6"></label>
|
<label>Prompt closeness <input type="number" id="a-fit" min="0" step="0.05" value="0.6"></label>
|
||||||
<label>Newness <input type="number" id="a-recency" min="0" step="0.05" value="0.2"></label>
|
<label>Newness <input type="number" id="a-recency" min="0" step="0.05" value="0.2"></label>
|
||||||
<label>Companies you like <input type="number" id="a-affinity" min="0" step="0.05" value="0.2"></label>
|
<label>Company like <input type="number" id="a-affinity" min="0" step="0.05" value="0.2"></label>
|
||||||
<label>Roles you liked <input type="number" id="a-taste" min="0" step="0.05" value="0.2"></label>
|
<label>Job like <input type="number" id="a-taste" min="0" step="0.05" value="0.2"></label>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
<span class="hint">The final score, which orders What I'm looking for on Roles: the weighted sum of the four over the sum of the weights. Newness halves every 14 days; Roles you liked is a role's closeness to the roles you liked, away from those you passed. Changing the weights re-sorts and reads nothing.</span>
|
<span class="hint">Orders What I'm looking for on Roles: the weighted sum of the four over the sum of the weights. Newness halves every 14 days; job like is a role's closeness to the roles you liked, away from those you passed. Changing the weights re-sorts and reads nothing.</span>
|
||||||
<fieldset class="weights">
|
<fieldset class="weights">
|
||||||
<legend>How much each counts in the CV match</legend>
|
<legend>CV match weights</legend>
|
||||||
<label>Skills <input type="number" id="a-cv-skills" min="0" step="5" value="30"></label>
|
<label>Skills <input type="number" id="a-cv-skills" min="0" step="5" value="30"></label>
|
||||||
<label>Experience <input type="number" id="a-cv-experience" min="0" step="5" value="25"></label>
|
<label>Experience <input type="number" id="a-cv-experience" min="0" step="5" value="25"></label>
|
||||||
<label>Career <input type="number" id="a-cv-career" min="0" step="5" value="30"></label>
|
<label>Career <input type="number" id="a-cv-career" min="0" step="5" value="30"></label>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
<span class="hint">The judge's three numbers are averaged with these shares into the CV match. Changing them re-sorts at once and asks the model for nothing; 0 leaves a number out, all three 0 means no CV match.</span>
|
<span class="hint">The judge rates skills, experience and career from 0 to 10; the CV match, from 0 to 100, is those three weighted by these shares. It orders What fits me on Roles. Changing the weights re-sorts at once and asks the model for nothing; 0 leaves a number out, all three 0 means no CV match.</span>
|
||||||
<fieldset class="weights limits">
|
|
||||||
<legend>The cut</legend>
|
|
||||||
<label>Fit to the prompts: the best <input type="number" id="a-cut-fit-top" min="0" step="10" value="500"></label>
|
|
||||||
<label>each at least <input type="number" id="a-cut-fit-min" min="0" max="1" step="0.05" value="0.6"></label>
|
|
||||||
<label>Closeness to the CV: the best <input type="number" id="a-cut-cv-top" min="0" step="10" value="150"></label>
|
|
||||||
<label>not already in, each at least <input type="number" id="a-cut-cv-min" min="0" max="1" step="0.05" value="0.6"></label>
|
|
||||||
</fieldset>
|
|
||||||
<span class="hint">Of the roles that pass the fetch gate, the cut holds the best by fit and, of the rest, the best by closeness to the CV. Liked and saved roles are always in, whatever their levels. The cut gets fetch text, extract and judge, so it sets what the models read. Both are levels from 0 to 1, the same in every run and free to compute; a liked company adds about 0.05 to them and a favourite about 0.1. For a broad agent the count stops a list, for a narrow one the minimum does, so roles that do not fit are not read to fill the count. An agent with no prompt has no fit list, and one with no CV no closeness list.</span>
|
|
||||||
<fieldset class="weights limits">
|
<fieldset class="weights limits">
|
||||||
<legend>After the ranking</legend>
|
<legend>After the ranking</legend>
|
||||||
<label>Summarise the top <input type="number" id="a-summarise-top" min="0" step="5" value="50"></label>
|
<label>Summarise the top <input type="number" id="a-summarise-top" min="0" step="5" value="30"> of each feed order</label>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
<span class="hint">Summarise writes a short summary and tags for the top of the feed by the final score, and for every role you liked or saved, whatever its score. It is no part of any score.</span>
|
<span class="hint">Summarise writes a short summary and tags for the top N of What I'm looking for and the top N of What fits me, at most twice N roles, and for every role you liked or saved. It is no part of any score.</span>
|
||||||
<details class="advanced">
|
|
||||||
<summary>Advanced</summary>
|
|
||||||
<div class="advanced-body">
|
|
||||||
<fieldset class="rules">
|
|
||||||
<legend>Other rules</legend>
|
|
||||||
<div id="a-rules" class="rule-rows"></div>
|
|
||||||
<button type="button" id="a-add-rule" class="secondary">Add rule</button>
|
|
||||||
<span class="hint">Must: a role must meet it. One of: a role must meet one of these or a Place line. Not: a role that meets it is left out. Values are separated by commas; salary is a number.</span>
|
|
||||||
</fieldset>
|
|
||||||
<label>Rules as sent
|
|
||||||
<textarea id="a-gates" class="code" rows="6" spellcheck="false" readonly>[]</textarea>
|
|
||||||
<span class="hint">Read only: what Save sends as the agent's gates, from the fetch gate and Other rules.</span></label>
|
|
||||||
</div>
|
|
||||||
</details>
|
|
||||||
<div class="agent-actions">
|
<div class="agent-actions">
|
||||||
<button type="submit">Save and score</button>
|
<button type="submit">Save and score</button>
|
||||||
<button type="button" id="a-cancel" class="secondary">Cancel</button>
|
<button type="button" id="a-cancel" class="secondary">Cancel</button>
|
||||||
|
|||||||
@@ -104,7 +104,7 @@
|
|||||||
|
|
||||||
<section class="card" id="company-boards" data-section>
|
<section class="card" id="company-boards" data-section>
|
||||||
<h2>Company boards</h2>
|
<h2>Company boards</h2>
|
||||||
<p class="lede">Every company roledesk has seen, and how many of its roles are open. Switching a board off stops reading it and closes its roles; hiding a company keeps reading it but leaves its roles out of the list and unscored. Like or favourite raises a company's roles a little in the cut and in the feed order, in each search agent's own ranking (a liked company about 0.05, a favourite about 0.1), and passes nothing the fetch gate rejects. All countries and All roles are for every agent: the first skips the place part of the fetch gate, the second the fetch terms and the titles to leave out.</p>
|
<p class="lede">Every company roledesk has seen, and how many of its roles are open. Switching a board off stops reading it and closes its roles; hiding a company keeps reading it but leaves its roles out of the list and unscored. Like or favourite raises a company's roles a little in the cut and in the feed order, in each search agent's own ranking (a liked company about 0.056, a favourite about 0.111 on the start weights), and passes nothing the fetch gate rejects. All locations and All roles are for every agent: the first skips the location gate and keeps the world gate, the second skips the role exclusion and the fetch terms.</p>
|
||||||
|
|
||||||
<form id="board-form" class="agent-form source-form" autocomplete="off">
|
<form id="board-form" class="agent-form source-form" autocomplete="off">
|
||||||
<h3>Follow a company</h3>
|
<h3>Follow a company</h3>
|
||||||
@@ -132,7 +132,7 @@
|
|||||||
<option value="favourite">Favourite</option>
|
<option value="favourite">Favourite</option>
|
||||||
<option value="liked">Liked</option>
|
<option value="liked">Liked</option>
|
||||||
<option value="hidden">Hidden</option>
|
<option value="hidden">Hidden</option>
|
||||||
<option value="all_countries">All countries</option>
|
<option value="all_countries">All locations</option>
|
||||||
<option value="all_roles">All roles</option>
|
<option value="all_roles">All roles</option>
|
||||||
<option value="on">On</option>
|
<option value="on">On</option>
|
||||||
<option value="off">Off</option>
|
<option value="off">Off</option>
|
||||||
|
|||||||
+38
-30
@@ -70,7 +70,6 @@ def _feed_filters(
|
|||||||
function: str = "",
|
function: str = "",
|
||||||
remote: str = "",
|
remote: str = "",
|
||||||
days: int | None = None,
|
days: int | None = None,
|
||||||
show_muted: bool = False,
|
|
||||||
include_read: bool = True,
|
include_read: bool = True,
|
||||||
tags: list[str] = Query(default=[]),
|
tags: list[str] = Query(default=[]),
|
||||||
tag_mode: Literal["any", "all"] = "any",
|
tag_mode: Literal["any", "all"] = "any",
|
||||||
@@ -96,7 +95,8 @@ def _feed_filters(
|
|||||||
them (`levels_between`), which is what the store filters on. `order=cv`
|
them (`levels_between`), which is what the store filters on. `order=cv`
|
||||||
is the score order with the agent's judged roles first by `cv_score`
|
is the score order with the agent's judged roles first by `cv_score`
|
||||||
(`by_cv`). A request that names an agent keeps to its fetch gate in
|
(`by_cv`). A request that names an agent keeps to its fetch gate in
|
||||||
every order, newest included (`fetch_gate`), unless `gated`."""
|
every order, newest included (`fetch_gate`), unless `gated`, and never
|
||||||
|
shows a company the agent hid."""
|
||||||
for token in tokens:
|
for token in tokens:
|
||||||
if not places.known(token):
|
if not places.known(token):
|
||||||
raise HTTPException(422, f"No place is known as {token!r}.")
|
raise HTTPException(422, f"No place is known as {token!r}.")
|
||||||
@@ -115,7 +115,7 @@ def _feed_filters(
|
|||||||
return {
|
return {
|
||||||
"q": q or None, "company": company or None, "place": place or None,
|
"q": q or None, "company": company or None, "place": place or None,
|
||||||
"function": function or None, "remote": remote or None, "days": days,
|
"function": function or None, "remote": remote or None, "days": days,
|
||||||
"hide_muted": not show_muted, "include_read": include_read, "tags": tags,
|
"include_read": include_read, "tags": tags,
|
||||||
"tag_mode": tag_mode, "agent": _agent_id(store, agent),
|
"tag_mode": tag_mode, "agent": _agent_id(store, agent),
|
||||||
"by_score": agent is not None and order in ("score", "cv"),
|
"by_score": agent is not None and order in ("score", "cv"),
|
||||||
"by_cv": agent is not None and order == "cv", "gated": gated,
|
"by_cv": agent is not None and order == "cv", "gated": gated,
|
||||||
@@ -155,7 +155,8 @@ def role_facets(store: StoreDep, filters: FeedFilters) -> Facets:
|
|||||||
def get_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole:
|
def get_role(uid: str, store: StoreDep, agent: int | None = None) -> ScoredRole:
|
||||||
"""One role, open or closed, for the read page, with `agent`'s vote and
|
"""One role, open or closed, for the read page, with `agent`'s vote and
|
||||||
score, the prompt that gave its fit, its `judgement` against the agent's
|
score, the prompt that gave its fit, its `judgement` against the agent's
|
||||||
CV and the language flag (the default agent's when absent;
|
CV (`skills`, `experience` and `career` 0 to 10, `cv_score` 0 to 100)
|
||||||
|
and the language flag (the default agent's when absent;
|
||||||
None when there is no agent). `copies` lists the open copies of a kept
|
None when there is no agent). `copies` lists the open copies of a kept
|
||||||
role (`RoleCopy`, for "Also posted"); a copy itself opens too, with
|
role (`RoleCopy`, for "Also posted"); a copy itself opens too, with
|
||||||
`dup_of` naming its kept role."""
|
`dup_of` naming its kept role."""
|
||||||
@@ -235,9 +236,9 @@ def list_roles(
|
|||||||
`Store.roles` for what each filter does.
|
`Store.roles` for what each filter does.
|
||||||
|
|
||||||
`agent` says whose votes, hidden companies and scores are meant. Each
|
`agent` says whose votes, hidden companies and scores are meant. Each
|
||||||
role carries that agent's `vote`; a company it hid is left out unless
|
role carries that agent's `vote`; a company it hid is always left out,
|
||||||
`show_muted` is set; `judged` keeps the liked, the unjudged or the
|
in every order, the total and the facets; `judged` keeps the liked, the
|
||||||
passed. Every order keeps to the roles that agent's fetch gate passes,
|
unjudged or the passed. Every order keeps to the roles that agent's fetch gate passes,
|
||||||
a gated one only with `gated=1`: with `order=score` (the default) best
|
a gated one only with `gated=1`: with `order=score` (the default) best
|
||||||
first by final score, carrying `score` and `gated_reason`; with
|
first by final score, carrying `score` and `gated_reason`; with
|
||||||
`order=newest` newest first, carrying `gated_reason` and no score; with
|
`order=newest` newest first, carrying `gated_reason` and no score; with
|
||||||
@@ -255,11 +256,11 @@ def list_roles(
|
|||||||
as well. `closing=1` keeps the roles whose `deadline` is today or in
|
as well. `closing=1` keeps the roles whose `deadline` is today or in
|
||||||
the next four days (Closing soon).
|
the next four days (Closing soon).
|
||||||
|
|
||||||
Each row carries `deadline`, the agent's `cv_score` (None when not
|
Each row carries `deadline`, the agent's `cv_score` (0 to 100, None
|
||||||
judged), and `lang_flag` and `lang_note`, the posting's languages
|
when not judged), and `lang_flag` and `lang_note`, the posting's languages
|
||||||
against the agent's (`app.models.language_flag`).
|
against the agent's (`app.models.language_flag`).
|
||||||
|
|
||||||
`company` and `place` are substrings, kept for links from other pages.
|
`company` and `place` are substrings, kept for older links and saved filters.
|
||||||
The rail's fields are `company_keys` (repeated, exact keys) and
|
The rail's fields are `company_keys` (repeated, exact keys) and
|
||||||
`places` (repeated place tokens, `app.places`: `country:DK`,
|
`places` (repeated place tokens, `app.places`: `country:DK`,
|
||||||
`region:skane`, `place:malmo`, matched on whole folded words), several
|
`region:skane`, `place:malmo`, matched on whole folded words), several
|
||||||
@@ -290,14 +291,15 @@ class FiltersOut(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/filters")
|
@router.get("/filters")
|
||||||
def list_filters(store: StoreDep, show_muted: bool = False,
|
def list_filters(store: StoreDep, agent: int | None = None) -> FiltersOut:
|
||||||
agent: int | None = None) -> FiltersOut:
|
|
||||||
"""The tag vocabulary for the feed's filter row, most-used first. The
|
"""The tag vocabulary for the feed's filter row, most-used first. The
|
||||||
companies `agent` hid (the default agent's without one) are left out
|
companies `agent` hid (the default agent's without one) are left out,
|
||||||
unless `show_muted`, matching `/api/roles`."""
|
matching `/api/roles`; with `agent` named, so is every role its fetch
|
||||||
|
gate rejects, so a count is what the feed lists for the tag."""
|
||||||
hidden_for = _agent_id(store, agent)
|
hidden_for = _agent_id(store, agent)
|
||||||
return FiltersOut(tags=[TagCount(tag=tag, count=count, unread=unread)
|
return FiltersOut(tags=[TagCount(tag=tag, count=count, unread=unread)
|
||||||
for tag, count, unread in store.tag_counts(hidden_for, show_muted)])
|
for tag, count, unread in store.tag_counts(
|
||||||
|
hidden_for, fetch_gate=agent is not None)])
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ suggestions
|
# ------------------------------------------------------------ suggestions
|
||||||
@@ -343,14 +345,17 @@ def suggest_companies(
|
|||||||
folded as places are (`places.fold`), so `orsted` finds Ørsted and
|
folded as places are (`places.fold`), so `orsted` finds Ørsted and
|
||||||
`kobenhavn` finds a name with København in it. By open roles, most
|
`kobenhavn` finds a name with København in it. By open roles, most
|
||||||
first. The companies `agent` (the default agent when absent) hid are
|
first. The companies `agent` (the default agent when absent) hid are
|
||||||
left out, as the feed leaves them out. An empty `q` suggests nothing."""
|
left out, as the feed leaves them out; with `agent` named, the count is
|
||||||
|
the roles its feed shows, past its fetch gate. An empty `q` suggests
|
||||||
|
nothing."""
|
||||||
text = places.fold(q)
|
text = places.fold(q)
|
||||||
if not text:
|
if not text:
|
||||||
return []
|
return []
|
||||||
aliases = store.company_aliases()
|
aliases = store.company_aliases()
|
||||||
|
companies = store.open_companies(_agent_id(store, agent), fetch_gate=agent is not None)
|
||||||
found = [CompanySuggestion(key=key, name=names[0] if names else key, count=count,
|
found = [CompanySuggestion(key=key, name=names[0] if names else key, count=count,
|
||||||
affinity=affinity)
|
affinity=affinity)
|
||||||
for key, names, count, affinity in store.open_companies(_agent_id(store, agent))
|
for key, names, count, affinity in companies
|
||||||
if any(_starts_a_word(text, places.fold(name)) for name in names)
|
if any(_starts_a_word(text, places.fold(name)) for name in names)
|
||||||
or any(_starts_a_word(text, fold) for fold in aliases.get(key, ()))]
|
or any(_starts_a_word(text, fold) for fold in aliases.get(key, ()))]
|
||||||
found.sort(key=lambda c: (-c.count, c.name.casefold()))
|
found.sort(key=lambda c: (-c.count, c.name.casefold()))
|
||||||
@@ -370,11 +375,13 @@ def suggest_places(
|
|||||||
open role's place line and every open remote role's area). The count of
|
open role's place line and every open remote role's area). The count of
|
||||||
`remote:any` is the remote roles in the scope of `agent`, the default
|
`remote:any` is the remote roles in the scope of `agent`, the default
|
||||||
agent when absent, as the feed counts them, and the other remote tokens
|
agent when absent, as the feed counts them, and the other remote tokens
|
||||||
are kept to that agent's scope, its `require` gates on country. An empty
|
are kept to that agent's scope, its `require` gates on country. With
|
||||||
`q` suggests nothing."""
|
`agent` named, every count is of the roles its feed shows, past its
|
||||||
|
fetch gate. An empty `q` suggests nothing."""
|
||||||
if not places.fold(q):
|
if not places.fold(q):
|
||||||
return []
|
return []
|
||||||
viewer = _agent_id(store, agent)
|
viewer = _agent_id(store, agent)
|
||||||
|
shown = viewer if agent is not None else None
|
||||||
gaz = store.gazetteer()
|
gaz = store.gazetteer()
|
||||||
found = store.agent(viewer) if viewer is not None else None
|
found = store.agent(viewer) if viewer is not None else None
|
||||||
scope = {p for g in (found.gates if found is not None else ())
|
scope = {p for g in (found.gates if found is not None else ())
|
||||||
@@ -382,10 +389,11 @@ def suggest_places(
|
|||||||
for v in g.values for p in gaz.lookup(f"country:{v}") or ()}
|
for v in g.values for p in gaz.lookup(f"country:{v}") or ()}
|
||||||
return [PlaceSuggestion(token=s.token, label=s.label, kind=s.kind, count=s.count)
|
return [PlaceSuggestion(token=s.token, label=s.label, kind=s.kind, count=s.count)
|
||||||
for s in places.suggest(
|
for s in places.suggest(
|
||||||
q, store.open_places(), limit, remote=store.open_remote_areas(),
|
q, store.open_places(shown), limit, remote=store.open_remote_areas(shown),
|
||||||
gaz=gaz, scope=scope,
|
gaz=gaz, scope=scope,
|
||||||
remote_any=lambda: store.count_roles(places=["remote:any"], agent=viewer,
|
remote_any=lambda: store.count_roles(places=["remote:any"], agent=viewer,
|
||||||
by_score=False))]
|
by_score=False,
|
||||||
|
fetch_gate=agent is not None))]
|
||||||
|
|
||||||
|
|
||||||
@router.get("/places/labels")
|
@router.get("/places/labels")
|
||||||
@@ -506,9 +514,10 @@ class CompanyOut(BaseModel):
|
|||||||
`careers_opened_at` are the Watchlist's (`Company`'s), the latter also
|
`careers_opened_at` are the Watchlist's (`Company`'s), the latter also
|
||||||
when a person last looked at the careers page. `by_hand` marks a company
|
when a person last looked at the careers page. `by_hand` marks a company
|
||||||
with no board shown that is not `from_search`, and `hand_reason` says why
|
with no board shown that is not `from_search`, and `hand_reason` says why
|
||||||
it is read by hand (`app.models.HAND_REASONS`). `all_countries` and
|
it is read by hand (`app.models.HAND_REASONS`). `all_countries` (All
|
||||||
`all_roles` are the company's two fetch overrides: its big boards read
|
locations) and `all_roles` (All roles) are the company's two fetch
|
||||||
with the agents' wide reach, and with no role-type narrowing. `target` is
|
overrides: its big boards read with the agents' wide reach, and with no
|
||||||
|
role-type narrowing. `target` is
|
||||||
the attached board in full, `host:slug`, where `board` is its host alone;
|
the attached board in full, `host:slug`, where `board` is its host alone;
|
||||||
the page compares it with the board typed to know whether a replacement
|
the page compares it with the board typed to know whether a replacement
|
||||||
closes anything. `ats_slug` cannot stand in for it, since the careers site
|
closes anything. `ats_slug` cannot stand in for it, since the careers site
|
||||||
@@ -771,7 +780,7 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep,
|
|||||||
"""-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the
|
"""-1 hidden, 0 neutral, 1 liked, 2 favourite, for `body.agent` or the
|
||||||
default agent. A key with roles and no company row gets one first (see
|
default agent. A key with roles and no company row gets one first (see
|
||||||
`Store.set_affinity`); 404 for a key with neither. A change here is not
|
`Store.set_affinity`); 404 for a key with neither. A change here is not
|
||||||
seen in Agent match until `score` has run, so it queues one
|
seen in "What I'm looking for" until `score` has run, so it queues one
|
||||||
(`_score_after_change`) and answers with its id."""
|
(`_score_after_change`) and answers with its id."""
|
||||||
if body.affinity not in AFFINITY:
|
if body.affinity not in AFFINITY:
|
||||||
raise HTTPException(422, "affinity is -1, 0, 1 or 2")
|
raise HTTPException(422, "affinity is -1, 0, 1 or 2")
|
||||||
@@ -826,7 +835,8 @@ def _override(store: Store, queue: Any, key: str, on: bool, setter: str) -> Comp
|
|||||||
@router.post("/companies/{key}/all-countries")
|
@router.post("/companies/{key}/all-countries")
|
||||||
def set_all_countries(key: str, body: OverrideBody, store: StoreDep,
|
def set_all_countries(key: str, body: OverrideBody, store: StoreDep,
|
||||||
queue: QueueDep) -> Company:
|
queue: QueueDep) -> Company:
|
||||||
"""All countries on or off: global, apart from affinity. On, the
|
"""All locations on or off (the field and the path keep the name
|
||||||
|
`all_countries`): global, apart from affinity. On, the
|
||||||
company's roles skip the place part of every agent's fetch gate (the
|
company's roles skip the place part of every agent's fetch gate (the
|
||||||
`any` gates, `jobs.score.failed_gates`), and its big boards are read
|
`any` gates, `jobs.score.failed_gates`), and its big boards are read
|
||||||
with the agents' wide reach, Europe today, rather than their places
|
with the agents' wide reach, Europe today, rather than their places
|
||||||
@@ -1241,7 +1251,7 @@ class AgentBody(BaseModel):
|
|||||||
it is stored as one prompt named after the agent. `fetch_terms` narrow
|
it is stored as one prompt named after the agent. `fetch_terms` narrow
|
||||||
what big boards send (`app/jobs/reach.py`); `[]` narrows nothing, and
|
what big boards send (`app/jobs/reach.py`); `[]` narrows nothing, and
|
||||||
leaving the key out keeps an existing agent's. `cv_text`,
|
leaving the key out keeps an existing agent's. `cv_text`,
|
||||||
`cv_languages`, `like_text`, `avoid_text` and `must_text` are what `judge` reads a
|
`cv_languages`, `like_text` and `avoid_text` are what `judge` reads a
|
||||||
role against (`app/jobs/judge.py`), kept the same way when left out or
|
role against (`app/jobs/judge.py`), kept the same way when left out or
|
||||||
null."""
|
null."""
|
||||||
|
|
||||||
@@ -1257,7 +1267,6 @@ class AgentBody(BaseModel):
|
|||||||
cv_languages: CvLanguages | None = None
|
cv_languages: CvLanguages | None = None
|
||||||
like_text: str | None = None
|
like_text: str | None = None
|
||||||
avoid_text: str | None = None
|
avoid_text: str | None = None
|
||||||
must_text: str | None = None
|
|
||||||
enabled: bool = True
|
enabled: bool = True
|
||||||
|
|
||||||
|
|
||||||
@@ -1297,8 +1306,7 @@ def save_agent(body: AgentBody, store: StoreDep, queue: QueueDep) -> Agent:
|
|||||||
body.enabled, agent_id=body.id, limits=body.limits,
|
body.enabled, agent_id=body.id, limits=body.limits,
|
||||||
prompts=body.prompts, fetch_terms=body.fetch_terms,
|
prompts=body.prompts, fetch_terms=body.fetch_terms,
|
||||||
cv_text=body.cv_text, cv_languages=body.cv_languages,
|
cv_text=body.cv_text, cv_languages=body.cv_languages,
|
||||||
like_text=body.like_text, avoid_text=body.avoid_text,
|
like_text=body.like_text, avoid_text=body.avoid_text)
|
||||||
must_text=body.must_text)
|
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(409, str(exc)) from exc
|
raise HTTPException(409, str(exc)) from exc
|
||||||
except KeyError:
|
except KeyError:
|
||||||
|
|||||||
+19
-9
@@ -1,6 +1,6 @@
|
|||||||
# auth
|
# auth
|
||||||
|
|
||||||
One password on a public hostname. scrypt, a lockout counted in the database, sessions you can revoke, passkeys, and one middleware in front of everything — the WebSocket included. No user management: there is one user, they are the person who deployed it, and the only questions are whether the door is locked and what opens it.
|
One password on a public hostname. scrypt, a lockout counted in the database, sessions you can revoke, passkeys, API tokens for scripts, and one middleware in front of everything — the WebSocket included. No user management: there is one user, they are the person who deployed it, and the only questions are whether the door is locked and what opens it: the password, a passkey, or a token.
|
||||||
|
|
||||||
Needs Python 3.12+, `fastapi` and `pydantic`. `webauthn` is optional and only for passkeys; every import of it is inside a function, and the passkey routes answer 501 with a sentence saying what to install when it is missing.
|
Needs Python 3.12+, `fastapi` and `pydantic`. `webauthn` is optional and only for passkeys; every import of it is inside a function, and the passkey routes answer 501 with a sentence saying what to install when it is missing.
|
||||||
|
|
||||||
@@ -11,7 +11,7 @@ cp -r boilerplate/auth yourproject/auth
|
|||||||
python -m pytest yourproject/auth/tests
|
python -m pytest yourproject/auth/tests
|
||||||
```
|
```
|
||||||
|
|
||||||
Then add `auth*` to that project's `pyproject.toml` packages, `"auth" = ["templates/*.html", "static/*"]` to its package data, and `auth/tests` to its testpaths. `docs/password-protection.md` in this repository is the reasoning behind every decision below, at length.
|
Then add `auth*` to that project's `pyproject.toml` packages, `"auth" = ["templates/*.html", "static/*"]` to its package data, and `auth/tests` to its testpaths. The boilerplate's `documentation/Password Protection.md` is the reasoning behind every decision below, at length.
|
||||||
|
|
||||||
## Five lines
|
## Five lines
|
||||||
|
|
||||||
@@ -27,7 +27,7 @@ auth.connect(app, auth.AuthStore(config.database), config.auth,
|
|||||||
|
|
||||||
Call it **first**, before the app's own routers. Starlette wraps the whole router in its middleware whenever the routes were added, so the socket and every page are covered either way — but written first it is also read first, and nobody reviewing the file has to work out whether the socket is behind the password.
|
Call it **first**, before the app's own routers. Starlette wraps the whole router in its middleware whenever the routes were added, so the socket and every page are covered either way — but written first it is also read first, and nobody reviewing the file has to work out whether the socket is behind the password.
|
||||||
|
|
||||||
That gives you `/login` (the page and the POST), `/logout`, `/api/account`, `/api/account/password`, `/api/account/sessions/{id}` and `/api/account/sessions/close`, the four passkey routes, `/_auth/passkey.js` and `/_auth/account.js` — and a startup bootstrap that makes sure there is a password at all.
|
That gives you `/login` (the page and the POST), `/logout`, `/api/account`, `/api/account/password`, `/api/account/sessions/{id}` and `/api/account/sessions/close`, the four passkey routes, the three token routes (`POST /api/account/tokens`, `POST /api/account/tokens/{id}/label`, `DELETE /api/account/tokens/{id}`), `/_auth/passkey.js` and `/_auth/account.js` — and a startup bootstrap that makes sure there is a password at all.
|
||||||
|
|
||||||
| `connect(app, store, config, ...)` | |
|
| `connect(app, store, config, ...)` | |
|
||||||
|---|---|
|
|---|---|
|
||||||
@@ -35,6 +35,7 @@ That gives you `/login` (the page and the POST), `/logout`, `/api/account`, `/ap
|
|||||||
| `config` | an `AuthConfig`. The app's own config model carries it as a field; see below. |
|
| `config` | an `AuthConfig`. The app's own config model carries it as a field; see below. |
|
||||||
| `host=` | the address the server binds — `config.server.host`. What `enabled: auto` decides from, and deliberately not taken from a request, since a `Host` header is written by whoever is talking. |
|
| `host=` | the address the server binds — `config.server.host`. What `enabled: auto` decides from, and deliberately not taken from a request, since a `Host` header is written by whoever is talking. |
|
||||||
| `open=` | extra paths reachable without a session, added to `middleware.OPEN`. A name ending in `/` is a prefix; any other is matched whole. `("/static/",)` is what an app with a stylesheet passes. |
|
| `open=` | extra paths reachable without a session, added to `middleware.OPEN`. A name ending in `/` is a prefix; any other is matched whole. `("/static/",)` is what an app with a stylesheet passes. |
|
||||||
|
| `session_only=` | extra paths an API token may not reach, added to `middleware.ACCOUNT` and matched the same way. Anything that reads or replaces the database as a file belongs here, since the file holds the password hash and the token rows. |
|
||||||
| `stylesheet=` | a URL the login page links, so it looks like the rest of the app. Optional: the page carries enough inline CSS to be presentable on its own. |
|
| `stylesheet=` | a URL the login page links, so it looks like the rest of the app. Optional: the page carries enough inline CSS to be presentable on its own. |
|
||||||
|
|
||||||
The account screen is two lines in a template, the same deal `<desk-jobs>` makes:
|
The account screen is two lines in a template, the same deal `<desk-jobs>` makes:
|
||||||
@@ -44,7 +45,7 @@ The account screen is two lines in a template, the same deal `<desk-jobs>` makes
|
|||||||
<auth-account></auth-account>
|
<auth-account></auth-account>
|
||||||
```
|
```
|
||||||
|
|
||||||
It draws the lot — locked or not, change the password, the sessions with revoke and close-others, the passkeys with register, rename and revoke, and sign out — and it is themed from outside by the same `--desk-jobs-*` custom properties the queue's elements read, so `desk-jobs, desk-schedules, auth-account { ... }` in one stylesheet is all three.
|
It draws the lot — locked or not, change the password, the sessions with revoke and close-others, the passkeys with register, rename and revoke, the API tokens with create, rename and revoke, and sign out — and it is themed from outside by the same `--desk-jobs-*` custom properties the queue's elements read, so `desk-jobs, desk-schedules, auth-account { ... }` in one stylesheet is all three. Under `40rem` a row's buttons drop under its name when they do not fit beside it and a form's fields take the width; under a coarse pointer every button is a thumb's height and no field is under 16px. The login page does the same for itself, since it is served before any of the app's CSS.
|
||||||
|
|
||||||
## The config block
|
## The config block
|
||||||
|
|
||||||
@@ -69,7 +70,7 @@ auth:
|
|||||||
cookie: app_session
|
cookie: app_session
|
||||||
```
|
```
|
||||||
|
|
||||||
`enabled: auto` is the setting that matters: locked unless the server binds a loopback address. Deciding from the bind address rather than defaulting to off prevents the failure that actually happens — you deploy, it works, and nothing anywhere tells you the door is open. Production says `always` anyway, because behind a TLS-terminating proxy every request looks local to the app and is not.
|
`enabled: auto` is the setting that matters: locked unless the server binds a loopback address. Deciding from the bind address rather than defaulting to off prevents the failure that actually happens — you deploy, it works, and nothing anywhere tells you the door is open. Production says `always` anyway, so the lock does not depend on the bind address or on how the process was started. Under `auto` an app left open on loopback answers any request carrying `X-Forwarded-For`, `X-Forwarded-Proto`, `X-Forwarded-Host` or `Forwarded` with 421 and logs a warning once, since a proxy on the same machine would otherwise serve it to the network without a password. The Vite dev server's proxy sends none of these headers.
|
||||||
|
|
||||||
`password_env` and `cookie` are settings because this folder is copied into projects that are not called `app`, and two of those on one hostname would otherwise share a cookie and sign each other out.
|
`password_env` and `cookie` are settings because this folder is copied into projects that are not called `app`, and two of those on one hostname would otherwise share a cookie and sign each other out.
|
||||||
|
|
||||||
@@ -77,33 +78,42 @@ auth:
|
|||||||
|
|
||||||
The password is never in the config file — only ever an scrypt hash in the database. The first locked start takes `$APP_PASSWORD` if it is set, and otherwise invents one and prints it to the log once, in a plain ASCII banner you can read back off `docker compose logs`.
|
The password is never in the config file — only ever an scrypt hash in the database. The first locked start takes `$APP_PASSWORD` if it is set, and otherwise invents one and prints it to the log once, in a plain ASCII banner you can read back off `docker compose logs`.
|
||||||
|
|
||||||
`APP_PASSWORD` is authoritative while it is set: the Account screen says so and declines to change it. That is the way back in when a login is lost — set it, get in, empty it, recreate the container, then change the password in the app. A variable that silently stopped applying the moment you changed the password in the browser would be worse than either rule.
|
`APP_PASSWORD` is authoritative while it is set: the Account screen says so and declines to change it. That is the way back in when a login is lost — set it, get in, empty it, recreate the container, then change the password in the app. A variable that silently stopped applying the moment you changed the password in the browser would be worse than either rule. When it replaces a stored password, the start closes every session, since whoever had the old password may be signed in, and logs at WARNING how many API tokens and passkeys remain; review them under Account, or tick the revoke box on the next password change.
|
||||||
|
|
||||||
## What the app has to provide
|
## What the app has to provide
|
||||||
|
|
||||||
Almost nothing, and that is the point.
|
Almost nothing, and that is the point.
|
||||||
|
|
||||||
- A SQLite path. `AuthStore` creates four `auth_*` tables in whatever file it is given, so it can share the app's own.
|
- A SQLite path. `AuthStore` creates five `auth_*` tables in whatever file it is given, so it can share the app's own.
|
||||||
- The bind host, for `enabled: auto`.
|
- The bind host, for `enabled: auto`.
|
||||||
- Nothing for the open list, as a rule. `/login`, `/login/`, `/_auth/` and `/health` are opened by this package; the app adds `/static/` if it serves one, or the login page arrives unstyled. Everything else stays shut — a module's own scripts and sockets included, so in this repository `/_jobs/jobs.js` and `/ws/jobs` need a session like the pages that load them.
|
- Nothing for the open list, as a rule. `/login`, `/login/`, `/_auth/` and `/health` are opened by this package; the app adds `/static/` if it serves one, or the login page arrives unstyled. Everything else stays shut — a module's own scripts and sockets included, so in this repository `/_jobs/jobs.js` and `/ws/jobs` need a session like the pages that load them.
|
||||||
- An error handler that keeps `{"detail": "..."}` for `/api`, `/ws` and `/_auth`, if it has one. Every refusal here is already that shape.
|
- An error handler that keeps `{"detail": "..."}` for `/api`, `/ws` and `/_auth`, if it has one. Every refusal here is already that shape.
|
||||||
|
- `app.state.bus`, if it holds sockets open: an object with a `sockets` tuple and an async `close(sockets, code, reason)`, which is what `pipeline.web.connect` puts there. Without one, a socket stays open after its credential ends until the tab reconnects.
|
||||||
|
|
||||||
What it must **not** do is protect its own routes. Everything is shut except the open list, which is four entries long and asserted in a test — so a router added next month is safe because it exists, not because somebody remembered.
|
What it must **not** do is protect its own routes. Everything is shut except the open list, which is four entries long and asserted in a test — so a router added next month is safe because it exists, not because somebody remembered.
|
||||||
|
|
||||||
## The decisions, in one paragraph each
|
## The decisions, in one paragraph each
|
||||||
|
|
||||||
|
**One door, in front of everything.** The door is one middleware, installed once by `connect`, in front of every route and the socket. A route checks no credential itself, and the only exceptions are the open list. A route that needs the password again, such as a restore whose body is the database, calls `auth.confirm_password(request, current)`, which is counted and gated like the front door; when the body is a file, it calls `auth.confirm_current(request)` instead, which reads `current` from an `X-Current-Password` header before the body, and answers 403 without counting when the header is missing. A socket is checked at its handshake and closed with 1008 when its credential ends: on a revoke by `close_ended_sockets`, which `/logout`, the password change and the session and token revokes run after their answer, and on expiry by `sweep_ended_sockets`, which `connect` runs on a locked app every `SWEEP` (60) seconds.
|
||||||
|
|
||||||
**scrypt, not SHA-256.** Memory-hard rather than merely slow: one guess costs 16 MB, which is what takes a stolen database file out of range of a GPU. A wrong answer costs a full derivation too, so the delay an attacker meets is the one the honest user meets.
|
**scrypt, not SHA-256.** Memory-hard rather than merely slow: one guess costs 16 MB, which is what takes a stolen database file out of range of a GPU. A wrong answer costs a full derivation too, so the delay an attacker meets is the one the honest user meets.
|
||||||
|
|
||||||
**A session is 32 random bytes, and the table holds only its SHA-256.** No signing key to generate, store, keep out of backups and rotate; signing out everywhere is one `DELETE`, which no self-contained token can offer without keeping exactly this state.
|
**A session is 32 random bytes, and the table holds only its SHA-256.** No signing key to generate, store, keep out of backups and rotate; signing out everywhere is one `DELETE`, which no self-contained token can offer without keeping exactly this state.
|
||||||
|
|
||||||
**The lockout is counted in the file and read before the check.** A counter on the process is a free reset for whoever is guessing, on a server that reloads. Read after instead of before, and hammering the button extends your own lockout — which punishes the owner far harder than the attacker who is not watching. Five wrong answers are free, then 30 s doubling to a quarter of an hour, with the exponent capped so a bot cannot turn arithmetic into a 500.
|
**The lockout is counted in the file and claimed before the check.** A counter on the process is a free reset for whoever is guessing, on a server that reloads. Each guess is claimed before its derivation, in one `BEGIN IMMEDIATE` that reads `locked_until` and counts the attempt, and a right answer clears the count afterwards; read before the derivation and written after it, thirty concurrent guesses all found the door open. A claim refused while locked counts nothing, so hammering the button does not extend your own lockout, which would punish the owner far harder than the attacker who is not watching. Five wrong answers are free, then 30 s doubling to a quarter of an hour, with the exponent capped so a bot cannot turn arithmetic into a 500.
|
||||||
|
|
||||||
**One counter, not one per address.** Behind a proxy the only address the app sees is one the caller wrote. Keyed on that, an attacker gets a fresh counter per request while the owner still gets locked out by their own typos. So the address is logged — one WARNING per refusal, `auth.router: wrong password from <addr>`, stripped to the characters an address is made of — and decided from at the layer that knows who connected. `docs/password-protection.md` has the nginx and fail2ban half.
|
**One counter, not one per address.** Behind a proxy the only address the app sees is one the caller wrote. Keyed on that, an attacker gets a fresh counter per request while the owner still gets locked out by their own typos. So the address is logged — one WARNING per refusal, `auth.middleware: wrong password from <addr>`, stripped to the characters an address is made of — and decided from at the layer that knows who connected. The address is the rightmost hop of `X-Forwarded-For`, which the proxy appended, else `X-Real-IP`, else the peer; the hops left of it are the caller's own writing. `documentation/Password Protection.md` has the nginx and fail2ban half.
|
||||||
|
|
||||||
**No CSRF token.** `SameSite=Lax` refuses to send this cookie on any cross-site request that is not a top-level navigation, and no `GET` here changes anything, so the attack a token would stop cannot be mounted. A second secret to rotate, and a second way for a request to fail, buys nothing.
|
**No CSRF token.** `SameSite=Lax` refuses to send this cookie on any cross-site request that is not a top-level navigation, and no `GET` here changes anything, so the attack a token would stop cannot be mounted. A second secret to rotate, and a second way for a request to fail, buys nothing.
|
||||||
|
|
||||||
**Passkeys need a real hostname.** A passkey is bound to an origin, and that binding is what makes it unphishable — so it needs HTTPS, with `localhost` the one exception, which is what lets it be tried before deploying. Behind a proxy, set `rp_id` and `origin` explicitly or every ceremony fails as an origin mismatch. Changing `rp_id` later invalidates every key registered against the old one. The password never goes away, which is why revoking a key can never lock you out.
|
**Passkeys need a real hostname.** A passkey is bound to an origin, and that binding is what makes it unphishable — so it needs HTTPS, with `localhost` the one exception, which is what lets it be tried before deploying. Behind a proxy, set `rp_id` and `origin` explicitly or every ceremony fails as an origin mismatch. Changing `rp_id` later invalidates every key registered against the old one. The password never goes away, which is why revoking a key can never lock you out.
|
||||||
|
|
||||||
|
**A passkey is added only with the password.** A session is not enough: a passkey stays valid after every session ends and after a password change, so a borrowed signed-in browser must not be able to add one. `POST /api/account/passkeys/options` takes `current`, checked behind the same lockout as the front door. An app with no password yet (loopback, never locked) registers without it. Changing the password leaves the passkeys registered; revoke any you do not recognise.
|
||||||
|
|
||||||
|
**An API token is a session for a script.** `POST /api/account/tokens` takes the password like a passkey does, and answers with the token once; the table keeps its SHA-256, a label, the first eight characters so the owner can tell the rows apart, and an optional expiry. A script sends it as `Authorization: Bearer <token>` and it is looked up on every request, so revoking the row takes effect on the next call. It opens everything a session opens except `/api/account`, `/logout` and what the app names in `connect(session_only=...)`, which answer 403 to a token: a token that leaked cannot mint another, revoke the owner's sessions or read the list of ways in. A route that downloads or restores the database belongs on that list, because the file holds the password hash and the token rows, and a token that could fetch or swap it would hold the account after all. A token the store does not know is a 401 and a WARNING line like a wrong password, and is not counted against the lockout, since 256 random bits are not guessed and the counter would only hand a stranger a way to lock the owner out. A password change leaves tokens in place, as it does passkeys, unless `revoke_all` is sent with it, which deletes every token and passkey; the answer counts the sessions closed and the tokens and passkeys revoked and left.
|
||||||
|
|
||||||
|
**Another site's page changes nothing.** `SameSite=Lax` covers a locked app, and an unlocked one on loopback has no cookie to withhold: a page open in the same browser could `POST` a `text/plain` body to any route that takes none, and open `/ws/jobs`, which CORS does not cover. So the middleware refuses a request that changes something when `Sec-Fetch-Site`, or `Origin` against `Host` or `auth.origin`, says it came from another site, refuses a body of a type a page can send without a preflight (the API is JSON), and refuses a socket handshake from another origin. A read is never refused, and a request with no `Origin` at all (`curl`, a script with a token) passes. The comparison is against the request's own `Host`, which a DNS rebinding page matches, so an app `auto` left open also answers 421 to any request whose `Host` is not a loopback name, a `.localhost` name or the host of `auth.origin`, whatever other headers it carries: a loopback app is reached as `localhost`, and nothing else should be naming it. A test client of an unlocked app therefore names one, e.g. `TestClient(app, base_url="http://127.0.0.1")`.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
+55
-16
@@ -9,27 +9,30 @@ That is the middleware in front of everything (the WebSocket included), the
|
|||||||
login page, the account API, the `<auth-account>` element, and the startup
|
login page, the account API, the `<auth-account>` element, and the startup
|
||||||
bootstrap that makes sure there is a password at all. No user management: there
|
bootstrap that makes sure there is a password at all. No user management: there
|
||||||
is one user, they are the person who deployed it, and the only questions are
|
is one user, they are the person who deployed it, and the only questions are
|
||||||
whether the door is locked and what opens it.
|
whether the door is locked and what opens it: the password, a passkey, or an
|
||||||
|
API token carried by a script as `Authorization: Bearer`.
|
||||||
|
|
||||||
| module | |
|
| module | |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `config.py` | `AuthConfig`: the `auth:` block of the app's `config.yaml` |
|
| `config.py` | `AuthConfig`: the `auth:` block of the app's `config.yaml` |
|
||||||
| `store.py` | `AuthStore`: scrypt, the lockout, sessions, passkeys, four `auth_*` tables in whatever SQLite file it is given |
|
| `store.py` | `AuthStore`: scrypt, the lockout, sessions, passkeys, API tokens, five `auth_*` tables in whatever SQLite file it is given |
|
||||||
| `models.py` | `Account`, `SessionInfo`, `Passkey` — what the account screen is told, which carries no credential |
|
| `models.py` | `Account`, `SessionInfo`, `Passkey`, `ApiToken` — what the account screen is told, which carries no credential |
|
||||||
| `middleware.py` | `RequireLogin`, `locked()`, the open list, the cookie |
|
| `middleware.py` | `RequireLogin`, `locked()`, the open list, the cookie, the bearer header, the same-site rule, `confirm_password`, `confirm_current` and `warn`, and `close_ended_sockets` with its sweep |
|
||||||
| `passkeys.py` | the two WebAuthn ceremonies; `webauthn` is an optional extra, imported lazily |
|
| `passkeys.py` | the two WebAuthn ceremonies; `webauthn` is an optional extra, imported lazily |
|
||||||
| `schemas.py` | the five request bodies |
|
| `schemas.py` | the seven request bodies |
|
||||||
| `router.py` | `/login`, `/logout`, `/api/account/*`, the passkey routes, `/_auth/*.js` |
|
| `router.py` | `/login`, `/logout`, `/api/account/*`, the passkey and token routes, `/_auth/*.js` |
|
||||||
| `templates/login.html` | the one page served without a session |
|
| `templates/login.html` | the one page served without a session |
|
||||||
| `static/` | `passkey.js` (the browser half of WebAuthn) and `account.js` (`<auth-account>`) |
|
| `static/` | `passkey.js` (the browser half of WebAuthn) and `account.js` (`<auth-account>`) |
|
||||||
|
|
||||||
The reasoning is `docs/password-protection.md`, and this package is that
|
The reasoning is the boilerplate's `documentation/Password Protection.md`, and this package is that
|
||||||
document in code. `auth/README.md` is how to copy the folder into a project
|
document in code. `auth/README.md` is how to copy the folder into a project
|
||||||
that has never heard of this one.
|
that has never heard of this one.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import contextlib
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import secrets
|
import secrets
|
||||||
@@ -41,23 +44,29 @@ from fastapi import FastAPI
|
|||||||
|
|
||||||
from auth import passkeys
|
from auth import passkeys
|
||||||
from auth.config import AuthConfig
|
from auth.config import AuthConfig
|
||||||
from auth.middleware import OPEN, RequireLogin, client_address, locked, token_from
|
from auth.middleware import (
|
||||||
from auth.models import Account, Passkey, SessionInfo
|
ACCOUNT, CURRENT_HEADER, OPEN, RequireLogin, bearer_from, client_address,
|
||||||
|
close_ended_sockets, confirm_current, confirm_password, credential_live, locked,
|
||||||
|
sweep_ended_sockets, token_from, warn,
|
||||||
|
)
|
||||||
|
from auth.models import Account, ApiToken, Passkey, SessionInfo
|
||||||
from auth.router import router
|
from auth.router import router
|
||||||
from auth.store import AuthStore
|
from auth.store import AuthStore
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"OPEN", "Account", "AuthConfig", "AuthStore", "Passkey", "RequireLogin",
|
"ACCOUNT", "CURRENT_HEADER", "OPEN", "Account", "ApiToken", "AuthConfig", "AuthStore",
|
||||||
"SessionInfo", "bootstrap", "client_address", "connect", "locked",
|
"Passkey", "RequireLogin", "SessionInfo", "bearer_from", "bootstrap", "client_address",
|
||||||
"passkeys", "router", "token_from",
|
"close_ended_sockets", "confirm_current", "confirm_password", "connect",
|
||||||
|
"credential_live", "locked", "passkeys", "router", "sweep_ended_sockets", "token_from",
|
||||||
|
"warn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def connect(app: FastAPI, store: AuthStore, config: AuthConfig, *,
|
def connect(app: FastAPI, store: AuthStore, config: AuthConfig, *,
|
||||||
open: Iterable[str] = (), stylesheet: str | None = None,
|
open: Iterable[str] = (), session_only: Iterable[str] = (),
|
||||||
host: str = "") -> AuthStore:
|
stylesheet: str | None = None, host: str = "") -> AuthStore:
|
||||||
"""Put `app` behind a password: middleware, routes, element, bootstrap.
|
"""Put `app` behind a password: middleware, routes, element, bootstrap.
|
||||||
|
|
||||||
Call it first, before the app's own routers. Starlette wraps the whole
|
Call it first, before the app's own routers. Starlette wraps the whole
|
||||||
@@ -72,13 +81,17 @@ def connect(app: FastAPI, store: AuthStore, config: AuthConfig, *,
|
|||||||
| keyword | |
|
| keyword | |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `open` | extra paths reachable without a session, added to `middleware.OPEN`. A name ending in `/` is a prefix; any other is matched whole. `("/static/",)` is what an app with a stylesheet passes. |
|
| `open` | extra paths reachable without a session, added to `middleware.OPEN`. A name ending in `/` is a prefix; any other is matched whole. `("/static/",)` is what an app with a stylesheet passes. |
|
||||||
|
| `session_only` | extra paths an API token may not reach, added to `middleware.ACCOUNT`, matched the same way. Anything that reads or replaces the database as a file belongs here: it holds the password hash and the tokens, so a token that could fetch or swap it would hold the account. A route listed here should also take the current password; a route that hands out or replaces the database as a file takes it in an `X-Current-Password` header, through `auth.confirm_current`, and any other through `auth.confirm_password`. |
|
||||||
| `stylesheet` | a URL the login page links, so it looks like the rest of the app. Optional: the page carries enough inline CSS to be presentable on its own. |
|
| `stylesheet` | a URL the login page links, so it looks like the rest of the app. Optional: the page carries enough inline CSS to be presentable on its own. |
|
||||||
| `host` | the address the server binds — `config.server.host`. What `enabled: auto` decides from, and deliberately not taken from a request, since a `Host` header is written by whoever is talking. |
|
| `host` | the address the server binds — `config.server.host`. What `enabled: auto` decides from, and deliberately not taken from a request, since a `Host` header is written by whoever is talking. |
|
||||||
|
|
||||||
The lifespan is wrapped rather than replaced, the same way
|
The lifespan is wrapped rather than replaced, the same way
|
||||||
`pipeline.web.connect` does it, so an app's own lifespan still runs inside
|
`pipeline.web.connect` does it, so an app's own lifespan still runs inside
|
||||||
this one: the password exists before the app's startup code, which may want
|
this one: the password exists before the app's startup code, which may want
|
||||||
to say something about it.
|
to say something about it. On a locked app it also runs
|
||||||
|
`sweep_ended_sockets` for as long as the app is up, as the task on
|
||||||
|
`app.state.auth_sweep` (None on an unlocked app), so a socket whose session
|
||||||
|
or token expires is closed within `middleware.SWEEP` seconds.
|
||||||
|
|
||||||
Returns the store, for the app to keep if it wants one.
|
Returns the store, for the app to keep if it wants one.
|
||||||
"""
|
"""
|
||||||
@@ -88,10 +101,12 @@ def connect(app: FastAPI, store: AuthStore, config: AuthConfig, *,
|
|||||||
#: the bind address, which only the caller knows.
|
#: the bind address, which only the caller knows.
|
||||||
app.state.auth_locked = locked(config, host)
|
app.state.auth_locked = locked(config, host)
|
||||||
app.state.auth_stylesheet = stylesheet or ""
|
app.state.auth_stylesheet = stylesheet or ""
|
||||||
|
#: The `sweep_ended_sockets` task while a locked app is up; see below.
|
||||||
|
app.state.auth_sweep = None
|
||||||
|
|
||||||
app.include_router(router)
|
app.include_router(router)
|
||||||
app.add_middleware(RequireLogin, store=store, config=config,
|
app.add_middleware(RequireLogin, store=store, config=config,
|
||||||
open=tuple(open), host=host)
|
open=tuple(open), host=host, session_only=tuple(session_only))
|
||||||
|
|
||||||
inner = app.router.lifespan_context
|
inner = app.router.lifespan_context
|
||||||
|
|
||||||
@@ -104,8 +119,18 @@ def connect(app: FastAPI, store: AuthStore, config: AuthConfig, *,
|
|||||||
# a loopback app before it is ever deployed, which is the order this
|
# a loopback app before it is ever deployed, which is the order this
|
||||||
# encourages, so an unlocked one can still have keys it cannot use.
|
# encourages, so an unlocked one can still have keys it cannot use.
|
||||||
passkeys.warn_if_missing(store)
|
passkeys.warn_if_missing(store)
|
||||||
|
# Expiry ends no request, so no route closes the socket it ended.
|
||||||
|
sweep = (asyncio.create_task(sweep_ended_sockets(app))
|
||||||
|
if app.state.auth_locked else None)
|
||||||
|
app.state.auth_sweep = sweep
|
||||||
|
try:
|
||||||
async with inner(this) as state:
|
async with inner(this) as state:
|
||||||
yield state
|
yield state
|
||||||
|
finally:
|
||||||
|
if sweep is not None:
|
||||||
|
sweep.cancel()
|
||||||
|
with contextlib.suppress(asyncio.CancelledError):
|
||||||
|
await sweep
|
||||||
|
|
||||||
app.router.lifespan_context = lifespan
|
app.router.lifespan_context = lifespan
|
||||||
return store
|
return store
|
||||||
@@ -124,13 +149,27 @@ def bootstrap(store: AuthStore, config: AuthConfig) -> None:
|
|||||||
|
|
||||||
Re-derived only when the stored hash does not already match, so a restart is
|
Re-derived only when the stored hash does not already match, so a restart is
|
||||||
not a write and "password last changed" keeps meaning something.
|
not a write and "password last changed" keeps meaning something.
|
||||||
|
|
||||||
|
When the variable replaces a password that was there, every session is
|
||||||
|
closed: that is the recovery, and whoever had the old password may be
|
||||||
|
signed in. API tokens and passkeys stay, since the environment cannot say
|
||||||
|
which of them are the owner's, and their counts are logged at WARNING so
|
||||||
|
the owner knows to review them under Account.
|
||||||
"""
|
"""
|
||||||
pinned = os.environ.get(config.password_env, "")
|
pinned = os.environ.get(config.password_env, "")
|
||||||
if pinned:
|
if pinned:
|
||||||
if not store.check_password(pinned):
|
if not store.check_password(pinned):
|
||||||
|
replaced = store.has_password()
|
||||||
store.set_password(pinned)
|
store.set_password(pinned)
|
||||||
|
if not replaced:
|
||||||
log.info("password set from %s", config.password_env)
|
log.info("password set from %s", config.password_env)
|
||||||
return
|
return
|
||||||
|
closed = store.close_all()
|
||||||
|
tokens, keys = store.counts()
|
||||||
|
log.warning("password reset from %s: closed %d sessions; %d API tokens and "
|
||||||
|
"%d passkeys remain, review them under Account",
|
||||||
|
config.password_env, closed, tokens, keys)
|
||||||
|
return
|
||||||
|
|
||||||
if store.has_password():
|
if store.has_password():
|
||||||
return
|
return
|
||||||
|
|||||||
+8
-8
@@ -22,7 +22,7 @@ from pydantic import BaseModel, ConfigDict, Field
|
|||||||
|
|
||||||
|
|
||||||
class AuthConfig(BaseModel):
|
class AuthConfig(BaseModel):
|
||||||
"""The `auth:` block of `config.yaml`. See docs/password-protection.md."""
|
"""The `auth:` block of `config.yaml`. See documentation/Password Protection.md."""
|
||||||
|
|
||||||
#: Unknown keys are refused rather than ignored, the same rule the app's
|
#: Unknown keys are refused rather than ignored, the same rule the app's
|
||||||
#: own sections keep: a setting you believe you have changed and have not
|
#: own sections keep: a setting you believe you have changed and have not
|
||||||
@@ -38,9 +38,9 @@ class AuthConfig(BaseModel):
|
|||||||
#: Deciding from the bind address rather than defaulting to off is the
|
#: Deciding from the bind address rather than defaulting to off is the
|
||||||
#: whole point. The failure this prevents is the one that actually happens:
|
#: whole point. The failure this prevents is the one that actually happens:
|
||||||
#: you deploy, it works, and nothing anywhere tells you the door is open.
|
#: you deploy, it works, and nothing anywhere tells you the door is open.
|
||||||
#: `always` locks it on localhost too and is what a deployment behind a
|
#: `always` locks it on localhost too and is what a deployment sets, so the
|
||||||
#: TLS-terminating proxy needs, since there every request looks local to
|
#: lock does not depend on the bind address or on how the process was
|
||||||
#: the app and is not. `never` is for putting some other authentication in
|
#: started. `never` is for putting some other authentication in
|
||||||
#: front of this one, and means exactly what it says.
|
#: front of this one, and means exactly what it says.
|
||||||
enabled: Literal["auto", "always", "never"] = "auto"
|
enabled: Literal["auto", "always", "never"] = "auto"
|
||||||
|
|
||||||
@@ -52,10 +52,10 @@ class AuthConfig(BaseModel):
|
|||||||
|
|
||||||
#: Whether the session cookie is marked `Secure`, so a browser will only
|
#: Whether the session cookie is marked `Secure`, so a browser will only
|
||||||
#: ever send it back over HTTPS. `auto` sets it when the request itself
|
#: ever send it back over HTTPS. `auto` sets it when the request itself
|
||||||
#: arrived over HTTPS, which is right when the app terminates TLS and wrong
|
#: arrived over HTTPS, which behind a reverse proxy that terminates TLS
|
||||||
#: behind a reverse proxy that terminates it for you -- there the app sees
|
#: holds only while the server trusts `X-Forwarded-Proto` from it; the app
|
||||||
#: plain HTTP and `auto` reads as "no". Set `always` when something else is
|
#: alone cannot tell. Set `always` when something else is doing TLS.
|
||||||
#: doing TLS. `never` exists for testing over http on a LAN and is the
|
#: `never` exists for testing over http on a LAN and is the
|
||||||
#: setting to be uncomfortable with.
|
#: setting to be uncomfortable with.
|
||||||
secure_cookie: Literal["auto", "always", "never"] = "auto"
|
secure_cookie: Literal["auto", "always", "never"] = "auto"
|
||||||
|
|
||||||
|
|||||||
+509
-26
@@ -1,8 +1,10 @@
|
|||||||
"""The lock on the door: is there a session, and what happens when there isn't.
|
"""The lock on the door: is there a session, and what happens when there isn't.
|
||||||
|
|
||||||
Three things live here — when the door is locked at all, who gets past it, and
|
Six things live here — when the door is locked at all, who gets past it, the
|
||||||
the cookie that says so. The password itself is `auth/store.py`; this module
|
cookie and the bearer header that say so, the rule that another site's page
|
||||||
never sees a hash.
|
gets nothing changed, the password asked again from behind a session
|
||||||
|
(`confirm_password`, and `confirm_current` for a header), and the closing of a socket whose credential has ended.
|
||||||
|
The password itself is `auth/store.py`; this module never sees a hash.
|
||||||
|
|
||||||
## Why a middleware and not a dependency
|
## Why a middleware and not a dependency
|
||||||
|
|
||||||
@@ -30,6 +32,66 @@ password is a second lock on it; bound to anything else, the app is on a
|
|||||||
network and needs its own. The alternative — default off, switch it on when you
|
network and needs its own. The alternative — default off, switch it on when you
|
||||||
deploy — fails silently in the exact case it exists for, because an app serving
|
deploy — fails silently in the exact case it exists for, because an app serving
|
||||||
the internet with no password looks and behaves precisely like one that is fine.
|
the internet with no password looks and behaves precisely like one that is fine.
|
||||||
|
A reverse proxy on the same machine defeats the bind address, so an app `auto`
|
||||||
|
left open refuses any request a proxy forwarded; see
|
||||||
|
`RequireLogin._refuse_proxied`.
|
||||||
|
|
||||||
|
## Three credentials, two carriers
|
||||||
|
|
||||||
|
A session cookie is what the password and a passkey open, and it is what a
|
||||||
|
browser carries. An API token (`AuthStore.create_token`) is what a script
|
||||||
|
carries, as `Authorization: Bearer <token>`, and it is looked up on every
|
||||||
|
request the way a session is, so revoking it takes effect on the next call.
|
||||||
|
The cookie is read first and wins; the header is read only when there is no
|
||||||
|
live session. A token opens every door the session does except the account:
|
||||||
|
a request carrying one for `/api/account` or `/logout` is answered 403
|
||||||
|
(`ACCOUNT`), so a token that leaked cannot mint another, revoke the owner's
|
||||||
|
sessions or read the list of ways in. An app adds its own session-only routes
|
||||||
|
through `connect(session_only=...)`: anything that reads or replaces the
|
||||||
|
database as a file belongs there, since the file holds the password hash and
|
||||||
|
these rows, and a token that could fetch or swap it would hold the account
|
||||||
|
after all. A token that is not live is a 401 and a
|
||||||
|
WARNING line, like a wrong password; it is not counted against the lockout,
|
||||||
|
since 256 random bits are not guessed and the counter would only hand a
|
||||||
|
stranger a way to lock the owner out.
|
||||||
|
|
||||||
|
## Another site's page
|
||||||
|
|
||||||
|
`SameSite=Lax` keeps the session cookie off a cross-site `POST`, which covers
|
||||||
|
a locked app. An unlocked one on loopback has no cookie to withhold, so a page
|
||||||
|
open in the same browser could `POST` to `http://127.0.0.1:8787/api/...` with
|
||||||
|
a `text/plain` body, which needs no preflight, and every route that takes no
|
||||||
|
body acted on it; a WebSocket handshake is not subject to CORS at all, so the
|
||||||
|
same page could open `/ws/jobs` and read every job and notice. So a request
|
||||||
|
that changes something is refused when the browser says it came from another
|
||||||
|
site (`Sec-Fetch-Site`, else `Origin` against `Host` or `AuthConfig.origin`),
|
||||||
|
and a body of a type another site could send without a preflight is refused
|
||||||
|
whoever sent it: the API is JSON. Locked or not, because the rule costs two
|
||||||
|
header reads and a locked app with `secure_cookie: never` on a LAN is a real
|
||||||
|
deployment. See `cross_site`.
|
||||||
|
|
||||||
|
That comparison is against the request's own `Host`, and a page served from
|
||||||
|
`rebind.example` whose name is pointed at 127.0.0.1 arrives with a `Host` and
|
||||||
|
an `Origin` that agree. So an app `auto` left open also refuses any request
|
||||||
|
whose `Host` is not a loopback name, a `.localhost` name or the host of
|
||||||
|
`AuthConfig.origin`, with the same 421 as a forwarded one, whatever else it
|
||||||
|
carries: a browser that sends no fetch metadata and no `Origin` on a read is
|
||||||
|
still a rebound page. A loopback app is reached as `localhost` or
|
||||||
|
`127.0.0.1`, and a request that names another host for it came through a
|
||||||
|
proxy or a rebinding, neither of which the bind address can see. A test
|
||||||
|
client names the host it is given (`base_url="http://127.0.0.1"`).
|
||||||
|
|
||||||
|
## A socket outlives its handshake
|
||||||
|
|
||||||
|
The door is checked when a socket opens, and the socket stays open for the
|
||||||
|
life of the tab. So a route that ends a credential (`/logout`, a password
|
||||||
|
change, a revoked session or token) runs `close_ended_sockets` after its
|
||||||
|
answer, which asks the store about every open socket's cookie or bearer header
|
||||||
|
and closes the ones it no longer knows with 1008. Expiry ends no request, so
|
||||||
|
`connect` also runs `sweep_ended_sockets`, which does the same every `SWEEP`
|
||||||
|
seconds. A tab's reconnect then meets the door. The sockets are the app's:
|
||||||
|
`app.state.bus`, which `pipeline.web.connect` sets, and an app without one has
|
||||||
|
nothing to close.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -37,11 +99,11 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import logging
|
import logging
|
||||||
from http.cookies import SimpleCookie
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote, urlsplit
|
||||||
|
|
||||||
from starlette.requests import HTTPConnection
|
from fastapi import HTTPException
|
||||||
|
from starlette.requests import HTTPConnection, cookie_parser
|
||||||
from starlette.responses import JSONResponse, RedirectResponse, Response
|
from starlette.responses import JSONResponse, RedirectResponse, Response
|
||||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||||
|
|
||||||
@@ -61,6 +123,22 @@ log = logging.getLogger(__name__)
|
|||||||
#: against the path only — a query string cannot widen this.
|
#: against the path only — a query string cannot widen this.
|
||||||
OPEN: tuple[str, ...] = ("/login", "/login/", "/_auth/", "/health")
|
OPEN: tuple[str, ...] = ("/login", "/login/", "/_auth/", "/health")
|
||||||
|
|
||||||
|
#: What a bearer token cannot reach: the account, which is managed from a
|
||||||
|
#: browser with a session, and the sign-out of a session a token has not got.
|
||||||
|
#: Matched as `OPEN` is. Everything else a session opens, a token opens.
|
||||||
|
ACCOUNT: tuple[str, ...] = ("/api/account", "/api/account/", "/logout")
|
||||||
|
|
||||||
|
#: What a request carrying a token for the account is told.
|
||||||
|
TOKEN_NOT_HERE = ("An API token opens the API, not the account. Sign in with the "
|
||||||
|
"password or a passkey to manage it.")
|
||||||
|
|
||||||
|
|
||||||
|
def matches(path: str, names: tuple[str, ...]) -> bool:
|
||||||
|
"""Whether `path` is one of `names`: a name ending in `/` is a prefix, any
|
||||||
|
other is matched whole. The one rule for `OPEN` and `ACCOUNT`."""
|
||||||
|
return any(path == name.rstrip("/") or (name.endswith("/") and path.startswith(name))
|
||||||
|
for name in names)
|
||||||
|
|
||||||
|
|
||||||
def locked(config: AuthConfig, host: str) -> bool:
|
def locked(config: AuthConfig, host: str) -> bool:
|
||||||
"""Whether this app should ask for a password. See `AuthConfig.enabled`.
|
"""Whether this app should ask for a password. See `AuthConfig.enabled`.
|
||||||
@@ -126,47 +204,113 @@ def token_from(conn: HTTPConnection | Scope, config: AuthConfig) -> str:
|
|||||||
|
|
||||||
Both cookie names are looked for, so flipping `secure_cookie` or moving
|
Both cookie names are looked for, so flipping `secure_cookie` or moving
|
||||||
behind TLS does not strand a live session under the other name.
|
behind TLS does not strand a live session under the other name.
|
||||||
|
|
||||||
|
Parsed per `;`-separated pair by Starlette's `cookie_parser`, as a browser
|
||||||
|
does, and not by `http.cookies.SimpleCookie`, whose `load` returns an
|
||||||
|
empty jar for the whole header when one cookie in it has a space or JSON
|
||||||
|
in its value: a sibling app on the parent domain setting `theme=dark mode`
|
||||||
|
signed the owner out on every request. A pair that is not a cookie is
|
||||||
|
skipped, so a garbage header is no cookie rather than a 500.
|
||||||
"""
|
"""
|
||||||
scope: Any = conn.scope if isinstance(conn, HTTPConnection) else conn
|
scope: Any = conn.scope if isinstance(conn, HTTPConnection) else conn
|
||||||
for key, value in scope.get("headers", ()):
|
for key, value in scope.get("headers", ()):
|
||||||
if key == b"cookie":
|
if key == b"cookie":
|
||||||
jar = SimpleCookie()
|
jar = cookie_parser(value.decode("latin-1"))
|
||||||
try:
|
|
||||||
jar.load(value.decode("latin-1"))
|
|
||||||
except Exception: # a cookie header that is not one
|
|
||||||
return ""
|
|
||||||
for name in cookie_names(config):
|
for name in cookie_names(config):
|
||||||
if name in jar:
|
if jar.get(name):
|
||||||
return jar[name].value
|
return jar[name]
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def bearer_from(conn: HTTPConnection | Scope) -> str:
|
||||||
|
"""The API token this request carries in `Authorization: Bearer`, or "".
|
||||||
|
|
||||||
|
Only that header and only that scheme: a token in a query string would
|
||||||
|
land in every access log between the script and the app, and `Basic` is
|
||||||
|
a password, which has its own door.
|
||||||
|
"""
|
||||||
|
value = _header(conn, b"authorization")
|
||||||
|
scheme, _, token = value.strip().partition(" ")
|
||||||
|
if scheme.lower() != "bearer":
|
||||||
|
return ""
|
||||||
|
return token.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _header(conn: HTTPConnection | Scope, name: bytes) -> str:
|
||||||
|
scope: Any = conn.scope if isinstance(conn, HTTPConnection) else conn
|
||||||
|
for key, value in scope.get("headers", ()):
|
||||||
|
if key == name:
|
||||||
|
return value.decode("latin-1")
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def credential_live(store: AuthStore, config: AuthConfig,
|
||||||
|
conn: HTTPConnection | Scope) -> bool:
|
||||||
|
"""Whether what this request carries still opens the door: its cookie's
|
||||||
|
session, else its bearer token. What `RequireLogin` decides from, asked
|
||||||
|
again of an open socket by `close_ended_sockets`."""
|
||||||
|
token = token_from(conn, config)
|
||||||
|
if token and store.touch(token) is not None:
|
||||||
|
return True
|
||||||
|
bearer = bearer_from(conn)
|
||||||
|
return bool(bearer) and store.touch_token(bearer) is not None
|
||||||
|
|
||||||
|
|
||||||
def client_address(conn: HTTPConnection | Scope) -> str:
|
def client_address(conn: HTTPConnection | Scope) -> str:
|
||||||
"""Who is asking, as well as it can be known, for the log to say so.
|
"""Who is asking, as well as it can be known, for the log to say so.
|
||||||
|
|
||||||
Behind a reverse proxy the peer is a container on the same Docker network
|
Behind a reverse proxy the peer is a container on the same Docker network
|
||||||
and says nothing about who is at the other end; the address worth writing
|
and says nothing about who is at the other end. The address worth writing
|
||||||
down is the first hop of `X-Forwarded-For`, which the proxy sets. Both are
|
down is the rightmost hop of `X-Forwarded-For`, the one the proxy in front
|
||||||
reported when they differ, because the header is only as trustworthy as
|
of this app appended for the connection it accepted; every hop left of it
|
||||||
whoever sent it — which is why nothing is *decided* from this, only logged.
|
was in the request when it arrived and is whatever the caller wrote. nginx's
|
||||||
See `AuthStore.check_password` for that argument in full.
|
`$proxy_add_x_forwarded_for` appends rather than replaces, so the first hop
|
||||||
|
let an attacker name the address a fail2ban filter on this line bans.
|
||||||
|
Without that header, `X-Real-IP`, which such a proxy sets outright. The
|
||||||
|
claimed address and the peer are both reported when they differ, because a
|
||||||
|
header is only as trustworthy as the proxy that set it, which is why
|
||||||
|
nothing is *decided* from this, only logged. See
|
||||||
|
`AuthStore.check_password` for that argument in full.
|
||||||
|
|
||||||
That header is also the one piece of a stranger's own writing that ends up
|
The header is also the one piece of a stranger's writing that can end up
|
||||||
in the log, so only the characters an address is made of survive the trip:
|
in the log, so only the characters an address is made of survive the trip:
|
||||||
a line in this log is something fail2ban reads and a person believes, and
|
a line in this log is something fail2ban reads and a person believes, and
|
||||||
neither should be forgeable by whoever is knocking.
|
neither should be forgeable by whoever is knocking. The address stays the
|
||||||
|
first word after `from`, so a filter is one regex.
|
||||||
"""
|
"""
|
||||||
scope: Any = conn.scope if isinstance(conn, HTTPConnection) else conn
|
scope: Any = conn.scope if isinstance(conn, HTTPConnection) else conn
|
||||||
peer = (scope.get("client") or ("", 0))[0] or "unknown"
|
peer = (scope.get("client") or ("", 0))[0] or "unknown"
|
||||||
|
forwarded = real = ""
|
||||||
for key, value in scope.get("headers", ()):
|
for key, value in scope.get("headers", ()):
|
||||||
|
# The last of each, should a header arrive twice: a proxy that adds
|
||||||
|
# one rather than appending writes it after the caller's.
|
||||||
if key == b"x-forwarded-for":
|
if key == b"x-forwarded-for":
|
||||||
first = value.decode("latin-1").split(",")[0]
|
forwarded = value.decode("latin-1")
|
||||||
claimed = "".join(c for c in first if c.isalnum() or c in ".:%-")[:45]
|
elif key == b"x-real-ip":
|
||||||
|
real = value.decode("latin-1")
|
||||||
|
hop = forwarded.split(",")[-1] if forwarded.strip() else real
|
||||||
|
claimed = "".join(c for c in hop if c.isalnum() or c in ".:%-")[:45]
|
||||||
if claimed and claimed != peer:
|
if claimed and claimed != peer:
|
||||||
return f"{claimed} via {peer}"
|
return f"{claimed} via {peer}"
|
||||||
return peer
|
return peer
|
||||||
|
|
||||||
|
|
||||||
|
#: Headers a reverse proxy adds. Any one of them on a request to an app that
|
||||||
|
#: `auto` left open because it binds loopback means the bind address is not the
|
||||||
|
#: whole story: a proxy on this machine is passing on someone else.
|
||||||
|
FORWARDED = frozenset({b"forwarded", b"x-forwarded-for", b"x-forwarded-proto",
|
||||||
|
b"x-forwarded-host"})
|
||||||
|
|
||||||
|
#: What such a request is answered with, for whoever runs the proxy.
|
||||||
|
PROXIED = ("This app binds loopback and so asks for no password, but this request "
|
||||||
|
"came through a proxy. Set auth.enabled: always in config.yaml and restart.")
|
||||||
|
|
||||||
|
#: What a browser script's request naming another host is answered with.
|
||||||
|
REBOUND = ("This app binds loopback and so asks for no password, but this request "
|
||||||
|
"names a host that is not localhost. Open the app as http://localhost, or "
|
||||||
|
"set auth.enabled: always in config.yaml and restart.")
|
||||||
|
|
||||||
|
|
||||||
#: Prefixes whose refusal is JSON rather than a redirect. A caller that asked
|
#: Prefixes whose refusal is JSON rather than a redirect. A caller that asked
|
||||||
#: for data and is sent a login page reads it as a 200 full of HTML where its
|
#: for data and is sent a login page reads it as a 200 full of HTML where its
|
||||||
#: JSON should be; `{"detail": ...}` with a 401 is something it can act on.
|
#: JSON should be; `{"detail": ...}` with a 401 is something it can act on.
|
||||||
@@ -187,20 +331,35 @@ class RequireLogin:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, app: ASGIApp, store: AuthStore, config: AuthConfig,
|
def __init__(self, app: ASGIApp, store: AuthStore, config: AuthConfig,
|
||||||
open: tuple[str, ...] = (), host: str = "") -> None:
|
open: tuple[str, ...] = (), host: str = "",
|
||||||
|
session_only: tuple[str, ...] = ()) -> None:
|
||||||
self.app = app
|
self.app = app
|
||||||
self.store = store
|
self.store = store
|
||||||
self.config = config
|
self.config = config
|
||||||
self.open = OPEN + tuple(open)
|
self.open = OPEN + tuple(open)
|
||||||
|
#: What a token may not reach: the account, and whatever the app adds.
|
||||||
|
self.session_only = (*ACCOUNT, *session_only)
|
||||||
self.locked = locked(config, host)
|
self.locked = locked(config, host)
|
||||||
|
#: Open only because `auto` saw loopback; see `_refuse_proxied`.
|
||||||
|
self.watch_proxy = config.enabled == "auto" and not self.locked
|
||||||
|
self.warned = False
|
||||||
|
|
||||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||||
if scope["type"] not in ("http", "websocket") or not self.locked:
|
if scope["type"] not in ("http", "websocket"):
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
# Another site's page, locked or not: see the module docstring.
|
||||||
|
refused = cross_site(scope, self.config.origin)
|
||||||
|
if refused is not None:
|
||||||
|
return await self._answer(scope, receive, send, *refused)
|
||||||
|
if not self.locked:
|
||||||
|
if self.watch_proxy and _forwarded(scope):
|
||||||
|
return await self._refuse_proxied(scope, receive, send, PROXIED)
|
||||||
|
if self.watch_proxy and _rebound(scope, self.config.origin):
|
||||||
|
return await self._refuse_proxied(scope, receive, send, REBOUND)
|
||||||
return await self.app(scope, receive, send)
|
return await self.app(scope, receive, send)
|
||||||
|
|
||||||
path = scope.get("path", "")
|
path = scope.get("path", "")
|
||||||
if any(path == p.rstrip("/") or (p.endswith("/") and path.startswith(p))
|
if matches(path, self.open):
|
||||||
for p in self.open):
|
|
||||||
return await self.app(scope, receive, send)
|
return await self.app(scope, receive, send)
|
||||||
|
|
||||||
token = token_from(scope, self.config)
|
token = token_from(scope, self.config)
|
||||||
@@ -214,8 +373,34 @@ class RequireLogin:
|
|||||||
scope["auth_token"] = token
|
scope["auth_token"] = token
|
||||||
return await self.app(scope, receive, send)
|
return await self.app(scope, receive, send)
|
||||||
|
|
||||||
|
bearer = bearer_from(scope)
|
||||||
|
if bearer:
|
||||||
|
held = await asyncio.to_thread(self.store.touch_token, bearer)
|
||||||
|
if held is not None:
|
||||||
|
if matches(path, self.session_only):
|
||||||
|
return await self._answer(scope, receive, send, 403, TOKEN_NOT_HERE)
|
||||||
|
# The row, for a route that wants to know it was a script.
|
||||||
|
scope["auth_bearer"] = held.id
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
# The same line a wrong password writes, for the same reader. A
|
||||||
|
# script with a bad token gets JSON whatever it asked for: a
|
||||||
|
# redirect to the login page is for a browser, and a browser does
|
||||||
|
# not send this header.
|
||||||
|
log.warning("bad token from %s", client_address(scope))
|
||||||
|
return await self._answer(scope, receive, send, 401,
|
||||||
|
"That token is not valid here.")
|
||||||
|
|
||||||
await self._refuse(scope, receive, send, path)
|
await self._refuse(scope, receive, send, path)
|
||||||
|
|
||||||
|
async def _answer(self, scope: Scope, receive: Receive, send: Send,
|
||||||
|
status: int, detail: str) -> None:
|
||||||
|
"""A refusal in the shape the caller can act on: `{"detail"}` with the
|
||||||
|
status, or a 1008 close before the socket is accepted."""
|
||||||
|
if scope["type"] == "websocket":
|
||||||
|
await send({"type": "websocket.close", "code": 1008})
|
||||||
|
return
|
||||||
|
await JSONResponse({"detail": detail}, status_code=status)(scope, receive, send)
|
||||||
|
|
||||||
async def _refuse(self, scope: Scope, receive: Receive, send: Send,
|
async def _refuse(self, scope: Scope, receive: Receive, send: Send,
|
||||||
path: str) -> None:
|
path: str) -> None:
|
||||||
"""No session. Say so in whatever the caller can understand."""
|
"""No session. Say so in whatever the caller can understand."""
|
||||||
@@ -244,3 +429,301 @@ class RequireLogin:
|
|||||||
else:
|
else:
|
||||||
response = JSONResponse({"detail": "Not signed in"}, status_code=401)
|
response = JSONResponse({"detail": "Not signed in"}, status_code=401)
|
||||||
await response(scope, receive, send)
|
await response(scope, receive, send)
|
||||||
|
|
||||||
|
async def _refuse_proxied(self, scope: Scope, receive: Receive, send: Send,
|
||||||
|
detail: str) -> None:
|
||||||
|
"""421 for a forwarded request to an app `auto` left open, and for a
|
||||||
|
request that names a host it is not reached under (`_rebound`).
|
||||||
|
|
||||||
|
`auto` reads loopback as "this machine only", and a reverse proxy on
|
||||||
|
the same machine makes that false while every request still arrives
|
||||||
|
from 127.0.0.1. The peer cannot tell the proxy from a local browser,
|
||||||
|
only the headers the proxy adds can, so a request carrying one is
|
||||||
|
refused rather than served unlocked. A client can add the headers
|
||||||
|
itself, which only gets it refused. The Vite dev server's proxy adds
|
||||||
|
none (`xfwd` is off) and forwards `Host` unchanged, so `npm run dev`
|
||||||
|
is unaffected. Logged once at WARNING, since every request through
|
||||||
|
the proxy would repeat it.
|
||||||
|
"""
|
||||||
|
if not self.warned:
|
||||||
|
self.warned = True
|
||||||
|
log.warning("refused a proxied or rebound request to an unlocked app (%s): %s",
|
||||||
|
client_address(scope), detail)
|
||||||
|
if scope["type"] == "websocket":
|
||||||
|
await send({"type": "websocket.close", "code": 1008})
|
||||||
|
return
|
||||||
|
await JSONResponse({"detail": detail}, status_code=421)(scope, receive, send)
|
||||||
|
|
||||||
|
|
||||||
|
def _forwarded(scope: Scope) -> bool:
|
||||||
|
return any(key in FORWARDED for key, _ in scope.get("headers", ()))
|
||||||
|
|
||||||
|
|
||||||
|
def _rebound(scope: Scope, configured_origin: str) -> bool:
|
||||||
|
"""Whether a request reaches this loopback app under a name that is not
|
||||||
|
loopback: the shape of a DNS rebinding, where the page's own `Origin` and
|
||||||
|
the `Host` it sends agree and `_from_here` cannot tell.
|
||||||
|
|
||||||
|
Every request is judged, whatever headers it carries. A browser that sends
|
||||||
|
neither `Sec-Fetch-Site` nor `Origin` on a read is still a page on the
|
||||||
|
rebound name reading the API, and the only names a loopback app is reached
|
||||||
|
under are a loopback address, `localhost` and its subdomains, and the host
|
||||||
|
of `AuthConfig.origin`. A missing or unparseable `Host` names none of
|
||||||
|
them and is refused too.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
host = urlsplit("//" + _header(scope, b"host")).hostname or ""
|
||||||
|
except ValueError: # an unparseable authority, e.g. `[`
|
||||||
|
return True
|
||||||
|
if _loopback(host) or host.endswith(".localhost"):
|
||||||
|
return False
|
||||||
|
allowed = urlsplit(configured_origin).hostname if configured_origin else ""
|
||||||
|
return not host or host != (allowed or "").lower()
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------ the password again
|
||||||
|
|
||||||
|
def claim_attempt(conn: HTTPConnection) -> None:
|
||||||
|
"""The lockout, in front of every place a password can be guessed: 429
|
||||||
|
while it holds, else the guess is counted before it is derived.
|
||||||
|
|
||||||
|
`AuthStore.claim_attempt` decides and counts in one write transaction, so
|
||||||
|
concurrent guesses each see the ones before them and a window admits five
|
||||||
|
of them, not a threadpool's worth. A refused claim counts nothing, so
|
||||||
|
hammering while locked out does not extend the lockout. The caller then
|
||||||
|
checks with `check_password(..., claimed=True)`, which clears the count
|
||||||
|
when the guess was right. Called from both doors rather than living inside
|
||||||
|
`check_password`, because the same comparison is a counted guess here and
|
||||||
|
a bootstrap comparison in `auth.bootstrap`, which must not be able to lock
|
||||||
|
anyone out of their own restart.
|
||||||
|
"""
|
||||||
|
wait = conn.app.state.auth.claim_attempt()
|
||||||
|
if wait:
|
||||||
|
warn(conn, "attempt while locked out")
|
||||||
|
raise HTTPException(429, f"Too many attempts. Try again in {_spell(wait)}.",
|
||||||
|
headers={"Retry-After": str(wait)})
|
||||||
|
|
||||||
|
|
||||||
|
def confirm_password(conn: HTTPConnection, current: str) -> None:
|
||||||
|
"""The password again, from someone already signed in: changing it, adding
|
||||||
|
a passkey, minting a token, and any app route that must not be one
|
||||||
|
borrowed browser away from the account (`connect(session_only=...)`).
|
||||||
|
Counted and gated like the front door, because `current` is a guess at the
|
||||||
|
password made from behind a session: 429 while locked out, 403 when wrong.
|
||||||
|
|
||||||
|
It has nothing to confirm on an app with no password yet, which refuses
|
||||||
|
every guess, so a route that can run there asks `has_password()` first, as
|
||||||
|
the account routes do."""
|
||||||
|
claim_attempt(conn)
|
||||||
|
if not conn.app.state.auth.check_password(current, claimed=True):
|
||||||
|
warn(conn, "wrong current password")
|
||||||
|
raise HTTPException(403, "That is not the current password.")
|
||||||
|
|
||||||
|
|
||||||
|
#: Where a route whose body is a file takes the current password. A header and
|
||||||
|
#: not a field, because the body is the file itself; read before the first byte
|
||||||
|
#: of that body, so a refusal costs no upload.
|
||||||
|
CURRENT_HEADER = "x-current-password"
|
||||||
|
|
||||||
|
|
||||||
|
def confirm_current(conn: HTTPConnection) -> None:
|
||||||
|
"""The current password from the `X-Current-Password` header, for a route
|
||||||
|
that hands out or replaces the database as a file (`connect(session_only=...)`).
|
||||||
|
|
||||||
|
A session is not enough for a route that replaces the credentials: the
|
||||||
|
restored file brings its own password hash, passkeys and tokens, so a
|
||||||
|
borrowed signed-in browser that could restore a doctored backup would own
|
||||||
|
the account. Call it before reading the body. A missing or empty header is
|
||||||
|
a WARNING line and a 403 that counts nothing against the lockout, since a
|
||||||
|
missing field is not a guess; a wrong one goes through `confirm_password`,
|
||||||
|
which counts it, warns and answers 403, or 429 while locked out. An app
|
||||||
|
that never had a password (loopback, never locked) has nothing to confirm.
|
||||||
|
|
||||||
|
The check is a deliberately slow derivation, so a coroutine route runs it
|
||||||
|
in the threadpool. newsdesk's `/api/restore` is the caller.
|
||||||
|
"""
|
||||||
|
if not conn.app.state.auth.has_password():
|
||||||
|
return
|
||||||
|
current = conn.headers.get(CURRENT_HEADER, "")
|
||||||
|
if not current:
|
||||||
|
warn(conn, f"no current password for {conn.scope.get('method', '')} "
|
||||||
|
f"{conn.scope.get('path', '')}")
|
||||||
|
raise HTTPException(403, "This takes the current password in X-Current-Password.")
|
||||||
|
confirm_password(conn, current)
|
||||||
|
|
||||||
|
|
||||||
|
def warn(conn: HTTPConnection, what: str) -> None:
|
||||||
|
"""Write down that someone got it wrong, and who.
|
||||||
|
|
||||||
|
The whole record of who is knocking. There is no access log worth the name —
|
||||||
|
uvicorn is run quietly — and no table of attempts, because the log is the
|
||||||
|
record and a second copy of it would only ever disagree. WARNING so it
|
||||||
|
survives that level, one line per refusal, and the address first after
|
||||||
|
`from` so a fail2ban filter is one regex; see documentation/Password Protection.md.
|
||||||
|
|
||||||
|
The message does not name the package, because the logger already does:
|
||||||
|
with the usual `%(name)s: %(message)s` this module is `auth.middleware`,
|
||||||
|
and a line reading `auth.middleware: auth: wrong password` says it twice
|
||||||
|
and reads like a bug in the logging.
|
||||||
|
|
||||||
|
Nothing is *decided* from the address — `AuthStore.check_password` says why
|
||||||
|
it cannot be — so this is where it earns its keep: at the layer that can
|
||||||
|
drop the connection instead of answering it.
|
||||||
|
"""
|
||||||
|
log.warning("%s from %s", what, client_address(conn))
|
||||||
|
|
||||||
|
|
||||||
|
def _spell(seconds: int) -> str:
|
||||||
|
if seconds < 60:
|
||||||
|
return f"{seconds} seconds"
|
||||||
|
return f"{round(seconds / 60)} minutes"
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------- sockets whose credential ended
|
||||||
|
|
||||||
|
#: How often, in seconds, `sweep_ended_sockets` asks about every open socket:
|
||||||
|
#: the longest an expired session or token keeps the socket it opened.
|
||||||
|
SWEEP = 60.0
|
||||||
|
|
||||||
|
|
||||||
|
async def close_ended_sockets(app: Any) -> int:
|
||||||
|
"""Close every open socket whose session or token has ended. Returns how
|
||||||
|
many it closed.
|
||||||
|
|
||||||
|
The door is checked at the handshake, and a socket stays open for the
|
||||||
|
life of the tab. So signing out, changing the password or revoking a
|
||||||
|
session or a token left the socket it had opened streaming to whoever
|
||||||
|
held it. Run after each of those, as a background task of the route, and
|
||||||
|
on a clock by `sweep_ended_sockets`, this asks the store about every
|
||||||
|
socket's cookie or bearer header (`credential_live`) and closes the ones
|
||||||
|
it no longer knows with 1008; a tab's reconnect then meets the door.
|
||||||
|
|
||||||
|
The sockets are `app.state.bus`: what `pipeline.web.connect` puts there,
|
||||||
|
or anything with a `sockets` tuple and an async `close(sockets, code,
|
||||||
|
reason)`. Nothing to do on an unlocked app, which has no sessions to
|
||||||
|
end, or on one without a bus.
|
||||||
|
"""
|
||||||
|
if not getattr(app.state, "auth_locked", False):
|
||||||
|
return 0
|
||||||
|
bus = getattr(app.state, "bus", None)
|
||||||
|
if bus is None:
|
||||||
|
return 0
|
||||||
|
store, config = app.state.auth, app.state.auth_config
|
||||||
|
sockets = bus.sockets
|
||||||
|
|
||||||
|
def ended() -> list[Any]:
|
||||||
|
return [ws for ws in sockets if not credential_live(store, config, ws)]
|
||||||
|
|
||||||
|
# `to_thread`, because the store is SQLite and this is the event loop.
|
||||||
|
gone = await asyncio.to_thread(ended)
|
||||||
|
return await bus.close(gone, 1008, "signed out") if gone else 0
|
||||||
|
|
||||||
|
|
||||||
|
async def sweep_ended_sockets(app: Any, every: float = SWEEP) -> None:
|
||||||
|
"""`close_ended_sockets` every `every` seconds, until cancelled.
|
||||||
|
|
||||||
|
The door is checked at the handshake only, and the routes that end a
|
||||||
|
credential close its sockets themselves. Expiry is seen by no route: a
|
||||||
|
session or token that runs out ends no request, so its socket streamed
|
||||||
|
until the tab reconnected. This closes it within `every` seconds.
|
||||||
|
`connect` starts it on a locked app and cancels it at shutdown. A sweep
|
||||||
|
that fails is logged and the next one runs, since a loop that died on
|
||||||
|
one busy database would leave every later expiry open.
|
||||||
|
"""
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(every)
|
||||||
|
try:
|
||||||
|
await close_ended_sockets(app)
|
||||||
|
except Exception:
|
||||||
|
log.warning("the socket sweep failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ another site
|
||||||
|
|
||||||
|
#: The methods that change something. No `GET` here changes anything (see the
|
||||||
|
#: module docstring of `auth/router.py`), so these are the ones to guard.
|
||||||
|
UNSAFE = frozenset({"POST", "PUT", "PATCH", "DELETE"})
|
||||||
|
|
||||||
|
#: The bodies another site's page may send without the browser asking first:
|
||||||
|
#: the three CORS-safelisted types, and a body with no type at all. A
|
||||||
|
#: cross-site request with any other type is preflighted, and nothing here
|
||||||
|
#: answers a preflight, so it never leaves the browser.
|
||||||
|
SIMPLE_BODIES = frozenset({"", "text/plain", "application/x-www-form-urlencoded",
|
||||||
|
"multipart/form-data"})
|
||||||
|
|
||||||
|
CROSS_SITE = ("This request came from another site's page, and nothing here "
|
||||||
|
"changes on another site's say-so.")
|
||||||
|
|
||||||
|
|
||||||
|
def cross_site(scope: Scope, configured_origin: str = "") -> tuple[int, str] | None:
|
||||||
|
"""Why a request that changes something is refused, or None.
|
||||||
|
|
||||||
|
Two checks, the first for the socket too:
|
||||||
|
|
||||||
|
- **Where it came from.** `Sec-Fetch-Site` when the browser sends it,
|
||||||
|
which a page cannot set; otherwise `Origin`, which must name this host
|
||||||
|
or `configured_origin` (`AuthConfig.origin`, for a proxy that rewrites
|
||||||
|
`Host`). No `Origin` at all is a client that is not a browser page
|
||||||
|
(`curl`, a script with a token, a test) and passes.
|
||||||
|
- **What the body is.** A body of a type another site could send without
|
||||||
|
a preflight is refused, whoever sent it: the API is JSON.
|
||||||
|
|
||||||
|
A read is never refused, wherever it came from.
|
||||||
|
"""
|
||||||
|
if scope["type"] == "websocket":
|
||||||
|
if not _from_here(scope, configured_origin):
|
||||||
|
return 403, "another site's page cannot open this socket"
|
||||||
|
return None
|
||||||
|
if scope.get("method") not in UNSAFE:
|
||||||
|
return None
|
||||||
|
if not _from_here(scope, configured_origin):
|
||||||
|
return 403, CROSS_SITE
|
||||||
|
kind = _header(scope, b"content-type").split(";")[0].strip().lower()
|
||||||
|
if _has_body(scope) and kind in SIMPLE_BODIES:
|
||||||
|
return 415, ("Send the body as application/json, not "
|
||||||
|
f"{kind or 'without a content type'}.")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _from_here(scope: Scope, configured: str) -> bool:
|
||||||
|
"""Whether the page that sent this is this app's own, as far as the
|
||||||
|
browser says.
|
||||||
|
|
||||||
|
`Sec-Fetch-Site` decides when present: the browser sets it and a page
|
||||||
|
cannot, and it stays right behind a proxy that rewrites `Host`, where an
|
||||||
|
`Origin` comparison would refuse the app's own page. `none` is a request
|
||||||
|
the user made by typing or from a bookmark. Without it, `Origin` against
|
||||||
|
`Host` (the Vite dev server forwards `Host` unchanged) or against the
|
||||||
|
configured origin. `null` is a sandboxed frame or a `file://` page and is
|
||||||
|
never this app.
|
||||||
|
"""
|
||||||
|
site = _header(scope, b"sec-fetch-site")
|
||||||
|
if site:
|
||||||
|
return site in ("same-origin", "none")
|
||||||
|
origin = _header(scope, b"origin")
|
||||||
|
if not origin:
|
||||||
|
return True
|
||||||
|
if origin == "null":
|
||||||
|
return False
|
||||||
|
if configured and origin.rstrip("/") == configured.rstrip("/"):
|
||||||
|
return True
|
||||||
|
parts = urlsplit(origin)
|
||||||
|
host = _header(scope, b"host")
|
||||||
|
return bool(host) and (_authority(parts.scheme, parts.netloc)
|
||||||
|
== _authority(parts.scheme, host))
|
||||||
|
|
||||||
|
|
||||||
|
def _authority(scheme: str, netloc: str) -> str:
|
||||||
|
"""`host[:port]`, lowercased, without the scheme's default port, so
|
||||||
|
`Host: app.example.com:443` and `Origin: https://app.example.com` agree."""
|
||||||
|
netloc = netloc.lower()
|
||||||
|
default = {"http": ":80", "ws": ":80", "https": ":443", "wss": ":443"}.get(scheme, "")
|
||||||
|
return netloc[: -len(default)] if default and netloc.endswith(default) else netloc
|
||||||
|
|
||||||
|
|
||||||
|
def _has_body(scope: Scope) -> bool:
|
||||||
|
length = _header(scope, b"content-length")
|
||||||
|
# The header's presence, not its value, says a body follows; an empty value
|
||||||
|
# is malformed and is not given the benefit of the doubt.
|
||||||
|
chunked = any(key == b"transfer-encoding" for key, _ in scope.get("headers", ()))
|
||||||
|
return (length.isdigit() and int(length) > 0) or chunked
|
||||||
+25
-2
@@ -1,8 +1,8 @@
|
|||||||
"""What the Account tab is told, and nothing more.
|
"""What the Account tab is told, and nothing more.
|
||||||
|
|
||||||
Three models, and the thing worth noticing about all three is what they cannot
|
Four models, and the thing worth noticing about all four is what they cannot
|
||||||
carry. No token, no hash, no key material — not even the public half of a
|
carry. No token, no hash, no key material — not even the public half of a
|
||||||
passkey. The list of ways into an app is the one page most worth reading and
|
passkey; of an API token, the first eight characters and never the rest. The list of ways into an app is the one page most worth reading and
|
||||||
least worth leaking, so the shape it is rendered from is one that could not
|
least worth leaking, so the shape it is rendered from is one that could not
|
||||||
leak a credential if it were logged whole.
|
leak a credential if it were logged whole.
|
||||||
"""
|
"""
|
||||||
@@ -47,6 +47,27 @@ class Passkey(BaseModel):
|
|||||||
transports: list[str] = Field(default_factory=list)
|
transports: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class ApiToken(BaseModel):
|
||||||
|
"""One token a script signs in with, as the account screen sees it.
|
||||||
|
|
||||||
|
The token itself is shown once, when it is made (`TokenCreated` in
|
||||||
|
`auth/schemas.py`), and is not here: this is the row, which is what the
|
||||||
|
revoke button sends back. `hint` is the token's opening, enough to match
|
||||||
|
the row to the environment variable that holds the whole.
|
||||||
|
"""
|
||||||
|
|
||||||
|
id: int
|
||||||
|
#: What you called it. Named for the machine or the job that holds it, so
|
||||||
|
#: the one to revoke can be told from the others.
|
||||||
|
label: str = ""
|
||||||
|
#: The token's first characters. See `auth.store.HINT`.
|
||||||
|
hint: str = ""
|
||||||
|
created_at: str = ""
|
||||||
|
last_used_at: str = ""
|
||||||
|
#: When it stops working, or "" for a token that does not expire.
|
||||||
|
expires_at: str = ""
|
||||||
|
|
||||||
|
|
||||||
class Account(BaseModel):
|
class Account(BaseModel):
|
||||||
"""What the account screen knows. Deliberately almost nothing.
|
"""What the account screen knows. Deliberately almost nothing.
|
||||||
|
|
||||||
@@ -79,3 +100,5 @@ class Account(BaseModel):
|
|||||||
#: honest common case is an insecure origin: WebAuthn refuses anything but
|
#: honest common case is an insecure origin: WebAuthn refuses anything but
|
||||||
#: HTTPS, localhost excepted, so this reads "you need TLS first".
|
#: HTTPS, localhost excepted, so this reads "you need TLS first".
|
||||||
passkeys_blocked: str = ""
|
passkeys_blocked: str = ""
|
||||||
|
#: The API tokens, expired ones included, each without its secret.
|
||||||
|
tokens: list[ApiToken] = Field(default_factory=list)
|
||||||
+9
-1
@@ -291,11 +291,19 @@ def login_verify(config: AuthConfig, request: Request, store: AuthStore,
|
|||||||
)
|
)
|
||||||
|
|
||||||
challenge = _challenge_from(credential)
|
challenge = _challenge_from(credential)
|
||||||
|
# Read before the challenge is spent, and inside a `try`: outside one, a
|
||||||
|
# `rawId` that is not base64url was a 500 from an endpoint anyone can
|
||||||
|
# reach, after the challenge had already been used up.
|
||||||
|
try:
|
||||||
|
raw_id = base64url_to_bytes(credential.get("rawId", ""))
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise PasskeyMalformed("That response was not a passkey: its rawId is "
|
||||||
|
"not base64url.") from exc
|
||||||
if not store.take_challenge(challenge, "login"):
|
if not store.take_challenge(challenge, "login"):
|
||||||
raise PasskeyError("That sign-in took too long, or was already used. "
|
raise PasskeyError("That sign-in took too long, or was already used. "
|
||||||
"Try again.")
|
"Try again.")
|
||||||
|
|
||||||
known = store.passkey_for(base64url_to_bytes(credential.get("rawId", "")))
|
known = store.passkey_for(raw_id)
|
||||||
if known is None:
|
if known is None:
|
||||||
# Deliberately the same sentence as a bad signature. There is one user,
|
# Deliberately the same sentence as a bad signature. There is one user,
|
||||||
# so this leaks little, but "that key is not registered" is an oracle
|
# so this leaks little, but "that key is not registered" is an oracle
|
||||||
|
|||||||
+119
-83
@@ -29,18 +29,20 @@ import os
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import unquote, urlsplit
|
from urllib.parse import unquote, urlsplit
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Request, Response
|
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, Response
|
||||||
from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
|
from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
|
||||||
from starlette.requests import HTTPConnection
|
from starlette.requests import HTTPConnection
|
||||||
|
|
||||||
from auth import passkeys as pk
|
from auth import passkeys as pk
|
||||||
from auth.config import AuthConfig
|
from auth.config import AuthConfig
|
||||||
from auth.middleware import (
|
from auth.middleware import (
|
||||||
client_address, cookie_name, cookie_names, is_secure, token_from,
|
claim_attempt, close_ended_sockets, confirm_password, cookie_name, cookie_names,
|
||||||
|
is_secure, token_from, warn,
|
||||||
)
|
)
|
||||||
from auth.models import Account
|
from auth.models import Account
|
||||||
from auth.schemas import (
|
from auth.schemas import (
|
||||||
LabelBody, LoginBody, PasskeyBody, PasskeyLoginBody, PasswordBody,
|
LabelBody, LoginBody, Ok, PasskeyAdded, PasskeyBody, PasskeyLoginBody,
|
||||||
|
PasskeyOptionsBody, PasswordBody, PasswordChanged, SignedOut, TokenBody, TokenCreated,
|
||||||
)
|
)
|
||||||
from auth.store import AuthStore
|
from auth.store import AuthStore
|
||||||
|
|
||||||
@@ -63,7 +65,7 @@ NO_STORE = {"Cache-Control": "no-store"}
|
|||||||
|
|
||||||
#: The shortest new password the account screen will take. Low, deliberately:
|
#: The shortest new password the account screen will take. Low, deliberately:
|
||||||
#: it is a floor against a typo, not a password policy. The honest advice is in
|
#: it is a floor against a typo, not a password policy. The honest advice is in
|
||||||
#: docs/password-protection.md — the generated password is sixteen random
|
#: documentation/Password Protection.md — the generated password is sixteen random
|
||||||
#: characters and anything you choose yourself is the weak half.
|
#: characters and anything you choose yourself is the weak half.
|
||||||
MIN_LENGTH = 8
|
MIN_LENGTH = 8
|
||||||
|
|
||||||
@@ -156,19 +158,20 @@ def login_page(request: Request) -> Response:
|
|||||||
def sign_in(request: Request, response: Response, body: LoginBody) -> dict[str, str]:
|
def sign_in(request: Request, response: Response, body: LoginBody) -> dict[str, str]:
|
||||||
"""Check the password, and if it is right, open a session.
|
"""Check the password, and if it is right, open a session.
|
||||||
|
|
||||||
The lockout is read *before* the password is checked, so hammering the
|
The attempt is claimed *before* the password is derived, which reads the
|
||||||
button while locked out does not extend the lockout — see
|
lockout and counts the guess in one write, so hammering the button while
|
||||||
`AuthStore.locked_for`. A wrong password and an unknown one are the same
|
locked out does not extend the lockout and concurrent guesses cannot all
|
||||||
answer here for the usual reason, which is thin cover with one account but
|
slip in under one reading of it; see `AuthStore.claim_attempt`. A wrong
|
||||||
costs nothing to keep.
|
password and an unknown one are the same answer here for the usual
|
||||||
|
reason, which is thin cover with one account but costs nothing to keep.
|
||||||
"""
|
"""
|
||||||
if not _locked(request):
|
if not _locked(request):
|
||||||
raise HTTPException(400, "This app is not password-protected.")
|
raise HTTPException(400, "This app is not password-protected.")
|
||||||
|
|
||||||
_refuse_while_locked(request)
|
claim_attempt(request)
|
||||||
|
|
||||||
if not _store(request).check_password(body.password):
|
if not _store(request).check_password(body.password, claimed=True):
|
||||||
_warn(request, "wrong password")
|
warn(request, "wrong password")
|
||||||
raise HTTPException(401, "That is not the password.")
|
raise HTTPException(401, "That is not the password.")
|
||||||
|
|
||||||
_open_session(request, response)
|
_open_session(request, response)
|
||||||
@@ -176,17 +179,22 @@ def sign_in(request: Request, response: Response, body: LoginBody) -> dict[str,
|
|||||||
|
|
||||||
|
|
||||||
@router.post("/logout", include_in_schema=False)
|
@router.post("/logout", include_in_schema=False)
|
||||||
def sign_out(request: Request, response: Response) -> dict[str, bool]:
|
def sign_out(request: Request, response: Response, tasks: BackgroundTasks) -> Ok:
|
||||||
"""End this session and clear the cookie. Other devices are untouched."""
|
"""End this session and clear the cookie. Other devices are untouched.
|
||||||
|
|
||||||
|
The socket this tab opened is closed too, after the answer; see
|
||||||
|
`auth.close_ended_sockets`. The same for the four routes below that end
|
||||||
|
a session or a token."""
|
||||||
token = token_from(request, _config(request))
|
token = token_from(request, _config(request))
|
||||||
if token:
|
if token:
|
||||||
_store(request).close_session(token)
|
_store(request).close_session(token)
|
||||||
|
tasks.add_task(close_ended_sockets, request.app)
|
||||||
# Both names, because which one was set depends on `secure_cookie` and on
|
# Both names, because which one was set depends on `secure_cookie` and on
|
||||||
# the scheme the login happened over — and clearing only one of them leaves
|
# the scheme the login happened over — and clearing only one of them leaves
|
||||||
# a cookie the browser will keep offering.
|
# a cookie the browser will keep offering.
|
||||||
for name in cookie_names(_config(request)):
|
for name in cookie_names(_config(request)):
|
||||||
response.delete_cookie(name, path="/")
|
response.delete_cookie(name, path="/")
|
||||||
return {"ok": True}
|
return Ok()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- the account
|
# ---------------------------------------------------------------- the account
|
||||||
@@ -205,11 +213,13 @@ def account(request: Request) -> Account:
|
|||||||
passkeys=store.passkeys(),
|
passkeys=store.passkeys(),
|
||||||
passkeys_available=pk.available(),
|
passkeys_available=pk.available(),
|
||||||
passkeys_blocked=pk.blocked(config, request),
|
passkeys_blocked=pk.blocked(config, request),
|
||||||
|
tokens=store.tokens(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/account/password")
|
@router.post("/api/account/password")
|
||||||
def change_password(request: Request, body: PasswordBody) -> dict[str, int]:
|
def change_password(request: Request, body: PasswordBody,
|
||||||
|
tasks: BackgroundTasks) -> PasswordChanged:
|
||||||
"""Change it, having proved you know the current one — or set the first.
|
"""Change it, having proved you know the current one — or set the first.
|
||||||
|
|
||||||
Asking for the current password matters even though you are already signed
|
Asking for the current password matters even though you are already signed
|
||||||
@@ -223,6 +233,13 @@ def change_password(request: Request, body: PasswordBody) -> dict[str, int]:
|
|||||||
the change ceremonial — and the cost when that is not the reason is signing
|
the change ceremonial — and the cost when that is not the reason is signing
|
||||||
in again on your phone, which is the right way round.
|
in again on your phone, which is the right way round.
|
||||||
|
|
||||||
|
API tokens and passkeys stay by default, because the cron job holding a
|
||||||
|
token is not what the password was changed against. `revoke_all` deletes
|
||||||
|
every one of them too, for the case the route exists for: an intruder
|
||||||
|
who had the password could have minted a token that never expires or
|
||||||
|
registered a passkey, and either outlives the change. The answer counts
|
||||||
|
what was closed and what remains, so the screen can say so.
|
||||||
|
|
||||||
Which is also why this is behind the same lockout as the front door, and not
|
Which is also why this is behind the same lockout as the front door, and not
|
||||||
only behind a session. `current` is the password, guessed here rather than
|
only behind a session. `current` is the password, guessed here rather than
|
||||||
at `/login`, and a borrowed unlocked laptop is exactly the situation the
|
at `/login`, and a borrowed unlocked laptop is exactly the situation the
|
||||||
@@ -244,32 +261,82 @@ def change_password(request: Request, body: PasswordBody) -> dict[str, int]:
|
|||||||
# app never lands here: `auth.bootstrap` gives it a password at startup,
|
# app never lands here: `auth.bootstrap` gives it a password at startup,
|
||||||
# so `current` is always required for the app that has a door.
|
# so `current` is always required for the app that has a door.
|
||||||
if store.has_password():
|
if store.has_password():
|
||||||
_refuse_while_locked(request)
|
confirm_password(request, body.current)
|
||||||
if not store.check_password(body.current):
|
|
||||||
_warn(request, "wrong current password")
|
|
||||||
raise HTTPException(403, "That is not the current password.")
|
|
||||||
if len(body.new) < MIN_LENGTH:
|
if len(body.new) < MIN_LENGTH:
|
||||||
raise HTTPException(400, f"Use at least {MIN_LENGTH} characters.")
|
raise HTTPException(400, f"Use at least {MIN_LENGTH} characters.")
|
||||||
|
|
||||||
store.set_password(body.new)
|
store.set_password(body.new)
|
||||||
closed = store.close_all(keep=token_from(request, config))
|
closed = store.close_all(keep=token_from(request, config))
|
||||||
return {"signed_out": closed}
|
tokens, keys = store.revoke_credentials() if body.revoke_all else (0, 0)
|
||||||
|
tokens_left, keys_left = store.counts()
|
||||||
|
tasks.add_task(close_ended_sockets, request.app)
|
||||||
|
return PasswordChanged(signed_out=closed, tokens_revoked=tokens, passkeys_revoked=keys,
|
||||||
|
tokens_left=tokens_left, passkeys_left=keys_left)
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/api/account/sessions/{session_id}")
|
@router.delete("/api/account/sessions/{session_id}")
|
||||||
def revoke(session_id: int, request: Request) -> dict[str, bool]:
|
def revoke(session_id: int, request: Request, tasks: BackgroundTasks) -> Ok:
|
||||||
"""Drop one session. Revoking your own is allowed — it is a logout with an
|
"""Drop one session. Revoking your own is allowed — it is a logout with an
|
||||||
extra click, and refusing it would be a rule to explain for no gain."""
|
extra click, and refusing it would be a rule to explain for no gain."""
|
||||||
if not _store(request).close_by_id(session_id):
|
if not _store(request).close_by_id(session_id):
|
||||||
raise HTTPException(404, "That session is already gone.")
|
raise HTTPException(404, "That session is already gone.")
|
||||||
return {"ok": True}
|
tasks.add_task(close_ended_sockets, request.app)
|
||||||
|
return Ok()
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/account/sessions/close")
|
@router.post("/api/account/sessions/close")
|
||||||
def close_others(request: Request) -> dict[str, int]:
|
def close_others(request: Request, tasks: BackgroundTasks) -> SignedOut:
|
||||||
"""Sign out everywhere but here. The reason sessions are rows."""
|
"""Sign out everywhere but here. The reason sessions are rows."""
|
||||||
|
closed = _store(request).close_all(keep=token_from(request, _config(request)))
|
||||||
|
tasks.add_task(close_ended_sockets, request.app)
|
||||||
|
return SignedOut(signed_out=closed)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------- tokens
|
||||||
|
#
|
||||||
|
# Three routes, all behind a session and none reachable with a token: the
|
||||||
|
# middleware answers a bearer request under `/api/account` with 403
|
||||||
|
# (`auth.middleware.ACCOUNT`), so a leaked token cannot mint another, revoke
|
||||||
|
# the owner's sessions or read the list of ways in. The token is used
|
||||||
|
# anywhere else as `Authorization: Bearer <token>`.
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/api/account/tokens")
|
||||||
|
def create_token(request: Request, body: TokenBody) -> TokenCreated:
|
||||||
|
"""Mint an API token for a script, once the password is confirmed.
|
||||||
|
|
||||||
|
A session is not enough, for the reason it is not enough for a passkey: a
|
||||||
|
token outlives every session and a password change, so a borrowed
|
||||||
|
signed-in browser that could mint one would keep a way in for good.
|
||||||
|
`current` is checked as `change_password` checks it, behind the same
|
||||||
|
lockout, and on the same condition. The token is in this answer and
|
||||||
|
nowhere else; the store keeps its hash.
|
||||||
|
"""
|
||||||
store = _store(request)
|
store = _store(request)
|
||||||
return {"signed_out": store.close_all(keep=token_from(request, _config(request)))}
|
if store.has_password():
|
||||||
|
confirm_password(request, body.current)
|
||||||
|
token_id, token = store.create_token(body.label, body.days)
|
||||||
|
made = next(t for t in store.tokens() if t.id == token_id)
|
||||||
|
return TokenCreated(id=token_id, token=token, hint=made.hint, label=made.label,
|
||||||
|
expires_at=made.expires_at)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/api/account/tokens/{token_id}/label")
|
||||||
|
def rename_token(token_id: int, body: LabelBody, request: Request) -> Ok:
|
||||||
|
if not _store(request).rename_token(token_id, body.label):
|
||||||
|
raise HTTPException(404, "That token is already gone.")
|
||||||
|
return Ok()
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/api/account/tokens/{token_id}")
|
||||||
|
def drop_token(token_id: int, request: Request, tasks: BackgroundTasks) -> Ok:
|
||||||
|
"""Revoke one. The next request carrying it is refused, whatever it holds
|
||||||
|
open: a token is looked up on every request, like a session, and a socket
|
||||||
|
it opened is closed as a session's is."""
|
||||||
|
if not _store(request).delete_token(token_id):
|
||||||
|
raise HTTPException(404, "That token is already gone.")
|
||||||
|
tasks.add_task(close_ended_sockets, request.app)
|
||||||
|
return Ok()
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------- passkeys
|
# ------------------------------------------------------------------- passkeys
|
||||||
@@ -281,24 +348,37 @@ def close_others(request: Request) -> dict[str, int]:
|
|||||||
|
|
||||||
|
|
||||||
@router.post("/api/account/passkeys/options")
|
@router.post("/api/account/passkeys/options")
|
||||||
def passkey_options(request: Request) -> dict[str, object]:
|
def passkey_options(request: Request, body: PasskeyOptionsBody) -> dict[str, object]:
|
||||||
"""Step one of registering: a challenge for `navigator.credentials.create`."""
|
"""Step one of registering: a challenge for `navigator.credentials.create`,
|
||||||
|
once the password is confirmed.
|
||||||
|
|
||||||
|
A session is not enough. A passkey outlives every session and a password
|
||||||
|
change, so a borrowed signed-in browser that could add one would keep a way
|
||||||
|
in for good. `current` is checked as `change_password` checks it, behind the
|
||||||
|
same lockout, and on the same condition: an app with no password (loopback,
|
||||||
|
never locked) has nothing to confirm and no session to borrow, so
|
||||||
|
registering there before deploying still works.
|
||||||
|
"""
|
||||||
_require_passkeys(request)
|
_require_passkeys(request)
|
||||||
return pk.register_options(_config(request), request, _store(request))
|
store = _store(request)
|
||||||
|
if store.has_password():
|
||||||
|
confirm_password(request, body.current)
|
||||||
|
return pk.register_options(_config(request), request, store)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/api/account/passkeys")
|
@router.post("/api/account/passkeys")
|
||||||
def add_passkey(request: Request, body: PasskeyBody) -> dict[str, int]:
|
def add_passkey(request: Request, body: PasskeyBody) -> PasskeyAdded:
|
||||||
"""Step two: check what the authenticator signed, and keep the public key.
|
"""Step two: check what the authenticator signed, and keep the public key.
|
||||||
|
|
||||||
Requires a session, like everything else under `/api/account`. Registering a
|
Requires a session, like everything else under `/api/account`, and a
|
||||||
passkey is granting a new way in, so it is gated on already being in — the
|
challenge from `passkey_options`, which only the password gets. The
|
||||||
password is what proves that, and the ceremony does not replace it.
|
challenge is single-use and bound to registering, so this step cannot be
|
||||||
|
reached without the password having been typed for it.
|
||||||
"""
|
"""
|
||||||
_require_passkeys(request)
|
_require_passkeys(request)
|
||||||
try:
|
try:
|
||||||
return {"id": pk.register_verify(_config(request), request, _store(request),
|
return PasskeyAdded(id=pk.register_verify(_config(request), request, _store(request),
|
||||||
body.credential, body.label)}
|
body.credential, body.label))
|
||||||
except pk.PasskeyError as exc:
|
except pk.PasskeyError as exc:
|
||||||
# `PasskeyMalformed` is one of these and answers the same way: from
|
# `PasskeyMalformed` is one of these and answers the same way: from
|
||||||
# behind a session, a body that is not a ceremony and a ceremony that
|
# behind a session, a body that is not a ceremony and a ceremony that
|
||||||
@@ -307,20 +387,20 @@ def add_passkey(request: Request, body: PasskeyBody) -> dict[str, int]:
|
|||||||
|
|
||||||
|
|
||||||
@router.post("/api/account/passkeys/{passkey_id}/label")
|
@router.post("/api/account/passkeys/{passkey_id}/label")
|
||||||
def rename_passkey(passkey_id: int, body: LabelBody, request: Request) -> dict[str, bool]:
|
def rename_passkey(passkey_id: int, body: LabelBody, request: Request) -> Ok:
|
||||||
"""Name it. A key you cannot name is one you will never dare revoke."""
|
"""Name it. A key you cannot name is one you will never dare revoke."""
|
||||||
if not _store(request).rename_passkey(passkey_id, body.label):
|
if not _store(request).rename_passkey(passkey_id, body.label):
|
||||||
raise HTTPException(404, "That passkey is already gone.")
|
raise HTTPException(404, "That passkey is already gone.")
|
||||||
return {"ok": True}
|
return Ok()
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/api/account/passkeys/{passkey_id}")
|
@router.delete("/api/account/passkeys/{passkey_id}")
|
||||||
def drop_passkey(passkey_id: int, request: Request) -> dict[str, bool]:
|
def drop_passkey(passkey_id: int, request: Request) -> Ok:
|
||||||
"""Revoke one. The password is still there, so this cannot lock you out —
|
"""Revoke one. The password is still there, so this cannot lock you out —
|
||||||
which is exactly why the password is still there."""
|
which is exactly why the password is still there."""
|
||||||
if not _store(request).delete_passkey(passkey_id):
|
if not _store(request).delete_passkey(passkey_id):
|
||||||
raise HTTPException(404, "That passkey is already gone.")
|
raise HTTPException(404, "That passkey is already gone.")
|
||||||
return {"ok": True}
|
return Ok()
|
||||||
|
|
||||||
|
|
||||||
@router.post("/login/passkey/options", include_in_schema=False)
|
@router.post("/login/passkey/options", include_in_schema=False)
|
||||||
@@ -363,7 +443,7 @@ def passkey_login(request: Request, response: Response,
|
|||||||
# been wrong about.
|
# been wrong about.
|
||||||
raise HTTPException(400, str(exc)) from None
|
raise HTTPException(400, str(exc)) from None
|
||||||
except pk.PasskeyError as exc:
|
except pk.PasskeyError as exc:
|
||||||
_warn(request, "passkey refused")
|
warn(request, "passkey refused")
|
||||||
raise HTTPException(401, str(exc)) from None
|
raise HTTPException(401, str(exc)) from None
|
||||||
|
|
||||||
_open_session(request, response)
|
_open_session(request, response)
|
||||||
@@ -406,44 +486,6 @@ def _open_session(request: Request, response: Response) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _refuse_while_locked(request: Request) -> None:
|
|
||||||
"""The lockout, in front of every place a password can be guessed.
|
|
||||||
|
|
||||||
Read *before* the guess is looked at, so hammering while locked out does not
|
|
||||||
extend the lockout — see `AuthStore.locked_for`. Called from both doors
|
|
||||||
rather than living inside `check_password`, because the store counts and
|
|
||||||
this decides: the same wrong answer is a refusal here and a bootstrap
|
|
||||||
comparison in `auth.bootstrap`, which must not be able to lock anyone out of
|
|
||||||
their own restart.
|
|
||||||
"""
|
|
||||||
wait = _store(request).locked_for()
|
|
||||||
if wait:
|
|
||||||
_warn(request, "attempt while locked out")
|
|
||||||
raise HTTPException(429, f"Too many attempts. Try again in {_spell(wait)}.",
|
|
||||||
headers={"Retry-After": str(wait)})
|
|
||||||
|
|
||||||
|
|
||||||
def _warn(request: Request, what: str) -> None:
|
|
||||||
"""Write down that someone got it wrong, and who.
|
|
||||||
|
|
||||||
The whole record of who is knocking. There is no access log worth the name —
|
|
||||||
uvicorn is run quietly — and no table of attempts, because the log is the
|
|
||||||
record and a second copy of it would only ever disagree. WARNING so it
|
|
||||||
survives that level, one line per refusal, and the address first after
|
|
||||||
`from` so a fail2ban filter is one regex; see docs/password-protection.md.
|
|
||||||
|
|
||||||
The message does not name the package, because the logger already does:
|
|
||||||
with the usual `%(name)s: %(message)s` this module is `auth.router`, and a
|
|
||||||
line reading `auth.router: auth: wrong password` says it twice and reads
|
|
||||||
like a bug in the logging.
|
|
||||||
|
|
||||||
Nothing is *decided* from the address — `AuthStore.check_password` says why
|
|
||||||
it cannot be — so this is where it earns its keep: at the layer that can
|
|
||||||
drop the connection instead of answering it.
|
|
||||||
"""
|
|
||||||
log.warning("%s from %s", what, client_address(request))
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_next(target: str) -> str:
|
def _safe_next(target: str) -> str:
|
||||||
r"""Where to go after signing in — only ever inside this app.
|
r"""Where to go after signing in — only ever inside this app.
|
||||||
|
|
||||||
@@ -481,9 +523,3 @@ def _local(target: str) -> bool:
|
|||||||
except ValueError: # an unparseable authority, e.g. `//[`
|
except ValueError: # an unparseable authority, e.g. `//[`
|
||||||
return False
|
return False
|
||||||
return not parts.scheme and not parts.netloc
|
return not parts.scheme and not parts.netloc
|
||||||
|
|
||||||
|
|
||||||
def _spell(seconds: int) -> str:
|
|
||||||
if seconds < 60:
|
|
||||||
return f"{seconds} seconds"
|
|
||||||
return f"{round(seconds / 60)} minutes"
|
|
||||||
+84
-2
@@ -1,10 +1,15 @@
|
|||||||
"""The five bodies this package accepts. Small on purpose.
|
"""The seven bodies this package accepts, and the five small answers it gives.
|
||||||
|
Small on purpose.
|
||||||
|
|
||||||
A ceremony's `credential` is passed through as an opaque dict — it is the
|
A ceremony's `credential` is passed through as an opaque dict — it is the
|
||||||
browser's `PublicKeyCredential` serialised by `toJSON()`, and its shape is the
|
browser's `PublicKeyCredential` serialised by `toJSON()`, and its shape is the
|
||||||
WebAuthn spec's business rather than ours. Restating it as a model here would
|
WebAuthn spec's business rather than ours. Restating it as a model here would
|
||||||
be a second, staler copy of a structure the library already parses and
|
be a second, staler copy of a structure the library already parses and
|
||||||
validates properly.
|
validates properly.
|
||||||
|
|
||||||
|
The answers are models rather than dicts so that the app's OpenAPI document,
|
||||||
|
which a client generates its types from, names every one of them. The JSON is
|
||||||
|
what the dicts were.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -34,6 +39,22 @@ class PasswordBody(BaseModel):
|
|||||||
|
|
||||||
current: str = Field(max_length=1000)
|
current: str = Field(max_length=1000)
|
||||||
new: str = Field(max_length=1000)
|
new: str = Field(max_length=1000)
|
||||||
|
#: Also delete every API token and passkey. Off by default, because a
|
||||||
|
#: token's cron job is not what a routine change is against; on when the
|
||||||
|
#: password may have been someone else's, since what they minted with it
|
||||||
|
#: outlives the change. None is off, and is the default rather than False
|
||||||
|
#: so the generated client type leaves the field optional, as it does
|
||||||
|
#: `TokenBody.days`: openapi-typescript makes a field with a non-null
|
||||||
|
#: default required, and a client that never asks should not have to say so.
|
||||||
|
revoke_all: bool | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class PasskeyOptionsBody(BaseModel):
|
||||||
|
"""Starting to register a passkey. `current` is the password, required for
|
||||||
|
the same reason as in `PasswordBody`: a passkey is a permanent way in, and
|
||||||
|
a borrowed session must not be able to mint one."""
|
||||||
|
|
||||||
|
current: str = Field(max_length=1000)
|
||||||
|
|
||||||
|
|
||||||
class PasskeyBody(BaseModel):
|
class PasskeyBody(BaseModel):
|
||||||
@@ -51,6 +72,67 @@ class PasskeyLoginBody(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class LabelBody(BaseModel):
|
class LabelBody(BaseModel):
|
||||||
"""Renaming a passkey."""
|
"""Renaming a passkey or a token."""
|
||||||
|
|
||||||
|
label: str = Field(max_length=100)
|
||||||
|
|
||||||
|
|
||||||
|
class TokenBody(BaseModel):
|
||||||
|
"""Minting an API token. `current` is the password, required for the same
|
||||||
|
reason as in `PasskeyOptionsBody`: a token is a standing way in, and a
|
||||||
|
borrowed session must not be able to mint one."""
|
||||||
|
|
||||||
|
current: str = Field(max_length=1000)
|
||||||
|
#: What to call it: the machine or the job that will hold it.
|
||||||
|
label: str = Field("", max_length=100)
|
||||||
|
#: How long it lives, in days. None is a token that does not expire, which
|
||||||
|
#: is what a cron job wants; bounded above so a typo cannot ask for a
|
||||||
|
#: thousand years and still be called dated.
|
||||||
|
days: int | None = Field(None, ge=1, le=3650)
|
||||||
|
|
||||||
|
|
||||||
|
class Ok(BaseModel):
|
||||||
|
"""A change that has nothing to report beyond having happened."""
|
||||||
|
|
||||||
|
ok: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class SignedOut(BaseModel):
|
||||||
|
"""How many other sessions "sign out everywhere else" closed."""
|
||||||
|
|
||||||
|
signed_out: int
|
||||||
|
|
||||||
|
|
||||||
|
class PasswordChanged(BaseModel):
|
||||||
|
"""What a password change closed and what it left, for the screen to say.
|
||||||
|
|
||||||
|
`signed_out` is the other sessions, always closed. The tokens and
|
||||||
|
passkeys are deleted only when `revoke_all` was asked for, and the
|
||||||
|
`_left` counts are what is still registered afterwards, expired tokens
|
||||||
|
included, so an owner who did not ask knows what to review."""
|
||||||
|
|
||||||
|
signed_out: int
|
||||||
|
tokens_revoked: int
|
||||||
|
passkeys_revoked: int
|
||||||
|
tokens_left: int
|
||||||
|
passkeys_left: int
|
||||||
|
|
||||||
|
|
||||||
|
class PasskeyAdded(BaseModel):
|
||||||
|
"""The row a registered passkey was given, which rename and revoke take."""
|
||||||
|
|
||||||
|
id: int
|
||||||
|
|
||||||
|
|
||||||
|
class TokenCreated(BaseModel):
|
||||||
|
"""A minted token: the one answer in this package that carries a credential.
|
||||||
|
|
||||||
|
`token` is shown once and stored nowhere but as a hash, so a caller that
|
||||||
|
loses it mints another. The rest is the row as `ApiToken` lists it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
id: int
|
||||||
|
token: str
|
||||||
|
hint: str
|
||||||
label: str
|
label: str
|
||||||
|
expires_at: str
|
||||||
+209
-12
@@ -4,9 +4,9 @@
|
|||||||
* <auth-account></auth-account>
|
* <auth-account></auth-account>
|
||||||
*
|
*
|
||||||
* That is the whole of it. The element asks `/api/account` what there is to
|
* That is the whole of it. The element asks `/api/account` what there is to
|
||||||
* know, and draws the four things there are to do about it: change the
|
* know, and draws the five things there are to do about it: change the
|
||||||
* password, look at what is signed in and revoke any of it, register and
|
* password, look at what is signed in and revoke any of it, register and
|
||||||
* revoke passkeys, and sign out.
|
* revoke passkeys, make and revoke API tokens for scripts, and sign out.
|
||||||
*
|
*
|
||||||
* One file, and the styles are in it rather than beside it, because the shadow
|
* One file, and the styles are in it rather than beside it, because the shadow
|
||||||
* root scopes them anyway and a second file is a second route, a second cache
|
* root scopes them anyway and a second file is a second route, a second cache
|
||||||
@@ -79,6 +79,18 @@ button {
|
|||||||
padding: 2px 9px;
|
padding: 2px 9px;
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
|
select {
|
||||||
|
font: inherit; color: inherit; background: transparent;
|
||||||
|
border: 1px solid color-mix(in srgb, var(--desk-jobs-fg) 25%, transparent);
|
||||||
|
border-radius: 6px; padding: 3px 7px;
|
||||||
|
}
|
||||||
|
select option { color: initial; }
|
||||||
|
label.check { display: flex; align-items: center; gap: 6px; flex: 1 1 100%; }
|
||||||
|
label.check input { min-width: 0; margin: 0; }
|
||||||
|
input.token { font-family: ui-monospace, "Cascadia Mono", Consolas, monospace; min-width: 20rem; }
|
||||||
|
.made { display: flex; flex-direction: column; gap: var(--desk-jobs-gap); }
|
||||||
|
.made .bar { display: flex; flex-wrap: wrap; gap: var(--desk-jobs-gap); align-items: center; }
|
||||||
|
|
||||||
button:hover { color: var(--desk-jobs-accent); }
|
button:hover { color: var(--desk-jobs-accent); }
|
||||||
button.danger:hover { color: var(--desk-jobs-error); }
|
button.danger:hover { color: var(--desk-jobs-error); }
|
||||||
button[disabled] { opacity: .4; cursor: default; }
|
button[disabled] { opacity: .4; cursor: default; }
|
||||||
@@ -87,6 +99,25 @@ button[disabled] { opacity: .4; cursor: default; }
|
|||||||
font-size: 12px; padding: 0 6px; border-radius: 999px;
|
font-size: 12px; padding: 0 6px; border-radius: 999px;
|
||||||
border: 1px solid currentColor; color: var(--desk-jobs-dim);
|
border: 1px solid currentColor; color: var(--desk-jobs-dim);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A finger: every button a thumb's height, and no field under 16px, which
|
||||||
|
* is where iOS zooms the page in on focus and never back out. Sizes only;
|
||||||
|
* the layout is the width query below, and neither reads the other. */
|
||||||
|
@media (pointer: coarse) {
|
||||||
|
button { min-height: 2.5rem; padding: 4px 12px; }
|
||||||
|
input, select { font-size: 16px; padding: 7px 9px; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A phone: a session's name and its buttons share a line while they fit and
|
||||||
|
* the buttons drop under it when they do not, still on the right; a form's
|
||||||
|
* fields take the width and its buttons share what is left. 40rem is
|
||||||
|
* app.css's number, on purpose. */
|
||||||
|
@media (max-width: 40rem) {
|
||||||
|
.row { flex-wrap: wrap; align-items: center; }
|
||||||
|
.row .acts { margin-left: auto; }
|
||||||
|
form input:not([type="checkbox"]), form select, .bar input { flex: 1 1 100%; min-width: 0; }
|
||||||
|
form button, .bar button { flex: 1 1 auto; }
|
||||||
|
}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
/** What the server says when this build has no `webauthn` installed. The
|
/** What the server says when this build has no `webauthn` installed. The
|
||||||
@@ -110,6 +141,11 @@ class AuthAccount extends HTMLElement {
|
|||||||
* the other way -- from an app that was restarted without the extra while
|
* the other way -- from an app that was restarted without the extra while
|
||||||
* this page was open. */
|
* this page was open. */
|
||||||
#passkeysGone = false;
|
#passkeysGone = false;
|
||||||
|
/** The token just made, held until Done: the server shows it once and keeps
|
||||||
|
* only its hash, so nothing reloaded from `/api/account` can bring it back. */
|
||||||
|
#made = null;
|
||||||
|
/** What Copy last did: "", "done" or "failed". */
|
||||||
|
#copy = "";
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
@@ -211,6 +247,7 @@ class AuthAccount extends HTMLElement {
|
|||||||
this.#panel.append(this.#password(account));
|
this.#panel.append(this.#password(account));
|
||||||
if (account.sessions.length) this.#panel.append(this.#sessions(account));
|
if (account.sessions.length) this.#panel.append(this.#sessions(account));
|
||||||
if (this.#showPasskeys(account)) this.#panel.append(this.#passkeys(account));
|
if (this.#showPasskeys(account)) this.#panel.append(this.#passkeys(account));
|
||||||
|
this.#panel.append(this.#tokens(account));
|
||||||
if (account.locked) this.#panel.append(this.#signOut());
|
if (account.locked) this.#panel.append(this.#signOut());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -255,22 +292,30 @@ class AuthAccount extends HTMLElement {
|
|||||||
const form = document.createElement("form");
|
const form = document.createElement("form");
|
||||||
const current = field("password", "Current password");
|
const current = field("password", "Current password");
|
||||||
const next = field("password", first ? "Choose a password" : "New password");
|
const next = field("password", first ? "Choose a password" : "New password");
|
||||||
|
// Tokens and passkeys survive a change unless this is ticked: the cron
|
||||||
|
// job holding a token is not what a routine change is against, but an
|
||||||
|
// intruder who had the password may have minted one.
|
||||||
|
const revoke = document.createElement("input");
|
||||||
|
revoke.type = "checkbox";
|
||||||
|
const revokeLabel = document.createElement("label");
|
||||||
|
revokeLabel.className = "check";
|
||||||
|
revokeLabel.append(revoke, "Also revoke every API token and passkey");
|
||||||
const go = document.createElement("button");
|
const go = document.createElement("button");
|
||||||
go.textContent = first ? "Set password" : "Change password";
|
go.textContent = first ? "Set password" : "Change password";
|
||||||
go.disabled = this.#busy;
|
go.disabled = this.#busy;
|
||||||
form.append(...(first ? [next, go] : [current, next, go]));
|
form.append(...(first ? [next, go] : [current, next, revokeLabel, go]));
|
||||||
form.addEventListener("submit", (event) => {
|
form.addEventListener("submit", (event) => {
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
this.#act(async () => {
|
this.#act(async () => {
|
||||||
const result = await this.#api("/api/account/password", {
|
const result = await this.#api("/api/account/password", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
body: JSON.stringify({ current: current.value, new: next.value }),
|
body: JSON.stringify({
|
||||||
|
current: current.value, new: next.value, revoke_all: !first && revoke.checked,
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
current.value = next.value = "";
|
current.value = next.value = "";
|
||||||
if (first) return this.#say("Password set.");
|
if (first) return this.#say("Password set.");
|
||||||
this.#say(result.signed_out
|
this.#say(changed(result));
|
||||||
? `Password changed. ${count(result.signed_out, "other session")} signed out.`
|
|
||||||
: "Password changed.");
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
section.append(form);
|
section.append(form);
|
||||||
@@ -279,7 +324,7 @@ class AuthAccount extends HTMLElement {
|
|||||||
// makes the change ceremonial. Said here so it is not a surprise.
|
// makes the change ceremonial. Said here so it is not a surprise.
|
||||||
section.append(this.#line(first
|
section.append(this.#line(first
|
||||||
? "Nothing asks for it until the app binds an address that is not loopback -- or `auth.enabled: always`. Set it before you deploy, not after."
|
? "Nothing asks for it until the app binds an address that is not loopback -- or `auth.enabled: always`. Set it before you deploy, not after."
|
||||||
: "Changing it signs out every other session, and keeps this one."));
|
: "Changing it signs out every other session and keeps this one. API tokens and passkeys stay unless you tick the box; tick it if someone else may have had the password."));
|
||||||
return section;
|
return section;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -344,13 +389,142 @@ class AuthAccount extends HTMLElement {
|
|||||||
section.append(this.#line(account.passkeys_blocked));
|
section.append(this.#line(account.passkeys_blocked));
|
||||||
return section;
|
return section;
|
||||||
}
|
}
|
||||||
const add = this.#button("Add a passkey", "", () => this.#register());
|
// A passkey outlasts every session, so adding one takes the password --
|
||||||
|
// see `passkey_options` in router.py. With none set there is nothing to
|
||||||
|
// confirm, and the server does not ask.
|
||||||
|
const asks = Boolean(account.password_set_at);
|
||||||
|
const form = document.createElement("form");
|
||||||
|
const current = field("password", "Password, to add a passkey");
|
||||||
|
current.autocomplete = "current-password";
|
||||||
|
const add = document.createElement("button");
|
||||||
|
add.textContent = "Add a passkey";
|
||||||
add.disabled = this.#busy;
|
add.disabled = this.#busy;
|
||||||
section.append(add);
|
form.append(...(asks ? [current, add] : [add]));
|
||||||
|
form.addEventListener("submit", (event) => {
|
||||||
|
event.preventDefault();
|
||||||
|
this.#register(current.value);
|
||||||
|
});
|
||||||
|
section.append(form);
|
||||||
return section;
|
return section;
|
||||||
}
|
}
|
||||||
|
|
||||||
async #register() {
|
#tokens(account) {
|
||||||
|
const section = this.#section("API tokens");
|
||||||
|
for (const token of account.tokens || []) {
|
||||||
|
const row = this.#row(
|
||||||
|
token.label || "Token",
|
||||||
|
[`${token.hint}…`,
|
||||||
|
`added ${when(token.created_at)}`,
|
||||||
|
token.last_used_at ? `last used ${when(token.last_used_at)}` : "never used",
|
||||||
|
lifetime(token),
|
||||||
|
].join(" · "));
|
||||||
|
const acts = row.querySelector(".acts");
|
||||||
|
acts.append(this.#button("Rename", "", () => {
|
||||||
|
const label = prompt("Call this token what?", token.label || "");
|
||||||
|
if (label === null) return;
|
||||||
|
this.#act(async () => {
|
||||||
|
await this.#api(`/api/account/tokens/${token.id}/label`, {
|
||||||
|
method: "POST", body: JSON.stringify({ label }),
|
||||||
|
});
|
||||||
|
this.#say("Renamed.");
|
||||||
|
});
|
||||||
|
}));
|
||||||
|
acts.append(this.#button("Revoke", "danger", () => this.#act(async () => {
|
||||||
|
await this.#api(`/api/account/tokens/${token.id}`, { method: "DELETE" });
|
||||||
|
// The script holding it is locked out on its next request, and
|
||||||
|
// nothing else changes: the password and the sessions stay.
|
||||||
|
this.#say("Token revoked.");
|
||||||
|
})));
|
||||||
|
section.append(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A token outlives every session and a password change, so making one
|
||||||
|
// takes the password, on the same condition as a passkey -- see
|
||||||
|
// `create_token` in router.py.
|
||||||
|
const asks = Boolean(account.password_set_at);
|
||||||
|
const form = document.createElement("form");
|
||||||
|
const label = field("text", "Name, e.g. the script that holds it");
|
||||||
|
label.required = false;
|
||||||
|
label.maxLength = 100;
|
||||||
|
const days = document.createElement("select");
|
||||||
|
days.setAttribute("aria-label", "How long it is valid");
|
||||||
|
for (const [value, text] of [["", "Does not expire"], ["30", "30 days"],
|
||||||
|
["90", "90 days"], ["365", "1 year"]]) {
|
||||||
|
const option = document.createElement("option");
|
||||||
|
option.value = value;
|
||||||
|
option.textContent = text;
|
||||||
|
days.append(option);
|
||||||
|
}
|
||||||
|
const current = field("password", "Password, to create a token");
|
||||||
|
current.autocomplete = "current-password";
|
||||||
|
const create = document.createElement("button");
|
||||||
|
create.textContent = "Create a token";
|
||||||
|
create.disabled = this.#busy;
|
||||||
|
form.append(label, days, ...(asks ? [current] : []), create);
|
||||||
|
form.addEventListener("submit", (event) => {
|
||||||
|
event.preventDefault();
|
||||||
|
this.#act(async () => {
|
||||||
|
const made = await this.#api("/api/account/tokens", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({
|
||||||
|
current: current.value, label: label.value.trim(),
|
||||||
|
days: days.value ? Number(days.value) : null,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
// In the one answer that made it and nowhere after, so it is kept on
|
||||||
|
// the element until Done rather than read back from the account.
|
||||||
|
this.#made = made;
|
||||||
|
this.#copy = "";
|
||||||
|
this.#say(`Token "${made.label || "Token"}" created. Copy it below; it is shown only once.`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
section.append(form);
|
||||||
|
if (this.#made) section.append(this.#shown(this.#made));
|
||||||
|
section.append(this.#line(
|
||||||
|
"A script sends a token as `Authorization: Bearer`. It cannot manage the account, and it is kept through a password change. Revoke it here when the script no longer needs it."));
|
||||||
|
return section;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The new token, read-only and selectable, with Copy and a sentence that
|
||||||
|
* it is the only time. Done drops it from the element, after which the
|
||||||
|
* page never has it again. */
|
||||||
|
#shown(made) {
|
||||||
|
const box = document.createElement("div");
|
||||||
|
box.className = "made";
|
||||||
|
const bar = document.createElement("div");
|
||||||
|
bar.className = "bar";
|
||||||
|
const value = document.createElement("input");
|
||||||
|
value.type = "text";
|
||||||
|
value.readOnly = true;
|
||||||
|
value.className = "token";
|
||||||
|
value.value = made.token;
|
||||||
|
value.setAttribute("aria-label", "The new token");
|
||||||
|
value.addEventListener("focus", () => value.select());
|
||||||
|
bar.append(value,
|
||||||
|
this.#button("Copy", "", async () => {
|
||||||
|
try {
|
||||||
|
await navigator.clipboard.writeText(made.token);
|
||||||
|
this.#copy = "done";
|
||||||
|
} catch {
|
||||||
|
// No clipboard on an insecure origin, or the browser refused: the
|
||||||
|
// field is selectable, so the token is still there to take by hand.
|
||||||
|
this.#copy = "failed";
|
||||||
|
}
|
||||||
|
this.#render();
|
||||||
|
}),
|
||||||
|
this.#button("Done", "", () => {
|
||||||
|
this.#made = null;
|
||||||
|
this.#copy = "";
|
||||||
|
this.#render();
|
||||||
|
}));
|
||||||
|
box.append(bar, this.#line(
|
||||||
|
"This is the only time the token is shown. Only a hash is stored, so it cannot be looked up again; if it is lost, revoke it and create another."));
|
||||||
|
if (this.#copy === "done") box.append(this.#line("Copied."));
|
||||||
|
if (this.#copy === "failed") box.append(this.#note("Could not copy. Select the token and copy it by hand.", true));
|
||||||
|
return box;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #register(current) {
|
||||||
const passkey = await loadPasskeyHelper();
|
const passkey = await loadPasskeyHelper();
|
||||||
if (!passkey || !passkey.supported()) {
|
if (!passkey || !passkey.supported()) {
|
||||||
this.#say("This browser will not create a passkey here. It needs HTTPS, or localhost.");
|
this.#say("This browser will not create a passkey here. It needs HTTPS, or localhost.");
|
||||||
@@ -361,7 +535,9 @@ class AuthAccount extends HTMLElement {
|
|||||||
// browser's own prompt happens between them, and a cancelled prompt is an
|
// browser's own prompt happens between them, and a cancelled prompt is an
|
||||||
// exception rather than an error status -- hence `explain`.
|
// exception rather than an error status -- hence `explain`.
|
||||||
await this.#act(async () => {
|
await this.#act(async () => {
|
||||||
const options = await this.#api("/api/account/passkeys/options", { method: "POST" });
|
const options = await this.#api("/api/account/passkeys/options", {
|
||||||
|
method: "POST", body: JSON.stringify({ current }),
|
||||||
|
});
|
||||||
let made;
|
let made;
|
||||||
try {
|
try {
|
||||||
made = await passkey.create(options);
|
made = await passkey.create(options);
|
||||||
@@ -485,6 +661,19 @@ function field(type, placeholder) {
|
|||||||
return input;
|
return input;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** What a password change closed and what it left, in a sentence or three. */
|
||||||
|
function changed(result) {
|
||||||
|
const said = ["Password changed."];
|
||||||
|
if (result.signed_out) said.push(`${count(result.signed_out, "other session")} signed out.`);
|
||||||
|
if (result.tokens_revoked || result.passkeys_revoked) {
|
||||||
|
said.push(`${count(result.tokens_revoked, "API token")} and ${count(result.passkeys_revoked, "passkey")} revoked.`);
|
||||||
|
}
|
||||||
|
if (result.tokens_left || result.passkeys_left) {
|
||||||
|
said.push(`${count(result.tokens_left, "API token")} and ${count(result.passkeys_left, "passkey")} remain; revoke any you do not recognise.`);
|
||||||
|
}
|
||||||
|
return said.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
function count(n, thing) {
|
function count(n, thing) {
|
||||||
return `${n} ${thing}${n === 1 ? "" : "s"}`;
|
return `${n} ${thing}${n === 1 ? "" : "s"}`;
|
||||||
}
|
}
|
||||||
@@ -498,6 +687,14 @@ function when(stamp) {
|
|||||||
return at.toLocaleString();
|
return at.toLocaleString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** What a token's row says about its end: no end, one to come, or one that
|
||||||
|
* has passed and left the row for the owner to clear. */
|
||||||
|
function lifetime(token) {
|
||||||
|
if (!token.expires_at) return "does not expire";
|
||||||
|
const gone = new Date(token.expires_at).getTime() <= Date.now();
|
||||||
|
return `${gone ? "expired" : "expires"} ${when(token.expires_at)}`;
|
||||||
|
}
|
||||||
|
|
||||||
if (!customElements.get("auth-account")) {
|
if (!customElements.get("auth-account")) {
|
||||||
customElements.define("auth-account", AuthAccount);
|
customElements.define("auth-account", AuthAccount);
|
||||||
}
|
}
|
||||||
|
|||||||
+207
-26
@@ -1,6 +1,7 @@
|
|||||||
"""One password, and the sessions it opens. Nothing else knows a hash.
|
"""One password, the sessions it opens, and the tokens that stand in for it
|
||||||
|
from a script. Nothing else knows a hash.
|
||||||
|
|
||||||
Four tables in whatever SQLite file it is given — the app's own, normally, so
|
Five tables in whatever SQLite file it is given — the app's own, normally, so
|
||||||
there is one file to back up and SQLite's writer lock keeps auth honest with
|
there is one file to back up and SQLite's writer lock keeps auth honest with
|
||||||
everything else in it. Every name starts `auth_`, because the file is shared:
|
everything else in it. Every name starts `auth_`, because the file is shared:
|
||||||
a `sessions` table is a name an app is entitled to want for itself.
|
a `sessions` table is a name an app is entitled to want for itself.
|
||||||
@@ -54,7 +55,7 @@ from contextlib import closing
|
|||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from auth.models import Passkey, SessionInfo
|
from auth.models import ApiToken, Passkey, SessionInfo
|
||||||
|
|
||||||
#: scrypt cost. `n` is the work factor, and 2**14 with `r=8` is the widely
|
#: scrypt cost. `n` is the work factor, and 2**14 with `r=8` is the widely
|
||||||
#: recommended interactive setting: about 16 MB and ~50 ms per guess, which is
|
#: recommended interactive setting: about 16 MB and ~50 ms per guess, which is
|
||||||
@@ -86,6 +87,12 @@ _MAX_LOCK = timedelta(minutes=15)
|
|||||||
#: the ceiling. See `_lockout` for what goes wrong without it.
|
#: the ceiling. See `_lockout` for what goes wrong without it.
|
||||||
_MAX_STEPS = math.ceil(math.log2(_MAX_LOCK / _FIRST_LOCK))
|
_MAX_STEPS = math.ceil(math.log2(_MAX_LOCK / _FIRST_LOCK))
|
||||||
|
|
||||||
|
#: How much of a token the account screen is shown, so the owner can tell
|
||||||
|
#: which environment variable holds which row. Eight of forty-three characters
|
||||||
|
#: from `token_urlsafe(32)` leaves 210 bits unknown, which is still past
|
||||||
|
#: anybody's ability to guess.
|
||||||
|
HINT = 8
|
||||||
|
|
||||||
#: How stale `last_seen_at` may get before a request bothers to update it.
|
#: How stale `last_seen_at` may get before a request bothers to update it.
|
||||||
#: Without this, every single request — every socket frame's sibling fetch,
|
#: Without this, every single request — every socket frame's sibling fetch,
|
||||||
#: every poll of a page left open — takes SQLite's write lock to move a
|
#: every poll of a page left open — takes SQLite's write lock to move a
|
||||||
@@ -96,7 +103,7 @@ _SEEN_EVERY = timedelta(minutes=5)
|
|||||||
|
|
||||||
#: The tables. Created on construction rather than by a migration step, because
|
#: The tables. Created on construction rather than by a migration step, because
|
||||||
#: the only schema change this has ever had is "it did not exist yet", and a
|
#: the only schema change this has ever had is "it did not exist yet", and a
|
||||||
#: migration runner for four `CREATE TABLE IF NOT EXISTS` is a thing to
|
#: migration runner for five `CREATE TABLE IF NOT EXISTS` is a thing to
|
||||||
#: maintain in exchange for nothing.
|
#: maintain in exchange for nothing.
|
||||||
SCHEMA = """
|
SCHEMA = """
|
||||||
CREATE TABLE IF NOT EXISTS auth_credential (
|
CREATE TABLE IF NOT EXISTS auth_credential (
|
||||||
@@ -130,6 +137,15 @@ CREATE TABLE IF NOT EXISTS auth_challenges (
|
|||||||
purpose TEXT NOT NULL,
|
purpose TEXT NOT NULL,
|
||||||
expires_at TEXT NOT NULL
|
expires_at TEXT NOT NULL
|
||||||
);
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS auth_tokens (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
token_hash TEXT NOT NULL UNIQUE,
|
||||||
|
hint TEXT NOT NULL DEFAULT '',
|
||||||
|
label TEXT NOT NULL DEFAULT '',
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
last_used_at TEXT NOT NULL DEFAULT '',
|
||||||
|
expires_at TEXT NOT NULL DEFAULT ''
|
||||||
|
);
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
@@ -151,7 +167,7 @@ def _derive(password: str, salt: bytes) -> bytes:
|
|||||||
|
|
||||||
|
|
||||||
class AuthStore:
|
class AuthStore:
|
||||||
"""The four `auth_*` tables, opened on the path it is handed.
|
"""The five `auth_*` tables, opened on the path it is handed.
|
||||||
|
|
||||||
store = AuthStore("data/app.db")
|
store = AuthStore("data/app.db")
|
||||||
|
|
||||||
@@ -207,13 +223,8 @@ class AuthStore:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def locked_for(self) -> int:
|
def locked_for(self) -> int:
|
||||||
"""Seconds until the next attempt will be looked at, or 0.
|
"""Seconds until the next attempt will be looked at, or 0. Reads only;
|
||||||
|
the doors ask `claim_attempt`, which decides and counts in one step.
|
||||||
Read before checking rather than after, so a locked-out caller is told
|
|
||||||
to wait instead of having a guess counted — otherwise a fast enough
|
|
||||||
attacker extends their own lockout forever and never notices, and, more
|
|
||||||
to the point, an honest person hammering the button digs a deeper hole
|
|
||||||
with every press.
|
|
||||||
"""
|
"""
|
||||||
with closing(self._connect()) as conn:
|
with closing(self._connect()) as conn:
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
@@ -223,9 +234,61 @@ class AuthStore:
|
|||||||
remaining = datetime.fromisoformat(row["locked_until"]) - _now()
|
remaining = datetime.fromisoformat(row["locked_until"]) - _now()
|
||||||
return max(0, int(remaining.total_seconds() + 0.999))
|
return max(0, int(remaining.total_seconds() + 0.999))
|
||||||
|
|
||||||
def check_password(self, password: str) -> bool:
|
def claim_attempt(self) -> int:
|
||||||
|
"""Claim one guess at the password before it is derived: 0 when the
|
||||||
|
guess may go ahead, else the seconds until one may.
|
||||||
|
|
||||||
|
Inside one `BEGIN IMMEDIATE`, `locked_until` is read and, when it has
|
||||||
|
passed, the attempt is counted as a failure up front and the lockout
|
||||||
|
it would earn is written; `check_password(..., claimed=True)` then
|
||||||
|
clears both when the guess was right. Read before scrypt and written
|
||||||
|
after it, thirty wrong guesses in parallel each found the door open
|
||||||
|
and each got a 401, so a window admitted a threadpool of guesses
|
||||||
|
rather than `_GRACE`. Now the sixth concurrent claim finds the lockout
|
||||||
|
the fifth wrote.
|
||||||
|
|
||||||
|
A refused claim writes nothing, so a caller locked out is told to wait
|
||||||
|
instead of having a guess counted: otherwise a fast enough attacker
|
||||||
|
extends their own lockout forever and never notices, and an honest
|
||||||
|
person hammering the button digs a deeper hole with every press. On a
|
||||||
|
database with no password there is nothing to count and nothing to
|
||||||
|
lock, and the claim is granted.
|
||||||
|
"""
|
||||||
|
with closing(self._connect()) as conn:
|
||||||
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT failures, locked_until FROM auth_credential WHERE id=1"
|
||||||
|
).fetchone()
|
||||||
|
wait = 0
|
||||||
|
if row is not None:
|
||||||
|
now = _now()
|
||||||
|
if row["locked_until"]:
|
||||||
|
remaining = datetime.fromisoformat(row["locked_until"]) - now
|
||||||
|
wait = max(0, int(remaining.total_seconds() + 0.999))
|
||||||
|
if not wait:
|
||||||
|
failures = int(row["failures"]) + 1
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE auth_credential SET failures=?, locked_until=? WHERE id=1",
|
||||||
|
(failures, _stamp(now + _lockout(failures))
|
||||||
|
if failures >= _GRACE else ""))
|
||||||
|
conn.commit()
|
||||||
|
except BaseException:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
return wait
|
||||||
|
|
||||||
|
def check_password(self, password: str, *, claimed: bool = False) -> bool:
|
||||||
"""Right or wrong, and record which.
|
"""Right or wrong, and record which.
|
||||||
|
|
||||||
|
`claimed` says the caller counted this guess with `claim_attempt`
|
||||||
|
already, which is what both doors do: a wrong answer then writes
|
||||||
|
nothing more and a right one clears the count. Without it a wrong
|
||||||
|
answer is counted here, after the derivation, which is what a caller
|
||||||
|
that is not a door wants: `auth.bootstrap` compares the pinned
|
||||||
|
password, and its comparison cannot lock anyone out because it is
|
||||||
|
followed by `set_password`, which clears the count.
|
||||||
|
|
||||||
`hmac.compare_digest` rather than `==`, so the comparison takes the same
|
`hmac.compare_digest` rather than `==`, so the comparison takes the same
|
||||||
time whatever the first wrong byte is. That is close to superstition at
|
time whatever the first wrong byte is. That is close to superstition at
|
||||||
this scale — the network jitter in front of it is larger than the signal
|
this scale — the network jitter in front of it is larger than the signal
|
||||||
@@ -247,29 +310,42 @@ class AuthStore:
|
|||||||
owner still gets locked out by their own typos: strictly worse than
|
owner still gets locked out by their own typos: strictly worse than
|
||||||
this, and it would look like protection. So the address is written to
|
this, and it would look like protection. So the address is written to
|
||||||
the log instead, at WARNING, where fail2ban can act on it at the layer
|
the log instead, at WARNING, where fail2ban can act on it at the layer
|
||||||
that actually knows who connected — see `_warn` in `auth/router.py`.
|
that actually knows who connected — see `warn` in `auth/middleware.py`.
|
||||||
What is left for this counter to do is make guessing pointless, which it
|
What is left for this counter to do is make guessing pointless, which it
|
||||||
does whoever is guessing.
|
does whoever is guessing.
|
||||||
"""
|
"""
|
||||||
with closing(self._connect()) as conn:
|
with closing(self._connect()) as conn:
|
||||||
row = conn.execute(
|
# `fetchall`, so no read is left open on this connection when it
|
||||||
"SELECT salt, hash, failures FROM auth_credential WHERE id=1"
|
# asks for the write lock below.
|
||||||
).fetchone()
|
rows = conn.execute(
|
||||||
if row is None:
|
"SELECT salt, hash FROM auth_credential WHERE id=1").fetchall()
|
||||||
|
if not rows:
|
||||||
_derive(password, b"no password here")
|
_derive(password, b"no password here")
|
||||||
return False
|
return False
|
||||||
ok = hmac.compare_digest(_derive(password, row["salt"]), row["hash"])
|
ok = hmac.compare_digest(_derive(password, rows[0]["salt"]), rows[0]["hash"])
|
||||||
with conn:
|
if claimed and not ok:
|
||||||
|
return False
|
||||||
|
# The count is raised in the database under the write lock, and
|
||||||
|
# the lockout is decided from what that raise returned. Read
|
||||||
|
# before scrypt and written after it, forty guesses in parallel
|
||||||
|
# each wrote "one more than before" and recorded one failure.
|
||||||
|
conn.execute("BEGIN IMMEDIATE")
|
||||||
|
try:
|
||||||
if ok:
|
if ok:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE auth_credential SET failures=0, locked_until='' WHERE id=1")
|
"UPDATE auth_credential SET failures=0, locked_until='' WHERE id=1")
|
||||||
else:
|
elif counted := conn.execute(
|
||||||
failures = int(row["failures"]) + 1
|
"UPDATE auth_credential SET failures = failures + 1 WHERE id=1 "
|
||||||
|
"RETURNING failures").fetchall():
|
||||||
|
failures = int(counted[0][0])
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE auth_credential SET failures=?, locked_until=? WHERE id=1",
|
"UPDATE auth_credential SET locked_until=? WHERE id=1",
|
||||||
(failures, _stamp(_now() + _lockout(failures))
|
(_stamp(_now() + _lockout(failures))
|
||||||
if failures >= _GRACE else ""),
|
if failures >= _GRACE else "",))
|
||||||
)
|
conn.commit()
|
||||||
|
except BaseException:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
return ok
|
return ok
|
||||||
|
|
||||||
def changed_at(self) -> str:
|
def changed_at(self) -> str:
|
||||||
@@ -368,6 +444,104 @@ class AuthStore:
|
|||||||
cur = conn.execute("DELETE FROM auth_sessions")
|
cur = conn.execute("DELETE FROM auth_sessions")
|
||||||
return cur.rowcount
|
return cur.rowcount
|
||||||
|
|
||||||
|
def revoke_credentials(self) -> tuple[int, int]:
|
||||||
|
"""Delete every API token and every passkey in one transaction, for a
|
||||||
|
password change made because the account may be in someone else's
|
||||||
|
hands. Returns `(tokens, passkeys)` deleted. The password stays, so
|
||||||
|
this cannot lock the owner out."""
|
||||||
|
with closing(self._connect()) as conn, conn:
|
||||||
|
tokens = conn.execute("DELETE FROM auth_tokens").rowcount
|
||||||
|
keys = conn.execute("DELETE FROM auth_passkeys").rowcount
|
||||||
|
return tokens, keys
|
||||||
|
|
||||||
|
def counts(self) -> tuple[int, int]:
|
||||||
|
"""`(tokens, passkeys)` registered, expired tokens included, for a
|
||||||
|
caller that reports what is left after a password change."""
|
||||||
|
with closing(self._connect()) as conn:
|
||||||
|
tokens = conn.execute("SELECT COUNT(*) FROM auth_tokens").fetchone()[0]
|
||||||
|
keys = conn.execute("SELECT COUNT(*) FROM auth_passkeys").fetchone()[0]
|
||||||
|
return int(tokens), int(keys)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- tokens
|
||||||
|
#
|
||||||
|
# A token is a session for a script: the same 32 random bytes, the same
|
||||||
|
# SHA-256 in the file, carried as `Authorization: Bearer` instead of a
|
||||||
|
# cookie. It differs from a session in what it is for. A session is a
|
||||||
|
# browser that typed the password once and is signed out by a password
|
||||||
|
# change; a token is minted on purpose, named, optionally dated, and
|
||||||
|
# survives a password change the way a passkey does, because the cron job
|
||||||
|
# holding it is not the thing the password was changed against. It is
|
||||||
|
# revoked by its row, from the account screen, which a token cannot reach
|
||||||
|
# (`auth.middleware.ACCOUNT`).
|
||||||
|
|
||||||
|
def create_token(self, label: str, days: int | None) -> tuple[int, str]:
|
||||||
|
"""Mint one. Returns `(row id, token)`; the token exists only in that
|
||||||
|
return value, so the caller shows it once and never again.
|
||||||
|
|
||||||
|
`days` of None is a token that does not expire: the cron job that
|
||||||
|
holds it is not going to type a new one, and the row's last-used
|
||||||
|
stamp is what tells an abandoned one from one in use.
|
||||||
|
"""
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
made = _now()
|
||||||
|
expires = _stamp(made + timedelta(days=days)) if days else ""
|
||||||
|
with closing(self._connect()) as conn, conn:
|
||||||
|
cur = conn.execute(
|
||||||
|
"""INSERT INTO auth_tokens
|
||||||
|
(token_hash, hint, label, created_at, last_used_at, expires_at)
|
||||||
|
VALUES (?,?,?,?,'',?)""",
|
||||||
|
(_token_hash(token), token[:HINT], label[:100], _stamp(made), expires),
|
||||||
|
)
|
||||||
|
return int(cur.lastrowid or 0), token
|
||||||
|
|
||||||
|
def touch_token(self, token: str) -> ApiToken | None:
|
||||||
|
"""The token this bearer value names, if it is live, marking it used.
|
||||||
|
|
||||||
|
Expiry is in the `WHERE` as it is for a session; a blank `expires_at`
|
||||||
|
is a token that does not. `last_used_at` is written at most once every
|
||||||
|
`_SEEN_EVERY`, for the reason `touch` gives.
|
||||||
|
"""
|
||||||
|
digest = _token_hash(token)
|
||||||
|
now = _now()
|
||||||
|
stamp = _stamp(now)
|
||||||
|
with closing(self._connect()) as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT * FROM auth_tokens WHERE token_hash=? "
|
||||||
|
"AND (expires_at = '' OR expires_at > ?)",
|
||||||
|
(digest, stamp),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
used = row["last_used_at"]
|
||||||
|
if not used or now - datetime.fromisoformat(used) >= _SEEN_EVERY:
|
||||||
|
with conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE auth_tokens SET last_used_at=? WHERE token_hash=?",
|
||||||
|
(stamp, digest))
|
||||||
|
used = stamp
|
||||||
|
return _api_token(row, used)
|
||||||
|
|
||||||
|
def tokens(self) -> list[ApiToken]:
|
||||||
|
"""Every token, newest first, the expired ones included: a row that
|
||||||
|
ran out is still the owner's to recognise and remove. No hash."""
|
||||||
|
with closing(self._connect()) as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT * FROM auth_tokens ORDER BY created_at DESC, id DESC").fetchall()
|
||||||
|
return [_api_token(r, r["last_used_at"]) for r in rows]
|
||||||
|
|
||||||
|
def rename_token(self, token_id: int, label: str) -> bool:
|
||||||
|
with closing(self._connect()) as conn, conn:
|
||||||
|
cur = conn.execute("UPDATE auth_tokens SET label=? WHERE id=?",
|
||||||
|
(label[:100], token_id))
|
||||||
|
return cur.rowcount > 0
|
||||||
|
|
||||||
|
def delete_token(self, token_id: int) -> bool:
|
||||||
|
"""Revoke one. By row id, as a session is: the screen has never seen
|
||||||
|
the token after the moment it was minted."""
|
||||||
|
with closing(self._connect()) as conn, conn:
|
||||||
|
cur = conn.execute("DELETE FROM auth_tokens WHERE id=?", (token_id,))
|
||||||
|
return cur.rowcount > 0
|
||||||
|
|
||||||
# -------------------------------------------------------------- passkeys
|
# -------------------------------------------------------------- passkeys
|
||||||
|
|
||||||
def add_passkey(self, credential_id: bytes, public_key: bytes,
|
def add_passkey(self, credential_id: bytes, public_key: bytes,
|
||||||
@@ -503,3 +677,10 @@ def _session(row: sqlite3.Row, last_seen: str, current: bool) -> SessionInfo:
|
|||||||
id=row["id"], label=row["label"], created_at=row["created_at"],
|
id=row["id"], label=row["label"], created_at=row["created_at"],
|
||||||
last_seen_at=last_seen, expires_at=row["expires_at"], current=current,
|
last_seen_at=last_seen, expires_at=row["expires_at"], current=current,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _api_token(row: sqlite3.Row, last_used: str) -> ApiToken:
|
||||||
|
return ApiToken(
|
||||||
|
id=row["id"], label=row["label"], hint=row["hint"], created_at=row["created_at"],
|
||||||
|
last_used_at=last_used, expires_at=row["expires_at"],
|
||||||
|
)
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8">
|
<meta charset="utf-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
||||||
<title>{name} — sign in</title>
|
<title>{name} — sign in</title>
|
||||||
<!-- The app's own stylesheet, if it passed one to `connect`. Optional on
|
<!-- The app's own stylesheet, if it passed one to `connect`. Optional on
|
||||||
purpose: this folder is copied into projects that have no stylesheet yet,
|
purpose: this folder is copied into projects that have no stylesheet yet,
|
||||||
@@ -42,6 +42,13 @@ body.login {
|
|||||||
* saying twice. Without it the passkey button is drawn on an app that has no
|
* saying twice. Without it the passkey button is drawn on an app that has no
|
||||||
* passkeys, and pressing it 404s. */
|
* passkeys, and pressing it 404s. */
|
||||||
[hidden] { display: none !important; }
|
[hidden] { display: none !important; }
|
||||||
|
/* A phone: the form clears the notch and the home indicator, and a finger
|
||||||
|
* gets a thumb's height of button and a field that does not make iOS zoom. */
|
||||||
|
body.login { padding: env(safe-area-inset-top) 0 env(safe-area-inset-bottom); }
|
||||||
|
@media (pointer: coarse) {
|
||||||
|
.signin input[type=password] { font-size: 16px; padding: 0.6rem 0.7rem; }
|
||||||
|
.signin button { min-height: 2.75rem; }
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body class="login">
|
<body class="login">
|
||||||
|
|||||||
@@ -68,8 +68,11 @@ def locked(make_client: Callable[..., TestClient]) -> TestClient:
|
|||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def open_app(make_client: Callable[..., TestClient]) -> TestClient:
|
def open_app(make_client: Callable[..., TestClient]) -> TestClient:
|
||||||
"""The same app on loopback under `auto` — no password asked for."""
|
"""The same app on loopback under `auto` — no password asked for. Reached
|
||||||
return make_client(enabled="auto", host="127.0.0.1", password=None)
|
as 127.0.0.1, which is the name an unlocked app answers a browser under
|
||||||
|
(`auth.middleware._rebound`)."""
|
||||||
|
return make_client(enabled="auto", host="127.0.0.1", password=None,
|
||||||
|
base_url="http://127.0.0.1")
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|||||||
+43
-4
@@ -5,7 +5,8 @@ one, so its tests are not allowed to reach for `app/` or `pipeline/` either --
|
|||||||
`tests/test_coupling.py` reads the imports of every file under this folder and
|
`tests/test_coupling.py` reads the imports of every file under this folder and
|
||||||
says so. What they need is a FastAPI app with a page, an API route, a socket,
|
says so. What they need is a FastAPI app with a page, an API route, a socket,
|
||||||
and a couple of neighbours to prove the open list is matched the way it claims.
|
and a couple of neighbours to prove the open list is matched the way it claims.
|
||||||
That is this file, and it is nine routes.
|
That is this file, and it is nine routes and a stand-in for the broadcaster
|
||||||
|
the socket subscribes to.
|
||||||
|
|
||||||
A module rather than a conftest fixture because two tests build the same
|
A module rather than a conftest fixture because two tests build the same
|
||||||
database twice, to see what a restart does, and a fixture cannot be asked for
|
database twice, to see what a restart does, and a fixture cannot be asked for
|
||||||
@@ -15,11 +16,32 @@ twice in one test.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
from fastapi import FastAPI, WebSocket
|
from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
|
||||||
from fastapi.responses import PlainTextResponse
|
from fastapi.responses import PlainTextResponse
|
||||||
|
|
||||||
from auth import AuthConfig, AuthStore, connect
|
from auth import AuthConfig, AuthStore, confirm_current, connect
|
||||||
|
|
||||||
|
|
||||||
|
class Bus:
|
||||||
|
"""What `auth.close_ended_sockets` reads off `app.state.bus`: the open
|
||||||
|
sockets and a way to close some. A stand-in for `pipeline.web.broadcaster.
|
||||||
|
Broadcaster`, which this folder may not import, with its two members."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.held: set[WebSocket] = set()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sockets(self) -> tuple[WebSocket, ...]:
|
||||||
|
return tuple(self.held)
|
||||||
|
|
||||||
|
async def close(self, sockets: Iterable[WebSocket], code: int, reason: str = "") -> int:
|
||||||
|
gone = [ws for ws in sockets if ws in self.held]
|
||||||
|
for ws in gone:
|
||||||
|
self.held.discard(ws)
|
||||||
|
await ws.close(code, reason)
|
||||||
|
return len(gone)
|
||||||
|
|
||||||
|
|
||||||
def build(db: str | os.PathLike[str], *, host: str = "0.0.0.0",
|
def build(db: str | os.PathLike[str], *, host: str = "0.0.0.0",
|
||||||
@@ -28,6 +50,7 @@ def build(db: str | os.PathLike[str], *, host: str = "0.0.0.0",
|
|||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
connect(app, AuthStore(db), AuthConfig(**settings), host=host,
|
connect(app, AuthStore(db), AuthConfig(**settings), host=host,
|
||||||
open=("/static/",), stylesheet="/static/app.css")
|
open=("/static/",), stylesheet="/static/app.css")
|
||||||
|
app.state.bus = bus = Bus()
|
||||||
|
|
||||||
@app.get("/", response_class=PlainTextResponse)
|
@app.get("/", response_class=PlainTextResponse)
|
||||||
def front() -> str:
|
def front() -> str:
|
||||||
@@ -41,6 +64,13 @@ def build(db: str | os.PathLike[str], *, host: str = "0.0.0.0",
|
|||||||
def thing() -> dict[str, bool]:
|
def thing() -> dict[str, bool]:
|
||||||
return {"ok": True}
|
return {"ok": True}
|
||||||
|
|
||||||
|
@app.post("/thing/replace")
|
||||||
|
def replace(request: Request) -> dict[str, bool]:
|
||||||
|
"""A route that swaps the database as a file would: the current
|
||||||
|
password first, in its header, before any body."""
|
||||||
|
confirm_current(request)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
def health() -> dict[str, bool]:
|
def health() -> dict[str, bool]:
|
||||||
return {"ok": True}
|
return {"ok": True}
|
||||||
@@ -61,8 +91,17 @@ def build(db: str | os.PathLike[str], *, host: str = "0.0.0.0",
|
|||||||
|
|
||||||
@app.websocket("/ws/jobs")
|
@app.websocket("/ws/jobs")
|
||||||
async def socket(ws: WebSocket) -> None:
|
async def socket(ws: WebSocket) -> None:
|
||||||
|
"""Held open until the client leaves or the bus closes it, as the
|
||||||
|
app's `/ws/jobs` is."""
|
||||||
await ws.accept()
|
await ws.accept()
|
||||||
|
bus.held.add(ws)
|
||||||
|
try:
|
||||||
await ws.send_json({"type": "jobs"})
|
await ws.send_json({"type": "jobs"})
|
||||||
await ws.close()
|
while True:
|
||||||
|
await ws.receive_text()
|
||||||
|
except WebSocketDisconnect:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
bus.held.discard(ws)
|
||||||
|
|
||||||
return app
|
return app
|
||||||
+217
-8
@@ -8,15 +8,19 @@ from __future__ import annotations
|
|||||||
import logging
|
import logging
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from contextlib import closing
|
from contextlib import closing
|
||||||
from urllib.parse import parse_qs, urlsplit
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from fastapi import Request
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
from starlette.requests import HTTPConnection
|
||||||
from starlette.websockets import WebSocketDisconnect
|
from starlette.websockets import WebSocketDisconnect
|
||||||
|
|
||||||
from auth import AuthStore
|
from auth import AuthStore
|
||||||
from auth.middleware import OPEN
|
from auth.middleware import OPEN
|
||||||
|
from auth.store import _GRACE
|
||||||
from auth.tests import PASSWORD, sign_in
|
from auth.tests import PASSWORD, sign_in
|
||||||
from auth.tests.harness import build
|
from auth.tests.harness import build
|
||||||
|
|
||||||
@@ -27,7 +31,9 @@ def test_loopback_under_auto_needs_no_password(open_app: TestClient):
|
|||||||
"""The default on a laptop: the machine's own login is already the door."""
|
"""The default on a laptop: the machine's own login is already the door."""
|
||||||
assert open_app.get("/").status_code == 200
|
assert open_app.get("/").status_code == 200
|
||||||
assert open_app.get("/api/thing").status_code == 200
|
assert open_app.get("/api/thing").status_code == 200
|
||||||
with open_app.websocket_connect("/ws/jobs") as socket:
|
# The handshake names `testserver` unless told otherwise, and an unlocked
|
||||||
|
# app answers only to the names it is reached under.
|
||||||
|
with open_app.websocket_connect("/ws/jobs", headers={"host": "127.0.0.1"}) as socket:
|
||||||
assert socket.receive_json()["type"] == "jobs"
|
assert socket.receive_json()["type"] == "jobs"
|
||||||
|
|
||||||
|
|
||||||
@@ -49,6 +55,49 @@ def test_never_is_open_off_loopback_too(make_client: Callable[..., TestClient]):
|
|||||||
assert client.get("/api/thing").status_code == 200
|
assert client.get("/api/thing").status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------ a proxy on the same machine
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("header", [
|
||||||
|
{"x-forwarded-for": "203.0.113.9"}, {"x-forwarded-proto": "https"},
|
||||||
|
{"x-forwarded-host": "app.example.com"}, {"forwarded": "for=203.0.113.9"},
|
||||||
|
])
|
||||||
|
def test_an_open_app_refuses_what_a_proxy_forwarded(open_app: TestClient, header):
|
||||||
|
"""`auto` on loopback is open because only this machine can reach it. A
|
||||||
|
reverse proxy on the same machine makes that false while every request
|
||||||
|
still comes from 127.0.0.1; its headers are the only sign, so a request
|
||||||
|
carrying one is refused rather than served unlocked."""
|
||||||
|
refused = open_app.get("/api/thing", headers=header)
|
||||||
|
assert refused.status_code == 421
|
||||||
|
assert "auth.enabled: always" in refused.json()["detail"]
|
||||||
|
assert open_app.get("/api/thing").status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_proxy_refusal_is_logged_once(open_app: TestClient, caplog):
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
for _ in range(3):
|
||||||
|
open_app.get("/api/thing", headers={"x-forwarded-for": "203.0.113.9"})
|
||||||
|
said = [r.getMessage() for r in caplog.records if "proxied" in r.getMessage()]
|
||||||
|
assert len(said) == 1 and "auth.enabled: always" in said[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_proxy_refusal_covers_the_socket(open_app: TestClient):
|
||||||
|
with pytest.raises(WebSocketDisconnect), \
|
||||||
|
open_app.websocket_connect("/ws/jobs", headers={"x-forwarded-for": "203.0.113.9"}):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_locked_or_never_locked_app_serves_a_proxy(
|
||||||
|
locked: TestClient, make_client: Callable[..., TestClient]):
|
||||||
|
"""`always` has its own door and `never` is the operator's word: neither
|
||||||
|
is second-guessed from the headers."""
|
||||||
|
forwarded = {"x-forwarded-for": "203.0.113.9", "x-forwarded-proto": "https"}
|
||||||
|
assert locked.post("/login", json={"password": PASSWORD},
|
||||||
|
headers=forwarded).status_code == 200
|
||||||
|
assert locked.get("/api/thing", headers=forwarded).status_code == 200
|
||||||
|
never = make_client(enabled="never", host="127.0.0.1")
|
||||||
|
assert never.get("/api/thing", headers=forwarded).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
# -------------------------------------------------------- the locked door
|
# -------------------------------------------------------- the locked door
|
||||||
|
|
||||||
def test_everything_is_shut(locked: TestClient):
|
def test_everything_is_shut(locked: TestClient):
|
||||||
@@ -150,6 +199,21 @@ def test_a_wrong_password_is_refused(locked: TestClient):
|
|||||||
assert locked.get("/api/thing").status_code == 401
|
assert locked.get("/api/thing").status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_wrong_guesses_in_parallel_meet_the_lockout(locked: TestClient):
|
||||||
|
"""The lockout was read before the derivation and the failure written
|
||||||
|
after it, so thirty wrong guesses at once all found the door open and all
|
||||||
|
got a 401. Each is claimed before it is derived now: the first `_GRACE`
|
||||||
|
are looked at and the rest are told to wait."""
|
||||||
|
def guess(_: int) -> int:
|
||||||
|
# A client per thread, so no two share a cookie jar or a portal.
|
||||||
|
return sign_in(TestClient(locked.app), "nope").status_code
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
|
said = list(pool.map(guess, range(30)))
|
||||||
|
assert said.count(401) == _GRACE, said
|
||||||
|
assert said.count(429) == 30 - _GRACE, said
|
||||||
|
|
||||||
|
|
||||||
def test_signing_in_opens_everything(locked: TestClient):
|
def test_signing_in_opens_everything(locked: TestClient):
|
||||||
assert sign_in(locked).status_code == 200
|
assert sign_in(locked).status_code == 200
|
||||||
assert locked.get("/api/thing").status_code == 200
|
assert locked.get("/api/thing").status_code == 200
|
||||||
@@ -352,6 +416,84 @@ def test_the_password_change_is_behind_the_same_lockout(signed_in: TestClient):
|
|||||||
assert sign_in(signed_in).status_code == 429
|
assert sign_in(signed_in).status_code == 429
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_app_route_asks_for_the_password_through_the_package(
|
||||||
|
signed_in: TestClient, caplog):
|
||||||
|
"""A route outside the package that must take the current password (a
|
||||||
|
restore, whose body is the database file) calls `confirm_password`
|
||||||
|
rather than writing its own check, so it is counted and gated like the
|
||||||
|
front door."""
|
||||||
|
from auth import confirm_password, warn
|
||||||
|
|
||||||
|
@signed_in.app.post("/api/restore")
|
||||||
|
def restore(request: Request) -> dict[str, bool]:
|
||||||
|
confirm_password(request, request.headers.get("x-current-password", ""))
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
wrong = signed_in.post("/api/restore", headers={"x-current-password": "nope"})
|
||||||
|
assert wrong.status_code == 403
|
||||||
|
assert "wrong current password from testclient" in caplog.text
|
||||||
|
assert {r.name for r in caplog.records if "from testclient" in r.getMessage()} == {
|
||||||
|
"auth.middleware"}
|
||||||
|
assert signed_in.post("/api/restore",
|
||||||
|
headers={"x-current-password": PASSWORD}).status_code == 200
|
||||||
|
for _ in range(5):
|
||||||
|
signed_in.post("/api/restore", headers={"x-current-password": "nope"})
|
||||||
|
refused = signed_in.post("/api/restore", headers={"x-current-password": PASSWORD})
|
||||||
|
assert refused.status_code == 429 and refused.headers["retry-after"]
|
||||||
|
assert sign_in(signed_in).status_code == 429
|
||||||
|
|
||||||
|
caplog.clear()
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
warn(HTTPConnection({"type": "http", "client": ("203.0.113.9", 1), "headers": []}),
|
||||||
|
"restore refused")
|
||||||
|
assert "restore refused from 203.0.113.9" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_file_route_without_the_header_is_refused_and_counts_nothing(
|
||||||
|
signed_in: TestClient, caplog):
|
||||||
|
"""A restore's body is the database, so the password rides in a header,
|
||||||
|
and a session alone is not enough: the restored file brings its own hash.
|
||||||
|
A missing header is a missing field and not a guess, so five of them leave
|
||||||
|
the right password working, at this route and at the front door."""
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
missing = signed_in.post("/thing/replace")
|
||||||
|
empty = signed_in.post("/thing/replace", headers={"x-current-password": ""})
|
||||||
|
for refused in (missing, empty):
|
||||||
|
assert refused.status_code == 403
|
||||||
|
assert "X-Current-Password" in refused.json()["detail"]
|
||||||
|
assert "no current password for POST /thing/replace from testclient" in caplog.text
|
||||||
|
|
||||||
|
for _ in range(5):
|
||||||
|
assert signed_in.post("/thing/replace").status_code == 403
|
||||||
|
assert signed_in.post("/thing/replace",
|
||||||
|
headers={"x-current-password": PASSWORD}).status_code == 200
|
||||||
|
assert sign_in(signed_in).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_file_route_counts_a_wrong_header(signed_in: TestClient, caplog):
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
wrong = signed_in.post("/thing/replace", headers={"x-current-password": "nope"})
|
||||||
|
assert wrong.status_code == 403
|
||||||
|
assert "wrong current password from testclient" in caplog.text
|
||||||
|
|
||||||
|
for _ in range(4):
|
||||||
|
signed_in.post("/thing/replace", headers={"x-current-password": "nope"})
|
||||||
|
refused = signed_in.post("/thing/replace", headers={"x-current-password": PASSWORD})
|
||||||
|
assert refused.status_code == 429 and refused.headers["retry-after"]
|
||||||
|
assert sign_in(signed_in).status_code == 429
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_file_route_takes_the_right_header(signed_in: TestClient):
|
||||||
|
right = signed_in.post("/thing/replace", headers={"x-current-password": PASSWORD})
|
||||||
|
assert right.status_code == 200 and right.json() == {"ok": True}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_file_route_on_an_app_with_no_password_needs_no_header(open_app: TestClient):
|
||||||
|
"""The laptop that was never locked has nothing to confirm."""
|
||||||
|
assert open_app.post("/thing/replace").status_code == 200
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------- the bootstrap
|
# ------------------------------------------------------- the bootstrap
|
||||||
|
|
||||||
def test_a_generated_password_is_printed_once(
|
def test_a_generated_password_is_printed_once(
|
||||||
@@ -429,6 +571,51 @@ def test_the_environment_variable_is_a_setting(
|
|||||||
assert client.get("/api/account").json()["pinned_by"] == "OTHER_PASSWORD"
|
assert client.get("/api/account").json()["pinned_by"] == "OTHER_PASSWORD"
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_through_the_environment_signs_everyone_out(
|
||||||
|
tmp_path, monkeypatch, caplog):
|
||||||
|
"""`APP_PASSWORD` is the way back in when the password is lost, and the
|
||||||
|
reason it is lost may be that someone else has it and is signed in. So a
|
||||||
|
pinned password that replaces the stored one closes every session. Tokens
|
||||||
|
and passkeys stay, since the environment cannot tell whose they are, and
|
||||||
|
the log says how many so the owner knows to review them."""
|
||||||
|
db = tmp_path / "recovered.db"
|
||||||
|
store = AuthStore(db)
|
||||||
|
store.set_password("the-one-the-intruder-has")
|
||||||
|
intruder = store.open_session("intruder", 30)
|
||||||
|
_, token = store.create_token("intruder's cron", None)
|
||||||
|
store.add_passkey(b"cred", b"key", 0, "", "intruder's key")
|
||||||
|
|
||||||
|
monkeypatch.setenv("APP_PASSWORD", PASSWORD)
|
||||||
|
with caplog.at_level(logging.WARNING), \
|
||||||
|
TestClient(build(db, enabled="always"), follow_redirects=False) as client:
|
||||||
|
assert client.get("/api/thing",
|
||||||
|
headers={"cookie": f"app_session={intruder}"}).status_code == 401
|
||||||
|
assert client.get("/api/thing",
|
||||||
|
headers={"authorization": f"Bearer {token}"}).status_code == 200
|
||||||
|
assert sign_in(client).status_code == 200
|
||||||
|
mine = client.cookies["app_session"]
|
||||||
|
said = [r for r in caplog.records if "password reset from APP_PASSWORD" in r.getMessage()]
|
||||||
|
assert len(said) == 1 and said[0].levelno == logging.WARNING
|
||||||
|
assert "closed 1 sessions; 1 API tokens and 1 passkeys remain" in said[0].getMessage()
|
||||||
|
|
||||||
|
# A restart with the same variable matches and closes nothing.
|
||||||
|
caplog.clear()
|
||||||
|
with caplog.at_level(logging.WARNING), TestClient(build(db, enabled="always")) as client:
|
||||||
|
assert client.get("/api/thing",
|
||||||
|
headers={"cookie": f"app_session={mine}"}).status_code == 200
|
||||||
|
assert "password reset" not in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_first_password_from_the_environment_is_not_a_recovery(
|
||||||
|
make_client: Callable[..., TestClient], monkeypatch, caplog):
|
||||||
|
monkeypatch.setenv("APP_PASSWORD", PASSWORD)
|
||||||
|
with caplog.at_level(logging.INFO):
|
||||||
|
client = make_client(enabled="always", password=None)
|
||||||
|
assert "password set from APP_PASSWORD" in caplog.text
|
||||||
|
assert "password reset" not in caplog.text
|
||||||
|
assert sign_in(client).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
def test_expired_sessions_are_purged_at_startup(tmp_path):
|
def test_expired_sessions_are_purged_at_startup(tmp_path):
|
||||||
"""Housekeeping, not a security boundary -- `touch` already refuses an
|
"""Housekeeping, not a security boundary -- `touch` already refuses an
|
||||||
expired row. Without it the table grows forever."""
|
expired row. Without it the table grows forever."""
|
||||||
@@ -452,8 +639,8 @@ def test_a_refused_login_says_who_it_was(locked: TestClient, caplog):
|
|||||||
nothing anywhere knows it happened."""
|
nothing anywhere knows it happened."""
|
||||||
with caplog.at_level(logging.WARNING):
|
with caplog.at_level(logging.WARNING):
|
||||||
assert sign_in(locked, "nope").status_code == 401
|
assert sign_in(locked, "nope").status_code == 401
|
||||||
# The logger is `auth.router`, so the line already says which package this
|
# The logger is `auth.middleware`, so the line already says which package
|
||||||
# came from and the message does not say it again.
|
# this came from and the message does not say it again.
|
||||||
assert "wrong password from testclient" in caplog.text
|
assert "wrong password from testclient" in caplog.text
|
||||||
assert "auth: wrong password" not in caplog.text
|
assert "auth: wrong password" not in caplog.text
|
||||||
|
|
||||||
@@ -465,15 +652,37 @@ def test_a_refused_password_change_says_so_too(signed_in: TestClient, caplog):
|
|||||||
assert "wrong current password" in caplog.text
|
assert "wrong current password" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_the_log_prefers_the_address_the_proxy_passed_on(locked: TestClient, caplog):
|
def test_the_log_names_the_hop_the_proxy_added(locked: TestClient, caplog):
|
||||||
"""Behind a proxy the peer is a container on its own network and says
|
"""Behind a proxy the peer is a container on its own network and says
|
||||||
nothing about who is at the other end. Both are written, because the header
|
nothing about who is at the other end. The proxy appends the address it
|
||||||
is only as good as whoever sent it -- which is also why nothing is decided
|
accepted the connection from to whatever `X-Forwarded-For` the caller
|
||||||
from it."""
|
sent, so the rightmost hop is the proxy's word and every hop left of it
|
||||||
|
the caller's: logging the first let an attacker name whom fail2ban bans.
|
||||||
|
Both the hop and the peer are written, because the header is only as good
|
||||||
|
as whoever set it -- which is also why nothing is decided from it."""
|
||||||
with caplog.at_level(logging.WARNING):
|
with caplog.at_level(logging.WARNING):
|
||||||
locked.post("/login", json={"password": "nope"},
|
locked.post("/login", json={"password": "nope"},
|
||||||
headers={"x-forwarded-for": "203.0.113.9, 10.9.9.9"})
|
headers={"x-forwarded-for": "203.0.113.9, 10.9.9.9"})
|
||||||
assert "203.0.113.9 via testclient" in caplog.text
|
(said,) = [r.getMessage() for r in caplog.records if r.name == "auth.middleware"]
|
||||||
|
assert said == "wrong password from 10.9.9.9 via testclient"
|
||||||
|
assert "203.0.113.9" not in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_log_falls_back_to_x_real_ip(locked: TestClient, caplog):
|
||||||
|
"""A proxy that sets `X-Real-IP` sets it outright. Read only when there is
|
||||||
|
no `X-Forwarded-For`, which a proxy that sends both keeps current too."""
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
locked.post("/login", json={"password": "nope"},
|
||||||
|
headers={"x-real-ip": "198.51.100.7"})
|
||||||
|
locked.post("/login", json={"password": "nope"},
|
||||||
|
headers={"x-real-ip": "198.51.100.7",
|
||||||
|
"x-forwarded-for": "203.0.113.9, 192.0.2.1"})
|
||||||
|
locked.post("/login", json={"password": "nope"},
|
||||||
|
headers={"x-forwarded-for": " ", "x-real-ip": "198.51.100.7"})
|
||||||
|
said = [r.getMessage() for r in caplog.records if r.name == "auth.middleware"]
|
||||||
|
assert said == ["wrong password from 198.51.100.7 via testclient",
|
||||||
|
"wrong password from 192.0.2.1 via testclient",
|
||||||
|
"wrong password from 198.51.100.7 via testclient"]
|
||||||
|
|
||||||
|
|
||||||
def test_a_forwarded_address_cannot_write_the_log_itself(locked: TestClient, caplog):
|
def test_a_forwarded_address_cannot_write_the_log_itself(locked: TestClient, caplog):
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ from auth.tests.softkey import SoftKey, b64 # noqa: E402 after the skip
|
|||||||
#: the real `blocked()` path rather than a configured override.
|
#: the real `blocked()` path rather than a configured override.
|
||||||
RP_ID = "localhost"
|
RP_ID = "localhost"
|
||||||
ORIGIN = "http://localhost"
|
ORIGIN = "http://localhost"
|
||||||
|
#: What starting a registration takes besides a session: the password.
|
||||||
|
CONFIRM = {"current": PASSWORD}
|
||||||
|
|
||||||
|
|
||||||
def unb64(text: str) -> bytes:
|
def unb64(text: str) -> bytes:
|
||||||
@@ -48,7 +50,7 @@ def client(make_client: Callable[..., TestClient]) -> TestClient:
|
|||||||
|
|
||||||
def register(client: TestClient, key: SoftKey, label: str = "Test key"):
|
def register(client: TestClient, key: SoftKey, label: str = "Test key"):
|
||||||
"""The whole registration ceremony, both round trips."""
|
"""The whole registration ceremony, both round trips."""
|
||||||
options = client.post("/api/account/passkeys/options").json()
|
options = client.post("/api/account/passkeys/options", json=CONFIRM).json()
|
||||||
made = key.register(unb64(options["challenge"]), RP_ID, ORIGIN)
|
made = key.register(unb64(options["challenge"]), RP_ID, ORIGIN)
|
||||||
return client.post("/api/account/passkeys",
|
return client.post("/api/account/passkeys",
|
||||||
json={"credential": made, "label": label})
|
json={"credential": made, "label": label})
|
||||||
@@ -66,7 +68,45 @@ def login(client: TestClient, key: SoftKey, origin: str = ORIGIN):
|
|||||||
def test_registering_needs_a_session(client: TestClient):
|
def test_registering_needs_a_session(client: TestClient):
|
||||||
"""A passkey is a new way in, so adding one is gated on already being in."""
|
"""A passkey is a new way in, so adding one is gated on already being in."""
|
||||||
client.post("/logout")
|
client.post("/logout")
|
||||||
assert client.post("/api/account/passkeys/options").status_code == 401
|
assert client.post("/api/account/passkeys/options", json=CONFIRM).status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_registering_needs_the_password_too(client: TestClient, caplog):
|
||||||
|
"""A passkey outlives every session and a password change, so a borrowed
|
||||||
|
signed-in browser must not be able to add one: a wrong or missing password
|
||||||
|
gets no challenge, is logged like the other doors, and is counted."""
|
||||||
|
with caplog.at_level("WARNING"):
|
||||||
|
wrong = client.post("/api/account/passkeys/options", json={"current": "nope"})
|
||||||
|
assert wrong.status_code == 403
|
||||||
|
assert "wrong current password" in caplog.text
|
||||||
|
assert client.post("/api/account/passkeys/options", json={}).status_code == 422
|
||||||
|
assert client.post("/api/account/passkeys/options").status_code == 422
|
||||||
|
assert client.get("/api/account").json()["passkeys"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_registering_is_behind_the_same_lockout(client: TestClient):
|
||||||
|
"""Guessing at the passkey door locks the front door too, and once locked
|
||||||
|
the right password is not looked at."""
|
||||||
|
for _ in range(5):
|
||||||
|
assert client.post("/api/account/passkeys/options",
|
||||||
|
json={"current": "nope"}).status_code == 403
|
||||||
|
refused = client.post("/api/account/passkeys/options", json=CONFIRM)
|
||||||
|
assert refused.status_code == 429
|
||||||
|
client.post("/logout")
|
||||||
|
assert sign_in(client).status_code == 429
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_app_with_no_password_registers_without_one(
|
||||||
|
make_client: Callable[..., TestClient]):
|
||||||
|
"""Loopback under `auto` and never locked: there is no password to confirm
|
||||||
|
and no session to borrow, so registering before deploying still works."""
|
||||||
|
open_app = make_client(host="127.0.0.1", password=None, base_url=ORIGIN)
|
||||||
|
assert not open_app.app.state.auth.has_password()
|
||||||
|
options = open_app.post("/api/account/passkeys/options", json={"current": ""})
|
||||||
|
assert options.status_code == 200
|
||||||
|
made = SoftKey().register(unb64(options.json()["challenge"]), RP_ID, ORIGIN)
|
||||||
|
assert open_app.post("/api/account/passkeys",
|
||||||
|
json={"credential": made}).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
def test_a_passkey_registers_and_is_listed(client: TestClient):
|
def test_a_passkey_registers_and_is_listed(client: TestClient):
|
||||||
@@ -95,14 +135,14 @@ def test_a_second_registration_is_excluded_not_duplicated(client: TestClient):
|
|||||||
say "you have one of these" rather than making a second."""
|
say "you have one of these" rather than making a second."""
|
||||||
key = SoftKey()
|
key = SoftKey()
|
||||||
register(client, key)
|
register(client, key)
|
||||||
options = client.post("/api/account/passkeys/options").json()
|
options = client.post("/api/account/passkeys/options", json=CONFIRM).json()
|
||||||
assert [c["id"] for c in options["excludeCredentials"]] == [b64(key.credential_id)]
|
assert [c["id"] for c in options["excludeCredentials"]] == [b64(key.credential_id)]
|
||||||
|
|
||||||
|
|
||||||
def test_a_stale_challenge_is_refused(client: TestClient):
|
def test_a_stale_challenge_is_refused(client: TestClient):
|
||||||
"""A challenge is single-use, so the same response cannot be sent twice."""
|
"""A challenge is single-use, so the same response cannot be sent twice."""
|
||||||
key = SoftKey()
|
key = SoftKey()
|
||||||
options = client.post("/api/account/passkeys/options").json()
|
options = client.post("/api/account/passkeys/options", json=CONFIRM).json()
|
||||||
made = key.register(unb64(options["challenge"]), RP_ID, ORIGIN)
|
made = key.register(unb64(options["challenge"]), RP_ID, ORIGIN)
|
||||||
|
|
||||||
assert client.post("/api/account/passkeys", json={"credential": made}).status_code == 200
|
assert client.post("/api/account/passkeys", json={"credential": made}).status_code == 200
|
||||||
@@ -224,6 +264,23 @@ def test_a_malformed_body_is_a_400_and_not_a_traceback(client: TestClient):
|
|||||||
assert client.post("/login/passkey", json={"credential": {}}).status_code == 400
|
assert client.post("/login/passkey", json={"credential": {}}).status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_raw_id_that_is_not_base64url_is_a_400_and_spends_nothing(client: TestClient):
|
||||||
|
"""`rawId` was decoded unguarded after the challenge was taken, so `"a"`
|
||||||
|
was a 500 from an endpoint anyone can reach and the challenge was gone.
|
||||||
|
Now it is refused before the challenge is spent: the same challenge,
|
||||||
|
answered properly, still signs in."""
|
||||||
|
key = SoftKey()
|
||||||
|
register(client, key)
|
||||||
|
client.post("/logout")
|
||||||
|
|
||||||
|
options = client.post("/login/passkey/options").json()
|
||||||
|
signed = key.sign(unb64(options["challenge"]), RP_ID, ORIGIN)
|
||||||
|
refused = client.post("/login/passkey", json={"credential": {**signed, "rawId": "a"}})
|
||||||
|
assert refused.status_code == 400
|
||||||
|
assert "rawId is not base64url" in refused.json()["detail"]
|
||||||
|
assert client.post("/login/passkey", json={"credential": signed}).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
def test_no_passkeys_registered_says_so(client: TestClient):
|
def test_no_passkeys_registered_says_so(client: TestClient):
|
||||||
client.post("/logout")
|
client.post("/logout")
|
||||||
assert client.post("/login/passkey/options").status_code == 404
|
assert client.post("/login/passkey/options").status_code == 404
|
||||||
@@ -243,7 +300,7 @@ def test_a_registration_challenge_cannot_be_answered_at_the_login_door(
|
|||||||
"""
|
"""
|
||||||
key = SoftKey()
|
key = SoftKey()
|
||||||
register(client, key)
|
register(client, key)
|
||||||
theirs = client.post("/api/account/passkeys/options").json()
|
theirs = client.post("/api/account/passkeys/options", json=CONFIRM).json()
|
||||||
client.post("/logout")
|
client.post("/logout")
|
||||||
|
|
||||||
signed = key.sign(unb64(theirs["challenge"]), RP_ID, ORIGIN)
|
signed = key.sign(unb64(theirs["challenge"]), RP_ID, ORIGIN)
|
||||||
@@ -352,7 +409,7 @@ def test_a_configured_rp_id_and_origin_win(make_client: Callable[..., TestClient
|
|||||||
client = make_client(enabled="always", rp_id="app.example.com",
|
client = make_client(enabled="always", rp_id="app.example.com",
|
||||||
origin="https://app.example.com")
|
origin="https://app.example.com")
|
||||||
sign_in(client)
|
sign_in(client)
|
||||||
options = client.post("/api/account/passkeys/options").json()
|
options = client.post("/api/account/passkeys/options", json=CONFIRM).json()
|
||||||
assert options["rp"]["id"] == "app.example.com"
|
assert options["rp"]["id"] == "app.example.com"
|
||||||
assert client.get("/api/account").json()["passkeys_blocked"] == ""
|
assert client.get("/api/account").json()["passkeys_blocked"] == ""
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""A socket whose credential ends is closed: by the route that ended it, or by
|
||||||
|
the sweep when it expired.
|
||||||
|
|
||||||
|
uv run pytest auth/tests/test_sockets.py -q
|
||||||
|
|
||||||
|
The door is checked at the handshake only, so before this a signed-out tab,
|
||||||
|
a revoked token or an expired session kept its socket streaming until the tab
|
||||||
|
reconnected. Every socket here is opened on the client the route is called
|
||||||
|
on: a client that is not entered runs its requests on an event loop of their
|
||||||
|
own, and the route's background task could not close a socket living there.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import contextlib
|
||||||
|
import logging
|
||||||
|
import sqlite3
|
||||||
|
from contextlib import closing
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from starlette.websockets import WebSocketDisconnect
|
||||||
|
|
||||||
|
import auth.middleware
|
||||||
|
from auth import AuthConfig, close_ended_sockets, sweep_ended_sockets
|
||||||
|
from auth.tests import PASSWORD
|
||||||
|
from auth.tests.harness import build
|
||||||
|
|
||||||
|
#: The cookie a login over plain http sets.
|
||||||
|
COOKIE = AuthConfig().cookie
|
||||||
|
LONG_AGO = "2000-01-01T00:00:00+00:00"
|
||||||
|
|
||||||
|
|
||||||
|
def closed_with(ws: Any) -> int:
|
||||||
|
"""Read frames until the server closes the socket; the close code.
|
||||||
|
|
||||||
|
Blocks on a socket the server left open, since the test client's receive
|
||||||
|
has no timeout, so each test asks the bus first, which does not block."""
|
||||||
|
for _ in range(50):
|
||||||
|
try:
|
||||||
|
ws.receive_json()
|
||||||
|
except WebSocketDisconnect as closed:
|
||||||
|
return closed.code
|
||||||
|
raise AssertionError("the socket stayed open")
|
||||||
|
|
||||||
|
|
||||||
|
def sweep_now(client: TestClient) -> int:
|
||||||
|
"""`close_ended_sockets` on the loop the client's sockets live on."""
|
||||||
|
assert client.portal is not None
|
||||||
|
return client.portal.call(close_ended_sockets, client.app)
|
||||||
|
|
||||||
|
|
||||||
|
def expire(client: TestClient, table: str) -> None:
|
||||||
|
with closing(sqlite3.connect(client.app.state.auth.path)) as conn, conn:
|
||||||
|
conn.execute(f"UPDATE {table} SET expires_at=?", (LONG_AGO,))
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- the routes
|
||||||
|
|
||||||
|
def test_signing_out_closes_the_socket_the_session_opened(signed_in: TestClient):
|
||||||
|
with signed_in.websocket_connect("/ws/jobs") as ws:
|
||||||
|
assert ws.receive_json()["type"] == "jobs"
|
||||||
|
assert signed_in.post("/logout").status_code == 200
|
||||||
|
assert signed_in.app.state.bus.sockets == ()
|
||||||
|
assert closed_with(ws) == 1008
|
||||||
|
|
||||||
|
|
||||||
|
def test_revoking_a_token_closes_its_socket(signed_in: TestClient):
|
||||||
|
"""The cookie is taken out so the bearer header is what opens the socket;
|
||||||
|
with the cookie in, the session would be what it stands on."""
|
||||||
|
made = signed_in.post("/api/account/tokens",
|
||||||
|
json={"current": PASSWORD, "label": "cron"}).json()
|
||||||
|
here = signed_in.cookies[COOKIE]
|
||||||
|
signed_in.cookies.clear()
|
||||||
|
with signed_in.websocket_connect(
|
||||||
|
"/ws/jobs", headers={"authorization": f"Bearer {made['token']}"}) as ws:
|
||||||
|
assert ws.receive_json()["type"] == "jobs"
|
||||||
|
signed_in.cookies.set(COOKIE, here)
|
||||||
|
assert signed_in.delete(f"/api/account/tokens/{made['id']}").status_code == 200
|
||||||
|
assert signed_in.app.state.bus.sockets == ()
|
||||||
|
assert closed_with(ws) == 1008
|
||||||
|
|
||||||
|
|
||||||
|
def test_closing_the_others_closes_their_sockets_and_keeps_this_one(
|
||||||
|
signed_in: TestClient):
|
||||||
|
here = signed_in.cookies[COOKIE]
|
||||||
|
there = signed_in.app.state.auth.open_session("phone", 1)
|
||||||
|
|
||||||
|
def as_(token: str) -> None:
|
||||||
|
signed_in.cookies.clear()
|
||||||
|
signed_in.cookies.set(COOKIE, token)
|
||||||
|
|
||||||
|
as_(there)
|
||||||
|
with signed_in.websocket_connect("/ws/jobs") as theirs:
|
||||||
|
assert theirs.receive_json()["type"] == "jobs"
|
||||||
|
as_(here)
|
||||||
|
with signed_in.websocket_connect("/ws/jobs") as mine:
|
||||||
|
assert mine.receive_json()["type"] == "jobs"
|
||||||
|
closed = signed_in.post("/api/account/sessions/close")
|
||||||
|
assert closed.json() == {"signed_out": 1}
|
||||||
|
assert len(signed_in.app.state.bus.sockets) == 1, "this tab's is open"
|
||||||
|
assert closed_with(theirs) == 1008
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- expiry
|
||||||
|
|
||||||
|
def test_an_expired_session_loses_its_socket(signed_in: TestClient):
|
||||||
|
"""Expiry ends no request, so no route sees it; the sweep's call does."""
|
||||||
|
with signed_in.websocket_connect("/ws/jobs") as ws:
|
||||||
|
assert ws.receive_json()["type"] == "jobs"
|
||||||
|
assert sweep_now(signed_in) == 0, "a live session keeps its socket"
|
||||||
|
expire(signed_in, "auth_sessions")
|
||||||
|
assert sweep_now(signed_in) == 1
|
||||||
|
assert closed_with(ws) == 1008
|
||||||
|
|
||||||
|
|
||||||
|
def test_nothing_is_closed_on_an_unlocked_app_or_one_without_a_bus(
|
||||||
|
open_app: TestClient, signed_in: TestClient):
|
||||||
|
with open_app.websocket_connect("/ws/jobs", headers={"host": "127.0.0.1"}) as ws:
|
||||||
|
assert ws.receive_json()["type"] == "jobs"
|
||||||
|
assert sweep_now(open_app) == 0
|
||||||
|
del signed_in.app.state.bus
|
||||||
|
expire(signed_in, "auth_sessions")
|
||||||
|
assert sweep_now(signed_in) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_sweep_runs_on_its_clock_and_survives_a_failure(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture):
|
||||||
|
calls: list[object] = []
|
||||||
|
|
||||||
|
async def counted(app: object) -> int:
|
||||||
|
calls.append(app)
|
||||||
|
if len(calls) == 1:
|
||||||
|
raise sqlite3.OperationalError("database is locked")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
monkeypatch.setattr(auth.middleware, "close_ended_sockets", counted)
|
||||||
|
|
||||||
|
async def run() -> asyncio.Task[None]:
|
||||||
|
task = asyncio.create_task(sweep_ended_sockets("the app", every=0.01))
|
||||||
|
for _ in range(200):
|
||||||
|
if len(calls) >= 2:
|
||||||
|
break
|
||||||
|
await asyncio.sleep(0.01)
|
||||||
|
task.cancel()
|
||||||
|
with contextlib.suppress(asyncio.CancelledError):
|
||||||
|
await task
|
||||||
|
return task
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
task = asyncio.run(run())
|
||||||
|
assert len(calls) >= 2 and calls[0] == "the app"
|
||||||
|
assert "the socket sweep failed" in caplog.text
|
||||||
|
assert task.cancelled()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_locked_app_sweeps_while_it_is_up(tmp_path: Path):
|
||||||
|
app = build(tmp_path / "a.db", enabled="always")
|
||||||
|
assert app.state.auth_sweep is None
|
||||||
|
with TestClient(app):
|
||||||
|
sweep = app.state.auth_sweep
|
||||||
|
assert isinstance(sweep, asyncio.Task) and not sweep.done()
|
||||||
|
assert sweep.done() and sweep.cancelled()
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unlocked_app_has_no_sweep(open_app: TestClient):
|
||||||
|
assert open_app.app.state.auth_sweep is None
|
||||||
@@ -53,7 +53,7 @@ def test_the_tables_are_namespaced(store: AuthStore):
|
|||||||
named = {name for (name,) in rows(
|
named = {name for (name,) in rows(
|
||||||
store, "SELECT name FROM sqlite_master WHERE type='table'")}
|
store, "SELECT name FROM sqlite_master WHERE type='table'")}
|
||||||
assert {"auth_credential", "auth_sessions", "auth_passkeys",
|
assert {"auth_credential", "auth_sessions", "auth_passkeys",
|
||||||
"auth_challenges"} <= named
|
"auth_challenges", "auth_tokens"} <= named
|
||||||
assert not {n for n in named if not n.startswith(("auth_", "sqlite_"))}
|
assert not {n for n in named if not n.startswith(("auth_", "sqlite_"))}
|
||||||
|
|
||||||
|
|
||||||
@@ -119,6 +119,71 @@ def test_the_delay_doubles_and_then_stops(store: AuthStore):
|
|||||||
assert store.check_password(PASSWORD) is True
|
assert store.check_password(PASSWORD) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_wrong_guesses_in_parallel_are_each_counted(store: AuthStore):
|
||||||
|
"""`failures` was read, scrypt ran, and the read plus one was written, so
|
||||||
|
forty wrong guesses at once recorded one failure and no lockout. The raise
|
||||||
|
is one statement under the write lock now."""
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
|
store.set_password(PASSWORD)
|
||||||
|
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||||
|
said = list(pool.map(lambda _: store.check_password("wrong"), range(40)))
|
||||||
|
assert said == [False] * 40
|
||||||
|
assert rows(store, "SELECT failures FROM auth_credential") == [(40,)]
|
||||||
|
assert store.locked_for() > 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_parallel_claims_admit_the_grace_and_no_more(store: AuthStore):
|
||||||
|
"""What both doors do: claim, then derive. A claim reads the lockout and
|
||||||
|
counts the attempt in one write transaction, so thirty claims at once
|
||||||
|
admit `_GRACE` guesses and tell the rest to wait, where a read before the
|
||||||
|
derivation and a write after it admitted all thirty."""
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
|
store.set_password(PASSWORD)
|
||||||
|
|
||||||
|
def guess(_: int) -> str:
|
||||||
|
if store.claim_attempt():
|
||||||
|
return "wait"
|
||||||
|
return "right" if store.check_password("wrong", claimed=True) else "wrong"
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
|
said = list(pool.map(guess, range(30)))
|
||||||
|
assert said.count("wrong") == _GRACE and said.count("wait") == 30 - _GRACE
|
||||||
|
assert rows(store, "SELECT failures FROM auth_credential") == [(_GRACE,)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_claimed_right_password_clears_the_count(store: AuthStore):
|
||||||
|
"""Counted up front, credited back: the fifth claim writes the lockout
|
||||||
|
before the derivation, and a right answer then clears it."""
|
||||||
|
store.set_password(PASSWORD)
|
||||||
|
for _ in range(_GRACE - 1):
|
||||||
|
assert store.claim_attempt() == 0
|
||||||
|
assert store.check_password("wrong", claimed=True) is False
|
||||||
|
assert store.claim_attempt() == 0
|
||||||
|
assert store.locked_for() > 0, "the fifth claim did not write the lockout"
|
||||||
|
assert store.check_password(PASSWORD, claimed=True) is True
|
||||||
|
assert store.locked_for() == 0
|
||||||
|
assert rows(store, "SELECT failures, locked_until FROM auth_credential") == [(0, "")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_claim_while_locked_counts_nothing(store: AuthStore):
|
||||||
|
"""Hammering while locked out does not extend the lockout."""
|
||||||
|
store.set_password(PASSWORD)
|
||||||
|
for _ in range(_GRACE):
|
||||||
|
assert store.claim_attempt() == 0
|
||||||
|
store.check_password("wrong", claimed=True)
|
||||||
|
before = rows(store, "SELECT failures, locked_until FROM auth_credential")
|
||||||
|
for _ in range(10):
|
||||||
|
assert store.claim_attempt() > 0
|
||||||
|
assert rows(store, "SELECT failures, locked_until FROM auth_credential") == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_claim_on_a_database_with_no_password_is_granted(store: AuthStore):
|
||||||
|
assert store.claim_attempt() == 0
|
||||||
|
assert store.check_password(PASSWORD, claimed=True) is False
|
||||||
|
|
||||||
|
|
||||||
# -------------------------------------------------------------- the sessions
|
# -------------------------------------------------------------- the sessions
|
||||||
|
|
||||||
def test_a_session_is_live_until_it_is_not(store: AuthStore):
|
def test_a_session_is_live_until_it_is_not(store: AuthStore):
|
||||||
|
|||||||
@@ -0,0 +1,465 @@
|
|||||||
|
"""API tokens: the third way in, for a script, and what it cannot do.
|
||||||
|
|
||||||
|
uv run pytest auth/tests/test_tokens.py -q
|
||||||
|
|
||||||
|
A token is minted with the password, carried as `Authorization: Bearer`, looked
|
||||||
|
up on every request like a session, and shut out of the account routes. The
|
||||||
|
same-site rule the middleware applies to every request is here too, because a
|
||||||
|
script's token is the one credential a browser never carries for it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import sqlite3
|
||||||
|
from collections.abc import Callable
|
||||||
|
from contextlib import closing
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from starlette.websockets import WebSocketDisconnect
|
||||||
|
|
||||||
|
from auth import AuthStore
|
||||||
|
from auth.middleware import ACCOUNT, _rebound, cross_site
|
||||||
|
from auth.store import HINT
|
||||||
|
from auth.tests import PASSWORD, sign_in
|
||||||
|
|
||||||
|
CONFIRM = {"current": PASSWORD}
|
||||||
|
|
||||||
|
|
||||||
|
def mint(client: TestClient, label: str = "cron", **body: object) -> dict:
|
||||||
|
answer = client.post("/api/account/tokens", json={**CONFIRM, "label": label, **body})
|
||||||
|
assert answer.status_code == 200, answer.text
|
||||||
|
return answer.json()
|
||||||
|
|
||||||
|
|
||||||
|
def bearer(token: str) -> dict[str, str]:
|
||||||
|
return {"authorization": f"Bearer {token}"}
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ the store
|
||||||
|
|
||||||
|
def test_the_token_is_not_in_the_file(store: AuthStore):
|
||||||
|
"""Hashed like a session token, and for the same reason: the file lets
|
||||||
|
nobody in."""
|
||||||
|
token_id, token = store.create_token("cron", None)
|
||||||
|
assert token.encode() not in store.path.read_bytes()
|
||||||
|
assert store.touch_token(token) is not None
|
||||||
|
assert store.touch_token(token).id == token_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_list_shows_the_opening_and_nothing_else(store: AuthStore):
|
||||||
|
_, token = store.create_token("cron", None)
|
||||||
|
(listed,) = store.tokens()
|
||||||
|
assert listed.hint == token[:HINT] and len(listed.hint) == HINT
|
||||||
|
assert listed.label == "cron" and listed.expires_at == "" and listed.last_used_at == ""
|
||||||
|
assert "token" not in listed.model_dump()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_dated_token_runs_out(store: AuthStore):
|
||||||
|
_, token = store.create_token("short", 30)
|
||||||
|
(listed,) = store.tokens()
|
||||||
|
expires = datetime.fromisoformat(listed.expires_at)
|
||||||
|
assert timedelta(days=29) < expires - datetime.now(UTC) <= timedelta(days=30)
|
||||||
|
with closing(sqlite3.connect(store.path)) as conn, conn:
|
||||||
|
conn.execute("UPDATE auth_tokens SET expires_at='2000-01-01T00:00:00+00:00'")
|
||||||
|
assert store.touch_token(token) is None
|
||||||
|
# Still listed, so the owner sees the row that ran out and removes it.
|
||||||
|
assert len(store.tokens()) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_using_a_token_stamps_it_once_in_a_while(store: AuthStore):
|
||||||
|
_, token = store.create_token("cron", None)
|
||||||
|
first = store.touch_token(token)
|
||||||
|
assert first is not None and first.last_used_at
|
||||||
|
with closing(sqlite3.connect(store.path)) as conn, conn:
|
||||||
|
conn.execute("UPDATE auth_tokens SET last_used_at='2000-01-01T00:00:00+00:00'")
|
||||||
|
again = store.touch_token(token)
|
||||||
|
assert again is not None and again.last_used_at > "2000-01-02"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_is_renamed_and_revoked_by_its_row(store: AuthStore):
|
||||||
|
token_id, token = store.create_token("cron", None)
|
||||||
|
assert store.rename_token(token_id, "the laptop") is True
|
||||||
|
assert store.tokens()[0].label == "the laptop"
|
||||||
|
assert store.delete_token(token_id) is True
|
||||||
|
assert store.touch_token(token) is None
|
||||||
|
assert store.delete_token(token_id) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_is_not_a_session(store: AuthStore):
|
||||||
|
"""Two tables, two lookups: a token in the cookie opens nothing, and a
|
||||||
|
session token in the header opens nothing."""
|
||||||
|
_, token = store.create_token("cron", None)
|
||||||
|
session = store.open_session("browser", 30)
|
||||||
|
assert store.touch(token) is None
|
||||||
|
assert store.touch_token(session) is None
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------- minting
|
||||||
|
|
||||||
|
def test_minting_needs_a_session(locked: TestClient):
|
||||||
|
assert locked.post("/api/account/tokens", json=CONFIRM).status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_minting_needs_the_password_too(signed_in: TestClient, caplog):
|
||||||
|
"""A token outlives every session and a password change, so a borrowed
|
||||||
|
signed-in browser must not be able to mint one."""
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
wrong = signed_in.post("/api/account/tokens", json={"current": "nope"})
|
||||||
|
assert wrong.status_code == 403
|
||||||
|
assert "wrong current password" in caplog.text
|
||||||
|
assert signed_in.get("/api/account").json()["tokens"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_minting_is_behind_the_same_lockout(signed_in: TestClient):
|
||||||
|
for _ in range(5):
|
||||||
|
signed_in.post("/api/account/tokens", json={"current": "nope"})
|
||||||
|
refused = signed_in.post("/api/account/tokens", json=CONFIRM)
|
||||||
|
assert refused.status_code == 429
|
||||||
|
assert sign_in(signed_in).status_code == 429
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_app_with_no_password_mints_without_one(open_app: TestClient):
|
||||||
|
"""The laptop before it is deployed, as with a passkey."""
|
||||||
|
made = open_app.post("/api/account/tokens", json={"current": "", "label": "here"})
|
||||||
|
assert made.status_code == 200 and made.json()["token"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_token_is_shown_once_and_listed_by_its_opening(signed_in: TestClient):
|
||||||
|
made = mint(signed_in, "cron", days=90)
|
||||||
|
assert made["hint"] == made["token"][:HINT] and made["label"] == "cron"
|
||||||
|
assert made["expires_at"]
|
||||||
|
(listed,) = signed_in.get("/api/account").json()["tokens"]
|
||||||
|
assert listed["id"] == made["id"] and listed["hint"] == made["hint"]
|
||||||
|
assert "token" not in listed
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_label_and_the_days_are_bounded(signed_in: TestClient):
|
||||||
|
assert signed_in.post("/api/account/tokens",
|
||||||
|
json={**CONFIRM, "days": 0}).status_code == 422
|
||||||
|
assert signed_in.post("/api/account/tokens",
|
||||||
|
json={**CONFIRM, "days": 4000}).status_code == 422
|
||||||
|
assert signed_in.post("/api/account/tokens",
|
||||||
|
json={**CONFIRM, "label": "x" * 101}).status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- the bearer
|
||||||
|
|
||||||
|
def test_a_token_opens_the_api_and_the_socket(signed_in: TestClient):
|
||||||
|
token = mint(signed_in)["token"]
|
||||||
|
script = TestClient(signed_in.app, follow_redirects=False)
|
||||||
|
assert script.get("/api/thing").status_code == 401
|
||||||
|
assert script.get("/api/thing", headers=bearer(token)).status_code == 200
|
||||||
|
with script.websocket_connect("/ws/jobs", headers=bearer(token)) as socket:
|
||||||
|
assert socket.receive_json()["type"] == "jobs"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_cannot_reach_the_account(signed_in: TestClient):
|
||||||
|
"""A leaked token must not be able to mint another, revoke the owner's
|
||||||
|
sessions, or read the list of ways in; and `/logout` is a session's."""
|
||||||
|
token = mint(signed_in)["token"]
|
||||||
|
script = TestClient(signed_in.app, follow_redirects=False)
|
||||||
|
for method, path in (("GET", "/api/account"),
|
||||||
|
("POST", "/api/account/tokens"),
|
||||||
|
("POST", "/api/account/sessions/close"),
|
||||||
|
("DELETE", "/api/account/sessions/1"),
|
||||||
|
("POST", "/api/account/password"),
|
||||||
|
("POST", "/logout")):
|
||||||
|
refused = script.request(method, path, headers=bearer(token), json=CONFIRM)
|
||||||
|
assert refused.status_code == 403, f"{method} {path}: {refused.status_code}"
|
||||||
|
assert "not the account" in refused.json()["detail"]
|
||||||
|
# And the names are matched whole: `/api/accounts` is not the account.
|
||||||
|
assert script.get("/api/accountx", headers=bearer(token)).status_code == 404
|
||||||
|
assert ACCOUNT == ("/api/account", "/api/account/", "/logout")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_cookie_session_wins_over_a_bad_header(signed_in: TestClient):
|
||||||
|
"""A browser extension that adds a stale header must not sign a browser out."""
|
||||||
|
assert signed_in.get("/api/account", headers=bearer("stale")).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_bad_token_is_a_401_and_a_line_in_the_log(locked: TestClient, caplog):
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
refused = locked.get("/api/thing", headers=bearer("not-a-token"))
|
||||||
|
assert refused.status_code == 401
|
||||||
|
assert refused.json()["detail"] == "That token is not valid here."
|
||||||
|
assert "bad token from testclient" in caplog.text
|
||||||
|
# JSON whatever was asked for: a script, not a browser, sent this header.
|
||||||
|
assert locked.get("/deep/page", headers=bearer("not-a-token")).status_code == 401
|
||||||
|
# And nothing is counted against the password: the right one still opens.
|
||||||
|
assert sign_in(locked).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_the_bearer_scheme_is_read(locked: TestClient):
|
||||||
|
assert locked.get("/api/thing",
|
||||||
|
headers={"authorization": "Basic dXNlcjpwYXNz"}).status_code == 401
|
||||||
|
assert locked.get("/api/thing?token=x").status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_revoked_token_is_refused_on_the_next_request(signed_in: TestClient):
|
||||||
|
made = mint(signed_in)
|
||||||
|
script = TestClient(signed_in.app, follow_redirects=False)
|
||||||
|
assert script.get("/api/thing", headers=bearer(made["token"])).status_code == 200
|
||||||
|
assert signed_in.delete(f"/api/account/tokens/{made['id']}").status_code == 200
|
||||||
|
assert script.get("/api/thing", headers=bearer(made["token"])).status_code == 401
|
||||||
|
assert signed_in.delete(f"/api/account/tokens/{made['id']}").status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_survives_a_password_change(signed_in: TestClient):
|
||||||
|
"""Like a passkey: the cron job holding it is not what the password was
|
||||||
|
changed against. The account screen is where to revoke it, and the answer
|
||||||
|
says how many remain so the screen can say so."""
|
||||||
|
token = mint(signed_in)["token"]
|
||||||
|
changed = signed_in.post("/api/account/password",
|
||||||
|
json={"current": PASSWORD, "new": "a-much-longer-one"})
|
||||||
|
assert changed.status_code == 200
|
||||||
|
assert changed.json() == {"signed_out": 0, "tokens_revoked": 0, "passkeys_revoked": 0,
|
||||||
|
"tokens_left": 1, "passkeys_left": 0}
|
||||||
|
script = TestClient(signed_in.app, follow_redirects=False)
|
||||||
|
assert script.get("/api/thing", headers=bearer(token)).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_password_change_can_revoke_every_token_and_passkey(signed_in: TestClient):
|
||||||
|
"""The case the route exists for: whoever had the password may have
|
||||||
|
minted a token that never expires or registered a passkey, and either
|
||||||
|
outlives the change. `revoke_all` deletes them with it."""
|
||||||
|
token = mint(signed_in)["token"]
|
||||||
|
mint(signed_in, "another")
|
||||||
|
signed_in.app.state.auth.add_passkey(b"cred", b"key", 0, "", "someone's key")
|
||||||
|
other = TestClient(signed_in.app, follow_redirects=False)
|
||||||
|
sign_in(other)
|
||||||
|
|
||||||
|
changed = signed_in.post("/api/account/password", json={
|
||||||
|
"current": PASSWORD, "new": "a-much-longer-one", "revoke_all": True})
|
||||||
|
assert changed.status_code == 200
|
||||||
|
assert changed.json() == {"signed_out": 1, "tokens_revoked": 2, "passkeys_revoked": 1,
|
||||||
|
"tokens_left": 0, "passkeys_left": 0}
|
||||||
|
script = TestClient(signed_in.app, follow_redirects=False)
|
||||||
|
assert script.get("/api/thing", headers=bearer(token)).status_code == 401
|
||||||
|
account = signed_in.get("/api/account").json()
|
||||||
|
assert account["tokens"] == [] and account["passkeys"] == []
|
||||||
|
assert other.get("/api/thing").status_code == 401
|
||||||
|
assert signed_in.get("/api/thing").status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_wrong_password_revokes_nothing(signed_in: TestClient):
|
||||||
|
token = mint(signed_in)["token"]
|
||||||
|
refused = signed_in.post("/api/account/password", json={
|
||||||
|
"current": "nope", "new": "a-much-longer-one", "revoke_all": True})
|
||||||
|
assert refused.status_code == 403
|
||||||
|
script = TestClient(signed_in.app, follow_redirects=False)
|
||||||
|
assert script.get("/api/thing", headers=bearer(token)).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_is_renamed_from_the_account(signed_in: TestClient):
|
||||||
|
made = mint(signed_in)
|
||||||
|
renamed = signed_in.post(f"/api/account/tokens/{made['id']}/label",
|
||||||
|
json={"label": "the pi"})
|
||||||
|
assert renamed.status_code == 200
|
||||||
|
assert signed_in.get("/api/account").json()["tokens"][0]["label"] == "the pi"
|
||||||
|
assert signed_in.post("/api/account/tokens/999/label",
|
||||||
|
json={"label": "x"}).status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_open_app_ignores_the_header(open_app: TestClient):
|
||||||
|
"""Unlocked, there is no door for a token to open; a stale one is not an error."""
|
||||||
|
assert open_app.get("/api/thing", headers=bearer("whatever")).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ another site
|
||||||
|
|
||||||
|
EVIL = "http://evil.example"
|
||||||
|
|
||||||
|
|
||||||
|
def test_another_sites_page_cannot_post_to_an_unlocked_app(open_app: TestClient):
|
||||||
|
"""No session cookie to withhold on loopback, so `SameSite` protected
|
||||||
|
nothing: a `text/plain` POST needs no preflight and reached every route
|
||||||
|
that takes no body."""
|
||||||
|
assert open_app.post("/api/account/sessions/close").status_code == 200
|
||||||
|
assert open_app.post("/api/account/sessions/close",
|
||||||
|
headers={"origin": "http://127.0.0.1"}).status_code == 200
|
||||||
|
assert open_app.post("/api/account/sessions/close",
|
||||||
|
headers={"sec-fetch-site": "same-origin"}).status_code == 200
|
||||||
|
for headers in ({"origin": EVIL}, {"origin": "null"},
|
||||||
|
{"sec-fetch-site": "cross-site"}, {"sec-fetch-site": "same-site"},
|
||||||
|
{"origin": "http://127.0.0.1", "sec-fetch-site": "cross-site"}):
|
||||||
|
refused = open_app.post("/api/account/sessions/close", headers=headers)
|
||||||
|
assert refused.status_code == 403, headers
|
||||||
|
assert "another site" in refused.json()["detail"]
|
||||||
|
plain = open_app.post("/api/account/sessions/close", content=b"x",
|
||||||
|
headers={"content-type": "text/plain"})
|
||||||
|
assert plain.status_code == 415 and "application/json" in plain.json()["detail"]
|
||||||
|
form = open_app.post("/api/account/sessions/close", content=b"a=1",
|
||||||
|
headers={"content-type": "application/x-www-form-urlencoded"})
|
||||||
|
assert form.status_code == 415
|
||||||
|
# A read is never refused, wherever it came from.
|
||||||
|
assert open_app.get("/api/thing", headers={"origin": EVIL}).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_transfer_encoding_header_is_a_body_whatever_its_value():
|
||||||
|
"""Its presence says a body follows; an empty value is malformed and is
|
||||||
|
not given the benefit of the doubt."""
|
||||||
|
for value in (b"chunked", b""):
|
||||||
|
scope = {"type": "http", "method": "POST",
|
||||||
|
"headers": [(b"content-type", b"text/plain"),
|
||||||
|
(b"transfer-encoding", value)]}
|
||||||
|
refused = cross_site(scope)
|
||||||
|
assert refused is not None and refused[0] == 415, value
|
||||||
|
scope["headers"] = [(b"content-type", b"text/plain")]
|
||||||
|
assert cross_site(scope) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_another_sites_page_cannot_open_the_socket(open_app: TestClient):
|
||||||
|
"""A WebSocket handshake is not subject to CORS, so any page could open
|
||||||
|
`/ws/jobs` on a loopback app and read every job and notice."""
|
||||||
|
with pytest.raises(WebSocketDisconnect), \
|
||||||
|
open_app.websocket_connect("/ws/jobs", headers={"origin": EVIL}):
|
||||||
|
pass
|
||||||
|
# The test client's handshake names `testserver` whatever the base URL,
|
||||||
|
# so the app's own page is spelt out: a loopback host, and its origin.
|
||||||
|
with open_app.websocket_connect("/ws/jobs", headers={
|
||||||
|
"host": "127.0.0.1", "origin": "http://127.0.0.1"}) as ws:
|
||||||
|
assert ws.receive_json()["type"] == "jobs"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_configured_origin_is_accepted_behind_a_proxy_that_rewrites_host(
|
||||||
|
make_client: Callable[..., TestClient]):
|
||||||
|
client = make_client(enabled="auto", host="127.0.0.1", password=None,
|
||||||
|
origin="https://app.example.com", base_url="http://127.0.0.1")
|
||||||
|
assert client.post("/api/account/sessions/close",
|
||||||
|
headers={"origin": "https://app.example.com"}).status_code == 200
|
||||||
|
assert client.post("/api/account/sessions/close",
|
||||||
|
headers={"origin": EVIL}).status_code == 403
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_rule_holds_on_a_locked_app_too(signed_in: TestClient):
|
||||||
|
"""A locked app with `secure_cookie: never` on a LAN is a real deployment,
|
||||||
|
and the two header reads cost nothing."""
|
||||||
|
assert signed_in.post("/api/account/sessions/close",
|
||||||
|
headers={"origin": EVIL}).status_code == 403
|
||||||
|
assert signed_in.post("/api/account/sessions/close").status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------- what a token must not reach
|
||||||
|
|
||||||
|
def test_an_app_names_what_a_token_may_not_reach(tmp_path):
|
||||||
|
"""A route that hands out or replaces the database as a file holds the
|
||||||
|
password hash and the token rows, so `connect(session_only=...)` shuts it
|
||||||
|
to a token the way the account is shut."""
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.responses import PlainTextResponse
|
||||||
|
|
||||||
|
from auth import AuthConfig, AuthStore, connect
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
db = tmp_path / "s.db"
|
||||||
|
store = AuthStore(db)
|
||||||
|
store.set_password(PASSWORD)
|
||||||
|
connect(app, store, AuthConfig(enabled="always"), host="0.0.0.0",
|
||||||
|
session_only=("/api/backups", "/api/backups/", "/api/restore"))
|
||||||
|
|
||||||
|
@app.get("/api/backups/{name}", response_class=PlainTextResponse)
|
||||||
|
def download(name: str) -> str:
|
||||||
|
return "the whole database"
|
||||||
|
|
||||||
|
@app.post("/api/restore", response_class=PlainTextResponse)
|
||||||
|
def restore() -> str:
|
||||||
|
return "swapped"
|
||||||
|
|
||||||
|
@app.get("/api/thing")
|
||||||
|
def thing() -> dict[str, bool]:
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
_, token = store.create_token("cron", None)
|
||||||
|
with TestClient(app, follow_redirects=False) as client:
|
||||||
|
assert client.get("/api/thing", headers=bearer(token)).status_code == 200
|
||||||
|
assert client.get("/api/backups/x.db.gz", headers=bearer(token)).status_code == 403
|
||||||
|
assert client.post("/api/restore", headers=bearer(token)).status_code == 403
|
||||||
|
sign_in(client)
|
||||||
|
assert client.get("/api/backups/x.db.gz").status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------- a rebound name
|
||||||
|
|
||||||
|
def test_a_rebinding_page_is_refused_on_an_unlocked_app(open_app: TestClient):
|
||||||
|
"""A page on `rebind.example`, pointed at 127.0.0.1, sends a `Host` and
|
||||||
|
an `Origin` that agree, and `Sec-Fetch-Site: same-origin`, so the
|
||||||
|
same-site rule cannot tell it from the app's own page. The name can:
|
||||||
|
a loopback app is reached as localhost, and a script naming anything
|
||||||
|
else for it is refused with the proxy's 421."""
|
||||||
|
page = {"host": "rebind.example:8787", "origin": "http://rebind.example:8787",
|
||||||
|
"sec-fetch-site": "same-origin"}
|
||||||
|
refused = open_app.post("/api/account/tokens", json={"current": ""}, headers=page)
|
||||||
|
assert refused.status_code == 421
|
||||||
|
assert "localhost" in refused.json()["detail"]
|
||||||
|
assert open_app.get("/api/thing", headers=page).status_code == 421
|
||||||
|
with pytest.raises(WebSocketDisconnect), \
|
||||||
|
open_app.websocket_connect("/ws/jobs", headers=page):
|
||||||
|
pass
|
||||||
|
# The app's own page, at any loopback name, is untouched, and so is a
|
||||||
|
# client that is not a page when it names one.
|
||||||
|
for host in ("localhost:8787", "127.0.0.1", "[::1]:8787", "app.localhost"):
|
||||||
|
own = {"host": host, "origin": f"http://{host}", "sec-fetch-site": "same-origin"}
|
||||||
|
assert open_app.get("/api/thing", headers=own).status_code == 200, host
|
||||||
|
assert open_app.get("/api/thing", headers={"host": host}).status_code == 200, host
|
||||||
|
assert open_app.post("/api/account/sessions/close",
|
||||||
|
headers={"host": "localhost"}).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_rebound_request_without_page_headers_is_refused_too(open_app: TestClient):
|
||||||
|
"""A browser that sends no `Sec-Fetch-Site`, and no `Origin` on a read, is
|
||||||
|
still a rebound page reading the API. The `Host` is judged whatever else
|
||||||
|
the request carries, so a name that is not loopback is refused, and so is
|
||||||
|
a `Host` that names nothing."""
|
||||||
|
for host in ("rebind.example", "rebind.example:8787", "testserver", "["):
|
||||||
|
refused = open_app.get("/api/thing", headers={"host": host})
|
||||||
|
assert refused.status_code == 421, host
|
||||||
|
assert open_app.post("/api/account/sessions/close",
|
||||||
|
headers={"host": "rebind.example"}).status_code == 421
|
||||||
|
with pytest.raises(WebSocketDisconnect), \
|
||||||
|
open_app.websocket_connect("/ws/jobs", headers={"host": "rebind.example"}):
|
||||||
|
pass
|
||||||
|
assert _rebound({"type": "http", "headers": []}, "") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_configured_origin_is_a_name_the_unlocked_app_answers_to(
|
||||||
|
make_client: Callable[..., TestClient]):
|
||||||
|
client = make_client(enabled="auto", host="127.0.0.1", password=None,
|
||||||
|
origin="https://app.example.com")
|
||||||
|
page = {"host": "app.example.com", "origin": "https://app.example.com",
|
||||||
|
"sec-fetch-site": "same-origin"}
|
||||||
|
assert client.get("/api/thing", headers=page).status_code == 200
|
||||||
|
page["host"] = page["origin"] = "other.example.com"
|
||||||
|
assert client.get("/api/thing", headers=page).status_code == 421
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------- a cookie that is not one
|
||||||
|
|
||||||
|
def test_a_cookie_header_that_is_not_one_is_read_as_no_cookie(locked: TestClient):
|
||||||
|
"""The middleware is in front of every request, so a header it cannot
|
||||||
|
read must be no cookie and not a 500 on every call that carried it."""
|
||||||
|
for header in ("a@b=1", ";;;", "=", '"', "app_session=", "app_session"):
|
||||||
|
refused = locked.get("/api/thing", headers={"cookie": header})
|
||||||
|
assert refused.status_code == 401, header
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_malformed_neighbour_cookie_does_not_sign_the_owner_out(signed_in: TestClient):
|
||||||
|
"""`SimpleCookie.load` returned an empty jar for the whole header when any
|
||||||
|
cookie in it had a space or JSON in its value, so a sibling app on the
|
||||||
|
parent domain setting `theme=dark mode` locked the owner into a login
|
||||||
|
loop. Read per pair, the session beside it still opens the door."""
|
||||||
|
from http.cookies import SimpleCookie
|
||||||
|
|
||||||
|
session = signed_in.cookies["app_session"]
|
||||||
|
jar = SimpleCookie()
|
||||||
|
jar.load(f"theme=dark mode; app_session={session}")
|
||||||
|
assert "app_session" not in jar, "the stdlib parser was the bug"
|
||||||
|
|
||||||
|
browser = TestClient(signed_in.app, follow_redirects=False)
|
||||||
|
for neighbour in ("theme=dark mode", 'x={"a":1}'):
|
||||||
|
for header in (f"{neighbour}; app_session={session}",
|
||||||
|
f"app_session={session}; {neighbour}"):
|
||||||
|
answer = browser.get("/api/thing", headers={"cookie": header})
|
||||||
|
assert answer.status_code == 200, header
|
||||||
@@ -55,7 +55,7 @@ def test_the_passkey_routes_answer_501_rather_than_crashing(
|
|||||||
"""501 is what the element watches for -- "this route exists and this build
|
"""501 is what the element watches for -- "this route exists and this build
|
||||||
cannot perform it" -- and it takes the passkey section off the screen."""
|
cannot perform it" -- and it takes the passkey section off the screen."""
|
||||||
for path in ("/api/account/passkeys/options", "/login/passkey/options"):
|
for path in ("/api/account/passkeys/options", "/login/passkey/options"):
|
||||||
answer = signed_in.post(path)
|
answer = signed_in.post(path, json={"current": PASSWORD})
|
||||||
assert answer.status_code == 501, path
|
assert answer.status_code == 501, path
|
||||||
assert "passkeys" in answer.json()["detail"]
|
assert "passkeys" in answer.json()["detail"]
|
||||||
assert "uv sync" in answer.json()["detail"]
|
assert "uv sync" in answer.json()["detail"]
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# The backend in the foreground until Ctrl-C: uv run app serve, on :8790. The profile's <name>-backend runs it.
|
||||||
|
# Ours: `-v` and `--config` go before `serve`, where the app takes them, and every other argument
|
||||||
|
# after it (`--reload`, `--port`), so `.run/serve.run.xml`'s `serve --reload` is `<name>-backend --reload`.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
# A shell started by wsl.exe has not read the rc file that puts uv on PATH.
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
before=() after=()
|
||||||
|
while (($#)); do
|
||||||
|
case $1 in
|
||||||
|
-v|--verbose) before+=("$1") ;;
|
||||||
|
-c|--config) before+=("$1" "${2:?$1 needs a file}"); shift ;;
|
||||||
|
--config=*) before+=("$1") ;;
|
||||||
|
*) after+=("$1") ;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
exec uv run app "${before[@]}" serve "${after[@]}"
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# The local gate on the working tree, uncommitted edits included, without promoting:
|
||||||
|
# the steps deploy/test-commit.sh runs on a commit; the profile's <name>-check runs it.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
exec sh deploy/test-commit.sh --here
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# roledesk's own, beside the boilerplate's backend and check: one pull in the foreground, `app job pull`.
|
||||||
|
# The profile's <name>-pull runs it; `.run/pull.run.xml` is the same command from PyCharm.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
# A shell started by wsl.exe has not read the rc file that puts uv on PATH.
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
exec uv run app job pull "$@"
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# roledesk's own, beside the boilerplate's backend and check: the queue drained with no web app, `app worker`.
|
||||||
|
# The profile's <name>-worker runs it; `.run/worker.run.xml` is the same command from PyCharm.
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
# A shell started by wsl.exe has not read the rc file that puts uv on PATH.
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
exec uv run app worker "$@"
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
This skill adapts rules from ai-job-search (https://github.com/MadsLorentzen/ai-job-search), used under the MIT License:
|
|
||||||
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2026 Mads Lorentzen
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
Binary file not shown.
Binary file not shown.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+59
-65
@@ -51,11 +51,10 @@ agent:
|
|||||||
max_tokens: 8192 # the answer's cap per call
|
max_tokens: 8192 # the answer's cap per call
|
||||||
cli_path: "" # where the `claude` binary is, if not on PATH
|
cli_path: "" # where the `claude` binary is, if not on PATH
|
||||||
|
|
||||||
# Title and description in, gate columns out (app/jobs/extract.py). Off by
|
# Title and description in, gate columns out: location, remote area, seniority,
|
||||||
# default: on the API backend every batch of 20 roles is a paid call. On, a
|
# languages and deadline (app/jobs/extract.py). Off by default: on the API
|
||||||
# pull reads only the cut of every enabled agent (its limits, see agents
|
# backend every batch of 20 roles is a paid call. On, a pull reads the cut of
|
||||||
# below), after a first score; a second score then applies the fetch gate and
|
# every enabled agent (its limits, see agents below). `enabled` is only the
|
||||||
# the cut to what was filled. `enabled` is only the
|
|
||||||
# default for a fresh database: the switch under Settings, Sources, Runs (or
|
# default for a fresh database: the switch under Settings, Sources, Runs (or
|
||||||
# `app stages extract on|off`) overrides it and needs no restart.
|
# `app stages extract on|off`) overrides it and needs no restart.
|
||||||
extract:
|
extract:
|
||||||
@@ -66,11 +65,11 @@ extract:
|
|||||||
thinking: off # off | adaptive
|
thinking: off # off | adaptive
|
||||||
|
|
||||||
# About 60 words on a role: what it is, the stack, where, the pay if stated
|
# About 60 words on a role: what it is, the stack, where, the pay if stated
|
||||||
# (app/jobs/summarise.py). Off by default. On, each pull summarises the top
|
# (app/jobs/summarise.py). Off by default. On, each pull summarises the
|
||||||
# of every enabled agent's feed that has none yet, the best summarise_top by
|
# summarise_top (an agent's limits) best roles of each enabled agent's two
|
||||||
# final score (its limits), and every role liked or saved, so the cost stays
|
# orders, What I'm looking for and What fits me, and every role liked or
|
||||||
# bounded however many roles are open. The Summarise button on
|
# saved, when they have no summary yet. The Summarise button on a role's page
|
||||||
# a role's page asks for that one role either way. `enabled` is only the
|
# asks for that one role either way. `enabled` is only the
|
||||||
# default for a fresh database: the switch under Settings, Sources, Runs (or
|
# default for a fresh database: the switch under Settings, Sources, Runs (or
|
||||||
# `app stages summarise on|off`) overrides it and needs no restart.
|
# `app stages summarise on|off`) overrides it and needs no restart.
|
||||||
summarise:
|
summarise:
|
||||||
@@ -83,12 +82,12 @@ summarise:
|
|||||||
# Only extract reads it.
|
# Only extract reads it.
|
||||||
|
|
||||||
# Each search agent's cut read against its CV: skills, experience and career 0
|
# Each search agent's cut read against its CV: skills, experience and career 0
|
||||||
# to 100, a total, and 1 to 3 strengths and gaps (app/jobs/judge.py), for the
|
# to 10, and 1 to 3 strengths and gaps (app/jobs/judge.py); the CV match, which
|
||||||
# CV match order on Roles. Off by default. On, each pull judges the cut of
|
# orders What fits me on Roles, weights the three. Off by default. On, each pull
|
||||||
# every enabled agent with a CV and a prompt, and only the roles not judged
|
# judges the cut of every enabled agent with a CV and a prompt, once per change
|
||||||
# yet or judged from another CV, prompt, like, avoid or must text, or role
|
# of its CV, prompts, likes, dislikes or must text, or of the role's text. The
|
||||||
# text. The CV rides in every call, so batches are
|
# CV rides in every call, so batches are small. `enabled` is only the default
|
||||||
# small. `enabled` is only the default for a fresh database: the switch under
|
# for a fresh database: the switch under
|
||||||
# Settings, Sources, Runs (or `app stages judge on|off`) overrides it and needs
|
# Settings, Sources, Runs (or `app stages judge on|off`) overrides it and needs
|
||||||
# no restart. on_ingest is accepted and ignored, as for summarise.
|
# no restart. on_ingest is accepted and ignored, as for summarise.
|
||||||
judge:
|
judge:
|
||||||
@@ -109,25 +108,23 @@ fetch_text:
|
|||||||
per_run: 1000 # reads per run
|
per_run: 1000 # reads per run
|
||||||
retry_days: 7 # days before a page that gave no text is read again
|
retry_days: 7 # days before a page that gave no text is read again
|
||||||
|
|
||||||
# A role and each agent's prompts as vectors, for the fit term of a score
|
# A role, each agent's prompts and its CV as vectors, for prompt closeness and
|
||||||
# (app/jobs/embed.py). Needs the `embeddings` extra; without it `embed` posts
|
# CV closeness (app/jobs/embed.py). Needs the `embeddings` extra; without it
|
||||||
# a notice and scores leave fit out. Changing the model re-embeds.
|
# `embed` posts a notice and scores leave closeness out. Changing the model
|
||||||
# multilingual-e5-small, not English-only MiniLM, because the corpus is now
|
# re-embeds. multilingual-e5-small, not English-only MiniLM, because the corpus
|
||||||
# European: measured against the "Copenhagen engineering" agent's intent, it
|
# is European (see docs/decisions.md). An e5 model's "query: " / "passage: "
|
||||||
# separates engineering from non-engineering roles better, especially among
|
# prefixes are added automatically, keyed off this name.
|
||||||
# Danish-country roles (see docs/decisions.md), for about 3x the CPU time. An
|
# calibration maps a model's cosines to 0..1 levels, as [lo, hi] for fit (prompt
|
||||||
# e5 model's "query: " / "passage: " prefixes are added automatically, keyed
|
# closeness), taste (job like) and cv (CV closeness, fit's band when left out):
|
||||||
# off this name.
|
# lo is 0, hi is 1, clipped outside. multilingual-e5-small has built-in
|
||||||
# calibration maps a model's cosines to 0..1 levels, as [lo, hi] for fit and
|
# constants (app/config.py, CALIBRATION); an entry here replaces them for that
|
||||||
# for taste: lo is 0, hi is 1, clipped outside. multilingual-e5-small has
|
# model. A model with none is min-max scaled per run instead, so its levels are
|
||||||
# built-in constants (app/config.py, CALIBRATION); an entry here replaces
|
# ranks, and the log says so.
|
||||||
# them for that model. A model with none is min-max scaled per run instead,
|
|
||||||
# so its fit is a rank, not a level, and the log says so.
|
|
||||||
embed:
|
embed:
|
||||||
model: intfloat/multilingual-e5-small
|
model: intfloat/multilingual-e5-small
|
||||||
per_run: 50000
|
per_run: 50000
|
||||||
calibration: {}
|
calibration: {}
|
||||||
# intfloat/multilingual-e5-small: {fit: [0.75, 0.88], taste: [-0.14, 0.19]}
|
# intfloat/multilingual-e5-small: {fit: [0.75, 0.88], taste: [-0.14, 0.19], cv: [0.77, 0.90]}
|
||||||
|
|
||||||
# One block per plugin, keyed by the plugin's name. The ats and search plugins
|
# One block per plugin, keyed by the plugin's name. The ats and search plugins
|
||||||
# read their boards and searches from the subscriptions table, so they need
|
# read their boards and searches from the subscriptions table, so they need
|
||||||
@@ -140,39 +137,36 @@ sources: {}
|
|||||||
# search:
|
# search:
|
||||||
# adzuna: {app_id: "...", app_key: "..."}
|
# adzuna: {app_id: "...", app_key: "..."}
|
||||||
|
|
||||||
# Starter search agents, imported once into an empty `agents` table. The gates
|
# Starter search agents, imported once into an empty `agents` table. The Search
|
||||||
# are the agent's fetch gate, each {field, values, mode: require|any|exclude}:
|
# agents page lists and edits them afterwards, and the Viewing as select on
|
||||||
# a require gate is the region, the any gates are one group a role must match
|
# Roles picks one by name. An agent has:
|
||||||
# one of ("in Denmark, or remote"), the place part, and an exclude gate leaves
|
# - name, and enabled (default true; false leaves it out of the scores and the
|
||||||
# out the roles that match one of its values (title: the titles to leave
|
# pull).
|
||||||
# out). A gate rejects only a role whose column is filled, since extract fills
|
# - prompts, a list of {name, text}, each what the agent looks for; a role is
|
||||||
# the rest for the roles in the cut. A company set to All countries skips the
|
# ranked by the prompt it is closest to. intent, as in config.yaml, is one
|
||||||
# any group, one set to All roles the exclude gates on title. Fields are
|
# prompt named after the agent.
|
||||||
# place, country, remote, remote_from, seniority, function, salary_min,
|
# - gates, its fetch gate, each {field, values, mode: require|any|exclude}: a
|
||||||
# currency, permit, languages, company and title. weights are the final
|
# require gate is the region (the world), the any gates are one group a role
|
||||||
# score's, the feed's order, default {fit: 0.6, recency: 0.2, affinity: 0.2,
|
# must match one of ("in Denmark, or remote"; the location), and an exclude
|
||||||
# taste: 0.2}; taste is how close a role is to the ones the agent liked minus
|
# gate leaves out the roles that match one of its values (title: the titles
|
||||||
# the ones it passed. A country is an ISO code, filled from the board's
|
# to leave out). A gate rejects only a role whose column is filled, since
|
||||||
# location line; EU is a role open anywhere in Europe.
|
# extract fills the rest for the roles in the cut. A company set to All
|
||||||
# limits is the agent's cut, the roles that get fetch text, extract and
|
# locations skips the any group, one set to All roles the exclude gates on
|
||||||
# judge: the best cut_fit_top by fit level at cut_fit_min or more, then the
|
# title. Fields are place, country, remote, remote_from, seniority,
|
||||||
# best cut_cv_top by CV closeness at cut_cv_min or more that the first list
|
# function, salary_min, currency, permit, languages, company and title. A
|
||||||
# leaves out, then every role liked or saved. Both levels are calibrated, so
|
# country is an ISO code, filled from the board's location line; EU is a
|
||||||
# a minimum means the same in every run, and a liked company's roles sit
|
# role open anywhere in Europe.
|
||||||
# about 0.05 higher, a favourite's about 0.1. summarise_top is how many of
|
# - weights, the want feed score's, which orders What I'm looking for: {fit:
|
||||||
# the best final scores summarise reads. The defaults are {cut_fit_top: 500,
|
# 0.6, recency: 0.2, affinity: 0.2, taste: 0.2}, that is prompt closeness,
|
||||||
# cut_fit_min: 0.6, cut_cv_top: 150, cut_cv_min: 0.6, summarise_top: 50}. For
|
# newness, company like and job like; taste is how close a role is to the
|
||||||
# a broad agent the count stops a list, for a narrow one the minimum. An
|
# ones the agent liked minus the ones it passed. cv_skills, cv_experience and
|
||||||
# agent without a prompt that has text has no fit, so nothing of it is in
|
# cv_career {30, 25, 30} weight the judge's three numbers in the CV match.
|
||||||
# the cut by fit.
|
# - limits, the analytics filter and the summarise size: {cut_fit_top: 500,
|
||||||
# prompts is a list of {name, text}, each what the agent looks for; a role's
|
# cut_fit_min: 0.6, cut_cv_top: 150, cut_cv_min: 0.6, summarise_top: 30}, the
|
||||||
# fit is its best prompt's, so one agent can hunt several kinds of role
|
# count and the minimum by prompt closeness and by CV closeness, and how deep
|
||||||
# without blending them into one paragraph. Votes, gates and limits stay the
|
# into each order summarise reads. See documentation/ADR Fetch prep and
|
||||||
# agent's, shared by its prompts. intent, as in config.yaml, is one prompt
|
# score.md.
|
||||||
# named after the agent.
|
# Two more, beside the starter in config.yaml:
|
||||||
# enabled (default true) leaves an agent out of the scores and the pull when
|
|
||||||
# false. The Search agents sub-tab lists and edits them; the order menu on
|
|
||||||
# Roles picks one by name. Two more, beside the starter in config.yaml:
|
|
||||||
agents: []
|
agents: []
|
||||||
# - name: Remote from Europe
|
# - name: Remote from Europe
|
||||||
# prompts:
|
# prompts:
|
||||||
@@ -191,7 +185,7 @@ agents: []
|
|||||||
# LV, LT, LU, MT, NL, PL, PT, RO, SK, SI, ES, SE, IS, LI, NO,
|
# LV, LT, LU, MT, NL, PL, PT, RO, SK, SI, ES, SE, IS, LI, NO,
|
||||||
# GB, CH, EU], mode: require}
|
# GB, CH, EU], mode: require}
|
||||||
# limits: {cut_fit_top: 1000, cut_fit_min: 0.5, cut_cv_top: 150, cut_cv_min: 0.6,
|
# limits: {cut_fit_top: 1000, cut_fit_min: 0.5, cut_cv_top: 150, cut_cv_min: 0.6,
|
||||||
# summarise_top: 50}
|
# summarise_top: 30}
|
||||||
# - name: Europe hybrid
|
# - name: Europe hybrid
|
||||||
# intent: >-
|
# intent: >-
|
||||||
# Senior backend or platform engineer in a European tech hub, hybrid or
|
# Senior backend or platform engineer in a European tech hub, hybrid or
|
||||||
|
|||||||
+4
-4
@@ -27,10 +27,10 @@ embed:
|
|||||||
model: intfloat/multilingual-e5-small
|
model: intfloat/multilingual-e5-small
|
||||||
sources: {}
|
sources: {}
|
||||||
# The starter ranking, imported once into an empty table. Edit it under
|
# The starter ranking, imported once into an empty table. Edit it under
|
||||||
# Settings, Ranking afterwards; this list is not read again. EU in a country
|
# Settings, Search agents afterwards; this list is not read again. EU in a
|
||||||
# gate is a role open anywhere in Europe ("Remote - Europe", "EMEA"). Which
|
# country gate is a role open anywhere in Europe ("Remote - Europe", "EMEA").
|
||||||
# roles get fetch text, extract and judge is each agent's cut, its `limits`, see
|
# Which roles get fetch text, extract and judge is each agent's cut, its
|
||||||
# config.example.yaml, which has two more agents.
|
# `limits`, see config.example.yaml, which has two more agents.
|
||||||
agents:
|
agents:
|
||||||
- name: Copenhagen engineering
|
- name: Copenhagen engineering
|
||||||
intent: >-
|
intent: >-
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Run steps 1, 2 and 3: test/promote, build image, push image.
|
|
||||||
.DESCRIPTION
|
|
||||||
Stops on the first failure. Server deployment is a separate operation --
|
|
||||||
nothing here connects to the server. Git branch publication is optional
|
|
||||||
through -PushBranch.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param([switch] $PushBranch)
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
& "$PSScriptRoot/1-test-and-promote.ps1" -PushBranch:$PushBranch
|
|
||||||
& "$PSScriptRoot/2-build-image.ps1"
|
|
||||||
& "$PSScriptRoot/3-push-image.ps1"
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Test the latest local main commit and fast-forward production to it.
|
|
||||||
.DESCRIPTION
|
|
||||||
Uses a fixed commit and an isolated worktree. PushBranch optionally publishes
|
|
||||||
production to origin; without it promotion stays local. Does not run Docker.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param([switch] $PushBranch, [switch] $NoTests)
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
. "$PSScriptRoot/_common.ps1"
|
|
||||||
|
|
||||||
$Source = Setting SOURCE 'main'
|
|
||||||
$Branch = Setting BRANCH 'production'
|
|
||||||
$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe')
|
|
||||||
|
|
||||||
Push-Location $root
|
|
||||||
try {
|
|
||||||
$commit = git rev-parse --verify "$Source^{commit}"
|
|
||||||
Assert-Ok 'resolve source commit'
|
|
||||||
$commit = $commit.Trim()
|
|
||||||
|
|
||||||
# ------------------------------------------------------- test the commit
|
|
||||||
|
|
||||||
# In a throwaway worktree, not here. The tests then run against exactly the
|
|
||||||
# tree that is about to become production, with none of the working copy's
|
|
||||||
# uncommitted edits in the way. Worktrees have no .venv of their own, so the
|
|
||||||
# interpreter comes from the main checkout and the code from the worktree.
|
|
||||||
#
|
|
||||||
# A checkout inside WSL has a Linux .venv, so there the same steps run in
|
|
||||||
# the distro through deploy/test-commit.sh.
|
|
||||||
if (-not $NoTests -and $Wsl.Success) {
|
|
||||||
Write-Step "tests on $Source, in WSL $($Wsl.Groups[1].Value)"
|
|
||||||
wsl.exe -d $Wsl.Groups[1].Value --cd (ConvertTo-LinuxPath $root) -- sh deploy/test-commit.sh $commit
|
|
||||||
Assert-Ok 'tests'
|
|
||||||
} elseif (-not $NoTests) {
|
|
||||||
Write-Step "tests on $Source"
|
|
||||||
if (-not (Test-Path $Python)) {
|
|
||||||
throw "no interpreter at $Python - set PYTHON in deploy\release.env"
|
|
||||||
}
|
|
||||||
$base = Join-Path ([IO.Path]::GetTempPath()) ("$Name-test-" + [guid]::NewGuid().ToString('N').Substring(0, 8))
|
|
||||||
$tree = Join-Path $base $Name
|
|
||||||
try {
|
|
||||||
git worktree add --detach $tree $commit | Out-Null
|
|
||||||
Assert-Ok 'git worktree add'
|
|
||||||
Push-Location $tree
|
|
||||||
try {
|
|
||||||
# Your project's checks. A gate that is red for weeks is not a
|
|
||||||
# gate -- it is a flag you learn to type -- so take one out when
|
|
||||||
# it stops meaning anything, and put it back when it is green.
|
|
||||||
#
|
|
||||||
# The starter's dev group is pytest and httpx, so pytest is the
|
|
||||||
# only gate that can run out of the box. Add ruff and mypy to
|
|
||||||
# [dependency-groups] dev in pyproject.toml, then uncomment the
|
|
||||||
# two lines below -- a gate whose tool is not installed fails
|
|
||||||
# every release for a reason that has nothing to do with the code.
|
|
||||||
& $Python -m pytest -q; Assert-Ok 'pytest'
|
|
||||||
# & $Python -m ruff check .; Assert-Ok 'ruff'
|
|
||||||
# & $Python -m mypy --cache-dir (Join-Path $root '.mypy_cache')
|
|
||||||
# Assert-Ok 'mypy'
|
|
||||||
} finally { Pop-Location }
|
|
||||||
} finally {
|
|
||||||
git worktree remove --force $tree 2>$null | Out-Null
|
|
||||||
Remove-ReleaseTemp @($base)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ------------------------------------------------------------- promote
|
|
||||||
|
|
||||||
# Fast-forward only. If production has commits main does not, that is a
|
|
||||||
# release made from somewhere else, and this stops rather than discarding it.
|
|
||||||
Write-Step "promote $Source -> $Branch"
|
|
||||||
git show-ref --verify --quiet "refs/heads/$Branch"
|
|
||||||
if ($LASTEXITCODE -eq 0) {
|
|
||||||
git merge-base --is-ancestor $Branch $commit
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
Write-Host "$Branch is not an ancestor of $Source - it has commits of its own." -ForegroundColor Red
|
|
||||||
throw "look at: git log --oneline $Source..$Branch"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
# Git publication is optional and independent of the image.
|
|
||||||
git branch -f $Branch $commit; Assert-Ok 'git branch -f'
|
|
||||||
|
|
||||||
if ($PushBranch) {
|
|
||||||
git push origin "${commit}:refs/heads/$Branch"
|
|
||||||
Assert-Ok 'publish production branch'
|
|
||||||
}
|
|
||||||
Write-Host "Promoted $Branch to $commit"
|
|
||||||
} finally { Pop-Location }
|
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Test the latest local main commit, fast-forward production to it, and export
|
||||||
|
# it for the image build on Windows.
|
||||||
|
#
|
||||||
|
# bash deploy/1-test-and-promote.sh [--push-branch] [--no-tests]
|
||||||
|
#
|
||||||
|
# Uses a fixed commit and an isolated worktree. --push-branch optionally
|
||||||
|
# publishes production to origin; without it promotion stays local. --no-tests
|
||||||
|
# skips the gates. The export is the production tree in <RELEASE_DIR>/<folder>/<sha>/
|
||||||
|
# and its sha in <RELEASE_DIR>/<folder>/production, RELEASE_DIR defaulting to
|
||||||
|
# release/ beside the checkout. Uncommitted work is never included.
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck source=_common.sh
|
||||||
|
. "$(dirname "${BASH_SOURCE[0]}")/_common.sh"
|
||||||
|
|
||||||
|
push_branch=0 no_tests=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--push-branch) push_branch=1 ;;
|
||||||
|
--no-tests) no_tests=1 ;;
|
||||||
|
-h|--help) sed -n '2,11p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
|
*) die "unknown option $arg" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
need_git
|
||||||
|
SOURCE=$(setting SOURCE main)
|
||||||
|
BRANCH=$(setting BRANCH production)
|
||||||
|
PYTHON=$(setting PYTHON "$root/.venv/bin/python")
|
||||||
|
|
||||||
|
cd "$root"
|
||||||
|
commit=$(resolve_commit "$SOURCE" source)
|
||||||
|
|
||||||
|
# ----------------------------------------------------------- test the commit
|
||||||
|
|
||||||
|
# In a throwaway worktree, not here. The tests then run against exactly the tree
|
||||||
|
# that is about to become production, with none of the working copy's
|
||||||
|
# uncommitted edits in the way. test-commit.sh makes the worktree, runs pytest,
|
||||||
|
# ruff, mypy and the client's check in it, and removes it. Worktrees have no
|
||||||
|
# .venv of their own, so the interpreter comes from this checkout.
|
||||||
|
if ((! no_tests)); then
|
||||||
|
step "tests on $SOURCE"
|
||||||
|
run_or_die tests env PYTHON="$PYTHON" sh "$here/test-commit.sh" "$commit"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------- promote
|
||||||
|
|
||||||
|
# Fast-forward only. If production has commits main does not, that is a release
|
||||||
|
# made from somewhere else, and this stops rather than discarding it.
|
||||||
|
step "promote $SOURCE -> $BRANCH"
|
||||||
|
if git show-ref --verify --quiet "refs/heads/$BRANCH"; then
|
||||||
|
if ! git merge-base --is-ancestor "$BRANCH" "$commit"; then
|
||||||
|
echo "$BRANCH is not an ancestor of $SOURCE - it has commits of its own." >&2
|
||||||
|
die "look at: git log --oneline $SOURCE..$BRANCH"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
# Git publication is optional and independent of the image.
|
||||||
|
run_or_die 'git branch -f' git branch -f "$BRANCH" "$commit"
|
||||||
|
|
||||||
|
if ((push_branch)); then
|
||||||
|
run_or_die 'publish production branch' git push origin "$commit:refs/heads/$BRANCH"
|
||||||
|
fi
|
||||||
|
echo "Promoted $BRANCH to $commit"
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------- export
|
||||||
|
|
||||||
|
# The image is built on Windows by wslc (2-build-image.ps1), which reads this
|
||||||
|
# folder through \\wsl.localhost. A tree rather than a tar, so Windows has
|
||||||
|
# nothing to unpack. wslc hands the Dockerfile to the builder on stdin, so the
|
||||||
|
# builder only finds the ignore rules at the root of the context:
|
||||||
|
# Dockerfile.dockerignore is copied there as .dockerignore. A tree that exists
|
||||||
|
# for this sha is kept, not rewritten, because wslc can build from a stale view
|
||||||
|
# of a folder deleted and recreated at the same path (microsoft/WSL#41287).
|
||||||
|
# The promote has happened by now; when the export fails, rerun this step.
|
||||||
|
name=$(basename "$root")
|
||||||
|
release=${RELEASE_DIR:-$(dirname "$root")/release}/$name
|
||||||
|
sha=$(git rev-parse --short "$commit") || die 'resolve image tag failed'
|
||||||
|
tree=$release/$sha
|
||||||
|
step "export $BRANCH ($sha) to $tree"
|
||||||
|
mkdir -p -- "$release" 2>/dev/null && [[ -w $release ]] ||
|
||||||
|
die "cannot write $release; $BRANCH is promoted, rerun this step once it is writable"
|
||||||
|
if [[ ! -d $tree ]]; then
|
||||||
|
part=$release/.part-$sha
|
||||||
|
rm -rf -- "$part"
|
||||||
|
mkdir -- "$part"
|
||||||
|
git archive --format=tar "$commit" | tar -x -C "$part" || die 'export failed'
|
||||||
|
if [[ -f $part/deploy/Dockerfile.dockerignore ]]; then
|
||||||
|
cp -- "$part/deploy/Dockerfile.dockerignore" "$part/.dockerignore"
|
||||||
|
fi
|
||||||
|
mv -- "$part" "$tree"
|
||||||
|
fi
|
||||||
|
for old in "$release"/*/; do
|
||||||
|
[[ ${old%/} == "$tree" ]] || rm -rf -- "${old%/}"
|
||||||
|
done
|
||||||
|
printf '%s\n' "$sha" > "$release/production"
|
||||||
|
echo "Exported $tree"
|
||||||
+12
-56
@@ -1,57 +1,13 @@
|
|||||||
<#
|
# Step 2, on Windows: build the image from the tree step 1 exported, with wslc.
|
||||||
.SYNOPSIS
|
# Tags <IMAGE>:<sha> and <IMAGE>:latest. Never pushes; 3-push-image.ps1 does.
|
||||||
Build a local Docker image from the committed production branch.
|
|
||||||
.DESCRIPTION
|
|
||||||
Does not test, promote, push Git branches, use SSH, or start a container.
|
|
||||||
The image stays local. Use 3-push-image.ps1 to publish it separately.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param()
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
. "$PSScriptRoot/_common.ps1"
|
. "$PSScriptRoot\_common.ps1"
|
||||||
|
Assert-Wslc
|
||||||
$Image = Setting IMAGE -Required
|
$sha = Get-ReleaseSha
|
||||||
$Branch = Setting BRANCH 'production'
|
$context = Join-Path $Release $sha
|
||||||
$Platform = Setting PLATFORM 'linux/amd64'
|
Write-Host "== build ${Image}:$sha from $context" -ForegroundColor Cyan
|
||||||
|
# -f is a path wslc opens itself, not one inside the context.
|
||||||
Push-Location $root
|
wslc build -t "${Image}:$sha" -t "${Image}:latest" -f "$context\deploy\Dockerfile" $context
|
||||||
try {
|
if ($LASTEXITCODE -ne 0) { throw "wslc build failed (exit $LASTEXITCODE)" }
|
||||||
$commit = git rev-parse --verify "$Branch^{commit}"
|
wslc image list
|
||||||
Assert-Ok 'resolve production commit'
|
Write-Host "Image: ${Image}:$sha"
|
||||||
$commit = $commit.Trim()
|
|
||||||
$sha = (git rev-parse --short $commit).Trim()
|
|
||||||
Assert-Ok 'resolve image tag'
|
|
||||||
$tagged = "${Image}:${sha}"
|
|
||||||
$latest = "${Image}:latest"
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- build
|
|
||||||
|
|
||||||
# `git archive` to a file and unpack it, rather than piping it. A PowerShell
|
|
||||||
# pipeline carries text: it would re-encode a tar stream and hand docker a
|
|
||||||
# corrupted context. Writing the archive out and building from the directory
|
|
||||||
# keeps the same guarantee -- the context is the committed tree and nothing
|
|
||||||
# else -- without a byte going through a pipe.
|
|
||||||
#
|
|
||||||
# The Dockerfile comes out of that same tree, at deploy/Dockerfile, so the
|
|
||||||
# recipe is versioned with the code it builds. Docker reads the ignore rules
|
|
||||||
# from deploy/Dockerfile.dockerignore beside it.
|
|
||||||
|
|
||||||
Write-Step "build $tagged"
|
|
||||||
$ctx = Join-Path ([IO.Path]::GetTempPath()) "$Name-build-$sha"
|
|
||||||
$tar = "$ctx.tar"
|
|
||||||
Remove-ReleaseTemp @($ctx, $tar)
|
|
||||||
New-Item -ItemType Directory -Path $ctx | Out-Null
|
|
||||||
try {
|
|
||||||
git archive --format=tar -o $tar $commit; Assert-Ok 'git archive'
|
|
||||||
tar -xf $tar -C $ctx; Assert-Ok 'tar -xf'
|
|
||||||
$dockerfile = Join-Path $ctx 'deploy/Dockerfile'
|
|
||||||
docker build --platform $Platform -f $dockerfile -t $tagged -t $latest $ctx
|
|
||||||
Assert-Ok 'docker build'
|
|
||||||
docker image ls $Image --format ' {{.Tag}} {{.Size}}'
|
|
||||||
} finally {
|
|
||||||
Remove-ReleaseTemp @($ctx, $tar)
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "Image: $tagged"
|
|
||||||
} finally { Pop-Location }
|
|
||||||
+19
-35
@@ -1,36 +1,20 @@
|
|||||||
<#
|
# Step 3, on Windows: push the image step 2 built, <IMAGE>:<sha> and :latest,
|
||||||
.SYNOPSIS
|
# with wslc. Never rebuilds: after a failed push, fix the login and run it again.
|
||||||
Publish the already-built production image to the registry.
|
|
||||||
.DESCRIPTION
|
|
||||||
Never rebuilds. If the push fails, fix access or connectivity and run this
|
|
||||||
again; the image is already on disk. `docker login` first, with a token
|
|
||||||
that has package-write access.
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param()
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
. "$PSScriptRoot/_common.ps1"
|
. "$PSScriptRoot\_common.ps1"
|
||||||
|
Assert-Wslc
|
||||||
$Image = Setting IMAGE -Required
|
$sha = Get-ReleaseSha
|
||||||
$Branch = Setting BRANCH 'production'
|
wslc image inspect "${Image}:$sha" | Out-Null
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw "no local ${Image}:$sha - run step 2 first" }
|
||||||
Push-Location $root
|
# latest must name this candidate, even if another image was built since.
|
||||||
try {
|
wslc tag "${Image}:$sha" "${Image}:latest"
|
||||||
$commit = git rev-parse --verify "$Branch^{commit}"
|
if ($LASTEXITCODE -ne 0) { throw "wslc tag failed (exit $LASTEXITCODE)" }
|
||||||
Assert-Ok 'resolve production commit'
|
foreach ($tag in $sha, 'latest') {
|
||||||
$sha = (git rev-parse --short $commit.Trim()).Trim()
|
Write-Host "== push ${Image}:$tag" -ForegroundColor Cyan
|
||||||
Assert-Ok 'resolve image tag'
|
wslc push "${Image}:$tag"
|
||||||
$tagged = "${Image}:${sha}"
|
if ($LASTEXITCODE -ne 0) {
|
||||||
$latest = "${Image}:latest"
|
Write-Host "If the push was refused: wslc login $(($Image -split '/')[0]) -u <user>, with a token that has package-write access" -ForegroundColor Yellow
|
||||||
docker image inspect $tagged --format '{{.Id}}' | Out-Null
|
throw "wslc push ${Image}:$tag failed (exit $LASTEXITCODE)"
|
||||||
Assert-Ok "find local $tagged (run step 2 first)"
|
}
|
||||||
# latest must name this candidate, even if another image was built since.
|
}
|
||||||
docker tag $tagged $latest
|
Write-Host "Published ${Image}:$sha and ${Image}:latest"
|
||||||
Assert-Ok 'tag latest image'
|
|
||||||
docker push $tagged
|
|
||||||
Assert-Ok 'push commit image'
|
|
||||||
docker push $latest
|
|
||||||
Assert-Ok 'push latest image'
|
|
||||||
Write-Host "Published $tagged and $latest"
|
|
||||||
} finally { Pop-Location }
|
|
||||||
+50
-16
@@ -1,21 +1,28 @@
|
|||||||
# A Python app, for a server rather than a laptop.
|
# A Python app, for a server rather than a laptop.
|
||||||
#
|
#
|
||||||
# Two stages so the runtime carries no build tooling: `uv` resolves and
|
# Two stages so the runtime carries no build tooling: `uv` resolves and
|
||||||
# installs into /app/.venv, and the second stage copies that venv and the
|
# installs into /app/.venv, and the second stage copies the venv and the source
|
||||||
# source next to it. The install is deliberately editable, which matters if
|
# next to it. The install is deliberately editable, which matters because the
|
||||||
# the app reads a file from beside its package (an example config, a template
|
# app reads files from beside its package (templates, `app/static`, `seed/`):
|
||||||
# directory): that path is true in a checkout and true here, and false in
|
# that path is true in a checkout and true here, and false in site-packages.
|
||||||
# site-packages.
|
# Ours: roledesk has no `frontend/` and no Node stage, since the server renders
|
||||||
|
# Jinja templates and serves `app/static` as it is.
|
||||||
#
|
#
|
||||||
# Already in place at deploy/Dockerfile, with its ignore file beside it.
|
# Already in place at deploy/Dockerfile, with its ignore file beside it.
|
||||||
# 2-build-image.ps1 reads it out of a `git archive` of the production commit,
|
# 2-build-image.ps1 builds with it from the `git archive` of the production
|
||||||
# so the recipe is versioned with the code it builds.
|
# commit that step 1 exports, so the recipe is versioned with the code it builds.
|
||||||
#
|
#
|
||||||
# The package is `app` and stays `app` — a project started from this repo
|
# The package is `app` and stays `app` (the domain noun is `role`). Edit the
|
||||||
# renames nothing. Edit the two `uv sync` lines when you add extras.
|
# two `uv sync` lines when you add extras.
|
||||||
|
#
|
||||||
|
# Each base image is pinned by tag and digest: the tag says which release, the
|
||||||
|
# digest makes a rebuild next month install the same bytes. The digest is the
|
||||||
|
# multi-platform index's. To move on, look up the new tag's digest on its
|
||||||
|
# registry (Docker Hub's tag API for python, ghcr.io for uv) and
|
||||||
|
# replace both halves.
|
||||||
|
|
||||||
ARG PYTHON=3.14-slim
|
ARG PYTHON=3.15-slim@sha256:2da676046630d7bf14c6764ae1143dc7fa0377a53f94bf5a642f0b214aca5bce
|
||||||
ARG UV=0.9.17
|
ARG UV=0.13.1@sha256:4a1f3666816dafadd20ceb71eac9a7d56d7ef30279f7d64048519ab901ada336
|
||||||
|
|
||||||
FROM ghcr.io/astral-sh/uv:${UV} AS uv
|
FROM ghcr.io/astral-sh/uv:${UV} AS uv
|
||||||
|
|
||||||
@@ -32,22 +39,38 @@ WORKDIR /app
|
|||||||
|
|
||||||
# Dependencies first and on their own layer, keyed on the lock file alone, so
|
# Dependencies first and on their own layer, keyed on the lock file alone, so
|
||||||
# editing a Python file rebuilds in seconds rather than re-resolving the tree.
|
# editing a Python file rebuilds in seconds rather than re-resolving the tree.
|
||||||
|
# The extras are the same on both `uv sync` lines: `passkeys`, without which
|
||||||
|
# the passkey routes answer 501, `embeddings`, the default ranking backend
|
||||||
|
# (torch comes from the CPU index named in pyproject.toml, [tool.uv.sources],
|
||||||
|
# not the CUDA build; the model itself is downloaded by the first `embed` job
|
||||||
|
# into HOME, which is the /data volume), and `llm`, the SDKs the compose file
|
||||||
|
# passes keys for.
|
||||||
COPY pyproject.toml uv.lock README.md ./
|
COPY pyproject.toml uv.lock README.md ./
|
||||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||||
uv sync --frozen --no-dev --no-install-project
|
uv sync --frozen --no-dev --no-install-project \
|
||||||
|
--extra passkeys --extra embeddings --extra llm
|
||||||
|
|
||||||
COPY app app
|
COPY app app
|
||||||
# The two module folders this repo ships beside the app, here and again in the
|
# The module folders this repo ships beside the app, here and again in the
|
||||||
# runtime stage below. `pyproject.toml` lists `pipeline*` and `auth*` in its
|
# runtime stage below. `pyproject.toml` lists `pipeline*`, `auth*`, `llm*` and
|
||||||
# packages as well, or the install leaves them out.
|
# `desk*` in its packages as well, or the install leaves them out.
|
||||||
COPY pipeline pipeline
|
COPY pipeline pipeline
|
||||||
# The door. A project that does not want one drops this line *and* the
|
# The door. A project that does not want one drops this line *and* the
|
||||||
# `auth.connect` call in `app/main.py` together: dropping the folder and
|
# `auth.connect` call in `app/main.py` together: dropping the folder and
|
||||||
# leaving the call is a container that fails at startup, and dropping the call
|
# leaving the call is a container that fails at startup, and dropping the call
|
||||||
# and leaving the folder is a container that starts with the door open.
|
# and leaving the folder is a container that starts with the door open.
|
||||||
COPY auth auth
|
COPY auth auth
|
||||||
|
# The model call. Nothing imports it until an app adds its SDKs, and then the
|
||||||
|
# folder has to be here already, so that is not a Dockerfile edit.
|
||||||
|
COPY llm llm
|
||||||
|
# Ours: the source contract and the generic desk modules, the plugin folder the
|
||||||
|
# config's `plugin_paths` names, and the data a first run imports (`seed/`).
|
||||||
|
COPY desk desk
|
||||||
|
COPY plugins plugins
|
||||||
|
COPY seed seed
|
||||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||||
uv sync --frozen --no-dev
|
uv sync --frozen --no-dev \
|
||||||
|
--extra passkeys --extra embeddings --extra llm
|
||||||
|
|
||||||
# ---------------------------------------------------------------- runtime
|
# ---------------------------------------------------------------- runtime
|
||||||
|
|
||||||
@@ -69,6 +92,10 @@ COPY pipeline pipeline
|
|||||||
# The login page and the two scripts are files inside this folder, served from
|
# The login page and the two scripts are files inside this folder, served from
|
||||||
# routes; see the build stage above.
|
# routes; see the build stage above.
|
||||||
COPY auth auth
|
COPY auth auth
|
||||||
|
COPY llm llm
|
||||||
|
COPY desk desk
|
||||||
|
COPY plugins plugins
|
||||||
|
COPY seed seed
|
||||||
|
|
||||||
ENV PATH=/app/.venv/bin:$PATH \
|
ENV PATH=/app/.venv/bin:$PATH \
|
||||||
PYTHONUNBUFFERED=1 \
|
PYTHONUNBUFFERED=1 \
|
||||||
@@ -79,6 +106,13 @@ ENV PATH=/app/.venv/bin:$PATH \
|
|||||||
# /data, and a CLI the app shells out to — the `claude` binary, for one —
|
# /data, and a CLI the app shells out to — the `claude` binary, for one —
|
||||||
# wants somewhere to write. An image-baked /home/app would be right for uid
|
# wants somewhere to write. An image-baked /home/app would be right for uid
|
||||||
# 1000 and unwritable for anyone else.
|
# 1000 and unwritable for anyone else.
|
||||||
|
#
|
||||||
|
# The image runs as 1000:1000, the compose file's default PUID:PGID and the
|
||||||
|
# owner docs/Install Server.md gives the data folder, so a `docker run` without a
|
||||||
|
# `user:` does not write root-owned files into it. /data is made here, before
|
||||||
|
# VOLUME, so an anonymous volume starts out owned by that uid too.
|
||||||
|
RUN install -d -o 1000 -g 1000 /data /data/home
|
||||||
|
USER 1000:1000
|
||||||
VOLUME /data
|
VOLUME /data
|
||||||
|
|
||||||
EXPOSE 8787
|
EXPOSE 8787
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
# Named for the Dockerfile beside it: Docker reads `<dockerfile>.dockerignore`
|
# Named for the Dockerfile beside it. Docker reads `<dockerfile>.dockerignore`
|
||||||
# before falling back to one at the root of the context, which is how the ignore
|
# before falling back to one at the root of the context, but wslc hands the
|
||||||
# rules get to live next to the recipe they belong to instead of at the top of
|
# Dockerfile to the builder on stdin, which leaves only the root one: step 1
|
||||||
# the repository.
|
# (1-test-and-promote.sh) copies this file to `.dockerignore` at the root of
|
||||||
|
# the tree it exports, so the rules still live next to the recipe here.
|
||||||
#
|
#
|
||||||
# 2-build-image.ps1 already hands docker a context extracted from `git archive`,
|
# That tree is a `git archive` of the production commit, so the untracked
|
||||||
# so the untracked heavyweights below are not there to exclude. They are listed
|
# heavyweights below are not there to exclude. They are listed for the other
|
||||||
# for the other way in — `docker build -f deploy/Dockerfile .` from a checkout —
|
# way in, `docker build -f deploy/Dockerfile .` from a checkout, where .venv
|
||||||
# where .venv alone can be several hundred megabytes.
|
# alone can be several hundred megabytes.
|
||||||
.git
|
.git
|
||||||
.venv
|
.venv
|
||||||
.claude
|
.claude
|
||||||
@@ -19,6 +20,7 @@ docs
|
|||||||
# per folder, so a fourth module is a fourth line here.
|
# per folder, so a fourth module is a fourth line here.
|
||||||
tests
|
tests
|
||||||
pipeline/tests
|
pipeline/tests
|
||||||
|
llm/tests
|
||||||
auth/tests
|
auth/tests
|
||||||
|
|
||||||
__pycache__
|
__pycache__
|
||||||
|
|||||||
+45
-21
@@ -1,52 +1,76 @@
|
|||||||
# Deploy
|
# Deploy
|
||||||
|
|
||||||
Everything needed to put this app on the rittershofer.org box behind SWAG: build an image on Windows, publish it to Gitea, run it as one more service in the compose file that already runs the proxy, back up its database, and give it a password that survives being on the open internet.
|
Everything needed to put roledesk on a server behind SWAG ([../docs/Install Server.md](../docs/Install%20Server.md) is the box it was written for, a copy of the boilerplate's file): test it in WSL, build an image on Windows with `wslc`, publish it to Gitea, run it as one more service in the compose file that already runs the proxy, back up its database, and give it a password that survives being on the open internet.
|
||||||
|
|
||||||
Extracted from newsdesk, which is the worked example — every decision here has been run in production at least once, and the comments say why rather than what.
|
This folder is a downstream copy of `../boilerplate/deploy` (commit 09e144d), refreshed with `git archive`; a fix to a shared script lands upstream first. roledesk's own facts are put back after a refresh: the Dockerfile without a frontend stage and with `desk`, `plugins`, `llm` and `seed`, the extras `passkeys`, `embeddings` and `llm`, our image name in `release.env.example`, `server/config.example.yaml` for roledesk's keys, `TABLES` and `REQUIRED_TABLES` naming `roles`, and `bin/pull.sh` and `bin/worker.sh`. `tests/test_deploy.py` pins them.
|
||||||
|
|
||||||
**It is already in place.** Nothing has to be copied and nothing has to be renamed: the package is `app`, the environment variables are `APP_`, and the files below are where the scripts expect them. `<app>` in the docs is your *service's* name — the hostname, the data directory, the image tag, the compose service — and it is the one thing you choose.
|
The package is `app`, and the files below are where the scripts expect them: the `APP_` variables, the compose service `app` and the hostname `app.example.com` in the templates are placeholders for the server, and `<app>` in the docs is the service's name: the hostname, the data directory, the image tag, the compose service. The domain noun is `role`, so "job" only ever means queued work.
|
||||||
|
|
||||||
## Where each file is
|
## Where each file is
|
||||||
|
|
||||||
| Here | | |
|
| Here | | |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `deploy/*.ps1` | the release scripts, run on Windows | |
|
| `deploy/1-test-and-promote.sh`, `test-commit.sh`, `_common.sh`, `db.sh` | the gate, the export and the database moves, run in the WSL distro | |
|
||||||
| `deploy/Dockerfile` | read out of the archived commit by step 2 | copies `app`, `pipeline` and `auth` |
|
| `deploy/2-build-image.ps1`, `3-push-image.ps1`, `_common.ps1` | the image build and push, run on Windows with `wslc` | the distro is a sealed sandbox and does not hold a container socket |
|
||||||
| `deploy/Dockerfile.dockerignore` | Docker finds it by that name | |
|
| `deploy/wsl-setup.cmd`, `wsl-setup.ps1` | the profile commands, written once per checkout on Windows | |
|
||||||
|
| `bin/backend.sh`, `check.sh`, and ours `pull.sh`, `worker.sh` | the dev commands, run in the WSL distro: the backend, the gate on the working tree, a pull, the worker | |
|
||||||
|
| `.run/serve`, `pull`, `worker` | PyCharm run configurations, left as they are | `serve --reload` = `bin/backend.sh --reload` = `<folder>-backend --reload`; `pull` = `bin/pull.sh` = `<folder>-pull`; `worker` = `bin/worker.sh` = `<folder>-worker` |
|
||||||
|
| `deploy/Dockerfile` | read out of the tree step 1 exports, by step 2 | no frontend stage; copies `app`, `pipeline`, `auth`, `llm`, `desk`, `plugins` and `seed` |
|
||||||
|
| `deploy/Dockerfile.dockerignore` | copied to `.dockerignore` at the root of the exported tree | `wslc` streams the Dockerfile, so only a root ignore file applies |
|
||||||
| `deploy/server/*` | committed here, then installed on the server | |
|
| `deploy/server/*` | committed here, then installed on the server | |
|
||||||
| `deploy/server/compose.yaml` | to `/srv/swag/docker-compose.yml`, as one `services:` block | |
|
| `deploy/server/compose.yaml` | to `/srv/swag/docker-compose.yml`, as one `services:` block | |
|
||||||
| `deploy/server/env.example` | to `/srv/swag/.env`, as appended variables, `chmod 600` | |
|
| `deploy/server/env.example` | to `/srv/swag/.env`, as appended variables, `chmod 600` | |
|
||||||
|
| `deploy/server/config.example.yaml` | to `$APP_DATA/config.yaml`, with the hostname filled in | `deploy.sh` refuses a config without `database` and `auth.enabled` |
|
||||||
| `deploy/server/deploy.sh`, `deploy/server/backup.sh` | to `/srv/swag/`, next to the compose file | |
|
| `deploy/server/deploy.sh`, `deploy/server/backup.sh` | to `/srv/swag/`, next to the compose file | |
|
||||||
|
| `deploy/server/import.sh`, `deploy/server/import_database.py` | uploaded and run by `db.sh push` | |
|
||||||
| `deploy/server/swag/app.subdomain.conf` | to SWAG's `proxy-confs/`, renamed to `<app>.subdomain.conf` | |
|
| `deploy/server/swag/app.subdomain.conf` | to SWAG's `proxy-confs/`, renamed to `<app>.subdomain.conf` | |
|
||||||
|
|
||||||
`db.ps1` expects `server/` beside it, which is why `deploy/server/` is committed here as well as installed on the server.
|
`db.sh` expects `server/` beside it, which is why `deploy/server/` is committed here as well as installed on the server.
|
||||||
|
|
||||||
**Keep the `.ps1` files to ASCII.** Windows PowerShell 5.1 reads a script without a byte-order mark as Windows-1252, so an em dash becomes three characters ending in a closing quote: the string ends in the middle of itself and the script fails to parse before it runs a line. Write `--`. newsdesk hit this on a comment nobody thought was code, and `tests/test_packaging.py` here holds every script in this folder to it.
|
**Keep every `.ps1` to ASCII.** Windows PowerShell 5.1 reads a script without a byte-order mark as Windows-1252, so an em dash becomes three characters ending in a closing quote: the string ends in the middle of itself and the script fails to parse before it runs a line. Write `--`. newsdesk hit this on a comment nobody thought was code, and `tests/test_packaging.py` here holds every `.ps1` in this folder to it, the release scripts included.
|
||||||
|
|
||||||
|
`wsl-setup.sh` and `wsl-setup.ps1` are the two halves of setting up a checkout that lives in WSL. `wsl-setup.sh` runs in the distro ([../docs/Install WSL.md, Git](../docs/Install%20WSL.md#git)). `wsl-setup.ps1`, or a double-click on `deploy\wsl-setup.cmd`, writes the commands below into the PowerShell profile, with `<folder>` the checkout folder's name (`jobboard` here) or `-Name`. PowerShell 7, the execution policy, `wslc`, the hand-paste fallback and what to check when a command fails are in [../docs/Install WSL.md, PS Profile commands](../docs/Install%20WSL.md#ps-profile-commands).
|
||||||
|
|
||||||
|
| Command | Runs | Does |
|
||||||
|
|---|---|---|
|
||||||
|
| `<folder>-cd` | `Set-Location` | PowerShell into the checkout's folder |
|
||||||
|
| `<folder>-sh` | `wsl.exe` | a shell in the distro, in the checkout |
|
||||||
|
| `<folder>-backend [-v] [--reload]` | bin/backend.sh in WSL | `uv run app serve` on :8790 |
|
||||||
|
| `<folder>-pull [args]` | bin/pull.sh in WSL | `uv run app job pull`, one pull in the foreground (roledesk's) |
|
||||||
|
| `<folder>-worker [args]` | bin/worker.sh in WSL | `uv run app worker`, the queue drained with no web app (roledesk's) |
|
||||||
|
| `<folder>-check` | bin/check.sh in WSL | the gate on the working tree, without promoting |
|
||||||
|
| `<folder>-1test [--push-branch] [--no-tests]` | deploy/1-test-and-promote.sh in WSL | test local main, fast-forward `production`, export it |
|
||||||
|
| `<folder>-2build` | deploy\2-build-image.ps1 on Windows | build the exported tree with `wslc` |
|
||||||
|
| `<folder>-3push` | deploy\3-push-image.ps1 on Windows | push the built image with `wslc` |
|
||||||
|
| `<folder>-0release [--push-branch] [--no-tests]` | the three above | steps 1, 2 and 3 in order, stopping at the first failure |
|
||||||
|
| `<folder>-db pull\|push` | deploy/db.sh in WSL | the database, to and from the server |
|
||||||
|
|
||||||
|
Each command is one line in the profile: `wsl.exe -d <DISTRO> --cd <CHECKOUT> -- bash <script> @args` for the ones that run in the distro, and for steps 2 and 3 the `.ps1` with the execution policy lifted for that run. `DISTRO` and `CHECKOUT` are read from `deploy/release.env` when the setup runs, so rerun it after either changes. `-backend`, `-pull` and `-worker` stay in the foreground until Ctrl-C. The `-0release` to `-3push` names carry the number of the script's file name, so `0<Tab>` after `<folder>-` finds them.
|
||||||
|
|
||||||
`.gitignore` already covers `deploy/release.env` and `deploy/backups/`.
|
`.gitignore` already covers `deploy/release.env` and `deploy/backups/`.
|
||||||
|
|
||||||
## The docs
|
## The docs
|
||||||
|
|
||||||
They are one directory up, in [../docs/](../docs), because they are about the hosting rather than about these scripts.
|
They are one directory up, in [../docs/](../docs), because they are about the hosting rather than about these scripts. `Install Server.md` and `Install WSL.md` are copies of the boilerplate's `documentation/` files; the two proposals stay in the boilerplate checkout.
|
||||||
|
|
||||||
- **[../docs/hosting.md](../docs/hosting.md)** — what the box is, the seven steps to add a service, what the templates already decide for you, and the things that have already gone wrong on it. Start here for a service that is not on the box yet.
|
- **[../docs/Install Server.md](../docs/Install%20Server.md)** — what the box is, the seven steps to add a service, what the templates already decide for you, and the things that have already gone wrong on it. Start here for a service that is not on the box yet.
|
||||||
- **[../docs/release.md](../docs/release.md)** — the Windows side: `release.env`, the four numbered scripts, what each one refuses to do, and why the tests run in a throwaway worktree.
|
- **[../docs/Install Server.md, Publishing a release](../docs/Install%20Server.md#publishing-a-release)** — the release side: `release.env`, the numbered scripts and where each runs, what each one refuses to do, and why the tests run in a throwaway worktree.
|
||||||
- **[../docs/server.md](../docs/server.md)** — installing, deploying, backups, moving a SQLite database in either direction, rolling back, and what to verify before relying on any of it.
|
- **[../docs/Install Server.md, Operating it](../docs/Install%20Server.md#operating-it)** — installing, deploying, backups, moving a SQLite database in either direction, rolling back, and what to verify before relying on any of it.
|
||||||
- **[../docs/llm-backends.md](../docs/llm-backends.md)** — asking Claude from a deployed app: API key or subscription token, selected by which one is in the environment, with a self-contained reference implementation.
|
- **[Proposal LLM Backends.md](../../boilerplate/documentation/Proposal%20LLM%20Backends.md)** (boilerplate checkout) — asking Claude from a deployed app: API key or subscription token, selected by which one is in the environment. `llm/` is the implementation of it, see [../llm/README.md](../llm/README.md).
|
||||||
- **[../docs/password-protection.md](../docs/password-protection.md)** — one password on a public hostname: the lockout, sessions, passkeys, and how much throttling is actually worth building. `auth/` is the implementation of it.
|
- **[Proposal Password Protection.md](../../boilerplate/documentation/Proposal%20Password%20Protection.md)** (boilerplate checkout) — one password on a public hostname: the lockout, sessions, passkeys, and how much throttling is actually worth building. `auth/` is the implementation of it, see [../auth/README.md](../auth/README.md).
|
||||||
|
|
||||||
## A new service, in order
|
## A new service, in order
|
||||||
|
|
||||||
1. DNS, then certificate, then registry login, then data directory — [../docs/hosting.md](../docs/hosting.md).
|
1. DNS, then certificate, then registry login, then data directory — [../docs/Install Server.md](../docs/Install%20Server.md).
|
||||||
2. Copy `release.env.example` to `release.env` and fill it in. That is the only file this step touches: everything else is already committed.
|
2. Copy `release.env.example` to `release.env` and fill it in. That is the only file this step touches: everything else is already committed.
|
||||||
3. `.\deploy\0-release.ps1` to test, promote, build and publish.
|
3. `<folder>-0release` from Windows to test, promote, build and publish. `wslc` and its registry login are on Windows, [../docs/Install Server.md, Publishing a release](../docs/Install%20Server.md#publishing-a-release).
|
||||||
4. Install the server files, edit `.env`, `bash deploy.sh` — [../docs/server.md](../docs/server.md).
|
4. Install the server files and `config.yaml`, edit `.env`, `bash deploy.sh` — [../docs/Install Server.md, Operating it](../docs/Install%20Server.md#operating-it).
|
||||||
5. Put the model credential in `.env` if the app needs one — [../docs/llm-backends.md](../docs/llm-backends.md).
|
5. Put the model credential in `.env` if the app needs one — [../llm/README.md](../llm/README.md).
|
||||||
6. Get in, rotate the bootstrap password, register a passkey — [../docs/password-protection.md](../docs/password-protection.md).
|
6. Get in, rotate the bootstrap password, register a passkey — [../auth/README.md](../auth/README.md).
|
||||||
7. Schedule `backup.sh`, then restore one of its backups somewhere else to prove it works.
|
7. Schedule `backup.sh`, then restore one of its backups somewhere else to prove it works.
|
||||||
|
|
||||||
Step 1 of `0-release.ps1` runs `pytest` in a throwaway worktree, with `ruff` and `mypy` commented out beside it: this repository's dev group is pytest and httpx, and a gate whose tool is not installed fails every release for a reason that has nothing to do with the code. Add the tools, then uncomment the gates.
|
Step 1 runs `pytest` in a throwaway worktree through `test-commit.sh`, with `ruff` and `mypy` commented out beside it, as the release has always run. The dev group has both, so uncommenting them is the owner's call: the tree has to pass them first.
|
||||||
|
|
||||||
## What this is not
|
## What this is not
|
||||||
|
|
||||||
Not a framework and not a generator: nothing here is imported or extended, it is copied and edited. That is the point — three services with three edited copies beats three services sharing one abstraction that has to be right for all of them. What this repository changes is only *where the copy comes from*: `boilerplate/deploy/` is the source of truth, and news and health hold copies of it.
|
Not a framework: nothing here is imported or extended, it is copied and edited. `boilerplate/deploy/` is the source of truth, and news, health, jobboard and pocketmoney hold copies of it.
|
||||||
+22
-66
@@ -1,76 +1,32 @@
|
|||||||
<#
|
# Dot-sourced by 2-build-image.ps1 and 3-push-image.ps1. Reads deploy\release.env
|
||||||
.SYNOPSIS
|
# beside it (KEY=VALUE lines, # comments, never run as code) into $Settings,
|
||||||
Shared plumbing for the release scripts. Dot-source it; do not run it.
|
# and works out where step 1 exported the release: release\<folder> beside the
|
||||||
|
# checkout, in the distro, seen from Windows through \\wsl.localhost.
|
||||||
.DESCRIPTION
|
|
||||||
Dot-sourcing runs in the caller's scope, so everything defined here --
|
|
||||||
$root, $settings, Setting, Assert-Ok, Write-Step, Remove-ReleaseTemp -- is
|
|
||||||
the caller's afterwards. One copy of the release.env reader, because three
|
|
||||||
copies is three places for it to drift.
|
|
||||||
#>
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
#: The repository root, worked out from this file rather than from the working
|
|
||||||
#: directory, so the scripts can be run from anywhere.
|
|
||||||
$root = Split-Path -Parent $PSScriptRoot
|
|
||||||
|
|
||||||
#: A checkout inside WSL, run from Windows as \\wsl.localhost\<distro>\... Its
|
|
||||||
#: .venv is a Linux one, so the steps that run Python hand that part to the
|
|
||||||
#: distro. $Wsl.Groups[1] is the distro; ConvertTo-LinuxPath turns a path under
|
|
||||||
#: the checkout into the distro's own. See docs/wsl.md.
|
|
||||||
$WslPattern = '^\\\\wsl(?:\.localhost|\$)\\([^\\]+)(\\.*)$'
|
|
||||||
$Wsl = [regex]::Match($root, $WslPattern)
|
|
||||||
function ConvertTo-LinuxPath([string] $Path) {
|
|
||||||
[regex]::Match($Path, $WslPattern).Groups[2].Value -replace '\\', '/'
|
|
||||||
}
|
|
||||||
|
|
||||||
function Assert-Ok([string] $What) {
|
|
||||||
if ($LASTEXITCODE -ne 0) { throw "$What failed (exit $LASTEXITCODE)" }
|
|
||||||
}
|
|
||||||
|
|
||||||
function Write-Step([string] $Text) {
|
|
||||||
Write-Host ''
|
|
||||||
Write-Host "== $Text" -ForegroundColor Cyan
|
|
||||||
}
|
|
||||||
|
|
||||||
$envFile = Join-Path $PSScriptRoot 'release.env'
|
$envFile = Join-Path $PSScriptRoot 'release.env'
|
||||||
if (-not (Test-Path $envFile)) {
|
if (-not (Test-Path $envFile)) { throw "no $envFile - copy release.env.example and fill it in" }
|
||||||
throw "no deploy\release.env - copy deploy\release.env.example and fill it in"
|
$Settings = @{}
|
||||||
}
|
|
||||||
$settings = @{}
|
|
||||||
foreach ($line in Get-Content $envFile) {
|
foreach ($line in Get-Content $envFile) {
|
||||||
if ($line -match '^\s*(#|$)') { continue }
|
if ($line -match '^\s*(#|$)') { continue }
|
||||||
# A line with no `=` is a typo, not a setting. Say so here rather than
|
|
||||||
# letting it surface later as a null reference three functions away.
|
|
||||||
if ($line -notmatch '=') { throw "release.env: not a KEY=VALUE line: $line" }
|
|
||||||
$key, $value = $line -split '=', 2
|
$key, $value = $line -split '=', 2
|
||||||
$settings[$key.Trim()] = $value.Trim().Trim('"')
|
$Settings[$key.Trim()] = "$value".Trim().Trim('"')
|
||||||
}
|
}
|
||||||
|
foreach ($key in 'DISTRO', 'CHECKOUT', 'IMAGE') {
|
||||||
function Setting([string] $Name, [string] $Default, [switch] $Required) {
|
if (-not $Settings[$key]) { throw "set $key in $envFile" }
|
||||||
if ($settings.ContainsKey($Name) -and $settings[$Name]) { return $settings[$Name] }
|
|
||||||
if ($Required) { throw "set $Name in deploy\release.env" }
|
|
||||||
return $Default
|
|
||||||
}
|
}
|
||||||
|
$Image = $Settings.IMAGE
|
||||||
|
$Checkout = $Settings.CHECKOUT.TrimEnd('/')
|
||||||
|
$Release = "\\wsl.localhost\$($Settings.DISTRO)" + ($Checkout -replace '/[^/]+$', '' -replace '/', '\') + '\release\' + ($Checkout -replace '^.*/', '')
|
||||||
|
|
||||||
#: What temporary directories this release is allowed to delete. Everything
|
function Assert-Wslc {
|
||||||
#: below is created by these scripts and named for it.
|
if (-not (Get-Command wslc -ErrorAction SilentlyContinue)) {
|
||||||
$Name = Setting NAME -Required
|
throw 'wslc is not on PATH - run wsl --update (WSL 2.9.3 or later), see https://learn.microsoft.com/windows/wsl/tutorials/wsl-containers'
|
||||||
|
|
||||||
# Only ever deletes a <name>-* entry directly under the temp directory. A
|
|
||||||
# guard, not a convenience: these paths are built by string concatenation and
|
|
||||||
# handed to `Remove-Item -Recurse -Force`, and one empty variable in that
|
|
||||||
# sentence is a very bad afternoon.
|
|
||||||
function Remove-ReleaseTemp([string[]] $Paths) {
|
|
||||||
$tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\') + '\'
|
|
||||||
foreach ($path in $Paths) {
|
|
||||||
$resolved = [IO.Path]::GetFullPath($path)
|
|
||||||
if (-not $resolved.StartsWith($tempRoot, [StringComparison]::OrdinalIgnoreCase) -or
|
|
||||||
-not ([IO.Path]::GetFileName($resolved)).StartsWith("$Name-")) {
|
|
||||||
throw "Unsafe release temporary path: $resolved"
|
|
||||||
}
|
|
||||||
if (Test-Path -LiteralPath $resolved) { Remove-Item -LiteralPath $resolved -Recurse -Force }
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# The tag step 1 exported, from release\<folder>\production.
|
||||||
|
function Get-ReleaseSha {
|
||||||
|
$file = Join-Path $Release 'production'
|
||||||
|
if (-not (Test-Path $file)) { throw "no $file - run step 1 first" }
|
||||||
|
(Get-Content $file -TotalCount 1).Trim()
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
# shellcheck shell=bash
|
||||||
|
# Shared plumbing for the release scripts that run in WSL. Source it; do not
|
||||||
|
# run it.
|
||||||
|
#
|
||||||
|
# Sourcing runs in the caller's shell, so everything defined here ($root,
|
||||||
|
# SETTINGS, setting, require_setting, die, step, run_or_die) is the caller's
|
||||||
|
# afterwards. The Windows half (steps 2 and 3) reads the same release.env with
|
||||||
|
# _common.ps1, which parses the same KEY=VALUE lines. The caller sets
|
||||||
|
# `set -euo pipefail` first.
|
||||||
|
|
||||||
|
#: The directory of these scripts and the repository root above it, worked out
|
||||||
|
#: from this file rather than from the working directory, so the scripts can be
|
||||||
|
#: run from anywhere.
|
||||||
|
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||||
|
# shellcheck disable=SC2034 # read by the scripts that source this file
|
||||||
|
root=$(dirname "$here")
|
||||||
|
|
||||||
|
die() { printf 'error: %s\n' "$*" >&2; exit 1; }
|
||||||
|
|
||||||
|
step() { printf '\n== %s\n' "$*"; }
|
||||||
|
|
||||||
|
# Runs a command and, when it fails, stops with the step's name and its exit
|
||||||
|
# code: `run_or_die 'git archive' git archive ...`.
|
||||||
|
run_or_die() {
|
||||||
|
local what=$1 rc=0
|
||||||
|
shift
|
||||||
|
"$@" || rc=$?
|
||||||
|
((rc == 0)) || die "$what failed (exit $rc)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- release.env
|
||||||
|
|
||||||
|
# Read, not sourced: sourcing would run whatever the file says and would break
|
||||||
|
# on a space in a value. A line is blank, a comment, or KEY=VALUE; anything else
|
||||||
|
# is a typo and is reported here rather than as an empty variable later. Keys
|
||||||
|
# and values are trimmed and surrounding double quotes are dropped. A CRLF file
|
||||||
|
# (saved from a Windows editor) reads the same as an LF one.
|
||||||
|
trim() {
|
||||||
|
local s=$1
|
||||||
|
s=${s#"${s%%[![:space:]]*}"}
|
||||||
|
s=${s%"${s##*[![:space:]]}"}
|
||||||
|
printf '%s' "$s"
|
||||||
|
}
|
||||||
|
|
||||||
|
declare -A SETTINGS=()
|
||||||
|
env_file=$here/release.env
|
||||||
|
[[ -f $env_file ]] || die "no deploy/release.env - copy deploy/release.env.example and fill it in"
|
||||||
|
while IFS= read -r line || [[ -n $line ]]; do
|
||||||
|
line=${line%$'\r'}
|
||||||
|
[[ $line =~ ^[[:space:]]*(#|$) ]] && continue
|
||||||
|
[[ $line == *=* ]] || die "release.env: not a KEY=VALUE line: $line"
|
||||||
|
key=$(trim "${line%%=*}")
|
||||||
|
value=$(trim "${line#*=}")
|
||||||
|
while [[ $value == \"* ]]; do value=${value#\"}; done
|
||||||
|
while [[ $value == *\" ]]; do value=${value%\"}; done
|
||||||
|
SETTINGS[$key]=$value
|
||||||
|
done < "$env_file"
|
||||||
|
unset line key value
|
||||||
|
|
||||||
|
# setting NAME [DEFAULT]: the value, or the default when unset or empty.
|
||||||
|
setting() {
|
||||||
|
local value=${SETTINGS[$1]-}
|
||||||
|
if [[ -n $value ]]; then printf '%s' "$value"; else printf '%s' "${2-}"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# require_setting NAME: the value, or a stop naming the missing setting.
|
||||||
|
require_setting() {
|
||||||
|
local value
|
||||||
|
value=$(setting "$1")
|
||||||
|
[[ -n $value ]] || die "set $1 in deploy/release.env"
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- shared checks
|
||||||
|
|
||||||
|
need_git() {
|
||||||
|
command -v git >/dev/null 2>&1 || die "git is not on PATH"
|
||||||
|
}
|
||||||
|
|
||||||
|
# The commit a branch or ref points at, or a stop saying which one did not
|
||||||
|
# resolve.
|
||||||
|
resolve_commit() {
|
||||||
|
local commit
|
||||||
|
commit=$(git rev-parse --verify "$1^{commit}") || die "resolve $2 commit failed"
|
||||||
|
printf '%s' "$commit"
|
||||||
|
}
|
||||||
-150
@@ -1,150 +0,0 @@
|
|||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
The database, in both directions.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
|
||||||
push this machine's database -> the server
|
|
||||||
pull a fresh snapshot from the server -> .\backups
|
|
||||||
|
|
||||||
`push` is the one-off at the start: production begins knowing what you
|
|
||||||
already have, rather than starting empty. `pull` is the one you keep
|
|
||||||
running -- on a schedule from the server itself (server\backup.sh under
|
|
||||||
cron) and by hand from here whenever you want the copy somewhere the
|
|
||||||
server is not.
|
|
||||||
|
|
||||||
`push` moves the database and nothing else. It does not stop the app before
|
|
||||||
or start it after: a deployment knows what else is moving and in what order,
|
|
||||||
and this does not. Stop the app on the server first -- the import is a file
|
|
||||||
swap under SQLite's feet and is refused while the app is up. Start it again
|
|
||||||
afterwards.
|
|
||||||
|
|
||||||
Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so
|
|
||||||
the .db file on disk is always missing whatever is still in the -wal beside
|
|
||||||
it, and copying the three files with the app running is a race you lose
|
|
||||||
quietly -- a database that opens fine and is short a day's writes.
|
|
||||||
|
|
||||||
After a push the two copies diverge from the first thing written on either.
|
|
||||||
There is no merge for that and this does not pretend there is: pick the one
|
|
||||||
that is authoritative, and use the other read-only or not at all.
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
.\deploy\db.ps1 push
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
.\deploy\db.ps1 pull
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[ValidateSet('push', 'pull')]
|
|
||||||
[string] $Action,
|
|
||||||
|
|
||||||
# Skip the confirmation before replacing the server's database.
|
|
||||||
[switch] $Yes
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
. "$PSScriptRoot/_common.ps1"
|
|
||||||
Set-Location $root
|
|
||||||
|
|
||||||
$SshHost = Setting SSH_HOST -Required
|
|
||||||
$RemoteDir = Setting REMOTE_DIR -Required
|
|
||||||
# Where the data actually is. Beside the compose file unless APP_DATA put it
|
|
||||||
# under a shared root; then REMOTE_DATA has to say so.
|
|
||||||
$RemoteData = Setting REMOTE_DATA "$RemoteDir/data"
|
|
||||||
$Python = Setting PYTHON (Join-Path $root '.venv\Scripts\python.exe')
|
|
||||||
$LocalDb = Setting LOCAL_DB (Join-Path $root "$Name.db")
|
|
||||||
|
|
||||||
switch ($Action) {
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- push
|
|
||||||
|
|
||||||
'push' {
|
|
||||||
if (-not (Test-Path $LocalDb)) { throw "no $LocalDb" }
|
|
||||||
|
|
||||||
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
|
||||||
$snap = Join-Path $root "$Name-$stamp.db.gz"
|
|
||||||
Write-Host "snapshotting $LocalDb"
|
|
||||||
|
|
||||||
# One Python call for both halves. `VACUUM INTO` is the only way to get a
|
|
||||||
# consistent single file out of a WAL database, and Python is already the
|
|
||||||
# thing that has to be here -- PowerShell has no gzip that writes the format
|
|
||||||
# the server's `gunzip` expects.
|
|
||||||
$snapshot = @'
|
|
||||||
import gzip, pathlib, shutil, sqlite3, sys, tempfile
|
|
||||||
source, target = sys.argv[1], pathlib.Path(sys.argv[2])
|
|
||||||
with tempfile.TemporaryDirectory() as work:
|
|
||||||
plain = pathlib.Path(work) / "snapshot.db"
|
|
||||||
db = sqlite3.connect(source)
|
|
||||||
try:
|
|
||||||
db.execute("VACUUM INTO ?", (str(plain),))
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
|
|
||||||
shutil.copyfileobj(raw, out)
|
|
||||||
'@
|
|
||||||
# A checkout inside WSL has a Linux .venv, and SQLite cannot lock a WAL
|
|
||||||
# database through the \\wsl.localhost share, so the snapshot is taken in
|
|
||||||
# the distro. The script goes in on stdin, past both shells.
|
|
||||||
if ($Wsl.Success) {
|
|
||||||
$snapshot | wsl.exe -d $Wsl.Groups[1].Value --cd (ConvertTo-LinuxPath $root) --exec .venv/bin/python - (ConvertTo-LinuxPath $LocalDb) (ConvertTo-LinuxPath $snap)
|
|
||||||
} else {
|
|
||||||
& $Python -c $snapshot $LocalDb $snap
|
|
||||||
}
|
|
||||||
Assert-Ok 'snapshot'
|
|
||||||
' {0:N1} MB' -f ((Get-Item $snap).Length / 1MB) | Write-Host
|
|
||||||
|
|
||||||
try {
|
|
||||||
Write-Host ''
|
|
||||||
Write-Host "This replaces the database on $SshHost with this one." -ForegroundColor Yellow
|
|
||||||
Write-Host 'Anything written there since is gone.' -ForegroundColor Yellow
|
|
||||||
Write-Host 'The app must already be stopped there, and stays stopped after.' -ForegroundColor Yellow
|
|
||||||
if (-not $Yes) {
|
|
||||||
$reply = Read-Host 'go ahead? [y/N]'
|
|
||||||
if ($reply -notmatch '^[yY]') { Write-Host 'stopped.'; return }
|
|
||||||
}
|
|
||||||
|
|
||||||
scp -C $snap "${SshHost}:$RemoteData/incoming.db.gz"; Assert-Ok 'scp database'
|
|
||||||
scp (Join-Path $PSScriptRoot 'server\import_database.py') "${SshHost}:$RemoteDir/import_database.py"
|
|
||||||
Assert-Ok 'scp importer'
|
|
||||||
} finally {
|
|
||||||
Remove-Item -Force $snap -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
|
|
||||||
# Base64 rather than stdin: PowerShell has no `<` for a native command, and
|
|
||||||
# piping the text would rewrite the script's line endings on the way.
|
|
||||||
$b64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes((Join-Path $PSScriptRoot 'server\import.sh')))
|
|
||||||
ssh $SshHost "echo $b64 | base64 -d | bash -s -- '$RemoteDir'"; Assert-Ok 'import'
|
|
||||||
Write-Host "pushed. The app is still stopped; start it when the rest of"
|
|
||||||
Write-Host 'the deployment is in place:'
|
|
||||||
Write-Host " ssh $SshHost `"cd '$RemoteDir' && docker compose up -d --wait $Name`""
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- pull
|
|
||||||
|
|
||||||
'pull' {
|
|
||||||
$into = Join-Path $root 'backups'
|
|
||||||
New-Item -ItemType Directory -Force -Path $into | Out-Null
|
|
||||||
|
|
||||||
# The server's own copy of backup.sh, not a streamed one. It is the same
|
|
||||||
# script cron runs, it is already there, and it locates its own directory
|
|
||||||
# through BASH_SOURCE -- which is exactly the thing a script read from stdin
|
|
||||||
# does not have. import.sh takes its directory as an argument and so can be
|
|
||||||
# streamed; this one cannot.
|
|
||||||
Write-Host "snapshotting on $SshHost"
|
|
||||||
ssh $SshHost "cd '$RemoteDir' && bash backup.sh"; Assert-Ok 'backup'
|
|
||||||
|
|
||||||
$newest = (ssh $SshHost "ls -1t '$RemoteData'/backups/$Name-*.db.gz | head -1").Trim()
|
|
||||||
Assert-Ok 'listing backups'
|
|
||||||
if (-not $newest) { throw 'no backup appeared on the server' }
|
|
||||||
|
|
||||||
scp -C "${SshHost}:$newest" $into; Assert-Ok 'scp'
|
|
||||||
$file = Split-Path -Leaf $newest
|
|
||||||
Write-Host "backups\$file"
|
|
||||||
Write-Host ''
|
|
||||||
Write-Host 'To read it here, gunzip it, or in Python:'
|
|
||||||
Write-Host " import gzip, shutil; shutil.copyfileobj(gzip.open(r'backups\$file'), open('read-only.db','wb'))"
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
+130
@@ -0,0 +1,130 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# The database, in both directions.
|
||||||
|
#
|
||||||
|
# bash deploy/db.sh push [--yes] this machine's database -> the server
|
||||||
|
# bash deploy/db.sh pull a fresh snapshot from the server -> backups/
|
||||||
|
#
|
||||||
|
# `push` is the one-off at the start: production begins knowing what you already
|
||||||
|
# have, rather than starting empty. `pull` is the one you keep running, on a
|
||||||
|
# schedule from the server itself (server/backup.sh under cron) and by hand from
|
||||||
|
# here whenever you want the copy somewhere the server is not.
|
||||||
|
#
|
||||||
|
# `push` moves the database and nothing else. It does not stop the app before or
|
||||||
|
# start it after: a deployment knows what else is moving and in what order, and
|
||||||
|
# this does not. Stop the app on the server first; the import is a file swap
|
||||||
|
# under SQLite's feet and is refused while the app is up. Start it again
|
||||||
|
# afterwards.
|
||||||
|
#
|
||||||
|
# Both go through SQLite's own `VACUUM INTO`. The database is in WAL mode, so the
|
||||||
|
# .db file on disk is always missing whatever is still in the -wal beside it, and
|
||||||
|
# copying the three files with the app running is a race you lose quietly: a
|
||||||
|
# database that opens fine and is short a day's writes.
|
||||||
|
#
|
||||||
|
# After a push the two copies diverge from the first thing written on either.
|
||||||
|
# There is no merge for that: pick the one that is authoritative, and use the
|
||||||
|
# other read-only or not at all.
|
||||||
|
#
|
||||||
|
# Settings from release.env: SSH_HOST, REMOTE_DIR and NAME (the compose service)
|
||||||
|
# are required; REMOTE_DATA (default REMOTE_DIR/data), PYTHON and LOCAL_DB
|
||||||
|
# (default the database the app's config names) are not.
|
||||||
|
set -euo pipefail
|
||||||
|
# shellcheck source=_common.sh
|
||||||
|
. "$(dirname "${BASH_SOURCE[0]}")/_common.sh"
|
||||||
|
|
||||||
|
action='' yes=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
push|pull) action=$arg ;;
|
||||||
|
--yes) yes=1 ;;
|
||||||
|
-h|--help) sed -n '2,29p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
|
*) die "unknown option $arg" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[[ -n $action ]] || die "say push or pull: bash deploy/db.sh push|pull"
|
||||||
|
|
||||||
|
ssh_host=$(require_setting SSH_HOST)
|
||||||
|
remote_dir=$(require_setting REMOTE_DIR)
|
||||||
|
# Where the data actually is. Beside the compose file unless APP_DATA put it
|
||||||
|
# under a shared root; then REMOTE_DATA has to say so.
|
||||||
|
remote_data=$(setting REMOTE_DATA "$remote_dir/data")
|
||||||
|
name=$(require_setting NAME)
|
||||||
|
python=$(setting PYTHON "$root/.venv/bin/python")
|
||||||
|
cd "$root"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------- push
|
||||||
|
|
||||||
|
if [[ $action == push ]]; then
|
||||||
|
[[ -x $python ]] || die "no interpreter at $python - run uv sync --all-extras, or set PYTHON in deploy/release.env"
|
||||||
|
# The file the app writes: `database` in config.yaml (or $APP_CONFIG),
|
||||||
|
# roledesk.db without one, asked of the app from the checkout's root.
|
||||||
|
local_db=$(setting LOCAL_DB)
|
||||||
|
if [[ -z $local_db ]]; then
|
||||||
|
local_db=$("$python" -c 'from app.config import load; print(load().database.absolute())') \
|
||||||
|
|| die "could not read the database path from the app's config; set LOCAL_DB in deploy/release.env"
|
||||||
|
fi
|
||||||
|
[[ -f $local_db ]] || die "no $local_db"
|
||||||
|
|
||||||
|
snap=$root/$name-$(date +%Y%m%d-%H%M%S).db.gz
|
||||||
|
trap 'rm -f -- "$snap"' EXIT
|
||||||
|
echo "snapshotting $local_db"
|
||||||
|
|
||||||
|
# One Python call for both halves: `VACUUM INTO` is the only way to get a
|
||||||
|
# consistent single file out of a WAL database, and it writes the gzip the
|
||||||
|
# server's `gunzip` expects.
|
||||||
|
run_or_die snapshot "$python" -c '
|
||||||
|
import gzip, pathlib, shutil, sqlite3, sys, tempfile
|
||||||
|
source, target = sys.argv[1], pathlib.Path(sys.argv[2])
|
||||||
|
with tempfile.TemporaryDirectory() as work:
|
||||||
|
plain = pathlib.Path(work) / "snapshot.db"
|
||||||
|
db = sqlite3.connect(source)
|
||||||
|
try:
|
||||||
|
db.execute("VACUUM INTO ?", (str(plain),))
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
with plain.open("rb") as raw, gzip.open(target, "wb") as out:
|
||||||
|
shutil.copyfileobj(raw, out)
|
||||||
|
' "$local_db" "$snap"
|
||||||
|
printf ' %s\n' "$(du -h "$snap" | cut -f1)"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "This replaces the database on $ssh_host with this one."
|
||||||
|
echo 'Anything written there since is gone.'
|
||||||
|
echo 'The app must already be stopped there, and stays stopped after.'
|
||||||
|
if ((! yes)); then
|
||||||
|
read -r -p 'go ahead? [y/N] ' reply || reply=''
|
||||||
|
[[ $reply =~ ^[yY] ]] || { echo 'stopped.'; exit 0; }
|
||||||
|
fi
|
||||||
|
|
||||||
|
run_or_die 'scp database' scp -C "$snap" "$ssh_host:$remote_data/incoming.db.gz"
|
||||||
|
run_or_die 'scp importer' scp "$here/server/import_database.py" "$ssh_host:$remote_dir/import_database.py"
|
||||||
|
rm -f -- "$snap"
|
||||||
|
|
||||||
|
# import.sh takes its directory as an argument, so it can be streamed.
|
||||||
|
run_or_die import ssh "$ssh_host" "bash -s -- '$remote_dir'" < "$here/server/import.sh"
|
||||||
|
echo 'pushed. The app is still stopped; start it when the rest of'
|
||||||
|
echo 'the deployment is in place:'
|
||||||
|
echo " ssh $ssh_host \"cd '$remote_dir' && docker compose up -d --wait $name\""
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------- pull
|
||||||
|
|
||||||
|
into=$root/backups
|
||||||
|
mkdir -p -- "$into"
|
||||||
|
|
||||||
|
# The server's own copy of backup.sh, not a streamed one: it is the same script
|
||||||
|
# cron runs, and it locates its own directory through BASH_SOURCE, which a
|
||||||
|
# script read from stdin does not have.
|
||||||
|
echo "snapshotting on $ssh_host"
|
||||||
|
run_or_die backup ssh "$ssh_host" "cd '$remote_dir' && bash backup.sh"
|
||||||
|
|
||||||
|
# shellcheck disable=SC2029 # the remote command is meant to expand here
|
||||||
|
newest=$(ssh "$ssh_host" "ls -1t '$remote_data'/backups/$name-*.db.gz | head -1") || die 'listing backups failed'
|
||||||
|
[[ -n $newest ]] || die 'no backup appeared on the server'
|
||||||
|
|
||||||
|
run_or_die scp scp -C "$ssh_host:$newest" "$into/"
|
||||||
|
file=$(basename "$newest")
|
||||||
|
echo "backups/$file"
|
||||||
|
echo
|
||||||
|
echo 'To read it here, gunzip it, or in Python:'
|
||||||
|
echo " import gzip, shutil; shutil.copyfileobj(gzip.open('backups/$file'), open('read-only.db','wb'))"
|
||||||
+26
-13
@@ -1,28 +1,41 @@
|
|||||||
# Copy to release.env beside these scripts and fill it in. Never committed:
|
# Copy to release.env. One file, read by both halves of the release: the bash
|
||||||
# it names your registry and, for db.ps1, your server.
|
# scripts (_common.sh), which run in the WSL distro that holds the checkout, and
|
||||||
|
# the PowerShell ones (_common.ps1, wsl-setup.ps1), which run on Windows. Both
|
||||||
|
# read KEY=VALUE lines and neither runs the file as shell, so $HOME and ~ are
|
||||||
|
# not expanded. Local main is the source; fetching is your choice. Never
|
||||||
|
# committed: it names your registry and, for db.sh, your server.
|
||||||
#
|
#
|
||||||
# Local main is the source. Fetching it first is your choice, not the script's.
|
# Where the checkout is: the distro's name and the path inside it, as
|
||||||
|
# docs/Install WSL.md lays them out, with the checkout folder's name last.
|
||||||
|
# wsl-setup.ps1 writes them into the profile commands, and steps 2 and 3 find
|
||||||
|
# step 1's export at \\wsl.localhost\<DISTRO>\<CHECKOUT's parent>\release\<folder>.
|
||||||
|
DISTRO=claudebox
|
||||||
|
CHECKOUT=/home/claude/cboxcode/jobboard
|
||||||
|
|
||||||
# The service's name. Used for the image tag, for the temporary directories
|
# The compose service, SERVICE in deploy/server/*.sh. Used by db.sh, on the
|
||||||
# these scripts create, and for the database file db.ps1 moves. One word, no
|
# Linux side, for the snapshot names (<NAME>-<stamp>.db.gz, as backup.sh writes
|
||||||
# spaces — it ends up in paths.
|
# them) and the start command it prints. One word, no spaces.
|
||||||
NAME=app
|
NAME=app
|
||||||
|
|
||||||
|
# Windows side, used by steps 2 and 3. wslc builds for the machine's own
|
||||||
|
# architecture (linux/amd64 here) and has no --platform.
|
||||||
IMAGE=git.rittershofer.org/wolf/app
|
IMAGE=git.rittershofer.org/wolf/app
|
||||||
PLATFORM=linux/amd64
|
|
||||||
|
# Linux side, used by step 1: which branch is tested, and which one it promotes.
|
||||||
BRANCH=production
|
BRANCH=production
|
||||||
SOURCE=main
|
SOURCE=main
|
||||||
|
|
||||||
# Defaults to the checkout's .venv/Scripts/python.exe. Only needed for the
|
# Linux side, used by step 1 and by db.sh's snapshot, in the distro. Defaults to
|
||||||
# tests in step 1 and for db.ps1's snapshot.
|
# the checkout's .venv/bin/python (a WSL path, /home/<user>/...).
|
||||||
# PYTHON=C:/Users/Coolkau/code/app/.venv/Scripts/python.exe
|
# PYTHON=/home/<user>/cboxcode/jobboard/.venv/bin/python
|
||||||
|
|
||||||
# db.ps1 uses SSH. The numbered release scripts never do. REMOTE_DIR is the
|
# db.sh uses SSH. The numbered release scripts never do. REMOTE_DIR is the
|
||||||
# directory holding the compose file; REMOTE_DATA is where APP_DATA points,
|
# directory holding the compose file; REMOTE_DATA is where APP_DATA points,
|
||||||
# and defaults to REMOTE_DIR/data when left blank.
|
# and defaults to REMOTE_DIR/data when left blank.
|
||||||
SSH_HOST=
|
SSH_HOST=
|
||||||
REMOTE_DIR=/srv/swag
|
REMOTE_DIR=/srv/swag
|
||||||
REMOTE_DATA=/srv/data/app
|
REMOTE_DATA=/srv/data/app
|
||||||
|
|
||||||
# The database this machine pushes, if it is not <root>/<NAME>.db.
|
# The database this machine pushes, if it is not the one the app's config names
|
||||||
# LOCAL_DB=C:/Users/Coolkau/code/app/app.db
|
# (`database` in config.yaml, roledesk.db in the checkout without one).
|
||||||
|
# LOCAL_DB=/home/<user>/cboxcode/jobboard/roledesk.db
|
||||||
+43
-10
@@ -13,9 +13,26 @@
|
|||||||
# self-contained and compacted file, and never blocks a writer for long enough
|
# self-contained and compacted file, and never blocks a writer for long enough
|
||||||
# to matter on a database of this size.
|
# to matter on a database of this size.
|
||||||
#
|
#
|
||||||
# Rename `app` and `APP_DATA` to your service's name.
|
# The database is the one the app's own config names, asked of the app inside
|
||||||
|
# the container, and it is opened read-only, so a path that names no file is
|
||||||
|
# an error rather than a new empty database and a snapshot of nothing.
|
||||||
|
#
|
||||||
|
# SERVICE is the compose service (deploy.sh says why it is here); APP_DATA is
|
||||||
|
# renamed with the other APP_ variables, and `from app.config` with the package.
|
||||||
|
|
||||||
|
SERVICE=app
|
||||||
|
# Tables every database this app writes has: the same set as REQUIRED_TABLES in
|
||||||
|
# import_database.py, which a test holds this line to. A snapshot without one
|
||||||
|
# of them is deleted and the run fails.
|
||||||
|
# Ours: `roles` is app/storage/schema.py's first table, where the boilerplate has `ticks`.
|
||||||
|
TABLES="jobs roles"
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
# A snapshot is the whole database, so it is owner only whatever the caller's
|
||||||
|
# umask is. This covers the folder made below; the umask inside the container
|
||||||
|
# (set in the snippet) covers the .db, and gzip gives the .gz the mode of the
|
||||||
|
# file it compresses.
|
||||||
|
umask 077
|
||||||
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
||||||
|
|
||||||
# Same source of truth as the compose file and deploy.sh.
|
# Same source of truth as the compose file and deploy.sh.
|
||||||
@@ -27,18 +44,34 @@ stamp=$(date +%Y%m%d-%H%M%S)
|
|||||||
|
|
||||||
mkdir -p "$DATA/backups"
|
mkdir -p "$DATA/backups"
|
||||||
|
|
||||||
docker compose exec -T app python -c "
|
# shellcheck disable=SC2086 # TABLES is a list of words on purpose
|
||||||
import sqlite3
|
docker compose exec -T "$SERVICE" python - "$SERVICE-$stamp" $TABLES <<'PY'
|
||||||
db = sqlite3.connect('/data/app.db')
|
import os, sqlite3, sys
|
||||||
db.execute('VACUUM INTO ?', ('/data/backups/app-$stamp.db',))
|
from contextlib import closing
|
||||||
db.close()
|
from pathlib import Path
|
||||||
"
|
|
||||||
gzip -f "$DATA/backups/app-$stamp.db"
|
from app.config import load
|
||||||
echo "$DATA/backups/app-$stamp.db.gz $(du -h "$DATA/backups/app-$stamp.db.gz" | cut -f1)"
|
|
||||||
|
os.umask(0o077)
|
||||||
|
name, required = sys.argv[1], set(sys.argv[2:])
|
||||||
|
database = load().database.absolute()
|
||||||
|
snapshot = Path("/data/backups") / f"{name}.db"
|
||||||
|
with closing(sqlite3.connect(f"{database.as_uri()}?mode=ro", uri=True)) as db:
|
||||||
|
db.execute("VACUUM INTO ?", (str(snapshot),))
|
||||||
|
with closing(sqlite3.connect(f"{snapshot.as_uri()}?mode=ro", uri=True)) as db:
|
||||||
|
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||||
|
missing = sorted(required - tables)
|
||||||
|
if missing:
|
||||||
|
snapshot.unlink()
|
||||||
|
sys.exit(f"{database} has no table {', '.join(missing)}: not this app's database")
|
||||||
|
print(f"snapshot of {database}")
|
||||||
|
PY
|
||||||
|
gzip -f "$DATA/backups/$SERVICE-$stamp.db"
|
||||||
|
echo "$DATA/backups/$SERVICE-$stamp.db.gz $(du -h "$DATA/backups/$SERVICE-$stamp.db.gz" | cut -f1)"
|
||||||
|
|
||||||
# Oldest first, everything past KEEP goes. Named by timestamp, so sorting the
|
# Oldest first, everything past KEEP goes. Named by timestamp, so sorting the
|
||||||
# names sorts them by age.
|
# names sorts them by age.
|
||||||
ls -1 "$DATA"/backups/app-*.db.gz 2>/dev/null \
|
ls -1 "$DATA"/backups/"$SERVICE"-*.db.gz 2>/dev/null \
|
||||||
| sort \
|
| sort \
|
||||||
| head -n -"$KEEP" \
|
| head -n -"$KEEP" \
|
||||||
| while read -r old; do rm -f "$old"; echo "removed $old"; done
|
| while read -r old; do rm -f "$old"; echo "removed $old"; done
|
||||||
@@ -9,7 +9,10 @@
|
|||||||
# `env.example` to the `.env` next to it, and it is `docker compose up -d app`
|
# `env.example` to the `.env` next to it, and it is `docker compose up -d app`
|
||||||
# from then on — which recreates this container and no other.
|
# from then on — which recreates this container and no other.
|
||||||
#
|
#
|
||||||
# Rename `app` and `APP_` throughout to your service's name.
|
# Rename `app` and `APP_` throughout to your service's name. The service and
|
||||||
|
# `container_name` are the name `SERVICE=` in deploy.sh, backup.sh and
|
||||||
|
# import.sh, `$upstream_app` in the proxy conf and `NAME=` in release.env
|
||||||
|
# repeat; tests/test_deploy.py reads it from this file and holds them to it.
|
||||||
|
|
||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
@@ -33,12 +36,12 @@ services:
|
|||||||
# Which model answers, decided here rather than in the image. With
|
# Which model answers, decided here rather than in the image. With
|
||||||
# `backend: auto` the API key wins when there is one and the
|
# `backend: auto` the API key wins when there is one and the
|
||||||
# subscription CLI is the fallback, so filling in ANTHROPIC_API_KEY
|
# subscription CLI is the fallback, so filling in ANTHROPIC_API_KEY
|
||||||
# later switches it over with no rebuild. See docs/llm-backends.md.
|
# later switches it over with no rebuild. See llm/README.md.
|
||||||
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
|
CLAUDE_CODE_OAUTH_TOKEN: ${CLAUDE_CODE_OAUTH_TOKEN:-}
|
||||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||||
# Normally empty. Set it to pin the password — the way back in when a
|
# Normally empty. Set it to pin the password — the way back in when a
|
||||||
# login is lost — then unset it and recreate to hand control of the
|
# login is lost — then unset it and recreate to hand control of the
|
||||||
# password back to the app. See docs/password-protection.md.
|
# password back to the app. See auth/README.md.
|
||||||
APP_PASSWORD: ${APP_PASSWORD:-}
|
APP_PASSWORD: ${APP_PASSWORD:-}
|
||||||
|
|
||||||
# No `ports:`. Nothing on the host reaches it; SWAG does, by container
|
# No `ports:`. Nothing on the host reaches it; SWAG does, by container
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# The server's config.yaml. Copy it to $APP_DATA/config.yaml, which the
|
||||||
|
# container reads as /data/config.yaml (APP_CONFIG in the Dockerfile), and set
|
||||||
|
# the hostname. deploy.sh refuses a config without `database` or `auth.enabled`.
|
||||||
|
# Every key left out keeps its default; ../../config.example.yaml lists them all.
|
||||||
|
# Ours: the keys are roledesk's (app/config.py); an unknown one is an error.
|
||||||
|
|
||||||
|
server:
|
||||||
|
host: 0.0.0.0 # the container's interface; SWAG is in front of it
|
||||||
|
port: 8787 # the port the proxy conf and the HEALTHCHECK use
|
||||||
|
|
||||||
|
database: /data/roledesk.db # in the volume; /app, the working directory, is not writable
|
||||||
|
|
||||||
|
auth:
|
||||||
|
enabled: always # the lock must not depend on how the process was started
|
||||||
|
secure_cookie: always # auto needs uvicorn to trust X-Forwarded-Proto, which only `app serve` off loopback sets
|
||||||
|
rp_id: app.example.com # the hostname passkeys are bound to; changing it voids every passkey
|
||||||
|
origin: https://app.example.com # what the browser's address bar shows, scheme included
|
||||||
|
|
||||||
|
# Who answers when a stage needs a model (extract, summarise, judge). `auto`
|
||||||
|
# takes ANTHROPIC_API_KEY when the compose file passes one and otherwise the
|
||||||
|
# subscription CLI with CLAUDE_CODE_OAUTH_TOKEN, so switching is an .env edit.
|
||||||
|
# The stages themselves are switched under Settings, Sources, Runs.
|
||||||
|
agent:
|
||||||
|
backend: auto
|
||||||
|
|
||||||
|
# The vectors behind prompt and CV closeness. The model is not in the image:
|
||||||
|
# the first `embed` job downloads it (~460 MB) into HOME, the /data volume, so
|
||||||
|
# it happens once per server and not once per release. Changing it re-embeds.
|
||||||
|
embed:
|
||||||
|
model: intfloat/multilingual-e5-small
|
||||||
+27
-10
@@ -1,9 +1,14 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Run on the server, next to the compose file, or with its directory as $1.
|
# Run on the server, next to the compose file, or with its directory as $1.
|
||||||
# No remote shell required: the release scripts on Windows publish an image
|
# No remote shell required: the release scripts publish an image
|
||||||
# and stop there, and this is the other half.
|
# and stop there, and this is the other half.
|
||||||
#
|
#
|
||||||
# Rename `app` and `APP_DATA` to your service's name.
|
# SERVICE is the compose service, the one name a project renames in these
|
||||||
|
# scripts (README, "Starting a project from it"); the tests read it from
|
||||||
|
# compose.yaml and hold every script to it. APP_DATA is renamed with the other
|
||||||
|
# APP_ variables.
|
||||||
|
SERVICE=app
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
||||||
|
|
||||||
@@ -15,24 +20,36 @@ cd "${1:-$(dirname "$(readlink -f "$0")")}"
|
|||||||
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
|
DATA=$(sed -n 's/^[[:space:]]*APP_DATA=\([^#]*\).*/\1/p' .env | tail -1 | xargs || true)
|
||||||
DATA=${DATA:-./data}
|
DATA=${DATA:-./data}
|
||||||
|
|
||||||
[[ -s "$DATA/config.yaml" ]] || {
|
config=$DATA/config.yaml
|
||||||
echo "Install $DATA/config.yaml before deploying." >&2
|
[[ -s $config ]] || {
|
||||||
|
echo "Install $config before deploying, from config.example.yaml beside this script." >&2
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
# The two keys whose defaults are wrong in a container: `auth.enabled: auto`
|
||||||
|
# ties the lock to how the process was started, and the default
|
||||||
|
# database is wherever the config file happens to be. Read with grep and awk,
|
||||||
|
# because the host need not have a Python with a YAML parser.
|
||||||
|
if ! grep -Eq '^database:[[:space:]]*[^[:space:]#]' "$config" ||
|
||||||
|
! awk '/^auth:/ { a = 1; next } /^[^[:space:]#]/ { a = 0 }
|
||||||
|
a && /^[[:space:]]+enabled:[[:space:]]*[^[:space:]#]/ { found = 1 }
|
||||||
|
END { exit !found }' "$config"; then
|
||||||
|
echo "$config must set database and auth.enabled; config.example.yaml beside this script has both." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
# Pull explicitly, here and nowhere else: the compose file has no pull policy,
|
# Pull explicitly, here and nowhere else: the compose file has no pull policy,
|
||||||
# so starting or importing never fetches an image on its own. A registry
|
# so starting or importing never fetches an image on its own. A registry
|
||||||
# failure must leave the running service alone, not deploy a cache.
|
# failure must leave the running service alone, not deploy a cache.
|
||||||
echo 'Pulling the configured image…'
|
echo 'Pulling the configured image…'
|
||||||
if ! docker compose pull app; then
|
if ! docker compose pull "$SERVICE"; then
|
||||||
echo 'Image pull failed; app was not restarted.' >&2
|
echo "Image pull failed; $SERVICE was not restarted." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
# Use the image just pulled. Do not start or recreate other services in the
|
# Use the image just pulled. Do not start or recreate other services in the
|
||||||
# file. `--wait` holds until the container's own HEALTHCHECK passes.
|
# file. `--wait` holds until the container's own HEALTHCHECK passes.
|
||||||
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 app; then
|
if ! docker compose up -d --no-deps --pull never --wait --wait-timeout 240 "$SERVICE"; then
|
||||||
docker compose ps app >&2 || true
|
docker compose ps "$SERVICE" >&2 || true
|
||||||
echo 'Deployment failed. Check docker compose logs app.' >&2
|
echo "Deployment failed. Check docker compose logs $SERVICE." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
docker compose ps app
|
docker compose ps "$SERVICE"
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
#
|
#
|
||||||
# chmod 600. Rename APP_ to your service's name throughout.
|
# chmod 600. Rename APP_ to your service's name throughout.
|
||||||
|
|
||||||
APP_IMAGE=git.rittershofer.org/wolf/app
|
APP_IMAGE=registry.example/you/app
|
||||||
APP_TAG=latest # follows published images; pin a commit tag for rollback
|
APP_TAG=latest # follows published images; pin a commit tag for rollback
|
||||||
|
|
||||||
# Where the app keeps its database, its config and its backups. Relative to
|
# Where the app keeps its database, its config and its backups. Relative to
|
||||||
@@ -20,7 +20,7 @@ PGID=1000
|
|||||||
TZ=Europe/Berlin
|
TZ=Europe/Berlin
|
||||||
|
|
||||||
# Which model answers. Set one or the other, not both — see
|
# Which model answers. Set one or the other, not both — see
|
||||||
# docs/llm-backends.md for what each one means and what it costs.
|
# llm/README.md for what each one means and what it costs.
|
||||||
#
|
#
|
||||||
# CLAUDE_CODE_OAUTH_TOKEN a year-long token for the Claude subscription, from
|
# CLAUDE_CODE_OAUTH_TOKEN a year-long token for the Claude subscription, from
|
||||||
# `claude setup-token` on a machine you are signed in
|
# `claude setup-token` on a machine you are signed in
|
||||||
@@ -34,7 +34,7 @@ ANTHROPIC_API_KEY=
|
|||||||
# `docker compose logs app`; after that it is the app's Account screen's to
|
# `docker compose logs app`; after that it is the app's Account screen's to
|
||||||
# change. Fill this in only to force one — it is the way back in — then empty
|
# change. Fill this in only to force one — it is the way back in — then empty
|
||||||
# it again and recreate, or the Account screen refuses to change what the
|
# it again and recreate, or the Account screen refuses to change what the
|
||||||
# environment pins. See docs/password-protection.md.
|
# environment pins. See auth/README.md.
|
||||||
APP_PASSWORD=
|
APP_PASSWORD=
|
||||||
|
|
||||||
# Unlimited until the box has been watched through the heaviest thing the app
|
# Unlimited until the box has been watched through the heaviest thing the app
|
||||||
|
|||||||
@@ -1,18 +1,21 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Called on the server after uploading incoming.db.gz and import_database.py.
|
# Called on the server after uploading incoming.db.gz and import_database.py.
|
||||||
# `db.ps1 push` does both and then runs this; by hand it is the same two files
|
# `db.sh push` does both and then runs this; by hand it is the same two files
|
||||||
# in the same two places.
|
# in the same two places.
|
||||||
#
|
#
|
||||||
# This replaces the database and does nothing else. It does not stop the app
|
# This replaces the database and does nothing else. It does not stop the app
|
||||||
# and it does not start it: whoever is running a deployment knows what else is
|
# and it does not start it: whoever is running a deployment knows what else is
|
||||||
# moving and in what order, and a tool that helpfully restarts things underneath
|
# moving and in what order, and a tool that helpfully restarts things underneath
|
||||||
# that is a tool fighting its caller. By hand it is
|
# that is a tool fighting its caller. By hand it is
|
||||||
# `docker compose stop app` and `docker compose up -d --wait app` around this.
|
# `docker compose stop app` and `docker compose up -d --wait app` around this,
|
||||||
|
# with SERVICE (deploy.sh says why it is here) in place of app.
|
||||||
#
|
#
|
||||||
# What it will not do is import into a running app. The replacement is a file
|
# What it will not do is import into a running app. The replacement is a file
|
||||||
# swap under SQLite's feet, and the live database is checkpointed first, which
|
# swap under SQLite's feet, and the live database is checkpointed first, which
|
||||||
# needs every writer gone — `import_database.py` refuses a busy database, and
|
# needs every writer gone — `import_database.py` refuses a busy database, and
|
||||||
# this refuses an app that is up, because the clearer error is the earlier one.
|
# this refuses an app that is up, because the clearer error is the earlier one.
|
||||||
|
SERVICE=app
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cd "${1:?server directory required}"
|
cd "${1:?server directory required}"
|
||||||
|
|
||||||
@@ -22,14 +25,14 @@ DATA=${DATA:-./data}
|
|||||||
|
|
||||||
[[ -s "$DATA/incoming.db.gz" && -s import_database.py ]]
|
[[ -s "$DATA/incoming.db.gz" && -s import_database.py ]]
|
||||||
|
|
||||||
if [[ -n "$(docker compose ps -q --status running app 2>/dev/null)" ]]; then
|
if [[ -n "$(docker compose ps -q --status running "$SERVICE" 2>/dev/null)" ]]; then
|
||||||
echo 'app is running. Stop it before importing:' >&2
|
echo "$SERVICE is running. Stop it before importing:" >&2
|
||||||
echo ' docker compose stop app' >&2
|
echo " docker compose stop $SERVICE" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# The one-off container uses compose's uid and its mounted /data. `--pull
|
# The one-off container uses compose's uid and its mounted /data. `--pull
|
||||||
# never`: an import runs on the image already here, whatever the installed
|
# never`: an import runs on the image already here, whatever the installed
|
||||||
# compose file says about pulling. Getting a new image is deploy.sh's job.
|
# compose file says about pulling. Getting a new image is deploy.sh's job.
|
||||||
docker compose run --rm --no-deps --pull never -T --entrypoint python app - < import_database.py
|
docker compose run --rm --no-deps --pull never -T --entrypoint python "$SERVICE" - < import_database.py
|
||||||
rm -f "$DATA/incoming.db.gz"
|
rm -f "$DATA/incoming.db.gz"
|
||||||
@@ -1,11 +1,13 @@
|
|||||||
"""Offline database replacement, run as the compose service's uid.
|
"""Offline database replacement, run as the compose service's uid.
|
||||||
|
|
||||||
The app container must be stopped. Only stdlib is used so this can be sent
|
The app container must be stopped. Sent over stdin to the image on the
|
||||||
over stdin to either the old or the new image during a deployment.
|
server, whose working directory is the app's: the database is the one the
|
||||||
|
app's config names (`from app.config` follows the package when it is renamed),
|
||||||
|
and the rest is stdlib, so the old image and the new one both run it.
|
||||||
|
|
||||||
Two things to edit for your service: `DATABASE` and `REQUIRED_TABLES`. The
|
`REQUIRED_TABLES` is the table check, which stops a right-shaped-but-wrong
|
||||||
table check is what stops a right-shaped-but-wrong database — last project's,
|
database (last project's, or a half-downloaded one) from landing on top of a
|
||||||
or a half-downloaded one — from landing on top of a good one.
|
good one. `tests/test_deploy.py` fails when the app stops creating one of them.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -19,13 +21,14 @@ from contextlib import closing
|
|||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
DATABASE = Path("/data/app.db")
|
|
||||||
INCOMING = Path("/data/incoming.db.gz")
|
INCOMING = Path("/data/incoming.db.gz")
|
||||||
BACKUPS = Path("/data/backups")
|
BACKUPS = Path("/data/backups")
|
||||||
|
|
||||||
#: Any one of these missing means this is not our database. Name tables that
|
#: Any one of these missing means this is not our database. Name tables that
|
||||||
#: have existed since the first schema, not ones a recent migration added.
|
#: have existed since the first schema, not ones a recent migration added:
|
||||||
REQUIRED_TABLES = {"items"}
|
#: `jobs` is the queue's and `roles` is `app/storage/schema.py`'s first table
|
||||||
|
#: (the boilerplate has `ticks`). backup.sh's TABLES is the same set.
|
||||||
|
REQUIRED_TABLES = {"jobs", "roles"}
|
||||||
|
|
||||||
|
|
||||||
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
|
def replace_database(database: Path, archive: Path, backups: Path) -> Path | None:
|
||||||
@@ -73,5 +76,10 @@ def replace_database(database: Path, archive: Path, backups: Path) -> Path | Non
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
saved = replace_database(DATABASE, INCOMING, BACKUPS)
|
from app.config import load
|
||||||
|
|
||||||
|
# The snapshot is the whole database, so it is owner only whatever the
|
||||||
|
# container's umask is; the gzip of it is made under the same mask.
|
||||||
|
os.umask(0o077)
|
||||||
|
saved = replace_database(load().database.absolute(), INCOMING, BACKUPS)
|
||||||
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")
|
print(f"Database replaced; previous snapshot: {saved or 'none (first import)'}")
|
||||||
Loaded 100 of 212 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user