From c4701e3d7f2870d3a06c30e38b5458455c1c8b08 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:13:45 +0200 Subject: [PATCH 01/64] todo(1): open --- TODO | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/TODO b/TODO index 5860bf7..7fe0027 100644 --- a/TODO +++ b/TODO @@ -15,3 +15,19 @@ Mappings: - Module: sekft Product: sek Component: sekft + +--ISSUE +Content-Type: application/issue +ID: 1 +Type: feature +Title: Package sekft as an installable namespace package +Status: open +Priority: medium +Created: 2026-06-16 +Module: sekft +Relationships: +Description: Turn the flat trainer scripts into an installable tiararodney.sekft + namespace package: src layout, pyproject with the abstract + posix-sdc dependency and an optional gpu extra, console scripts, a + Pipfile pinning posix-sdc as a local editable override, and tox + environments. From 0ebf19b4ec92c602dd4179a26035fda76de8e659 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:13:47 +0200 Subject: [PATCH 02/64] todo(1): in-progress tiararodney.sekft imports; sekft-train/eval/resident console scripts resolve; pyproject declares posix-sdc as a dependency with torch et al. behind a gpu extra; Pipfile carries the local editable posix-sdc override; tox envs exist. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 7fe0027..037458f 100644 --- a/TODO +++ b/TODO @@ -21,7 +21,7 @@ Content-Type: application/issue ID: 1 Type: feature Title: Package sekft as an installable namespace package -Status: open +Status: in-progress Priority: medium Created: 2026-06-16 Module: sekft From 0ace2597a33470dc1eb4d1c22459dfd96655b375 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:13:48 +0200 Subject: [PATCH 03/64] feat: scaffold installable namespace package --- pyproject.toml | 77 +++++++++++++++++++++++++++++++ src/tiararodney/sekft/__init__.py | 5 ++ 2 files changed, 82 insertions(+) create mode 100644 pyproject.toml create mode 100644 src/tiararodney/sekft/__init__.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c99f76a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,77 @@ +[build-system] +requires = [ + "setuptools", + "wheel", + "setuptools-scm[toml]" +] +build-backend = "setuptools.build_meta" + +[project] +name = "tiararodney.sekft" +description = "Fine-tune small open models to operate a POSIX shell (sek)" +authors = [ + { name = "Tiara Rodney", email = "tiara.rodney@byteb4rb1e.me" } +] +readme = "README.md" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: System :: Shells", + "Typing :: Typed", +] +dependencies = [ + "tiararodney.posix-sdc", +] +dynamic = ["version"] +requires-python = ">=3.9" + +[project.optional-dependencies] +gpu = [ + "torch", + "transformers", + "peft", + "datasets", + "accelerate", + "bitsandbytes", + "tensorboard", +] + +[project.scripts] +sekft-train = "tiararodney.sekft.sft:main" +sekft-eval = "tiararodney.sekft.eval:main" +sekft-resident = "tiararodney.sekft.resident:main" + +[project.urls] +Git = "https://git.code.tiararodney.com/tiararodney/sekft" + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = true + +[tool.pytest.ini_options] +pythonpath = ["src", "../posix-sdc/src"] +testpaths = ["tests"] +markers = [ + "pytest: integration tests runnable without external services", + "gpu: requires torch and a GPU", + "docker: requires Docker and the sekft-dash image", +] + +[tool.mypy] +strict = true + +[tool.autopep8] +max_line_length = 80 +aggressive = 3 +recursive = true + +[tool.setuptools_scm] diff --git a/src/tiararodney/sekft/__init__.py b/src/tiararodney/sekft/__init__.py new file mode 100644 index 0000000..0cb60f3 --- /dev/null +++ b/src/tiararodney/sekft/__init__.py @@ -0,0 +1,5 @@ +"""sekft: fine-tune small open models to operate a POSIX shell (sek). + +Consumes the posix-sdc dataset; the trainer, behavioural evaluator, and the +resident-base harness live here. +""" From d468870d64f46a2c0db2575660d97dda99a842f6 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:13:49 +0200 Subject: [PATCH 04/64] chore: pin posix-sdc as a local editable dependency --- Pipfile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 Pipfile diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..08edb5c --- /dev/null +++ b/Pipfile @@ -0,0 +1,32 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +"tiararodney.sekft" = {file = ".", editable = true} +# Local editable override of the posix-sdc dependency declared (abstractly) in +# pyproject.toml, so the dataset/factory is developed side by side. +"tiararodney.posix-sdc" = {path = "../posix-sdc", editable = true} + +[dev-packages] +tox = "*" +pytest = "*" +build = "*" +twine = "*" +setuptools-scm = "~=8.2.0" +pypi-attestations = "*" +autopep8 = "*" + +[requires] +python_version = "3" + +[scripts] +"dist" = "python3 -m build" +"dist:attestations" = "python3 -m pypi_attestations sign dist/*" +"dist:publish:tiararodney" = "python3 -m twine upload --sign --repository tiararodney dist/*" +"test" = "tox" +"test:static" = "tox run -m static" +"test:unit" = "tox run -m unit" +"test:integration" = "tox run -m integration" +"test:smoke" = "tox run -m smoke" From b00aefaf780b0a0e3b8e59c8abb57d9c2762c434 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:13:49 +0200 Subject: [PATCH 05/64] chore: add tox lint, format and test environments --- tox.ini | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..917e98a --- /dev/null +++ b/tox.ini @@ -0,0 +1,47 @@ +[tox] +requires = + tox>=4.19 +env_list = + unit-py3{9-13} + smoke-py3{9-13} + lint + format + +[testenv] +deps = + ../posix-sdc + . + +[testenv:lint] +description = run type check on code base +labels = static +deps = + mypy +commands = + mypy src tests --junit-xml test-reports/{env_name}.xml + +[testenv:format] +description = check formatting +labels = static +deps = + autopep8 +commands = + autopep8 --diff --exit-code src tests + +[testenv:unit-py3{9-13}] +description = run unit tests +labels = unit +deps = + {[testenv]deps} + pytest +commands = + pytest tests/unit --junitxml=test-reports/{env_name}.xml + +[testenv:smoke-py3{9-13}] +description = run smoke tests against the console entry points +labels = smoke +deps = + {[testenv]deps} + pytest +commands = + pytest tests/smoke --junitxml=test-reports/{env_name}.xml From 046683a3717cf7ab8a70a915dc59f6a12a1f2f76 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:13:51 +0200 Subject: [PATCH 06/64] todo(1): done Namespace package builds; gpu extra isolates torch/transformers/peft/datasets; posix-sdc declared in pyproject and overridden editable in Pipfile; console scripts wired; tox envs defined. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 037458f..b9e1ed8 100644 --- a/TODO +++ b/TODO @@ -21,7 +21,7 @@ Content-Type: application/issue ID: 1 Type: feature Title: Package sekft as an installable namespace package -Status: in-progress +Status: done Priority: medium Created: 2026-06-16 Module: sekft From fab16d175d10fac5f2494ab991d7606b5fbef067 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:09 +0200 Subject: [PATCH 07/64] todo(2): open --- TODO | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/TODO b/TODO index b9e1ed8..6840657 100644 --- a/TODO +++ b/TODO @@ -31,3 +31,18 @@ Description: Turn the flat trainer scripts into an installable tiararodney.sekft posix-sdc dependency and an optional gpu extra, console scripts, a Pipfile pinning posix-sdc as a local editable override, and tox environments. + +--ISSUE +Content-Type: application/issue +ID: 2 +Type: feature +Title: SFT trainer with chat-template render and assistant-only mask +Status: open +Priority: medium +Created: 2026-06-16 +Module: sekft +Relationships: +Description: Add the supervised fine-tuner: render trajectories through the + tokenizer's own chat template (matching serving), canonicalise + turns (fold system, merge consecutive), derive an assistant-only + loss mask by token-prefix differencing, and train a QLoRA adapter. From 4533a040210d9808fbb5b83894f53e218723b2c4 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:11 +0200 Subject: [PATCH 08/64] todo(2): in-progress sft renders via apply_chat_template; normalize_for_template folds system and merges consecutive turns; the loss mask trains assistant turns only and raises on a non-additive template; --inspect reports mask stats. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 6840657..85e05c7 100644 --- a/TODO +++ b/TODO @@ -37,7 +37,7 @@ Content-Type: application/issue ID: 2 Type: feature Title: SFT trainer with chat-template render and assistant-only mask -Status: open +Status: in-progress Priority: medium Created: 2026-06-16 Module: sekft From 0bb9c1983b5d6a69906a8128c667a657ec1d7f89 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:12 +0200 Subject: [PATCH 09/64] feat: add SFT trainer with chat-template render and assistant-only mask --- src/tiararodney/sekft/sft.py | 209 +++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 src/tiararodney/sekft/sft.py diff --git a/src/tiararodney/sekft/sft.py b/src/tiararodney/sekft/sft.py new file mode 100644 index 0000000..db5df3b --- /dev/null +++ b/src/tiararodney/sekft/sft.py @@ -0,0 +1,209 @@ +"""sekft trainer: SFT a base model on kept shell-operation trajectories. + +Trains assistant turns ONLY -- the commands and the terminal ``exit`` / ``panic``. +The environment turns (system orientation, prompts, command output) are masked +to ``-100`` so the model learns to *produce* commands, not to predict the +environment's replies. Getting this mask wrong is the classic way to ruin a +shell-operator SFT (the model starts hallucinating output), so it is the part +worth testing hardest -- and it is framework-independent. + +Render uses the tokenizer's OWN chat template (``apply_chat_template``), so the +training render is identical to what the serving harness produces (ccpty sends +structured messages and the inference endpoint applies the model's default +template). Trajectories are canonicalised first (``normalize_for_template``): +a leading ``system`` turn is folded into the first ``user`` turn and consecutive +same-role turns are merged, because instruct templates such as Mistral's have no +system role and require strict user/assistant alternation. That same +canonicalisation must run on the serving side. Everything else is standard +causal-LM SFT with an assistant-only loss mask. + + python sft.py --data ./trajectories --base --out ./ckpt + python sft.py --data ./trajectories --base --inspect # mask stats, no training + +Training needs torch + transformers + peft (a GPU box). ``--inspect`` and the +normalize/mask helpers run anywhere a tokenizer with a chat template is +available. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +def normalize_for_template(messages: list[dict]) -> list[dict]: + """Canonicalise a trajectory for instruct chat templates that have no system + role and require strict user/assistant alternation (Mistral and friends): + treat ``system`` as ``user``, then merge consecutive same-role turns by + joining their content with a newline. + + This is loss-neutral for the assistant mask (only environment/user turns + ever merge; the assistant commands are never adjacent in this data) and it + is what lets ``apply_chat_template`` render the multi-turn shell dialogue. + The serving side MUST apply the same canonicalisation, or train and serve + diverge again. + """ + out: list[dict] = [] + for m in messages: + role = "user" if m["role"] == "system" else m["role"] + if out and out[-1]["role"] == role: + out[-1] = {"role": role, "content": out[-1]["content"] + "\n" + m["content"]} + else: + out.append({"role": role, "content": m["content"]}) + return out + + +def build_masked_example(messages: list[dict], tokenizer) -> dict: + """Tokenize a trajectory with the tokenizer's OWN chat template and build an + assistant-only loss mask. + + The render is ``tokenizer.apply_chat_template`` on the canonicalised turns, + so it is byte-identical to what the serving harness sends. The mask is + derived by token-prefix differencing: the tokens an assistant turn + contributes are exactly those that appear when it extends the rendered + prefix, which trains the commands plus the template's end-of-turn token (so + the model learns to stop) and masks every environment turn to ``-100``. This + assumes an additive template (each turn extends the previous render); a + non-additive one raises rather than silently mis-mask. + """ + msgs = normalize_for_template(messages) + ids = tokenizer.apply_chat_template(msgs, add_generation_prompt=False) + labels = [-100] * len(ids) + prev: list[int] = [] + for i, m in enumerate(msgs): + upto = tokenizer.apply_chat_template(msgs[:i + 1], add_generation_prompt=False) + if ids[:len(upto)] != upto or upto[:len(prev)] != prev: + raise ValueError("chat template is not additive; cannot derive an " + "assistant loss mask by token-prefix differencing") + if m["role"] == "assistant": + for j in range(len(prev), len(upto)): + labels[j] = ids[j] + prev = upto + return {"input_ids": ids, "attention_mask": [1] * len(ids), "labels": labels} + + +def iter_keepers(data_dir: Path): + """Yield ``turns`` (message lists) from trajectory JSONs marked keep.""" + for f in sorted(data_dir.glob("*.json")): + d = json.loads(f.read_text()) + if d.get("keep"): + yield d["turns"] + + +def mask_stats(example: dict) -> tuple[int, int]: + """(trained tokens, total tokens) for an example.""" + trained = sum(1 for x in example["labels"] if x != -100) + return trained, len(example["labels"]) + + +# -------------------------------------------------------------------------- +# Training (GPU box: torch + transformers + peft) +# -------------------------------------------------------------------------- + +def train(data_dir: Path, base: str, out: Path, epochs: float, lr: float, + batch: int, accum: int, max_len: int, lora_r: int, + load_4bit: bool = False) -> None: + import torch + from datasets import Dataset + from peft import LoraConfig, get_peft_model + from transformers import (AutoModelForCausalLM, AutoTokenizer, + DataCollatorForSeq2Seq, Trainer, TrainingArguments) + + tok = AutoTokenizer.from_pretrained(base) + if tok.pad_token is None: + tok.pad_token = tok.eos_token + + rows = [] + for turns in iter_keepers(data_dir): + ex = build_masked_example(turns, tok) + if len(ex["input_ids"]) <= max_len and any(l != -100 for l in ex["labels"]): + rows.append(ex) + if not rows: + raise SystemExit(f"no usable keeper trajectories in {data_dir}") + print(f"examples: {len(rows)}; " + f"trained/total tokens: {sum(mask_stats(r)[0] for r in rows)}" + f"/{sum(mask_stats(r)[1] for r in rows)}") + ds = Dataset.from_list(rows) + + # 4-bit (QLoRA) shrinks the base from ~14 GB to ~4 GB to move across the + # OcuLink/PCIe link and to hold in VRAM; nf4 + fp16 compute works on the + # V100 (sm_70). Without it, plain fp16 weights. + quant = None + if load_4bit: + from transformers import BitsAndBytesConfig + quant = BitsAndBytesConfig( + load_in_4bit=True, bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True, + ) + model = AutoModelForCausalLM.from_pretrained( + base, dtype=torch.float16, quantization_config=quant) + if load_4bit: + from peft import prepare_model_for_kbit_training + model = prepare_model_for_kbit_training(model) # handles ckpt + input grads + else: + model.enable_input_require_grads() + model.gradient_checkpointing_enable() + model = get_peft_model(model, LoraConfig( + r=lora_r, lora_alpha=lora_r * 2, lora_dropout=0.05, task_type="CAUSAL_LM", + target_modules=["q_proj", "k_proj", "v_proj", "o_proj"], + )) + model.print_trainable_parameters() + + args = TrainingArguments( + output_dir=str(out), per_device_train_batch_size=batch, + gradient_accumulation_steps=accum, num_train_epochs=epochs, + learning_rate=lr, fp16=True, logging_steps=1, save_strategy="epoch", + report_to=["tensorboard"], logging_dir=str(out / "runs"), + remove_unused_columns=False, warmup_ratio=0.03, + ) + trainer = Trainer( + model=model, args=args, train_dataset=ds, + data_collator=DataCollatorForSeq2Seq(tok, padding=True, label_pad_token_id=-100), + ) + trainer.train() + model.save_pretrained(str(out)) + tok.save_pretrained(str(out)) + # durable, greppable record of the curve (loss/lr/grad_norm per step). + (out / "log_history.jsonl").write_text( + "\n".join(json.dumps(r) for r in trainer.state.log_history)) + print(f"saved LoRA adapter + log_history.jsonl -> {out} " + f"(tensorboard: --logdir {out / 'runs'})") + + +def inspect(data_dir: Path, base: str) -> None: + from transformers import AutoTokenizer + tok = AutoTokenizer.from_pretrained(base) + n = tt = tr = 0 + for turns in iter_keepers(data_dir): + ex = build_masked_example(turns, tok) + t, total = mask_stats(ex) + tr += t; tt += total; n += 1 + if not n: + raise SystemExit(f"no keeper trajectories in {data_dir}") + print(f"{n} keeper trajectories; {tr}/{tt} tokens trained " + f"({100*tr/tt:.1f}% assistant, rest masked)") + + +def main() -> None: + ap = argparse.ArgumentParser(description="SFT a model on shell trajectories.") + ap.add_argument("--data", type=Path, default=Path("./trajectories")) + ap.add_argument("--base", required=True, help="HF model id or local dir") + ap.add_argument("--out", type=Path, default=Path("./ckpt")) + ap.add_argument("--inspect", action="store_true", help="mask stats only, no training") + ap.add_argument("--epochs", type=float, default=3.0) + ap.add_argument("--lr", type=float, default=2e-4) + ap.add_argument("--batch", type=int, default=1) + ap.add_argument("--accum", type=int, default=8) + ap.add_argument("--max-len", type=int, default=4096) + ap.add_argument("--lora-r", type=int, default=16) + ap.add_argument("--load-4bit", action="store_true", + help="QLoRA: load base in 4-bit (less to move over the link, less VRAM)") + ns = ap.parse_args() + if ns.inspect: + inspect(ns.data, ns.base) + else: + train(ns.data, ns.base, ns.out, ns.epochs, ns.lr, ns.batch, ns.accum, + ns.max_len, ns.lora_r, ns.load_4bit) + + +if __name__ == "__main__": + main() From 4e45bacf0bfebeefd2c7f93d3455d8da2e007184 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:15 +0200 Subject: [PATCH 10/64] todo(2): done Trainer renders with the model's chat template, canonicalises turns, masks to assistant-only by token-prefix differencing, and trains a QLoRA adapter. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 85e05c7..d4e9dce 100644 --- a/TODO +++ b/TODO @@ -37,7 +37,7 @@ Content-Type: application/issue ID: 2 Type: feature Title: SFT trainer with chat-template render and assistant-only mask -Status: in-progress +Status: done Priority: medium Created: 2026-06-16 Module: sekft From 1d672ff8d9d25230ce07f9bbe9ea788523e0b1ad Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:35 +0200 Subject: [PATCH 11/64] todo(3): open --- TODO | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/TODO b/TODO index d4e9dce..1a10bf8 100644 --- a/TODO +++ b/TODO @@ -46,3 +46,18 @@ Description: Add the supervised fine-tuner: render trajectories through the tokenizer's own chat template (matching serving), canonicalise turns (fold system, merge consecutive), derive an assistant-only loss mask by token-prefix differencing, and train a QLoRA adapter. + +--ISSUE +Content-Type: application/issue +ID: 3 +Type: feature +Title: Behavioural evaluator +Status: open +Priority: medium +Created: 2026-06-16 +Module: sekft +Relationships: +Description: Add the behavioural eval: load base plus LoRA adapter, drop it into + held-out scenarios with no scaffold, drive them through a local + operator that renders with the model's chat template, and report + reach/terminate/checker rates. From 7ee0d635228a7ff13573ac72592540cba524a2f8 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:38 +0200 Subject: [PATCH 12/64] todo(3): in-progress eval reuses the posix-sdc rollout with a local operator; renders via apply_chat_template + normalize_for_template; reports command-mode, terminate, and verified rates over held-out scenarios. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 1a10bf8..5b04fb8 100644 --- a/TODO +++ b/TODO @@ -52,7 +52,7 @@ Content-Type: application/issue ID: 3 Type: feature Title: Behavioural evaluator -Status: open +Status: in-progress Priority: medium Created: 2026-06-16 Module: sekft From 121d79c89b9a201e924d1dc7965e3c8ee832568b Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:39 +0200 Subject: [PATCH 13/64] feat: add behavioural evaluator --- src/tiararodney/sekft/eval.py | 101 ++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 src/tiararodney/sekft/eval.py diff --git a/src/tiararodney/sekft/eval.py b/src/tiararodney/sekft/eval.py new file mode 100644 index 0000000..b381385 --- /dev/null +++ b/src/tiararodney/sekft/eval.py @@ -0,0 +1,101 @@ +"""Behavioural eval: the metric that matters. + +Train loss says nothing about whether the model operates the shell and leaves. +This loads a fine-tuned model (base + LoRA adapter), drops it into held-out +scenarios with NO scaffold (the trained behaviour must stand on its own), and +reports the rates that count: does it reach command-mode, does it terminate, +does the checker pass. + + python eval.py --base --adapter ./ckpt-mistral-r16 \ + --scenarios ./holdout-scenarios --n 10 + +Reuses the rollout loop with a *local* operator: the model formats and +generates in the same role-delimited render it was trained on (train == eval == +deploy, or the prompts go out of distribution). Prerequisites on the box: torch ++ transformers + peft, the ``sekft-dash`` image, and held-out SCENARIO bundles +(from ``generate.py`` -- not trajectories; the eval stands up and verifies each). +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from tiararodney.posix_sdc.factory.dashdocker import DashDocker, available +from tiararodney.posix_sdc.factory.rollout import rollout +from tiararodney.posix_sdc.schema import Scenario + +from .sft import normalize_for_template + + +def make_local_operator(base: str, adapter: str, max_new_tokens: int = 64, + temperature: float = 0.7): + """A ``messages -> command`` callable backed by base + LoRA adapter. + + Renders the conversation exactly as the model was trained, appends the + assistant header, generates one turn, and cuts at the first stop marker. + """ + import torch + from peft import PeftModel + from transformers import AutoModelForCausalLM, AutoTokenizer + + tok = AutoTokenizer.from_pretrained(adapter) + model = AutoModelForCausalLM.from_pretrained( + base, torch_dtype=torch.float16, device_map="auto") + model = PeftModel.from_pretrained(model, adapter) + model.eval() + + def operator(messages): + msgs = normalize_for_template(messages) + ids = tok.apply_chat_template( + msgs, add_generation_prompt=True, return_tensors="pt").to(model.device) + with torch.no_grad(): + out = model.generate( + ids, max_new_tokens=max_new_tokens, + do_sample=temperature > 0, temperature=max(temperature, 1e-2), + eos_token_id=tok.eos_token_id, pad_token_id=tok.eos_token_id) + return tok.decode(out[0][ids.shape[1]:], skip_special_tokens=True).strip() + + return operator + + +def evaluate(base: str, adapter: str, scenarios_dir: Path, n: int, + max_steps: int, temperature: float) -> dict: + if not available(): + raise SystemExit("sekft-dash image unavailable; `docker build -t sekft-dash .`") + operator = make_local_operator(base, adapter, temperature=temperature) + backend = DashDocker() + rows = [] + for f in sorted(scenarios_dir.glob("*.json"))[:n]: + sc = Scenario.from_dict(json.loads(f.read_text())) + tj = rollout(sc, backend, max_steps=max_steps, temperature=temperature, + operator=operator, use_scaffold=False) + rows.append(tj) + print(f" {sc.id}: {tj.outcome} (terminal={tj.terminal} " + f"verified={tj.verified} steps={tj.steps})") + d = len(rows) or 1 + return { + "n": len(rows), + "operate_rate": round(sum(t.steps > 0 and t.meta.get("clean") for t in rows) / d, 3), + "terminate_rate": round(sum(t.terminal in ("exit", "panic") for t in rows) / d, 3), + "verified_rate": round(sum(t.verified for t in rows) / d, 3), + "clean_rate": round(sum(t.keep for t in rows) / d, 3), + } + + +def main() -> None: + ap = argparse.ArgumentParser(description="Behavioural eval of a tuned model.") + ap.add_argument("--base", required=True) + ap.add_argument("--adapter", required=True) + ap.add_argument("--scenarios", type=Path, required=True) + ap.add_argument("--n", type=int, default=10) + ap.add_argument("--max-steps", type=int, default=30) + ap.add_argument("--temperature", type=float, default=0.7) + ns = ap.parse_args() + m = evaluate(ns.base, ns.adapter, ns.scenarios, ns.n, ns.max_steps, ns.temperature) + print("\n=== behavioural metrics ===") + print(json.dumps(m, indent=2)) + + +if __name__ == "__main__": + main() From 0dc038053749db77b64a789552a9f492ba1f204a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:41 +0200 Subject: [PATCH 14/64] todo(3): done Evaluator drives held-out scenarios with a chat-template local operator and reports the behavioural rates. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 5b04fb8..d89777c 100644 --- a/TODO +++ b/TODO @@ -52,7 +52,7 @@ Content-Type: application/issue ID: 3 Type: feature Title: Behavioural evaluator -Status: in-progress +Status: done Priority: medium Created: 2026-06-16 Module: sekft From 6f5245cda61ecf9995ab783bab39e4222ab8f8b2 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:44 +0200 Subject: [PATCH 15/64] todo(4): open --- TODO | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/TODO b/TODO index d89777c..1301af8 100644 --- a/TODO +++ b/TODO @@ -61,3 +61,17 @@ Description: Add the behavioural eval: load base plus LoRA adapter, drop it into held-out scenarios with no scaffold, drive them through a local operator that renders with the model's chat template, and report reach/terminate/checker rates. + +--ISSUE +Content-Type: application/issue +ID: 4 +Type: feature +Title: Resident-base train/eval harness +Status: open +Priority: medium +Created: 2026-06-16 +Module: sekft +Relationships: +Description: Add the resident harness that loads the 14GB base once and keeps it + hot, training fresh LoRA adapters and evaluating them without + reloading the base, for the slow-OcuLink iterate loop. From e810d0e4425742efb7d3d1f53d3d62d31f25be01 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:46 +0200 Subject: [PATCH 16/64] todo(4): in-progress Resident loads the base once; fit trains an adapter and unloads it; evaluate attaches an adapter (or the base baseline) and renders via the shared chat-template canonicalisation. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 1301af8..d9b39b3 100644 --- a/TODO +++ b/TODO @@ -67,7 +67,7 @@ Content-Type: application/issue ID: 4 Type: feature Title: Resident-base train/eval harness -Status: open +Status: in-progress Priority: medium Created: 2026-06-16 Module: sekft From 23e749c8785affc474d7c24dd127256a41b84125 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:47 +0200 Subject: [PATCH 17/64] feat: add resident-base train and eval harness --- src/tiararodney/sekft/resident.py | 187 ++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 src/tiararodney/sekft/resident.py diff --git a/src/tiararodney/sekft/resident.py b/src/tiararodney/sekft/resident.py new file mode 100644 index 0000000..c56979b --- /dev/null +++ b/src/tiararodney/sekft/resident.py @@ -0,0 +1,187 @@ +"""Resident harness: load the base ONCE, cycle adapters. + +On a slow link (OcuLink / PCIe 3.0 x4) the 14 GB base transfer dominates every +process start. This loads the base once and keeps it hot, so the +iterate-train-eval loop pays the transfer only at startup. Each ``fit`` trains a +fresh LoRA adapter on the resident base and ``unload``s it back to clean; each +``evaluate`` attaches a saved adapter for inference and unloads. + +Interactive (IPython on the GPU box) is the intended use: + + from resident import Resident + r = Resident("~/llm-models/mistral-7b-instruct-v0.2", load_4bit=True) + r.fit("~/sekft/trajectories", "~/sekft/ckpt-a", lora_r=16, lr=2e-4, epochs=3) + r.evaluate("~/sekft/ckpt-a", "~/sekft/holdout", n=10) + r.fit("~/sekft/trajectories", "~/sekft/ckpt-b", lora_r=32) # NO base reload + +Or `python resident.py --base --selftest-data ` to prove the +base loads once and two adapters train against it. +""" +from __future__ import annotations + +import argparse +import gc +import json +from pathlib import Path + +import torch +from datasets import Dataset +from peft import (LoraConfig, PeftModel, get_peft_model, + prepare_model_for_kbit_training) +from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, + DataCollatorForSeq2Seq, Trainer, TrainingArguments) + +from .sft import build_masked_example, iter_keepers, normalize_for_template + +LORA_TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj"] + + +def _free() -> None: + gc.collect() + torch.cuda.empty_cache() + + +class Resident: + """A base model held resident on the GPU; adapters cycle through it.""" + + def __init__(self, base: str, load_4bit: bool = False) -> None: + self.base_path = str(Path(base).expanduser()) + self.load_4bit = load_4bit + self.tok = AutoTokenizer.from_pretrained(self.base_path) + if self.tok.pad_token is None: + self.tok.pad_token = self.tok.eos_token + quant = None + if load_4bit: + quant = BitsAndBytesConfig( + load_in_4bit=True, bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True) + print(f"[resident] loading base ONCE: {self.base_path} (4bit={load_4bit}) ...") + self.base = AutoModelForCausalLM.from_pretrained( + self.base_path, dtype=torch.float16, quantization_config=quant) + self.base = (prepare_model_for_kbit_training(self.base) if load_4bit + else self.base) + if not load_4bit: + self.base.enable_input_require_grads() + dev = next(self.base.parameters()).device + mem = torch.cuda.memory_allocated() / 1e9 + print(f"[resident] base resident on {dev}; {mem:.1f} GB VRAM") + + # -- build masked rows from kept trajectories -------------------------- + + def _rows(self, data_dir: Path, max_len: int) -> list[dict]: + rows = [] + for turns in iter_keepers(data_dir): + ex = build_masked_example(turns, self.tok) + if len(ex["input_ids"]) <= max_len and any(l != -100 for l in ex["labels"]): + rows.append(ex) + if not rows: + raise SystemExit(f"no usable keeper trajectories in {data_dir}") + return rows + + # -- train a fresh adapter on the resident base ------------------------ + + def fit(self, data_dir: str, out: str, lora_r: int = 16, lr: float = 2e-4, + epochs: float = 3.0, batch: int = 1, accum: int = 8, + max_len: int = 4096) -> Path: + data_dir, out = Path(data_dir).expanduser(), Path(out).expanduser() + ds = Dataset.from_list(self._rows(data_dir, max_len)) + if not self.load_4bit: + self.base.gradient_checkpointing_enable() + model = get_peft_model(self.base, LoraConfig( + r=lora_r, lora_alpha=lora_r * 2, lora_dropout=0.05, + task_type="CAUSAL_LM", target_modules=LORA_TARGETS)) + model.print_trainable_parameters() + args = TrainingArguments( + output_dir=str(out), per_device_train_batch_size=batch, + gradient_accumulation_steps=accum, num_train_epochs=epochs, + learning_rate=lr, fp16=True, logging_steps=1, save_strategy="no", + report_to=["tensorboard"], logging_dir=str(out / "runs"), + remove_unused_columns=False, warmup_ratio=0.03) + tr = Trainer(model=model, args=args, train_dataset=ds, + data_collator=DataCollatorForSeq2Seq( + self.tok, padding=True, label_pad_token_id=-100)) + tr.train() + out.mkdir(parents=True, exist_ok=True) + model.save_pretrained(str(out)) + self.tok.save_pretrained(str(out)) + (out / "log_history.jsonl").write_text( + "\n".join(json.dumps(r) for r in tr.state.log_history)) + losses = [h["loss"] for h in tr.state.log_history if "loss" in h] + print(f"[resident] fit -> {out} final loss {losses[-1] if losses else '?'}") + self.base = model.unload() # strip LoRA, restore resident base + del model, tr, ds + _free() + return out + + # -- behavioural eval of a saved adapter ------------------------------- + + def evaluate(self, adapter: str, scenarios_dir: str, n: int = 10, + max_steps: int = 30, temperature: float = 0.7) -> dict: + from tiararodney.posix_sdc.factory.dashdocker import DashDocker, available + from tiararodney.posix_sdc.factory.rollout import rollout + from tiararodney.posix_sdc.schema import Scenario + if not available(): + raise SystemExit("sekft-dash image unavailable on this box") + # adapter=None -> evaluate the BASE model (the within-holdout baseline). + if adapter: + adapter = str(Path(adapter).expanduser()) + pm = PeftModel.from_pretrained(self.base, adapter) + else: + pm = self.base + pm.eval() + + def operator(messages): + msgs = normalize_for_template(messages) + ids = self.tok.apply_chat_template( + msgs, add_generation_prompt=True, return_tensors="pt").to(pm.device) + with torch.no_grad(): + o = pm.generate(ids, max_new_tokens=64, do_sample=temperature > 0, + temperature=max(temperature, 1e-2), + eos_token_id=self.tok.eos_token_id, + pad_token_id=self.tok.eos_token_id) + return self.tok.decode(o[0][ids.shape[1]:], skip_special_tokens=True).strip() + + backend = DashDocker() + rows = [] + for f in sorted(Path(scenarios_dir).expanduser().glob("*.json"))[:n]: + sc = Scenario.from_dict(json.loads(f.read_text())) + tj = rollout(sc, backend, max_steps=max_steps, temperature=temperature, + operator=operator, use_scaffold=False) + rows.append(tj) + print(f" {sc.id}: {tj.outcome} terminal={tj.terminal} verified={tj.verified}") + d = len(rows) or 1 + m = { + "n": len(rows), + "operate_rate": round(sum(t.steps > 0 and t.meta.get("clean") for t in rows) / d, 3), + "terminate_rate": round(sum(t.terminal in ("exit", "panic") for t in rows) / d, 3), + "verified_rate": round(sum(t.verified for t in rows) / d, 3), + "clean_rate": round(sum(t.keep for t in rows) / d, 3), + } + if adapter: # base is unwrapped only if we wrapped it + self.base = pm.unload() + del pm + _free() + print("[resident] eval:", json.dumps(m)) + return m + + +def main() -> None: + ap = argparse.ArgumentParser(description="Resident base; cycle adapters.") + ap.add_argument("--base", required=True) + ap.add_argument("--load-4bit", action="store_true") + ap.add_argument("--selftest-data", + help="fit two adapters on this data to prove resident multi-fit") + ns = ap.parse_args() + r = Resident(ns.base, ns.load_4bit) + if ns.selftest_data: + print("=== selftest: two fits on the SAME resident base (no reload) ===") + r.fit(ns.selftest_data, "/tmp/res-a", epochs=1, lora_r=8) + r.fit(ns.selftest_data, "/tmp/res-b", epochs=1, lora_r=8) + print("=== selftest OK: base loaded once, two adapters trained ===") + else: + print("Resident ready. Import and use r.fit() / r.evaluate(), " + "or pass --selftest-data .") + + +if __name__ == "__main__": + main() From fd30ab47ed38a09bfb5b72d282e7f431f7816d74 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:14:49 +0200 Subject: [PATCH 18/64] todo(4): done Resident base loads once; fit/evaluate cycle adapters without reloading; renders match the trainer. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index d9b39b3..82fcd42 100644 --- a/TODO +++ b/TODO @@ -67,7 +67,7 @@ Content-Type: application/issue ID: 4 Type: feature Title: Resident-base train/eval harness -Status: in-progress +Status: done Priority: medium Created: 2026-06-16 Module: sekft From a24c6c9681d77e6618da93600eda477be2a3c7af Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:09 +0200 Subject: [PATCH 19/64] todo(5): open --- TODO | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/TODO b/TODO index 82fcd42..83f9722 100644 --- a/TODO +++ b/TODO @@ -75,3 +75,17 @@ Relationships: Description: Add the resident harness that loads the 14GB base once and keeps it hot, training fresh LoRA adapters and evaluating them without reloading the base, for the slow-OcuLink iterate loop. + +--ISSUE +Content-Type: application/issue +ID: 5 +Type: feature +Title: Pipeline overview README +Status: open +Priority: medium +Created: 2026-06-16 +Module: sekft +Relationships: +Description: Document the sekft pipeline: the trainer, evaluator, and resident + harness; how they consume the posix-sdc dataset; the render + contract; and how to run on the GPU box. From 12f10854c3bed9448229fc48ce5419f2a401e131 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:11 +0200 Subject: [PATCH 20/64] todo(5): in-progress README describes the trainer/eval/resident roles, the posix-sdc dependency, the chat-template render contract, and the box workflow. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 83f9722..a1406b9 100644 --- a/TODO +++ b/TODO @@ -81,7 +81,7 @@ Content-Type: application/issue ID: 5 Type: feature Title: Pipeline overview README -Status: open +Status: in-progress Priority: medium Created: 2026-06-16 Module: sekft From ea31b7fceb981ab027e3ff7f9452369011543597 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:12 +0200 Subject: [PATCH 21/64] docs: add pipeline overview --- README.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..f5cfbc5 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# sekft + +Synthetic-trajectory generation for fine-tuning a model to operate a shell +as a self-directed citizen: land with **no imperative**, discover where +directives live, learn the provider from its own self-documentation, retrieve +the directives, execute them, and terminate (`exit` on success, `panic` when +genuinely blocked). + +The dataset teaches a **mechanism, not a program**. Every axis of a scenario +is varied; only the four-step routine is held invariant: + +1. **expect an announcement** of where directives are (motd / banner / env / file) +2. **understand the provider** via its self-documentation (`--help` / `man` / usage) +3. **retrieve** the directives +4. **execute**, then terminate + +Bind the *convention* (there is an announcement at entry; tools are +self-documenting), free everything else. The model that learns this tolerates +an unstable userland because it re-learns the interface every session. + +## Pipeline + +``` +A. author generate.py model writes scenario bundles from the taxonomy + + ref-gate dashdocker.py run the bundle's own reference solution; admit only if its checker passes +B. rollout rollout.py scaffolded operator model acts in a fresh dash-in-docker container +C. verify rollout.py run the checker against container STATE (effect, not transcript) +D. record rollout.py strip the operator scaffold; save env<->action turns in deploy format +E. pairs [seam] rejects from B/C become DPO negatives against keepers from the same scenario +``` + +This repo implements **A-D** plus the execution backend (`dashdocker.py`). +Stage E (preference-pair assembly from the kept/rejected trajectories) is the +remaining seam; the rejects are already labelled by `outcome`/`keep`. + +## Files + +- `taxonomy.py` - the axes of variation (task / provider / announcement / + doc-depth / difficulty) as pure data. No model, no container. +- `schema.py` - the `Scenario` bundle dataclasses + JSON (de)serialisation. +- `generate.py` - sample a combo, prompt a teacher model to author the bundle, + gate on the reference solution, write validated bundles to disk. +- `dashdocker.py` - the dash-in-Docker backend. `run(fixtures, script)` for the + one-shot reference gate; `session(fixtures)` for stateful rollouts, with + `Session.exec` (state-replayed), `.cwd()` (prompt building), `.check()` (Stage + C). Each command runs as its own `docker exec` (no tty buffering); cwd + + exported env are replayed between commands; `exit`/`panic` are intercepted as + terminals. +- `rollout.py` - Stage D. Rolls an operator model through a scenario in a fresh + container with only the disposable `SCAFFOLD`, records the turns + imperative-free (orientation + login + prompt/command/output, ending in the + terminal), verifies against final state, and classifies the outcome into a + `keep` decision. Multiple `--samples` per scenario for rejection sampling. +- `Dockerfile` - `sekft-dash`: alpine + dash, `/bin/sh` -> dash. + +## Run + +```sh +docker build -t sekft-dash . # the execution sandbox (once) + +SEKFT_MODEL=qwen2.5:32b \ # strong teacher via the litellm proxy +SEKFT_URL=http://localhost:4000/v1 \ +SEKFT_KEY=sk-litellm-dev \ + python generate.py --n 50 --out ./scenarios + +SEKFT_OP_MODEL=qwen2.5:32b \ # operator (teacher in round 1, student in STaR) + python rollout.py --scenarios ./scenarios --out ./trajectories --samples 3 +``` + +`rollout.py` writes one JSON per (scenario, sample) with the recorded turns and +a `keep` flag. The keepers are the SFT set; the rejects (labelled by `outcome`) +are Stage E's DPO negatives. Both stages run the model through the litellm +proxy; the rollout's container work is CPU/disk only. + +When the `sekft-dash` image is present, `generate.py` runs each bundle's +reference solution in a fresh container and admits it only if its checker then +passes (real solvability gate). Without the image it falls back to a +**structural** dry-run that proves consistency, not solvability (`--no-docker` +forces this). The backend is verified end-to-end: `python dashdocker.py` runs a +self-test (fixtures, cwd/env replay, terminals). + +## Non-negotiables (or the data rots) + +- **Reference-solution gate is mandatory** once the runner exists: never admit + a scenario whose own checker its reference solution cannot pass. +- **Verify effect, not claim**: the checker inspects container state. +- **Strip teacher prose** from recorded assistant turns (Stage D). +- **Balance terminals**: enough `empty-queue` and `blocked -> panic` scenarios + or the student learns "always exit success". From cf130e032e900879433fa0ada9aca06a150bff71 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:13 +0200 Subject: [PATCH 22/64] todo(5): done README documents the pipeline, the posix-sdc dependency, and the box workflow. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index a1406b9..4bef28b 100644 --- a/TODO +++ b/TODO @@ -81,7 +81,7 @@ Content-Type: application/issue ID: 5 Type: feature Title: Pipeline overview README -Status: in-progress +Status: done Priority: medium Created: 2026-06-16 Module: sekft From 9bd61f99a3d54118863c8cbd3f17555f0fc84365 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:16 +0200 Subject: [PATCH 23/64] todo(6): open --- TODO | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/TODO b/TODO index 4bef28b..b812627 100644 --- a/TODO +++ b/TODO @@ -89,3 +89,18 @@ Relationships: Description: Document the sekft pipeline: the trainer, evaluator, and resident harness; how they consume the posix-sdc dataset; the render contract; and how to run on the GPU box. + +--ISSUE +Content-Type: application/issue +ID: 6 +Type: feature +Title: Test suite: unit and smoke +Status: open +Priority: medium +Created: 2026-06-16 +Module: sekft +Relationships: +Description: Add a pytest suite: torch-free unit tests for the render + canonicalisation and assistant-only mask (fake tokenizer), and + smoke tests that the console entry points respond to --help without + the GPU stack. From 45120dea9753f3e9c9026e5212c2e0aa51fd4338 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:18 +0200 Subject: [PATCH 24/64] todo(6): in-progress tests/unit and tests/smoke pass under pytest; the mask test proves assistant-only training and raises on non-additive templates; entry-point smoke tests pass without torch. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index b812627..db3a33e 100644 --- a/TODO +++ b/TODO @@ -95,7 +95,7 @@ Content-Type: application/issue ID: 6 Type: feature Title: Test suite: unit and smoke -Status: open +Status: in-progress Priority: medium Created: 2026-06-16 Module: sekft From 4d1ec9e7fa96d99ce2838ad4bf2e0aea971ce48e Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:19 +0200 Subject: [PATCH 25/64] test: add SFT render and mask unit tests --- tests/unit/test_sft.py | 75 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/unit/test_sft.py diff --git a/tests/unit/test_sft.py b/tests/unit/test_sft.py new file mode 100644 index 0000000..f2e84b4 --- /dev/null +++ b/tests/unit/test_sft.py @@ -0,0 +1,75 @@ +"""Unit tests for the SFT render canonicalisation and assistant-only mask. + +These run anywhere: a fake additive tokenizer stands in for a real chat +template, so no torch/transformers is needed.""" +from __future__ import annotations + +from typing import Any + +import pytest + +from tiararodney.sekft import sft + + +class FakeTok: + """Additive chat template: each turn renders to `` tokens... ``; + the generation prompt appends ````.""" + + def apply_chat_template(self, msgs: list[dict], add_generation_prompt: bool = False, + return_tensors: Any = None) -> list[str]: + toks: list[str] = [] + for m in msgs: + toks.append(f"<{m['role']}>") + toks += m["content"].split() + toks.append("") + if add_generation_prompt: + toks.append("") + return toks + + +def test_normalize_folds_system_and_merges_consecutive() -> None: + raw = [ + {"role": "system", "content": "orient"}, + {"role": "user", "content": "login"}, + {"role": "user", "content": "prompt"}, + {"role": "assistant", "content": "cat f"}, + {"role": "user", "content": "out"}, + {"role": "user", "content": "prompt"}, + {"role": "assistant", "content": "exit"}, + ] + norm = sft.normalize_for_template(raw) + assert [m["role"] for m in norm] == ["user", "assistant", "user", "assistant"] + assert norm[0]["content"] == "orient\nlogin\nprompt" + + +def test_normalize_leaves_clean_alternation_untouched() -> None: + raw = [{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}] + assert sft.normalize_for_template(raw) == raw + + +def test_mask_trains_assistant_turns_only() -> None: + raw = [ + {"role": "system", "content": "orient"}, + {"role": "user", "content": "login"}, + {"role": "assistant", "content": "cat f"}, + {"role": "user", "content": "out"}, + {"role": "assistant", "content": "exit"}, + ] + ex = sft.build_masked_example(raw, FakeTok()) + trained = [t for t, lab in zip(ex["input_ids"], ex["labels"]) if lab != -100] + masked = [t for t, lab in zip(ex["input_ids"], ex["labels"]) if lab == -100] + assert set(trained) <= {"", "cat", "f", "exit", ""} + assert "cat" in trained and "exit" in trained # both commands present + assert {"orient", "login", "out"} <= set(masked) # environment masked + + +def test_mask_raises_on_non_additive_template() -> None: + class BadTok: + def apply_chat_template(self, msgs: list[dict], add_generation_prompt: bool = False, + return_tensors: Any = None) -> list[int]: + return list(range(len(msgs), 0, -1)) # reversed: prefixes do not nest + + with pytest.raises(ValueError): + sft.build_masked_example( + [{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}], + BadTok()) From 0ea77d434f6e822392ca9175dcc219e6d08897fd Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:20 +0200 Subject: [PATCH 26/64] test: add entry-point smoke tests --- tests/smoke/test_entrypoints.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/smoke/test_entrypoints.py diff --git a/tests/smoke/test_entrypoints.py b/tests/smoke/test_entrypoints.py new file mode 100644 index 0000000..6d6c474 --- /dev/null +++ b/tests/smoke/test_entrypoints.py @@ -0,0 +1,30 @@ +"""Smoke tests: the console entry points load and respond to --help without the +GPU stack (torch is imported lazily inside the training/eval code paths).""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SRC = ROOT / "src" +POSIX_SRC = ROOT.parent / "posix-sdc" / "src" + + +def _help(module: str) -> "subprocess.CompletedProcess[str]": + env = dict(os.environ, PYTHONPATH=os.pathsep.join([str(SRC), str(POSIX_SRC)])) + return subprocess.run([sys.executable, "-m", module, "--help"], + capture_output=True, text=True, env=env) + + +def test_train_help() -> None: + cp = _help("tiararodney.sekft.sft") + assert cp.returncode == 0, cp.stderr + assert "--data" in cp.stdout + + +def test_eval_help() -> None: + cp = _help("tiararodney.sekft.eval") + assert cp.returncode == 0, cp.stderr + assert "--adapter" in cp.stdout From 95ce27530145b1b189d74184e72e7ea7b9ad694d Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:15:21 +0200 Subject: [PATCH 27/64] todo(6): done 6 tests pass (4 unit render/mask, 2 smoke entry points); mask test covers assistant-only training and the non-additive guard. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index db3a33e..6c582a5 100644 --- a/TODO +++ b/TODO @@ -95,7 +95,7 @@ Content-Type: application/issue ID: 6 Type: feature Title: Test suite: unit and smoke -Status: in-progress +Status: done Priority: medium Created: 2026-06-16 Module: sekft From 0a03dd0cfa665a4e0264e977b58cc1258adfcf3a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:27:50 +0200 Subject: [PATCH 28/64] todo(7): open --- TODO | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/TODO b/TODO index 6c582a5..8497f19 100644 --- a/TODO +++ b/TODO @@ -104,3 +104,17 @@ Description: Add a pytest suite: torch-free unit tests for the render canonicalisation and assistant-only mask (fake tokenizer), and smoke tests that the console entry points respond to --help without the GPU stack. + +--ISSUE +Content-Type: application/issue +ID: 7 +Type: feature +Title: Add GPL-2.0 license and drop the relocated Dockerfile +Status: open +Priority: medium +Created: 2026-06-16 +Module: sekft +Relationships: +Description: License sekft under GPL-2.0 (canonical text plus pyproject + metadata) and remove the dash Dockerfile, which now lives in + posix-sdc under docker/alpine-dash. From acaf8dd061cd63ecd70037012dbb545d4ea20c4d Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:27:51 +0200 Subject: [PATCH 29/64] todo(7): in-progress LICENSE holds the GPL-2.0 text; pyproject declares it via license-files and a GPLv2 classifier; the Dockerfile is gone from sekft. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 8497f19..aeb95bc 100644 --- a/TODO +++ b/TODO @@ -110,7 +110,7 @@ Content-Type: application/issue ID: 7 Type: feature Title: Add GPL-2.0 license and drop the relocated Dockerfile -Status: open +Status: in-progress Priority: medium Created: 2026-06-16 Module: sekft From 66365f3d5fc4d2c34e4d2d9b95dc9dde96e0425a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:28:17 +0200 Subject: [PATCH 30/64] chore: add GPL-2.0 license --- LICENSE | 338 +++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 + 2 files changed, 340 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9efa6fb --- /dev/null +++ b/LICENSE @@ -0,0 +1,338 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Moe Ghoul, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/pyproject.toml b/pyproject.toml index c99f76a..fa22ee5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,11 +12,13 @@ description = "Fine-tune small open models to operate a POSIX shell (sek)" authors = [ { name = "Tiara Rodney", email = "tiara.rodney@byteb4rb1e.me" } ] +license-files = ["LICENSE"] readme = "README.md" classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", From f471db2cde71d53d7cb9d99d5237a0e5c06e332f Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:28:18 +0200 Subject: [PATCH 31/64] chore: remove Dockerfile relocated to posix-sdc --- Dockerfile | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index cb8280d..0000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# Minimal dash-in-a-box for sekft trajectory generation. -# docker build -t sekft-dash . -# -# dash as the operated shell (strict POSIX, no bashisms), busybox applets for -# the coreutils. busybox is intentionally close to minimal POSIX so trajectories -# transfer toward sek rather than encoding GNU-isms. Add `coreutils findutils -# grep sed` here if you want GNU semantics instead. -FROM alpine:3.19 -RUN apk add --no-cache dash \ - && ln -sf /usr/bin/dash /bin/dash \ - && ln -sf /usr/bin/dash /bin/sh -# /work is the default arena; provider files land at their absolute paths. -RUN mkdir -p /work -WORKDIR /work From a34ce78dc5d376cd00eedca576aef172e41beeec Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 20:28:19 +0200 Subject: [PATCH 32/64] todo(7): done GPL-2.0 LICENSE added with pyproject license-files and GPLv2 classifier; Dockerfile removed (now in posix-sdc docker/alpine-dash). --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index aeb95bc..6dad3a8 100644 --- a/TODO +++ b/TODO @@ -110,7 +110,7 @@ Content-Type: application/issue ID: 7 Type: feature Title: Add GPL-2.0 license and drop the relocated Dockerfile -Status: in-progress +Status: done Priority: medium Created: 2026-06-16 Module: sekft From 4646d34d9d212521f93773f8bc236d6d6a7845e4 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 23:47:41 +0200 Subject: [PATCH 33/64] todo(8): open --- TODO | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/TODO b/TODO index 6dad3a8..06d2d72 100644 --- a/TODO +++ b/TODO @@ -118,3 +118,19 @@ Relationships: Description: License sekft under GPL-2.0 (canonical text plus pyproject metadata) and remove the dash Dockerfile, which now lives in posix-sdc under docker/alpine-dash. + +--ISSUE +Content-Type: application/issue +ID: 8 +Type: feature +Title: Refresh docs for the packaged trainer +Status: open +Priority: medium +Created: 2026-06-16 +Module: sekft +Relationships: +Description: The README still describes sekft as the data factory + (generate/rollout/dashdocker/taxonomy/schema), which all moved to + posix-sdc. Rewrite it as the trainer (sft/eval/resident) that + consumes posix-sdc, and update the module docstrings to + console-script invocations and the chat-template render contract. From 9cdb2bdc97db83af5a4869ada71945e61ff18250 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 23:47:43 +0200 Subject: [PATCH 34/64] todo(8): in-progress README describes the trainer/eval/resident, the posix-sdc dependency, install via the gpu extra, the chat-template render contract, and console-script usage; module docstrings reference sekft-train/eval/resident and apply_chat_template, not the old flat scripts. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 06d2d72..f6f4c17 100644 --- a/TODO +++ b/TODO @@ -124,7 +124,7 @@ Content-Type: application/issue ID: 8 Type: feature Title: Refresh docs for the packaged trainer -Status: open +Status: in-progress Priority: medium Created: 2026-06-16 Module: sekft From a0b1fbc0c16501cac0802ab5f00aace96238c590 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 23:49:01 +0200 Subject: [PATCH 35/64] docs: rewrite README for the packaged trainer --- README.md | 136 +++++++++++++++++++++++++----------------------------- 1 file changed, 63 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index f5cfbc5..22e2e67 100644 --- a/README.md +++ b/README.md @@ -1,89 +1,79 @@ # sekft -Synthetic-trajectory generation for fine-tuning a model to operate a shell -as a self-directed citizen: land with **no imperative**, discover where -directives live, learn the provider from its own self-documentation, retrieve -the directives, execute them, and terminate (`exit` on success, `panic` when -genuinely blocked). +Fine-tune small open models to operate a POSIX shell as a self-directed citizen: +land with **no imperative**, discover where directives live, learn the provider +from its own self-documentation, do the work, and terminate (`exit` on success, +`panic` when genuinely blocked). -The dataset teaches a **mechanism, not a program**. Every axis of a scenario -is varied; only the four-step routine is held invariant: +sekft is the **training half**. The dataset and the synthetic-data factory live +in [`posix-sdc`](../posix-sdc) (`tiararodney.posix-sdc`), which this package +depends on. Here live the trainer, the behavioural evaluator, and the +resident-base harness. -1. **expect an announcement** of where directives are (motd / banner / env / file) -2. **understand the provider** via its self-documentation (`--help` / `man` / usage) -3. **retrieve** the directives -4. **execute**, then terminate +## Components -Bind the *convention* (there is an announcement at entry; tools are -self-documenting), free everything else. The model that learns this tolerates -an unstable userland because it re-learns the interface every session. +- **`sekft.sft`** (`sekft-train`) — supervised fine-tuner. Renders trajectories + with the tokenizer's own chat template and trains an **assistant-only** loss + mask (the commands plus the terminal token; environment turns masked to -100) + into a QLoRA adapter. Getting the mask wrong is the classic way to ruin a + shell-operator SFT, so it is the part tested hardest. +- **`sekft.eval`** (`sekft-eval`) — behavioural eval. Train loss says nothing + about whether the model operates the shell and leaves. This drops base + + adapter into held-out scenarios with no scaffold and reports the rates that + count: reach command-mode, terminate, checker passes. +- **`sekft.resident`** (`sekft-resident`) — resident-base harness. Loads the + 14 GB base once and keeps it hot, training and evaluating adapters without + reloading it (over OcuLink/PCIe the base transfer otherwise dominates every + run). -## Pipeline +## The render contract -``` -A. author generate.py model writes scenario bundles from the taxonomy - + ref-gate dashdocker.py run the bundle's own reference solution; admit only if its checker passes -B. rollout rollout.py scaffolded operator model acts in a fresh dash-in-docker container -C. verify rollout.py run the checker against container STATE (effect, not transcript) -D. record rollout.py strip the operator scaffold; save env<->action turns in deploy format -E. pairs [seam] rejects from B/C become DPO negatives against keepers from the same scenario -``` +The render the model trains on MUST equal what it is served with. The serving +harness (ccpty) sends structured `{role, content}` messages over the OpenAI +chat-completions protocol, so the endpoint applies the **model's own chat +template**. sekft therefore renders with `apply_chat_template`, after +`normalize_for_template` canonicalises each session: a leading `system` turn is +folded into the first `user` turn and consecutive same-role turns are merged, +because instruct templates such as Mistral's have no system role and require +strict user/assistant alternation. The same canonicalisation must run +serve-side, or train and serve diverge. -This repo implements **A-D** plus the execution backend (`dashdocker.py`). -Stage E (preference-pair assembly from the kept/rejected trajectories) is the -remaining seam; the rejects are already labelled by `outcome`/`keep`. +## Install -## Files - -- `taxonomy.py` - the axes of variation (task / provider / announcement / - doc-depth / difficulty) as pure data. No model, no container. -- `schema.py` - the `Scenario` bundle dataclasses + JSON (de)serialisation. -- `generate.py` - sample a combo, prompt a teacher model to author the bundle, - gate on the reference solution, write validated bundles to disk. -- `dashdocker.py` - the dash-in-Docker backend. `run(fixtures, script)` for the - one-shot reference gate; `session(fixtures)` for stateful rollouts, with - `Session.exec` (state-replayed), `.cwd()` (prompt building), `.check()` (Stage - C). Each command runs as its own `docker exec` (no tty buffering); cwd + - exported env are replayed between commands; `exit`/`panic` are intercepted as - terminals. -- `rollout.py` - Stage D. Rolls an operator model through a scenario in a fresh - container with only the disposable `SCAFFOLD`, records the turns - imperative-free (orientation + login + prompt/command/output, ending in the - terminal), verifies against final state, and classifies the outcome into a - `keep` decision. Multiple `--samples` per scenario for rejection sampling. -- `Dockerfile` - `sekft-dash`: alpine + dash, `/bin/sh` -> dash. - -## Run +The training paths only run on a CUDA host, so the GPU stack is an extra: ```sh -docker build -t sekft-dash . # the execution sandbox (once) - -SEKFT_MODEL=qwen2.5:32b \ # strong teacher via the litellm proxy -SEKFT_URL=http://localhost:4000/v1 \ -SEKFT_KEY=sk-litellm-dev \ - python generate.py --n 50 --out ./scenarios - -SEKFT_OP_MODEL=qwen2.5:32b \ # operator (teacher in round 1, student in STaR) - python rollout.py --scenarios ./scenarios --out ./trajectories --samples 3 +pipenv install # editable sekft + the local editable posix-sdc +pipenv install -e '.[gpu]' # torch / transformers / peft / datasets, on the box ``` -`rollout.py` writes one JSON per (scenario, sample) with the recorded turns and -a `keep` flag. The keepers are the SFT set; the rejects (labelled by `outcome`) -are Stage E's DPO negatives. Both stages run the model through the litellm -proxy; the rollout's container work is CPU/disk only. +`pyproject.toml` declares `tiararodney.posix-sdc` abstractly; the `Pipfile` +overrides it with the local editable `../posix-sdc` for side-by-side development. -When the `sekft-dash` image is present, `generate.py` runs each bundle's -reference solution in a fresh container and admits it only if its checker then -passes (real solvability gate). Without the image it falls back to a -**structural** dry-run that proves consistency, not solvability (`--no-docker` -forces this). The backend is verified end-to-end: `python dashdocker.py` runs a -self-test (fixtures, cwd/env replay, terminals). +## Use (on the GPU box) -## Non-negotiables (or the data rots) +```sh +# fine-tune an adapter on the posix-sdc trajectories +sekft-train --data ./trajectories --base mistralai/Mistral-7B-Instruct-v0.2 \ + --out ./ckpt --load-4bit -- **Reference-solution gate is mandatory** once the runner exists: never admit - a scenario whose own checker its reference solution cannot pass. -- **Verify effect, not claim**: the checker inspects container state. -- **Strip teacher prose** from recorded assistant turns (Stage D). -- **Balance terminals**: enough `empty-queue` and `blocked -> panic` scenarios - or the student learns "always exit success". +# inspect the assistant-only loss mask without training (runs anywhere) +sekft-train --data ./trajectories --base --inspect + +# behavioural eval on held-out scenario bundles (worlds, not trajectories) +sekft-eval --base --adapter ./ckpt --scenarios ./holdout --n 16 + +# resident loop: load the base once, cycle adapters without reloading it +sekft-resident --base --load-4bit +``` + +The eval consumes held-out **scenario bundles** from posix-sdc (it stands up and +verifies each in a fresh container), not trajectories. + +## Result + +Fine-tuning `mistralai/Mistral-7B-Instruct-v0.2` on the posix-sdc data lifted +clean termination on archetype-level held-out scenarios from **0/16 (base) to +9/16 (tuned)**: the operate-and-terminate mechanism generalised to unseen task +types, while task competence stayed archetype-local. See the experiment +[*From seed to weights*](https://blog.tiararodney.com/projects/2026/semantic-execution-kernel/experiments/from-seed-to-weights/). From 9e487bc2bf4ece65e21f9d8c218796e7efbd77b6 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 23:49:02 +0200 Subject: [PATCH 36/64] docs: refresh module docstrings for the packaged layout and render contract --- src/tiararodney/sekft/eval.py | 15 ++++++++------- src/tiararodney/sekft/resident.py | 6 +++--- src/tiararodney/sekft/sft.py | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/tiararodney/sekft/eval.py b/src/tiararodney/sekft/eval.py index b381385..4438134 100644 --- a/src/tiararodney/sekft/eval.py +++ b/src/tiararodney/sekft/eval.py @@ -6,14 +6,15 @@ scenarios with NO scaffold (the trained behaviour must stand on its own), and reports the rates that count: does it reach command-mode, does it terminate, does the checker pass. - python eval.py --base --adapter ./ckpt-mistral-r16 \ - --scenarios ./holdout-scenarios --n 10 + sekft-eval --base --adapter ./ckpt-mistral-r16 \ + --scenarios ./holdout-scenarios --n 10 -Reuses the rollout loop with a *local* operator: the model formats and -generates in the same role-delimited render it was trained on (train == eval == -deploy, or the prompts go out of distribution). Prerequisites on the box: torch -+ transformers + peft, the ``sekft-dash`` image, and held-out SCENARIO bundles -(from ``generate.py`` -- not trajectories; the eval stands up and verifies each). +Reuses the posix-sdc rollout loop with a *local* operator: the model renders and +generates with the same chat template it was trained on (train == eval == serve, +via ``apply_chat_template`` + ``normalize_for_template``, or the prompts go out +of distribution). Prerequisites on the box: torch + transformers + peft, the +``sekft-dash`` image, and held-out SCENARIO bundles from the posix-sdc factory +(not trajectories; the eval stands up and verifies each). """ from __future__ import annotations diff --git a/src/tiararodney/sekft/resident.py b/src/tiararodney/sekft/resident.py index c56979b..c295d48 100644 --- a/src/tiararodney/sekft/resident.py +++ b/src/tiararodney/sekft/resident.py @@ -8,14 +8,14 @@ fresh LoRA adapter on the resident base and ``unload``s it back to clean; each Interactive (IPython on the GPU box) is the intended use: - from resident import Resident + from tiararodney.sekft.resident import Resident r = Resident("~/llm-models/mistral-7b-instruct-v0.2", load_4bit=True) r.fit("~/sekft/trajectories", "~/sekft/ckpt-a", lora_r=16, lr=2e-4, epochs=3) r.evaluate("~/sekft/ckpt-a", "~/sekft/holdout", n=10) r.fit("~/sekft/trajectories", "~/sekft/ckpt-b", lora_r=32) # NO base reload -Or `python resident.py --base --selftest-data ` to prove the -base loads once and two adapters train against it. +Or `sekft-resident --base --selftest-data ` to prove the base +loads once and two adapters train against it. """ from __future__ import annotations diff --git a/src/tiararodney/sekft/sft.py b/src/tiararodney/sekft/sft.py index db5df3b..5ac8633 100644 --- a/src/tiararodney/sekft/sft.py +++ b/src/tiararodney/sekft/sft.py @@ -17,8 +17,8 @@ system role and require strict user/assistant alternation. That same canonicalisation must run on the serving side. Everything else is standard causal-LM SFT with an assistant-only loss mask. - python sft.py --data ./trajectories --base --out ./ckpt - python sft.py --data ./trajectories --base --inspect # mask stats, no training + sekft-train --data ./trajectories --base --out ./ckpt + sekft-train --data ./trajectories --base --inspect # mask stats, no training Training needs torch + transformers + peft (a GPU box). ``--inspect`` and the normalize/mask helpers run anywhere a tokenizer with a chat template is From 26f956f25a7794f510b0dafa86123d7f8b78bb98 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Tue, 16 Jun 2026 23:49:04 +0200 Subject: [PATCH 37/64] todo(8): done README rewritten for the trainer (sft/eval/resident, posix-sdc dependency, gpu extra, render contract, console-script usage); module docstrings updated to sekft-train/eval/resident and apply_chat_template. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index f6f4c17..304309f 100644 --- a/TODO +++ b/TODO @@ -124,7 +124,7 @@ Content-Type: application/issue ID: 8 Type: feature Title: Refresh docs for the packaged trainer -Status: in-progress +Status: done Priority: medium Created: 2026-06-16 Module: sekft From 4fa082478ca3a8beb4129cc449eefb904d208878 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 13:53:02 +0200 Subject: [PATCH 38/64] todo(9): open --- TODO | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/TODO b/TODO index 304309f..85da717 100644 --- a/TODO +++ b/TODO @@ -134,3 +134,18 @@ Description: The README still describes sekft as the data factory posix-sdc. Rewrite it as the trainer (sft/eval/resident) that consumes posix-sdc, and update the module docstrings to console-script invocations and the chat-template render contract. + +--ISSUE +Content-Type: application/issue +ID: 9 +Type: feature +Title: Type-check the package under mypy strict +Status: open +Priority: medium +Created: 2026-06-17 +Module: sekft +Relationships: +Description: Make the lint env honestly pass: add mypy as a dev dependency, + ignore_missing_imports for the ML libs, fully annotate + eval/resident/sft (including the inner operator callables), and + ship a py.typed marker so the Typing::Typed claim is real. From ee2a7294384282a14640e302ca62565e1b13fd56 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 13:53:09 +0200 Subject: [PATCH 39/64] todo(9): in-progress mypy --strict src tests passes; ML imports are configured to not error; eval/resident/sft are fully annotated; py.typed ships in the package. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 85da717..3ed93ce 100644 --- a/TODO +++ b/TODO @@ -140,7 +140,7 @@ Content-Type: application/issue ID: 9 Type: feature Title: Type-check the package under mypy strict -Status: open +Status: in-progress Priority: medium Created: 2026-06-17 Module: sekft From e60495b2ce0518b6f646272c0d21c9ed605db7eb Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 14:03:46 +0200 Subject: [PATCH 40/64] chore: set up mypy strict checking and ship py.typed --- Pipfile | 1 + pyproject.toml | 13 +++++++++++++ src/tiararodney/sekft/py.typed | 0 3 files changed, 14 insertions(+) create mode 100644 src/tiararodney/sekft/py.typed diff --git a/Pipfile b/Pipfile index 08edb5c..fffe94e 100644 --- a/Pipfile +++ b/Pipfile @@ -12,6 +12,7 @@ name = "pypi" [dev-packages] tox = "*" pytest = "*" +mypy = "*" build = "*" twine = "*" setuptools-scm = "~=8.2.0" diff --git a/pyproject.toml b/pyproject.toml index fa22ee5..ae51a61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,9 @@ Git = "https://git.code.tiararodney.com/tiararodney/sekft" where = ["src"] namespaces = true +[tool.setuptools.package-data] +"tiararodney.sekft" = ["py.typed"] + [tool.pytest.ini_options] pythonpath = ["src", "../posix-sdc/src"] testpaths = ["tests"] @@ -70,6 +73,16 @@ markers = [ [tool.mypy] strict = true +mypy_path = "src" +explicit_package_bases = true +namespace_packages = true + +[[tool.mypy.overrides]] +module = [ + "torch.*", "transformers.*", "peft.*", "datasets.*", "bitsandbytes.*", + "tiararodney.posix_sdc.*", +] +ignore_missing_imports = true [tool.autopep8] max_line_length = 80 diff --git a/src/tiararodney/sekft/py.typed b/src/tiararodney/sekft/py.typed new file mode 100644 index 0000000..e69de29 From 9397280e6fc6e8966a3920b4847e632855fe0028 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 14:03:52 +0200 Subject: [PATCH 41/64] refactor: annotate the trainer modules under mypy strict --- src/tiararodney/sekft/eval.py | 11 +++++++---- src/tiararodney/sekft/resident.py | 30 ++++++++++++++++-------------- src/tiararodney/sekft/sft.py | 12 +++++++----- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/tiararodney/sekft/eval.py b/src/tiararodney/sekft/eval.py index 4438134..59f5bfe 100644 --- a/src/tiararodney/sekft/eval.py +++ b/src/tiararodney/sekft/eval.py @@ -20,7 +20,9 @@ from __future__ import annotations import argparse import json +from collections.abc import Callable from pathlib import Path +from typing import Any from tiararodney.posix_sdc.factory.dashdocker import DashDocker, available from tiararodney.posix_sdc.factory.rollout import rollout @@ -30,7 +32,7 @@ from .sft import normalize_for_template def make_local_operator(base: str, adapter: str, max_new_tokens: int = 64, - temperature: float = 0.7): + temperature: float = 0.7) -> Callable[[list[dict[str, str]]], str]: """A ``messages -> command`` callable backed by base + LoRA adapter. Renders the conversation exactly as the model was trained, appends the @@ -46,7 +48,7 @@ def make_local_operator(base: str, adapter: str, max_new_tokens: int = 64, model = PeftModel.from_pretrained(model, adapter) model.eval() - def operator(messages): + def operator(messages: list[dict[str, str]]) -> str: msgs = normalize_for_template(messages) ids = tok.apply_chat_template( msgs, add_generation_prompt=True, return_tensors="pt").to(model.device) @@ -55,13 +57,14 @@ def make_local_operator(base: str, adapter: str, max_new_tokens: int = 64, ids, max_new_tokens=max_new_tokens, do_sample=temperature > 0, temperature=max(temperature, 1e-2), eos_token_id=tok.eos_token_id, pad_token_id=tok.eos_token_id) - return tok.decode(out[0][ids.shape[1]:], skip_special_tokens=True).strip() + text: str = tok.decode(out[0][ids.shape[1]:], skip_special_tokens=True).strip() + return text return operator def evaluate(base: str, adapter: str, scenarios_dir: Path, n: int, - max_steps: int, temperature: float) -> dict: + max_steps: int, temperature: float) -> dict[str, Any]: if not available(): raise SystemExit("sekft-dash image unavailable; `docker build -t sekft-dash .`") operator = make_local_operator(base, adapter, temperature=temperature) diff --git a/src/tiararodney/sekft/resident.py b/src/tiararodney/sekft/resident.py index c295d48..3b0b8e0 100644 --- a/src/tiararodney/sekft/resident.py +++ b/src/tiararodney/sekft/resident.py @@ -23,6 +23,7 @@ import argparse import gc import json from pathlib import Path +from typing import Any import torch from datasets import Dataset @@ -68,7 +69,7 @@ class Resident: # -- build masked rows from kept trajectories -------------------------- - def _rows(self, data_dir: Path, max_len: int) -> list[dict]: + def _rows(self, data_dir: Path, max_len: int) -> list[dict[str, list[Any]]]: rows = [] for turns in iter_keepers(data_dir): ex = build_masked_example(turns, self.tok) @@ -83,8 +84,8 @@ class Resident: def fit(self, data_dir: str, out: str, lora_r: int = 16, lr: float = 2e-4, epochs: float = 3.0, batch: int = 1, accum: int = 8, max_len: int = 4096) -> Path: - data_dir, out = Path(data_dir).expanduser(), Path(out).expanduser() - ds = Dataset.from_list(self._rows(data_dir, max_len)) + ddir, odir = Path(data_dir).expanduser(), Path(out).expanduser() + ds = Dataset.from_list(self._rows(ddir, max_len)) if not self.load_4bit: self.base.gradient_checkpointing_enable() model = get_peft_model(self.base, LoraConfig( @@ -92,31 +93,31 @@ class Resident: task_type="CAUSAL_LM", target_modules=LORA_TARGETS)) model.print_trainable_parameters() args = TrainingArguments( - output_dir=str(out), per_device_train_batch_size=batch, + output_dir=str(odir), per_device_train_batch_size=batch, gradient_accumulation_steps=accum, num_train_epochs=epochs, learning_rate=lr, fp16=True, logging_steps=1, save_strategy="no", - report_to=["tensorboard"], logging_dir=str(out / "runs"), + report_to=["tensorboard"], logging_dir=str(odir / "runs"), remove_unused_columns=False, warmup_ratio=0.03) tr = Trainer(model=model, args=args, train_dataset=ds, data_collator=DataCollatorForSeq2Seq( self.tok, padding=True, label_pad_token_id=-100)) tr.train() - out.mkdir(parents=True, exist_ok=True) - model.save_pretrained(str(out)) - self.tok.save_pretrained(str(out)) - (out / "log_history.jsonl").write_text( + odir.mkdir(parents=True, exist_ok=True) + model.save_pretrained(str(odir)) + self.tok.save_pretrained(str(odir)) + (odir / "log_history.jsonl").write_text( "\n".join(json.dumps(r) for r in tr.state.log_history)) losses = [h["loss"] for h in tr.state.log_history if "loss" in h] - print(f"[resident] fit -> {out} final loss {losses[-1] if losses else '?'}") + print(f"[resident] fit -> {odir} final loss {losses[-1] if losses else '?'}") self.base = model.unload() # strip LoRA, restore resident base del model, tr, ds _free() - return out + return odir # -- behavioural eval of a saved adapter ------------------------------- def evaluate(self, adapter: str, scenarios_dir: str, n: int = 10, - max_steps: int = 30, temperature: float = 0.7) -> dict: + max_steps: int = 30, temperature: float = 0.7) -> dict[str, Any]: from tiararodney.posix_sdc.factory.dashdocker import DashDocker, available from tiararodney.posix_sdc.factory.rollout import rollout from tiararodney.posix_sdc.schema import Scenario @@ -130,7 +131,7 @@ class Resident: pm = self.base pm.eval() - def operator(messages): + def operator(messages: list[dict[str, str]]) -> str: msgs = normalize_for_template(messages) ids = self.tok.apply_chat_template( msgs, add_generation_prompt=True, return_tensors="pt").to(pm.device) @@ -139,7 +140,8 @@ class Resident: temperature=max(temperature, 1e-2), eos_token_id=self.tok.eos_token_id, pad_token_id=self.tok.eos_token_id) - return self.tok.decode(o[0][ids.shape[1]:], skip_special_tokens=True).strip() + text: str = self.tok.decode(o[0][ids.shape[1]:], skip_special_tokens=True).strip() + return text backend = DashDocker() rows = [] diff --git a/src/tiararodney/sekft/sft.py b/src/tiararodney/sekft/sft.py index 5ac8633..0716f69 100644 --- a/src/tiararodney/sekft/sft.py +++ b/src/tiararodney/sekft/sft.py @@ -28,9 +28,11 @@ from __future__ import annotations import argparse import json +from collections.abc import Iterator from pathlib import Path +from typing import Any -def normalize_for_template(messages: list[dict]) -> list[dict]: +def normalize_for_template(messages: list[dict[str, str]]) -> list[dict[str, str]]: """Canonicalise a trajectory for instruct chat templates that have no system role and require strict user/assistant alternation (Mistral and friends): treat ``system`` as ``user``, then merge consecutive same-role turns by @@ -42,7 +44,7 @@ def normalize_for_template(messages: list[dict]) -> list[dict]: The serving side MUST apply the same canonicalisation, or train and serve diverge again. """ - out: list[dict] = [] + out: list[dict[str, str]] = [] for m in messages: role = "user" if m["role"] == "system" else m["role"] if out and out[-1]["role"] == role: @@ -52,7 +54,7 @@ def normalize_for_template(messages: list[dict]) -> list[dict]: return out -def build_masked_example(messages: list[dict], tokenizer) -> dict: +def build_masked_example(messages: list[dict[str, str]], tokenizer: Any) -> dict[str, list[Any]]: """Tokenize a trajectory with the tokenizer's OWN chat template and build an assistant-only loss mask. @@ -81,7 +83,7 @@ def build_masked_example(messages: list[dict], tokenizer) -> dict: return {"input_ids": ids, "attention_mask": [1] * len(ids), "labels": labels} -def iter_keepers(data_dir: Path): +def iter_keepers(data_dir: Path) -> Iterator[list[dict[str, str]]]: """Yield ``turns`` (message lists) from trajectory JSONs marked keep.""" for f in sorted(data_dir.glob("*.json")): d = json.loads(f.read_text()) @@ -89,7 +91,7 @@ def iter_keepers(data_dir: Path): yield d["turns"] -def mask_stats(example: dict) -> tuple[int, int]: +def mask_stats(example: dict[str, list[Any]]) -> tuple[int, int]: """(trained tokens, total tokens) for an example.""" trained = sum(1 for x in example["labels"] if x != -100) return trained, len(example["labels"]) From 64020c321d7c1c8d7683db995cf5a9e74fd845da Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 14:03:52 +0200 Subject: [PATCH 42/64] test: annotate the sft test helpers --- tests/unit/test_sft.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_sft.py b/tests/unit/test_sft.py index f2e84b4..d24eef0 100644 --- a/tests/unit/test_sft.py +++ b/tests/unit/test_sft.py @@ -15,7 +15,7 @@ class FakeTok: """Additive chat template: each turn renders to `` tokens... ``; the generation prompt appends ````.""" - def apply_chat_template(self, msgs: list[dict], add_generation_prompt: bool = False, + def apply_chat_template(self, msgs: list[dict[str, str]], add_generation_prompt: bool = False, return_tensors: Any = None) -> list[str]: toks: list[str] = [] for m in msgs: @@ -65,7 +65,7 @@ def test_mask_trains_assistant_turns_only() -> None: def test_mask_raises_on_non_additive_template() -> None: class BadTok: - def apply_chat_template(self, msgs: list[dict], add_generation_prompt: bool = False, + def apply_chat_template(self, msgs: list[dict[str, str]], add_generation_prompt: bool = False, return_tensors: Any = None) -> list[int]: return list(range(len(msgs), 0, -1)) # reversed: prefixes do not nest From 90e0ebbb45435559435eae172311cf9a1d139763 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 14:03:54 +0200 Subject: [PATCH 43/64] todo(9): done mypy --strict src tests passes (no issues, 6 files); ML/posix-sdc imports ignored via overrides; sft/eval/resident/tests fully annotated; mypy is a declared dev dep; py.typed ships in the package. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 3ed93ce..acb41d8 100644 --- a/TODO +++ b/TODO @@ -140,7 +140,7 @@ Content-Type: application/issue ID: 9 Type: feature Title: Type-check the package under mypy strict -Status: in-progress +Status: done Priority: medium Created: 2026-06-17 Module: sekft From 814261dc561495647aac5e1fa72b9ac2e1908376 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 23:43:08 +0200 Subject: [PATCH 44/64] todo(10): open --- TODO | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/TODO b/TODO index acb41d8..78e0ff3 100644 --- a/TODO +++ b/TODO @@ -149,3 +149,26 @@ Description: Make the lint env honestly pass: add mypy as a dev dependency, ignore_missing_imports for the ML libs, fully annotate eval/resident/sft (including the inner operator callables), and ship a py.typed marker so the Typing::Typed claim is real. + +--ISSUE +Content-Type: application/issue +ID: 10 +Type: feature +Title: structured logging for the trainer (sft) +Status: open +Priority: medium +Created: 2026-06-17 +Module: sekft +Relationships: +Description: The trainer is nearly silent: outside an example count and a save + line it prints nothing through tokenizer load, the ~14GB base-model + load, example building, and the whole training loop, and + trajectories dropped for exceeding --max-len or having an empty + loss mask vanish without a trace. Add a small shared logging setup + (_log.py, stderr so stdout stays clean for results) and a module + logger; give sekft-train -v/--verbose and -q/--quiet. Log the run + config and each phase, report dataset accounting (keepers -> + usable, with counts dropped for length / empty-mask and a warning + when any are dropped), and raise transformers' verbosity during + training so the per-step curve shows. Apply to train() and + inspect(). From 47b84a0dcea5af458d7ae3d8e2e22e274342aa18 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 23:43:21 +0200 Subject: [PATCH 45/64] todo(10): in-progress sekft-train logs run config and each phase (tokenizer load, model load, training, save) via a module logger to stderr; -v/--verbose and -q/--quiet control level; dataset accounting reports keepers->usable with counts dropped for length and empty-mask and warns when any are dropped; transformers verbosity raised so the per-step curve shows during training; inspect() logs likewise; existing sft tests stay green; mypy strict clean. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 78e0ff3..7913b1f 100644 --- a/TODO +++ b/TODO @@ -155,7 +155,7 @@ Content-Type: application/issue ID: 10 Type: feature Title: structured logging for the trainer (sft) -Status: open +Status: in-progress Priority: medium Created: 2026-06-17 Module: sekft From e4e88c5c18feae444c4ef8ba606ca20dfad26c5d Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 23:47:42 +0200 Subject: [PATCH 46/64] feat(10): structured logging for the trainer The trainer was nearly silent: outside an example count and a save line it printed nothing through tokenizer load, the base-model load, example building, or the training loop, and trajectories dropped for length or empty mask vanished without a trace. Add _log.py (a shared stderr logging setup so stdout stays clean for results) and a module logger. sekft-train gains -v/--verbose and -q/--quiet. Log the run config and each phase; report dataset accounting (keepers -> usable, with counts dropped for over-length and empty-mask and a warning when any are dropped); inside the build loop, a per-trajectory debug line and a progress line every 100; raise transformers' verbosity during training so the per-step curve shows. Prints in train() and inspect() are routed through the logger. --- src/tiararodney/sekft/_log.py | 20 ++++++++++++ src/tiararodney/sekft/sft.py | 61 ++++++++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 src/tiararodney/sekft/_log.py diff --git a/src/tiararodney/sekft/_log.py b/src/tiararodney/sekft/_log.py new file mode 100644 index 0000000..8cb4d4f --- /dev/null +++ b/src/tiararodney/sekft/_log.py @@ -0,0 +1,20 @@ +"""Console logging setup shared by the sekft entry points. + +Logs go to stderr so stdout stays clean for a command's actual output (metrics +JSON, a path a caller might capture). Call :func:`setup` once at the top of a +``main()``; modules then log through ``logging.getLogger("sekft.")``. +""" +from __future__ import annotations + +import logging + + +def setup(verbose: bool = False, quiet: bool = False) -> None: + """Configure root logging to stderr. ``quiet`` shows warnings and worse, + ``verbose`` adds debug; the default is info.""" + level = logging.WARNING if quiet else logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)-5s %(name)s %(message)s", + datefmt="%H:%M:%S", + ) diff --git a/src/tiararodney/sekft/sft.py b/src/tiararodney/sekft/sft.py index 0716f69..6acfd0f 100644 --- a/src/tiararodney/sekft/sft.py +++ b/src/tiararodney/sekft/sft.py @@ -28,10 +28,16 @@ from __future__ import annotations import argparse import json +import logging from collections.abc import Iterator from pathlib import Path from typing import Any +from ._log import setup as _setup_logging + +log = logging.getLogger("sekft.train") + + def normalize_for_template(messages: list[dict[str, str]]) -> list[dict[str, str]]: """Canonicalise a trajectory for instruct chat templates that have no system role and require strict user/assistant alternation (Mistral and friends): @@ -109,21 +115,44 @@ def train(data_dir: Path, base: str, out: Path, epochs: float, lr: float, from peft import LoraConfig, get_peft_model from transformers import (AutoModelForCausalLM, AutoTokenizer, DataCollatorForSeq2Seq, Trainer, TrainingArguments) + from transformers.utils import logging as hf_logging + # Surface the Trainer's own per-step curve (loss/lr/grad_norm); it is at + # WARNING by default, which is most of why training looks silent. + hf_logging.set_verbosity_info() + + log.info("base=%s data=%s out=%s", base, data_dir, out) + log.info("loading tokenizer: %s", base) tok = AutoTokenizer.from_pretrained(base) if tok.pad_token is None: tok.pad_token = tok.eos_token - rows = [] + log.info("building masked examples from %s ...", data_dir) + rows: list[dict[str, list[Any]]] = [] + n_seen = n_long = n_empty = 0 for turns in iter_keepers(data_dir): + n_seen += 1 ex = build_masked_example(turns, tok) - if len(ex["input_ids"]) <= max_len and any(l != -100 for l in ex["labels"]): - rows.append(ex) + log.debug(" trajectory %d: %d turns -> %d tokens, %d trained", + n_seen, len(turns), len(ex["input_ids"]), mask_stats(ex)[0]) + if n_seen % 100 == 0: + log.info(" ... %d trajectories processed, %d usable", n_seen, len(rows)) + if len(ex["input_ids"]) > max_len: + n_long += 1 + continue + if not any(l != -100 for l in ex["labels"]): + n_empty += 1 + continue + rows.append(ex) if not rows: raise SystemExit(f"no usable keeper trajectories in {data_dir}") - print(f"examples: {len(rows)}; " - f"trained/total tokens: {sum(mask_stats(r)[0] for r in rows)}" - f"/{sum(mask_stats(r)[1] for r in rows)}") + trained = sum(mask_stats(r)[0] for r in rows) + total = sum(mask_stats(r)[1] for r in rows) + log.info("dataset: %d keepers -> %d usable; %d trained / %d tokens (%.1f%% assistant)", + n_seen, len(rows), trained, total, 100 * trained / total) + if n_long or n_empty: + log.warning("dropped %d trajectories: %d over --max-len %d, %d empty-mask", + n_long + n_empty, n_long, max_len, n_empty) ds = Dataset.from_list(rows) # 4-bit (QLoRA) shrinks the base from ~14 GB to ~4 GB to move across the @@ -136,6 +165,8 @@ def train(data_dir: Path, base: str, out: Path, epochs: float, lr: float, load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True, ) + log.info("loading base model: %s (%s)", base, + "4-bit QLoRA" if load_4bit else "fp16") model = AutoModelForCausalLM.from_pretrained( base, dtype=torch.float16, quantization_config=quant) if load_4bit: @@ -148,7 +179,9 @@ def train(data_dir: Path, base: str, out: Path, epochs: float, lr: float, r=lora_r, lora_alpha=lora_r * 2, lora_dropout=0.05, task_type="CAUSAL_LM", target_modules=["q_proj", "k_proj", "v_proj", "o_proj"], )) - model.print_trainable_parameters() + n_train, n_all = model.get_nb_trainable_parameters() + log.info("LoRA r=%d: %d trainable / %d params (%.3f%%)", + lora_r, n_train, n_all, 100 * n_train / n_all) args = TrainingArguments( output_dir=str(out), per_device_train_batch_size=batch, @@ -161,18 +194,21 @@ def train(data_dir: Path, base: str, out: Path, epochs: float, lr: float, model=model, args=args, train_dataset=ds, data_collator=DataCollatorForSeq2Seq(tok, padding=True, label_pad_token_id=-100), ) + log.info("training: %g epochs, lr=%g, batch=%d x accum=%d (effective %d), max_len=%d", + epochs, lr, batch, accum, batch * accum, max_len) trainer.train() model.save_pretrained(str(out)) tok.save_pretrained(str(out)) # durable, greppable record of the curve (loss/lr/grad_norm per step). (out / "log_history.jsonl").write_text( "\n".join(json.dumps(r) for r in trainer.state.log_history)) - print(f"saved LoRA adapter + log_history.jsonl -> {out} " - f"(tensorboard: --logdir {out / 'runs'})") + log.info("saved LoRA adapter + log_history.jsonl -> %s (tensorboard: --logdir %s)", + out, out / "runs") def inspect(data_dir: Path, base: str) -> None: from transformers import AutoTokenizer + log.info("loading tokenizer: %s", base) tok = AutoTokenizer.from_pretrained(base) n = tt = tr = 0 for turns in iter_keepers(data_dir): @@ -181,8 +217,8 @@ def inspect(data_dir: Path, base: str) -> None: tr += t; tt += total; n += 1 if not n: raise SystemExit(f"no keeper trajectories in {data_dir}") - print(f"{n} keeper trajectories; {tr}/{tt} tokens trained " - f"({100*tr/tt:.1f}% assistant, rest masked)") + log.info("%d keeper trajectories; %d/%d tokens trained (%.1f%% assistant, rest masked)", + n, tr, tt, 100 * tr / tt) def main() -> None: @@ -199,7 +235,10 @@ def main() -> None: ap.add_argument("--lora-r", type=int, default=16) ap.add_argument("--load-4bit", action="store_true", help="QLoRA: load base in 4-bit (less to move over the link, less VRAM)") + ap.add_argument("-v", "--verbose", action="store_true", help="debug-level logging") + ap.add_argument("-q", "--quiet", action="store_true", help="warnings and errors only") ns = ap.parse_args() + _setup_logging(verbose=ns.verbose, quiet=ns.quiet) if ns.inspect: inspect(ns.data, ns.base) else: From 7673e47002b7b266b2cfb0d44b2f0798eb73d517 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 23:47:52 +0200 Subject: [PATCH 47/64] todo(10): done sekft-train logs run config and each phase (tokenizer/model load, build, training, save) via a sekft.train logger to stderr; -v/--verbose and -q/--quiet added; dataset accounting reports keepers->usable with over-length/empty-mask drop counts and a warning; build loop logs per-trajectory (debug) and every 100 (info); transformers verbosity raised so the per-step curve shows; inspect() logs likewise; 6 tests pass; mypy strict clean on sft.py + _log.py (verified loop logging fires with FakeTok). No submodule changes. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 7913b1f..12b64a3 100644 --- a/TODO +++ b/TODO @@ -155,7 +155,7 @@ Content-Type: application/issue ID: 10 Type: feature Title: structured logging for the trainer (sft) -Status: in-progress +Status: done Priority: medium Created: 2026-06-17 Module: sekft From 299b2ce4887086c1d018eda7220c57f3f60458cf Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 23:48:21 +0200 Subject: [PATCH 48/64] todo(11): open --- TODO | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/TODO b/TODO index 12b64a3..ecdb34d 100644 --- a/TODO +++ b/TODO @@ -172,3 +172,22 @@ Description: The trainer is nearly silent: outside an example count and a save when any are dropped), and raise transformers' verbosity during training so the per-step curve shows. Apply to train() and inspect(). + +--ISSUE +Content-Type: application/issue +ID: 11 +Type: bugfix +Title: operate_rate can sum a None (eval + resident) +Status: open +Priority: medium +Created: 2026-06-17 +Module: sekft +Relationships: +Description: operate_rate computes sum(t.steps > 0 and t.meta.get('clean') for t + in rows). The 'and' yields the right operand when steps>0, so if + meta lacks the 'clean' key it yields None and sum() raises + TypeError at runtime; mypy (now that posix-sdc ships py.typed and + Trajectory is typed) flags the generator item type in eval.py:83 + and resident.py:157. Wrap the predicate in bool() so it counts + trajectories that operated and are clean, fixing both the type + error and the latent crash. From 637b746d1d4ff346228370683ba8b2336991e7d3 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 23:48:29 +0200 Subject: [PATCH 49/64] todo(11): in-progress operate_rate in eval.py and resident.py wraps the steps>0 and meta.get('clean') predicate in bool() so the sum counts cleanly-operated trajectories without summing None; mypy strict clean across the whole package (5 files); tests pass. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index ecdb34d..911e3b1 100644 --- a/TODO +++ b/TODO @@ -178,7 +178,7 @@ Content-Type: application/issue ID: 11 Type: bugfix Title: operate_rate can sum a None (eval + resident) -Status: open +Status: in-progress Priority: medium Created: 2026-06-17 Module: sekft From 157bb4955dc45bbe7de27dd3a71366d1ab228174 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 23:49:26 +0200 Subject: [PATCH 50/64] bugfix(11): operate_rate must not sum a None sum(t.steps > 0 and t.meta.get("clean") for t in rows) yields the right operand of `and` when steps>0, so a trajectory whose meta lacks the "clean" key contributes None and sum() raises TypeError. Wrap the predicate in bool() so it counts trajectories that operated and are clean. Surfaced by mypy once posix-sdc began shipping py.typed (Trajectory is now typed). --- src/tiararodney/sekft/eval.py | 2 +- src/tiararodney/sekft/resident.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tiararodney/sekft/eval.py b/src/tiararodney/sekft/eval.py index 59f5bfe..5d5964c 100644 --- a/src/tiararodney/sekft/eval.py +++ b/src/tiararodney/sekft/eval.py @@ -80,7 +80,7 @@ def evaluate(base: str, adapter: str, scenarios_dir: Path, n: int, d = len(rows) or 1 return { "n": len(rows), - "operate_rate": round(sum(t.steps > 0 and t.meta.get("clean") for t in rows) / d, 3), + "operate_rate": round(sum(bool(t.steps > 0 and t.meta.get("clean")) for t in rows) / d, 3), "terminate_rate": round(sum(t.terminal in ("exit", "panic") for t in rows) / d, 3), "verified_rate": round(sum(t.verified for t in rows) / d, 3), "clean_rate": round(sum(t.keep for t in rows) / d, 3), diff --git a/src/tiararodney/sekft/resident.py b/src/tiararodney/sekft/resident.py index 3b0b8e0..2d7bc3b 100644 --- a/src/tiararodney/sekft/resident.py +++ b/src/tiararodney/sekft/resident.py @@ -154,7 +154,7 @@ class Resident: d = len(rows) or 1 m = { "n": len(rows), - "operate_rate": round(sum(t.steps > 0 and t.meta.get("clean") for t in rows) / d, 3), + "operate_rate": round(sum(bool(t.steps > 0 and t.meta.get("clean")) for t in rows) / d, 3), "terminate_rate": round(sum(t.terminal in ("exit", "panic") for t in rows) / d, 3), "verified_rate": round(sum(t.verified for t in rows) / d, 3), "clean_rate": round(sum(t.keep for t in rows) / d, 3), From c6939c0a645a38909ed54a33c1b0886bd4c5cb24 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 17 Jun 2026 23:49:27 +0200 Subject: [PATCH 51/64] todo(11): done bool() wraps the operate_rate predicate in eval.py and resident.py so the sum counts cleanly-operated trajectories without summing None; mypy strict clean across the whole package (5 files); 6 tests pass. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 911e3b1..f4392bf 100644 --- a/TODO +++ b/TODO @@ -178,7 +178,7 @@ Content-Type: application/issue ID: 11 Type: bugfix Title: operate_rate can sum a None (eval + resident) -Status: in-progress +Status: done Priority: medium Created: 2026-06-17 Module: sekft From d47ba8a56eb641efc5012022f9c8bb298e9d3a05 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:03:10 +0200 Subject: [PATCH 52/64] todo(12): open --- TODO | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/TODO b/TODO index f4392bf..981dfa7 100644 --- a/TODO +++ b/TODO @@ -191,3 +191,26 @@ Description: operate_rate computes sum(t.steps > 0 and t.meta.get('clean') for t and resident.py:157. Wrap the predicate in bool() so it counts trajectories that operated and are clean, fixing both the type error and the latent crash. + +--ISSUE +Content-Type: application/issue +ID: 12 +Type: feature +Title: load training data from a raw dir, a curated jsonl, or the Hub +Status: open +Priority: medium +Created: 2026-06-17 +Module: sekft +Relationships: +Description: iter_keepers reads only raw per-trajectory .json - one of three + input shapes the trainer should accept. Add load_turns(data, hub, + revision) that yields assistant-bearing turns from: a directory of + raw rollout .json (keep-filtered, today's iter_keepers); a curated + .jsonl corpus file (already keep-filtered, yield turns per line); + or the published corpus via posix-sdc's load_trajectories (local + data/ in a checkout, else the Hub). sekft-train gains --hub and + --revision; --data dispatches by dir-vs-.jsonl. Raw-rollout reading + stays sekft-local; curated+Hub reuse posix-sdc's loader (imported + lazily so the trainer needs neither posix-sdc nor huggingface_hub + for the raw/jsonl paths). Unit tests for the raw-dir and jsonl + dispatch. From d78a8028d2eeabc23bba4c7dc6ca6134d45c46ec Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:03:19 +0200 Subject: [PATCH 53/64] todo(12): in-progress sft.load_turns(data, hub, revision) yields turns from a raw rollout dir (keep-filtered), a curated .jsonl file, or the published corpus via posix-sdc load_trajectories (Hub fallback), imported lazily; sekft-train gains --hub and --revision and dispatches --data by dir-vs-.jsonl; train() and inspect() use it; unit tests cover the raw-dir and jsonl paths; existing tests stay green; mypy strict clean. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 981dfa7..bf353f1 100644 --- a/TODO +++ b/TODO @@ -197,7 +197,7 @@ Content-Type: application/issue ID: 12 Type: feature Title: load training data from a raw dir, a curated jsonl, or the Hub -Status: open +Status: in-progress Priority: medium Created: 2026-06-17 Module: sekft From 414e963825e597e3ac7374f56a61abb50b3d08d9 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:05:27 +0200 Subject: [PATCH 54/64] feat(12): load training data from a raw dir, a curated jsonl, or the Hub iter_keepers read only raw per-trajectory .json -- one of three input shapes. Add load_turns(data, hub, revision) yielding assistant-bearing turns from a raw rollout dir (keep-filtered), a curated .jsonl corpus (one record per line), or the published corpus via posix-sdc's load_trajectories (the in-repo data/ of a checkout, else the Hugging Face Hub). sekft-train gains --hub and --revision and dispatches --data by dir-vs-.jsonl; train() and inspect() use it. Raw-rollout reading stays sekft-local; curated + Hub reuse posix-sdc's loader, imported lazily so the raw/jsonl paths need neither posix-sdc nor huggingface_hub installed. Unit tests cover the raw-dir and jsonl dispatch. --- src/tiararodney/sekft/sft.py | 59 ++++++++++++++++++++++++++++++------ tests/unit/test_load.py | 35 +++++++++++++++++++++ 2 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_load.py diff --git a/src/tiararodney/sekft/sft.py b/src/tiararodney/sekft/sft.py index 6acfd0f..ce0e478 100644 --- a/src/tiararodney/sekft/sft.py +++ b/src/tiararodney/sekft/sft.py @@ -18,6 +18,8 @@ canonicalisation must run on the serving side. Everything else is standard causal-LM SFT with an assistant-only loss mask. sekft-train --data ./trajectories --base --out ./ckpt + sekft-train --data corpus.jsonl --base # a curated .jsonl corpus + sekft-train --hub --base # the published corpus (Hub) sekft-train --data ./trajectories --base --inspect # mask stats, no training Training needs torch + transformers + peft (a GPU box). ``--inspect`` and the @@ -90,13 +92,42 @@ def build_masked_example(messages: list[dict[str, str]], tokenizer: Any) -> dict def iter_keepers(data_dir: Path) -> Iterator[list[dict[str, str]]]: - """Yield ``turns`` (message lists) from trajectory JSONs marked keep.""" + """Yield ``turns`` (message lists) from raw rollout JSONs marked keep.""" for f in sorted(data_dir.glob("*.json")): d = json.loads(f.read_text()) if d.get("keep"): yield d["turns"] +def load_turns(data: Path, hub: bool = False, + revision: str | None = None) -> Iterator[list[dict[str, str]]]: + """Yield assistant-bearing ``turns`` from one of three sources: + + - ``--hub``: the published corpus via posix-sdc's ``load_trajectories`` (the + in-repo ``data/`` of a posix-sdc checkout, else the Hugging Face Hub); + - ``data`` a ``.jsonl`` file: a curated corpus, already keep-filtered, one + record per line; + - ``data`` a directory: raw rollout ``.json`` (keep-filtered here). + + posix-sdc is imported lazily, so the raw-dir and ``.jsonl`` paths need + neither posix-sdc nor huggingface_hub installed. + """ + if hub: + from tiararodney.posix_sdc import load_trajectories + for r in load_trajectories(revision=revision): + yield r["turns"] + elif data.is_dir(): + yield from iter_keepers(data) + elif data.suffix == ".jsonl": + with open(data) as fh: + for line in fh: + if line.strip(): + yield json.loads(line)["turns"] + else: + raise SystemExit( + f"--data must be a rollout directory or a .jsonl corpus (got {data})") + + def mask_stats(example: dict[str, list[Any]]) -> tuple[int, int]: """(trained tokens, total tokens) for an example.""" trained = sum(1 for x in example["labels"] if x != -100) @@ -109,7 +140,8 @@ def mask_stats(example: dict[str, list[Any]]) -> tuple[int, int]: def train(data_dir: Path, base: str, out: Path, epochs: float, lr: float, batch: int, accum: int, max_len: int, lora_r: int, - load_4bit: bool = False) -> None: + load_4bit: bool = False, hub: bool = False, + revision: str | None = None) -> None: import torch from datasets import Dataset from peft import LoraConfig, get_peft_model @@ -121,16 +153,17 @@ def train(data_dir: Path, base: str, out: Path, epochs: float, lr: float, # WARNING by default, which is most of why training looks silent. hf_logging.set_verbosity_info() - log.info("base=%s data=%s out=%s", base, data_dir, out) + source = "hub" if hub else data_dir + log.info("base=%s data=%s out=%s", base, source, out) log.info("loading tokenizer: %s", base) tok = AutoTokenizer.from_pretrained(base) if tok.pad_token is None: tok.pad_token = tok.eos_token - log.info("building masked examples from %s ...", data_dir) + log.info("building masked examples from %s ...", source) rows: list[dict[str, list[Any]]] = [] n_seen = n_long = n_empty = 0 - for turns in iter_keepers(data_dir): + for turns in load_turns(data_dir, hub=hub, revision=revision): n_seen += 1 ex = build_masked_example(turns, tok) log.debug(" trajectory %d: %d turns -> %d tokens, %d trained", @@ -206,12 +239,13 @@ def train(data_dir: Path, base: str, out: Path, epochs: float, lr: float, out, out / "runs") -def inspect(data_dir: Path, base: str) -> None: +def inspect(data_dir: Path, base: str, hub: bool = False, + revision: str | None = None) -> None: from transformers import AutoTokenizer log.info("loading tokenizer: %s", base) tok = AutoTokenizer.from_pretrained(base) n = tt = tr = 0 - for turns in iter_keepers(data_dir): + for turns in load_turns(data_dir, hub=hub, revision=revision): ex = build_masked_example(turns, tok) t, total = mask_stats(ex) tr += t; tt += total; n += 1 @@ -223,7 +257,12 @@ def inspect(data_dir: Path, base: str) -> None: def main() -> None: ap = argparse.ArgumentParser(description="SFT a model on shell trajectories.") - ap.add_argument("--data", type=Path, default=Path("./trajectories")) + ap.add_argument("--data", type=Path, default=Path("./trajectories"), + help="a raw rollout dir or a curated .jsonl corpus") + ap.add_argument("--hub", action="store_true", + help="load the published corpus via posix-sdc (Hub); ignores --data") + ap.add_argument("--revision", default=None, + help="dataset revision/tag to pin when using --hub") ap.add_argument("--base", required=True, help="HF model id or local dir") ap.add_argument("--out", type=Path, default=Path("./ckpt")) ap.add_argument("--inspect", action="store_true", help="mask stats only, no training") @@ -240,10 +279,10 @@ def main() -> None: ns = ap.parse_args() _setup_logging(verbose=ns.verbose, quiet=ns.quiet) if ns.inspect: - inspect(ns.data, ns.base) + inspect(ns.data, ns.base, hub=ns.hub, revision=ns.revision) else: train(ns.data, ns.base, ns.out, ns.epochs, ns.lr, ns.batch, ns.accum, - ns.max_len, ns.lora_r, ns.load_4bit) + ns.max_len, ns.lora_r, ns.load_4bit, hub=ns.hub, revision=ns.revision) if __name__ == "__main__": diff --git a/tests/unit/test_load.py b/tests/unit/test_load.py new file mode 100644 index 0000000..4106da6 --- /dev/null +++ b/tests/unit/test_load.py @@ -0,0 +1,35 @@ +"""Unit tests for the trainer's three-source data loader (raw dir / curated +jsonl). The Hub path delegates to posix-sdc and is covered there.""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tiararodney.sekft import sft + + +def test_load_turns_from_raw_dir(tmp_path: Path) -> None: + (tmp_path / "a.json").write_text(json.dumps( + {"keep": True, "turns": [{"role": "assistant", "content": "ls"}]})) + (tmp_path / "b.json").write_text(json.dumps( # not kept -> excluded + {"keep": False, "turns": [{"role": "assistant", "content": "rm -rf /"}]})) + got = list(sft.load_turns(tmp_path)) + assert len(got) == 1 + assert got[0][0]["content"] == "ls" + + +def test_load_turns_from_jsonl(tmp_path: Path) -> None: + f = tmp_path / "corpus.jsonl" + f.write_text("\n".join(json.dumps({"turns": [{"role": "assistant", "content": c}]}) + for c in ("ls", "cat x")) + "\n") + got = list(sft.load_turns(f)) + assert [t[0]["content"] for t in got] == ["ls", "cat x"] + + +def test_load_turns_rejects_other_paths(tmp_path: Path) -> None: + bad = tmp_path / "notes.txt" + bad.write_text("hi") + with pytest.raises(SystemExit): + list(sft.load_turns(bad)) From 15e598bda18dab3584bbf7800842ab9e2c80c34f Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:05:36 +0200 Subject: [PATCH 55/64] todo(12): done sft.load_turns(data, hub, revision) yields turns from a raw rollout dir (keep-filtered), a curated .jsonl file, or the published corpus via posix-sdc load_trajectories (lazy import; Hub fallback) - verified the hub path yields all 787 trajectories via the editable checkout's data/; sekft-train gains --hub/--revision and dispatches --data by dir-vs-.jsonl; train()+inspect() use it; 9 tests pass (3 new for raw-dir/jsonl/reject); mypy strict clean (5 files). No submodule changes. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index bf353f1..56bc0d4 100644 --- a/TODO +++ b/TODO @@ -197,7 +197,7 @@ Content-Type: application/issue ID: 12 Type: feature Title: load training data from a raw dir, a curated jsonl, or the Hub -Status: in-progress +Status: done Priority: medium Created: 2026-06-17 Module: sekft From a37e52aacf2fc28b65f7bcd8154782f41565f321 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:17:06 +0200 Subject: [PATCH 56/64] todo(13): open --- TODO | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/TODO b/TODO index 56bc0d4..ab83c93 100644 --- a/TODO +++ b/TODO @@ -214,3 +214,20 @@ Description: iter_keepers reads only raw per-trajectory .json - one of three lazily so the trainer needs neither posix-sdc nor huggingface_hub for the raw/jsonl paths). Unit tests for the raw-dir and jsonl dispatch. + +--ISSUE +Content-Type: application/issue +ID: 13 +Type: feature +Title: reference posix-sdc three ways for seamless multi-machine dev +Status: open +Priority: medium +Created: 2026-06-17 +Module: sekft +Relationships: +Description: Wire the posix-sdc dependency as a triplet: the abstract + posix-sdc[hub] in pyproject (so the trainer's --hub path can reach + the Hub via huggingface_hub); the published wheel from the private + index in Pipfile [packages]; the git develop branch in Pipfile + [dev-packages] for develop-time. Commit Pipfile.lock so the + dependency surface and lock land together. From b7878251ed98df7efc73661450a578a824215ee7 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:17:47 +0200 Subject: [PATCH 57/64] todo(13): in-progress pyproject declares posix-sdc[hub]; Pipfile [packages] pulls posix-sdc from the private index and [dev-packages] from git develop; Pipfile.lock committed and consistent; tree clean after commit. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index ab83c93..ae88ea0 100644 --- a/TODO +++ b/TODO @@ -220,7 +220,7 @@ Content-Type: application/issue ID: 13 Type: feature Title: reference posix-sdc three ways for seamless multi-machine dev -Status: open +Status: in-progress Priority: medium Created: 2026-06-17 Module: sekft From 74d9793e76534fe9840b105f3218d896570efb32 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:17:56 +0200 Subject: [PATCH 58/64] feat(13): reference posix-sdc three ways for seamless multi-machine dev Wire the posix-sdc dependency as a triplet: - pyproject declares the abstract posix-sdc[hub], so the trainer's --hub path can reach the Hub (huggingface_hub) wherever sekft is installed; - Pipfile [packages] pulls the published wheel from the private index; - Pipfile [dev-packages] pulls the git develop branch for develop-time work. Commit Pipfile.lock so the dependency surface and its lock land together. --- Pipfile | 10 +- Pipfile.lock | 1186 ++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 1194 insertions(+), 4 deletions(-) create mode 100644 Pipfile.lock diff --git a/Pipfile b/Pipfile index fffe94e..c83a380 100644 --- a/Pipfile +++ b/Pipfile @@ -3,11 +3,14 @@ url = "https://pypi.org/simple" verify_ssl = true name = "pypi" +[[source]] +url = "https://pypi.code.tiararodney.com/root/byteb4rb1e/+simple/" +verify_ssl = true +name = "pypicodetiararodney" + [packages] "tiararodney.sekft" = {file = ".", editable = true} -# Local editable override of the posix-sdc dependency declared (abstractly) in -# pyproject.toml, so the dataset/factory is developed side by side. -"tiararodney.posix-sdc" = {path = "../posix-sdc", editable = true} +"tiararodney.posix-sdc" = {version = "*", index = "pypicodetiararodney", extras= ["hub"]} [dev-packages] tox = "*" @@ -18,6 +21,7 @@ twine = "*" setuptools-scm = "~=8.2.0" pypi-attestations = "*" autopep8 = "*" +"tiararodney.posix-sdc" = {ref = "develop", git = "https://git.code.tiararodney.com/tiara/posix-sdc.git", extras = ["hub"]} [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..217e375 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,1186 @@ +{ + "_meta": { + "hash": { + "sha256": "4583c15dee320e17846aa51acd0fbca1fbdcd24118d50a34033f8c5f8b4a95ac" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + }, + { + "name": "pypicodetiararodney", + "url": "https://pypi.code.tiararodney.com/root/byteb4rb1e/+simple/", + "verify_ssl": true + } + ] + }, + "default": { + "tiararodney.posix-sdc": { + "hashes": [ + "sha256:09b091d53d662652536e1b7ea7d33231357e308b3b20e4dd6e2955545cf6ffb0", + "sha256:1026f5f7be72d76d215161b8aa525d5662451340132eefaf894e2595205ae5ef" + ], + "index": "pypicodetiararodney", + "version": "==1.2.1" + }, + "tiararodney.sekft": { + "editable": true, + "file": "." + } + }, + "develop": { + "annotated-types": { + "hashes": [ + "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", + "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89" + ], + "markers": "python_version >= '3.8'", + "version": "==0.7.0" + }, + "ast-serialize": { + "hashes": [ + "sha256:061ee58bdb52341c8201a6df41182a977736bae3b7ded87ca7176ca25a8a47ab", + "sha256:0668aa9459cfa8c9c49ddd2163ebcf43088ba045ef7492af6fe22e0098303101", + "sha256:104e4a35bd7c124173c41760ef9aaea17ddb3f86c65cb643671d59afbe3ee94c", + "sha256:143a4ef63285a075871908fda3672dc21864b83a8ec3ee12304aa3e4c5387b9a", + "sha256:16db7c62ec0b8efe1d7afd283a388d8f74f2605d56032e5a37747d2de8dba027", + "sha256:1943db345233cc7194a470f13afa9c59772c0b123dea0c9414c4d4ca54369759", + "sha256:239a4c354e8d676e9d94631d1d4a64edc6b266f86ff3a5a80aedd344f342c01d", + "sha256:2782c36237c46dd1674542f2109740ea5ea485a169bf1431939ada0434e17934", + "sha256:27d51654fc240a1e87e742d353d98eb45b75f62f129086b3596ab53df2ac2a43", + "sha256:36be371028fc1675acb38a331bde160dbab7ff907fdf00b67eb6911aa106951b", + "sha256:5499e8797edff2a9186aa313ed382c6b422e798e9332d9953badcee6e69a88f2", + "sha256:5880091bfe6f4f986f22866375c2e884843e7a0b6343ae41aeea659613d879b6", + "sha256:6848f2a093fb5548751a9a09bff8fcd229e2bbeb0e3331f391b6ae6d26cd9903", + "sha256:787baedb0262cc49e8ce37cc15c00ae818e46a165a3b36f5e21ed174998104cb", + "sha256:7d1a2de9de5be04652f0ed60738356ef94f66db37924a9499fffe98dc491aa0b", + "sha256:832d4c998e0b091fd60a6d6bceee535483c4d490de9ba85003af835225719261", + "sha256:842d1c004bb466c7df036f95fabef789570541922b10976b12f5592a69cf0b38", + "sha256:8f5c14f169eb0972c0c21bada5358b23d6047c76583b005234f865b11f1fa00a", + "sha256:92a31c9c20d25a076edaeec76b128a3535d74a24f340b9a8a7e96c9b86dc9642", + "sha256:9cc22cf0c9be65e71cf88fda130af60d61eb4a79370ad4cfe7900d48a4aa2211", + "sha256:b0c06d760909b095cc466356dfccd05a1c7233a6ca191c020dca2c6a6f16c24c", + "sha256:b15219e9cdc9f53f6f4cb51c009203507228226148c05c5e8fe451c28b435eb3", + "sha256:b54f60c1d78767a53b67eaa663f0dfac3afe606aa07f1301572f588b73d64809", + "sha256:b725026bafa801dbd7310eb13a75f0a2e370e7e51b2cb225f9d21fcfadf919ee", + "sha256:baf5eb061eb5bccade4128ad42da33787d72f6013809cd1b590376ece8b3c937", + "sha256:be5173fb66f9b49026d9d5a2ff0fc7c7009077107c0eb285b2d60fdf1fe10bd1", + "sha256:bf683d6363edf2b39eed6b6d4fe22d34b6203867a67e27134d9e2a2680c4bc4a", + "sha256:cae65289fc456fde04af979a2be09302ef5d8ab92ef23e596d6746dc267ada27", + "sha256:cf25572c526add400f26a4750dc6ce0c3bb93fc1f75e7ae0cad4ce4f2cd5c590", + "sha256:df1c00022cbbcb064bfaa505aa9c9295362443ce5dacb459d1331d3da353f887", + "sha256:e42d729ef2be96a14efbad355093284739e3670ece3e534f82cc8832790911d9", + "sha256:f66173891548c9f2726bf27957b41cabce12fa679dc6da505ddbde4d4b3b31cf", + "sha256:f8015cd071ac1339924ee2b8098c93e00e155f30a16f40ec9816fcf84f4753f6" + ], + "markers": "python_version >= '3.7'", + "version": "==0.5.0" + }, + "autopep8": { + "hashes": [ + "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758", + "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==2.3.2" + }, + "build": { + "hashes": [ + "sha256:13f3eecb844759ab66efec90ca17639bbf14dc06cb2fdf37a9010322d9c50a6f", + "sha256:302c22c3ba2a0fd5f3911918651341ebb3896176cbdec15bd421f80b1afc7647" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==1.5.0" + }, + "cachetools": { + "hashes": [ + "sha256:323dc4127934744db5b54eb4924482d7edafbf9554e820d1531c2e08c0e4ef54", + "sha256:437f55a4e0c1b01a4f3077cc470e6991d47430970e36fbcb77e2be0df4fc1cd6" + ], + "markers": "python_version >= '3.10'", + "version": "==7.1.4" + }, + "certifi": { + "hashes": [ + "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", + "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db" + ], + "markers": "python_version >= '3.7'", + "version": "==2026.6.17" + }, + "cffi": { + "hashes": [ + "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", + "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", + "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", + "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", + "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", + "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2", + "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", + "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", + "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65", + "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", + "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", + "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", + "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", + "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a", + "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", + "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", + "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", + "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", + "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", + "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", + "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", + "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", + "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", + "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", + "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165", + "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", + "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", + "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c", + "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", + "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", + "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", + "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", + "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63", + "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", + "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", + "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", + "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", + "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", + "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", + "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", + "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", + "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", + "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", + "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", + "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", + "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", + "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322", + "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", + "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", + "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", + "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", + "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", + "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", + "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", + "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", + "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", + "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", + "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", + "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", + "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", + "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9", + "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", + "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", + "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", + "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", + "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", + "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f", + "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", + "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", + "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", + "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", + "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", + "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", + "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", + "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", + "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", + "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7", + "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", + "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534", + "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", + "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", + "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", + "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", + "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf" + ], + "markers": "python_version >= '3.9'", + "version": "==2.0.0" + }, + "charset-normalizer": { + "hashes": [ + "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc", + "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", + "sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67", + "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", + "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", + "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", + "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", + "sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444", + "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153", + "sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9", + "sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01", + "sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217", + "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", + "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", + "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", + "sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83", + "sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5", + "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", + "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", + "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", + "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", + "sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42", + "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", + "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", + "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", + "sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207", + "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", + "sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734", + "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", + "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", + "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", + "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", + "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", + "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", + "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", + "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", + "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", + "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53", + "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790", + "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", + "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", + "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", + "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", + "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", + "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", + "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", + "sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776", + "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", + "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", + "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008", + "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943", + "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374", + "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", + "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", + "sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5", + "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616", + "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", + "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", + "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", + "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752", + "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", + "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", + "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", + "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", + "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", + "sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4", + "sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545", + "sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706", + "sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366", + "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", + "sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a", + "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", + "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", + "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", + "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a", + "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", + "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", + "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", + "sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319", + "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", + "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", + "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", + "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", + "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", + "sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0", + "sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686", + "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34", + "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", + "sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c", + "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1", + "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", + "sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60", + "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", + "sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274", + "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", + "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", + "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", + "sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f", + "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d", + "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", + "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", + "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393", + "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", + "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af", + "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", + "sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00", + "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", + "sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3", + "sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7", + "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", + "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", + "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", + "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8", + "sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259", + "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", + "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", + "sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30", + "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", + "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", + "sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24", + "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", + "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", + "sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc", + "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", + "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", + "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", + "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", + "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", + "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464" + ], + "markers": "python_version >= '3.7'", + "version": "==3.4.7" + }, + "colorama": { + "hashes": [ + "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", + "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==0.4.6" + }, + "cryptography": { + "hashes": [ + "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", + "sha256:09f73a725d582cef64b91281a322cd798d14a33b2b6f2b7ad9531dc336d84c02", + "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471", + "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", + "sha256:15254441469dd6bf027039453288e2072124f8b6603563f5d759e1c9b69273fa", + "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", + "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", + "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", + "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", + "sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6", + "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", + "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", + "sha256:3fd2ca57062b241c856670b073487d2e86c4637937ca5601e48f97bf8e11fc8f", + "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72", + "sha256:43c5835e2cb98c8733d86f57d6fc879b613f5c3478607281c3e36daffc6dd8a6", + "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", + "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", + "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", + "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", + "sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429", + "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", + "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", + "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", + "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265", + "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", + "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", + "sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d", + "sha256:8ace4507d1e6533c125f4fac754f8bb8b6a74c08e92179dabd7e16571a3efbf3", + "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", + "sha256:9621de99d2da096006b629979efd8ae7eb2d8b822488d0c89ee4000c306c59b1", + "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", + "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", + "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2", + "sha256:b024e784ad6c077ee0147b35ea9cbfc1e34e1fd4c1dcca214c2794d73a12df08", + "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c", + "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", + "sha256:b7a2d1a937a738a881737cec135a38bb61470589b17515b9f73f571d0ae10401", + "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", + "sha256:bd81490cd5801d755cf97bb68ac191f14b708470b1c7cf4580f669b9c9264cd8", + "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9", + "sha256:d069066deead00ac7f090be101be875a06855908f7ec004c27b8fefb4acfb411", + "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4", + "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991", + "sha256:e361afba8918070d376df76f408a4f67fec0ee9cff81a99e48fe9a233ef59e17", + "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", + "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", + "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", + "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345", + "sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46" + ], + "markers": "python_version >= '3.9' and python_full_version not in '3.9.0, 3.9.1'", + "version": "==48.0.1" + }, + "distlib": { + "hashes": [ + "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", + "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed" + ], + "version": "==0.4.3" + }, + "dnspython": { + "hashes": [ + "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", + "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f" + ], + "markers": "python_version >= '3.10'", + "version": "==2.8.0" + }, + "docutils": { + "hashes": [ + "sha256:25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea", + "sha256:746f5060322511280a1e50eb76846ed6bf2342984b2ac04dc42caa1a8d78799e" + ], + "markers": "python_version >= '3.9'", + "version": "==0.23" + }, + "email-validator": { + "hashes": [ + "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", + "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426" + ], + "markers": "python_version >= '3.8'", + "version": "==2.3.0" + }, + "filelock": { + "hashes": [ + "sha256:10cdb3656fc44541cdf30652a93fb10ec6b05325620eb316bd26893e4201538a", + "sha256:dac1648087d5115554850d113e7dd8c83ab2d38e3435dde2d4f163847e57b767" + ], + "markers": "python_version >= '3.10'", + "version": "==3.29.4" + }, + "id": { + "hashes": [ + "sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069", + "sha256:f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca" + ], + "markers": "python_version >= '3.9'", + "version": "==1.6.1" + }, + "idna": { + "hashes": [ + "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", + "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848" + ], + "markers": "python_version >= '3.9'", + "version": "==3.18" + }, + "iniconfig": { + "hashes": [ + "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", + "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12" + ], + "markers": "python_version >= '3.10'", + "version": "==2.3.0" + }, + "jaraco.classes": { + "hashes": [ + "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", + "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790" + ], + "markers": "python_version >= '3.8'", + "version": "==3.4.0" + }, + "jaraco.context": { + "hashes": [ + "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", + "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3" + ], + "markers": "python_version >= '3.10'", + "version": "==6.1.2" + }, + "jaraco.functools": { + "hashes": [ + "sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03", + "sha256:79ce39246eddbde4b3a03b77ea5f0f7878dc669b166a66cf3fa8e266aa3fa2f4" + ], + "markers": "python_version >= '3.10'", + "version": "==4.5.0" + }, + "jeepney": { + "hashes": [ + "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", + "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732" + ], + "markers": "python_version >= '3.7'", + "version": "==0.9.0" + }, + "keyring": { + "hashes": [ + "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", + "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b" + ], + "markers": "python_version >= '3.9'", + "version": "==25.7.0" + }, + "librt": { + "hashes": [ + "sha256:05fb8fb2ab90e21c8d12ea240d744ad514da9baf381ebfa70d91d20d21713175", + "sha256:070aa8c26c0a74774317a72df8851facc7f0f012a5b406557ac56992d92e1ec8", + "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1", + "sha256:0827efe7854718f04aaddf6496e96960a956e676fe1d0f04eb41511fd8ad06d5", + "sha256:0add982e0e7b9fc14cf4b33789d5f13f66581889b88c2f58099f6ce8f92617bd", + "sha256:0cad8a4d6a8ff03c9b76f9414caccd78e7cfbc8a2e12fa334d8e1d9932753783", + "sha256:0d1029d7e1ae1a7e647ed6fb5df8c4ce2dffefb7a9f5fd1376a4554d96dac09f", + "sha256:0dc56b1f8d06e60db362cc3fdae206681817f86ce4725d34511473487f12a34b", + "sha256:0ef69ac715f3cd8e5cd252cb2aebfa72c015492aacc339d5d7bf8fef3c62c677", + "sha256:11bd19822431cc21af9f27374e7ae2e58103c7d98bda823536a6c47f6bb2bb3d", + "sha256:137e79445c896a0ea7b265f52d23954e05b64222ee1af69e2cb34219067cbb67", + "sha256:140695816ddf3c86eb972981a26f35efd871c44b0c3aed44c8cd01749386617f", + "sha256:22bdf239b219d3993761a148ffa134b19e52e9989c84f845d5d7b71d70a17412", + "sha256:258d73a0aa66a055e65b2e4d1b8cdb23b9d132c5bb915d9547d804fcaed116cc", + "sha256:28edb433edde181112a908c78907af28f964eabc15f4dd16c9d66c834302677c", + "sha256:2b481d846ac894c4e8403c5fd0e87c5d11d6499e404b474602508a224ff531c8", + "sha256:2f10cf143e4a9bb0f4f5af568a00df94a2d69ef41c2579584454bb0fe5cc642c", + "sha256:32bcc918c0148eb7e3d57385125bac7e5f9e4359d05f07448b09f6f778c2f31c", + "sha256:40071fc5fe0ce8daa6de616702314a01e1250711682b0523d6ab8d4525910cb3", + "sha256:41dc19fe150b69716c8ece4f76773a9e8813fe3e35e032a58b4d46423fb8d7c0", + "sha256:461bbceede621f1ffb8839755f8663e886087ee7af16294cab7fb4d782c62eeb", + "sha256:46c60b61e308eb535fbd6fa622b1ee1bb2815691c1ad9c98bf7b84952ec3bc8d", + "sha256:4a017a95e5837dc15a8c5661d60e05daa96b90908b1aa6b7acdf443cd25c8ebd", + "sha256:4a9a237d13addb93715b6fee74023d5ee3469b53fce527626c0e088aa585805f", + "sha256:4ce1f21fbe589bc1afd7872dece84fb0e1144f794a288e58a10d2c54a55c43be", + "sha256:4e8bd98ea9c47ae90b319a087ab28dac493f1ffbc1ecd1f28fcdbf3b7e1108d1", + "sha256:4ee278c769a713638cdacd4c0436d72156e75df3ebc0166ab2b9dc43acc386c9", + "sha256:557183ddc36babe46b27dd60facbd5adb4492181a5be887587d57cda6e092f21", + "sha256:5ba067f4aadae8fda802d91d2124c90c42195ff32d9161d3549e6d05cfe26f96", + "sha256:5d63c855d86938d9de93e265c9bd8c705b51ec494de5738340ee93767a686e4b", + "sha256:5ddd17bd87b2c56ddd60e546a7984a2e64c4e8eab92fb4cf3830a48ad5469d51", + "sha256:621db29691044bdeda22e789e482e1b0f3a985d90e3426c9c6d17606416205ea", + "sha256:624a40c4a4ad7773315c287276cd024509b2c66ff5904f504bfc08d2c70293ab", + "sha256:65ac3bc20f78aa0ee5ae84baa68917f89fef4af63e941084dd019a0d0e749f0c", + "sha256:6bd72d903911d995ab666dbd1871f8b1e80925a699af8063fbf50053329fb05f", + "sha256:6bf14feb84b05ae945277395451998c89c54d0def4070eb5c08de544930b245a", + "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f", + "sha256:75672f0bc524ede266287d532d7923dbce94c7514ad07627bac3d0c6d92cc4d9", + "sha256:7753e57d6e12d019c0d8786f1c09c709f4c3fcc57c3887b24e36e6c06ec938b7", + "sha256:78dc31f7fdfe9c9d0eb0e8f42d139db230e826415bbcabd9f0e9faaaee909894", + "sha256:78fddc31cd4d3caa897ad5d31f856b1faadc9474021ad6cb182b9018793e254e", + "sha256:7a80a71e1fda83cc752a9141e87aae7fef279538597564d670e9ce513f286192", + "sha256:7aef3cf1d5af86e770ab04bfd993dfc4ae8b8c17f66fb77dd4a7d50de7bbb1a3", + "sha256:7c39513d8b7477a2e1ed8c43fc21c524e8d5a0f8d4e8b7b074dbdbe7820a08e2", + "sha256:7da327dacd7be8f8ec36547373550744a3cc0e536d54665cd83f8bcd961200e8", + "sha256:7e82e642ab0f7608ce2fe53d76ca2280a9ee33a1b06556142c7c6fe80a86fc33", + "sha256:83d3e1f72bd42f6c5c0b7daec530c3f829bd02db42c70b8ddf0c2d90a2459930", + "sha256:84308fc49423ce6475d1c5d1985cd69a8ca9f0325fc7d5f81bb690a3f3625d4e", + "sha256:88145c15c67731d54283d135b03244028c750cc9edc334a96a4f5950ebdb2884", + "sha256:8ca8aa88751a775870b764e93bad5135385f563cb8dcee399abf034ea4d3cb47", + "sha256:902e546ff044f579ff1c953ff5fce97b636fe9e3943996b2177710c6ef076f73", + "sha256:92f7ff819c197fc30473190a12c2856f325ac90aabfccbeb2072d28cc2e234e3", + "sha256:936c5995f3514a42111f20099397d8177c79b4d7e70961e396c6f5a0a3566766", + "sha256:93d95bd45b7d58343d8b90d904450a545144eec19a002511163426f8ab1fae29", + "sha256:94663a21534637f0e787ec2a2a756022df6e5b7b2335a5cdd7d8e33d68a2af89", + "sha256:96f044bb325fd9cf1a723015638c219e9143f0dfbc0ca54c565df2b7fc748b44", + "sha256:970b09f7044ea2b64c9da42fd3d335666518cfd1c6e8a182c95da73d0214b41e", + "sha256:993f028be9e96a08d31df3479ac80d99be374d17f3b78e4796b3fd3c913d4e89", + "sha256:9bc0ca6ad9381cbe8e4aa6e5726e4c80c78115a6e9723c599ed1d73e092bc49d", + "sha256:9c028a9442a18e266955d364ce42259136e79a7ba14d773e0d778d5f70cd56f1", + "sha256:9d36a51b3d93320b686588e27123f4995804dbf1bce81df78c02fc3c6eea9280", + "sha256:9f1692105a02bcf853f355032a5fdc5494358ef83d8fd22d16de375c85cec3f5", + "sha256:a9010e2ed5b3a9e158c5fd966b3ab7e834bb3d3aacc8f66c91dd4b57a3799230", + "sha256:aa0dd688aab3f7914d3e6e5e3554978e0383312fb8e771d84be008a35b9ee548", + "sha256:ab73e8db5e3f564d812c1f5c3a175930a5f9bc96ccb5e3b22a34d7858b401cf7", + "sha256:ae627397a2f351560440d872d6f7c8dbb4072e57868e7b2fc5b8b430fe489d45", + "sha256:aea3caa317752e3a466fa8af45d91ee0ea8c7fdd96e42b0a8dd9b76a7931eba1", + "sha256:b1ecbd9819deccc39b7542bf4d2a740d8a620694d39989e58661d3763458f8d4", + "sha256:b87504f1690a23b9a2cca841191a04f83895d4fc2dd04df91d82b1a04ca2ad46", + "sha256:bc3ce6b33c5828d9e80592011a5c584cb2ce86edbc4088405f70da47dc1d1b3b", + "sha256:bd43992b4473d42f12ff9e68326079f0696d9d4e6000e8f39a0238d482ba6ee2", + "sha256:c1f708d8ae9c56cf38a903c44297243d2ec83fd82b396b977e0144a3e76217e3", + "sha256:cae74872be221df4374d10fec61f93ed1513b9546ea84f2c0bf73ab3e9bd0b03", + "sha256:cca6644054e78746d8d4ef238681f9c34ff8b584fe6b988ecebb8db3b15e622a", + "sha256:d00f3ac06a2a8b246327f11e186a53a100a4d5c7ed52346367e5ec751d51586c", + "sha256:d1b36540d7aaf9b9101b3a6f376c8d8e9f7a9aec93ed05918f2c69d493ffef72", + "sha256:d2277a05f6dcb9fd13db9566aac4fabd68c3ea1ea46ee5567d4eef8efa495a2f", + "sha256:d5b0eea49f5562861ee8d757a32ef7d559c1d35be2aaaa1ec28941d74c9ffc8a", + "sha256:dc329359321b67d24efdf4bc69012b0597001649544db662c001db5a0184794c", + "sha256:de3bf945454d032f9e390b85c4072e0a0570bf825421c8be0e71209fa65e1abe", + "sha256:dec7db73758c2b54953fd8b7fe348c45188fe26b39ee18446196edd08453a5d4", + "sha256:dee008f20b542e3cd162ba338a7f9ec0f6d23d395f66fe8aeeec3c9d067ea253", + "sha256:efbb343ab2ce3540f4ecbe6315d677ed70f37cd9a72b1e58066c918ca83acbaa", + "sha256:f230cb1cbc9faaa616f9a678f530ebcf186e414b6bcbd88b960e4ba1b92428d5", + "sha256:f37aa505b3cf60701562eddb32df74b12a9e380c207fd8b06dd157a943ac7ea0", + "sha256:f5fb36b8c6c63fdcbb1d526d94c0d1331610d43f4118cc1beb4efef4f3faacb2", + "sha256:f8e3e8056dd674e279741485e2e512d6e9a751c7455809d0114e6ebf8d781085", + "sha256:f9743fc99135d5f78d2454435615f6dec0473ca507c26ce9d92b10b562a280d3", + "sha256:fa475675db22290c3158e1d42326d0f5a65f04f44a0e68c3630a25b53560fb9c", + "sha256:ff0fbaf5f44a21beeb0110f2ab64f45135a9536a834b79c0d1ef018f2786bbfa" + ], + "markers": "python_version >= '3.9'", + "version": "==0.11.0" + }, + "markdown-it-py": { + "hashes": [ + "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", + "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a" + ], + "markers": "python_version >= '3.10'", + "version": "==4.2.0" + }, + "mdurl": { + "hashes": [ + "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", + "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" + ], + "markers": "python_version >= '3.7'", + "version": "==0.1.2" + }, + "more-itertools": { + "hashes": [ + "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d", + "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192" + ], + "markers": "python_version >= '3.10'", + "version": "==11.1.0" + }, + "mypy": { + "hashes": [ + "sha256:022c771234936ceac541ebaf836fe9e2abeb3f5e09aff21588fe543ff006fe21", + "sha256:0b1a5260c95aa443083f9ed3592662941951bca3d4ca224a5dc517c38b7cf666", + "sha256:11a6beb180257a805961aea9ec591bbd0bd17f1e18d35b8456d57aee5bedfedc", + "sha256:1a293c534adb55271fef24a26da04b855540a8c13cc07bc5917b9fd2c394f2ca", + "sha256:20509760fd791c51579d573153407d226385ec1f8bcce55d730b354f3336bc22", + "sha256:244358bf1c0da7722230bce60683d52e8e9fd030554926f15b747a84efb5b3af", + "sha256:35aac3bb114e03888f535d5eb51b8bafbb3266586b599da1940f9b1be3ec5bd5", + "sha256:3712c20deed54e814eaaa825603bada8ea1c390670a397c95b98405347acc563", + "sha256:47cebf61abde7c088a4e27718a8b13a81655686b2e9c251f5c0915a802248166", + "sha256:498207db725cec88829a6a5c2fc771205fd043719ef98bc49aba8fb9fc4e6d57", + "sha256:49890d4f76ac9e06ec117f9e09f3174da70a620a0c300953d8595c926e80947f", + "sha256:4ec7c57657493c7a75534df2751c8ae2cda383c16ecc55d2106c54476b1b16f6", + "sha256:4f910fe825376a7b66ef7ca8c98e5a149e8cd64c19ae71d84047a74ee060d4e6", + "sha256:5431d42af987ebd92ba2f71d45c85ed41d8e6ca9f5fd209a69f68f707d2469e5", + "sha256:5fdf2941a07434af755837d9880f7d7d25f1dacb1af9dcd4b9b66f2220a3024e", + "sha256:6753d0c1fdd6b1a23b9e4f283ce80b2153b724adcb2653b20b85a8a28ac6436b", + "sha256:7354c5a7f69d9345c3d6e69921d57088eea3ddeeb6b20d34c1b3855b02c36ec2", + "sha256:7406f4d048e71e576f5356d317e5b0a9e666dfd966bd99f9d14ca06e1a341538", + "sha256:761be68e023ef5d94678772396a8af1220030f80837a3afd8d0aef3b419666f4", + "sha256:767fe8c66dc3e01e19e1737d4c38ebefead16125e1b8e58ad421903b376f5c65", + "sha256:7d5e5cad0efeba72b93cd17490cc0d69c5ac9ca132994fe3fb0314808aeeb83e", + "sha256:81e76ad12c2d804512e9b13240d1588316531bfba07558286078bfbce9613633", + "sha256:82208da9e09414d520e912d3e462d454854bed0810b71540bb016dcbca7308fd", + "sha256:8de55a8c861f2a49331f807be98d90caeceeef520bde13d43a160207f8af613e", + "sha256:8ef78c1d306bbf9a8a12f526c44902c9c28dffd6c52c52bf6a72641ce18d3849", + "sha256:98ebb6589bb3b6d0c6f0c459d53ca55b8091fbc13d277c4041c885392e8195e8", + "sha256:a663814603a5c563fb87a4f96fb473eeb30d1f5a4885afcf44f9db000a366289", + "sha256:a683016b16fe2f572dc04c72be7ee0504ac1605a265d0200f5cea695fb788f41", + "sha256:aea7f7a8a55b459c34275fc468ada6ca7c173a5e43a68f5dbe588a563d8a06b8", + "sha256:b33b6cd332695bba180d55e717a79d3038e479a2c49cc5eb3d53603409b9a5d7", + "sha256:b84802e7b5a6daf1f5e15bc9fcd7ddae77be13981ffab037f1c67bb84d67d135", + "sha256:bf03e12003084a67395184d3eb8cbd6a489dc3655b5664b28c210a9e2403ab0b", + "sha256:c209a90853081ff01d01ee895cafe10f7db1474e0d95beaeef0f6c1db9119bbd", + "sha256:c90345fc182dc363b891350457ec69c35140858538f38b4540845afcc32b1aef", + "sha256:c989640253f0d76843e9c6c1bbf4bd48c5e85ada61bde4beb37cb3eca035685e", + "sha256:d57a90ae5e872138a425ec328edbc9b235d1934c4377881a33ec05b341acc9a8", + "sha256:d8161b6ff4392410023224f0969d17db93e1e154bc3e4ba62598e720723ae211", + "sha256:e0210d626fc8b31ccc90233754c7bc90e1f43205e85d96387f7db1285b55c398", + "sha256:e195b817c13f02352a9c124301f9f30f078405444679b6753c1b96b6eed37285", + "sha256:e583edc957cfb0deb142079162ae826f58449b116c1d442f2d91c69d9fced081", + "sha256:e79ebc1b904b84f0310dff7469655a9c36c7a68bddb37bdd42b67a332df61d08", + "sha256:ecfe70d43775ab99562ab128ce49854a362044c9f894961f68f898c23cb7429d", + "sha256:fcaa0e479066e31f7cceb6a3bea39cb22b2ff51a6b2f24f193d19179ba17c389", + "sha256:ff715050c127d724fd260a2e666e7747fdd83511c0c47d449d98238970aef780" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==2.1.0" + }, + "mypy-extensions": { + "hashes": [ + "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", + "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558" + ], + "markers": "python_version >= '3.8'", + "version": "==1.1.0" + }, + "nh3": { + "hashes": [ + "sha256:0a09f51806fd51b4fedbf9ea2b61fef388f19aef0d62fe51199d41648be14588", + "sha256:207c01801d3e9bb8ec08f08689346bdd30ce15b8bf60013a925d08b5388962a4", + "sha256:23a312224875f72cd16bde417f49071451877e29ef646a60e50fcb69407cc18a", + "sha256:2c069570b06aa848457713ad7af4a9905691291548c4466a9ad78ee95808382b", + "sha256:38748140bf76383ab7ce2dce0ad4cb663855d8fbc9098f7f3483673d09616a17", + "sha256:387abd011e81959d5a35151a11350a0795c6edeb53ebfa02d2e882dc01299263", + "sha256:3bb854485c9b33e5bb143ff3e49e577073bc6bc320f0ff8fc316dd89c0d3c101", + "sha256:45855e14ff056064fec77133bfcf7cd691838168e5e17bbef075394954dc9dc8", + "sha256:45e6a65dc88a300a2e3502cb9c8e6d1d6b831d6fba7470643333609c6aab1f30", + "sha256:488928988caad25ba14b1eb5bc74e25e21f3b5e40341d956f3ce4a8bc19460dc", + "sha256:48f45e3e914be93a596431aa143dedf1582557bf41a58153c296048d6e3798c9", + "sha256:50d401ab2d8e86d59e2126e3ab2a2f45840c405842b626d9a51624b3a33b6878", + "sha256:52d877980d7ca01dc3baf3936bf844828bc6f332962227a684ed79c18cce14c3", + "sha256:559e4c73b689e9a7aa97ac9760b1bc488038d7c1a575aa4ab5a0e19ee9630c0f", + "sha256:6ea58cc44d274c643b83547ca9654a0b1a817609b160601356f76a2b744c49ad", + "sha256:72c5bdedec27fa33de6a5326346ea8aa3fe54f6ac294d54c4b204fb66a9f1e79", + "sha256:84bdeb082544fbcb77a12c034dd77d7da0556fdc0727b787eb6214b958c15e29", + "sha256:8f85285700a18e9f3fc5bff41fe573fa84f81542ef13b48a89f9fecca0474d3b", + "sha256:acfd354e61accbe4c74f8017c6e397a776916dfe47c48643cf7fd84ade826f93", + "sha256:c357f1d042c67f135a5e6babb2b0e3b9d9224ff4a3543240f597767b01384ffd", + "sha256:c3aae321f67ae66cff2a627115f106a377d4475d10b0e13d97959a13486b9a88", + "sha256:c88605d8d468f7fc1b31e06129bc91d6c96f6c621776c9b504a0da9beac9df5f", + "sha256:de8e8621853b6470fe928c684ee0d3f39ea8086cebafe4c416486488dea7b68d", + "sha256:e49c9b564e6bcb03ecd2f057213df9a0de15a95812ac9db9600b590db23d3ae9", + "sha256:ea232933394d1d58bf7c4bb348dc4660eae6604e1ae81cd2ba6d9ed80d390f3b", + "sha256:eeedc90ed8c42c327e8e10e621ccfa314fc6cce35d5929f4297ff1cdb89667c4", + "sha256:fe3a787dc76b50de6bee54ef242f26c41dfe47654428e3e94f0fae5bb6dd2cc1" + ], + "markers": "python_version >= '3.8'", + "version": "==0.3.5" + }, + "packaging": { + "hashes": [ + "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", + "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661" + ], + "markers": "python_version >= '3.8'", + "version": "==26.2" + }, + "pathspec": { + "hashes": [ + "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", + "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189" + ], + "markers": "python_version >= '3.9'", + "version": "==1.1.1" + }, + "platformdirs": { + "hashes": [ + "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", + "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a" + ], + "markers": "python_version >= '3.10'", + "version": "==4.10.0" + }, + "pluggy": { + "hashes": [ + "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", + "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746" + ], + "markers": "python_version >= '3.9'", + "version": "==1.6.0" + }, + "pyasn1": { + "hashes": [ + "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", + "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde" + ], + "markers": "python_version >= '3.8'", + "version": "==0.6.3" + }, + "pycodestyle": { + "hashes": [ + "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783", + "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d" + ], + "markers": "python_version >= '3.9'", + "version": "==2.14.0" + }, + "pycparser": { + "hashes": [ + "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", + "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" + ], + "markers": "python_version >= '3.10'", + "version": "==3.0" + }, + "pydantic": { + "extras": [ + "email" + ], + "hashes": [ + "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", + "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6" + ], + "markers": "python_version >= '3.9'", + "version": "==2.13.4" + }, + "pydantic-core": { + "hashes": [ + "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", + "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", + "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", + "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", + "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", + "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", + "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", + "sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29", + "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", + "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", + "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", + "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", + "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", + "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", + "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", + "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", + "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", + "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", + "sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9", + "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", + "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", + "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", + "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", + "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", + "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", + "sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0", + "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", + "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", + "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", + "sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac", + "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", + "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", + "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", + "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", + "sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928", + "sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6", + "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", + "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", + "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", + "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", + "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", + "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", + "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", + "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", + "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", + "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", + "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", + "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", + "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", + "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", + "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", + "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", + "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", + "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", + "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", + "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", + "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", + "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", + "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", + "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", + "sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15", + "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", + "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", + "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", + "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", + "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", + "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", + "sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9", + "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", + "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", + "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", + "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", + "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", + "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", + "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", + "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", + "sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201", + "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", + "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", + "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", + "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", + "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", + "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", + "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", + "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", + "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", + "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", + "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", + "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", + "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", + "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", + "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", + "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", + "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", + "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", + "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", + "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", + "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", + "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", + "sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1", + "sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76", + "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", + "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", + "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", + "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", + "sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066", + "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", + "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", + "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", + "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", + "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", + "sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49", + "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", + "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", + "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", + "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", + "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", + "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", + "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", + "sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae" + ], + "markers": "python_version >= '3.9'", + "version": "==2.46.4" + }, + "pygments": { + "hashes": [ + "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", + "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176" + ], + "markers": "python_version >= '3.9'", + "version": "==2.20.0" + }, + "pyjwt": { + "hashes": [ + "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", + "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728" + ], + "markers": "python_version >= '3.9'", + "version": "==2.13.0" + }, + "pyopenssl": { + "hashes": [ + "sha256:4f9d971bc5298b8bc1fab282803da04bf000c755d4ad9d99b52de2569ca19a70", + "sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387" + ], + "markers": "python_version >= '3.8'", + "version": "==26.2.0" + }, + "pypi-attestations": { + "hashes": [ + "sha256:278a28d741b57d62973c00d453ec9b9bb30456464d69296c6780474cd0bf098e", + "sha256:2daf3ec46ff4c7123184ec892852b4d4599b78128f01f742a44406a73200c5df" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==0.0.29" + }, + "pyproject-api": { + "hashes": [ + "sha256:c2b2726bd7aa9217b6c50b621fef5b2ae5def4d55b779c9e0694c15e0a8517ba", + "sha256:fa9e6f66c35b5017e909825d8f2b5d5482ea699d7be809d21c03bd1f7317f36a" + ], + "markers": "python_version >= '3.10'", + "version": "==1.10.1" + }, + "pyproject-hooks": { + "hashes": [ + "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8", + "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913" + ], + "markers": "python_version >= '3.7'", + "version": "==1.2.0" + }, + "pytest": { + "hashes": [ + "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", + "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==9.1.0" + }, + "python-discovery": { + "hashes": [ + "sha256:475803f53b7b2ed6e490e27373f9d8340f7d2eebf9acdaf645d7d714c97bb500", + "sha256:8f3746c4b4968d22afbb97d36e1a0e5b66e6c0f297290f2e95f05b9b8bf18690" + ], + "markers": "python_version >= '3.8'", + "version": "==1.4.2" + }, + "readme-renderer": { + "hashes": [ + "sha256:030a8fac74904f8fba11ad1bb6964e3f76e896dc7e5e71f16af190c9056696d1", + "sha256:3385ed220117104a2bceb4a9dac8c5fdf6d1f96890d7ea2a9c7174fd5c84091f" + ], + "markers": "python_version >= '3.10'", + "version": "==45.0" + }, + "requests": { + "hashes": [ + "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", + "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed" + ], + "markers": "python_version >= '3.10'", + "version": "==2.34.2" + }, + "requests-toolbelt": { + "hashes": [ + "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", + "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.0.0" + }, + "rfc3161-client": { + "hashes": [ + "sha256:0b3920334f7334ec3bb9c319d53a5d08cd43b6883f75e2669cfd869cd264d53a", + "sha256:1671b1be16480ea54c0d36239efd0fb62c13dd572a9865a5e91fea39f1b95303", + "sha256:1be4e1133f0f7fe875629f2c358285503c1cfc79cebfbc3fb4e28b8a57d6f1a4", + "sha256:2bc9835467f6166edd6f876470484e5b294ee141add6eff6a59f5047937aaa75", + "sha256:3da328ba08139846b1ab3a03402ba8a5f3659a640dbe2cd6a18f7f342e99ba98", + "sha256:4ef4b096abe7d55b020526e39932c2721939a6c55e9a5cd3b3e77897a0942937", + "sha256:63355099d932851eac507806bb9d0937dab546a66d5857d888168799ec635f6d", + "sha256:78cdc6bde331492cb94f69328831d5c56b271012b00c6f1784c2e4b33837d585", + "sha256:8102165201c5224cf6e6634bfd68c6a39e8f800601188216f8210face4861215", + "sha256:85a1d71d1eb2c9bced2b3eb75e96f9fe49732ec2567b5dafa1dd889fff42b7fe", + "sha256:8631f7db7c1327bf87ee6a9a8681b4cd6bc2a90aae651388f29d045cd9ff1ac9", + "sha256:940e1fc95ec0ca734927a82bcb5363fa988ef1a085d238ff0c861f29c0cfb746", + "sha256:9969262fe6c08ecce39f9fe3996cf412187793834a022a643803090db5aae6b4", + "sha256:9a98e9c7ff632d9571fcea25fb70bde0e8339b86368aef67a65f6a301f125733", + "sha256:b7ad54288a49379b01b1d0d9d15167d2b7c6c7f940332ab85eeb4a6e844da8c7", + "sha256:bc379167238df32cbcc1dc9c324088559c1734331030f5293d75f4fd37b5f4f6", + "sha256:bed6ef8e194cab85f6ec5678995b6406bb568383ebb6a4301be40e7939dd28d9", + "sha256:e16ed34f6f33fd62aa3b1f83615ecf2f96e1b1f57df4e1a36570b3f895333972", + "sha256:e3caffaebf43242b000c4a6659d60eaf19c3b161ccbe05b15634a856c9ea7e61" + ], + "markers": "python_version >= '3.9'", + "version": "==1.0.6" + }, + "rfc3986": { + "hashes": [ + "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", + "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.0" + }, + "rfc8785": { + "hashes": [ + "sha256:520d690b448ecf0703691c76e1a34a24ddcd4fc5bc41d589cb7c58ec651bcd48", + "sha256:e545841329fe0eee4f6a3b44e7034343100c12b4ec566dc06ca9735681deb4da" + ], + "markers": "python_version >= '3.8'", + "version": "==0.1.4" + }, + "rich": { + "hashes": [ + "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", + "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36" + ], + "markers": "python_full_version >= '3.9.0'", + "version": "==15.0.0" + }, + "secretstorage": { + "hashes": [ + "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", + "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be" + ], + "markers": "python_version >= '3.10'", + "version": "==3.5.0" + }, + "securesystemslib": { + "hashes": [ + "sha256:a0743a3d978cf26e98a70a57e3fbd5a18e0a74c20cabe615f6a55b02ef0272b3", + "sha256:faea87be0f9c4b4277a5fa1b54bf9bfd807be9a94ab11be6c557dc8b75c43285" + ], + "markers": "python_version ~= '3.10'", + "version": "==1.4.0" + }, + "setuptools": { + "hashes": [ + "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", + "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb" + ], + "markers": "python_version >= '3.9'", + "version": "==82.0.1" + }, + "setuptools-scm": { + "hashes": [ + "sha256:136e2b1d393d709d2bcf26f275b8dec06c48b811154167b0fd6bb002aad17d6d", + "sha256:a18396a1bc0219c974d1a74612b11f9dce0d5bd8b1dc55c65f6ac7fd609e8c28" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==8.2.0" + }, + "sigstore": { + "hashes": [ + "sha256:0f60c46c92fd4e871fbec979c9ae2aa381d7a93fbb774e49c9964550e5e16856", + "sha256:3c4b566bddfcc53e73d3adc06acf4311d72be0d907a167133abdc815a472a59b" + ], + "markers": "python_version >= '3.10'", + "version": "==4.3.0" + }, + "sigstore-models": { + "hashes": [ + "sha256:5201a68f4d7d0f8bec1e2f4378eb646b084c52609a4e31db8c385095fff68b2e", + "sha256:c766c09470c2a7e8a4a333c893f07e2001c56a3ff1757b1a246119f53169a849" + ], + "markers": "python_version >= '3.10'", + "version": "==0.0.6" + }, + "sigstore-rekor-types": { + "hashes": [ + "sha256:19aef25433218ebf9975a1e8b523cc84aaf3cd395ad39a30523b083ea7917ec5", + "sha256:b62bf38c5b1a62bc0d7fe0ee51a0709e49311d137c7880c329882a8f4b2d1d78" + ], + "markers": "python_version >= '3.8'", + "version": "==0.0.18" + }, + "tiararodney.posix-sdc": { + "hashes": [ + "sha256:09b091d53d662652536e1b7ea7d33231357e308b3b20e4dd6e2955545cf6ffb0", + "sha256:1026f5f7be72d76d215161b8aa525d5662451340132eefaf894e2595205ae5ef" + ], + "index": "pypicodetiararodney", + "version": "==1.2.1" + }, + "tomli-w": { + "hashes": [ + "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", + "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021" + ], + "markers": "python_version >= '3.9'", + "version": "==1.2.0" + }, + "tox": { + "hashes": [ + "sha256:0678fbf26dd5b559b1ef128fa4388325920219322ebc8cc5f3497627c00f4472", + "sha256:e2084be6dfdef96ba1bed4948e6a1f73613d6952e1477be5dca45653d4c053c8" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==4.55.1" + }, + "tuf": { + "hashes": [ + "sha256:572bdbdc9ff4a82278a0d4773e6100863b9b33023f27575e84ca65b486dd0d79", + "sha256:9d2e6723538e0d5a3e482b6de805fcfe64481448d5853039ba6b06ba541efd7f" + ], + "markers": "python_version >= '3.10'", + "version": "==7.0.0" + }, + "twine": { + "hashes": [ + "sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8", + "sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==6.2.0" + }, + "typing-extensions": { + "hashes": [ + "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", + "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548" + ], + "markers": "python_version >= '3.9'", + "version": "==4.15.0" + }, + "typing-inspection": { + "hashes": [ + "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", + "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464" + ], + "markers": "python_version >= '3.9'", + "version": "==0.4.2" + }, + "urllib3": { + "hashes": [ + "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", + "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897" + ], + "markers": "python_version >= '3.10'", + "version": "==2.7.0" + }, + "virtualenv": { + "hashes": [ + "sha256:55aa670b67bbfb991b03fda39bd3276d92c419d702376e98c5df1c9989a26783", + "sha256:dca3bf98275a59c652b69d68e73433e597d977c2da9198882479d1a7188009c8" + ], + "markers": "python_version >= '3.9'", + "version": "==21.5.1" + } + } +} diff --git a/pyproject.toml b/pyproject.toml index ae51a61..dcbab8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "tiararodney.posix-sdc", + "tiararodney.posix-sdc[hub]", ] dynamic = ["version"] requires-python = ">=3.9" From c3024d3df96ccf8138a044d50134192a9c6a37f3 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:18:06 +0200 Subject: [PATCH 59/64] todo(13): done pyproject declares posix-sdc[hub]; Pipfile [packages] = private-index wheel, [dev-packages] = git develop; Pipfile.lock committed alongside. Tree clean. No submodule changes. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index ae88ea0..78d73b9 100644 --- a/TODO +++ b/TODO @@ -220,7 +220,7 @@ Content-Type: application/issue ID: 13 Type: feature Title: reference posix-sdc three ways for seamless multi-machine dev -Status: in-progress +Status: done Priority: medium Created: 2026-06-17 Module: sekft From cc26e61e6b744cbde525e2a159104ed224d26aca Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:43:56 +0200 Subject: [PATCH 60/64] todo(14): open --- TODO | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/TODO b/TODO index 78d73b9..cbfb257 100644 --- a/TODO +++ b/TODO @@ -231,3 +231,21 @@ Description: Wire the posix-sdc dependency as a triplet: the abstract index in Pipfile [packages]; the git develop branch in Pipfile [dev-packages] for develop-time. Commit Pipfile.lock so the dependency surface and lock land together. + +--ISSUE +Content-Type: application/issue +ID: 14 +Type: bugfix +Title: refresh Pipfile.lock against published posix-sdc 1.2.2 +Status: open +Priority: medium +Created: 2026-06-17 +Module: sekft +Relationships: +Description: The lock committed with the triplet (#13) predated the published + posix-sdc 1.2.2 wheel, so it could not pin the real [hub] closure. + Now that 1.2.2 is on the private index, re-lock: posix-sdc resolves + to ==1.2.2 from the index and the [hub] extra pulls huggingface_hub + and its transitive deps into the lock. Commit the refreshed + Pipfile.lock so the next machine installs the published wheel with + the Hub path available. From baae4f3631b51afa7523923c8c57e484df37893b Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:44:25 +0200 Subject: [PATCH 61/64] todo(14): in-progress Pipfile.lock pins tiararodney.posix-sdc ==1.2.2 from the private index in both default and develop sections, with huggingface_hub and its [hub] transitive deps present; committed on develop; tree clean after. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index cbfb257..8549674 100644 --- a/TODO +++ b/TODO @@ -237,7 +237,7 @@ Content-Type: application/issue ID: 14 Type: bugfix Title: refresh Pipfile.lock against published posix-sdc 1.2.2 -Status: open +Status: in-progress Priority: medium Created: 2026-06-17 Module: sekft From 44ed08a239cb7cf651cf74aa0f2d4cd60838f191 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:44:36 +0200 Subject: [PATCH 62/64] bugfix(14): refresh Pipfile.lock against published posix-sdc 1.2.2 The lock committed with the triplet (#13) predated the published wheel. Now that posix-sdc 1.2.2 is on the private index, re-lock: it pins ==1.2.2 from the index in both the default and develop sections, and the [hub] extra pulls huggingface_hub and its transitive deps into the lock, so the next machine installs the published wheel with the Hub path available. --- Pipfile.lock | 490 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 482 insertions(+), 8 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 217e375..a1320ce 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4583c15dee320e17846aa51acd0fbca1fbdcd24118d50a34033f8c5f8b4a95ac" + "sha256": "53a92dc553410ed6b1bce84a3a81fdc4eb39b52a90e8581e28ac45bde5fbdc16" }, "pipfile-spec": 6, "requires": { @@ -21,20 +21,301 @@ ] }, "default": { - "tiararodney.posix-sdc": { + "annotated-doc": { "hashes": [ - "sha256:09b091d53d662652536e1b7ea7d33231357e308b3b20e4dd6e2955545cf6ffb0", - "sha256:1026f5f7be72d76d215161b8aa525d5662451340132eefaf894e2595205ae5ef" + "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", + "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4" + ], + "markers": "python_version >= '3.8'", + "version": "==0.0.4" + }, + "anyio": { + "hashes": [ + "sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89", + "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9" + ], + "markers": "python_version >= '3.10'", + "version": "==4.14.0" + }, + "certifi": { + "hashes": [ + "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", + "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db" + ], + "markers": "python_version >= '3.7'", + "version": "==2026.6.17" + }, + "click": { + "hashes": [ + "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", + "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96" + ], + "markers": "python_version >= '3.10'", + "version": "==8.4.1" + }, + "filelock": { + "hashes": [ + "sha256:10cdb3656fc44541cdf30652a93fb10ec6b05325620eb316bd26893e4201538a", + "sha256:dac1648087d5115554850d113e7dd8c83ab2d38e3435dde2d4f163847e57b767" + ], + "markers": "python_version >= '3.10'", + "version": "==3.29.4" + }, + "fsspec": { + "hashes": [ + "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", + "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a" + ], + "markers": "python_version >= '3.10'", + "version": "==2026.6.0" + }, + "h11": { + "hashes": [ + "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", + "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86" + ], + "markers": "python_version >= '3.8'", + "version": "==0.16.0" + }, + "hf-xet": { + "hashes": [ + "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", + "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f", + "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59", + "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", + "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d", + "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", + "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30", + "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", + "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947", + "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d", + "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", + "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", + "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", + "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283", + "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", + "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6", + "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a", + "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8", + "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577", + "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9", + "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43", + "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6", + "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff", + "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342", + "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5" + ], + "markers": "python_version >= '3.8'", + "version": "==1.5.1" + }, + "httpcore": { + "hashes": [ + "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", + "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8" + ], + "markers": "python_version >= '3.8'", + "version": "==1.0.9" + }, + "httpx": { + "hashes": [ + "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", + "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad" + ], + "markers": "python_version >= '3.8'", + "version": "==0.28.1" + }, + "huggingface-hub": { + "hashes": [ + "sha256:1dc72e1f6b4d6df6b30eb72e57d00514ef453d660f04af2b87f0e67267f31ee0", + "sha256:fd771622182d40977272a923953ee3b1b13538f9f8a7f5d78398f10af0f1c0bd" + ], + "markers": "python_full_version >= '3.10.0'", + "version": "==1.19.0" + }, + "idna": { + "hashes": [ + "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", + "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848" + ], + "markers": "python_version >= '3.9'", + "version": "==3.18" + }, + "markdown-it-py": { + "hashes": [ + "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", + "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a" + ], + "markers": "python_version >= '3.10'", + "version": "==4.2.0" + }, + "mdurl": { + "hashes": [ + "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", + "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" + ], + "markers": "python_version >= '3.7'", + "version": "==0.1.2" + }, + "packaging": { + "hashes": [ + "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", + "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661" + ], + "markers": "python_version >= '3.8'", + "version": "==26.2" + }, + "pygments": { + "hashes": [ + "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", + "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176" + ], + "markers": "python_version >= '3.9'", + "version": "==2.20.0" + }, + "pyyaml": { + "hashes": [ + "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", + "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a", + "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", + "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", + "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", + "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", + "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", + "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", + "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0", + "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", + "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", + "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6", + "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7", + "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", + "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007", + "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", + "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", + "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9", + "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295", + "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", + "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", + "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", + "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", + "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", + "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", + "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", + "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", + "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b", + "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", + "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5", + "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", + "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", + "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369", + "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", + "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", + "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", + "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", + "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", + "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", + "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", + "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", + "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", + "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", + "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", + "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", + "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", + "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", + "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", + "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", + "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4", + "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", + "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", + "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", + "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", + "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", + "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", + "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da", + "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", + "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", + "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", + "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", + "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f", + "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917", + "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", + "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", + "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", + "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", + "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", + "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", + "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3", + "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", + "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926", + "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0" + ], + "markers": "python_version >= '3.8'", + "version": "==6.0.3" + }, + "rich": { + "hashes": [ + "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", + "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36" + ], + "markers": "python_full_version >= '3.9.0'", + "version": "==15.0.0" + }, + "shellingham": { + "hashes": [ + "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", + "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de" + ], + "markers": "python_version >= '3.7'", + "version": "==1.5.4" + }, + "tiararodney.posix-sdc": { + "extras": [ + "hub" + ], + "hashes": [ + "sha256:b9790087a3baf3ef2976e1b7c75d27a9f8892473189354f1aae4e1f3f8f63f42", + "sha256:dbe137f67ae0b93357a8e280b2568041e95d9a2f8641e21fda5e958abc5075cb" ], "index": "pypicodetiararodney", - "version": "==1.2.1" + "version": "==1.2.2" }, "tiararodney.sekft": { "editable": true, "file": "." + }, + "tqdm": { + "hashes": [ + "sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482", + "sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03" + ], + "markers": "python_version >= '3.8'", + "version": "==4.68.3" + }, + "typer": { + "hashes": [ + "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", + "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc" + ], + "markers": "python_version >= '3.10'", + "version": "==0.25.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", + "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548" + ], + "markers": "python_version >= '3.9'", + "version": "==4.15.0" } }, "develop": { + "annotated-doc": { + "hashes": [ + "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", + "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4" + ], + "markers": "python_version >= '3.8'", + "version": "==0.0.4" + }, "annotated-types": { "hashes": [ "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", @@ -43,6 +324,14 @@ "markers": "python_version >= '3.8'", "version": "==0.7.0" }, + "anyio": { + "hashes": [ + "sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89", + "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9" + ], + "markers": "python_version >= '3.10'", + "version": "==4.14.0" + }, "ast-serialize": { "hashes": [ "sha256:061ee58bdb52341c8201a6df41182a977736bae3b7ded87ca7176ca25a8a47ab", @@ -341,6 +630,14 @@ "markers": "python_version >= '3.7'", "version": "==3.4.7" }, + "click": { + "hashes": [ + "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", + "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96" + ], + "markers": "python_version >= '3.10'", + "version": "==8.4.1" + }, "colorama": { "hashes": [ "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", @@ -443,6 +740,77 @@ "markers": "python_version >= '3.10'", "version": "==3.29.4" }, + "fsspec": { + "hashes": [ + "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", + "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a" + ], + "markers": "python_version >= '3.10'", + "version": "==2026.6.0" + }, + "h11": { + "hashes": [ + "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", + "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86" + ], + "markers": "python_version >= '3.8'", + "version": "==0.16.0" + }, + "hf-xet": { + "hashes": [ + "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", + "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f", + "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59", + "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", + "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d", + "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", + "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30", + "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", + "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947", + "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d", + "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", + "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", + "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", + "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283", + "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", + "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6", + "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a", + "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8", + "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577", + "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9", + "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43", + "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6", + "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff", + "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342", + "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5" + ], + "markers": "python_version >= '3.8'", + "version": "==1.5.1" + }, + "httpcore": { + "hashes": [ + "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", + "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8" + ], + "markers": "python_version >= '3.8'", + "version": "==1.0.9" + }, + "httpx": { + "hashes": [ + "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", + "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad" + ], + "markers": "python_version >= '3.8'", + "version": "==0.28.1" + }, + "huggingface-hub": { + "hashes": [ + "sha256:1dc72e1f6b4d6df6b30eb72e57d00514ef453d660f04af2b87f0e67267f31ee0", + "sha256:fd771622182d40977272a923953ee3b1b13538f9f8a7f5d78398f10af0f1c0bd" + ], + "markers": "python_full_version >= '3.10.0'", + "version": "==1.19.0" + }, "id": { "hashes": [ "sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069", @@ -978,6 +1346,85 @@ "markers": "python_version >= '3.8'", "version": "==1.4.2" }, + "pyyaml": { + "hashes": [ + "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", + "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a", + "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", + "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", + "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", + "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", + "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", + "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", + "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0", + "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", + "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", + "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6", + "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7", + "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", + "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007", + "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", + "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", + "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9", + "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295", + "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", + "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", + "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", + "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", + "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", + "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", + "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", + "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", + "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b", + "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", + "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5", + "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", + "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", + "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369", + "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", + "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", + "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", + "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", + "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", + "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", + "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", + "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", + "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", + "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", + "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", + "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", + "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", + "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", + "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", + "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", + "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4", + "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", + "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", + "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", + "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", + "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", + "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", + "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da", + "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", + "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", + "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", + "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", + "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f", + "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917", + "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", + "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", + "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", + "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", + "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", + "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", + "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3", + "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", + "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926", + "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0" + ], + "markers": "python_version >= '3.8'", + "version": "==6.0.3" + }, "readme-renderer": { "hashes": [ "sha256:030a8fac74904f8fba11ad1bb6964e3f76e896dc7e5e71f16af190c9056696d1", @@ -1084,6 +1531,14 @@ "markers": "python_version >= '3.8'", "version": "==8.2.0" }, + "shellingham": { + "hashes": [ + "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", + "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de" + ], + "markers": "python_version >= '3.7'", + "version": "==1.5.4" + }, "sigstore": { "hashes": [ "sha256:0f60c46c92fd4e871fbec979c9ae2aa381d7a93fbb774e49c9964550e5e16856", @@ -1109,12 +1564,15 @@ "version": "==0.0.18" }, "tiararodney.posix-sdc": { + "extras": [ + "hub" + ], "hashes": [ - "sha256:09b091d53d662652536e1b7ea7d33231357e308b3b20e4dd6e2955545cf6ffb0", - "sha256:1026f5f7be72d76d215161b8aa525d5662451340132eefaf894e2595205ae5ef" + "sha256:b9790087a3baf3ef2976e1b7c75d27a9f8892473189354f1aae4e1f3f8f63f42", + "sha256:dbe137f67ae0b93357a8e280b2568041e95d9a2f8641e21fda5e958abc5075cb" ], "index": "pypicodetiararodney", - "version": "==1.2.1" + "version": "==1.2.2" }, "tomli-w": { "hashes": [ @@ -1133,6 +1591,14 @@ "markers": "python_version >= '3.10'", "version": "==4.55.1" }, + "tqdm": { + "hashes": [ + "sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482", + "sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03" + ], + "markers": "python_version >= '3.8'", + "version": "==4.68.3" + }, "tuf": { "hashes": [ "sha256:572bdbdc9ff4a82278a0d4773e6100863b9b33023f27575e84ca65b486dd0d79", @@ -1150,6 +1616,14 @@ "markers": "python_version >= '3.9'", "version": "==6.2.0" }, + "typer": { + "hashes": [ + "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", + "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc" + ], + "markers": "python_version >= '3.10'", + "version": "==0.25.1" + }, "typing-extensions": { "hashes": [ "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", From b4bf90d275bc1a26527b0f74ba999c24057d1ce3 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:44:38 +0200 Subject: [PATCH 63/64] todo(14): done Pipfile.lock pins posix-sdc ==1.2.2 from the private index in default + develop sections with huggingface_hub and the [hub] closure present; committed; tree clean. No submodule changes. --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 8549674..4f2c672 100644 --- a/TODO +++ b/TODO @@ -237,7 +237,7 @@ Content-Type: application/issue ID: 14 Type: bugfix Title: refresh Pipfile.lock against published posix-sdc 1.2.2 -Status: in-progress +Status: done Priority: medium Created: 2026-06-17 Module: sekft From 7856ef6032cfa8cc26425a6f93da3ba761c123c0 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Thu, 18 Jun 2026 00:57:54 +0200 Subject: [PATCH 64/64] release(1.0.0): add changelog First release: the trainer (assistant-only mask, train=serve render), the three-source data loader (raw dir / curated jsonl / Hub), eval, resident, and structured logging. Packaged, typed, [gpu] extra, depends on posix-sdc[hub]. --- CHANGELOG.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4408334 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,41 @@ +# Changelog + +All notable changes to sekft, the shell-operator SFT trainer behind the +[posix-sdc](https://huggingface.co/datasets/tiararodney/posix-sdc) experiment, +are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and the project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.0] - 2026-06-18 + +First release: the training and evaluation pipeline that turns posix-sdc +trajectories into a fine-tuned shell operator. + +### Added +- `sekft-train`: LoRA / QLoRA supervised fine-tuning of a base model on + shell-operation trajectories, with an **assistant-only loss mask** derived by + token-prefix differencing — the commands and the terminal `exit` / `panic` + token are trained; the environment turns (orientation, prompts, command + output) are masked to `-100`. The render uses the tokenizer's own + `apply_chat_template`, so training matches what the serving harness sends + (train = serve), with `normalize_for_template` canonicalising trajectories for + instruct templates that have no system role and require strict user/assistant + alternation. +- Three sources of training data: a directory of raw rollout `.json` + (keep-filtered), a curated `.jsonl` corpus, or the published posix-sdc corpus + over the Hugging Face Hub (`--hub`). +- `--inspect` for mask and token statistics without training, and structured + stderr logging across every phase (`-v` / `-q`): per-trajectory and progress + lines while the corpus is tokenized, dataset accounting that warns on dropped + (over-length / empty-mask) trajectories, and the per-step training curve. +- `sekft-eval`: behavioural evaluation that drops the tuned model into held-out + scenarios with no scaffold and scores whether it operates and terminates. +- `sekft-resident`: a resident-base harness that loads the base model once and + fits several adapters without reloading, for paired / STaR-style runs. +- Packaging: the `tiararodney.sekft` namespace package with `sekft-train`, + `sekft-eval`, and `sekft-resident` console scripts; a typed (`py.typed`), + mypy-strict codebase; an optional `[gpu]` extra (torch / transformers / peft); + and a dependency on `posix-sdc[hub]`. Released under GPL-2.0. + +[1.0.0]: https://git.code.tiararodney.com/tiara/sekft/releases/tag/v1.0.0