From e310930d9e651aafe29ddff14acf097aa7518471 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 16 Oct 2025 23:07:06 +0200 Subject: [PATCH 01/93] Frontend basics --- app/api/models.py | 46 +++++++++- app/api/v1.py | 87 +++++++++++++++++- app/main.py | 21 +++-- app/services/passwords.py | 31 +++---- app/static/css/style.css | 66 ++++++++++++++ app/static/js/app.js | 187 ++++++++++++++++++++++++++++++++++++++ app/templates/index.html | 85 +++++++++++------ app/web/__init__.py | 3 + app/web/passwords.py | 138 ++++++++++++++++++++++++++++ pyproject.toml | 1 + 10 files changed, 607 insertions(+), 58 deletions(-) create mode 100644 app/static/css/style.css create mode 100644 app/static/js/app.js create mode 100644 app/web/__init__.py create mode 100644 app/web/passwords.py diff --git a/app/api/models.py b/app/api/models.py index 35afedb..ce7c5c0 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -1,5 +1,6 @@ from dataclasses import dataclass -from typing import List +from enum import Enum +from typing import List, Optional @dataclass @@ -10,6 +11,49 @@ class Request: commit_url: str changelist: List[str] + @dataclass class Response: status: int + + +class EntryKind(str, Enum): + simple = "simple" + complex = "complex" + + +@dataclass +class EntrySimpleDTO: + key: str + value: str # mapowane na title/password + + +@dataclass +class EntryComplexDTO: + title: str + username: str + password: str + url: Optional[str] = None + notes: Optional[str] = None + + +@dataclass +class GroupDTO: + name: str + path: str + + +@dataclass +class EntryNodeDTO: + kind: EntryKind + title: str + path: str + + +@dataclass +class NodeDTO: + # Drzewo: grupa lub wpisy w grupach + name: str + path: str + groups: List["NodeDTO"] + entries: List[EntryNodeDTO] diff --git a/app/api/v1.py b/app/api/v1.py index fc52bd4..31427e8 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,6 +1,9 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends, Body, Query +from fastapi.responses import JSONResponse +from fastapi_utils.cbv import cbv -from app.api.models import Request, Response +from app.api.models import * +from app.web import PasswordsController router = APIRouter() @@ -17,3 +20,83 @@ async def health(): @router.post("/ci", summary="CI Webhook") async def ci(request: Request): return Response(200) + + +@cbv(router) +class TreeController: + svc: PasswordsController = Depends(PasswordsController.dep) + + @router.get("/tree", response_model=NodeDTO) + def get_tree(self) -> NodeDTO: + return self.svc.get_tree() + + @router.post("/group", response_model=GroupDTO) + def create_group( + self, + name: str = Body(..., embed=True), + parent_path: str | None = Body(None, embed=True), + ) -> GroupDTO: + return self.svc.create_group(name=name, parent_path=parent_path) + + @router.patch("/group/rename", response_model=GroupDTO) + def rename_group( + self, + path: str = Body(..., embed=True), + new_name: str = Body(..., embed=True), + ) -> GroupDTO: + return self.svc.rename_group(path, new_name) + + @router.delete("/group") + def delete_group( + self, + path: str = Query(...), + ): + self.svc.delete_group(path) + return JSONResponse({"ok": True}) + + @router.post("/entry", response_model=NodeDTO) + def create_entry( + self, + kind: EntryKind = Body(..., embed=True), + parent_path: str = Body(..., embed=True), + data: dict = Body(..., embed=True), + ) -> NodeDTO: + if kind == EntryKind.simple: + dto = EntrySimpleDTO(**data) + return self.svc.create_entry_simple(parent_path, dto) + dto = EntryComplexDTO(**data) + return self.svc.create_entry_complex(parent_path, dto) + + @router.patch("/entry", response_model=NodeDTO) + def update_entry( + self, + path: str = Body(..., embed=True), + kind: EntryKind = Body(..., embed=True), + data: dict = Body(..., embed=True), + ) -> NodeDTO: + if kind == EntryKind.simple: + dto = EntrySimpleDTO(**data) + return self.svc.update_entry_simple(path, dto) + dto = EntryComplexDTO(**data) + return self.svc.update_entry_complex(path, dto) + + @router.patch("/entry/move", response_model=NodeDTO) + def move_entry( + self, + path: str = Body(..., embed=True), + target_group_path: str = Body(..., embed=True), + ) -> NodeDTO: + return self.svc.move_entry(path, target_group_path) + + @router.delete("/entry") + def delete_entry( + self, + path: str = Query(...), + ): + self.svc.delete_entry(path) + return JSONResponse({"ok": True}) + + @router.post("/save") + def save_changes(self): + self.svc.save() + return JSONResponse({"ok": True}) diff --git a/app/main.py b/app/main.py index 5bf8c98..f173a4a 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,7 @@ import logging from fastapi import FastAPI from injectable import load_injection_container +from fastapi.staticfiles import StaticFiles from app.config import get_settings from app.core.core import WebhookProcessor @@ -12,15 +13,15 @@ class KarlApplication: from starlette.types import Receive, Scope, Send def __init__(self) -> None: self._set_logging() - _app = FastAPI(title="Karl", version="0.1.0") - self._set_routes(_app) - self._set_events(_app) + _instance = FastAPI(title="Karl", version="0.1.0") + self._set_routes(_instance) + self._set_events(_instance) self._init_services() - self._app = _app + self._instance = _instance async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - await self._app.__call__(scope, receive, send) + await self._instance.__call__(scope, receive, send) def _set_logging(self): logging.basicConfig(level=logging.INFO, handlers=[LoggingHandler()]) @@ -39,14 +40,14 @@ class KarlApplication: logging_logger.handlers = [external_handler] logging_logger.propagate = False - def _set_routes(self, app: FastAPI): + def _set_routes(self, instance: FastAPI): from app.core.router import router as core_router - app.include_router(core_router) + instance.include_router(core_router) from app.api.v1 import router as api_v1_router - app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) - pass + instance.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) + instance.mount("/static", StaticFiles(directory="app/static"), name="static") - def _set_events(self, app: FastAPI): + def _set_events(self, instance: FastAPI): pass def _init_services(self): diff --git a/app/services/passwords.py b/app/services/passwords.py index 18d8519..fea898c 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -2,6 +2,7 @@ import os.path from injectable import injectable from pykeepass import PyKeePass, create_database, Group +import shutil @injectable(singleton=True) @@ -12,29 +13,27 @@ class Passwords: with open(settings.kp.secret, "r") as fh: secret = fh.read() - - self._kp_org = self.__get_or_create_store(settings.kp.file, secret) - self._kp = self.__get_lock(settings.kp.file, secret) + self._path = settings.kp.file + self._kp_org = self._open_or_create(self._path, secret) + self._kp = self._open_lock(self._path, secret) @staticmethod - def __get_or_create_store(path, passwd) -> PyKeePass: + def _open_or_create(path, password) -> PyKeePass: if os.path.exists(path): - return PyKeePass( - path, - password=passwd, - ) - return create_database(path, passwd) + return PyKeePass(path, password=password) + return create_database(path, password) @staticmethod - def __get_lock(path, passwd) -> PyKeePass: + def _open_lock(path, password) -> PyKeePass: lock_path = path + ".lock" - import shutil shutil.copyfile(path, lock_path) - return Passwords.__get_or_create_store(lock_path, passwd) + return Passwords._open_or_create(lock_path, password) @property - def store(self): - return self._kp.root_group + def kp(self) -> PyKeePass: + return self._kp - def save(self, group: Group): - pass + def save(self): + # nadpisz plik źródłowy zmianami z lock + self._kp.save() + shutil.copyfile(self._path + ".lock", self._path) diff --git a/app/static/css/style.css b/app/static/css/style.css new file mode 100644 index 0000000..1cd3176 --- /dev/null +++ b/app/static/css/style.css @@ -0,0 +1,66 @@ +html, body { + height: 100%; + margin: 0; + font-family: system-ui, -apple-system, Segoe UI, Roboto, Ubuntu, Cantarell, Noto Sans, Helvetica, Arial; +} + +.layout { + display: grid; + grid-template-columns: 320px 1fr; + height: 100vh; +} + +.left { + border-right: 1px solid #ddd; + padding: 12px; + overflow: auto; +} + +.right { + padding: 12px; + overflow: auto; +} + +ul { + list-style: none; + padding-left: 16px; +} + +.node { + cursor: pointer; +} + +.toolbar { + display: flex; + gap: 8px; + margin-bottom: 12px; +} + +.row { + margin-bottom: 8px; + display: flex; + gap: 8px; + align-items: center; +} + +input[type="text"], input[type="url"], input[type="password"], textarea, select { + width: 100%; + padding: 8px; + border: 1px solid #ccc; + border-radius: 6px; +} + +.section { + border: 1px solid #eee; + border-radius: 8px; + padding: 12px; + margin-bottom: 12px; +} + +.mono { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace; +} + +.danger { + color: #b10000 +} diff --git a/app/static/js/app.js b/app/static/js/app.js new file mode 100644 index 0000000..610f994 --- /dev/null +++ b/app/static/js/app.js @@ -0,0 +1,187 @@ +let selectedNode = { type: "group", path: "/" }; // type: "group" | "entry" +let currentKind = "simple"; + +const treeDiv = document.getElementById("tree"); +const currentPathEl = document.getElementById("current-path"); + +const editorSimple = document.getElementById("editor-simple"); +const editorComplex = document.getElementById("editor-complex"); +const kindSelect = document.getElementById("entry-kind"); + +function showEditor(kind) { + editorSimple.style.display = (kind === "simple") ? "block" : "none"; + editorComplex.style.display = (kind === "complex") ? "block" : "none"; +} + +kindSelect.addEventListener("change", () => { + currentKind = kindSelect.value; + showEditor(currentKind); +}); + +async function api(path, opts = {}) { + const resp = await fetch(path, opts); + if (!resp.ok) { + const txt = await resp.text(); + throw new Error("API error: " + txt); + } + if (resp.headers.get("Content-Type")?.includes("application/json")) { + return resp.json(); + } + return null; +} + +function renderTree(node) { + const ul = document.createElement("ul"); + + const mkEntryLi = (e) => { + const li = document.createElement("li"); + li.textContent = "🔑 " + e.title; + li.className = "node"; + li.onclick = () => { + selectedNode = { type: "entry", path: e.path, kind: e.kind }; + currentPathEl.textContent = e.path; + kindSelect.value = e.kind; + showEditor(e.kind); + if (e.kind === "simple") { + document.getElementById("s-key").value = e.title; + document.getElementById("s-value").value = ""; + } else { + document.getElementById("c-title").value = e.title; + document.getElementById("c-username").value = ""; + document.getElementById("c-password").value = ""; + document.getElementById("c-url").value = ""; + document.getElementById("c-notes").value = ""; + } + }; + return li; + }; + + const mkGroupLi = (g) => { + const li = document.createElement("li"); + const header = document.createElement("div"); + header.textContent = "📂 " + (g.name || "/"); + header.className = "node"; + header.onclick = () => { + selectedNode = { type: "group", path: g.path }; + currentPathEl.textContent = g.path; + }; + li.appendChild(header); + + const inner = renderTreeChildren(g); + li.appendChild(inner); + return li; + }; + + function renderTreeChildren(node) { + const wrap = document.createElement("ul"); + node.groups.forEach(sg => wrap.appendChild(mkGroupLi(sg))); + node.entries.forEach(e => wrap.appendChild(mkEntryLi(e))); + return wrap; + } + + ul.appendChild(mkGroupLi(node)); + treeDiv.innerHTML = ""; + treeDiv.appendChild(ul); +} + +async function refreshTree() { + const data = await api("/api/v1/tree"); + renderTree(data); +} + +// Toolbar lewej kolumny +document.getElementById("btn-refresh").onclick = refreshTree; + +document.getElementById("btn-add-group").onclick = async () => { + const name = prompt("Nazwa nowej grupy:"); + if (!name) return; + await api("/api/v1/group", { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ name, parent_path: selectedNode.type === "group" ? selectedNode.path : "/" }) + }); + await refreshTree(); +}; + +document.getElementById("btn-add-entry").onclick = async () => { + const parent_path = selectedNode.type === "group" ? selectedNode.path : "/"; + const kind = kindSelect.value; + if (kind === "simple") { + const key = prompt("Klucz (title):"); if (!key) return; + const value = prompt("Wartość (password):"); if (value === null) return; + await api("/api/v1/entry", { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ kind, parent_path, data: { key, value } }) + }); + } else { + const title = prompt("Tytuł:"); if (!title) return; + await api("/api/v1/entry", { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ kind, parent_path, data: { title, username:"", password:"", url:"", notes:"" } }) + }); + } + await refreshTree(); +}; + +// Edytory +document.getElementById("s-save").onclick = async () => { + if (selectedNode.type !== "entry") { alert("Wybierz wpis"); return; } + const data = { key: document.getElementById("s-key").value, value: document.getElementById("s-value").value }; + await api("/api/v1/entry", { + method: "PATCH", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ path: selectedNode.path, kind: "simple", data }) + }); + await refreshTree(); +}; +document.getElementById("s-delete").onclick = async () => { + if (selectedNode.type !== "entry") return; + if (!confirm("Usunąć wpis?")) return; + await api("/api/v1/entry?path=" + encodeURIComponent(selectedNode.path), { method: "DELETE" }); + await refreshTree(); +}; + +document.getElementById("c-save").onclick = async () => { + if (selectedNode.type !== "entry") { alert("Wybierz wpis"); return; } + const data = { + title: document.getElementById("c-title").value, + username: document.getElementById("c-username").value, + password: document.getElementById("c-password").value, + url: document.getElementById("c-url").value, + notes: document.getElementById("c-notes").value + }; + await api("/api/v1/entry", { + method: "PATCH", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ path: selectedNode.path, kind: "complex", data }) + }); + await refreshTree(); +}; +document.getElementById("c-delete").onclick = async () => { + if (selectedNode.type !== "entry") return; + if (!confirm("Usunąć wpis?")) return; + await api("/api/v1/entry?path=" + encodeURIComponent(selectedNode.path), { method: "DELETE" }); + await refreshTree(); +}; + +document.getElementById("btn-move-entry").onclick = async () => { + if (selectedNode.type !== "entry") { alert("Wybierz wpis"); return; } + const target = document.getElementById("target-group").value || "/"; + await api("/api/v1/entry/move", { + method: "PATCH", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ path: selectedNode.path, target_group_path: target }) + }); + await refreshTree(); +}; + +document.getElementById("btn-save-all").onclick = async () => { + await api("/api/v1/save", { method: "POST" }); + alert("Zapisano do bazy"); +}; + +// Start +showEditor(currentKind); +refreshTree().catch(err => console.error(err)); diff --git a/app/templates/index.html b/app/templates/index.html index ebbe8ca..4e37ff3 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -4,36 +4,63 @@ {{ title }} - + -

{{ title }}

-

To jest prosta strona Jinja2 serwowana przez FastAPI.

- -

Uruchamianie serwera lokalnie: uv run app lub uv run uvicorn app.main:app --reload

+
+
+
+ + + +
+
+
+
+
+ + +
+ + + + + +
+
+ + + +
+
+ +
+ Aktualna ścieżka: / +
+
+
+ diff --git a/app/web/__init__.py b/app/web/__init__.py new file mode 100644 index 0000000..ba466bf --- /dev/null +++ b/app/web/__init__.py @@ -0,0 +1,3 @@ +from .passwords import PasswordsController + +__all__ = ["PasswordsController"] diff --git a/app/web/passwords.py b/app/web/passwords.py new file mode 100644 index 0000000..92d3758 --- /dev/null +++ b/app/web/passwords.py @@ -0,0 +1,138 @@ +from typing import Optional, List + +from pykeepass import Group, Entry + +from app.api.models import EntryKind, EntrySimpleDTO, EntryComplexDTO, GroupDTO, NodeDTO, EntryNodeDTO +from app.services import Passwords + + +class PasswordsController: + def __init__(self, passwords: Passwords): + self._pw = passwords + + @staticmethod + def dep() -> "PasswordsController": + # prosta fabryka/DI mostkująca injectable + return PasswordsController(Passwords()) + + # Helpers + def _group_by_path(self, path: Optional[str]) -> Group: + if not path or path == "/": + return self._pw.kp.root_group + parts = [p for p in path.split("/") if p] + g = self._pw.kp.root_group + for p in parts: + g = next((x for x in g.subgroups if x.name == p), None) + if g is None: + raise ValueError(f"Group not found: {path}") + return g + + def _ensure_group(self, parent: Group, name: str) -> Group: + g = next((x for x in parent.subgroups if x.name == name), None) + if g: + return g + return self._pw.kp.add_group(parent, name) + + def _entry_by_path(self, path: str) -> Entry: + # path: /Group1/Sub/Title + parts = [p for p in path.split("/") if p] + if not parts: + raise ValueError("Invalid entry path") + title = parts[-1] + group_path = "/" + "/".join(parts[:-1]) if len(parts) > 1 else "/" + g = self._group_by_path(group_path) + e = next((x for x in g.entries if x.title == title), None) + if not e: + raise ValueError(f"Entry not found: {path}") + return e + + def _node_of_group(self, g: Group) -> NodeDTO: + def group_path(gr: Group) -> str: + names: List[str] = [] + cur = gr + while cur and cur.name: + names.append(cur.name) + cur = cur.parent + names.reverse() + return "/" + "/".join(names) if names else "/" + + path = group_path(g) + groups = [self._node_of_group(sg) for sg in g.subgroups] + entries = [EntryNodeDTO( + kind=EntryKind.complex if (e.username or e.url or e.notes) else EntryKind.simple, + title=e.title, + path=path.rstrip("/") + "/" + e.title + ) for e in g.entries] + return NodeDTO(name=g.name or "", path=path, groups=groups, entries=entries) + + # Tree + def get_tree(self) -> NodeDTO: + return self._node_of_group(self._pw.kp.root_group) + + # Groups + def create_group(self, name: str, parent_path: Optional[str]) -> GroupDTO: + parent = self._group_by_path(parent_path) + g = self._ensure_group(parent, name) + return GroupDTO(name=g.name, path=self._node_of_group(g).path) + + def rename_group(self, path: str, new_name: str) -> GroupDTO: + g = self._group_by_path(path) + g.name = new_name + return GroupDTO(name=g.name, path=self._node_of_group(g).path) + + def delete_group(self, path: str) -> None: + g = self._group_by_path(path) + if g is self._pw.kp.root_group: + raise ValueError("Cannot delete root group") + self._pw.kp.delete_group(g) + + # Entries + def create_entry_simple(self, parent_path: str, dto: EntrySimpleDTO) -> NodeDTO: + parent = self._group_by_path(parent_path) + title = dto.key + password = dto.value + self._pw.kp.add_entry(parent, title=title, username="", password=password) + return self._node_of_group(parent) + + def create_entry_complex(self, parent_path: str, dto: EntryComplexDTO) -> NodeDTO: + parent = self._group_by_path(parent_path) + self._pw.kp.add_entry( + parent, + title=dto.title, + username=dto.username or "", + password=dto.password or "", + url=dto.url or "", + notes=dto.notes or "", + ) + return self._node_of_group(parent) + + def update_entry_simple(self, path: str, dto: EntrySimpleDTO) -> NodeDTO: + e = self._entry_by_path(path) + parent = e.group + e.title = dto.key + e.password = dto.value + return self._node_of_group(parent) + + def update_entry_complex(self, path: str, dto: EntryComplexDTO) -> NodeDTO: + e = self._entry_by_path(path) + parent = e.group + e.title = dto.title + e.username = dto.username + e.password = dto.password + e.url = dto.url or "" + e.notes = dto.notes or "" + return self._node_of_group(parent) + + def move_entry(self, path: str, target_group_path: str) -> NodeDTO: + e = self._entry_by_path(path) + target = self._group_by_path(target_group_path) + self._pw.kp.move_entry(e, target) + return self._node_of_group(target) + + def delete_entry(self, path: str) -> None: + e = self._entry_by_path(path) + self._pw.kp.delete_entry(e) + + # Persist + def save(self): + self._pw.save() diff --git a/pyproject.toml b/pyproject.toml index c4e6d55..072a9fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.12" authors = [{ name = "Piotr Dec" }] dependencies = [ "fastapi>=0.119.0", + "fastapi-utils>=0.8.0", "uvicorn[standard]>=0.30.0", "jinja2>=3.1.4", "pydantic-settings>=2.4.0", From 5224fe78b66323651bb50f0160217ce593fd7ec0 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 16 Oct 2025 23:10:04 +0200 Subject: [PATCH 02/93] fix: Group tree path --- app/web/passwords.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/app/web/passwords.py b/app/web/passwords.py index 92d3758..cd2c76b 100644 --- a/app/web/passwords.py +++ b/app/web/passwords.py @@ -47,16 +47,8 @@ class PasswordsController: return e def _node_of_group(self, g: Group) -> NodeDTO: - def group_path(gr: Group) -> str: - names: List[str] = [] - cur = gr - while cur and cur.name: - names.append(cur.name) - cur = cur.parent - names.reverse() - return "/" + "/".join(names) if names else "/" - - path = group_path(g) + raw_path = g.path + path = "/" + "/".join(raw_path) if raw_path else "/" groups = [self._node_of_group(sg) for sg in g.subgroups] entries = [EntryNodeDTO( kind=EntryKind.complex if (e.username or e.url or e.notes) else EntryKind.simple, From 7c0ef155678b0dba879e4ffad9f5e7c2f77f93d4 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 17 Oct 2025 20:41:52 +0200 Subject: [PATCH 03/93] fix: front-end autowire --- app/api/v1.py | 9 ++++++++- app/main.py | 2 +- app/services/passwords.py | 2 +- app/web/passwords.py | 13 +++++-------- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/app/api/v1.py b/app/api/v1.py index 31427e8..b1785a7 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,6 +1,7 @@ from fastapi import APIRouter, Depends, Body, Query from fastapi.responses import JSONResponse from fastapi_utils.cbv import cbv +from injectable import inject from app.api.models import * from app.web import PasswordsController @@ -17,14 +18,20 @@ async def root(): async def health(): return {"status": "ok"} + @router.post("/ci", summary="CI Webhook") async def ci(request: Request): return Response(200) +class AutowireSupport: + @staticmethod + def password_controller(): + return inject(PasswordsController) + @cbv(router) class TreeController: - svc: PasswordsController = Depends(PasswordsController.dep) + svc: PasswordsController = Depends(AutowireSupport.password_controller) @router.get("/tree", response_model=NodeDTO) def get_tree(self) -> NodeDTO: diff --git a/app/main.py b/app/main.py index f173a4a..03f6c34 100644 --- a/app/main.py +++ b/app/main.py @@ -13,6 +13,7 @@ class KarlApplication: from starlette.types import Receive, Scope, Send def __init__(self) -> None: self._set_logging() + load_injection_container() _instance = FastAPI(title="Karl", version="0.1.0") self._set_routes(_instance) self._set_events(_instance) @@ -52,7 +53,6 @@ class KarlApplication: def _init_services(self): logger = logging.getLogger(__name__) - load_injection_container() webhook_service = WebhookProcessor() logger.info(webhook_service.health) diff --git a/app/services/passwords.py b/app/services/passwords.py index fea898c..9b9412a 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -12,7 +12,7 @@ class Passwords: settings = get_settings() with open(settings.kp.secret, "r") as fh: - secret = fh.read() + secret = fh.read().splitlines()[0] self._path = settings.kp.file self._kp_org = self._open_or_create(self._path, secret) self._kp = self._open_lock(self._path, secret) diff --git a/app/web/passwords.py b/app/web/passwords.py index cd2c76b..b0e3d51 100644 --- a/app/web/passwords.py +++ b/app/web/passwords.py @@ -1,20 +1,17 @@ -from typing import Optional, List +from typing import Optional, List, Annotated +from injectable import autowired, injectable, Autowired from pykeepass import Group, Entry from app.api.models import EntryKind, EntrySimpleDTO, EntryComplexDTO, GroupDTO, NodeDTO, EntryNodeDTO from app.services import Passwords - +@injectable(singleton=True) class PasswordsController: - def __init__(self, passwords: Passwords): + @autowired + def __init__(self, passwords: Annotated[Passwords, Autowired]): self._pw = passwords - @staticmethod - def dep() -> "PasswordsController": - # prosta fabryka/DI mostkująca injectable - return PasswordsController(Passwords()) - # Helpers def _group_by_path(self, path: Optional[str]) -> Group: if not path or path == "/": From e3be433e3724cf391684b2f764bdb0346fd75c42 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 17 Oct 2025 21:12:38 +0200 Subject: [PATCH 04/93] fix: FastAPI Jinja wrapper --- app/core/router.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/core/router.py b/app/core/router.py index 0d1a300..475be95 100644 --- a/app/core/router.py +++ b/app/core/router.py @@ -2,19 +2,17 @@ from fastapi import APIRouter, Request from fastapi.responses import HTMLResponse from jinja2 import Environment, FileSystemLoader, select_autoescape +from fastapi.templating import Jinja2Templates router = APIRouter() # Inicjalizacja Jinja2 -templates_env = Environment( - loader=FileSystemLoader("app/templates"), - autoescape=select_autoescape(["html", "xml"]), -) +templates = Jinja2Templates(directory="app/templates") # Przykładowy endpoint HTML @router.get("/", response_class=HTMLResponse) async def index(request: Request) -> HTMLResponse: - template = templates_env.get_template("index.html") - html = template.render(title="Strona główna", request=request) - return HTMLResponse(content=html) + return templates.TemplateResponse(name="index.html", + request=request, + context={"title": "Strona Główna"}) From 0c9a3a8527b826a6cf2cd1252e4d8c6eb679fb2d Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 17 Oct 2025 21:12:49 +0200 Subject: [PATCH 05/93] formatting --- app/web/passwords.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/web/passwords.py b/app/web/passwords.py index b0e3d51..ec57742 100644 --- a/app/web/passwords.py +++ b/app/web/passwords.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Annotated +from typing import Optional, Annotated from injectable import autowired, injectable, Autowired from pykeepass import Group, Entry @@ -6,6 +6,7 @@ from pykeepass import Group, Entry from app.api.models import EntryKind, EntrySimpleDTO, EntryComplexDTO, GroupDTO, NodeDTO, EntryNodeDTO from app.services import Passwords + @injectable(singleton=True) class PasswordsController: @autowired From 5f155f6cb266cf38d04ef7816c68be85d8c3bd26 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 17 Oct 2025 21:35:13 +0200 Subject: [PATCH 06/93] directory for external files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 8395e24..b6f4f9b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ uv.lock __pycache__/ **/dist/ + +dump/ From de6184273a69aeed2aac4e3330795ea48f4a8ad7 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 17 Oct 2025 23:16:38 +0200 Subject: [PATCH 07/93] Webhook model changed --- app/api/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index 35afedb..b27765b 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -4,11 +4,11 @@ from typing import List @dataclass class Request: - build_id: str - build_url: str - commit_id: str - commit_url: str - changelist: List[str] + _id: str + commit: str + message: str + started: str + files: List[str] @dataclass class Response: From 120e929469d792310ee2c7a09c0c491338d45a5b Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 21 Oct 2025 17:41:41 +0200 Subject: [PATCH 08/93] APIv1 cbv --- app/api/v1.py | 24 ++++++++++++++++-------- app/core/injects.py | 10 ++++++++++ app/model/webhook.py | 11 +++++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 app/core/injects.py create mode 100644 app/model/webhook.py diff --git a/app/api/v1.py b/app/api/v1.py index fc52bd4..e628d1f 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,6 +1,10 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends +from fastapi_utils.cbv import cbv +from starlette.responses import JSONResponse, Response -from app.api.models import Request, Response +from app.api.models import Request +from app.core.core import WebhookProcessor +from app.core.injects import AutowireSupport router = APIRouter() @@ -10,10 +14,14 @@ async def root(): return {"message": "Witaj w API v1"} -@router.get("/health", summary="Health check") -async def health(): - return {"status": "ok"} +@cbv(router) +class APIv1: + webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) -@router.post("/ci", summary="CI Webhook") -async def ci(request: Request): - return Response(200) + @router.get("/health", summary="Health check") + async def health(self) -> JSONResponse: + return JSONResponse({"status": "ok"}) + + @router.post("/ci", summary="CI Webhook") + async def ci(self, request: Request): + return Response(status_code=201) diff --git a/app/core/injects.py b/app/core/injects.py new file mode 100644 index 0000000..3dfcc66 --- /dev/null +++ b/app/core/injects.py @@ -0,0 +1,10 @@ +from injectable import inject + +from app.core.core import WebhookProcessor + + +class AutowireSupport: + + @staticmethod + def webhook_processor(): + return inject(WebhookProcessor) diff --git a/app/model/webhook.py b/app/model/webhook.py new file mode 100644 index 0000000..70470b2 --- /dev/null +++ b/app/model/webhook.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import List + + +@dataclass +class WebhookRequest: + _id: str + commit: str + message: str + started: str + files: List[str] From 1d7c4c2dfdbe3fbefbaa5feb75eb243e93b653e6 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 21 Oct 2025 17:50:53 +0200 Subject: [PATCH 09/93] Mappings & processing WIP --- app/api/v1.py | 5 ++++- app/core/core.py | 6 +++++- app/model/webhook.py | 2 +- pyproject.toml | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app/api/v1.py b/app/api/v1.py index e628d1f..d6355fb 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,3 +1,4 @@ +from automapper import mapper from fastapi import APIRouter, Depends from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response @@ -5,6 +6,7 @@ from starlette.responses import JSONResponse, Response from app.api.models import Request from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport +from app.model.webhook import WebhookEvent router = APIRouter() @@ -20,8 +22,9 @@ class APIv1: @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: - return JSONResponse({"status": "ok"}) + return JSONResponse({"status": self.webhook_service.health}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): + self.webhook_service.process_ci_event(mapper.to(WebhookEvent).map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index 9e3acec..a7096d2 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -3,10 +3,11 @@ from typing import Annotated from injectable import injectable, autowired, Autowired from app.model.healthcheck import HealthCheck +from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords -# @injectable +@injectable class WebhookProcessor: @autowired def __init__(self, docker: Annotated[DockerService, Autowired], @@ -16,6 +17,9 @@ class WebhookProcessor: self._git = git self._keepass = keepass + def process_ci_event(self, event: WebhookEvent): + pass + @property def health(self) -> HealthCheck: return HealthCheck( diff --git a/app/model/webhook.py b/app/model/webhook.py index 70470b2..da9ad8d 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -3,7 +3,7 @@ from typing import List @dataclass -class WebhookRequest: +class WebhookEvent: _id: str commit: str message: str diff --git a/pyproject.toml b/pyproject.toml index c4e6d55..9b97797 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "pykeepass>=4.1.1.post1", "docker>=7.1.0", "injectable==4.0.1", + "py-automapper>=2.2.0", ] [project.optional-dependencies] From 2dec6d53844b5b3fbd5c9a1a8b639011b2da8309 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 19:42:57 +0200 Subject: [PATCH 10/93] Mappings enhancement --- app/api/v1.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/api/v1.py b/app/api/v1.py index d6355fb..eba78a4 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -19,6 +19,10 @@ async def root(): @cbv(router) class APIv1: webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) + logger = __import__('logging').getLogger(__name__) + + def __init__(self): + mapper.add(Request, WebhookEvent) @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: @@ -26,5 +30,5 @@ class APIv1: @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.process_ci_event(mapper.to(WebhookEvent).map(request)) + self.webhook_service.process_ci_event(mapper.map(request)) return Response(status_code=201) From 569aefeccb500ded48ca345009a0592ac91a76d4 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 22:53:49 +0200 Subject: [PATCH 11/93] logging enhancements --- .gitignore | 1 + app/config/settings.py | 6 ++++++ app/main.py | 25 ++++++++++++++-------- app/util/logging.py | 47 +++++++++++++++++++++++++++++++++--------- app/web/__init__.py | 0 app/web/middlewares.py | 24 +++++++++++++++++++++ config/config.yaml | 3 +++ run.sh | 2 +- 8 files changed, 88 insertions(+), 20 deletions(-) create mode 100644 app/web/__init__.py create mode 100644 app/web/middlewares.py diff --git a/.gitignore b/.gitignore index 8395e24..4b259c4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ uv.lock __pycache__/ **/dist/ +**/*.log diff --git a/app/config/settings.py b/app/config/settings.py index 7e072aa..2017c14 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -6,6 +6,11 @@ from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict +class LoggingConfig(BaseModel): + level: str = "INFO" + path: Path = Path("logs/karl.log") + + class AppConfig(BaseModel): host: str = "127.0.0.1" port: int = 8081 @@ -26,6 +31,7 @@ class KeePassConfig(BaseModel): class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__") + logging: LoggingConfig = LoggingConfig() app: AppConfig = AppConfig() git: GitConfig = GitConfig() kp: KeePassConfig = KeePassConfig() diff --git a/app/main.py b/app/main.py index 5bf8c98..424548e 100644 --- a/app/main.py +++ b/app/main.py @@ -4,8 +4,7 @@ from fastapi import FastAPI from injectable import load_injection_container from app.config import get_settings -from app.core.core import WebhookProcessor -from app.util.logging import LoggingHandler, ExternalLoggingHandler +from app.util.logging import HandlerFactory class KarlApplication: @@ -13,6 +12,7 @@ class KarlApplication: def __init__(self) -> None: self._set_logging() _app = FastAPI(title="Karl", version="0.1.0") + self._set_middlewares(_app) self._set_routes(_app) self._set_events(_app) self._init_services() @@ -23,7 +23,12 @@ class KarlApplication: await self._app.__call__(scope, receive, send) def _set_logging(self): - logging.basicConfig(level=logging.INFO, handlers=[LoggingHandler()]) + settings = get_settings() + logging.addLevelName(5, "TRACE") + logging.Logger.trace = lambda s, msg, *args, **kwargs: s.log(5, msg, *args, **kwargs) + logging.basicConfig(level=settings.logging.level, + handlers=HandlerFactory.create(HandlerFactory.Target.ALL, handler_prefix='karl.', + file_path=settings.logging.path)) loggers = ( "uvicorn", @@ -33,12 +38,16 @@ class KarlApplication: "asyncio", "starlette", ) - external_handler = ExternalLoggingHandler() + external_handlers = HandlerFactory.create(HandlerFactory.Target.ALL, file_path=settings.logging.path) for logger_name in loggers: logging_logger = logging.getLogger(logger_name) - logging_logger.handlers = [external_handler] + logging_logger.handlers = external_handlers logging_logger.propagate = False + def _set_middlewares(self, app: FastAPI): + from app.web.middlewares import LoggingMiddleware + app.add_middleware(LoggingMiddleware) + def _set_routes(self, app: FastAPI): from app.core.router import router as core_router app.include_router(core_router) @@ -52,11 +61,9 @@ class KarlApplication: def _init_services(self): logger = logging.getLogger(__name__) load_injection_container() - webhook_service = WebhookProcessor() - logger.info(webhook_service.health) -def app(): +def run(): return KarlApplication() @@ -65,7 +72,7 @@ if __name__ == "__main__": settings = get_settings() uvicorn.run( - "app.main:app", + "app.main:run", factory=True, host=settings.app.host, port=settings.app.port, diff --git a/app/util/logging.py b/app/util/logging.py index 29992b6..3e9e77a 100644 --- a/app/util/logging.py +++ b/app/util/logging.py @@ -1,4 +1,8 @@ -from logging import Formatter, StreamHandler +from enum import Enum, auto +from logging import Formatter, StreamHandler, Handler +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path +from typing import List class NamingCache: @@ -37,7 +41,7 @@ class ApplicationFormatter(Formatter): from datetime import datetime timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') level = record.levelname.replace('WARNING', 'WARN').rjust(5) - thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? + thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"] message = record.getMessage() formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}" @@ -48,12 +52,35 @@ class ApplicationFormatter(Formatter): return formatted -class LoggingHandler(StreamHandler): - def __init__(self): - super().__init__() - self.setFormatter(ApplicationFormatter(handler_prefix='karl.')) +class HandlerFactory: + class Target(Enum): + CONSOLE = auto() + FILE = auto() + ALL = auto() -class ExternalLoggingHandler(StreamHandler): - def __init__(self): - super().__init__() - self.setFormatter(ApplicationFormatter()) + @staticmethod + def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]: + def console_handler(prefix: str = ''): + handler = StreamHandler() + handler.setFormatter(ApplicationFormatter(prefix)) + handler.setLevel('INFO') + return handler + + def file_handler(prefix: str = ''): + handler = TimedRotatingFileHandler(file_path, when='midnight', backupCount=30) + handler.setFormatter(ApplicationFormatter(prefix)) + handler.setLevel('TRACE') + return handler + + handlers = [] + match target: + case HandlerFactory.Target.CONSOLE: + handlers.append(console_handler(handler_prefix)) + case HandlerFactory.Target.FILE: + handlers.append(file_handler(handler_prefix)) + case HandlerFactory.Target.ALL: + handlers.append(file_handler(handler_prefix)) + handlers.append(console_handler(handler_prefix)) + case _: + raise ValueError(f"Unknown target: {target}") + return handlers diff --git a/app/web/__init__.py b/app/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/web/middlewares.py b/app/web/middlewares.py new file mode 100644 index 0000000..d162ed1 --- /dev/null +++ b/app/web/middlewares.py @@ -0,0 +1,24 @@ +import logging + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request + +logger = logging.getLogger(__name__) + + +class LoggingMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + client = f"{request.client.host}:{request.client.port}" + match request.method: + case "POST" | "PUT" | "DELETE" if request.headers.get("Content-Type") == "application/json": + body = await request.body() + logger.trace(f"Request from {client}: {body.decode()}") + case "GET": + logger.trace(f"Request from {client}") + case _: + logger.trace(f"Request from {client} (content-type:{request.headers.get("Content-Type")})") + + response = await call_next(request) + logger.trace(f"Respone: {response.status_code} {type(response)}") + + return response diff --git a/config/config.yaml b/config/config.yaml index 856d24e..6c2043e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,3 +1,6 @@ +logging: + level: "TRACE" + path: "logs/karl.log" app: host: "127.0.0.1" port: 8081 diff --git a/run.sh b/run.sh index 8d2e2b9..cf3a77b 100644 --- a/run.sh +++ b/run.sh @@ -1 +1 @@ -uvicorn app.main:app --factory --reload +uvicorn app.main:run --factory --reload From 3845cc7ecf3bf357cb7aed77777222721bb2f0ba Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 23:58:15 +0200 Subject: [PATCH 12/93] queue --- app/api/v1.py | 2 +- app/core/core.py | 15 ++++++++++---- app/core/queue.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 app/core/queue.py diff --git a/app/api/v1.py b/app/api/v1.py index eba78a4..3950b3b 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -30,5 +30,5 @@ class APIv1: @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.process_ci_event(mapper.map(request)) + self.webhook_service.enqueue(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index a7096d2..adc68fd 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -1,23 +1,30 @@ +import uuid from typing import Annotated from injectable import injectable, autowired, Autowired +from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result from app.model.healthcheck import HealthCheck from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords -@injectable -class WebhookProcessor: +@injectable(singleton=True) +class WebhookProcessor(EnqueuedProcessor): @autowired def __init__(self, docker: Annotated[DockerService, Autowired], git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired]): + keepass: Annotated[Passwords, Autowired], + queue: Annotated[ProcessQueue, Autowired]): + super().__init__(queue) self._docker = docker self._git = git self._keepass = keepass - def process_ci_event(self, event: WebhookEvent): + def enqueue(self, event: WebhookEvent): + self._enqueue(Task(uuid.UUID(), self, event)) + + def _process(self, task: Task) -> Result: pass @property diff --git a/app/core/queue.py b/app/core/queue.py new file mode 100644 index 0000000..4788c3a --- /dev/null +++ b/app/core/queue.py @@ -0,0 +1,50 @@ +import time +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass +from multiprocessing import Queue, Process + +from injectable import injectable + + +@dataclass +class Task: + _id: uuid.UUID + processor: 'EnqueuedProcessor' + payload: object + +@dataclass +class Result: + _id: uuid.UUID + success: bool + error: str | None = None + +@injectable(singleton=True) +class ProcessQueue: + def __init__(self): + self._q = Queue() + self._process_thread = Process(target=self._run, args=(self._q,)) + self._process_thread.start() + + def put(self, task: Task): + self._q.put(task) + + def _run(self, queue: Queue): + while True: + if queue.empty(): + time.sleep(10) + continue + task = queue.get() + task.processor._process(task.payload) + + +class EnqueuedProcessor(ABC): + def __init__(self, queue: ProcessQueue): + self._queue = queue + + def _enqueue(self, task: Task): + self._queue.put(task) + + @abstractmethod + def _process(self, task: Task) -> Result: + pass From 1440ec51b71efb5f1603c6b9fc68c2e00f0417f2 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 30 Oct 2025 21:05:06 +0100 Subject: [PATCH 13/93] Generics & visitor pattern? --- app/core/core.py | 20 ++++++++++++++++++-- app/core/queue.py | 8 +++++--- app/services/containers.py | 5 +++++ app/services/vcs.py | 6 +++++- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/app/core/core.py b/app/core/core.py index adc68fd..90d49d0 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -1,5 +1,6 @@ +import logging import uuid -from typing import Annotated +from typing import Annotated, List from injectable import injectable, autowired, Autowired @@ -8,6 +9,7 @@ from app.model.healthcheck import HealthCheck from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords +logger = logging.getLogger(__name__) @injectable(singleton=True) class WebhookProcessor(EnqueuedProcessor): @@ -24,7 +26,21 @@ class WebhookProcessor(EnqueuedProcessor): def enqueue(self, event: WebhookEvent): self._enqueue(Task(uuid.UUID(), self, event)) - def _process(self, task: Task) -> Result: + def _process(self, task: Task[WebhookEvent]) -> Result: + event: WebhookEvent = task.payload + # TODO: persist event data + commit_hash = self._git.get_new_commit_hash() + if commit_hash != event.commit: + logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") + return Result(task.id, False, "Commit hash mismatch") + # TODO: persist commit data + service = self._get_service(event.files) + + + + return Result(task.id, True) + + def _get_service(self, files: List[str]) -> str: pass @property diff --git a/app/core/queue.py b/app/core/queue.py index 4788c3a..3e92cd6 100644 --- a/app/core/queue.py +++ b/app/core/queue.py @@ -3,15 +3,17 @@ import uuid from abc import ABC, abstractmethod from dataclasses import dataclass from multiprocessing import Queue, Process +from typing import TypeVar from injectable import injectable +T = TypeVar('T') @dataclass -class Task: - _id: uuid.UUID +class Task[T]: + id: uuid.UUID processor: 'EnqueuedProcessor' - payload: object + payload: T @dataclass class Result: diff --git a/app/services/containers.py b/app/services/containers.py index ef891e3..a89343b 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -1,14 +1,19 @@ +import logging + import docker from docker.models.containers import Container from injectable import injectable from app.model.containers import Tree, Compose, SimpleContainer +logger = logging.getLogger(__name__) + @injectable(singleton=True) class DockerService: def __init__(self): self._client = docker.from_env() + # logger.info(f"Docker client initialized. Plugins: {self._client.plugins()}") self._tree = self._init_tree() def _init_tree(self) -> Tree: diff --git a/app/services/vcs.py b/app/services/vcs.py index a7d83b0..abb5468 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -13,15 +13,19 @@ class GitService: self._repo.git.checkout(self._settings.git.branch) self._origin: Remote = self._repo.remotes.origin - def get_modified_compose(self) -> str | None: self._update() return self._diff() + def get_new_commit_hash(self) -> str: + self._update() + return self._repo.head.commit.hexsha + @staticmethod def _check_preconditions(config: GitConfig) -> Repo: def clone(): return Repo.clone_from(config.url, config.path, branch=config.branch) + import os if not config.path.exists(): return clone() From 87e8af3f729fa24fa02c5758ae9af7a9406b9698 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 31 Oct 2025 00:01:45 +0100 Subject: [PATCH 14/93] Event bus basics --- app/api/v1.py | 14 +++++++++---- app/core/core.py | 10 +++++----- app/core/injects.py | 5 +++++ app/core/woodpecker.py | 22 +++++++++++++++++++++ app/events/__init__.py | 41 +++++++++++++++++++++++++++++++++++++++ app/main.py | 6 +----- app/model/webhook.py | 4 +++- app/services/passwords.py | 2 +- 8 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 app/core/woodpecker.py create mode 100644 app/events/__init__.py diff --git a/app/api/v1.py b/app/api/v1.py index 3950b3b..4fbe31f 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,4 +1,4 @@ -from automapper import mapper +from automapper import mapper, exceptions from fastapi import APIRouter, Depends from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response @@ -6,6 +6,7 @@ from starlette.responses import JSONResponse, Response from app.api.models import Request from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport +from app.events import SimpleEventBus from app.model.webhook import WebhookEvent router = APIRouter() @@ -19,16 +20,21 @@ async def root(): @cbv(router) class APIv1: webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) + event_bus: SimpleEventBus = Depends(AutowireSupport.event_bus) logger = __import__('logging').getLogger(__name__) def __init__(self): - mapper.add(Request, WebhookEvent) + try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie + mapper.add(Request, WebhookEvent) + except exceptions.DuplicatedRegistrationError: + pass @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: - return JSONResponse({"status": self.webhook_service.health}) + # TODO: JSON serialize + return JSONResponse({"status": self.webhook_service.health.healthy}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.enqueue(mapper.map(request)) + self.event_bus.publish(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index 90d49d0..3b16ea1 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -3,6 +3,7 @@ import uuid from typing import Annotated, List from injectable import injectable, autowired, Autowired +from typing_extensions import deprecated from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result from app.model.healthcheck import HealthCheck @@ -11,20 +12,19 @@ from app.services import DockerService, GitService, Passwords logger = logging.getLogger(__name__) +@deprecated("Use event bus instead.") @injectable(singleton=True) -class WebhookProcessor(EnqueuedProcessor): +class WebhookProcessor: @autowired def __init__(self, docker: Annotated[DockerService, Autowired], git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired], - queue: Annotated[ProcessQueue, Autowired]): - super().__init__(queue) + keepass: Annotated[Passwords, Autowired]): self._docker = docker self._git = git self._keepass = keepass def enqueue(self, event: WebhookEvent): - self._enqueue(Task(uuid.UUID(), self, event)) + pass def _process(self, task: Task[WebhookEvent]) -> Result: event: WebhookEvent = task.payload diff --git a/app/core/injects.py b/app/core/injects.py index 3dfcc66..0d7ff9c 100644 --- a/app/core/injects.py +++ b/app/core/injects.py @@ -1,6 +1,7 @@ from injectable import inject from app.core.core import WebhookProcessor +from app.events import SimpleEventBus class AutowireSupport: @@ -8,3 +9,7 @@ class AutowireSupport: @staticmethod def webhook_processor(): return inject(WebhookProcessor) + + @staticmethod + def event_bus(): + return inject(SimpleEventBus) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py new file mode 100644 index 0000000..8236bf5 --- /dev/null +++ b/app/core/woodpecker.py @@ -0,0 +1,22 @@ +import logging + +from injectable import injectable + +from app.events import SimpleEventBus +from app.model.webhook import WebhookEvent + +logger = logging.getLogger(__name__) + +@injectable +class Woodpecker: + + @SimpleEventBus.on(WebhookEvent) + def on_event(self, event): # TODO: caller nie działa -> brakuje instancji klasy? + logger.info(f"Received event: {event}") + pass + + +@SimpleEventBus.on(WebhookEvent) +def on_event2(event): # TODO: Tu działa + logger.info(f"F2: Received event: {event}") + pass diff --git a/app/events/__init__.py b/app/events/__init__.py new file mode 100644 index 0000000..fbd92d2 --- /dev/null +++ b/app/events/__init__.py @@ -0,0 +1,41 @@ +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import wraps +from typing import Dict, List, Callable + +from injectable import injectable, inject + + +@dataclass +class Event: + pass + + +@injectable(singleton=True) +class SimpleEventBus: + def __init__(self): + self._handlers: Dict[type, List[Callable]] = {} + self._executor = ThreadPoolExecutor() + + def publish(self, event: Event) -> None: + for handler in self._handlers.get(type(event), []): + # Fire-and-forget execution + self._executor.submit(handler, event) + + def subscribe(self, event_type: type, handler: Callable) -> None: + if event_type not in self._handlers: + self._handlers[event_type] = [] + self._handlers[event_type].append(handler) + + @staticmethod + def on(event: type) -> Callable: + def outer(func): + inject(SimpleEventBus).subscribe(event, func) + + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + return outer diff --git a/app/main.py b/app/main.py index 424548e..669d2d4 100644 --- a/app/main.py +++ b/app/main.py @@ -11,11 +11,11 @@ class KarlApplication: from starlette.types import Receive, Scope, Send def __init__(self) -> None: self._set_logging() + load_injection_container() _app = FastAPI(title="Karl", version="0.1.0") self._set_middlewares(_app) self._set_routes(_app) self._set_events(_app) - self._init_services() self._app = _app @@ -58,10 +58,6 @@ class KarlApplication: def _set_events(self, app: FastAPI): pass - def _init_services(self): - logger = logging.getLogger(__name__) - load_injection_container() - def run(): return KarlApplication() diff --git a/app/model/webhook.py b/app/model/webhook.py index da9ad8d..e8c3b0d 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,9 +1,11 @@ from dataclasses import dataclass from typing import List +from app.events import Event + @dataclass -class WebhookEvent: +class WebhookEvent(Event): _id: str commit: str message: str diff --git a/app/services/passwords.py b/app/services/passwords.py index 18d8519..6ff8406 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -11,7 +11,7 @@ class Passwords: settings = get_settings() with open(settings.kp.secret, "r") as fh: - secret = fh.read() + secret = fh.read().splitlines()[0] self._kp_org = self.__get_or_create_store(settings.kp.file, secret) self._kp = self.__get_lock(settings.kp.file, secret) From 29dfc13a48a4910780de9a10e0f5990a9e49ec54 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 31 Oct 2025 23:30:35 +0100 Subject: [PATCH 15/93] Woodpecker event registered --- app/core/woodpecker.py | 21 ++++++++++++--------- app/events/__init__.py | 6 +++--- config/config.yaml | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 8236bf5..9b6ffb4 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,22 +1,25 @@ import logging +from typing import Annotated -from injectable import injectable +from injectable import injectable, Autowired, autowired, inject, injectable_factory from app.events import SimpleEventBus from app.model.webhook import WebhookEvent logger = logging.getLogger(__name__) -@injectable -class Woodpecker: - @SimpleEventBus.on(WebhookEvent) - def on_event(self, event): # TODO: caller nie działa -> brakuje instancji klasy? +@injectable(singleton=True) +class Woodpecker: + @autowired + def __init__(self, event_bus: Annotated[SimpleEventBus, Autowired]): + logger.info("Woodpecker initialized.") + event_bus.subscribe(WebhookEvent, self.on_ci_event) + + def on_ci_event(self, event): logger.info(f"Received event: {event}") pass -@SimpleEventBus.on(WebhookEvent) -def on_event2(event): # TODO: Tu działa - logger.info(f"F2: Received event: {event}") - pass +instance = Woodpecker(inject(SimpleEventBus)) +injectable_factory(Woodpecker)(lambda: instance) diff --git a/app/events/__init__.py b/app/events/__init__.py index fbd92d2..5d54a87 100644 --- a/app/events/__init__.py +++ b/app/events/__init__.py @@ -18,9 +18,9 @@ class SimpleEventBus: self._executor = ThreadPoolExecutor() def publish(self, event: Event) -> None: - for handler in self._handlers.get(type(event), []): - # Fire-and-forget execution - self._executor.submit(handler, event) + for handler in self._handlers.get(type(event), []): + # Fire-and-forget execution + self._executor.submit(handler, event) def subscribe(self, event_type: type, handler: Callable) -> None: if event_type not in self._handlers: diff --git a/config/config.yaml b/config/config.yaml index 6c2043e..c4fc44a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,7 +4,7 @@ logging: app: host: "127.0.0.1" port: 8081 - reload: true + reload: false git: path: "F:/IdeaProjects/paas/karl/.compose_repository" branch: "main" From 34ee5f87549a7838404e1c8218d344032d6367ca Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sat, 1 Nov 2025 23:02:28 +0100 Subject: [PATCH 16/93] CI events flow simplified --- app/api/v1.py | 16 ++++++------ app/core/core.py | 51 ------------------------------------- app/core/injects.py | 11 +++----- app/core/queue.py | 52 -------------------------------------- app/core/woodpecker.py | 57 +++++++++++++++++++++++++++++++++++------- app/events/__init__.py | 41 ------------------------------ app/model/webhook.py | 5 +--- 7 files changed, 59 insertions(+), 174 deletions(-) delete mode 100644 app/core/core.py delete mode 100644 app/core/queue.py delete mode 100644 app/events/__init__.py diff --git a/app/api/v1.py b/app/api/v1.py index 4fbe31f..a012834 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -4,10 +4,9 @@ from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response from app.api.models import Request -from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport -from app.events import SimpleEventBus -from app.model.webhook import WebhookEvent +from app.core.woodpecker import Woodpecker +from app.model.webhook import WoodpeckerEvent router = APIRouter() @@ -19,22 +18,21 @@ async def root(): @cbv(router) class APIv1: - webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) - event_bus: SimpleEventBus = Depends(AutowireSupport.event_bus) + woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) logger = __import__('logging').getLogger(__name__) def __init__(self): - try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie - mapper.add(Request, WebhookEvent) + try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie + mapper.add(Request, WoodpeckerEvent) except exceptions.DuplicatedRegistrationError: pass @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: # TODO: JSON serialize - return JSONResponse({"status": self.webhook_service.health.healthy}) + return JSONResponse({"status": "unknown"}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.event_bus.publish(mapper.map(request)) + self.woodpecker.on_ci_event(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py deleted file mode 100644 index 3b16ea1..0000000 --- a/app/core/core.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging -import uuid -from typing import Annotated, List - -from injectable import injectable, autowired, Autowired -from typing_extensions import deprecated - -from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result -from app.model.healthcheck import HealthCheck -from app.model.webhook import WebhookEvent -from app.services import DockerService, GitService, Passwords - -logger = logging.getLogger(__name__) - -@deprecated("Use event bus instead.") -@injectable(singleton=True) -class WebhookProcessor: - @autowired - def __init__(self, docker: Annotated[DockerService, Autowired], - git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired]): - self._docker = docker - self._git = git - self._keepass = keepass - - def enqueue(self, event: WebhookEvent): - pass - - def _process(self, task: Task[WebhookEvent]) -> Result: - event: WebhookEvent = task.payload - # TODO: persist event data - commit_hash = self._git.get_new_commit_hash() - if commit_hash != event.commit: - logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") - return Result(task.id, False, "Commit hash mismatch") - # TODO: persist commit data - service = self._get_service(event.files) - - - - return Result(task.id, True) - - def _get_service(self, files: List[str]) -> str: - pass - - @property - def health(self) -> HealthCheck: - return HealthCheck( - self._docker is not None and self._git is not None and self._keepass is not None, - f"Docker: {self._docker is not None}, Git: {self._git is not None}, KeePass: {self._keepass is not None}" - ) diff --git a/app/core/injects.py b/app/core/injects.py index 0d7ff9c..115ae95 100644 --- a/app/core/injects.py +++ b/app/core/injects.py @@ -1,15 +1,10 @@ from injectable import inject -from app.core.core import WebhookProcessor -from app.events import SimpleEventBus +from app.core.woodpecker import Woodpecker class AutowireSupport: @staticmethod - def webhook_processor(): - return inject(WebhookProcessor) - - @staticmethod - def event_bus(): - return inject(SimpleEventBus) + def woodpecker(): + return inject(Woodpecker) diff --git a/app/core/queue.py b/app/core/queue.py deleted file mode 100644 index 3e92cd6..0000000 --- a/app/core/queue.py +++ /dev/null @@ -1,52 +0,0 @@ -import time -import uuid -from abc import ABC, abstractmethod -from dataclasses import dataclass -from multiprocessing import Queue, Process -from typing import TypeVar - -from injectable import injectable - -T = TypeVar('T') - -@dataclass -class Task[T]: - id: uuid.UUID - processor: 'EnqueuedProcessor' - payload: T - -@dataclass -class Result: - _id: uuid.UUID - success: bool - error: str | None = None - -@injectable(singleton=True) -class ProcessQueue: - def __init__(self): - self._q = Queue() - self._process_thread = Process(target=self._run, args=(self._q,)) - self._process_thread.start() - - def put(self, task: Task): - self._q.put(task) - - def _run(self, queue: Queue): - while True: - if queue.empty(): - time.sleep(10) - continue - task = queue.get() - task.processor._process(task.payload) - - -class EnqueuedProcessor(ABC): - def __init__(self, queue: ProcessQueue): - self._queue = queue - - def _enqueue(self, task: Task): - self._queue.put(task) - - @abstractmethod - def _process(self, task: Task) -> Result: - pass diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 9b6ffb4..9ba242d 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,25 +1,64 @@ import logging +from collections import deque +from multiprocessing import Process, Lock from typing import Annotated -from injectable import injectable, Autowired, autowired, inject, injectable_factory +from injectable import injectable, Autowired, autowired -from app.events import SimpleEventBus -from app.model.webhook import WebhookEvent +from app.model.webhook import WoodpeckerEvent +from app.services import Passwords, GitService, DockerService logger = logging.getLogger(__name__) +class WoodpeckerRunner(Process): + def __init__(self, event: WoodpeckerEvent): + super().__init__() + self._event = event + + def run(self): + super().run() + """ + event: WebhookEvent = task.payload + # TODO: persist event data + commit_hash = self._git.get_new_commit_hash() + if commit_hash != event.commit: + logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") + return Result(task.id, False, "Commit hash mismatch") + # TODO: persist commit data + service = self._get_service(event.files) + + """ + + @injectable(singleton=True) class Woodpecker: @autowired - def __init__(self, event_bus: Annotated[SimpleEventBus, Autowired]): + def __init__(self, passwords: Annotated[Passwords, Autowired]): + self._passwords = passwords + self._git = GitService() + self._docker = DockerService() + self._runner: WoodpeckerRunner | None = None + self._pending = deque() + self._lock = Lock() logger.info("Woodpecker initialized.") - event_bus.subscribe(WebhookEvent, self.on_ci_event) - def on_ci_event(self, event): + def on_ci_event(self, event: WoodpeckerEvent): logger.info(f"Received event: {event}") + with self._lock: + if len(self._pending) > 0 or self._runner is not None: + self._pending.append(event) + return + self._start_runner(event) + + def _start_runner(self, event: WoodpeckerEvent): pass - -instance = Woodpecker(inject(SimpleEventBus)) -injectable_factory(Woodpecker)(lambda: instance) + def _on_runner_completed(self): + logger.info("Runner completed.") + self._runner.join() + with self._lock: + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) diff --git a/app/events/__init__.py b/app/events/__init__.py deleted file mode 100644 index 5d54a87..0000000 --- a/app/events/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from functools import wraps -from typing import Dict, List, Callable - -from injectable import injectable, inject - - -@dataclass -class Event: - pass - - -@injectable(singleton=True) -class SimpleEventBus: - def __init__(self): - self._handlers: Dict[type, List[Callable]] = {} - self._executor = ThreadPoolExecutor() - - def publish(self, event: Event) -> None: - for handler in self._handlers.get(type(event), []): - # Fire-and-forget execution - self._executor.submit(handler, event) - - def subscribe(self, event_type: type, handler: Callable) -> None: - if event_type not in self._handlers: - self._handlers[event_type] = [] - self._handlers[event_type].append(handler) - - @staticmethod - def on(event: type) -> Callable: - def outer(func): - inject(SimpleEventBus).subscribe(event, func) - - @wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - - return wrapper - - return outer diff --git a/app/model/webhook.py b/app/model/webhook.py index e8c3b0d..364e17b 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,11 +1,8 @@ from dataclasses import dataclass from typing import List -from app.events import Event - - @dataclass -class WebhookEvent(Event): +class WoodpeckerEvent: _id: str commit: str message: str From 312631f0b57fe7bacc9fc2dbcea91a832180fe16 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 00:35:33 +0100 Subject: [PATCH 17/93] WoodpeckerRunner & some VCS changes --- app/core/woodpecker.py | 66 +++++++++++++++++++++++++++++++++--------- app/services/vcs.py | 23 ++------------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 9ba242d..ae8553b 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -12,23 +12,50 @@ logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, event: WoodpeckerEvent): - super().__init__() + def __init__(self, git: GitService, docker: DockerService, success_callback=None, error_callback=None): + super().__init__(daemon=True) + self._git = git + self._docker = docker + self._success_callback = success_callback + self._error_callback = error_callback + self._event: WoodpeckerEvent | None = None + + def process_event(self, event: WoodpeckerEvent): self._event = event + self.start() def run(self): - super().run() - """ - event: WebhookEvent = task.payload - # TODO: persist event data - commit_hash = self._git.get_new_commit_hash() - if commit_hash != event.commit: - logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") - return Result(task.id, False, "Commit hash mismatch") - # TODO: persist commit data - service = self._get_service(event.files) + try: + service = self.get_service(self._event.files) + if service is None: + logger.info("No service found.") + return self._success_callback() - """ + self._git.checkout(self._event.commit) + """ + TODO: + check for *.mo.* files + subs mo from pass + docker compose up -d -f service/docker-compose.yml + """ + + return self._success_callback() + except Exception as e: + return self._error_callback(e) + + def get_service(self, files: list[str]) -> str | None: + supported_files = [] + for f in files: + f_parts = f.split("/") + if f_parts[0] in ["compose", "files"]: + supported_files.append(f[1]) + match len(set(supported_files)): + case 0: + return None + case 1: + return supported_files[0] + case _: + raise Exception("Multiple services are not supported.") @injectable(singleton=True) @@ -52,7 +79,9 @@ class Woodpecker: self._start_runner(event) def _start_runner(self, event: WoodpeckerEvent): - pass + with self._lock: + self._runner = WoodpeckerRunner(self._git, self._docker, self._on_runner_completed) + self._runner.process_event(event) def _on_runner_completed(self): logger.info("Runner completed.") @@ -62,3 +91,12 @@ class Woodpecker: if len(self._pending) > 0: event = self._pending.popleft() self._start_runner(event) + + def _on_runner_error(self, t: Exception): + logger.error(f"Runner error: {t}", exc_info=True) + self._runner.join() + with self._lock: + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) diff --git a/app/services/vcs.py b/app/services/vcs.py index abb5468..00ef3c2 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -13,14 +13,6 @@ class GitService: self._repo.git.checkout(self._settings.git.branch) self._origin: Remote = self._repo.remotes.origin - def get_modified_compose(self) -> str | None: - self._update() - return self._diff() - - def get_new_commit_hash(self) -> str: - self._update() - return self._repo.head.commit.hexsha - @staticmethod def _check_preconditions(config: GitConfig) -> Repo: def clone(): @@ -34,16 +26,5 @@ class GitService: return clone() return Repo(config.path) - def _update(self): - self._origin.pull() - - def _diff(self) -> str | None: - diff = self._repo.head.commit.diff("HEAD~1") - composes = [f for f in diff if f.a_path.endswith("docker-compose.yml")] - match len(composes): - case 0: - return None - case 1: - return composes[0].a_path - case _: - raise Exception("Multiple compose files modified") + def checkout(self, sha: str): + pass From 1341b022d42b1bd4377799b53a02f7d9edf026c7 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:04:21 +0100 Subject: [PATCH 18/93] Woodpecker main loop almost finished --- app/core/woodpecker.py | 18 ++++++++++-------- app/services/containers.py | 4 ++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index ae8553b..caa29d5 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,10 +1,12 @@ import logging from collections import deque from multiprocessing import Process, Lock +from pathlib import Path from typing import Annotated from injectable import injectable, Autowired, autowired +from app.config import get_settings from app.model.webhook import WoodpeckerEvent from app.services import Passwords, GitService, DockerService @@ -12,10 +14,12 @@ logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, git: GitService, docker: DockerService, success_callback=None, error_callback=None): + def __init__(self, git: GitService, docker: DockerService, passwords: Passwords, + success_callback=None, error_callback=None): super().__init__(daemon=True) self._git = git self._docker = docker + self._passwords = passwords self._success_callback = success_callback self._error_callback = error_callback self._event: WoodpeckerEvent | None = None @@ -30,14 +34,12 @@ class WoodpeckerRunner(Process): if service is None: logger.info("No service found.") return self._success_callback() - + service_path = f"{get_settings().git.path}/compose/{service}/docker-compose.yml" self._git.checkout(self._event.commit) - """ - TODO: - check for *.mo.* files - subs mo from pass - docker compose up -d -f service/docker-compose.yml - """ + for file in self._event.files: + if file.__contains__('.mo.'): + pass + self._docker.reload(Path(service_path)) return self._success_callback() except Exception as e: diff --git a/app/services/containers.py b/app/services/containers.py index a89343b..20f0098 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path import docker from docker.models.containers import Container @@ -33,3 +34,6 @@ class DockerService: @property def tree(self) -> Tree: return self._tree + + def reload(self, compose_path: Path): + pass From 8ee950940e027db2e4b0202f5815e3df724d3589 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:15:12 +0100 Subject: [PATCH 19/93] Mo basics --- app/core/woodpecker.py | 21 ++++++++++++--------- app/services/mo.py | 16 ++++++++++++++++ app/services/passwords.py | 33 ++++++++++++++++----------------- 3 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 app/services/mo.py diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index caa29d5..1892c0b 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -8,21 +8,23 @@ from injectable import injectable, Autowired, autowired from app.config import get_settings from app.model.webhook import WoodpeckerEvent -from app.services import Passwords, GitService, DockerService +from app.services import GitService, DockerService +from app.services.mo import Mo logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, git: GitService, docker: DockerService, passwords: Passwords, + def __init__(self, git: GitService, docker: DockerService, mo: Mo, success_callback=None, error_callback=None): super().__init__(daemon=True) self._git = git self._docker = docker - self._passwords = passwords + self._mo = mo self._success_callback = success_callback self._error_callback = error_callback self._event: WoodpeckerEvent | None = None + self._root = get_settings().git.path def process_event(self, event: WoodpeckerEvent): self._event = event @@ -34,12 +36,12 @@ class WoodpeckerRunner(Process): if service is None: logger.info("No service found.") return self._success_callback() - service_path = f"{get_settings().git.path}/compose/{service}/docker-compose.yml" + service_path = f"{self._root}/compose/{service}/docker-compose.yml" self._git.checkout(self._event.commit) for file in self._event.files: if file.__contains__('.mo.'): - pass - self._docker.reload(Path(service_path)) + self._mo.process(Path(f"{self._root}{file}").absolute()) + self._docker.reload(Path(service_path).absolute()) return self._success_callback() except Exception as e: @@ -63,8 +65,8 @@ class WoodpeckerRunner(Process): @injectable(singleton=True) class Woodpecker: @autowired - def __init__(self, passwords: Annotated[Passwords, Autowired]): - self._passwords = passwords + def __init__(self, mo: Annotated[Mo, Autowired]): + self._mo = mo self._git = GitService() self._docker = DockerService() self._runner: WoodpeckerRunner | None = None @@ -82,7 +84,8 @@ class Woodpecker: def _start_runner(self, event: WoodpeckerEvent): with self._lock: - self._runner = WoodpeckerRunner(self._git, self._docker, self._on_runner_completed) + self._runner = WoodpeckerRunner(self._git, self._docker, self._mo, + self._on_runner_completed, self._on_runner_error) self._runner.process_event(event) def _on_runner_completed(self): diff --git a/app/services/mo.py b/app/services/mo.py new file mode 100644 index 0000000..3b1520d --- /dev/null +++ b/app/services/mo.py @@ -0,0 +1,16 @@ +from pathlib import Path +from typing import Annotated + +from injectable import injectable, autowired, Autowired + +from app.services import Passwords + + +@injectable +class Mo: + @autowired + def __init__(self, passwords: Annotated[Passwords, Autowired]): + self._passwords = passwords + + def process(self, mo_file: Path): + pass diff --git a/app/services/passwords.py b/app/services/passwords.py index 6ff8406..e3a181c 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -1,7 +1,8 @@ import os.path +import shutil from injectable import injectable -from pykeepass import PyKeePass, create_database, Group +from pykeepass import PyKeePass, create_database @injectable(singleton=True) @@ -12,29 +13,27 @@ class Passwords: with open(settings.kp.secret, "r") as fh: secret = fh.read().splitlines()[0] - - self._kp_org = self.__get_or_create_store(settings.kp.file, secret) - self._kp = self.__get_lock(settings.kp.file, secret) + self._path = settings.kp.file + self._kp_org = self._open_or_create(self._path, secret) + self._kp = self._open_lock(self._path, secret) @staticmethod - def __get_or_create_store(path, passwd) -> PyKeePass: + def _open_or_create(path, password) -> PyKeePass: if os.path.exists(path): - return PyKeePass( - path, - password=passwd, - ) - return create_database(path, passwd) + return PyKeePass(path, password=password) + return create_database(path, password) @staticmethod - def __get_lock(path, passwd) -> PyKeePass: + def _open_lock(path, password) -> PyKeePass: lock_path = path + ".lock" - import shutil shutil.copyfile(path, lock_path) - return Passwords.__get_or_create_store(lock_path, passwd) + return Passwords._open_or_create(lock_path, password) @property - def store(self): - return self._kp.root_group + def kp(self) -> PyKeePass: + return self._kp - def save(self, group: Group): - pass + def save(self): + # nadpisz plik źródłowy zmianami z lock + self._kp.save() + shutil.copyfile(self._path + ".lock", self._path) From 3c3be6d79c2dde279d940b6d72978de3745004d5 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:59:40 +0100 Subject: [PATCH 20/93] chevron removed --- app/services/mo.py | 10 +++++++++- app/services/passwords.py | 5 ++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/services/mo.py b/app/services/mo.py index 3b1520d..a1ae32c 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -1,4 +1,5 @@ from pathlib import Path +from string import Template from typing import Annotated from injectable import injectable, autowired, Autowired @@ -13,4 +14,11 @@ class Mo: self._passwords = passwords def process(self, mo_file: Path): - pass + raw = '' + with open(mo_file, "r") as mo: + raw = mo.read() + tpl = Template(raw) + rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) + de_mo_ified = str(mo_file).replace(".mo", "") + with open(de_mo_ified, "w") as mo: + mo.write(rendered) diff --git a/app/services/passwords.py b/app/services/passwords.py index e3a181c..1c656c6 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -29,9 +29,8 @@ class Passwords: shutil.copyfile(path, lock_path) return Passwords._open_or_create(lock_path, password) - @property - def kp(self) -> PyKeePass: - return self._kp + def get_values(self, keys: list[str]) -> dict[str, str]: + return {} def save(self): # nadpisz plik źródłowy zmianami z lock From e14180cbe7e5641aaff3762db8d8e958964aedc2 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:16:54 +0100 Subject: [PATCH 21/93] passwords#get_values --- app/services/passwords.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/services/passwords.py b/app/services/passwords.py index 1c656c6..7f30567 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -30,7 +30,15 @@ class Passwords: return Passwords._open_or_create(lock_path, password) def get_values(self, keys: list[str]) -> dict[str, str]: - return {} + output = {} + for k in keys: + key_parts = k.split(".") + path = key_parts[:-2] if len(key_parts) > 2 else None + entry_name = key_parts[-2] + field_name = key_parts[-1] + kp_entry = self._kp_org.find_entries(path=path, first=True, name=entry_name)[0] + output[k] = kp_entry[field_name] + return output def save(self): # nadpisz plik źródłowy zmianami z lock From 9166790de90d90f1766ca6a17050365d0523ef55 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:50:24 +0100 Subject: [PATCH 22/93] mo processing test --- app/services/mo.py | 6 +++++- app/services/passwords.py | 4 ++-- tests/__init__.py | 0 tests/files/test1/test.mo.yaml | 3 +++ tests/test_mo.py | 21 +++++++++++++++++++++ 5 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/files/test1/test.mo.yaml create mode 100644 tests/test_mo.py diff --git a/app/services/mo.py b/app/services/mo.py index a1ae32c..d917e62 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -6,6 +6,10 @@ from injectable import injectable, autowired, Autowired from app.services import Passwords +class DotTemplate(Template): + # Pozwala na kropki w nazwach placeholderów, np. ${user.name.first} + idpattern = r'[_a-zA-Z][_a-zA-Z0-9.]*' + @injectable class Mo: @@ -17,7 +21,7 @@ class Mo: raw = '' with open(mo_file, "r") as mo: raw = mo.read() - tpl = Template(raw) + tpl = DotTemplate(raw) rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) de_mo_ified = str(mo_file).replace(".mo", "") with open(de_mo_ified, "w") as mo: diff --git a/app/services/passwords.py b/app/services/passwords.py index 7f30567..9b11ca4 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -36,8 +36,8 @@ class Passwords: path = key_parts[:-2] if len(key_parts) > 2 else None entry_name = key_parts[-2] field_name = key_parts[-1] - kp_entry = self._kp_org.find_entries(path=path, first=True, name=entry_name)[0] - output[k] = kp_entry[field_name] + kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name)[0] + output[k] = kp_entry[field_name] # TODO: TypeError: 'Entry' object is not subscriptable return output def save(self): diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/files/test1/test.mo.yaml b/tests/files/test1/test.mo.yaml new file mode 100644 index 0000000..af55ecc --- /dev/null +++ b/tests/files/test1/test.mo.yaml @@ -0,0 +1,3 @@ +value: ${sample.password} +nested: ${some.nested.value.password} +custom: ${custom.field} diff --git a/tests/test_mo.py b/tests/test_mo.py new file mode 100644 index 0000000..731dada --- /dev/null +++ b/tests/test_mo.py @@ -0,0 +1,21 @@ +import os +from pathlib import Path +from unittest import TestCase + +from app.services import Passwords +from app.services.mo import Mo + + +class TestMo(TestCase): + def test_process(self): + mo = Mo(Passwords()) + mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) + self.assertTrue(os.path.exists('tests/files/test1/test.mo')) + with open('tests/files/test1/test.mo', 'r') as f: + content = f.read() + self.assertFalse(content.__contains__('${')) + """ + value: some_oass + nested: nested_pass + custom: custom_content + """ From e3a37419e8c59dd4049306f354ee523033dea380 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:50:24 +0100 Subject: [PATCH 23/93] mo processing test --- tests/test_mo.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_mo.py b/tests/test_mo.py index 731dada..200b3d8 100644 --- a/tests/test_mo.py +++ b/tests/test_mo.py @@ -2,6 +2,8 @@ import os from pathlib import Path from unittest import TestCase +import yaml + from app.services import Passwords from app.services.mo import Mo @@ -11,9 +13,13 @@ class TestMo(TestCase): mo = Mo(Passwords()) mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) self.assertTrue(os.path.exists('tests/files/test1/test.mo')) - with open('tests/files/test1/test.mo', 'r') as f: + with open('tests/files/test1/test.yaml', 'r') as f: content = f.read() self.assertFalse(content.__contains__('${')) + parsed = yaml.load(content, Loader=yaml.FullLoader) + self.assertEqual(parsed['value'], 'some_oass') + self.assertEqual(parsed['nested'], 'nested_pass') + self.assertEqual(parsed['custom'], 'custom_content') """ value: some_oass nested: nested_pass From 3dc27cc86838d5683a3356e5b5886308ae3a1379 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 21:56:59 +0100 Subject: [PATCH 24/93] Simple/Complex templates / parsers --- app/services/mo.py | 15 ++++++++++++--- app/services/passwords.py | 20 +++++++++++++++++--- tests/files/test1/test.mo.yaml | 6 +++--- tests/test_mo.py | 16 ++++++---------- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/app/services/mo.py b/app/services/mo.py index d917e62..9a35642 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -6,11 +6,16 @@ from injectable import injectable, autowired, Autowired from app.services import Passwords -class DotTemplate(Template): + +class SimpleValueTemplate(Template): # Pozwala na kropki w nazwach placeholderów, np. ${user.name.first} idpattern = r'[_a-zA-Z][_a-zA-Z0-9.]*' +class ComplexValueTemplate(SimpleValueTemplate): + delimiter = '@' + + @injectable class Mo: @autowired @@ -21,8 +26,12 @@ class Mo: raw = '' with open(mo_file, "r") as mo: raw = mo.read() - tpl = DotTemplate(raw) - rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) + cmp = ComplexValueTemplate(raw) + rendered = cmp.substitute(self._passwords.get_values(cmp.get_identifiers())) + smp = SimpleValueTemplate(rendered) + ids = [_id + '.password' for _id in smp.get_identifiers()] + mappings = {k.replace('.password', ''): v for k, v in self._passwords.get_values(ids).items()} + rendered = smp.substitute(mappings) de_mo_ified = str(mo_file).replace(".mo", "") with open(de_mo_ified, "w") as mo: mo.write(rendered) diff --git a/app/services/passwords.py b/app/services/passwords.py index 9b11ca4..408ec5e 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -33,13 +33,27 @@ class Passwords: output = {} for k in keys: key_parts = k.split(".") - path = key_parts[:-2] if len(key_parts) > 2 else None + path = key_parts[:-1] if len(key_parts) > 2 else None entry_name = key_parts[-2] field_name = key_parts[-1] - kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name)[0] - output[k] = kp_entry[field_name] # TODO: TypeError: 'Entry' object is not subscriptable + kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name) + output[k] = self._get_field_value(kp_entry, field_name) return output + @staticmethod + def _get_field_value(kp_entry, field_name): + if kp_entry is None: + return None + match field_name: + case "username": + return kp_entry.username + case "password": + return kp_entry.password + case "url": + return kp_entry.url + case _: + return kp_entry.get_custom_property(field_name) + def save(self): # nadpisz plik źródłowy zmianami z lock self._kp.save() diff --git a/tests/files/test1/test.mo.yaml b/tests/files/test1/test.mo.yaml index af55ecc..572b953 100644 --- a/tests/files/test1/test.mo.yaml +++ b/tests/files/test1/test.mo.yaml @@ -1,3 +1,3 @@ -value: ${sample.password} -nested: ${some.nested.value.password} -custom: ${custom.field} +value: ${sample} +nested: ${some.nested.value} +custom: @{custom.field} diff --git a/tests/test_mo.py b/tests/test_mo.py index 200b3d8..f685e34 100644 --- a/tests/test_mo.py +++ b/tests/test_mo.py @@ -10,18 +10,14 @@ from app.services.mo import Mo class TestMo(TestCase): def test_process(self): + target_path = Path('tests/files/test1/test.yaml') mo = Mo(Passwords()) mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) - self.assertTrue(os.path.exists('tests/files/test1/test.mo')) - with open('tests/files/test1/test.yaml', 'r') as f: + self.assertTrue(os.path.exists(target_path)) + with open(target_path, 'r') as f: content = f.read() self.assertFalse(content.__contains__('${')) parsed = yaml.load(content, Loader=yaml.FullLoader) - self.assertEqual(parsed['value'], 'some_oass') - self.assertEqual(parsed['nested'], 'nested_pass') - self.assertEqual(parsed['custom'], 'custom_content') - """ - value: some_oass - nested: nested_pass - custom: custom_content - """ + self.assertEqual('some_pass', parsed['value']) + self.assertEqual('nested_pass', parsed['nested']) + self.assertEqual('custom_content', parsed['custom']) From 8ca668f07e7bc4463102ce2b2b1afffff3f21838 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:20:24 +0100 Subject: [PATCH 25/93] git checkout sha --- app/services/vcs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/vcs.py b/app/services/vcs.py index 00ef3c2..0e97ec8 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -27,4 +27,5 @@ class GitService: return Repo(config.path) def checkout(self, sha: str): - pass + self._origin.fetch() + self._repo.git.checkout(sha) From 4c8716cc50a8f1edf5005ac74277329368fc7a15 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:31:00 +0100 Subject: [PATCH 26/93] Fixed cbv --- app/services/containers.py | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/services/containers.py b/app/services/containers.py index 20f0098..6e927bf 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -36,4 +36,6 @@ class DockerService: return self._tree def reload(self, compose_path: Path): - pass + cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"] + # TODO: subprocess + diff --git a/pyproject.toml b/pyproject.toml index 9b97797..1791992 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "docker>=7.1.0", "injectable==4.0.1", "py-automapper>=2.2.0", + "fastapi-utils>=0.8.0", ] [project.optional-dependencies] From 324938133f5430939f6deceef37008d9a07606f0 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:42:37 +0100 Subject: [PATCH 27/93] Global formatting fix --- app/api/models.py | 1 + app/model/passwords.py | 5 ++++- app/model/webhook.py | 1 + app/services/containers.py | 1 - app/templates/index.html | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index b27765b..8e8e4d2 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -10,6 +10,7 @@ class Request: started: str files: List[str] + @dataclass class Response: status: int diff --git a/app/model/passwords.py b/app/model/passwords.py index d795d2c..253dbd9 100644 --- a/app/model/passwords.py +++ b/app/model/passwords.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from typing import Type + # TODO: unnecessary? @dataclass @@ -8,6 +9,7 @@ class PathItem: name: str t: Type + @dataclass class Path: path: list[PathItem] = field(default_factory=list) @@ -43,7 +45,8 @@ class Password: def path(self): return self.group.path.append(self.name, type(self)) + class UnencryptedPassword(Password): - def __init__(self, name: str, value: str, group: Group): + def __init__(self, name: str, value: str, group: Group): super().__init__(name, group) self.value = value diff --git a/app/model/webhook.py b/app/model/webhook.py index 364e17b..a2ef7dc 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import List + @dataclass class WoodpeckerEvent: _id: str diff --git a/app/services/containers.py b/app/services/containers.py index 6e927bf..f754ddc 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -38,4 +38,3 @@ class DockerService: def reload(self, compose_path: Path): cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"] # TODO: subprocess - diff --git a/app/templates/index.html b/app/templates/index.html index ebbe8ca..ac9bbbe 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -3,7 +3,7 @@ {{ title }} - +