From 60c4aeb4e38dda07165efa465a6fb9c106f8ae42 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 6 Oct 2025 23:12:53 +0200 Subject: [PATCH 01/99] Services outline --- app/services/__init__.py | 0 app/services/ci.py | 0 app/services/containers.py | 0 app/services/passwd.py | 0 app/services/vcs.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/services/__init__.py create mode 100644 app/services/ci.py create mode 100644 app/services/containers.py create mode 100644 app/services/passwd.py create mode 100644 app/services/vcs.py diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/ci.py b/app/services/ci.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/containers.py b/app/services/containers.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/passwd.py b/app/services/passwd.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/vcs.py b/app/services/vcs.py new file mode 100644 index 0000000..e69de29 From 2cde0de07a7cf341e04ae2b43b99117f58b01ecb Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 6 Oct 2025 23:45:30 +0200 Subject: [PATCH 02/99] Config outline --- app/__main__.py | 5 ++++- app/main.py | 28 +++++++++++++++++++++++++++- config/config.yaml | 4 ++++ pyproject.toml | 1 + 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 config/config.yaml diff --git a/app/__main__.py b/app/__main__.py index 0b3284c..0ae0fbb 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -1,4 +1,7 @@ if __name__ == '__main__': - from main import run + try: + from main import run + except ImportError: + from .main import run run() diff --git a/app/main.py b/app/main.py index 22fe12a..c299144 100644 --- a/app/main.py +++ b/app/main.py @@ -1,3 +1,6 @@ +import os + +import yaml from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse from jinja2 import Environment, FileSystemLoader, select_autoescape @@ -10,7 +13,26 @@ templates_env = Environment( autoescape=select_autoescape(["html", "xml"]), ) + +def load_config() -> dict: + """ + Ładuje konfigurację z pliku YAML. + Domyślna ścieżka: config/config.yaml + Można nadpisać przez APP_CONFIG_FILE. + """ + config_path = os.getenv("APP_CONFIG_FILE", "config/config.yaml") + if not os.path.exists(config_path): + # Zwróć minimalną domyślną konfigurację, jeśli plik nie istnieje + return { + "app": {"host": "127.0.0.1", "port": 8000, "reload": True}, + } + with open(config_path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + app = FastAPI(title="Karl", version="0.1.0") +# Załaduj konfigurację do stanu aplikacji +app.state.config = load_config() # Rejestracja routera API pod /api/v1 app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) @@ -26,4 +48,8 @@ async def index(request: Request) -> HTMLResponse: def run() -> None: import uvicorn - uvicorn.run("app.main:app", host="127.0.0.1", port=8000, reload=True) + cfg = getattr(app.state, "config", {}) + host = cfg.get("app", {}).get("host", "127.0.0.1") + port = int(cfg.get("app", {}).get("port", 8000)) + reload = bool(cfg.get("app", {}).get("reload", True)) + uvicorn.run("app.main:app", host=host, port=port, reload=reload) diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..8afdd9a --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,4 @@ +app: + host: "127.0.0.1" + port: 8000 + reload: true diff --git a/pyproject.toml b/pyproject.toml index 5185ae8..43f7787 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "fastapi>=0.115.0", "uvicorn[standard]>=0.30.0", "jinja2>=3.1.4", + "pyyaml>=6.0.2", ] [project.optional-dependencies] From b18488a6d39486cc15427c5ba281b72172a86353 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 7 Oct 2025 00:06:04 +0200 Subject: [PATCH 03/99] Pydantic --- app/config/__init__.py | 1 + app/config/settings.py | 28 ++++++++++++++++++++++++++++ app/main.py | 35 ++++++++--------------------------- pyproject.toml | 1 + 4 files changed, 38 insertions(+), 27 deletions(-) create mode 100644 app/config/__init__.py create mode 100644 app/config/settings.py diff --git a/app/config/__init__.py b/app/config/__init__.py new file mode 100644 index 0000000..1fb2e97 --- /dev/null +++ b/app/config/__init__.py @@ -0,0 +1 @@ +from .settings import get_settings diff --git a/app/config/settings.py b/app/config/settings.py new file mode 100644 index 0000000..cf95017 --- /dev/null +++ b/app/config/settings.py @@ -0,0 +1,28 @@ +from functools import lru_cache + +from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import BaseModel +from pathlib import Path +import yaml + +class AppConfig(BaseModel): + host: str = "127.0.0.1" + port: int = 8000 + reload: bool = True + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__") + app: AppConfig = AppConfig() + + @classmethod + def from_yaml(cls, path: Path | str = "config/config.yaml") -> "Settings": + p = Path(path) + data = {} + if p.exists(): + with p.open("r", encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + return cls(**data) + +@lru_cache +def get_settings() -> Settings: + return Settings.from_yaml() diff --git a/app/main.py b/app/main.py index c299144..bb425d1 100644 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,9 @@ -import os - -import yaml from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse from jinja2 import Environment, FileSystemLoader, select_autoescape from app.api.v1 import router as api_v1_router +from app.config import get_settings # Inicjalizacja Jinja2 templates_env = Environment( @@ -13,26 +11,7 @@ templates_env = Environment( autoescape=select_autoescape(["html", "xml"]), ) - -def load_config() -> dict: - """ - Ładuje konfigurację z pliku YAML. - Domyślna ścieżka: config/config.yaml - Można nadpisać przez APP_CONFIG_FILE. - """ - config_path = os.getenv("APP_CONFIG_FILE", "config/config.yaml") - if not os.path.exists(config_path): - # Zwróć minimalną domyślną konfigurację, jeśli plik nie istnieje - return { - "app": {"host": "127.0.0.1", "port": 8000, "reload": True}, - } - with open(config_path, "r", encoding="utf-8") as f: - return yaml.safe_load(f) or {} - - app = FastAPI(title="Karl", version="0.1.0") -# Załaduj konfigurację do stanu aplikacji -app.state.config = load_config() # Rejestracja routera API pod /api/v1 app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) @@ -48,8 +27,10 @@ async def index(request: Request) -> HTMLResponse: def run() -> None: import uvicorn - cfg = getattr(app.state, "config", {}) - host = cfg.get("app", {}).get("host", "127.0.0.1") - port = int(cfg.get("app", {}).get("port", 8000)) - reload = bool(cfg.get("app", {}).get("reload", True)) - uvicorn.run("app.main:app", host=host, port=port, reload=reload) + settings = get_settings() + uvicorn.run( + "app.main:app", + host=settings.app.host, + port=settings.app.port, + reload=settings.app.reload, + ) diff --git a/pyproject.toml b/pyproject.toml index 43f7787..c7db9ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "fastapi>=0.115.0", "uvicorn[standard]>=0.30.0", "jinja2>=3.1.4", + "pydantic-settings>=2.4.0", "pyyaml>=6.0.2", ] From 1d615e9f7da68b3c6d08ed4ce59f56edde5500e9 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 7 Oct 2025 00:18:37 +0200 Subject: [PATCH 04/99] VCS settings --- app/config/settings.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/config/settings.py b/app/config/settings.py index cf95017..a20f301 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -1,18 +1,25 @@ from functools import lru_cache - -from pydantic_settings import BaseSettings, SettingsConfigDict -from pydantic import BaseModel from pathlib import Path + import yaml +from pydantic import BaseModel +from pydantic_settings import BaseSettings, SettingsConfigDict + class AppConfig(BaseModel): host: str = "127.0.0.1" port: int = 8000 reload: bool = True +class GitConfig(BaseModel): + directory: str = "/opt/repo/sample" + branch: str = "master" + remote: str = "origin" + class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__") app: AppConfig = AppConfig() + git: GitConfig = GitConfig() @classmethod def from_yaml(cls, path: Path | str = "config/config.yaml") -> "Settings": @@ -23,6 +30,7 @@ class Settings(BaseSettings): data = yaml.safe_load(fh) or {} return cls(**data) + @lru_cache def get_settings() -> Settings: return Settings.from_yaml() From a55628ce35a7484161676ac971d33a7c4a22ca37 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 7 Oct 2025 00:34:39 +0200 Subject: [PATCH 05/99] VCS service --- app/services/vcs.py | 28 ++++++++++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 29 insertions(+) diff --git a/app/services/vcs.py b/app/services/vcs.py index e69de29..c137108 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -0,0 +1,28 @@ +from git import Repo, Remote + +from app.config import get_settings + + +class GitService: + def __init__(self): + self._settings = get_settings() + self._repo = Repo(self._settings.git.directory) + self._origin: Remote = self._repo.remotes.origin + + def get_modified_compose(self) -> str | None: + self._update() + return self._diff() + + def _update(self): + self._origin.pull() + + def _diff(self) -> str | None: + diff = self._repo.head.commit.diff("HEAD~1") + composes = [f for f in diff if f.a_path.endswith("docker-compose.yml")] + match len(composes): + case 0: + return None + case 1: + return composes[0].a_path + case _: + raise Exception("Multiple compose files modified") diff --git a/pyproject.toml b/pyproject.toml index c7db9ab..1b8f359 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "jinja2>=3.1.4", "pydantic-settings>=2.4.0", "pyyaml>=6.0.2", + "gitpython>=3.1.45" ] [project.optional-dependencies] From 162c0adf13ad1633b4454eb19fe587aa2c1eb03c Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 8 Oct 2025 00:05:13 +0200 Subject: [PATCH 06/99] Keepass outline --- app/config/settings.py | 15 ++++-- app/{services/passwd.py => model/__init__.py} | 0 app/model/passwords.py | 49 +++++++++++++++++++ app/services/passwords.py | 38 ++++++++++++++ app/services/system.py | 1 + app/util/__init__.py | 0 app/util/dicts.py | 11 +++++ config/config.yaml | 3 ++ pyproject.toml | 1 + 9 files changed, 115 insertions(+), 3 deletions(-) rename app/{services/passwd.py => model/__init__.py} (100%) create mode 100644 app/model/passwords.py create mode 100644 app/services/passwords.py create mode 100644 app/services/system.py create mode 100644 app/util/__init__.py create mode 100644 app/util/dicts.py diff --git a/app/config/settings.py b/app/config/settings.py index cf95017..33a7019 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -1,18 +1,26 @@ from functools import lru_cache - -from pydantic_settings import BaseSettings, SettingsConfigDict -from pydantic import BaseModel from pathlib import Path + import yaml +from pydantic import BaseModel +from pydantic_settings import BaseSettings, SettingsConfigDict + class AppConfig(BaseModel): host: str = "127.0.0.1" port: int = 8000 reload: bool = True + +class KeePassConfig(BaseModel): + file: str = "database.kdbx" + secret: Path | str = "/run/secrets/kp_secret" + + class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__") app: AppConfig = AppConfig() + kp: KeePassConfig = KeePassConfig() @classmethod def from_yaml(cls, path: Path | str = "config/config.yaml") -> "Settings": @@ -23,6 +31,7 @@ class Settings(BaseSettings): data = yaml.safe_load(fh) or {} return cls(**data) + @lru_cache def get_settings() -> Settings: return Settings.from_yaml() diff --git a/app/services/passwd.py b/app/model/__init__.py similarity index 100% rename from app/services/passwd.py rename to app/model/__init__.py diff --git a/app/model/passwords.py b/app/model/passwords.py new file mode 100644 index 0000000..d795d2c --- /dev/null +++ b/app/model/passwords.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass, field +from typing import Type + +# TODO: unnecessary? + +@dataclass +class PathItem: + name: str + t: Type + +@dataclass +class Path: + path: list[PathItem] = field(default_factory=list) + + def append(self, name, t): + self.path.append(PathItem(name, t)) + + def __str__(self): + return "/".join([i.name for i in self.path]) + + +@dataclass +class Group: + name: str + passwords: list["Password"] = field(default_factory=list) + parent: "Group|None" = None + + @property + def path(self): + if self.parent is None: + new_path = Path() + new_path.append(self.name, type(self)) + return new_path + return self.parent.path.append(self.name, type(self)) + + +@dataclass +class Password: + name: str + group: Group + + @property + def path(self): + return self.group.path.append(self.name, type(self)) + +class UnencryptedPassword(Password): + def __init__(self, name: str, value: str, group: Group): + super().__init__(name, group) + self.value = value diff --git a/app/services/passwords.py b/app/services/passwords.py new file mode 100644 index 0000000..9dca423 --- /dev/null +++ b/app/services/passwords.py @@ -0,0 +1,38 @@ +import os.path + +from pykeepass import PyKeePass, create_database, Group + + +class Passwords: + def __init__(self): + from app.config import get_settings + settings = get_settings() + + with open(settings.kp.secret, "r") as fh: + secret = fh.read() + + self._kp_org = self.__get_or_create_store(settings.kp.file, secret) + self._kp = self.__get_lock(settings.kp.file, secret) + + @staticmethod + def __get_or_create_store(path, passwd) -> PyKeePass: + if os.path.exists(path): + return PyKeePass( + path, + password=passwd, + ) + return create_database(path, passwd) + + @staticmethod + def __get_lock(path, passwd) -> PyKeePass: + lock_path = path + ".lock" + import shutil + shutil.copyfile(path, lock_path) + return Passwords.__get_or_create_store(lock_path, passwd) + + @property + def store(self): + return self._kp.root_group + + def save(self, group: Group): + pass diff --git a/app/services/system.py b/app/services/system.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/app/services/system.py @@ -0,0 +1 @@ + diff --git a/app/util/__init__.py b/app/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/util/dicts.py b/app/util/dicts.py new file mode 100644 index 0000000..faf7f65 --- /dev/null +++ b/app/util/dicts.py @@ -0,0 +1,11 @@ +from types import SimpleNamespace + + +class NestedNamespace(SimpleNamespace): + def __init__(self, dictionary, **kwargs): + super().__init__(**kwargs) + for key, value in dictionary.items(): + if isinstance(value, dict): + self.__setattr__(key, NestedNamespace(value)) + else: + self.__setattr__(key, value) diff --git a/config/config.yaml b/config/config.yaml index 8afdd9a..5b47197 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,3 +2,6 @@ app: host: "127.0.0.1" port: 8000 reload: true +kp: + file: "config/kp.kdbx" + secret: "config/secret.txt" diff --git a/pyproject.toml b/pyproject.toml index c7db9ab..0f8ebcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "jinja2>=3.1.4", "pydantic-settings>=2.4.0", "pyyaml>=6.0.2", + "pykeepass>=4.1.1.post1" ] [project.optional-dependencies] From 7e189271e37b3bb98a6d601be70980343b328ae8 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 8 Oct 2025 00:55:32 +0200 Subject: [PATCH 07/99] Docker outline --- app/model/__init__.py | 0 app/model/containers.py | 40 ++++++++++++++++++++++++++++++++++++++ app/services/containers.py | 28 ++++++++++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 69 insertions(+) create mode 100644 app/model/__init__.py create mode 100644 app/model/containers.py diff --git a/app/model/__init__.py b/app/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/model/containers.py b/app/model/containers.py new file mode 100644 index 0000000..2bdf6aa --- /dev/null +++ b/app/model/containers.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass, field +from datetime import datetime + +from docker.models.containers import Container + + +@dataclass +class SimpleContainer: + name: str + image: str + status: str + health: str + created: datetime + + @staticmethod + def from_container(container: Container): + created = datetime.strptime(container.attrs['Created'].split('.')[0], '%Y-%m-%dT%H:%M:%S') + return SimpleContainer( + name=container.name, + image=container.image.tags[0], + status=container.status, + health=container.health, + created=created + ) + + +@dataclass +class Compose: + directory: str + containers: list[SimpleContainer] = field(default_factory=list) + + @property + def last_modified(self): + return max(self.containers, key=lambda c: c.created).created + + +@dataclass +class Tree: + composes: dict[str, Compose] = field(default_factory=dict) + containers: list[SimpleContainer] = field(default_factory=list) diff --git a/app/services/containers.py b/app/services/containers.py index e69de29..6a4814c 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -0,0 +1,28 @@ +import docker +from docker.models.containers import Container + +from app.model.containers import Tree, Compose, SimpleContainer + + +class DockerService: + def __init__(self): + self._client = docker.from_env() + self._tree = self._init_tree() + + def _init_tree(self) -> Tree: + tree = Tree() + container: Container + for container in self._client.containers.list(): + labels = container.labels + working_dir = labels.get("com.docker.compose.project.working_dir") + if working_dir: + if tree.composes.get(working_dir) is None: + tree.composes[working_dir] = Compose(working_dir) + tree.composes[working_dir].containers.append(SimpleContainer.from_container(container)) + else: + tree.containers.append(SimpleContainer.from_container(container)) + return tree + + @property + def tree(self) -> Tree: + return self._tree diff --git a/pyproject.toml b/pyproject.toml index c7db9ab..98bed9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "jinja2>=3.1.4", "pydantic-settings>=2.4.0", "pyyaml>=6.0.2", + "docker>=7.1.0" ] [project.optional-dependencies] From e1c32b6a20e3b4aca2ae399411909a7cc1279737 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 8 Oct 2025 01:12:12 +0200 Subject: [PATCH 08/99] CI API --- app/api/models.py | 15 +++++++++++++++ app/api/v1.py | 13 ++++++++++++- app/services/ci.py | 0 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 app/api/models.py delete mode 100644 app/services/ci.py diff --git a/app/api/models.py b/app/api/models.py new file mode 100644 index 0000000..35afedb --- /dev/null +++ b/app/api/models.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from typing import List + + +@dataclass +class Request: + build_id: str + build_url: str + commit_id: str + commit_url: str + changelist: List[str] + +@dataclass +class Response: + status: int diff --git a/app/api/v1.py b/app/api/v1.py index 1cd9f50..fc52bd4 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,8 +1,19 @@ from fastapi import APIRouter +from app.api.models import Request, Response + router = APIRouter() @router.get("/", summary="Main API") async def root(): - return {"message": "Witaj w API v12"} + return {"message": "Witaj w API v1"} + + +@router.get("/health", summary="Health check") +async def health(): + return {"status": "ok"} + +@router.post("/ci", summary="CI Webhook") +async def ci(request: Request): + return Response(200) diff --git a/app/services/ci.py b/app/services/ci.py deleted file mode 100644 index e69de29..0000000 From 2d3699ad00d0059a3c28aece297f5a1527a335a7 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 9 Oct 2025 20:38:37 +0200 Subject: [PATCH 09/99] Core outline --- .gitignore | 1 + app/core/__init__.py | 0 app/core/core.py | 28 ++++++++++++++++++++++++++++ app/main.py | 5 +++++ app/model/healthcheck.py | 7 +++++++ app/services/__init__.py | 5 +++++ config/secret.txt | 1 + 7 files changed, 47 insertions(+) create mode 100644 app/core/__init__.py create mode 100644 app/core/core.py create mode 100644 app/model/healthcheck.py create mode 100644 config/secret.txt diff --git a/.gitignore b/.gitignore index 790a54f..b85bc8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .idea *.iml uv.lock +**/*.kdbx* __pycache__/ **/dist/ diff --git a/app/core/__init__.py b/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/core/core.py b/app/core/core.py new file mode 100644 index 0000000..bb6cad4 --- /dev/null +++ b/app/core/core.py @@ -0,0 +1,28 @@ +from app.model.healthcheck import HealthCheck +from app.services import DockerService, GitService, Passwords + + +class WebhookProcessor: + def __init__(self): + try: + self._docker = DockerService() + except Exception as e: + self._docker = None + print(e) + try: + self._git = GitService() + except Exception as e: + self._git = None + print(f"{type(e).__name__}: {e}") + try: + self._keepass = Passwords() + except Exception as e: + self._keepass = None + print(e) + + @property + def health(self) -> HealthCheck: + return HealthCheck( + self._docker is not None and self._git is not None and self._keepass is not None, + f"Docker: {self._docker is not None}, Git: {self._git is not None}, KeePass: {self._keepass is not None}" + ) diff --git a/app/main.py b/app/main.py index bb425d1..4b13b8a 100644 --- a/app/main.py +++ b/app/main.py @@ -4,6 +4,7 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape from app.api.v1 import router as api_v1_router from app.config import get_settings +from app.core.core import WebhookProcessor # Inicjalizacja Jinja2 templates_env = Environment( @@ -15,6 +16,10 @@ app = FastAPI(title="Karl", version="0.1.0") # Rejestracja routera API pod /api/v1 app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) +# app.add_event_handler() + +webhook_service = WebhookProcessor() +print(webhook_service.health) # Przykładowy endpoint HTML diff --git a/app/model/healthcheck.py b/app/model/healthcheck.py new file mode 100644 index 0000000..ac483f3 --- /dev/null +++ b/app/model/healthcheck.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass +class HealthCheck: + healthy: bool + message: str diff --git a/app/services/__init__.py b/app/services/__init__.py index e69de29..febb742 100644 --- a/app/services/__init__.py +++ b/app/services/__init__.py @@ -0,0 +1,5 @@ +from .containers import DockerService +from .passwords import Passwords +from .vcs import GitService + +__all__ = ["GitService", "Passwords", "DockerService"] diff --git a/config/secret.txt b/config/secret.txt new file mode 100644 index 0000000..b5f9078 --- /dev/null +++ b/config/secret.txt @@ -0,0 +1 @@ +supersecret From 232920683ae1daa9ada6deddee831c1d237ef854 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 12 Oct 2025 20:30:43 +0200 Subject: [PATCH 10/99] Basic injects --- app/core/core.py | 28 ++++++++++++---------------- app/main.py | 2 ++ app/services/containers.py | 2 ++ app/services/passwords.py | 2 ++ app/services/vcs.py | 9 +++++++-- config/config.yaml | 2 ++ pyproject.toml | 3 ++- 7 files changed, 29 insertions(+), 19 deletions(-) diff --git a/app/core/core.py b/app/core/core.py index bb6cad4..9e3acec 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -1,24 +1,20 @@ +from typing import Annotated + +from injectable import injectable, autowired, Autowired + from app.model.healthcheck import HealthCheck from app.services import DockerService, GitService, Passwords +# @injectable class WebhookProcessor: - def __init__(self): - try: - self._docker = DockerService() - except Exception as e: - self._docker = None - print(e) - try: - self._git = GitService() - except Exception as e: - self._git = None - print(f"{type(e).__name__}: {e}") - try: - self._keepass = Passwords() - except Exception as e: - self._keepass = None - print(e) + @autowired + def __init__(self, docker: Annotated[DockerService, Autowired], + git: Annotated[GitService, Autowired], + keepass: Annotated[Passwords, Autowired]): + self._docker = docker + self._git = git + self._keepass = keepass @property def health(self) -> HealthCheck: diff --git a/app/main.py b/app/main.py index 4b13b8a..eb0d6ea 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,6 @@ from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse +from injectable import load_injection_container from jinja2 import Environment, FileSystemLoader, select_autoescape from app.api.v1 import router as api_v1_router @@ -18,6 +19,7 @@ app = FastAPI(title="Karl", version="0.1.0") app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) # app.add_event_handler() +load_injection_container() webhook_service = WebhookProcessor() print(webhook_service.health) diff --git a/app/services/containers.py b/app/services/containers.py index 6a4814c..ef891e3 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -1,9 +1,11 @@ import docker from docker.models.containers import Container +from injectable import injectable from app.model.containers import Tree, Compose, SimpleContainer +@injectable(singleton=True) class DockerService: def __init__(self): self._client = docker.from_env() diff --git a/app/services/passwords.py b/app/services/passwords.py index 9dca423..18d8519 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -1,8 +1,10 @@ import os.path +from injectable import injectable from pykeepass import PyKeePass, create_database, Group +@injectable(singleton=True) class Passwords: def __init__(self): from app.config import get_settings diff --git a/app/services/vcs.py b/app/services/vcs.py index c137108..8e58277 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -1,13 +1,18 @@ from git import Repo, Remote +from injectable import injectable from app.config import get_settings +@injectable(singleton=True) class GitService: def __init__(self): self._settings = get_settings() - self._repo = Repo(self._settings.git.directory) - self._origin: Remote = self._repo.remotes.origin + try: # TODO: clone if not exists + self._repo = Repo(self._settings.git.directory) + self._origin: Remote = self._repo.remotes.origin + except: + self._repo = None def get_modified_compose(self) -> str | None: self._update() diff --git a/config/config.yaml b/config/config.yaml index 5b47197..c48d6e9 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,6 +2,8 @@ app: host: "127.0.0.1" port: 8000 reload: true +git: + directory: "F:/IdeaProjects/paas/karl/.compose_repository" kp: file: "config/kp.kdbx" secret: "config/secret.txt" diff --git a/pyproject.toml b/pyproject.toml index 13d3179..c79d3af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,8 @@ dependencies = [ "pyyaml>=6.0.2", "gitpython>=3.1.45", "pykeepass>=4.1.1.post1", - "docker>=7.1.0" + "docker>=7.1.0", + "injectable==4.0.1", ] [project.optional-dependencies] From d45fe704b6f8de9b7b065f2006d62b4489c7b51a Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 14 Oct 2025 19:55:43 +0200 Subject: [PATCH 11/99] VCS fix for non-existing code base --- app/config/__init__.py | 6 ++++++ app/config/settings.py | 3 ++- app/services/vcs.py | 19 +++++++++++++++++-- config/config.yaml | 3 +++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/app/config/__init__.py b/app/config/__init__.py index 1fb2e97..7829e5c 100644 --- a/app/config/__init__.py +++ b/app/config/__init__.py @@ -1 +1,7 @@ +from .settings import AppConfig +from .settings import GitConfig +from .settings import KeePassConfig +from .settings import Settings from .settings import get_settings + +__all__ = [AppConfig, GitConfig, KeePassConfig, Settings, get_settings] diff --git a/app/config/settings.py b/app/config/settings.py index c831957..06f75e3 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -13,7 +13,8 @@ class AppConfig(BaseModel): class GitConfig(BaseModel): - directory: str = "/opt/repo/sample" + path: Path = Path("/opt/repo/sample") + url: str = "ssh://git@hattori.ztsh.eu:29418/paas/heimdall.git" branch: str = "master" remote: str = "origin" diff --git a/app/services/vcs.py b/app/services/vcs.py index c137108..2f0a1a2 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -1,18 +1,33 @@ from git import Repo, Remote -from app.config import get_settings +from app.config import GitConfig, get_settings class GitService: def __init__(self): self._settings = get_settings() - self._repo = Repo(self._settings.git.directory) + self._repo = self._check_preconditions(self._settings.git) + if self._repo.head.ref.name != self._settings.git.branch: + self._repo.git.checkout(self._settings.git.branch) self._origin: Remote = self._repo.remotes.origin + def get_modified_compose(self) -> str | None: self._update() return self._diff() + @staticmethod + def _check_preconditions(config: GitConfig) -> Repo: + def clone(): + return Repo.clone_from(config.url, config.path, branch=config.branch) + import os + if not config.path.exists(): + return clone() + if not (config.path / ".git").exists(): + os.rmdir(config.path) + return clone() + return Repo(config.path) + def _update(self): self._origin.pull() diff --git a/config/config.yaml b/config/config.yaml index 5b47197..6d4400f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,6 +2,9 @@ app: host: "127.0.0.1" port: 8000 reload: true +git: + path: "F:/IdeaProjects/paas/karl/.compose_repository" + branch: "main" kp: file: "config/kp.kdbx" secret: "config/secret.txt" From 27d1332b35c70c076d1c8a488fd31d23771918b5 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 15 Oct 2025 23:57:19 +0200 Subject: [PATCH 12/99] fix: logging.getLogger gets properly configured logger --- app/__init__.py | 0 app/__main__.py | 7 ---- app/core/router.py | 20 ++++++++++++ app/main.py | 79 ++++++++++++++++++++++++++++++--------------- app/util/logging.py | 59 +++++++++++++++++++++++++++++++++ run.sh | 2 +- 6 files changed, 133 insertions(+), 34 deletions(-) delete mode 100644 app/__init__.py delete mode 100644 app/__main__.py create mode 100644 app/core/router.py create mode 100644 app/util/logging.py diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/__main__.py b/app/__main__.py deleted file mode 100644 index 0ae0fbb..0000000 --- a/app/__main__.py +++ /dev/null @@ -1,7 +0,0 @@ -if __name__ == '__main__': - try: - from main import run - except ImportError: - from .main import run - - run() diff --git a/app/core/router.py b/app/core/router.py new file mode 100644 index 0000000..0d1a300 --- /dev/null +++ b/app/core/router.py @@ -0,0 +1,20 @@ +from fastapi import APIRouter, Request + +from fastapi.responses import HTMLResponse +from jinja2 import Environment, FileSystemLoader, select_autoescape + +router = APIRouter() + +# Inicjalizacja Jinja2 +templates_env = Environment( + loader=FileSystemLoader("app/templates"), + autoescape=select_autoescape(["html", "xml"]), +) + + +# Przykładowy endpoint HTML +@router.get("/", response_class=HTMLResponse) +async def index(request: Request) -> HTMLResponse: + template = templates_env.get_template("index.html") + html = template.render(title="Strona główna", request=request) + return HTMLResponse(content=html) diff --git a/app/main.py b/app/main.py index 4b13b8a..7eb04a5 100644 --- a/app/main.py +++ b/app/main.py @@ -1,41 +1,68 @@ -from fastapi import FastAPI, Request -from fastapi.responses import HTMLResponse -from jinja2 import Environment, FileSystemLoader, select_autoescape +import logging + +from fastapi import FastAPI -from app.api.v1 import router as api_v1_router from app.config import get_settings from app.core.core import WebhookProcessor - -# Inicjalizacja Jinja2 -templates_env = Environment( - loader=FileSystemLoader("app/templates"), - autoescape=select_autoescape(["html", "xml"]), -) - -app = FastAPI(title="Karl", version="0.1.0") - -# Rejestracja routera API pod /api/v1 -app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) -# app.add_event_handler() - -webhook_service = WebhookProcessor() -print(webhook_service.health) +from app.util.logging import LoggingHandler, ExternalLoggingHandler -# Przykładowy endpoint HTML -@app.get("/", response_class=HTMLResponse) -async def index(request: Request) -> HTMLResponse: - template = templates_env.get_template("index.html") - html = template.render(title="Strona główna", request=request) - return HTMLResponse(content=html) +class KarlApplication: + def __init__(self) -> None: + self._set_logging() + app = FastAPI(title="Karl", version="0.1.0") + self._set_routes(app) + self._set_events(app) + self._init_services() + pass + + def _set_logging(self): + logging.basicConfig(level=logging.INFO, handlers=[LoggingHandler()]) + + loggers = ( + "uvicorn", + "uvicorn.access", + "uvicorn.error", + "fastapi", + "asyncio", + "starlette", + ) + external_handler = ExternalLoggingHandler() + for logger_name in loggers: + logging_logger = logging.getLogger(logger_name) + logging_logger.handlers = [external_handler] + logging_logger.propagate = False + + def _set_routes(self, app: FastAPI): + from app.core.router import router as core_router + app.include_router(core_router) + from app.api.v1 import router as api_v1_router + app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) + pass + + def _set_events(self, app: FastAPI): + pass + + def _init_services(self): + logger = logging.getLogger(__name__) + + webhook_service = WebhookProcessor() + logger.info(webhook_service.health) -def run() -> None: +def app(): + return KarlApplication() + + +if __name__ == "__main__": import uvicorn + settings = get_settings() uvicorn.run( "app.main:app", + factory=True, host=settings.app.host, port=settings.app.port, reload=settings.app.reload, + log_config=None, ) diff --git a/app/util/logging.py b/app/util/logging.py new file mode 100644 index 0000000..29992b6 --- /dev/null +++ b/app/util/logging.py @@ -0,0 +1,59 @@ +from logging import Formatter, StreamHandler + + +class NamingCache: + def __init__(self): + self._cache = {} + + def __getitem__(self, key): + if key not in self._cache: + self._cache[key] = self.shorten(key) + return self._cache[key] + + @staticmethod + def shorten(logger_name: str) -> str: + target_length = 18 + if len(logger_name) > target_length: + parts = logger_name.split('.') + part = 0 + while len(logger_name) > target_length: + if part == len(parts) - 1: + logger_name = f'...{logger_name[-(target_length - 3):]}' + break + parts[part] = parts[part][0] + logger_name = '.'.join(parts) + part += 1 + + return logger_name.ljust(target_length) + + +class ApplicationFormatter(Formatter): + def __init__(self, handler_prefix: str = ''): + super().__init__() + self._logger_names = NamingCache() + self._handler_prefix = handler_prefix + + def format(self, record): + from datetime import datetime + timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') + level = record.levelname.replace('WARNING', 'WARN').rjust(5) + thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? + logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"] + message = record.getMessage() + formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}" + + if record.exc_info: + formatted += "\n" + self.formatException(record.exc_info) + + return formatted + + +class LoggingHandler(StreamHandler): + def __init__(self): + super().__init__() + self.setFormatter(ApplicationFormatter(handler_prefix='karl.')) + +class ExternalLoggingHandler(StreamHandler): + def __init__(self): + super().__init__() + self.setFormatter(ApplicationFormatter()) diff --git a/run.sh b/run.sh index c65ef75..8d2e2b9 100644 --- a/run.sh +++ b/run.sh @@ -1 +1 @@ -uv run uvicorn app.main:app --reload +uvicorn app.main:app --factory --reload From 876b3333971c2a5cf701f1076157679ea2f4f1df Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 16 Oct 2025 21:33:25 +0200 Subject: [PATCH 13/99] fix: ASGI run & ports fix --- .gitignore | 1 + app/config/settings.py | 2 +- app/main.py | 15 ++++++++++----- config/config.yaml | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index b85bc8b..8395e24 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.iml uv.lock **/*.kdbx* +.compose_repository __pycache__/ **/dist/ diff --git a/app/config/settings.py b/app/config/settings.py index 06f75e3..7e072aa 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -8,7 +8,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict class AppConfig(BaseModel): host: str = "127.0.0.1" - port: int = 8000 + port: int = 8081 reload: bool = True diff --git a/app/main.py b/app/main.py index 22e83b4..5bf8c98 100644 --- a/app/main.py +++ b/app/main.py @@ -1,7 +1,7 @@ import logging -from injectable import load_injection_container from fastapi import FastAPI +from injectable import load_injection_container from app.config import get_settings from app.core.core import WebhookProcessor @@ -9,13 +9,18 @@ from app.util.logging import LoggingHandler, ExternalLoggingHandler class KarlApplication: + from starlette.types import Receive, Scope, Send def __init__(self) -> None: self._set_logging() - app = FastAPI(title="Karl", version="0.1.0") - self._set_routes(app) - self._set_events(app) + _app = FastAPI(title="Karl", version="0.1.0") + self._set_routes(_app) + self._set_events(_app) self._init_services() - pass + + self._app = _app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + await self._app.__call__(scope, receive, send) def _set_logging(self): logging.basicConfig(level=logging.INFO, handlers=[LoggingHandler()]) diff --git a/config/config.yaml b/config/config.yaml index 6d4400f..856d24e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,6 +1,6 @@ app: host: "127.0.0.1" - port: 8000 + port: 8081 reload: true git: path: "F:/IdeaProjects/paas/karl/.compose_repository" From 8565ce19fec269a8aaa18866646b95e48a2b6d0c Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 16 Oct 2025 21:41:56 +0200 Subject: [PATCH 14/99] FastAPI bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c79d3af..c4e6d55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.12" authors = [{ name = "Piotr Dec" }] dependencies = [ - "fastapi>=0.115.0", + "fastapi>=0.119.0", "uvicorn[standard]>=0.30.0", "jinja2>=3.1.4", "pydantic-settings>=2.4.0", From de6184273a69aeed2aac4e3330795ea48f4a8ad7 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 17 Oct 2025 23:16:38 +0200 Subject: [PATCH 15/99] Webhook model changed --- app/api/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index 35afedb..b27765b 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -4,11 +4,11 @@ from typing import List @dataclass class Request: - build_id: str - build_url: str - commit_id: str - commit_url: str - changelist: List[str] + _id: str + commit: str + message: str + started: str + files: List[str] @dataclass class Response: From 120e929469d792310ee2c7a09c0c491338d45a5b Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 21 Oct 2025 17:41:41 +0200 Subject: [PATCH 16/99] APIv1 cbv --- app/api/v1.py | 24 ++++++++++++++++-------- app/core/injects.py | 10 ++++++++++ app/model/webhook.py | 11 +++++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 app/core/injects.py create mode 100644 app/model/webhook.py diff --git a/app/api/v1.py b/app/api/v1.py index fc52bd4..e628d1f 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,6 +1,10 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends +from fastapi_utils.cbv import cbv +from starlette.responses import JSONResponse, Response -from app.api.models import Request, Response +from app.api.models import Request +from app.core.core import WebhookProcessor +from app.core.injects import AutowireSupport router = APIRouter() @@ -10,10 +14,14 @@ async def root(): return {"message": "Witaj w API v1"} -@router.get("/health", summary="Health check") -async def health(): - return {"status": "ok"} +@cbv(router) +class APIv1: + webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) -@router.post("/ci", summary="CI Webhook") -async def ci(request: Request): - return Response(200) + @router.get("/health", summary="Health check") + async def health(self) -> JSONResponse: + return JSONResponse({"status": "ok"}) + + @router.post("/ci", summary="CI Webhook") + async def ci(self, request: Request): + return Response(status_code=201) diff --git a/app/core/injects.py b/app/core/injects.py new file mode 100644 index 0000000..3dfcc66 --- /dev/null +++ b/app/core/injects.py @@ -0,0 +1,10 @@ +from injectable import inject + +from app.core.core import WebhookProcessor + + +class AutowireSupport: + + @staticmethod + def webhook_processor(): + return inject(WebhookProcessor) diff --git a/app/model/webhook.py b/app/model/webhook.py new file mode 100644 index 0000000..70470b2 --- /dev/null +++ b/app/model/webhook.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import List + + +@dataclass +class WebhookRequest: + _id: str + commit: str + message: str + started: str + files: List[str] From 1d7c4c2dfdbe3fbefbaa5feb75eb243e93b653e6 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 21 Oct 2025 17:50:53 +0200 Subject: [PATCH 17/99] Mappings & processing WIP --- app/api/v1.py | 5 ++++- app/core/core.py | 6 +++++- app/model/webhook.py | 2 +- pyproject.toml | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app/api/v1.py b/app/api/v1.py index e628d1f..d6355fb 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,3 +1,4 @@ +from automapper import mapper from fastapi import APIRouter, Depends from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response @@ -5,6 +6,7 @@ from starlette.responses import JSONResponse, Response from app.api.models import Request from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport +from app.model.webhook import WebhookEvent router = APIRouter() @@ -20,8 +22,9 @@ class APIv1: @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: - return JSONResponse({"status": "ok"}) + return JSONResponse({"status": self.webhook_service.health}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): + self.webhook_service.process_ci_event(mapper.to(WebhookEvent).map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index 9e3acec..a7096d2 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -3,10 +3,11 @@ from typing import Annotated from injectable import injectable, autowired, Autowired from app.model.healthcheck import HealthCheck +from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords -# @injectable +@injectable class WebhookProcessor: @autowired def __init__(self, docker: Annotated[DockerService, Autowired], @@ -16,6 +17,9 @@ class WebhookProcessor: self._git = git self._keepass = keepass + def process_ci_event(self, event: WebhookEvent): + pass + @property def health(self) -> HealthCheck: return HealthCheck( diff --git a/app/model/webhook.py b/app/model/webhook.py index 70470b2..da9ad8d 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -3,7 +3,7 @@ from typing import List @dataclass -class WebhookRequest: +class WebhookEvent: _id: str commit: str message: str diff --git a/pyproject.toml b/pyproject.toml index c4e6d55..9b97797 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "pykeepass>=4.1.1.post1", "docker>=7.1.0", "injectable==4.0.1", + "py-automapper>=2.2.0", ] [project.optional-dependencies] From 2dec6d53844b5b3fbd5c9a1a8b639011b2da8309 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 19:42:57 +0200 Subject: [PATCH 18/99] Mappings enhancement --- app/api/v1.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/api/v1.py b/app/api/v1.py index d6355fb..eba78a4 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -19,6 +19,10 @@ async def root(): @cbv(router) class APIv1: webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) + logger = __import__('logging').getLogger(__name__) + + def __init__(self): + mapper.add(Request, WebhookEvent) @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: @@ -26,5 +30,5 @@ class APIv1: @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.process_ci_event(mapper.to(WebhookEvent).map(request)) + self.webhook_service.process_ci_event(mapper.map(request)) return Response(status_code=201) From 569aefeccb500ded48ca345009a0592ac91a76d4 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 22:53:49 +0200 Subject: [PATCH 19/99] logging enhancements --- .gitignore | 1 + app/config/settings.py | 6 ++++++ app/main.py | 25 ++++++++++++++-------- app/util/logging.py | 47 +++++++++++++++++++++++++++++++++--------- app/web/__init__.py | 0 app/web/middlewares.py | 24 +++++++++++++++++++++ config/config.yaml | 3 +++ run.sh | 2 +- 8 files changed, 88 insertions(+), 20 deletions(-) create mode 100644 app/web/__init__.py create mode 100644 app/web/middlewares.py diff --git a/.gitignore b/.gitignore index 8395e24..4b259c4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ uv.lock __pycache__/ **/dist/ +**/*.log diff --git a/app/config/settings.py b/app/config/settings.py index 7e072aa..2017c14 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -6,6 +6,11 @@ from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict +class LoggingConfig(BaseModel): + level: str = "INFO" + path: Path = Path("logs/karl.log") + + class AppConfig(BaseModel): host: str = "127.0.0.1" port: int = 8081 @@ -26,6 +31,7 @@ class KeePassConfig(BaseModel): class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__") + logging: LoggingConfig = LoggingConfig() app: AppConfig = AppConfig() git: GitConfig = GitConfig() kp: KeePassConfig = KeePassConfig() diff --git a/app/main.py b/app/main.py index 5bf8c98..424548e 100644 --- a/app/main.py +++ b/app/main.py @@ -4,8 +4,7 @@ from fastapi import FastAPI from injectable import load_injection_container from app.config import get_settings -from app.core.core import WebhookProcessor -from app.util.logging import LoggingHandler, ExternalLoggingHandler +from app.util.logging import HandlerFactory class KarlApplication: @@ -13,6 +12,7 @@ class KarlApplication: def __init__(self) -> None: self._set_logging() _app = FastAPI(title="Karl", version="0.1.0") + self._set_middlewares(_app) self._set_routes(_app) self._set_events(_app) self._init_services() @@ -23,7 +23,12 @@ class KarlApplication: await self._app.__call__(scope, receive, send) def _set_logging(self): - logging.basicConfig(level=logging.INFO, handlers=[LoggingHandler()]) + settings = get_settings() + logging.addLevelName(5, "TRACE") + logging.Logger.trace = lambda s, msg, *args, **kwargs: s.log(5, msg, *args, **kwargs) + logging.basicConfig(level=settings.logging.level, + handlers=HandlerFactory.create(HandlerFactory.Target.ALL, handler_prefix='karl.', + file_path=settings.logging.path)) loggers = ( "uvicorn", @@ -33,12 +38,16 @@ class KarlApplication: "asyncio", "starlette", ) - external_handler = ExternalLoggingHandler() + external_handlers = HandlerFactory.create(HandlerFactory.Target.ALL, file_path=settings.logging.path) for logger_name in loggers: logging_logger = logging.getLogger(logger_name) - logging_logger.handlers = [external_handler] + logging_logger.handlers = external_handlers logging_logger.propagate = False + def _set_middlewares(self, app: FastAPI): + from app.web.middlewares import LoggingMiddleware + app.add_middleware(LoggingMiddleware) + def _set_routes(self, app: FastAPI): from app.core.router import router as core_router app.include_router(core_router) @@ -52,11 +61,9 @@ class KarlApplication: def _init_services(self): logger = logging.getLogger(__name__) load_injection_container() - webhook_service = WebhookProcessor() - logger.info(webhook_service.health) -def app(): +def run(): return KarlApplication() @@ -65,7 +72,7 @@ if __name__ == "__main__": settings = get_settings() uvicorn.run( - "app.main:app", + "app.main:run", factory=True, host=settings.app.host, port=settings.app.port, diff --git a/app/util/logging.py b/app/util/logging.py index 29992b6..3e9e77a 100644 --- a/app/util/logging.py +++ b/app/util/logging.py @@ -1,4 +1,8 @@ -from logging import Formatter, StreamHandler +from enum import Enum, auto +from logging import Formatter, StreamHandler, Handler +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path +from typing import List class NamingCache: @@ -37,7 +41,7 @@ class ApplicationFormatter(Formatter): from datetime import datetime timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') level = record.levelname.replace('WARNING', 'WARN').rjust(5) - thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? + thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"] message = record.getMessage() formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}" @@ -48,12 +52,35 @@ class ApplicationFormatter(Formatter): return formatted -class LoggingHandler(StreamHandler): - def __init__(self): - super().__init__() - self.setFormatter(ApplicationFormatter(handler_prefix='karl.')) +class HandlerFactory: + class Target(Enum): + CONSOLE = auto() + FILE = auto() + ALL = auto() -class ExternalLoggingHandler(StreamHandler): - def __init__(self): - super().__init__() - self.setFormatter(ApplicationFormatter()) + @staticmethod + def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]: + def console_handler(prefix: str = ''): + handler = StreamHandler() + handler.setFormatter(ApplicationFormatter(prefix)) + handler.setLevel('INFO') + return handler + + def file_handler(prefix: str = ''): + handler = TimedRotatingFileHandler(file_path, when='midnight', backupCount=30) + handler.setFormatter(ApplicationFormatter(prefix)) + handler.setLevel('TRACE') + return handler + + handlers = [] + match target: + case HandlerFactory.Target.CONSOLE: + handlers.append(console_handler(handler_prefix)) + case HandlerFactory.Target.FILE: + handlers.append(file_handler(handler_prefix)) + case HandlerFactory.Target.ALL: + handlers.append(file_handler(handler_prefix)) + handlers.append(console_handler(handler_prefix)) + case _: + raise ValueError(f"Unknown target: {target}") + return handlers diff --git a/app/web/__init__.py b/app/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/web/middlewares.py b/app/web/middlewares.py new file mode 100644 index 0000000..d162ed1 --- /dev/null +++ b/app/web/middlewares.py @@ -0,0 +1,24 @@ +import logging + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request + +logger = logging.getLogger(__name__) + + +class LoggingMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + client = f"{request.client.host}:{request.client.port}" + match request.method: + case "POST" | "PUT" | "DELETE" if request.headers.get("Content-Type") == "application/json": + body = await request.body() + logger.trace(f"Request from {client}: {body.decode()}") + case "GET": + logger.trace(f"Request from {client}") + case _: + logger.trace(f"Request from {client} (content-type:{request.headers.get("Content-Type")})") + + response = await call_next(request) + logger.trace(f"Respone: {response.status_code} {type(response)}") + + return response diff --git a/config/config.yaml b/config/config.yaml index 856d24e..6c2043e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,3 +1,6 @@ +logging: + level: "TRACE" + path: "logs/karl.log" app: host: "127.0.0.1" port: 8081 diff --git a/run.sh b/run.sh index 8d2e2b9..cf3a77b 100644 --- a/run.sh +++ b/run.sh @@ -1 +1 @@ -uvicorn app.main:app --factory --reload +uvicorn app.main:run --factory --reload From 3845cc7ecf3bf357cb7aed77777222721bb2f0ba Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 23:58:15 +0200 Subject: [PATCH 20/99] queue --- app/api/v1.py | 2 +- app/core/core.py | 15 ++++++++++---- app/core/queue.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 app/core/queue.py diff --git a/app/api/v1.py b/app/api/v1.py index eba78a4..3950b3b 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -30,5 +30,5 @@ class APIv1: @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.process_ci_event(mapper.map(request)) + self.webhook_service.enqueue(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index a7096d2..adc68fd 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -1,23 +1,30 @@ +import uuid from typing import Annotated from injectable import injectable, autowired, Autowired +from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result from app.model.healthcheck import HealthCheck from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords -@injectable -class WebhookProcessor: +@injectable(singleton=True) +class WebhookProcessor(EnqueuedProcessor): @autowired def __init__(self, docker: Annotated[DockerService, Autowired], git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired]): + keepass: Annotated[Passwords, Autowired], + queue: Annotated[ProcessQueue, Autowired]): + super().__init__(queue) self._docker = docker self._git = git self._keepass = keepass - def process_ci_event(self, event: WebhookEvent): + def enqueue(self, event: WebhookEvent): + self._enqueue(Task(uuid.UUID(), self, event)) + + def _process(self, task: Task) -> Result: pass @property diff --git a/app/core/queue.py b/app/core/queue.py new file mode 100644 index 0000000..4788c3a --- /dev/null +++ b/app/core/queue.py @@ -0,0 +1,50 @@ +import time +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass +from multiprocessing import Queue, Process + +from injectable import injectable + + +@dataclass +class Task: + _id: uuid.UUID + processor: 'EnqueuedProcessor' + payload: object + +@dataclass +class Result: + _id: uuid.UUID + success: bool + error: str | None = None + +@injectable(singleton=True) +class ProcessQueue: + def __init__(self): + self._q = Queue() + self._process_thread = Process(target=self._run, args=(self._q,)) + self._process_thread.start() + + def put(self, task: Task): + self._q.put(task) + + def _run(self, queue: Queue): + while True: + if queue.empty(): + time.sleep(10) + continue + task = queue.get() + task.processor._process(task.payload) + + +class EnqueuedProcessor(ABC): + def __init__(self, queue: ProcessQueue): + self._queue = queue + + def _enqueue(self, task: Task): + self._queue.put(task) + + @abstractmethod + def _process(self, task: Task) -> Result: + pass From 1440ec51b71efb5f1603c6b9fc68c2e00f0417f2 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 30 Oct 2025 21:05:06 +0100 Subject: [PATCH 21/99] Generics & visitor pattern? --- app/core/core.py | 20 ++++++++++++++++++-- app/core/queue.py | 8 +++++--- app/services/containers.py | 5 +++++ app/services/vcs.py | 6 +++++- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/app/core/core.py b/app/core/core.py index adc68fd..90d49d0 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -1,5 +1,6 @@ +import logging import uuid -from typing import Annotated +from typing import Annotated, List from injectable import injectable, autowired, Autowired @@ -8,6 +9,7 @@ from app.model.healthcheck import HealthCheck from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords +logger = logging.getLogger(__name__) @injectable(singleton=True) class WebhookProcessor(EnqueuedProcessor): @@ -24,7 +26,21 @@ class WebhookProcessor(EnqueuedProcessor): def enqueue(self, event: WebhookEvent): self._enqueue(Task(uuid.UUID(), self, event)) - def _process(self, task: Task) -> Result: + def _process(self, task: Task[WebhookEvent]) -> Result: + event: WebhookEvent = task.payload + # TODO: persist event data + commit_hash = self._git.get_new_commit_hash() + if commit_hash != event.commit: + logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") + return Result(task.id, False, "Commit hash mismatch") + # TODO: persist commit data + service = self._get_service(event.files) + + + + return Result(task.id, True) + + def _get_service(self, files: List[str]) -> str: pass @property diff --git a/app/core/queue.py b/app/core/queue.py index 4788c3a..3e92cd6 100644 --- a/app/core/queue.py +++ b/app/core/queue.py @@ -3,15 +3,17 @@ import uuid from abc import ABC, abstractmethod from dataclasses import dataclass from multiprocessing import Queue, Process +from typing import TypeVar from injectable import injectable +T = TypeVar('T') @dataclass -class Task: - _id: uuid.UUID +class Task[T]: + id: uuid.UUID processor: 'EnqueuedProcessor' - payload: object + payload: T @dataclass class Result: diff --git a/app/services/containers.py b/app/services/containers.py index ef891e3..a89343b 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -1,14 +1,19 @@ +import logging + import docker from docker.models.containers import Container from injectable import injectable from app.model.containers import Tree, Compose, SimpleContainer +logger = logging.getLogger(__name__) + @injectable(singleton=True) class DockerService: def __init__(self): self._client = docker.from_env() + # logger.info(f"Docker client initialized. Plugins: {self._client.plugins()}") self._tree = self._init_tree() def _init_tree(self) -> Tree: diff --git a/app/services/vcs.py b/app/services/vcs.py index a7d83b0..abb5468 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -13,15 +13,19 @@ class GitService: self._repo.git.checkout(self._settings.git.branch) self._origin: Remote = self._repo.remotes.origin - def get_modified_compose(self) -> str | None: self._update() return self._diff() + def get_new_commit_hash(self) -> str: + self._update() + return self._repo.head.commit.hexsha + @staticmethod def _check_preconditions(config: GitConfig) -> Repo: def clone(): return Repo.clone_from(config.url, config.path, branch=config.branch) + import os if not config.path.exists(): return clone() From 87e8af3f729fa24fa02c5758ae9af7a9406b9698 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 31 Oct 2025 00:01:45 +0100 Subject: [PATCH 22/99] Event bus basics --- app/api/v1.py | 14 +++++++++---- app/core/core.py | 10 +++++----- app/core/injects.py | 5 +++++ app/core/woodpecker.py | 22 +++++++++++++++++++++ app/events/__init__.py | 41 +++++++++++++++++++++++++++++++++++++++ app/main.py | 6 +----- app/model/webhook.py | 4 +++- app/services/passwords.py | 2 +- 8 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 app/core/woodpecker.py create mode 100644 app/events/__init__.py diff --git a/app/api/v1.py b/app/api/v1.py index 3950b3b..4fbe31f 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,4 +1,4 @@ -from automapper import mapper +from automapper import mapper, exceptions from fastapi import APIRouter, Depends from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response @@ -6,6 +6,7 @@ from starlette.responses import JSONResponse, Response from app.api.models import Request from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport +from app.events import SimpleEventBus from app.model.webhook import WebhookEvent router = APIRouter() @@ -19,16 +20,21 @@ async def root(): @cbv(router) class APIv1: webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) + event_bus: SimpleEventBus = Depends(AutowireSupport.event_bus) logger = __import__('logging').getLogger(__name__) def __init__(self): - mapper.add(Request, WebhookEvent) + try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie + mapper.add(Request, WebhookEvent) + except exceptions.DuplicatedRegistrationError: + pass @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: - return JSONResponse({"status": self.webhook_service.health}) + # TODO: JSON serialize + return JSONResponse({"status": self.webhook_service.health.healthy}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.enqueue(mapper.map(request)) + self.event_bus.publish(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index 90d49d0..3b16ea1 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -3,6 +3,7 @@ import uuid from typing import Annotated, List from injectable import injectable, autowired, Autowired +from typing_extensions import deprecated from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result from app.model.healthcheck import HealthCheck @@ -11,20 +12,19 @@ from app.services import DockerService, GitService, Passwords logger = logging.getLogger(__name__) +@deprecated("Use event bus instead.") @injectable(singleton=True) -class WebhookProcessor(EnqueuedProcessor): +class WebhookProcessor: @autowired def __init__(self, docker: Annotated[DockerService, Autowired], git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired], - queue: Annotated[ProcessQueue, Autowired]): - super().__init__(queue) + keepass: Annotated[Passwords, Autowired]): self._docker = docker self._git = git self._keepass = keepass def enqueue(self, event: WebhookEvent): - self._enqueue(Task(uuid.UUID(), self, event)) + pass def _process(self, task: Task[WebhookEvent]) -> Result: event: WebhookEvent = task.payload diff --git a/app/core/injects.py b/app/core/injects.py index 3dfcc66..0d7ff9c 100644 --- a/app/core/injects.py +++ b/app/core/injects.py @@ -1,6 +1,7 @@ from injectable import inject from app.core.core import WebhookProcessor +from app.events import SimpleEventBus class AutowireSupport: @@ -8,3 +9,7 @@ class AutowireSupport: @staticmethod def webhook_processor(): return inject(WebhookProcessor) + + @staticmethod + def event_bus(): + return inject(SimpleEventBus) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py new file mode 100644 index 0000000..8236bf5 --- /dev/null +++ b/app/core/woodpecker.py @@ -0,0 +1,22 @@ +import logging + +from injectable import injectable + +from app.events import SimpleEventBus +from app.model.webhook import WebhookEvent + +logger = logging.getLogger(__name__) + +@injectable +class Woodpecker: + + @SimpleEventBus.on(WebhookEvent) + def on_event(self, event): # TODO: caller nie działa -> brakuje instancji klasy? + logger.info(f"Received event: {event}") + pass + + +@SimpleEventBus.on(WebhookEvent) +def on_event2(event): # TODO: Tu działa + logger.info(f"F2: Received event: {event}") + pass diff --git a/app/events/__init__.py b/app/events/__init__.py new file mode 100644 index 0000000..fbd92d2 --- /dev/null +++ b/app/events/__init__.py @@ -0,0 +1,41 @@ +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import wraps +from typing import Dict, List, Callable + +from injectable import injectable, inject + + +@dataclass +class Event: + pass + + +@injectable(singleton=True) +class SimpleEventBus: + def __init__(self): + self._handlers: Dict[type, List[Callable]] = {} + self._executor = ThreadPoolExecutor() + + def publish(self, event: Event) -> None: + for handler in self._handlers.get(type(event), []): + # Fire-and-forget execution + self._executor.submit(handler, event) + + def subscribe(self, event_type: type, handler: Callable) -> None: + if event_type not in self._handlers: + self._handlers[event_type] = [] + self._handlers[event_type].append(handler) + + @staticmethod + def on(event: type) -> Callable: + def outer(func): + inject(SimpleEventBus).subscribe(event, func) + + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + return outer diff --git a/app/main.py b/app/main.py index 424548e..669d2d4 100644 --- a/app/main.py +++ b/app/main.py @@ -11,11 +11,11 @@ class KarlApplication: from starlette.types import Receive, Scope, Send def __init__(self) -> None: self._set_logging() + load_injection_container() _app = FastAPI(title="Karl", version="0.1.0") self._set_middlewares(_app) self._set_routes(_app) self._set_events(_app) - self._init_services() self._app = _app @@ -58,10 +58,6 @@ class KarlApplication: def _set_events(self, app: FastAPI): pass - def _init_services(self): - logger = logging.getLogger(__name__) - load_injection_container() - def run(): return KarlApplication() diff --git a/app/model/webhook.py b/app/model/webhook.py index da9ad8d..e8c3b0d 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,9 +1,11 @@ from dataclasses import dataclass from typing import List +from app.events import Event + @dataclass -class WebhookEvent: +class WebhookEvent(Event): _id: str commit: str message: str diff --git a/app/services/passwords.py b/app/services/passwords.py index 18d8519..6ff8406 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -11,7 +11,7 @@ class Passwords: settings = get_settings() with open(settings.kp.secret, "r") as fh: - secret = fh.read() + secret = fh.read().splitlines()[0] self._kp_org = self.__get_or_create_store(settings.kp.file, secret) self._kp = self.__get_lock(settings.kp.file, secret) From 29dfc13a48a4910780de9a10e0f5990a9e49ec54 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 31 Oct 2025 23:30:35 +0100 Subject: [PATCH 23/99] Woodpecker event registered --- app/core/woodpecker.py | 21 ++++++++++++--------- app/events/__init__.py | 6 +++--- config/config.yaml | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 8236bf5..9b6ffb4 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,22 +1,25 @@ import logging +from typing import Annotated -from injectable import injectable +from injectable import injectable, Autowired, autowired, inject, injectable_factory from app.events import SimpleEventBus from app.model.webhook import WebhookEvent logger = logging.getLogger(__name__) -@injectable -class Woodpecker: - @SimpleEventBus.on(WebhookEvent) - def on_event(self, event): # TODO: caller nie działa -> brakuje instancji klasy? +@injectable(singleton=True) +class Woodpecker: + @autowired + def __init__(self, event_bus: Annotated[SimpleEventBus, Autowired]): + logger.info("Woodpecker initialized.") + event_bus.subscribe(WebhookEvent, self.on_ci_event) + + def on_ci_event(self, event): logger.info(f"Received event: {event}") pass -@SimpleEventBus.on(WebhookEvent) -def on_event2(event): # TODO: Tu działa - logger.info(f"F2: Received event: {event}") - pass +instance = Woodpecker(inject(SimpleEventBus)) +injectable_factory(Woodpecker)(lambda: instance) diff --git a/app/events/__init__.py b/app/events/__init__.py index fbd92d2..5d54a87 100644 --- a/app/events/__init__.py +++ b/app/events/__init__.py @@ -18,9 +18,9 @@ class SimpleEventBus: self._executor = ThreadPoolExecutor() def publish(self, event: Event) -> None: - for handler in self._handlers.get(type(event), []): - # Fire-and-forget execution - self._executor.submit(handler, event) + for handler in self._handlers.get(type(event), []): + # Fire-and-forget execution + self._executor.submit(handler, event) def subscribe(self, event_type: type, handler: Callable) -> None: if event_type not in self._handlers: diff --git a/config/config.yaml b/config/config.yaml index 6c2043e..c4fc44a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,7 +4,7 @@ logging: app: host: "127.0.0.1" port: 8081 - reload: true + reload: false git: path: "F:/IdeaProjects/paas/karl/.compose_repository" branch: "main" From 34ee5f87549a7838404e1c8218d344032d6367ca Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sat, 1 Nov 2025 23:02:28 +0100 Subject: [PATCH 24/99] CI events flow simplified --- app/api/v1.py | 16 ++++++------ app/core/core.py | 51 ------------------------------------- app/core/injects.py | 11 +++----- app/core/queue.py | 52 -------------------------------------- app/core/woodpecker.py | 57 +++++++++++++++++++++++++++++++++++------- app/events/__init__.py | 41 ------------------------------ app/model/webhook.py | 5 +--- 7 files changed, 59 insertions(+), 174 deletions(-) delete mode 100644 app/core/core.py delete mode 100644 app/core/queue.py delete mode 100644 app/events/__init__.py diff --git a/app/api/v1.py b/app/api/v1.py index 4fbe31f..a012834 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -4,10 +4,9 @@ from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response from app.api.models import Request -from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport -from app.events import SimpleEventBus -from app.model.webhook import WebhookEvent +from app.core.woodpecker import Woodpecker +from app.model.webhook import WoodpeckerEvent router = APIRouter() @@ -19,22 +18,21 @@ async def root(): @cbv(router) class APIv1: - webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) - event_bus: SimpleEventBus = Depends(AutowireSupport.event_bus) + woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) logger = __import__('logging').getLogger(__name__) def __init__(self): - try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie - mapper.add(Request, WebhookEvent) + try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie + mapper.add(Request, WoodpeckerEvent) except exceptions.DuplicatedRegistrationError: pass @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: # TODO: JSON serialize - return JSONResponse({"status": self.webhook_service.health.healthy}) + return JSONResponse({"status": "unknown"}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.event_bus.publish(mapper.map(request)) + self.woodpecker.on_ci_event(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py deleted file mode 100644 index 3b16ea1..0000000 --- a/app/core/core.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging -import uuid -from typing import Annotated, List - -from injectable import injectable, autowired, Autowired -from typing_extensions import deprecated - -from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result -from app.model.healthcheck import HealthCheck -from app.model.webhook import WebhookEvent -from app.services import DockerService, GitService, Passwords - -logger = logging.getLogger(__name__) - -@deprecated("Use event bus instead.") -@injectable(singleton=True) -class WebhookProcessor: - @autowired - def __init__(self, docker: Annotated[DockerService, Autowired], - git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired]): - self._docker = docker - self._git = git - self._keepass = keepass - - def enqueue(self, event: WebhookEvent): - pass - - def _process(self, task: Task[WebhookEvent]) -> Result: - event: WebhookEvent = task.payload - # TODO: persist event data - commit_hash = self._git.get_new_commit_hash() - if commit_hash != event.commit: - logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") - return Result(task.id, False, "Commit hash mismatch") - # TODO: persist commit data - service = self._get_service(event.files) - - - - return Result(task.id, True) - - def _get_service(self, files: List[str]) -> str: - pass - - @property - def health(self) -> HealthCheck: - return HealthCheck( - self._docker is not None and self._git is not None and self._keepass is not None, - f"Docker: {self._docker is not None}, Git: {self._git is not None}, KeePass: {self._keepass is not None}" - ) diff --git a/app/core/injects.py b/app/core/injects.py index 0d7ff9c..115ae95 100644 --- a/app/core/injects.py +++ b/app/core/injects.py @@ -1,15 +1,10 @@ from injectable import inject -from app.core.core import WebhookProcessor -from app.events import SimpleEventBus +from app.core.woodpecker import Woodpecker class AutowireSupport: @staticmethod - def webhook_processor(): - return inject(WebhookProcessor) - - @staticmethod - def event_bus(): - return inject(SimpleEventBus) + def woodpecker(): + return inject(Woodpecker) diff --git a/app/core/queue.py b/app/core/queue.py deleted file mode 100644 index 3e92cd6..0000000 --- a/app/core/queue.py +++ /dev/null @@ -1,52 +0,0 @@ -import time -import uuid -from abc import ABC, abstractmethod -from dataclasses import dataclass -from multiprocessing import Queue, Process -from typing import TypeVar - -from injectable import injectable - -T = TypeVar('T') - -@dataclass -class Task[T]: - id: uuid.UUID - processor: 'EnqueuedProcessor' - payload: T - -@dataclass -class Result: - _id: uuid.UUID - success: bool - error: str | None = None - -@injectable(singleton=True) -class ProcessQueue: - def __init__(self): - self._q = Queue() - self._process_thread = Process(target=self._run, args=(self._q,)) - self._process_thread.start() - - def put(self, task: Task): - self._q.put(task) - - def _run(self, queue: Queue): - while True: - if queue.empty(): - time.sleep(10) - continue - task = queue.get() - task.processor._process(task.payload) - - -class EnqueuedProcessor(ABC): - def __init__(self, queue: ProcessQueue): - self._queue = queue - - def _enqueue(self, task: Task): - self._queue.put(task) - - @abstractmethod - def _process(self, task: Task) -> Result: - pass diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 9b6ffb4..9ba242d 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,25 +1,64 @@ import logging +from collections import deque +from multiprocessing import Process, Lock from typing import Annotated -from injectable import injectable, Autowired, autowired, inject, injectable_factory +from injectable import injectable, Autowired, autowired -from app.events import SimpleEventBus -from app.model.webhook import WebhookEvent +from app.model.webhook import WoodpeckerEvent +from app.services import Passwords, GitService, DockerService logger = logging.getLogger(__name__) +class WoodpeckerRunner(Process): + def __init__(self, event: WoodpeckerEvent): + super().__init__() + self._event = event + + def run(self): + super().run() + """ + event: WebhookEvent = task.payload + # TODO: persist event data + commit_hash = self._git.get_new_commit_hash() + if commit_hash != event.commit: + logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") + return Result(task.id, False, "Commit hash mismatch") + # TODO: persist commit data + service = self._get_service(event.files) + + """ + + @injectable(singleton=True) class Woodpecker: @autowired - def __init__(self, event_bus: Annotated[SimpleEventBus, Autowired]): + def __init__(self, passwords: Annotated[Passwords, Autowired]): + self._passwords = passwords + self._git = GitService() + self._docker = DockerService() + self._runner: WoodpeckerRunner | None = None + self._pending = deque() + self._lock = Lock() logger.info("Woodpecker initialized.") - event_bus.subscribe(WebhookEvent, self.on_ci_event) - def on_ci_event(self, event): + def on_ci_event(self, event: WoodpeckerEvent): logger.info(f"Received event: {event}") + with self._lock: + if len(self._pending) > 0 or self._runner is not None: + self._pending.append(event) + return + self._start_runner(event) + + def _start_runner(self, event: WoodpeckerEvent): pass - -instance = Woodpecker(inject(SimpleEventBus)) -injectable_factory(Woodpecker)(lambda: instance) + def _on_runner_completed(self): + logger.info("Runner completed.") + self._runner.join() + with self._lock: + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) diff --git a/app/events/__init__.py b/app/events/__init__.py deleted file mode 100644 index 5d54a87..0000000 --- a/app/events/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from functools import wraps -from typing import Dict, List, Callable - -from injectable import injectable, inject - - -@dataclass -class Event: - pass - - -@injectable(singleton=True) -class SimpleEventBus: - def __init__(self): - self._handlers: Dict[type, List[Callable]] = {} - self._executor = ThreadPoolExecutor() - - def publish(self, event: Event) -> None: - for handler in self._handlers.get(type(event), []): - # Fire-and-forget execution - self._executor.submit(handler, event) - - def subscribe(self, event_type: type, handler: Callable) -> None: - if event_type not in self._handlers: - self._handlers[event_type] = [] - self._handlers[event_type].append(handler) - - @staticmethod - def on(event: type) -> Callable: - def outer(func): - inject(SimpleEventBus).subscribe(event, func) - - @wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - - return wrapper - - return outer diff --git a/app/model/webhook.py b/app/model/webhook.py index e8c3b0d..364e17b 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,11 +1,8 @@ from dataclasses import dataclass from typing import List -from app.events import Event - - @dataclass -class WebhookEvent(Event): +class WoodpeckerEvent: _id: str commit: str message: str From 312631f0b57fe7bacc9fc2dbcea91a832180fe16 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 00:35:33 +0100 Subject: [PATCH 25/99] WoodpeckerRunner & some VCS changes --- app/core/woodpecker.py | 66 +++++++++++++++++++++++++++++++++--------- app/services/vcs.py | 23 ++------------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 9ba242d..ae8553b 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -12,23 +12,50 @@ logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, event: WoodpeckerEvent): - super().__init__() + def __init__(self, git: GitService, docker: DockerService, success_callback=None, error_callback=None): + super().__init__(daemon=True) + self._git = git + self._docker = docker + self._success_callback = success_callback + self._error_callback = error_callback + self._event: WoodpeckerEvent | None = None + + def process_event(self, event: WoodpeckerEvent): self._event = event + self.start() def run(self): - super().run() - """ - event: WebhookEvent = task.payload - # TODO: persist event data - commit_hash = self._git.get_new_commit_hash() - if commit_hash != event.commit: - logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") - return Result(task.id, False, "Commit hash mismatch") - # TODO: persist commit data - service = self._get_service(event.files) + try: + service = self.get_service(self._event.files) + if service is None: + logger.info("No service found.") + return self._success_callback() - """ + self._git.checkout(self._event.commit) + """ + TODO: + check for *.mo.* files + subs mo from pass + docker compose up -d -f service/docker-compose.yml + """ + + return self._success_callback() + except Exception as e: + return self._error_callback(e) + + def get_service(self, files: list[str]) -> str | None: + supported_files = [] + for f in files: + f_parts = f.split("/") + if f_parts[0] in ["compose", "files"]: + supported_files.append(f[1]) + match len(set(supported_files)): + case 0: + return None + case 1: + return supported_files[0] + case _: + raise Exception("Multiple services are not supported.") @injectable(singleton=True) @@ -52,7 +79,9 @@ class Woodpecker: self._start_runner(event) def _start_runner(self, event: WoodpeckerEvent): - pass + with self._lock: + self._runner = WoodpeckerRunner(self._git, self._docker, self._on_runner_completed) + self._runner.process_event(event) def _on_runner_completed(self): logger.info("Runner completed.") @@ -62,3 +91,12 @@ class Woodpecker: if len(self._pending) > 0: event = self._pending.popleft() self._start_runner(event) + + def _on_runner_error(self, t: Exception): + logger.error(f"Runner error: {t}", exc_info=True) + self._runner.join() + with self._lock: + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) diff --git a/app/services/vcs.py b/app/services/vcs.py index abb5468..00ef3c2 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -13,14 +13,6 @@ class GitService: self._repo.git.checkout(self._settings.git.branch) self._origin: Remote = self._repo.remotes.origin - def get_modified_compose(self) -> str | None: - self._update() - return self._diff() - - def get_new_commit_hash(self) -> str: - self._update() - return self._repo.head.commit.hexsha - @staticmethod def _check_preconditions(config: GitConfig) -> Repo: def clone(): @@ -34,16 +26,5 @@ class GitService: return clone() return Repo(config.path) - def _update(self): - self._origin.pull() - - def _diff(self) -> str | None: - diff = self._repo.head.commit.diff("HEAD~1") - composes = [f for f in diff if f.a_path.endswith("docker-compose.yml")] - match len(composes): - case 0: - return None - case 1: - return composes[0].a_path - case _: - raise Exception("Multiple compose files modified") + def checkout(self, sha: str): + pass From 1341b022d42b1bd4377799b53a02f7d9edf026c7 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:04:21 +0100 Subject: [PATCH 26/99] Woodpecker main loop almost finished --- app/core/woodpecker.py | 18 ++++++++++-------- app/services/containers.py | 4 ++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index ae8553b..caa29d5 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,10 +1,12 @@ import logging from collections import deque from multiprocessing import Process, Lock +from pathlib import Path from typing import Annotated from injectable import injectable, Autowired, autowired +from app.config import get_settings from app.model.webhook import WoodpeckerEvent from app.services import Passwords, GitService, DockerService @@ -12,10 +14,12 @@ logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, git: GitService, docker: DockerService, success_callback=None, error_callback=None): + def __init__(self, git: GitService, docker: DockerService, passwords: Passwords, + success_callback=None, error_callback=None): super().__init__(daemon=True) self._git = git self._docker = docker + self._passwords = passwords self._success_callback = success_callback self._error_callback = error_callback self._event: WoodpeckerEvent | None = None @@ -30,14 +34,12 @@ class WoodpeckerRunner(Process): if service is None: logger.info("No service found.") return self._success_callback() - + service_path = f"{get_settings().git.path}/compose/{service}/docker-compose.yml" self._git.checkout(self._event.commit) - """ - TODO: - check for *.mo.* files - subs mo from pass - docker compose up -d -f service/docker-compose.yml - """ + for file in self._event.files: + if file.__contains__('.mo.'): + pass + self._docker.reload(Path(service_path)) return self._success_callback() except Exception as e: diff --git a/app/services/containers.py b/app/services/containers.py index a89343b..20f0098 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path import docker from docker.models.containers import Container @@ -33,3 +34,6 @@ class DockerService: @property def tree(self) -> Tree: return self._tree + + def reload(self, compose_path: Path): + pass From 8ee950940e027db2e4b0202f5815e3df724d3589 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:15:12 +0100 Subject: [PATCH 27/99] Mo basics --- app/core/woodpecker.py | 21 ++++++++++++--------- app/services/mo.py | 16 ++++++++++++++++ app/services/passwords.py | 33 ++++++++++++++++----------------- 3 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 app/services/mo.py diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index caa29d5..1892c0b 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -8,21 +8,23 @@ from injectable import injectable, Autowired, autowired from app.config import get_settings from app.model.webhook import WoodpeckerEvent -from app.services import Passwords, GitService, DockerService +from app.services import GitService, DockerService +from app.services.mo import Mo logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, git: GitService, docker: DockerService, passwords: Passwords, + def __init__(self, git: GitService, docker: DockerService, mo: Mo, success_callback=None, error_callback=None): super().__init__(daemon=True) self._git = git self._docker = docker - self._passwords = passwords + self._mo = mo self._success_callback = success_callback self._error_callback = error_callback self._event: WoodpeckerEvent | None = None + self._root = get_settings().git.path def process_event(self, event: WoodpeckerEvent): self._event = event @@ -34,12 +36,12 @@ class WoodpeckerRunner(Process): if service is None: logger.info("No service found.") return self._success_callback() - service_path = f"{get_settings().git.path}/compose/{service}/docker-compose.yml" + service_path = f"{self._root}/compose/{service}/docker-compose.yml" self._git.checkout(self._event.commit) for file in self._event.files: if file.__contains__('.mo.'): - pass - self._docker.reload(Path(service_path)) + self._mo.process(Path(f"{self._root}{file}").absolute()) + self._docker.reload(Path(service_path).absolute()) return self._success_callback() except Exception as e: @@ -63,8 +65,8 @@ class WoodpeckerRunner(Process): @injectable(singleton=True) class Woodpecker: @autowired - def __init__(self, passwords: Annotated[Passwords, Autowired]): - self._passwords = passwords + def __init__(self, mo: Annotated[Mo, Autowired]): + self._mo = mo self._git = GitService() self._docker = DockerService() self._runner: WoodpeckerRunner | None = None @@ -82,7 +84,8 @@ class Woodpecker: def _start_runner(self, event: WoodpeckerEvent): with self._lock: - self._runner = WoodpeckerRunner(self._git, self._docker, self._on_runner_completed) + self._runner = WoodpeckerRunner(self._git, self._docker, self._mo, + self._on_runner_completed, self._on_runner_error) self._runner.process_event(event) def _on_runner_completed(self): diff --git a/app/services/mo.py b/app/services/mo.py new file mode 100644 index 0000000..3b1520d --- /dev/null +++ b/app/services/mo.py @@ -0,0 +1,16 @@ +from pathlib import Path +from typing import Annotated + +from injectable import injectable, autowired, Autowired + +from app.services import Passwords + + +@injectable +class Mo: + @autowired + def __init__(self, passwords: Annotated[Passwords, Autowired]): + self._passwords = passwords + + def process(self, mo_file: Path): + pass diff --git a/app/services/passwords.py b/app/services/passwords.py index 6ff8406..e3a181c 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -1,7 +1,8 @@ import os.path +import shutil from injectable import injectable -from pykeepass import PyKeePass, create_database, Group +from pykeepass import PyKeePass, create_database @injectable(singleton=True) @@ -12,29 +13,27 @@ class Passwords: with open(settings.kp.secret, "r") as fh: secret = fh.read().splitlines()[0] - - self._kp_org = self.__get_or_create_store(settings.kp.file, secret) - self._kp = self.__get_lock(settings.kp.file, secret) + self._path = settings.kp.file + self._kp_org = self._open_or_create(self._path, secret) + self._kp = self._open_lock(self._path, secret) @staticmethod - def __get_or_create_store(path, passwd) -> PyKeePass: + def _open_or_create(path, password) -> PyKeePass: if os.path.exists(path): - return PyKeePass( - path, - password=passwd, - ) - return create_database(path, passwd) + return PyKeePass(path, password=password) + return create_database(path, password) @staticmethod - def __get_lock(path, passwd) -> PyKeePass: + def _open_lock(path, password) -> PyKeePass: lock_path = path + ".lock" - import shutil shutil.copyfile(path, lock_path) - return Passwords.__get_or_create_store(lock_path, passwd) + return Passwords._open_or_create(lock_path, password) @property - def store(self): - return self._kp.root_group + def kp(self) -> PyKeePass: + return self._kp - def save(self, group: Group): - pass + def save(self): + # nadpisz plik źródłowy zmianami z lock + self._kp.save() + shutil.copyfile(self._path + ".lock", self._path) From 3c3be6d79c2dde279d940b6d72978de3745004d5 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:59:40 +0100 Subject: [PATCH 28/99] chevron removed --- app/services/mo.py | 10 +++++++++- app/services/passwords.py | 5 ++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/services/mo.py b/app/services/mo.py index 3b1520d..a1ae32c 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -1,4 +1,5 @@ from pathlib import Path +from string import Template from typing import Annotated from injectable import injectable, autowired, Autowired @@ -13,4 +14,11 @@ class Mo: self._passwords = passwords def process(self, mo_file: Path): - pass + raw = '' + with open(mo_file, "r") as mo: + raw = mo.read() + tpl = Template(raw) + rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) + de_mo_ified = str(mo_file).replace(".mo", "") + with open(de_mo_ified, "w") as mo: + mo.write(rendered) diff --git a/app/services/passwords.py b/app/services/passwords.py index e3a181c..1c656c6 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -29,9 +29,8 @@ class Passwords: shutil.copyfile(path, lock_path) return Passwords._open_or_create(lock_path, password) - @property - def kp(self) -> PyKeePass: - return self._kp + def get_values(self, keys: list[str]) -> dict[str, str]: + return {} def save(self): # nadpisz plik źródłowy zmianami z lock From e14180cbe7e5641aaff3762db8d8e958964aedc2 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:16:54 +0100 Subject: [PATCH 29/99] passwords#get_values --- app/services/passwords.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/services/passwords.py b/app/services/passwords.py index 1c656c6..7f30567 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -30,7 +30,15 @@ class Passwords: return Passwords._open_or_create(lock_path, password) def get_values(self, keys: list[str]) -> dict[str, str]: - return {} + output = {} + for k in keys: + key_parts = k.split(".") + path = key_parts[:-2] if len(key_parts) > 2 else None + entry_name = key_parts[-2] + field_name = key_parts[-1] + kp_entry = self._kp_org.find_entries(path=path, first=True, name=entry_name)[0] + output[k] = kp_entry[field_name] + return output def save(self): # nadpisz plik źródłowy zmianami z lock From 9166790de90d90f1766ca6a17050365d0523ef55 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:50:24 +0100 Subject: [PATCH 30/99] mo processing test --- app/services/mo.py | 6 +++++- app/services/passwords.py | 4 ++-- tests/__init__.py | 0 tests/files/test1/test.mo.yaml | 3 +++ tests/test_mo.py | 21 +++++++++++++++++++++ 5 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/files/test1/test.mo.yaml create mode 100644 tests/test_mo.py diff --git a/app/services/mo.py b/app/services/mo.py index a1ae32c..d917e62 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -6,6 +6,10 @@ from injectable import injectable, autowired, Autowired from app.services import Passwords +class DotTemplate(Template): + # Pozwala na kropki w nazwach placeholderów, np. ${user.name.first} + idpattern = r'[_a-zA-Z][_a-zA-Z0-9.]*' + @injectable class Mo: @@ -17,7 +21,7 @@ class Mo: raw = '' with open(mo_file, "r") as mo: raw = mo.read() - tpl = Template(raw) + tpl = DotTemplate(raw) rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) de_mo_ified = str(mo_file).replace(".mo", "") with open(de_mo_ified, "w") as mo: diff --git a/app/services/passwords.py b/app/services/passwords.py index 7f30567..9b11ca4 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -36,8 +36,8 @@ class Passwords: path = key_parts[:-2] if len(key_parts) > 2 else None entry_name = key_parts[-2] field_name = key_parts[-1] - kp_entry = self._kp_org.find_entries(path=path, first=True, name=entry_name)[0] - output[k] = kp_entry[field_name] + kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name)[0] + output[k] = kp_entry[field_name] # TODO: TypeError: 'Entry' object is not subscriptable return output def save(self): diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/files/test1/test.mo.yaml b/tests/files/test1/test.mo.yaml new file mode 100644 index 0000000..af55ecc --- /dev/null +++ b/tests/files/test1/test.mo.yaml @@ -0,0 +1,3 @@ +value: ${sample.password} +nested: ${some.nested.value.password} +custom: ${custom.field} diff --git a/tests/test_mo.py b/tests/test_mo.py new file mode 100644 index 0000000..731dada --- /dev/null +++ b/tests/test_mo.py @@ -0,0 +1,21 @@ +import os +from pathlib import Path +from unittest import TestCase + +from app.services import Passwords +from app.services.mo import Mo + + +class TestMo(TestCase): + def test_process(self): + mo = Mo(Passwords()) + mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) + self.assertTrue(os.path.exists('tests/files/test1/test.mo')) + with open('tests/files/test1/test.mo', 'r') as f: + content = f.read() + self.assertFalse(content.__contains__('${')) + """ + value: some_oass + nested: nested_pass + custom: custom_content + """ From e3a37419e8c59dd4049306f354ee523033dea380 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:50:24 +0100 Subject: [PATCH 31/99] mo processing test --- tests/test_mo.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_mo.py b/tests/test_mo.py index 731dada..200b3d8 100644 --- a/tests/test_mo.py +++ b/tests/test_mo.py @@ -2,6 +2,8 @@ import os from pathlib import Path from unittest import TestCase +import yaml + from app.services import Passwords from app.services.mo import Mo @@ -11,9 +13,13 @@ class TestMo(TestCase): mo = Mo(Passwords()) mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) self.assertTrue(os.path.exists('tests/files/test1/test.mo')) - with open('tests/files/test1/test.mo', 'r') as f: + with open('tests/files/test1/test.yaml', 'r') as f: content = f.read() self.assertFalse(content.__contains__('${')) + parsed = yaml.load(content, Loader=yaml.FullLoader) + self.assertEqual(parsed['value'], 'some_oass') + self.assertEqual(parsed['nested'], 'nested_pass') + self.assertEqual(parsed['custom'], 'custom_content') """ value: some_oass nested: nested_pass From 3dc27cc86838d5683a3356e5b5886308ae3a1379 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 21:56:59 +0100 Subject: [PATCH 32/99] Simple/Complex templates / parsers --- app/services/mo.py | 15 ++++++++++++--- app/services/passwords.py | 20 +++++++++++++++++--- tests/files/test1/test.mo.yaml | 6 +++--- tests/test_mo.py | 16 ++++++---------- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/app/services/mo.py b/app/services/mo.py index d917e62..9a35642 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -6,11 +6,16 @@ from injectable import injectable, autowired, Autowired from app.services import Passwords -class DotTemplate(Template): + +class SimpleValueTemplate(Template): # Pozwala na kropki w nazwach placeholderów, np. ${user.name.first} idpattern = r'[_a-zA-Z][_a-zA-Z0-9.]*' +class ComplexValueTemplate(SimpleValueTemplate): + delimiter = '@' + + @injectable class Mo: @autowired @@ -21,8 +26,12 @@ class Mo: raw = '' with open(mo_file, "r") as mo: raw = mo.read() - tpl = DotTemplate(raw) - rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) + cmp = ComplexValueTemplate(raw) + rendered = cmp.substitute(self._passwords.get_values(cmp.get_identifiers())) + smp = SimpleValueTemplate(rendered) + ids = [_id + '.password' for _id in smp.get_identifiers()] + mappings = {k.replace('.password', ''): v for k, v in self._passwords.get_values(ids).items()} + rendered = smp.substitute(mappings) de_mo_ified = str(mo_file).replace(".mo", "") with open(de_mo_ified, "w") as mo: mo.write(rendered) diff --git a/app/services/passwords.py b/app/services/passwords.py index 9b11ca4..408ec5e 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -33,13 +33,27 @@ class Passwords: output = {} for k in keys: key_parts = k.split(".") - path = key_parts[:-2] if len(key_parts) > 2 else None + path = key_parts[:-1] if len(key_parts) > 2 else None entry_name = key_parts[-2] field_name = key_parts[-1] - kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name)[0] - output[k] = kp_entry[field_name] # TODO: TypeError: 'Entry' object is not subscriptable + kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name) + output[k] = self._get_field_value(kp_entry, field_name) return output + @staticmethod + def _get_field_value(kp_entry, field_name): + if kp_entry is None: + return None + match field_name: + case "username": + return kp_entry.username + case "password": + return kp_entry.password + case "url": + return kp_entry.url + case _: + return kp_entry.get_custom_property(field_name) + def save(self): # nadpisz plik źródłowy zmianami z lock self._kp.save() diff --git a/tests/files/test1/test.mo.yaml b/tests/files/test1/test.mo.yaml index af55ecc..572b953 100644 --- a/tests/files/test1/test.mo.yaml +++ b/tests/files/test1/test.mo.yaml @@ -1,3 +1,3 @@ -value: ${sample.password} -nested: ${some.nested.value.password} -custom: ${custom.field} +value: ${sample} +nested: ${some.nested.value} +custom: @{custom.field} diff --git a/tests/test_mo.py b/tests/test_mo.py index 200b3d8..f685e34 100644 --- a/tests/test_mo.py +++ b/tests/test_mo.py @@ -10,18 +10,14 @@ from app.services.mo import Mo class TestMo(TestCase): def test_process(self): + target_path = Path('tests/files/test1/test.yaml') mo = Mo(Passwords()) mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) - self.assertTrue(os.path.exists('tests/files/test1/test.mo')) - with open('tests/files/test1/test.yaml', 'r') as f: + self.assertTrue(os.path.exists(target_path)) + with open(target_path, 'r') as f: content = f.read() self.assertFalse(content.__contains__('${')) parsed = yaml.load(content, Loader=yaml.FullLoader) - self.assertEqual(parsed['value'], 'some_oass') - self.assertEqual(parsed['nested'], 'nested_pass') - self.assertEqual(parsed['custom'], 'custom_content') - """ - value: some_oass - nested: nested_pass - custom: custom_content - """ + self.assertEqual('some_pass', parsed['value']) + self.assertEqual('nested_pass', parsed['nested']) + self.assertEqual('custom_content', parsed['custom']) From 8ca668f07e7bc4463102ce2b2b1afffff3f21838 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:20:24 +0100 Subject: [PATCH 33/99] git checkout sha --- app/services/vcs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/vcs.py b/app/services/vcs.py index 00ef3c2..0e97ec8 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -27,4 +27,5 @@ class GitService: return Repo(config.path) def checkout(self, sha: str): - pass + self._origin.fetch() + self._repo.git.checkout(sha) From 4c8716cc50a8f1edf5005ac74277329368fc7a15 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:31:00 +0100 Subject: [PATCH 34/99] Fixed cbv --- app/services/containers.py | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/services/containers.py b/app/services/containers.py index 20f0098..6e927bf 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -36,4 +36,6 @@ class DockerService: return self._tree def reload(self, compose_path: Path): - pass + cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"] + # TODO: subprocess + diff --git a/pyproject.toml b/pyproject.toml index 9b97797..1791992 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "docker>=7.1.0", "injectable==4.0.1", "py-automapper>=2.2.0", + "fastapi-utils>=0.8.0", ] [project.optional-dependencies] From 324938133f5430939f6deceef37008d9a07606f0 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:42:37 +0100 Subject: [PATCH 35/99] Global formatting fix --- app/api/models.py | 1 + app/model/passwords.py | 5 ++++- app/model/webhook.py | 1 + app/services/containers.py | 1 - app/templates/index.html | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index b27765b..8e8e4d2 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -10,6 +10,7 @@ class Request: started: str files: List[str] + @dataclass class Response: status: int diff --git a/app/model/passwords.py b/app/model/passwords.py index d795d2c..253dbd9 100644 --- a/app/model/passwords.py +++ b/app/model/passwords.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from typing import Type + # TODO: unnecessary? @dataclass @@ -8,6 +9,7 @@ class PathItem: name: str t: Type + @dataclass class Path: path: list[PathItem] = field(default_factory=list) @@ -43,7 +45,8 @@ class Password: def path(self): return self.group.path.append(self.name, type(self)) + class UnencryptedPassword(Password): - def __init__(self, name: str, value: str, group: Group): + def __init__(self, name: str, value: str, group: Group): super().__init__(name, group) self.value = value diff --git a/app/model/webhook.py b/app/model/webhook.py index 364e17b..a2ef7dc 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import List + @dataclass class WoodpeckerEvent: _id: str diff --git a/app/services/containers.py b/app/services/containers.py index 6e927bf..f754ddc 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -38,4 +38,3 @@ class DockerService: def reload(self, compose_path: Path): cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"] # TODO: subprocess - diff --git a/app/templates/index.html b/app/templates/index.html index ebbe8ca..ac9bbbe 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -3,7 +3,7 @@ {{ title }} - +