From de6184273a69aeed2aac4e3330795ea48f4a8ad7 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 17 Oct 2025 23:16:38 +0200 Subject: [PATCH 01/87] Webhook model changed --- app/api/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index 35afedb..b27765b 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -4,11 +4,11 @@ from typing import List @dataclass class Request: - build_id: str - build_url: str - commit_id: str - commit_url: str - changelist: List[str] + _id: str + commit: str + message: str + started: str + files: List[str] @dataclass class Response: From 120e929469d792310ee2c7a09c0c491338d45a5b Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 21 Oct 2025 17:41:41 +0200 Subject: [PATCH 02/87] APIv1 cbv --- app/api/v1.py | 24 ++++++++++++++++-------- app/core/injects.py | 10 ++++++++++ app/model/webhook.py | 11 +++++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 app/core/injects.py create mode 100644 app/model/webhook.py diff --git a/app/api/v1.py b/app/api/v1.py index fc52bd4..e628d1f 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,6 +1,10 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends +from fastapi_utils.cbv import cbv +from starlette.responses import JSONResponse, Response -from app.api.models import Request, Response +from app.api.models import Request +from app.core.core import WebhookProcessor +from app.core.injects import AutowireSupport router = APIRouter() @@ -10,10 +14,14 @@ async def root(): return {"message": "Witaj w API v1"} -@router.get("/health", summary="Health check") -async def health(): - return {"status": "ok"} +@cbv(router) +class APIv1: + webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) -@router.post("/ci", summary="CI Webhook") -async def ci(request: Request): - return Response(200) + @router.get("/health", summary="Health check") + async def health(self) -> JSONResponse: + return JSONResponse({"status": "ok"}) + + @router.post("/ci", summary="CI Webhook") + async def ci(self, request: Request): + return Response(status_code=201) diff --git a/app/core/injects.py b/app/core/injects.py new file mode 100644 index 0000000..3dfcc66 --- /dev/null +++ b/app/core/injects.py @@ -0,0 +1,10 @@ +from injectable import inject + +from app.core.core import WebhookProcessor + + +class AutowireSupport: + + @staticmethod + def webhook_processor(): + return inject(WebhookProcessor) diff --git a/app/model/webhook.py b/app/model/webhook.py new file mode 100644 index 0000000..70470b2 --- /dev/null +++ b/app/model/webhook.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import List + + +@dataclass +class WebhookRequest: + _id: str + commit: str + message: str + started: str + files: List[str] From 1d7c4c2dfdbe3fbefbaa5feb75eb243e93b653e6 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Tue, 21 Oct 2025 17:50:53 +0200 Subject: [PATCH 03/87] Mappings & processing WIP --- app/api/v1.py | 5 ++++- app/core/core.py | 6 +++++- app/model/webhook.py | 2 +- pyproject.toml | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/app/api/v1.py b/app/api/v1.py index e628d1f..d6355fb 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,3 +1,4 @@ +from automapper import mapper from fastapi import APIRouter, Depends from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response @@ -5,6 +6,7 @@ from starlette.responses import JSONResponse, Response from app.api.models import Request from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport +from app.model.webhook import WebhookEvent router = APIRouter() @@ -20,8 +22,9 @@ class APIv1: @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: - return JSONResponse({"status": "ok"}) + return JSONResponse({"status": self.webhook_service.health}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): + self.webhook_service.process_ci_event(mapper.to(WebhookEvent).map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index 9e3acec..a7096d2 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -3,10 +3,11 @@ from typing import Annotated from injectable import injectable, autowired, Autowired from app.model.healthcheck import HealthCheck +from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords -# @injectable +@injectable class WebhookProcessor: @autowired def __init__(self, docker: Annotated[DockerService, Autowired], @@ -16,6 +17,9 @@ class WebhookProcessor: self._git = git self._keepass = keepass + def process_ci_event(self, event: WebhookEvent): + pass + @property def health(self) -> HealthCheck: return HealthCheck( diff --git a/app/model/webhook.py b/app/model/webhook.py index 70470b2..da9ad8d 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -3,7 +3,7 @@ from typing import List @dataclass -class WebhookRequest: +class WebhookEvent: _id: str commit: str message: str diff --git a/pyproject.toml b/pyproject.toml index c4e6d55..9b97797 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "pykeepass>=4.1.1.post1", "docker>=7.1.0", "injectable==4.0.1", + "py-automapper>=2.2.0", ] [project.optional-dependencies] From 2dec6d53844b5b3fbd5c9a1a8b639011b2da8309 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 19:42:57 +0200 Subject: [PATCH 04/87] Mappings enhancement --- app/api/v1.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/api/v1.py b/app/api/v1.py index d6355fb..eba78a4 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -19,6 +19,10 @@ async def root(): @cbv(router) class APIv1: webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) + logger = __import__('logging').getLogger(__name__) + + def __init__(self): + mapper.add(Request, WebhookEvent) @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: @@ -26,5 +30,5 @@ class APIv1: @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.process_ci_event(mapper.to(WebhookEvent).map(request)) + self.webhook_service.process_ci_event(mapper.map(request)) return Response(status_code=201) From 569aefeccb500ded48ca345009a0592ac91a76d4 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 22:53:49 +0200 Subject: [PATCH 05/87] logging enhancements --- .gitignore | 1 + app/config/settings.py | 6 ++++++ app/main.py | 25 ++++++++++++++-------- app/util/logging.py | 47 +++++++++++++++++++++++++++++++++--------- app/web/__init__.py | 0 app/web/middlewares.py | 24 +++++++++++++++++++++ config/config.yaml | 3 +++ run.sh | 2 +- 8 files changed, 88 insertions(+), 20 deletions(-) create mode 100644 app/web/__init__.py create mode 100644 app/web/middlewares.py diff --git a/.gitignore b/.gitignore index 8395e24..4b259c4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ uv.lock __pycache__/ **/dist/ +**/*.log diff --git a/app/config/settings.py b/app/config/settings.py index 7e072aa..2017c14 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -6,6 +6,11 @@ from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict +class LoggingConfig(BaseModel): + level: str = "INFO" + path: Path = Path("logs/karl.log") + + class AppConfig(BaseModel): host: str = "127.0.0.1" port: int = 8081 @@ -26,6 +31,7 @@ class KeePassConfig(BaseModel): class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__") + logging: LoggingConfig = LoggingConfig() app: AppConfig = AppConfig() git: GitConfig = GitConfig() kp: KeePassConfig = KeePassConfig() diff --git a/app/main.py b/app/main.py index 5bf8c98..424548e 100644 --- a/app/main.py +++ b/app/main.py @@ -4,8 +4,7 @@ from fastapi import FastAPI from injectable import load_injection_container from app.config import get_settings -from app.core.core import WebhookProcessor -from app.util.logging import LoggingHandler, ExternalLoggingHandler +from app.util.logging import HandlerFactory class KarlApplication: @@ -13,6 +12,7 @@ class KarlApplication: def __init__(self) -> None: self._set_logging() _app = FastAPI(title="Karl", version="0.1.0") + self._set_middlewares(_app) self._set_routes(_app) self._set_events(_app) self._init_services() @@ -23,7 +23,12 @@ class KarlApplication: await self._app.__call__(scope, receive, send) def _set_logging(self): - logging.basicConfig(level=logging.INFO, handlers=[LoggingHandler()]) + settings = get_settings() + logging.addLevelName(5, "TRACE") + logging.Logger.trace = lambda s, msg, *args, **kwargs: s.log(5, msg, *args, **kwargs) + logging.basicConfig(level=settings.logging.level, + handlers=HandlerFactory.create(HandlerFactory.Target.ALL, handler_prefix='karl.', + file_path=settings.logging.path)) loggers = ( "uvicorn", @@ -33,12 +38,16 @@ class KarlApplication: "asyncio", "starlette", ) - external_handler = ExternalLoggingHandler() + external_handlers = HandlerFactory.create(HandlerFactory.Target.ALL, file_path=settings.logging.path) for logger_name in loggers: logging_logger = logging.getLogger(logger_name) - logging_logger.handlers = [external_handler] + logging_logger.handlers = external_handlers logging_logger.propagate = False + def _set_middlewares(self, app: FastAPI): + from app.web.middlewares import LoggingMiddleware + app.add_middleware(LoggingMiddleware) + def _set_routes(self, app: FastAPI): from app.core.router import router as core_router app.include_router(core_router) @@ -52,11 +61,9 @@ class KarlApplication: def _init_services(self): logger = logging.getLogger(__name__) load_injection_container() - webhook_service = WebhookProcessor() - logger.info(webhook_service.health) -def app(): +def run(): return KarlApplication() @@ -65,7 +72,7 @@ if __name__ == "__main__": settings = get_settings() uvicorn.run( - "app.main:app", + "app.main:run", factory=True, host=settings.app.host, port=settings.app.port, diff --git a/app/util/logging.py b/app/util/logging.py index 29992b6..3e9e77a 100644 --- a/app/util/logging.py +++ b/app/util/logging.py @@ -1,4 +1,8 @@ -from logging import Formatter, StreamHandler +from enum import Enum, auto +from logging import Formatter, StreamHandler, Handler +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path +from typing import List class NamingCache: @@ -37,7 +41,7 @@ class ApplicationFormatter(Formatter): from datetime import datetime timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') level = record.levelname.replace('WARNING', 'WARN').rjust(5) - thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? + thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"] message = record.getMessage() formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}" @@ -48,12 +52,35 @@ class ApplicationFormatter(Formatter): return formatted -class LoggingHandler(StreamHandler): - def __init__(self): - super().__init__() - self.setFormatter(ApplicationFormatter(handler_prefix='karl.')) +class HandlerFactory: + class Target(Enum): + CONSOLE = auto() + FILE = auto() + ALL = auto() -class ExternalLoggingHandler(StreamHandler): - def __init__(self): - super().__init__() - self.setFormatter(ApplicationFormatter()) + @staticmethod + def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]: + def console_handler(prefix: str = ''): + handler = StreamHandler() + handler.setFormatter(ApplicationFormatter(prefix)) + handler.setLevel('INFO') + return handler + + def file_handler(prefix: str = ''): + handler = TimedRotatingFileHandler(file_path, when='midnight', backupCount=30) + handler.setFormatter(ApplicationFormatter(prefix)) + handler.setLevel('TRACE') + return handler + + handlers = [] + match target: + case HandlerFactory.Target.CONSOLE: + handlers.append(console_handler(handler_prefix)) + case HandlerFactory.Target.FILE: + handlers.append(file_handler(handler_prefix)) + case HandlerFactory.Target.ALL: + handlers.append(file_handler(handler_prefix)) + handlers.append(console_handler(handler_prefix)) + case _: + raise ValueError(f"Unknown target: {target}") + return handlers diff --git a/app/web/__init__.py b/app/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/web/middlewares.py b/app/web/middlewares.py new file mode 100644 index 0000000..d162ed1 --- /dev/null +++ b/app/web/middlewares.py @@ -0,0 +1,24 @@ +import logging + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request + +logger = logging.getLogger(__name__) + + +class LoggingMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + client = f"{request.client.host}:{request.client.port}" + match request.method: + case "POST" | "PUT" | "DELETE" if request.headers.get("Content-Type") == "application/json": + body = await request.body() + logger.trace(f"Request from {client}: {body.decode()}") + case "GET": + logger.trace(f"Request from {client}") + case _: + logger.trace(f"Request from {client} (content-type:{request.headers.get("Content-Type")})") + + response = await call_next(request) + logger.trace(f"Respone: {response.status_code} {type(response)}") + + return response diff --git a/config/config.yaml b/config/config.yaml index 856d24e..6c2043e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,3 +1,6 @@ +logging: + level: "TRACE" + path: "logs/karl.log" app: host: "127.0.0.1" port: 8081 diff --git a/run.sh b/run.sh index 8d2e2b9..cf3a77b 100644 --- a/run.sh +++ b/run.sh @@ -1 +1 @@ -uvicorn app.main:app --factory --reload +uvicorn app.main:run --factory --reload From 3845cc7ecf3bf357cb7aed77777222721bb2f0ba Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Wed, 22 Oct 2025 23:58:15 +0200 Subject: [PATCH 06/87] queue --- app/api/v1.py | 2 +- app/core/core.py | 15 ++++++++++---- app/core/queue.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 app/core/queue.py diff --git a/app/api/v1.py b/app/api/v1.py index eba78a4..3950b3b 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -30,5 +30,5 @@ class APIv1: @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.process_ci_event(mapper.map(request)) + self.webhook_service.enqueue(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index a7096d2..adc68fd 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -1,23 +1,30 @@ +import uuid from typing import Annotated from injectable import injectable, autowired, Autowired +from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result from app.model.healthcheck import HealthCheck from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords -@injectable -class WebhookProcessor: +@injectable(singleton=True) +class WebhookProcessor(EnqueuedProcessor): @autowired def __init__(self, docker: Annotated[DockerService, Autowired], git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired]): + keepass: Annotated[Passwords, Autowired], + queue: Annotated[ProcessQueue, Autowired]): + super().__init__(queue) self._docker = docker self._git = git self._keepass = keepass - def process_ci_event(self, event: WebhookEvent): + def enqueue(self, event: WebhookEvent): + self._enqueue(Task(uuid.UUID(), self, event)) + + def _process(self, task: Task) -> Result: pass @property diff --git a/app/core/queue.py b/app/core/queue.py new file mode 100644 index 0000000..4788c3a --- /dev/null +++ b/app/core/queue.py @@ -0,0 +1,50 @@ +import time +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass +from multiprocessing import Queue, Process + +from injectable import injectable + + +@dataclass +class Task: + _id: uuid.UUID + processor: 'EnqueuedProcessor' + payload: object + +@dataclass +class Result: + _id: uuid.UUID + success: bool + error: str | None = None + +@injectable(singleton=True) +class ProcessQueue: + def __init__(self): + self._q = Queue() + self._process_thread = Process(target=self._run, args=(self._q,)) + self._process_thread.start() + + def put(self, task: Task): + self._q.put(task) + + def _run(self, queue: Queue): + while True: + if queue.empty(): + time.sleep(10) + continue + task = queue.get() + task.processor._process(task.payload) + + +class EnqueuedProcessor(ABC): + def __init__(self, queue: ProcessQueue): + self._queue = queue + + def _enqueue(self, task: Task): + self._queue.put(task) + + @abstractmethod + def _process(self, task: Task) -> Result: + pass From 1440ec51b71efb5f1603c6b9fc68c2e00f0417f2 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 30 Oct 2025 21:05:06 +0100 Subject: [PATCH 07/87] Generics & visitor pattern? --- app/core/core.py | 20 ++++++++++++++++++-- app/core/queue.py | 8 +++++--- app/services/containers.py | 5 +++++ app/services/vcs.py | 6 +++++- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/app/core/core.py b/app/core/core.py index adc68fd..90d49d0 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -1,5 +1,6 @@ +import logging import uuid -from typing import Annotated +from typing import Annotated, List from injectable import injectable, autowired, Autowired @@ -8,6 +9,7 @@ from app.model.healthcheck import HealthCheck from app.model.webhook import WebhookEvent from app.services import DockerService, GitService, Passwords +logger = logging.getLogger(__name__) @injectable(singleton=True) class WebhookProcessor(EnqueuedProcessor): @@ -24,7 +26,21 @@ class WebhookProcessor(EnqueuedProcessor): def enqueue(self, event: WebhookEvent): self._enqueue(Task(uuid.UUID(), self, event)) - def _process(self, task: Task) -> Result: + def _process(self, task: Task[WebhookEvent]) -> Result: + event: WebhookEvent = task.payload + # TODO: persist event data + commit_hash = self._git.get_new_commit_hash() + if commit_hash != event.commit: + logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") + return Result(task.id, False, "Commit hash mismatch") + # TODO: persist commit data + service = self._get_service(event.files) + + + + return Result(task.id, True) + + def _get_service(self, files: List[str]) -> str: pass @property diff --git a/app/core/queue.py b/app/core/queue.py index 4788c3a..3e92cd6 100644 --- a/app/core/queue.py +++ b/app/core/queue.py @@ -3,15 +3,17 @@ import uuid from abc import ABC, abstractmethod from dataclasses import dataclass from multiprocessing import Queue, Process +from typing import TypeVar from injectable import injectable +T = TypeVar('T') @dataclass -class Task: - _id: uuid.UUID +class Task[T]: + id: uuid.UUID processor: 'EnqueuedProcessor' - payload: object + payload: T @dataclass class Result: diff --git a/app/services/containers.py b/app/services/containers.py index ef891e3..a89343b 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -1,14 +1,19 @@ +import logging + import docker from docker.models.containers import Container from injectable import injectable from app.model.containers import Tree, Compose, SimpleContainer +logger = logging.getLogger(__name__) + @injectable(singleton=True) class DockerService: def __init__(self): self._client = docker.from_env() + # logger.info(f"Docker client initialized. Plugins: {self._client.plugins()}") self._tree = self._init_tree() def _init_tree(self) -> Tree: diff --git a/app/services/vcs.py b/app/services/vcs.py index a7d83b0..abb5468 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -13,15 +13,19 @@ class GitService: self._repo.git.checkout(self._settings.git.branch) self._origin: Remote = self._repo.remotes.origin - def get_modified_compose(self) -> str | None: self._update() return self._diff() + def get_new_commit_hash(self) -> str: + self._update() + return self._repo.head.commit.hexsha + @staticmethod def _check_preconditions(config: GitConfig) -> Repo: def clone(): return Repo.clone_from(config.url, config.path, branch=config.branch) + import os if not config.path.exists(): return clone() From 87e8af3f729fa24fa02c5758ae9af7a9406b9698 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 31 Oct 2025 00:01:45 +0100 Subject: [PATCH 08/87] Event bus basics --- app/api/v1.py | 14 +++++++++---- app/core/core.py | 10 +++++----- app/core/injects.py | 5 +++++ app/core/woodpecker.py | 22 +++++++++++++++++++++ app/events/__init__.py | 41 +++++++++++++++++++++++++++++++++++++++ app/main.py | 6 +----- app/model/webhook.py | 4 +++- app/services/passwords.py | 2 +- 8 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 app/core/woodpecker.py create mode 100644 app/events/__init__.py diff --git a/app/api/v1.py b/app/api/v1.py index 3950b3b..4fbe31f 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,4 +1,4 @@ -from automapper import mapper +from automapper import mapper, exceptions from fastapi import APIRouter, Depends from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response @@ -6,6 +6,7 @@ from starlette.responses import JSONResponse, Response from app.api.models import Request from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport +from app.events import SimpleEventBus from app.model.webhook import WebhookEvent router = APIRouter() @@ -19,16 +20,21 @@ async def root(): @cbv(router) class APIv1: webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) + event_bus: SimpleEventBus = Depends(AutowireSupport.event_bus) logger = __import__('logging').getLogger(__name__) def __init__(self): - mapper.add(Request, WebhookEvent) + try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie + mapper.add(Request, WebhookEvent) + except exceptions.DuplicatedRegistrationError: + pass @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: - return JSONResponse({"status": self.webhook_service.health}) + # TODO: JSON serialize + return JSONResponse({"status": self.webhook_service.health.healthy}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.webhook_service.enqueue(mapper.map(request)) + self.event_bus.publish(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py index 90d49d0..3b16ea1 100644 --- a/app/core/core.py +++ b/app/core/core.py @@ -3,6 +3,7 @@ import uuid from typing import Annotated, List from injectable import injectable, autowired, Autowired +from typing_extensions import deprecated from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result from app.model.healthcheck import HealthCheck @@ -11,20 +12,19 @@ from app.services import DockerService, GitService, Passwords logger = logging.getLogger(__name__) +@deprecated("Use event bus instead.") @injectable(singleton=True) -class WebhookProcessor(EnqueuedProcessor): +class WebhookProcessor: @autowired def __init__(self, docker: Annotated[DockerService, Autowired], git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired], - queue: Annotated[ProcessQueue, Autowired]): - super().__init__(queue) + keepass: Annotated[Passwords, Autowired]): self._docker = docker self._git = git self._keepass = keepass def enqueue(self, event: WebhookEvent): - self._enqueue(Task(uuid.UUID(), self, event)) + pass def _process(self, task: Task[WebhookEvent]) -> Result: event: WebhookEvent = task.payload diff --git a/app/core/injects.py b/app/core/injects.py index 3dfcc66..0d7ff9c 100644 --- a/app/core/injects.py +++ b/app/core/injects.py @@ -1,6 +1,7 @@ from injectable import inject from app.core.core import WebhookProcessor +from app.events import SimpleEventBus class AutowireSupport: @@ -8,3 +9,7 @@ class AutowireSupport: @staticmethod def webhook_processor(): return inject(WebhookProcessor) + + @staticmethod + def event_bus(): + return inject(SimpleEventBus) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py new file mode 100644 index 0000000..8236bf5 --- /dev/null +++ b/app/core/woodpecker.py @@ -0,0 +1,22 @@ +import logging + +from injectable import injectable + +from app.events import SimpleEventBus +from app.model.webhook import WebhookEvent + +logger = logging.getLogger(__name__) + +@injectable +class Woodpecker: + + @SimpleEventBus.on(WebhookEvent) + def on_event(self, event): # TODO: caller nie działa -> brakuje instancji klasy? + logger.info(f"Received event: {event}") + pass + + +@SimpleEventBus.on(WebhookEvent) +def on_event2(event): # TODO: Tu działa + logger.info(f"F2: Received event: {event}") + pass diff --git a/app/events/__init__.py b/app/events/__init__.py new file mode 100644 index 0000000..fbd92d2 --- /dev/null +++ b/app/events/__init__.py @@ -0,0 +1,41 @@ +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import wraps +from typing import Dict, List, Callable + +from injectable import injectable, inject + + +@dataclass +class Event: + pass + + +@injectable(singleton=True) +class SimpleEventBus: + def __init__(self): + self._handlers: Dict[type, List[Callable]] = {} + self._executor = ThreadPoolExecutor() + + def publish(self, event: Event) -> None: + for handler in self._handlers.get(type(event), []): + # Fire-and-forget execution + self._executor.submit(handler, event) + + def subscribe(self, event_type: type, handler: Callable) -> None: + if event_type not in self._handlers: + self._handlers[event_type] = [] + self._handlers[event_type].append(handler) + + @staticmethod + def on(event: type) -> Callable: + def outer(func): + inject(SimpleEventBus).subscribe(event, func) + + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + return outer diff --git a/app/main.py b/app/main.py index 424548e..669d2d4 100644 --- a/app/main.py +++ b/app/main.py @@ -11,11 +11,11 @@ class KarlApplication: from starlette.types import Receive, Scope, Send def __init__(self) -> None: self._set_logging() + load_injection_container() _app = FastAPI(title="Karl", version="0.1.0") self._set_middlewares(_app) self._set_routes(_app) self._set_events(_app) - self._init_services() self._app = _app @@ -58,10 +58,6 @@ class KarlApplication: def _set_events(self, app: FastAPI): pass - def _init_services(self): - logger = logging.getLogger(__name__) - load_injection_container() - def run(): return KarlApplication() diff --git a/app/model/webhook.py b/app/model/webhook.py index da9ad8d..e8c3b0d 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,9 +1,11 @@ from dataclasses import dataclass from typing import List +from app.events import Event + @dataclass -class WebhookEvent: +class WebhookEvent(Event): _id: str commit: str message: str diff --git a/app/services/passwords.py b/app/services/passwords.py index 18d8519..6ff8406 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -11,7 +11,7 @@ class Passwords: settings = get_settings() with open(settings.kp.secret, "r") as fh: - secret = fh.read() + secret = fh.read().splitlines()[0] self._kp_org = self.__get_or_create_store(settings.kp.file, secret) self._kp = self.__get_lock(settings.kp.file, secret) From 29dfc13a48a4910780de9a10e0f5990a9e49ec54 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 31 Oct 2025 23:30:35 +0100 Subject: [PATCH 09/87] Woodpecker event registered --- app/core/woodpecker.py | 21 ++++++++++++--------- app/events/__init__.py | 6 +++--- config/config.yaml | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 8236bf5..9b6ffb4 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,22 +1,25 @@ import logging +from typing import Annotated -from injectable import injectable +from injectable import injectable, Autowired, autowired, inject, injectable_factory from app.events import SimpleEventBus from app.model.webhook import WebhookEvent logger = logging.getLogger(__name__) -@injectable -class Woodpecker: - @SimpleEventBus.on(WebhookEvent) - def on_event(self, event): # TODO: caller nie działa -> brakuje instancji klasy? +@injectable(singleton=True) +class Woodpecker: + @autowired + def __init__(self, event_bus: Annotated[SimpleEventBus, Autowired]): + logger.info("Woodpecker initialized.") + event_bus.subscribe(WebhookEvent, self.on_ci_event) + + def on_ci_event(self, event): logger.info(f"Received event: {event}") pass -@SimpleEventBus.on(WebhookEvent) -def on_event2(event): # TODO: Tu działa - logger.info(f"F2: Received event: {event}") - pass +instance = Woodpecker(inject(SimpleEventBus)) +injectable_factory(Woodpecker)(lambda: instance) diff --git a/app/events/__init__.py b/app/events/__init__.py index fbd92d2..5d54a87 100644 --- a/app/events/__init__.py +++ b/app/events/__init__.py @@ -18,9 +18,9 @@ class SimpleEventBus: self._executor = ThreadPoolExecutor() def publish(self, event: Event) -> None: - for handler in self._handlers.get(type(event), []): - # Fire-and-forget execution - self._executor.submit(handler, event) + for handler in self._handlers.get(type(event), []): + # Fire-and-forget execution + self._executor.submit(handler, event) def subscribe(self, event_type: type, handler: Callable) -> None: if event_type not in self._handlers: diff --git a/config/config.yaml b/config/config.yaml index 6c2043e..c4fc44a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,7 +4,7 @@ logging: app: host: "127.0.0.1" port: 8081 - reload: true + reload: false git: path: "F:/IdeaProjects/paas/karl/.compose_repository" branch: "main" From 34ee5f87549a7838404e1c8218d344032d6367ca Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sat, 1 Nov 2025 23:02:28 +0100 Subject: [PATCH 10/87] CI events flow simplified --- app/api/v1.py | 16 ++++++------ app/core/core.py | 51 ------------------------------------- app/core/injects.py | 11 +++----- app/core/queue.py | 52 -------------------------------------- app/core/woodpecker.py | 57 +++++++++++++++++++++++++++++++++++------- app/events/__init__.py | 41 ------------------------------ app/model/webhook.py | 5 +--- 7 files changed, 59 insertions(+), 174 deletions(-) delete mode 100644 app/core/core.py delete mode 100644 app/core/queue.py delete mode 100644 app/events/__init__.py diff --git a/app/api/v1.py b/app/api/v1.py index 4fbe31f..a012834 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -4,10 +4,9 @@ from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response from app.api.models import Request -from app.core.core import WebhookProcessor from app.core.injects import AutowireSupport -from app.events import SimpleEventBus -from app.model.webhook import WebhookEvent +from app.core.woodpecker import Woodpecker +from app.model.webhook import WoodpeckerEvent router = APIRouter() @@ -19,22 +18,21 @@ async def root(): @cbv(router) class APIv1: - webhook_service: WebhookProcessor = Depends(AutowireSupport.webhook_processor) - event_bus: SimpleEventBus = Depends(AutowireSupport.event_bus) + woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) logger = __import__('logging').getLogger(__name__) def __init__(self): - try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie - mapper.add(Request, WebhookEvent) + try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie + mapper.add(Request, WoodpeckerEvent) except exceptions.DuplicatedRegistrationError: pass @router.get("/health", summary="Health check") async def health(self) -> JSONResponse: # TODO: JSON serialize - return JSONResponse({"status": self.webhook_service.health.healthy}) + return JSONResponse({"status": "unknown"}) @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.event_bus.publish(mapper.map(request)) + self.woodpecker.on_ci_event(mapper.map(request)) return Response(status_code=201) diff --git a/app/core/core.py b/app/core/core.py deleted file mode 100644 index 3b16ea1..0000000 --- a/app/core/core.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging -import uuid -from typing import Annotated, List - -from injectable import injectable, autowired, Autowired -from typing_extensions import deprecated - -from app.core.queue import EnqueuedProcessor, ProcessQueue, Task, Result -from app.model.healthcheck import HealthCheck -from app.model.webhook import WebhookEvent -from app.services import DockerService, GitService, Passwords - -logger = logging.getLogger(__name__) - -@deprecated("Use event bus instead.") -@injectable(singleton=True) -class WebhookProcessor: - @autowired - def __init__(self, docker: Annotated[DockerService, Autowired], - git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired]): - self._docker = docker - self._git = git - self._keepass = keepass - - def enqueue(self, event: WebhookEvent): - pass - - def _process(self, task: Task[WebhookEvent]) -> Result: - event: WebhookEvent = task.payload - # TODO: persist event data - commit_hash = self._git.get_new_commit_hash() - if commit_hash != event.commit: - logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") - return Result(task.id, False, "Commit hash mismatch") - # TODO: persist commit data - service = self._get_service(event.files) - - - - return Result(task.id, True) - - def _get_service(self, files: List[str]) -> str: - pass - - @property - def health(self) -> HealthCheck: - return HealthCheck( - self._docker is not None and self._git is not None and self._keepass is not None, - f"Docker: {self._docker is not None}, Git: {self._git is not None}, KeePass: {self._keepass is not None}" - ) diff --git a/app/core/injects.py b/app/core/injects.py index 0d7ff9c..115ae95 100644 --- a/app/core/injects.py +++ b/app/core/injects.py @@ -1,15 +1,10 @@ from injectable import inject -from app.core.core import WebhookProcessor -from app.events import SimpleEventBus +from app.core.woodpecker import Woodpecker class AutowireSupport: @staticmethod - def webhook_processor(): - return inject(WebhookProcessor) - - @staticmethod - def event_bus(): - return inject(SimpleEventBus) + def woodpecker(): + return inject(Woodpecker) diff --git a/app/core/queue.py b/app/core/queue.py deleted file mode 100644 index 3e92cd6..0000000 --- a/app/core/queue.py +++ /dev/null @@ -1,52 +0,0 @@ -import time -import uuid -from abc import ABC, abstractmethod -from dataclasses import dataclass -from multiprocessing import Queue, Process -from typing import TypeVar - -from injectable import injectable - -T = TypeVar('T') - -@dataclass -class Task[T]: - id: uuid.UUID - processor: 'EnqueuedProcessor' - payload: T - -@dataclass -class Result: - _id: uuid.UUID - success: bool - error: str | None = None - -@injectable(singleton=True) -class ProcessQueue: - def __init__(self): - self._q = Queue() - self._process_thread = Process(target=self._run, args=(self._q,)) - self._process_thread.start() - - def put(self, task: Task): - self._q.put(task) - - def _run(self, queue: Queue): - while True: - if queue.empty(): - time.sleep(10) - continue - task = queue.get() - task.processor._process(task.payload) - - -class EnqueuedProcessor(ABC): - def __init__(self, queue: ProcessQueue): - self._queue = queue - - def _enqueue(self, task: Task): - self._queue.put(task) - - @abstractmethod - def _process(self, task: Task) -> Result: - pass diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 9b6ffb4..9ba242d 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,25 +1,64 @@ import logging +from collections import deque +from multiprocessing import Process, Lock from typing import Annotated -from injectable import injectable, Autowired, autowired, inject, injectable_factory +from injectable import injectable, Autowired, autowired -from app.events import SimpleEventBus -from app.model.webhook import WebhookEvent +from app.model.webhook import WoodpeckerEvent +from app.services import Passwords, GitService, DockerService logger = logging.getLogger(__name__) +class WoodpeckerRunner(Process): + def __init__(self, event: WoodpeckerEvent): + super().__init__() + self._event = event + + def run(self): + super().run() + """ + event: WebhookEvent = task.payload + # TODO: persist event data + commit_hash = self._git.get_new_commit_hash() + if commit_hash != event.commit: + logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") + return Result(task.id, False, "Commit hash mismatch") + # TODO: persist commit data + service = self._get_service(event.files) + + """ + + @injectable(singleton=True) class Woodpecker: @autowired - def __init__(self, event_bus: Annotated[SimpleEventBus, Autowired]): + def __init__(self, passwords: Annotated[Passwords, Autowired]): + self._passwords = passwords + self._git = GitService() + self._docker = DockerService() + self._runner: WoodpeckerRunner | None = None + self._pending = deque() + self._lock = Lock() logger.info("Woodpecker initialized.") - event_bus.subscribe(WebhookEvent, self.on_ci_event) - def on_ci_event(self, event): + def on_ci_event(self, event: WoodpeckerEvent): logger.info(f"Received event: {event}") + with self._lock: + if len(self._pending) > 0 or self._runner is not None: + self._pending.append(event) + return + self._start_runner(event) + + def _start_runner(self, event: WoodpeckerEvent): pass - -instance = Woodpecker(inject(SimpleEventBus)) -injectable_factory(Woodpecker)(lambda: instance) + def _on_runner_completed(self): + logger.info("Runner completed.") + self._runner.join() + with self._lock: + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) diff --git a/app/events/__init__.py b/app/events/__init__.py deleted file mode 100644 index 5d54a87..0000000 --- a/app/events/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from functools import wraps -from typing import Dict, List, Callable - -from injectable import injectable, inject - - -@dataclass -class Event: - pass - - -@injectable(singleton=True) -class SimpleEventBus: - def __init__(self): - self._handlers: Dict[type, List[Callable]] = {} - self._executor = ThreadPoolExecutor() - - def publish(self, event: Event) -> None: - for handler in self._handlers.get(type(event), []): - # Fire-and-forget execution - self._executor.submit(handler, event) - - def subscribe(self, event_type: type, handler: Callable) -> None: - if event_type not in self._handlers: - self._handlers[event_type] = [] - self._handlers[event_type].append(handler) - - @staticmethod - def on(event: type) -> Callable: - def outer(func): - inject(SimpleEventBus).subscribe(event, func) - - @wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - - return wrapper - - return outer diff --git a/app/model/webhook.py b/app/model/webhook.py index e8c3b0d..364e17b 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,11 +1,8 @@ from dataclasses import dataclass from typing import List -from app.events import Event - - @dataclass -class WebhookEvent(Event): +class WoodpeckerEvent: _id: str commit: str message: str From 312631f0b57fe7bacc9fc2dbcea91a832180fe16 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 00:35:33 +0100 Subject: [PATCH 11/87] WoodpeckerRunner & some VCS changes --- app/core/woodpecker.py | 66 +++++++++++++++++++++++++++++++++--------- app/services/vcs.py | 23 ++------------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index 9ba242d..ae8553b 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -12,23 +12,50 @@ logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, event: WoodpeckerEvent): - super().__init__() + def __init__(self, git: GitService, docker: DockerService, success_callback=None, error_callback=None): + super().__init__(daemon=True) + self._git = git + self._docker = docker + self._success_callback = success_callback + self._error_callback = error_callback + self._event: WoodpeckerEvent | None = None + + def process_event(self, event: WoodpeckerEvent): self._event = event + self.start() def run(self): - super().run() - """ - event: WebhookEvent = task.payload - # TODO: persist event data - commit_hash = self._git.get_new_commit_hash() - if commit_hash != event.commit: - logger.warning(f"Commit hash mismatch: {commit_hash} != {event.commit}") - return Result(task.id, False, "Commit hash mismatch") - # TODO: persist commit data - service = self._get_service(event.files) + try: + service = self.get_service(self._event.files) + if service is None: + logger.info("No service found.") + return self._success_callback() - """ + self._git.checkout(self._event.commit) + """ + TODO: + check for *.mo.* files + subs mo from pass + docker compose up -d -f service/docker-compose.yml + """ + + return self._success_callback() + except Exception as e: + return self._error_callback(e) + + def get_service(self, files: list[str]) -> str | None: + supported_files = [] + for f in files: + f_parts = f.split("/") + if f_parts[0] in ["compose", "files"]: + supported_files.append(f[1]) + match len(set(supported_files)): + case 0: + return None + case 1: + return supported_files[0] + case _: + raise Exception("Multiple services are not supported.") @injectable(singleton=True) @@ -52,7 +79,9 @@ class Woodpecker: self._start_runner(event) def _start_runner(self, event: WoodpeckerEvent): - pass + with self._lock: + self._runner = WoodpeckerRunner(self._git, self._docker, self._on_runner_completed) + self._runner.process_event(event) def _on_runner_completed(self): logger.info("Runner completed.") @@ -62,3 +91,12 @@ class Woodpecker: if len(self._pending) > 0: event = self._pending.popleft() self._start_runner(event) + + def _on_runner_error(self, t: Exception): + logger.error(f"Runner error: {t}", exc_info=True) + self._runner.join() + with self._lock: + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) diff --git a/app/services/vcs.py b/app/services/vcs.py index abb5468..00ef3c2 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -13,14 +13,6 @@ class GitService: self._repo.git.checkout(self._settings.git.branch) self._origin: Remote = self._repo.remotes.origin - def get_modified_compose(self) -> str | None: - self._update() - return self._diff() - - def get_new_commit_hash(self) -> str: - self._update() - return self._repo.head.commit.hexsha - @staticmethod def _check_preconditions(config: GitConfig) -> Repo: def clone(): @@ -34,16 +26,5 @@ class GitService: return clone() return Repo(config.path) - def _update(self): - self._origin.pull() - - def _diff(self) -> str | None: - diff = self._repo.head.commit.diff("HEAD~1") - composes = [f for f in diff if f.a_path.endswith("docker-compose.yml")] - match len(composes): - case 0: - return None - case 1: - return composes[0].a_path - case _: - raise Exception("Multiple compose files modified") + def checkout(self, sha: str): + pass From 1341b022d42b1bd4377799b53a02f7d9edf026c7 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:04:21 +0100 Subject: [PATCH 12/87] Woodpecker main loop almost finished --- app/core/woodpecker.py | 18 ++++++++++-------- app/services/containers.py | 4 ++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index ae8553b..caa29d5 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -1,10 +1,12 @@ import logging from collections import deque from multiprocessing import Process, Lock +from pathlib import Path from typing import Annotated from injectable import injectable, Autowired, autowired +from app.config import get_settings from app.model.webhook import WoodpeckerEvent from app.services import Passwords, GitService, DockerService @@ -12,10 +14,12 @@ logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, git: GitService, docker: DockerService, success_callback=None, error_callback=None): + def __init__(self, git: GitService, docker: DockerService, passwords: Passwords, + success_callback=None, error_callback=None): super().__init__(daemon=True) self._git = git self._docker = docker + self._passwords = passwords self._success_callback = success_callback self._error_callback = error_callback self._event: WoodpeckerEvent | None = None @@ -30,14 +34,12 @@ class WoodpeckerRunner(Process): if service is None: logger.info("No service found.") return self._success_callback() - + service_path = f"{get_settings().git.path}/compose/{service}/docker-compose.yml" self._git.checkout(self._event.commit) - """ - TODO: - check for *.mo.* files - subs mo from pass - docker compose up -d -f service/docker-compose.yml - """ + for file in self._event.files: + if file.__contains__('.mo.'): + pass + self._docker.reload(Path(service_path)) return self._success_callback() except Exception as e: diff --git a/app/services/containers.py b/app/services/containers.py index a89343b..20f0098 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path import docker from docker.models.containers import Container @@ -33,3 +34,6 @@ class DockerService: @property def tree(self) -> Tree: return self._tree + + def reload(self, compose_path: Path): + pass From 8ee950940e027db2e4b0202f5815e3df724d3589 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:15:12 +0100 Subject: [PATCH 13/87] Mo basics --- app/core/woodpecker.py | 21 ++++++++++++--------- app/services/mo.py | 16 ++++++++++++++++ app/services/passwords.py | 33 ++++++++++++++++----------------- 3 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 app/services/mo.py diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py index caa29d5..1892c0b 100644 --- a/app/core/woodpecker.py +++ b/app/core/woodpecker.py @@ -8,21 +8,23 @@ from injectable import injectable, Autowired, autowired from app.config import get_settings from app.model.webhook import WoodpeckerEvent -from app.services import Passwords, GitService, DockerService +from app.services import GitService, DockerService +from app.services.mo import Mo logger = logging.getLogger(__name__) class WoodpeckerRunner(Process): - def __init__(self, git: GitService, docker: DockerService, passwords: Passwords, + def __init__(self, git: GitService, docker: DockerService, mo: Mo, success_callback=None, error_callback=None): super().__init__(daemon=True) self._git = git self._docker = docker - self._passwords = passwords + self._mo = mo self._success_callback = success_callback self._error_callback = error_callback self._event: WoodpeckerEvent | None = None + self._root = get_settings().git.path def process_event(self, event: WoodpeckerEvent): self._event = event @@ -34,12 +36,12 @@ class WoodpeckerRunner(Process): if service is None: logger.info("No service found.") return self._success_callback() - service_path = f"{get_settings().git.path}/compose/{service}/docker-compose.yml" + service_path = f"{self._root}/compose/{service}/docker-compose.yml" self._git.checkout(self._event.commit) for file in self._event.files: if file.__contains__('.mo.'): - pass - self._docker.reload(Path(service_path)) + self._mo.process(Path(f"{self._root}{file}").absolute()) + self._docker.reload(Path(service_path).absolute()) return self._success_callback() except Exception as e: @@ -63,8 +65,8 @@ class WoodpeckerRunner(Process): @injectable(singleton=True) class Woodpecker: @autowired - def __init__(self, passwords: Annotated[Passwords, Autowired]): - self._passwords = passwords + def __init__(self, mo: Annotated[Mo, Autowired]): + self._mo = mo self._git = GitService() self._docker = DockerService() self._runner: WoodpeckerRunner | None = None @@ -82,7 +84,8 @@ class Woodpecker: def _start_runner(self, event: WoodpeckerEvent): with self._lock: - self._runner = WoodpeckerRunner(self._git, self._docker, self._on_runner_completed) + self._runner = WoodpeckerRunner(self._git, self._docker, self._mo, + self._on_runner_completed, self._on_runner_error) self._runner.process_event(event) def _on_runner_completed(self): diff --git a/app/services/mo.py b/app/services/mo.py new file mode 100644 index 0000000..3b1520d --- /dev/null +++ b/app/services/mo.py @@ -0,0 +1,16 @@ +from pathlib import Path +from typing import Annotated + +from injectable import injectable, autowired, Autowired + +from app.services import Passwords + + +@injectable +class Mo: + @autowired + def __init__(self, passwords: Annotated[Passwords, Autowired]): + self._passwords = passwords + + def process(self, mo_file: Path): + pass diff --git a/app/services/passwords.py b/app/services/passwords.py index 6ff8406..e3a181c 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -1,7 +1,8 @@ import os.path +import shutil from injectable import injectable -from pykeepass import PyKeePass, create_database, Group +from pykeepass import PyKeePass, create_database @injectable(singleton=True) @@ -12,29 +13,27 @@ class Passwords: with open(settings.kp.secret, "r") as fh: secret = fh.read().splitlines()[0] - - self._kp_org = self.__get_or_create_store(settings.kp.file, secret) - self._kp = self.__get_lock(settings.kp.file, secret) + self._path = settings.kp.file + self._kp_org = self._open_or_create(self._path, secret) + self._kp = self._open_lock(self._path, secret) @staticmethod - def __get_or_create_store(path, passwd) -> PyKeePass: + def _open_or_create(path, password) -> PyKeePass: if os.path.exists(path): - return PyKeePass( - path, - password=passwd, - ) - return create_database(path, passwd) + return PyKeePass(path, password=password) + return create_database(path, password) @staticmethod - def __get_lock(path, passwd) -> PyKeePass: + def _open_lock(path, password) -> PyKeePass: lock_path = path + ".lock" - import shutil shutil.copyfile(path, lock_path) - return Passwords.__get_or_create_store(lock_path, passwd) + return Passwords._open_or_create(lock_path, password) @property - def store(self): - return self._kp.root_group + def kp(self) -> PyKeePass: + return self._kp - def save(self, group: Group): - pass + def save(self): + # nadpisz plik źródłowy zmianami z lock + self._kp.save() + shutil.copyfile(self._path + ".lock", self._path) From 3c3be6d79c2dde279d940b6d72978de3745004d5 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 22:59:40 +0100 Subject: [PATCH 14/87] chevron removed --- app/services/mo.py | 10 +++++++++- app/services/passwords.py | 5 ++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/services/mo.py b/app/services/mo.py index 3b1520d..a1ae32c 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -1,4 +1,5 @@ from pathlib import Path +from string import Template from typing import Annotated from injectable import injectable, autowired, Autowired @@ -13,4 +14,11 @@ class Mo: self._passwords = passwords def process(self, mo_file: Path): - pass + raw = '' + with open(mo_file, "r") as mo: + raw = mo.read() + tpl = Template(raw) + rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) + de_mo_ified = str(mo_file).replace(".mo", "") + with open(de_mo_ified, "w") as mo: + mo.write(rendered) diff --git a/app/services/passwords.py b/app/services/passwords.py index e3a181c..1c656c6 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -29,9 +29,8 @@ class Passwords: shutil.copyfile(path, lock_path) return Passwords._open_or_create(lock_path, password) - @property - def kp(self) -> PyKeePass: - return self._kp + def get_values(self, keys: list[str]) -> dict[str, str]: + return {} def save(self): # nadpisz plik źródłowy zmianami z lock From e14180cbe7e5641aaff3762db8d8e958964aedc2 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:16:54 +0100 Subject: [PATCH 15/87] passwords#get_values --- app/services/passwords.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/services/passwords.py b/app/services/passwords.py index 1c656c6..7f30567 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -30,7 +30,15 @@ class Passwords: return Passwords._open_or_create(lock_path, password) def get_values(self, keys: list[str]) -> dict[str, str]: - return {} + output = {} + for k in keys: + key_parts = k.split(".") + path = key_parts[:-2] if len(key_parts) > 2 else None + entry_name = key_parts[-2] + field_name = key_parts[-1] + kp_entry = self._kp_org.find_entries(path=path, first=True, name=entry_name)[0] + output[k] = kp_entry[field_name] + return output def save(self): # nadpisz plik źródłowy zmianami z lock From 9166790de90d90f1766ca6a17050365d0523ef55 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:50:24 +0100 Subject: [PATCH 16/87] mo processing test --- app/services/mo.py | 6 +++++- app/services/passwords.py | 4 ++-- tests/__init__.py | 0 tests/files/test1/test.mo.yaml | 3 +++ tests/test_mo.py | 21 +++++++++++++++++++++ 5 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/files/test1/test.mo.yaml create mode 100644 tests/test_mo.py diff --git a/app/services/mo.py b/app/services/mo.py index a1ae32c..d917e62 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -6,6 +6,10 @@ from injectable import injectable, autowired, Autowired from app.services import Passwords +class DotTemplate(Template): + # Pozwala na kropki w nazwach placeholderów, np. ${user.name.first} + idpattern = r'[_a-zA-Z][_a-zA-Z0-9.]*' + @injectable class Mo: @@ -17,7 +21,7 @@ class Mo: raw = '' with open(mo_file, "r") as mo: raw = mo.read() - tpl = Template(raw) + tpl = DotTemplate(raw) rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) de_mo_ified = str(mo_file).replace(".mo", "") with open(de_mo_ified, "w") as mo: diff --git a/app/services/passwords.py b/app/services/passwords.py index 7f30567..9b11ca4 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -36,8 +36,8 @@ class Passwords: path = key_parts[:-2] if len(key_parts) > 2 else None entry_name = key_parts[-2] field_name = key_parts[-1] - kp_entry = self._kp_org.find_entries(path=path, first=True, name=entry_name)[0] - output[k] = kp_entry[field_name] + kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name)[0] + output[k] = kp_entry[field_name] # TODO: TypeError: 'Entry' object is not subscriptable return output def save(self): diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/files/test1/test.mo.yaml b/tests/files/test1/test.mo.yaml new file mode 100644 index 0000000..af55ecc --- /dev/null +++ b/tests/files/test1/test.mo.yaml @@ -0,0 +1,3 @@ +value: ${sample.password} +nested: ${some.nested.value.password} +custom: ${custom.field} diff --git a/tests/test_mo.py b/tests/test_mo.py new file mode 100644 index 0000000..731dada --- /dev/null +++ b/tests/test_mo.py @@ -0,0 +1,21 @@ +import os +from pathlib import Path +from unittest import TestCase + +from app.services import Passwords +from app.services.mo import Mo + + +class TestMo(TestCase): + def test_process(self): + mo = Mo(Passwords()) + mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) + self.assertTrue(os.path.exists('tests/files/test1/test.mo')) + with open('tests/files/test1/test.mo', 'r') as f: + content = f.read() + self.assertFalse(content.__contains__('${')) + """ + value: some_oass + nested: nested_pass + custom: custom_content + """ From e3a37419e8c59dd4049306f354ee523033dea380 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Sun, 2 Nov 2025 23:50:24 +0100 Subject: [PATCH 17/87] mo processing test --- tests/test_mo.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_mo.py b/tests/test_mo.py index 731dada..200b3d8 100644 --- a/tests/test_mo.py +++ b/tests/test_mo.py @@ -2,6 +2,8 @@ import os from pathlib import Path from unittest import TestCase +import yaml + from app.services import Passwords from app.services.mo import Mo @@ -11,9 +13,13 @@ class TestMo(TestCase): mo = Mo(Passwords()) mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) self.assertTrue(os.path.exists('tests/files/test1/test.mo')) - with open('tests/files/test1/test.mo', 'r') as f: + with open('tests/files/test1/test.yaml', 'r') as f: content = f.read() self.assertFalse(content.__contains__('${')) + parsed = yaml.load(content, Loader=yaml.FullLoader) + self.assertEqual(parsed['value'], 'some_oass') + self.assertEqual(parsed['nested'], 'nested_pass') + self.assertEqual(parsed['custom'], 'custom_content') """ value: some_oass nested: nested_pass From 3dc27cc86838d5683a3356e5b5886308ae3a1379 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 21:56:59 +0100 Subject: [PATCH 18/87] Simple/Complex templates / parsers --- app/services/mo.py | 15 ++++++++++++--- app/services/passwords.py | 20 +++++++++++++++++--- tests/files/test1/test.mo.yaml | 6 +++--- tests/test_mo.py | 16 ++++++---------- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/app/services/mo.py b/app/services/mo.py index d917e62..9a35642 100644 --- a/app/services/mo.py +++ b/app/services/mo.py @@ -6,11 +6,16 @@ from injectable import injectable, autowired, Autowired from app.services import Passwords -class DotTemplate(Template): + +class SimpleValueTemplate(Template): # Pozwala na kropki w nazwach placeholderów, np. ${user.name.first} idpattern = r'[_a-zA-Z][_a-zA-Z0-9.]*' +class ComplexValueTemplate(SimpleValueTemplate): + delimiter = '@' + + @injectable class Mo: @autowired @@ -21,8 +26,12 @@ class Mo: raw = '' with open(mo_file, "r") as mo: raw = mo.read() - tpl = DotTemplate(raw) - rendered = tpl.substitute(self._passwords.get_values(tpl.get_identifiers())) + cmp = ComplexValueTemplate(raw) + rendered = cmp.substitute(self._passwords.get_values(cmp.get_identifiers())) + smp = SimpleValueTemplate(rendered) + ids = [_id + '.password' for _id in smp.get_identifiers()] + mappings = {k.replace('.password', ''): v for k, v in self._passwords.get_values(ids).items()} + rendered = smp.substitute(mappings) de_mo_ified = str(mo_file).replace(".mo", "") with open(de_mo_ified, "w") as mo: mo.write(rendered) diff --git a/app/services/passwords.py b/app/services/passwords.py index 9b11ca4..408ec5e 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -33,13 +33,27 @@ class Passwords: output = {} for k in keys: key_parts = k.split(".") - path = key_parts[:-2] if len(key_parts) > 2 else None + path = key_parts[:-1] if len(key_parts) > 2 else None entry_name = key_parts[-2] field_name = key_parts[-1] - kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name)[0] - output[k] = kp_entry[field_name] # TODO: TypeError: 'Entry' object is not subscriptable + kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name) + output[k] = self._get_field_value(kp_entry, field_name) return output + @staticmethod + def _get_field_value(kp_entry, field_name): + if kp_entry is None: + return None + match field_name: + case "username": + return kp_entry.username + case "password": + return kp_entry.password + case "url": + return kp_entry.url + case _: + return kp_entry.get_custom_property(field_name) + def save(self): # nadpisz plik źródłowy zmianami z lock self._kp.save() diff --git a/tests/files/test1/test.mo.yaml b/tests/files/test1/test.mo.yaml index af55ecc..572b953 100644 --- a/tests/files/test1/test.mo.yaml +++ b/tests/files/test1/test.mo.yaml @@ -1,3 +1,3 @@ -value: ${sample.password} -nested: ${some.nested.value.password} -custom: ${custom.field} +value: ${sample} +nested: ${some.nested.value} +custom: @{custom.field} diff --git a/tests/test_mo.py b/tests/test_mo.py index 200b3d8..f685e34 100644 --- a/tests/test_mo.py +++ b/tests/test_mo.py @@ -10,18 +10,14 @@ from app.services.mo import Mo class TestMo(TestCase): def test_process(self): + target_path = Path('tests/files/test1/test.yaml') mo = Mo(Passwords()) mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) - self.assertTrue(os.path.exists('tests/files/test1/test.mo')) - with open('tests/files/test1/test.yaml', 'r') as f: + self.assertTrue(os.path.exists(target_path)) + with open(target_path, 'r') as f: content = f.read() self.assertFalse(content.__contains__('${')) parsed = yaml.load(content, Loader=yaml.FullLoader) - self.assertEqual(parsed['value'], 'some_oass') - self.assertEqual(parsed['nested'], 'nested_pass') - self.assertEqual(parsed['custom'], 'custom_content') - """ - value: some_oass - nested: nested_pass - custom: custom_content - """ + self.assertEqual('some_pass', parsed['value']) + self.assertEqual('nested_pass', parsed['nested']) + self.assertEqual('custom_content', parsed['custom']) From 8ca668f07e7bc4463102ce2b2b1afffff3f21838 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:20:24 +0100 Subject: [PATCH 19/87] git checkout sha --- app/services/vcs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/vcs.py b/app/services/vcs.py index 00ef3c2..0e97ec8 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -27,4 +27,5 @@ class GitService: return Repo(config.path) def checkout(self, sha: str): - pass + self._origin.fetch() + self._repo.git.checkout(sha) From 4c8716cc50a8f1edf5005ac74277329368fc7a15 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:31:00 +0100 Subject: [PATCH 20/87] Fixed cbv --- app/services/containers.py | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/services/containers.py b/app/services/containers.py index 20f0098..6e927bf 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -36,4 +36,6 @@ class DockerService: return self._tree def reload(self, compose_path: Path): - pass + cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"] + # TODO: subprocess + diff --git a/pyproject.toml b/pyproject.toml index 9b97797..1791992 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "docker>=7.1.0", "injectable==4.0.1", "py-automapper>=2.2.0", + "fastapi-utils>=0.8.0", ] [project.optional-dependencies] From 324938133f5430939f6deceef37008d9a07606f0 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Mon, 3 Nov 2025 22:42:37 +0100 Subject: [PATCH 21/87] Global formatting fix --- app/api/models.py | 1 + app/model/passwords.py | 5 ++++- app/model/webhook.py | 1 + app/services/containers.py | 1 - app/templates/index.html | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index b27765b..8e8e4d2 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -10,6 +10,7 @@ class Request: started: str files: List[str] + @dataclass class Response: status: int diff --git a/app/model/passwords.py b/app/model/passwords.py index d795d2c..253dbd9 100644 --- a/app/model/passwords.py +++ b/app/model/passwords.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from typing import Type + # TODO: unnecessary? @dataclass @@ -8,6 +9,7 @@ class PathItem: name: str t: Type + @dataclass class Path: path: list[PathItem] = field(default_factory=list) @@ -43,7 +45,8 @@ class Password: def path(self): return self.group.path.append(self.name, type(self)) + class UnencryptedPassword(Password): - def __init__(self, name: str, value: str, group: Group): + def __init__(self, name: str, value: str, group: Group): super().__init__(name, group) self.value = value diff --git a/app/model/webhook.py b/app/model/webhook.py index 364e17b..a2ef7dc 100644 --- a/app/model/webhook.py +++ b/app/model/webhook.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import List + @dataclass class WoodpeckerEvent: _id: str diff --git a/app/services/containers.py b/app/services/containers.py index 6e927bf..f754ddc 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -38,4 +38,3 @@ class DockerService: def reload(self, compose_path: Path): cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"] # TODO: subprocess - diff --git a/app/templates/index.html b/app/templates/index.html index ebbe8ca..ac9bbbe 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -3,7 +3,7 @@ {{ title }} - +