From 7808a663427f04653aaf47e030bb6b7eb87d6217 Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Thu, 11 Dec 2025 22:55:17 +0100 Subject: [PATCH 01/32] feat: new event bus --- pyproject.toml | 1 + src/karl/api/v1.py | 9 ++++++--- src/karl/core/events.py | 11 +++++++++++ src/karl/core/injects.py | 5 +++++ src/karl/core/woodpecker.py | 8 ++++++-- src/karl/model/webhook.py | 6 +++--- 6 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 src/karl/core/events.py diff --git a/pyproject.toml b/pyproject.toml index fa843df..4738496 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "fastapi-utils>=0.8.0", "keyring>=25.6.0", "keyring-backend>=0.1.0", + "bubus>=1.5.6", ] [dependency-groups] diff --git a/src/karl/api/v1.py b/src/karl/api/v1.py index 9bbbfbc..3bc8090 100644 --- a/src/karl/api/v1.py +++ b/src/karl/api/v1.py @@ -1,4 +1,7 @@ +import logging + from automapper import mapper, exceptions +from bubus import EventBus from fastapi import APIRouter, Depends from fastapi_utils.cbv import cbv from starlette.responses import JSONResponse, Response @@ -9,7 +12,7 @@ from core.woodpecker import Woodpecker from model.webhook import WoodpeckerEvent router = APIRouter() - +logger = logging.getLogger(__name__) @router.get("/", summary="Main API") async def root(): @@ -19,7 +22,7 @@ async def root(): @cbv(router) class APIv1: woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) - logger = __import__('logging').getLogger(__name__) + bus: EventBus = Depends(AutowireSupport.bus) def __init__(self): try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie @@ -34,5 +37,5 @@ class APIv1: @router.post("/ci", summary="CI Webhook") async def ci(self, request: Request): - self.woodpecker.on_ci_event(mapper.map(request)) + await self.bus.dispatch(mapper.map(request)) return Response(status_code=201) diff --git a/src/karl/core/events.py b/src/karl/core/events.py new file mode 100644 index 0000000..456968b --- /dev/null +++ b/src/karl/core/events.py @@ -0,0 +1,11 @@ +import logging + +from bubus import EventBus +from injectable import injectable_factory + +logger = logging.getLogger(__name__) + +@injectable_factory(EventBus, singleton=True) +def event_bus_factory() -> EventBus: + logger.info("Creating event bus...") + return EventBus() diff --git a/src/karl/core/injects.py b/src/karl/core/injects.py index 623fa59..e7081a7 100644 --- a/src/karl/core/injects.py +++ b/src/karl/core/injects.py @@ -1,3 +1,4 @@ +from bubus import EventBus from injectable import inject from core.woodpecker import Woodpecker @@ -8,3 +9,7 @@ class AutowireSupport: @staticmethod def woodpecker(): return inject(Woodpecker) + + @staticmethod + def bus(): + return inject(EventBus) diff --git a/src/karl/core/woodpecker.py b/src/karl/core/woodpecker.py index 2e83c07..19ec9df 100644 --- a/src/karl/core/woodpecker.py +++ b/src/karl/core/woodpecker.py @@ -4,6 +4,7 @@ from pathlib import Path from threading import RLock, Thread from typing import Annotated +from bubus import EventBus from injectable import injectable, Autowired, autowired from config import get_settings @@ -65,17 +66,20 @@ class WoodpeckerRunner(Thread): @injectable(singleton=True) class Woodpecker: @autowired - def __init__(self, mo: Annotated[Mo, Autowired]): + def __init__(self, mo: Annotated[Mo, Autowired], + bus: Annotated[EventBus, Autowired]): self._mo = mo + self._bus = bus self._git = GitService() self._docker = DockerService() self._runner: WoodpeckerRunner | None = None self._pending = deque() self._lock = RLock() + bus.on(WoodpeckerEvent, self.on_ci_event) logger.info("Woodpecker initialized.") def on_ci_event(self, event: WoodpeckerEvent): - logger.info(f"Received event: {event}") + logger.info(f"Received event: {event.event_id}") with self._lock: logger.debug("Lock acquired [on-ci-event]") if len(self._pending) > 0 or self._runner is not None: diff --git a/src/karl/model/webhook.py b/src/karl/model/webhook.py index a2ef7dc..e97c073 100644 --- a/src/karl/model/webhook.py +++ b/src/karl/model/webhook.py @@ -1,9 +1,9 @@ -from dataclasses import dataclass from typing import List +from bubus import BaseEvent -@dataclass -class WoodpeckerEvent: + +class WoodpeckerEvent(BaseEvent): _id: str commit: str message: str From 71de3c76c6c8733026424950fb6fe0cee79d5e6e Mon Sep 17 00:00:00 2001 From: Piotr Dec Date: Fri, 12 Dec 2025 21:24:53 +0100 Subject: [PATCH 02/32] feat: WoodpeckerRunner events --- src/karl/core/woodpecker.py | 72 ++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/src/karl/core/woodpecker.py b/src/karl/core/woodpecker.py index 19ec9df..24af84c 100644 --- a/src/karl/core/woodpecker.py +++ b/src/karl/core/woodpecker.py @@ -1,10 +1,11 @@ +import asyncio import logging from collections import deque from pathlib import Path from threading import RLock, Thread from typing import Annotated -from bubus import EventBus +from bubus import EventBus, BaseEvent from injectable import injectable, Autowired, autowired from config import get_settings @@ -15,15 +16,20 @@ from services.mo import Mo logger = logging.getLogger(__name__) +class RunnerResult(BaseEvent): + success: bool = False + throwable: Exception | None = None + + class WoodpeckerRunner(Thread): + @autowired def __init__(self, git: GitService, docker: DockerService, mo: Mo, - success_callback=None, error_callback=None): + bus: Annotated[EventBus, Autowired]): super().__init__(daemon=True) self._git = git self._docker = docker self._mo = mo - self._success_callback = success_callback - self._error_callback = error_callback + self._bus = bus self._event: WoodpeckerEvent | None = None self._root = get_settings().git.path @@ -32,21 +38,27 @@ class WoodpeckerRunner(Thread): self.start() def run(self): + async def dispatch(r: RunnerResult): + await self._bus.dispatch(r) + + result = RunnerResult() try: service = self.get_service(self._event.files) if service is None: logger.info("No service found.") - return self._success_callback() - service_path = f"{self._root}/compose/{service}/docker-compose.yml" - self._git.checkout(self._event.commit) - for file in self._event.files: - if file.__contains__('.mo.'): - self._mo.process(Path(f"{self._root}{file}").absolute()) - self._docker.reload(Path(service_path).absolute()) - - return self._success_callback() + result.success = True + else: + service_path = f"{self._root}/compose/{service}/docker-compose.yml" + self._git.checkout(self._event.commit) + for file in self._event.files: + if file.__contains__('.mo.'): + self._mo.process(Path(f"{self._root}{file}").absolute()) + self._docker.reload(Path(service_path).absolute()) + result.success = True except Exception as e: - return self._error_callback(e) + result.throwable = e + + asyncio.run(dispatch(result)) def get_service(self, files: list[str]) -> str | None: supported_files = [] @@ -76,40 +88,34 @@ class Woodpecker: self._pending = deque() self._lock = RLock() bus.on(WoodpeckerEvent, self.on_ci_event) + bus.on(RunnerResult, self._on_runner_completed) logger.info("Woodpecker initialized.") - def on_ci_event(self, event: WoodpeckerEvent): - logger.info(f"Received event: {event.event_id}") + async def on_ci_event(self, event: WoodpeckerEvent): + logger.debug(f"Received WoodpeckerEvent: {event.event_id}") with self._lock: logger.debug("Lock acquired [on-ci-event]") if len(self._pending) > 0 or self._runner is not None: self._pending.append(event) - return - self._start_runner(event) + else: + self._start_runner(event) def _start_runner(self, event: WoodpeckerEvent): with self._lock: logger.debug("Lock acquired [start-runner]") - self._runner = WoodpeckerRunner(self._git, self._docker, self._mo, - self._on_runner_completed, self._on_runner_error) + self._runner = WoodpeckerRunner(self._git, self._docker, self._mo) self._runner.process_event(event) - def _on_runner_completed(self): - logger.info("Runner completed.") - self._runner.join() + def _on_runner_completed(self, result: RunnerResult): + logger.debug(f"Received RunnerResult: {result.event_id}") + logger.info(f"Runner completed {'successfully' if result.success else 'with error'}.") + if result.throwable is not None: + logger.error(f"Runner error: {result.throwable}", exc_info=True) + self._runner.join(timeout=1) + logger.debug("Runner joined.") with self._lock: logger.debug("Lock acquired [on-runner-completed]") self._runner = None if len(self._pending) > 0: event = self._pending.popleft() self._start_runner(event) - - def _on_runner_error(self, t: Exception): - logger.error(f"Runner error: {t}", exc_info=True) - self._runner.join() - with self._lock: - logger.debug("Lock acquired [on-runner-error]") - self._runner = None - if len(self._pending) > 0: - event = self._pending.popleft() - self._start_runner(event) From e2fd185971041117a594efd76215d88f9b700f49 Mon Sep 17 00:00:00 2001 From: stawros Date: Fri, 12 Dec 2025 21:40:03 +0100 Subject: [PATCH 03/32] fix 0.1.1 (#17) Reviewed-on: https://hattori.ztsh.eu/iac/karl/pulls/17 --- .dockerignore | 62 +++++++++++++ .editorconfig | 3 + .gitignore | 13 +++ .python-version | 1 + .woodpecker/dev.yaml | 14 +++ .woodpecker/latest.yaml | 14 +++ Dockerfile | 33 +++++++ app/__init__.py => README.md | 0 app/__main__.py | 4 - app/api/v1.py | 8 -- config/config.yaml | 13 +++ config/secret.txt | 1 + docker-compose.yaml | 22 +++++ pyproject.toml | 28 ++++-- run.sh | 2 +- src/karl/__init__.py | 19 ++++ {app => src/karl}/api/__init__.py | 0 src/karl/api/models.py | 16 ++++ src/karl/api/v1.py | 41 +++++++++ src/karl/config/__init__.py | 7 ++ src/karl/config/settings.py | 54 +++++++++++ src/karl/core/__init__.py | 0 src/karl/core/events.py | 11 +++ src/karl/core/injects.py | 15 +++ app/main.py => src/karl/core/router.py | 17 +--- src/karl/core/woodpecker.py | 121 +++++++++++++++++++++++++ src/karl/main.py | 63 +++++++++++++ src/karl/model/__init__.py | 0 src/karl/model/containers.py | 40 ++++++++ src/karl/model/healthcheck.py | 7 ++ src/karl/model/passwords.py | 52 +++++++++++ src/karl/model/webhook.py | 11 +++ src/karl/services/__init__.py | 5 + src/karl/services/containers.py | 59 ++++++++++++ src/karl/services/mo.py | 30 ++++++ src/karl/services/passwords.py | 75 +++++++++++++++ src/karl/services/system.py | 1 + src/karl/services/vcs.py | 31 +++++++ {app => src/karl}/templates/index.html | 2 +- src/karl/util/__init__.py | 0 src/karl/util/dicts.py | 11 +++ src/karl/util/logging.py | 93 +++++++++++++++++++ src/karl/web/__init__.py | 0 src/karl/web/middlewares.py | 24 +++++ tests/__init__.py | 0 tests/files/test1/test.mo.yaml | 5 + tests/test_mo.py | 50 ++++++++++ 47 files changed, 1043 insertions(+), 35 deletions(-) create mode 100644 .dockerignore create mode 100644 .python-version create mode 100644 .woodpecker/dev.yaml create mode 100644 .woodpecker/latest.yaml create mode 100644 Dockerfile rename app/__init__.py => README.md (100%) delete mode 100644 app/__main__.py delete mode 100644 app/api/v1.py create mode 100644 config/config.yaml create mode 100644 config/secret.txt create mode 100644 docker-compose.yaml create mode 100644 src/karl/__init__.py rename {app => src/karl}/api/__init__.py (100%) create mode 100644 src/karl/api/models.py create mode 100644 src/karl/api/v1.py create mode 100644 src/karl/config/__init__.py create mode 100644 src/karl/config/settings.py create mode 100644 src/karl/core/__init__.py create mode 100644 src/karl/core/events.py create mode 100644 src/karl/core/injects.py rename app/main.py => src/karl/core/router.py (56%) create mode 100644 src/karl/core/woodpecker.py create mode 100644 src/karl/main.py create mode 100644 src/karl/model/__init__.py create mode 100644 src/karl/model/containers.py create mode 100644 src/karl/model/healthcheck.py create mode 100644 src/karl/model/passwords.py create mode 100644 src/karl/model/webhook.py create mode 100644 src/karl/services/__init__.py create mode 100644 src/karl/services/containers.py create mode 100644 src/karl/services/mo.py create mode 100644 src/karl/services/passwords.py create mode 100644 src/karl/services/system.py create mode 100644 src/karl/services/vcs.py rename {app => src/karl}/templates/index.html (93%) create mode 100644 src/karl/util/__init__.py create mode 100644 src/karl/util/dicts.py create mode 100644 src/karl/util/logging.py create mode 100644 src/karl/web/__init__.py create mode 100644 src/karl/web/middlewares.py create mode 100644 tests/__init__.py create mode 100644 tests/files/test1/test.mo.yaml create mode 100644 tests/test_mo.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..daf4f93 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,62 @@ +# JB Toolkit +.idea + +# Git +.git +.gitignore +.gitattributes + +# CI +.woodpecker/ + +# Virtual env +.venv/ + +# Docker +docker-compose.yml +Dockerfile +.docker +.dockerignore + +# Byte-compiled / optimized / DLL files +**/__pycache__/ +**/*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +target/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Logs +**/*.log + +# Documentation +docs/_build/ + +# Python mode for VIM +.ropeproject +**/.ropeproject + +# Vim swap files +**/*.swp + +# Project specific +.compose_repository/ +config/ +tests/ diff --git a/.editorconfig b/.editorconfig index 46daf4a..0aca624 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,3 +8,6 @@ trim_trailing_whitespace = true [*.py] indent_size = 2 + +[*.yaml] +indent_size = 2 diff --git a/.gitignore b/.gitignore index 790a54f..52364cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,19 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# Virtual environments +.venv .idea *.iml uv.lock +**/*.kdbx* +.compose_repository __pycache__/ **/dist/ +**/*.log diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/.woodpecker/dev.yaml b/.woodpecker/dev.yaml new file mode 100644 index 0000000..2ff2fdc --- /dev/null +++ b/.woodpecker/dev.yaml @@ -0,0 +1,14 @@ +steps: + - name: build + image: woodpeckerci/plugin-docker-buildx:5.2.2 + settings: + platforms: linux/amd64 + repo: hattori.ztsh.eu/iac/karl + registry: hattori.ztsh.eu + tags: dev-${CI_PIPELINE_NUMBER} + username: stawros + password: + from_secret: hattori-packages + when: + - event: pull_request + evaluate: 'CI_COMMIT_PULL_REQUEST_LABELS contains "ci-ready"' diff --git a/.woodpecker/latest.yaml b/.woodpecker/latest.yaml new file mode 100644 index 0000000..123d5aa --- /dev/null +++ b/.woodpecker/latest.yaml @@ -0,0 +1,14 @@ +steps: + - name: build + image: woodpeckerci/plugin-docker-buildx:5.2.2 + settings: + platforms: linux/amd64 + repo: hattori.ztsh.eu/iac/karl + registry: hattori.ztsh.eu + tags: latest + username: stawros + password: + from_secret: hattori-packages + when: + - event: [ tag, push, manual ] + branch: master diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d10579b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM ghcr.io/astral-sh/uv:0.9-python3.12-alpine AS builder + +WORKDIR /app + +RUN apk update \ + && apk add gcc python3-dev musl-dev linux-headers + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + uv sync --no-install-workspace + +ADD . /app + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked + +FROM python:3.12-alpine3.22 + +RUN apk update --no-cache \ + && apk add --no-cache git + +COPY --from=builder --chown=app:app /app/.venv /app/.venv +COPY --from=builder --chown=app:app /app/src /app/src + +ENV PYTHONPATH="/app" + +EXPOSE 8081 + +WORKDIR /app + +ENTRYPOINT ["/app/.venv/bin/python"] + +CMD ["/app/src/karl/__init__.py"] diff --git a/app/__init__.py b/README.md similarity index 100% rename from app/__init__.py rename to README.md diff --git a/app/__main__.py b/app/__main__.py deleted file mode 100644 index 0b3284c..0000000 --- a/app/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -if __name__ == '__main__': - from main import run - - run() diff --git a/app/api/v1.py b/app/api/v1.py deleted file mode 100644 index 1cd9f50..0000000 --- a/app/api/v1.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi import APIRouter - -router = APIRouter() - - -@router.get("/", summary="Main API") -async def root(): - return {"message": "Witaj w API v12"} diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..c4fc44a --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,13 @@ +logging: + level: "TRACE" + path: "logs/karl.log" +app: + host: "127.0.0.1" + port: 8081 + reload: false +git: + path: "F:/IdeaProjects/paas/karl/.compose_repository" + branch: "main" +kp: + file: "config/kp.kdbx" + secret: "config/secret.txt" diff --git a/config/secret.txt b/config/secret.txt new file mode 100644 index 0000000..b5f9078 --- /dev/null +++ b/config/secret.txt @@ -0,0 +1 @@ +supersecret diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..ef1b364 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,22 @@ +services: + karl: + image: hattori.ztsh.eu/iac/karl:dev-36 + container_name: karl + restart: always + networks: + - web + volumes: + - /etc/localtime:/etc/localtime:ro + - /var/run/docker.sock:/var/run/docker.sock + - ./data/config:/app/config + - ./data/logs:/app/logs + secrets: + - kp_secret + +secrets: + kp_secret: + file: ./data/kp_secret + +networks: + web: + external: true diff --git a/pyproject.toml b/pyproject.toml index 5185ae8..4738496 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,28 +5,40 @@ description = "Because name 'Jenkins' was already taken. Greatest composer ever. readme = "README.md" requires-python = ">=3.12" authors = [{ name = "Piotr Dec" }] + dependencies = [ - "fastapi>=0.115.0", + "fastapi>=0.119.0", "uvicorn[standard]>=0.30.0", "jinja2>=3.1.4", + "pydantic-settings>=2.4.0", + "pyyaml>=6.0.2", + "gitpython>=3.1.45", + "pykeepass>=4.1.1.post1", + "docker>=7.1.0", + "injectable==4.0.1", + "py-automapper>=2.2.0", + "fastapi-utils>=0.8.0", + "keyring>=25.6.0", + "keyring-backend>=0.1.0", + "bubus>=1.5.6", ] -[project.optional-dependencies] +[dependency-groups] dev = [ "httpx>=0.27.0", - "pytest>=8.3.0", - "pytest-asyncio>=0.23.0", + "pytest==9.0.1", + "pytest-asyncio>=1.3.0", "ruff>=0.6.0", "mypy>=1.11.0", "types-Jinja2>=2.11.9", ] [project.scripts] -app = "app.main:run" +karl = "karl.main:run" -[tool.uv] -# uv automatycznie wykrywa dependencies z [project] -# Możesz dodać tu własne ustawienia cache/mirrors, jeśli potrzebne. +[build-system] +requires = ["uv_build>=0.8.23,<0.9.0"] +build-backend = "uv_build" [tool.ruff] line-length = 120 diff --git a/run.sh b/run.sh index c65ef75..cf3a77b 100644 --- a/run.sh +++ b/run.sh @@ -1 +1 @@ -uv run uvicorn app.main:app --reload +uvicorn app.main:run --factory --reload diff --git a/src/karl/__init__.py b/src/karl/__init__.py new file mode 100644 index 0000000..5dec085 --- /dev/null +++ b/src/karl/__init__.py @@ -0,0 +1,19 @@ +from config import get_settings + + +def main() -> None: + import uvicorn + + settings = get_settings() + uvicorn.run( + "karl.main:run", + factory=True, + host=settings.app.host, + port=settings.app.port, + reload=settings.app.reload, + log_config=None, + ) + + +if __name__ == "__main__": + main() diff --git a/app/api/__init__.py b/src/karl/api/__init__.py similarity index 100% rename from app/api/__init__.py rename to src/karl/api/__init__.py diff --git a/src/karl/api/models.py b/src/karl/api/models.py new file mode 100644 index 0000000..8e8e4d2 --- /dev/null +++ b/src/karl/api/models.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from typing import List + + +@dataclass +class Request: + _id: str + commit: str + message: str + started: str + files: List[str] + + +@dataclass +class Response: + status: int diff --git a/src/karl/api/v1.py b/src/karl/api/v1.py new file mode 100644 index 0000000..3bc8090 --- /dev/null +++ b/src/karl/api/v1.py @@ -0,0 +1,41 @@ +import logging + +from automapper import mapper, exceptions +from bubus import EventBus +from fastapi import APIRouter, Depends +from fastapi_utils.cbv import cbv +from starlette.responses import JSONResponse, Response + +from api.models import Request +from core.injects import AutowireSupport +from core.woodpecker import Woodpecker +from model.webhook import WoodpeckerEvent + +router = APIRouter() +logger = logging.getLogger(__name__) + +@router.get("/", summary="Main API") +async def root(): + return {"message": "Witaj w API v1"} + + +@cbv(router) +class APIv1: + woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) + bus: EventBus = Depends(AutowireSupport.bus) + + def __init__(self): + try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie + mapper.add(Request, WoodpeckerEvent) + except exceptions.DuplicatedRegistrationError: + pass + + @router.get("/health", summary="Health check") + async def health(self) -> JSONResponse: + # TODO: JSON serialize + return JSONResponse({"status": "unknown"}) + + @router.post("/ci", summary="CI Webhook") + async def ci(self, request: Request): + await self.bus.dispatch(mapper.map(request)) + return Response(status_code=201) diff --git a/src/karl/config/__init__.py b/src/karl/config/__init__.py new file mode 100644 index 0000000..7829e5c --- /dev/null +++ b/src/karl/config/__init__.py @@ -0,0 +1,7 @@ +from .settings import AppConfig +from .settings import GitConfig +from .settings import KeePassConfig +from .settings import Settings +from .settings import get_settings + +__all__ = [AppConfig, GitConfig, KeePassConfig, Settings, get_settings] diff --git a/src/karl/config/settings.py b/src/karl/config/settings.py new file mode 100644 index 0000000..79e66c9 --- /dev/null +++ b/src/karl/config/settings.py @@ -0,0 +1,54 @@ +from functools import lru_cache +from pathlib import Path + +import yaml +from pydantic import BaseModel +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class LoggingConfig(BaseModel): + level: str = "INFO" + path: Path | None = None + + +class AppConfig(BaseModel): + host: str = "127.0.0.1" + port: int = 8081 + reload: bool = False + + +class GitConfig(BaseModel): + path: Path = Path("/opt/repo/sample") + url: str = "ssh://git@hattori.ztsh.eu:29418/paas/heimdall.git" + branch: str = "master" + remote: str = "origin" + + +class KeePassConfig(BaseModel): + file: str = "database.kdbx" + secret: Path | str = "/run/secrets/kp_secret" + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__") + logging: LoggingConfig = LoggingConfig() + app: AppConfig = AppConfig() + git: GitConfig = GitConfig() + kp: KeePassConfig = KeePassConfig() + + @classmethod + def from_yaml(cls, path: Path | str = "config/config.yaml") -> "Settings": + p = Path(path) + data = {} + if p.exists(): + with p.open("r", encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + else: + import sys, os + sys.stderr.write(f"Warning: Config file {os.path.realpath(p)} not found.\n") + return cls(**data) + + +@lru_cache +def get_settings() -> Settings: + return Settings.from_yaml() diff --git a/src/karl/core/__init__.py b/src/karl/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/karl/core/events.py b/src/karl/core/events.py new file mode 100644 index 0000000..456968b --- /dev/null +++ b/src/karl/core/events.py @@ -0,0 +1,11 @@ +import logging + +from bubus import EventBus +from injectable import injectable_factory + +logger = logging.getLogger(__name__) + +@injectable_factory(EventBus, singleton=True) +def event_bus_factory() -> EventBus: + logger.info("Creating event bus...") + return EventBus() diff --git a/src/karl/core/injects.py b/src/karl/core/injects.py new file mode 100644 index 0000000..e7081a7 --- /dev/null +++ b/src/karl/core/injects.py @@ -0,0 +1,15 @@ +from bubus import EventBus +from injectable import inject + +from core.woodpecker import Woodpecker + + +class AutowireSupport: + + @staticmethod + def woodpecker(): + return inject(Woodpecker) + + @staticmethod + def bus(): + return inject(EventBus) diff --git a/app/main.py b/src/karl/core/router.py similarity index 56% rename from app/main.py rename to src/karl/core/router.py index 22fe12a..0d1a300 100644 --- a/app/main.py +++ b/src/karl/core/router.py @@ -1,8 +1,9 @@ -from fastapi import FastAPI, Request +from fastapi import APIRouter, Request + from fastapi.responses import HTMLResponse from jinja2 import Environment, FileSystemLoader, select_autoescape -from app.api.v1 import router as api_v1_router +router = APIRouter() # Inicjalizacja Jinja2 templates_env = Environment( @@ -10,20 +11,10 @@ templates_env = Environment( autoescape=select_autoescape(["html", "xml"]), ) -app = FastAPI(title="Karl", version="0.1.0") - -# Rejestracja routera API pod /api/v1 -app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) - # Przykładowy endpoint HTML -@app.get("/", response_class=HTMLResponse) +@router.get("/", response_class=HTMLResponse) async def index(request: Request) -> HTMLResponse: template = templates_env.get_template("index.html") html = template.render(title="Strona główna", request=request) return HTMLResponse(content=html) - - -def run() -> None: - import uvicorn - uvicorn.run("app.main:app", host="127.0.0.1", port=8000, reload=True) diff --git a/src/karl/core/woodpecker.py b/src/karl/core/woodpecker.py new file mode 100644 index 0000000..24af84c --- /dev/null +++ b/src/karl/core/woodpecker.py @@ -0,0 +1,121 @@ +import asyncio +import logging +from collections import deque +from pathlib import Path +from threading import RLock, Thread +from typing import Annotated + +from bubus import EventBus, BaseEvent +from injectable import injectable, Autowired, autowired + +from config import get_settings +from model.webhook import WoodpeckerEvent +from services import GitService, DockerService +from services.mo import Mo + +logger = logging.getLogger(__name__) + + +class RunnerResult(BaseEvent): + success: bool = False + throwable: Exception | None = None + + +class WoodpeckerRunner(Thread): + @autowired + def __init__(self, git: GitService, docker: DockerService, mo: Mo, + bus: Annotated[EventBus, Autowired]): + super().__init__(daemon=True) + self._git = git + self._docker = docker + self._mo = mo + self._bus = bus + self._event: WoodpeckerEvent | None = None + self._root = get_settings().git.path + + def process_event(self, event: WoodpeckerEvent): + self._event = event + self.start() + + def run(self): + async def dispatch(r: RunnerResult): + await self._bus.dispatch(r) + + result = RunnerResult() + try: + service = self.get_service(self._event.files) + if service is None: + logger.info("No service found.") + result.success = True + else: + service_path = f"{self._root}/compose/{service}/docker-compose.yml" + self._git.checkout(self._event.commit) + for file in self._event.files: + if file.__contains__('.mo.'): + self._mo.process(Path(f"{self._root}{file}").absolute()) + self._docker.reload(Path(service_path).absolute()) + result.success = True + except Exception as e: + result.throwable = e + + asyncio.run(dispatch(result)) + + def get_service(self, files: list[str]) -> str | None: + supported_files = [] + for f in files: + f_parts = f.split("/") + if f_parts[0] in ["compose", "files"]: + supported_files.append(f[1]) + match len(set(supported_files)): + case 0: + return None + case 1: + return supported_files[0] + case _: + raise Exception("Multiple services are not supported.") + + +@injectable(singleton=True) +class Woodpecker: + @autowired + def __init__(self, mo: Annotated[Mo, Autowired], + bus: Annotated[EventBus, Autowired]): + self._mo = mo + self._bus = bus + self._git = GitService() + self._docker = DockerService() + self._runner: WoodpeckerRunner | None = None + self._pending = deque() + self._lock = RLock() + bus.on(WoodpeckerEvent, self.on_ci_event) + bus.on(RunnerResult, self._on_runner_completed) + logger.info("Woodpecker initialized.") + + async def on_ci_event(self, event: WoodpeckerEvent): + logger.debug(f"Received WoodpeckerEvent: {event.event_id}") + with self._lock: + logger.debug("Lock acquired [on-ci-event]") + if len(self._pending) > 0 or self._runner is not None: + self._pending.append(event) + else: + self._start_runner(event) + + def _start_runner(self, event: WoodpeckerEvent): + with self._lock: + logger.debug("Lock acquired [start-runner]") + self._runner = WoodpeckerRunner(self._git, self._docker, self._mo) + self._runner.process_event(event) + + def _on_runner_completed(self, result: RunnerResult): + logger.debug(f"Received RunnerResult: {result.event_id}") + logger.info(f"Runner completed {'successfully' if result.success else 'with error'}.") + if result.throwable is not None: + logger.error(f"Runner error: {result.throwable}", exc_info=True) + self._runner.join(timeout=1) + logger.debug("Runner joined.") + with self._lock: + logger.debug("Lock acquired [on-runner-completed]") + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) diff --git a/src/karl/main.py b/src/karl/main.py new file mode 100644 index 0000000..dbd5033 --- /dev/null +++ b/src/karl/main.py @@ -0,0 +1,63 @@ +import logging + +from fastapi import FastAPI +from injectable import load_injection_container + +from config import get_settings +from util.logging import HandlerFactory + + +class KarlApplication: + from starlette.types import Receive, Scope, Send + def __init__(self) -> None: + self._set_logging() + load_injection_container() + _app = FastAPI(title="Karl", version="0.1.0") + self._set_middlewares(_app) + self._set_routes(_app) + self._set_events(_app) + + self._app = _app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + await self._app.__call__(scope, receive, send) + + def _set_logging(self): + settings = get_settings() + logging.addLevelName(5, "TRACE") + logging.Logger.trace = lambda s, msg, *args, **kwargs: s.log(5, msg, *args, **kwargs) + logging.basicConfig(level=settings.logging.level, + handlers=HandlerFactory.create(HandlerFactory.Target.ALL, handler_prefix='karl.', + file_path=settings.logging.path)) + + loggers = ( + "uvicorn", + "uvicorn.access", + "uvicorn.error", + "fastapi", + "asyncio", + "starlette", + ) + external_handlers = HandlerFactory.create(HandlerFactory.Target.ALL, file_path=settings.logging.path) + for logger_name in loggers: + logging_logger = logging.getLogger(logger_name) + logging_logger.handlers = external_handlers + logging_logger.propagate = False + + def _set_middlewares(self, app: FastAPI): + from web.middlewares import LoggingMiddleware + app.add_middleware(LoggingMiddleware) + + def _set_routes(self, app: FastAPI): + from core.router import router as core_router + app.include_router(core_router) + from api.v1 import router as api_v1_router + app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) + pass + + def _set_events(self, app: FastAPI): + pass + + +def run(): + return KarlApplication() diff --git a/src/karl/model/__init__.py b/src/karl/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/karl/model/containers.py b/src/karl/model/containers.py new file mode 100644 index 0000000..2bdf6aa --- /dev/null +++ b/src/karl/model/containers.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass, field +from datetime import datetime + +from docker.models.containers import Container + + +@dataclass +class SimpleContainer: + name: str + image: str + status: str + health: str + created: datetime + + @staticmethod + def from_container(container: Container): + created = datetime.strptime(container.attrs['Created'].split('.')[0], '%Y-%m-%dT%H:%M:%S') + return SimpleContainer( + name=container.name, + image=container.image.tags[0], + status=container.status, + health=container.health, + created=created + ) + + +@dataclass +class Compose: + directory: str + containers: list[SimpleContainer] = field(default_factory=list) + + @property + def last_modified(self): + return max(self.containers, key=lambda c: c.created).created + + +@dataclass +class Tree: + composes: dict[str, Compose] = field(default_factory=dict) + containers: list[SimpleContainer] = field(default_factory=list) diff --git a/src/karl/model/healthcheck.py b/src/karl/model/healthcheck.py new file mode 100644 index 0000000..ac483f3 --- /dev/null +++ b/src/karl/model/healthcheck.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass +class HealthCheck: + healthy: bool + message: str diff --git a/src/karl/model/passwords.py b/src/karl/model/passwords.py new file mode 100644 index 0000000..253dbd9 --- /dev/null +++ b/src/karl/model/passwords.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass, field +from typing import Type + + +# TODO: unnecessary? + +@dataclass +class PathItem: + name: str + t: Type + + +@dataclass +class Path: + path: list[PathItem] = field(default_factory=list) + + def append(self, name, t): + self.path.append(PathItem(name, t)) + + def __str__(self): + return "/".join([i.name for i in self.path]) + + +@dataclass +class Group: + name: str + passwords: list["Password"] = field(default_factory=list) + parent: "Group|None" = None + + @property + def path(self): + if self.parent is None: + new_path = Path() + new_path.append(self.name, type(self)) + return new_path + return self.parent.path.append(self.name, type(self)) + + +@dataclass +class Password: + name: str + group: Group + + @property + def path(self): + return self.group.path.append(self.name, type(self)) + + +class UnencryptedPassword(Password): + def __init__(self, name: str, value: str, group: Group): + super().__init__(name, group) + self.value = value diff --git a/src/karl/model/webhook.py b/src/karl/model/webhook.py new file mode 100644 index 0000000..e97c073 --- /dev/null +++ b/src/karl/model/webhook.py @@ -0,0 +1,11 @@ +from typing import List + +from bubus import BaseEvent + + +class WoodpeckerEvent(BaseEvent): + _id: str + commit: str + message: str + started: str + files: List[str] diff --git a/src/karl/services/__init__.py b/src/karl/services/__init__.py new file mode 100644 index 0000000..febb742 --- /dev/null +++ b/src/karl/services/__init__.py @@ -0,0 +1,5 @@ +from .containers import DockerService +from .passwords import Passwords +from .vcs import GitService + +__all__ = ["GitService", "Passwords", "DockerService"] diff --git a/src/karl/services/containers.py b/src/karl/services/containers.py new file mode 100644 index 0000000..8803913 --- /dev/null +++ b/src/karl/services/containers.py @@ -0,0 +1,59 @@ +import logging +from pathlib import Path + +import docker +from docker.models.containers import Container +from injectable import injectable + +from model.containers import Tree, Compose, SimpleContainer + +logger = logging.getLogger(__name__) + + +@injectable(singleton=True) +class DockerService: + def __init__(self): + self._client = docker.from_env() + # logger.info(f"Docker client initialized. Plugins: {self._client.plugins()}") + self._tree = self._init_tree() + + def _init_tree(self) -> Tree: + tree = Tree() + container: Container + for container in self._client.containers.list(): + labels = container.labels + working_dir = labels.get("com.docker.compose.project.working_dir") + if working_dir: + if tree.composes.get(working_dir) is None: + tree.composes[working_dir] = Compose(working_dir) + tree.composes[working_dir].containers.append(SimpleContainer.from_container(container)) + else: + tree.containers.append(SimpleContainer.from_container(container)) + return tree + + @property + def tree(self) -> Tree: + return self._tree + + def reload(self, compose_path: Path): + # TODO: Won't work in docker container + cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"] + import subprocess + try: + process = subprocess.run( + cmd, + capture_output=True, + text=True, + check=False + ) + if process.returncode != 0: + logger.error(f"Docker compose failed with code {process.returncode}") + logger.error(f"stderr: {process.stderr}") + raise Exception(f"Docker compose failed: {process.stderr}") + + logger.info(f"Docker compose executed successfully") + logger.debug(f"stdout: {process.stdout}") + return process.stdout, process.stderr, process.returncode + except Exception as e: + logger.error(f"Failed to execute docker compose command: {e}") + raise e diff --git a/src/karl/services/mo.py b/src/karl/services/mo.py new file mode 100644 index 0000000..9d5756e --- /dev/null +++ b/src/karl/services/mo.py @@ -0,0 +1,30 @@ +from pathlib import Path +from string import Template +from typing import Annotated + +from injectable import injectable, autowired, Autowired + +from services import Passwords + + +class ValueTemplate(Template): + # Pozwala na kropki i ukośniki w nazwach placeholderów, np. ${user.name/first} + idpattern = r'[_a-zA-Z][_a-zA-Z0-9.\/]*' + + +@injectable +class Mo: + @autowired + def __init__(self, passwords: Annotated[Passwords, Autowired]): + self._passwords = passwords + + def process(self, mo_file: Path): + raw = '' + with open(mo_file, "r") as mo: + raw = mo.read() + parsed = ValueTemplate(raw) + mappings = self._passwords.get_values(parsed.get_identifiers()) + rendered = parsed.safe_substitute(mappings) + de_mo_ified = str(mo_file).replace(".mo", "") + with open(de_mo_ified, "w") as mo: + mo.write(rendered) diff --git a/src/karl/services/passwords.py b/src/karl/services/passwords.py new file mode 100644 index 0000000..a76dac8 --- /dev/null +++ b/src/karl/services/passwords.py @@ -0,0 +1,75 @@ +import os.path +from contextlib import contextmanager +from typing import Any, Generator + +import keyring +from injectable import injectable +from pykeepass import PyKeePass, create_database + +class KeyRequest: + def __init__(self, prompt: str): + self.field_name = None + self.entry_name = None + self.path = None + self._parse_prompt(prompt) + + def _parse_prompt(self, prompt: str): + prompt_parts = prompt.split("/") + key = None + match len(prompt_parts): + case 1: + self.field_name = 'password' + key = prompt_parts[0] + case 2: + self.field_name = prompt_parts[1] + key = prompt_parts[0] + case _: + key = None + if key is None: + return + key_parts = key.split(".") + self.path = key_parts[:] if len(key_parts) > 1 else None + self.entry_name = key_parts[-1] + +@injectable(singleton=True) +class Passwords: + def __init__(self): + from config import get_settings + settings = get_settings() + import keyring_backend + keyring.set_keyring(keyring=keyring_backend.Backend()) + + with open(settings.kp.secret, "r") as fh: + keyring.set_password("karl", "kp", fh.read().splitlines()[0]) + self._path = settings.kp.file + + @contextmanager + def open(self, mode: str = "r") -> Generator[PyKeePass | Any, Any, None]: + kp = PyKeePass(self._path, password=keyring.get_password("karl", "kp")) \ + if os.path.exists(self._path) else create_database(self._path, password=keyring.get_password("karl", "kp")) + yield kp + if mode == "rw": + kp.save() + + def get_values(self, keys: list[str]) -> dict[str, str]: + output = {} + for k in keys: + request = KeyRequest(k) + with self.open() as kp: + kp_entry = kp.find_entries(path=request.path, first=True, title=request.entry_name) + output[k] = self._get_field_value(kp_entry, request.field_name) + return output + + @staticmethod + def _get_field_value(kp_entry, field_name): + if kp_entry is None: + return None + match field_name: + case "username": + return kp_entry.username + case "password": + return kp_entry.password + case "url": + return kp_entry.url + case _: + return kp_entry.get_custom_property(field_name) diff --git a/src/karl/services/system.py b/src/karl/services/system.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/karl/services/system.py @@ -0,0 +1 @@ + diff --git a/src/karl/services/vcs.py b/src/karl/services/vcs.py new file mode 100644 index 0000000..8796787 --- /dev/null +++ b/src/karl/services/vcs.py @@ -0,0 +1,31 @@ +from git import Repo, Remote +from injectable import injectable + +from config import GitConfig, get_settings + + +@injectable(singleton=True) +class GitService: + def __init__(self): + self._settings = get_settings() + self._repo = self._check_preconditions(self._settings.git) + if self._repo.head.ref.name != self._settings.git.branch: + self._repo.git.checkout(self._settings.git.branch) + self._origin: Remote = self._repo.remotes.origin + + @staticmethod + def _check_preconditions(config: GitConfig) -> Repo: + def clone(): + return Repo.clone_from(config.url, config.path, branch=config.branch) + + import os + if not config.path.exists(): + return clone() + if not (config.path / ".git").exists(): + os.rmdir(config.path) + return clone() + return Repo(config.path) + + def checkout(self, sha: str): + self._origin.fetch() + self._repo.git.checkout(sha) diff --git a/app/templates/index.html b/src/karl/templates/index.html similarity index 93% rename from app/templates/index.html rename to src/karl/templates/index.html index ebbe8ca..ac9bbbe 100644 --- a/app/templates/index.html +++ b/src/karl/templates/index.html @@ -3,7 +3,7 @@ {{ title }} - +