diff --git a/.gitignore b/.gitignore index 8395e24..4b259c4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ uv.lock __pycache__/ **/dist/ +**/*.log diff --git a/app/api/models.py b/app/api/models.py index 35afedb..b27765b 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -4,11 +4,11 @@ from typing import List @dataclass class Request: - build_id: str - build_url: str - commit_id: str - commit_url: str - changelist: List[str] + _id: str + commit: str + message: str + started: str + files: List[str] @dataclass class Response: diff --git a/app/api/v1.py b/app/api/v1.py index fc52bd4..a012834 100644 --- a/app/api/v1.py +++ b/app/api/v1.py @@ -1,6 +1,12 @@ -from fastapi import APIRouter +from automapper import mapper, exceptions +from fastapi import APIRouter, Depends +from fastapi_utils.cbv import cbv +from starlette.responses import JSONResponse, Response -from app.api.models import Request, Response +from app.api.models import Request +from app.core.injects import AutowireSupport +from app.core.woodpecker import Woodpecker +from app.model.webhook import WoodpeckerEvent router = APIRouter() @@ -10,10 +16,23 @@ async def root(): return {"message": "Witaj w API v1"} -@router.get("/health", summary="Health check") -async def health(): - return {"status": "ok"} +@cbv(router) +class APIv1: + woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) + logger = __import__('logging').getLogger(__name__) -@router.post("/ci", summary="CI Webhook") -async def ci(request: Request): - return Response(200) + def __init__(self): + try: # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie + mapper.add(Request, WoodpeckerEvent) + except exceptions.DuplicatedRegistrationError: + pass + + @router.get("/health", summary="Health check") + async def health(self) -> JSONResponse: + # TODO: JSON serialize + return JSONResponse({"status": "unknown"}) + + @router.post("/ci", summary="CI Webhook") + async def ci(self, request: Request): + self.woodpecker.on_ci_event(mapper.map(request)) + return Response(status_code=201) diff --git a/app/config/settings.py b/app/config/settings.py index 7e072aa..2017c14 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -6,6 +6,11 @@ from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict +class LoggingConfig(BaseModel): + level: str = "INFO" + path: Path = Path("logs/karl.log") + + class AppConfig(BaseModel): host: str = "127.0.0.1" port: int = 8081 @@ -26,6 +31,7 @@ class KeePassConfig(BaseModel): class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__") + logging: LoggingConfig = LoggingConfig() app: AppConfig = AppConfig() git: GitConfig = GitConfig() kp: KeePassConfig = KeePassConfig() diff --git a/app/core/core.py b/app/core/core.py deleted file mode 100644 index 9e3acec..0000000 --- a/app/core/core.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import Annotated - -from injectable import injectable, autowired, Autowired - -from app.model.healthcheck import HealthCheck -from app.services import DockerService, GitService, Passwords - - -# @injectable -class WebhookProcessor: - @autowired - def __init__(self, docker: Annotated[DockerService, Autowired], - git: Annotated[GitService, Autowired], - keepass: Annotated[Passwords, Autowired]): - self._docker = docker - self._git = git - self._keepass = keepass - - @property - def health(self) -> HealthCheck: - return HealthCheck( - self._docker is not None and self._git is not None and self._keepass is not None, - f"Docker: {self._docker is not None}, Git: {self._git is not None}, KeePass: {self._keepass is not None}" - ) diff --git a/app/core/injects.py b/app/core/injects.py new file mode 100644 index 0000000..115ae95 --- /dev/null +++ b/app/core/injects.py @@ -0,0 +1,10 @@ +from injectable import inject + +from app.core.woodpecker import Woodpecker + + +class AutowireSupport: + + @staticmethod + def woodpecker(): + return inject(Woodpecker) diff --git a/app/core/woodpecker.py b/app/core/woodpecker.py new file mode 100644 index 0000000..1892c0b --- /dev/null +++ b/app/core/woodpecker.py @@ -0,0 +1,107 @@ +import logging +from collections import deque +from multiprocessing import Process, Lock +from pathlib import Path +from typing import Annotated + +from injectable import injectable, Autowired, autowired + +from app.config import get_settings +from app.model.webhook import WoodpeckerEvent +from app.services import GitService, DockerService +from app.services.mo import Mo + +logger = logging.getLogger(__name__) + + +class WoodpeckerRunner(Process): + def __init__(self, git: GitService, docker: DockerService, mo: Mo, + success_callback=None, error_callback=None): + super().__init__(daemon=True) + self._git = git + self._docker = docker + self._mo = mo + self._success_callback = success_callback + self._error_callback = error_callback + self._event: WoodpeckerEvent | None = None + self._root = get_settings().git.path + + def process_event(self, event: WoodpeckerEvent): + self._event = event + self.start() + + def run(self): + try: + service = self.get_service(self._event.files) + if service is None: + logger.info("No service found.") + return self._success_callback() + service_path = f"{self._root}/compose/{service}/docker-compose.yml" + self._git.checkout(self._event.commit) + for file in self._event.files: + if file.__contains__('.mo.'): + self._mo.process(Path(f"{self._root}{file}").absolute()) + self._docker.reload(Path(service_path).absolute()) + + return self._success_callback() + except Exception as e: + return self._error_callback(e) + + def get_service(self, files: list[str]) -> str | None: + supported_files = [] + for f in files: + f_parts = f.split("/") + if f_parts[0] in ["compose", "files"]: + supported_files.append(f[1]) + match len(set(supported_files)): + case 0: + return None + case 1: + return supported_files[0] + case _: + raise Exception("Multiple services are not supported.") + + +@injectable(singleton=True) +class Woodpecker: + @autowired + def __init__(self, mo: Annotated[Mo, Autowired]): + self._mo = mo + self._git = GitService() + self._docker = DockerService() + self._runner: WoodpeckerRunner | None = None + self._pending = deque() + self._lock = Lock() + logger.info("Woodpecker initialized.") + + def on_ci_event(self, event: WoodpeckerEvent): + logger.info(f"Received event: {event}") + with self._lock: + if len(self._pending) > 0 or self._runner is not None: + self._pending.append(event) + return + self._start_runner(event) + + def _start_runner(self, event: WoodpeckerEvent): + with self._lock: + self._runner = WoodpeckerRunner(self._git, self._docker, self._mo, + self._on_runner_completed, self._on_runner_error) + self._runner.process_event(event) + + def _on_runner_completed(self): + logger.info("Runner completed.") + self._runner.join() + with self._lock: + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) + + def _on_runner_error(self, t: Exception): + logger.error(f"Runner error: {t}", exc_info=True) + self._runner.join() + with self._lock: + self._runner = None + if len(self._pending) > 0: + event = self._pending.popleft() + self._start_runner(event) diff --git a/app/main.py b/app/main.py index 5bf8c98..669d2d4 100644 --- a/app/main.py +++ b/app/main.py @@ -4,18 +4,18 @@ from fastapi import FastAPI from injectable import load_injection_container from app.config import get_settings -from app.core.core import WebhookProcessor -from app.util.logging import LoggingHandler, ExternalLoggingHandler +from app.util.logging import HandlerFactory class KarlApplication: from starlette.types import Receive, Scope, Send def __init__(self) -> None: self._set_logging() + load_injection_container() _app = FastAPI(title="Karl", version="0.1.0") + self._set_middlewares(_app) self._set_routes(_app) self._set_events(_app) - self._init_services() self._app = _app @@ -23,7 +23,12 @@ class KarlApplication: await self._app.__call__(scope, receive, send) def _set_logging(self): - logging.basicConfig(level=logging.INFO, handlers=[LoggingHandler()]) + settings = get_settings() + logging.addLevelName(5, "TRACE") + logging.Logger.trace = lambda s, msg, *args, **kwargs: s.log(5, msg, *args, **kwargs) + logging.basicConfig(level=settings.logging.level, + handlers=HandlerFactory.create(HandlerFactory.Target.ALL, handler_prefix='karl.', + file_path=settings.logging.path)) loggers = ( "uvicorn", @@ -33,12 +38,16 @@ class KarlApplication: "asyncio", "starlette", ) - external_handler = ExternalLoggingHandler() + external_handlers = HandlerFactory.create(HandlerFactory.Target.ALL, file_path=settings.logging.path) for logger_name in loggers: logging_logger = logging.getLogger(logger_name) - logging_logger.handlers = [external_handler] + logging_logger.handlers = external_handlers logging_logger.propagate = False + def _set_middlewares(self, app: FastAPI): + from app.web.middlewares import LoggingMiddleware + app.add_middleware(LoggingMiddleware) + def _set_routes(self, app: FastAPI): from app.core.router import router as core_router app.include_router(core_router) @@ -49,14 +58,8 @@ class KarlApplication: def _set_events(self, app: FastAPI): pass - def _init_services(self): - logger = logging.getLogger(__name__) - load_injection_container() - webhook_service = WebhookProcessor() - logger.info(webhook_service.health) - -def app(): +def run(): return KarlApplication() @@ -65,7 +68,7 @@ if __name__ == "__main__": settings = get_settings() uvicorn.run( - "app.main:app", + "app.main:run", factory=True, host=settings.app.host, port=settings.app.port, diff --git a/app/model/webhook.py b/app/model/webhook.py new file mode 100644 index 0000000..364e17b --- /dev/null +++ b/app/model/webhook.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass +from typing import List + +@dataclass +class WoodpeckerEvent: + _id: str + commit: str + message: str + started: str + files: List[str] diff --git a/app/services/containers.py b/app/services/containers.py index ef891e3..6e927bf 100644 --- a/app/services/containers.py +++ b/app/services/containers.py @@ -1,14 +1,20 @@ +import logging +from pathlib import Path + import docker from docker.models.containers import Container from injectable import injectable from app.model.containers import Tree, Compose, SimpleContainer +logger = logging.getLogger(__name__) + @injectable(singleton=True) class DockerService: def __init__(self): self._client = docker.from_env() + # logger.info(f"Docker client initialized. Plugins: {self._client.plugins()}") self._tree = self._init_tree() def _init_tree(self) -> Tree: @@ -28,3 +34,8 @@ class DockerService: @property def tree(self) -> Tree: return self._tree + + def reload(self, compose_path: Path): + cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"] + # TODO: subprocess + diff --git a/app/services/mo.py b/app/services/mo.py new file mode 100644 index 0000000..9a35642 --- /dev/null +++ b/app/services/mo.py @@ -0,0 +1,37 @@ +from pathlib import Path +from string import Template +from typing import Annotated + +from injectable import injectable, autowired, Autowired + +from app.services import Passwords + + +class SimpleValueTemplate(Template): + # Pozwala na kropki w nazwach placeholderów, np. ${user.name.first} + idpattern = r'[_a-zA-Z][_a-zA-Z0-9.]*' + + +class ComplexValueTemplate(SimpleValueTemplate): + delimiter = '@' + + +@injectable +class Mo: + @autowired + def __init__(self, passwords: Annotated[Passwords, Autowired]): + self._passwords = passwords + + def process(self, mo_file: Path): + raw = '' + with open(mo_file, "r") as mo: + raw = mo.read() + cmp = ComplexValueTemplate(raw) + rendered = cmp.substitute(self._passwords.get_values(cmp.get_identifiers())) + smp = SimpleValueTemplate(rendered) + ids = [_id + '.password' for _id in smp.get_identifiers()] + mappings = {k.replace('.password', ''): v for k, v in self._passwords.get_values(ids).items()} + rendered = smp.substitute(mappings) + de_mo_ified = str(mo_file).replace(".mo", "") + with open(de_mo_ified, "w") as mo: + mo.write(rendered) diff --git a/app/services/passwords.py b/app/services/passwords.py index 18d8519..408ec5e 100644 --- a/app/services/passwords.py +++ b/app/services/passwords.py @@ -1,7 +1,8 @@ import os.path +import shutil from injectable import injectable -from pykeepass import PyKeePass, create_database, Group +from pykeepass import PyKeePass, create_database @injectable(singleton=True) @@ -11,30 +12,49 @@ class Passwords: settings = get_settings() with open(settings.kp.secret, "r") as fh: - secret = fh.read() - - self._kp_org = self.__get_or_create_store(settings.kp.file, secret) - self._kp = self.__get_lock(settings.kp.file, secret) + secret = fh.read().splitlines()[0] + self._path = settings.kp.file + self._kp_org = self._open_or_create(self._path, secret) + self._kp = self._open_lock(self._path, secret) @staticmethod - def __get_or_create_store(path, passwd) -> PyKeePass: + def _open_or_create(path, password) -> PyKeePass: if os.path.exists(path): - return PyKeePass( - path, - password=passwd, - ) - return create_database(path, passwd) + return PyKeePass(path, password=password) + return create_database(path, password) @staticmethod - def __get_lock(path, passwd) -> PyKeePass: + def _open_lock(path, password) -> PyKeePass: lock_path = path + ".lock" - import shutil shutil.copyfile(path, lock_path) - return Passwords.__get_or_create_store(lock_path, passwd) + return Passwords._open_or_create(lock_path, password) - @property - def store(self): - return self._kp.root_group + def get_values(self, keys: list[str]) -> dict[str, str]: + output = {} + for k in keys: + key_parts = k.split(".") + path = key_parts[:-1] if len(key_parts) > 2 else None + entry_name = key_parts[-2] + field_name = key_parts[-1] + kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name) + output[k] = self._get_field_value(kp_entry, field_name) + return output - def save(self, group: Group): - pass + @staticmethod + def _get_field_value(kp_entry, field_name): + if kp_entry is None: + return None + match field_name: + case "username": + return kp_entry.username + case "password": + return kp_entry.password + case "url": + return kp_entry.url + case _: + return kp_entry.get_custom_property(field_name) + + def save(self): + # nadpisz plik źródłowy zmianami z lock + self._kp.save() + shutil.copyfile(self._path + ".lock", self._path) diff --git a/app/services/vcs.py b/app/services/vcs.py index a7d83b0..0e97ec8 100644 --- a/app/services/vcs.py +++ b/app/services/vcs.py @@ -13,15 +13,11 @@ class GitService: self._repo.git.checkout(self._settings.git.branch) self._origin: Remote = self._repo.remotes.origin - - def get_modified_compose(self) -> str | None: - self._update() - return self._diff() - @staticmethod def _check_preconditions(config: GitConfig) -> Repo: def clone(): return Repo.clone_from(config.url, config.path, branch=config.branch) + import os if not config.path.exists(): return clone() @@ -30,16 +26,6 @@ class GitService: return clone() return Repo(config.path) - def _update(self): - self._origin.pull() - - def _diff(self) -> str | None: - diff = self._repo.head.commit.diff("HEAD~1") - composes = [f for f in diff if f.a_path.endswith("docker-compose.yml")] - match len(composes): - case 0: - return None - case 1: - return composes[0].a_path - case _: - raise Exception("Multiple compose files modified") + def checkout(self, sha: str): + self._origin.fetch() + self._repo.git.checkout(sha) diff --git a/app/util/logging.py b/app/util/logging.py index 29992b6..3e9e77a 100644 --- a/app/util/logging.py +++ b/app/util/logging.py @@ -1,4 +1,8 @@ -from logging import Formatter, StreamHandler +from enum import Enum, auto +from logging import Formatter, StreamHandler, Handler +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path +from typing import List class NamingCache: @@ -37,7 +41,7 @@ class ApplicationFormatter(Formatter): from datetime import datetime timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') level = record.levelname.replace('WARNING', 'WARN').rjust(5) - thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? + thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"] message = record.getMessage() formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}" @@ -48,12 +52,35 @@ class ApplicationFormatter(Formatter): return formatted -class LoggingHandler(StreamHandler): - def __init__(self): - super().__init__() - self.setFormatter(ApplicationFormatter(handler_prefix='karl.')) +class HandlerFactory: + class Target(Enum): + CONSOLE = auto() + FILE = auto() + ALL = auto() -class ExternalLoggingHandler(StreamHandler): - def __init__(self): - super().__init__() - self.setFormatter(ApplicationFormatter()) + @staticmethod + def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]: + def console_handler(prefix: str = ''): + handler = StreamHandler() + handler.setFormatter(ApplicationFormatter(prefix)) + handler.setLevel('INFO') + return handler + + def file_handler(prefix: str = ''): + handler = TimedRotatingFileHandler(file_path, when='midnight', backupCount=30) + handler.setFormatter(ApplicationFormatter(prefix)) + handler.setLevel('TRACE') + return handler + + handlers = [] + match target: + case HandlerFactory.Target.CONSOLE: + handlers.append(console_handler(handler_prefix)) + case HandlerFactory.Target.FILE: + handlers.append(file_handler(handler_prefix)) + case HandlerFactory.Target.ALL: + handlers.append(file_handler(handler_prefix)) + handlers.append(console_handler(handler_prefix)) + case _: + raise ValueError(f"Unknown target: {target}") + return handlers diff --git a/app/web/__init__.py b/app/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/web/middlewares.py b/app/web/middlewares.py new file mode 100644 index 0000000..d162ed1 --- /dev/null +++ b/app/web/middlewares.py @@ -0,0 +1,24 @@ +import logging + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request + +logger = logging.getLogger(__name__) + + +class LoggingMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + client = f"{request.client.host}:{request.client.port}" + match request.method: + case "POST" | "PUT" | "DELETE" if request.headers.get("Content-Type") == "application/json": + body = await request.body() + logger.trace(f"Request from {client}: {body.decode()}") + case "GET": + logger.trace(f"Request from {client}") + case _: + logger.trace(f"Request from {client} (content-type:{request.headers.get("Content-Type")})") + + response = await call_next(request) + logger.trace(f"Respone: {response.status_code} {type(response)}") + + return response diff --git a/config/config.yaml b/config/config.yaml index 856d24e..c4fc44a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,7 +1,10 @@ +logging: + level: "TRACE" + path: "logs/karl.log" app: host: "127.0.0.1" port: 8081 - reload: true + reload: false git: path: "F:/IdeaProjects/paas/karl/.compose_repository" branch: "main" diff --git a/pyproject.toml b/pyproject.toml index c4e6d55..1791992 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,8 @@ dependencies = [ "pykeepass>=4.1.1.post1", "docker>=7.1.0", "injectable==4.0.1", + "py-automapper>=2.2.0", + "fastapi-utils>=0.8.0", ] [project.optional-dependencies] diff --git a/run.sh b/run.sh index 8d2e2b9..cf3a77b 100644 --- a/run.sh +++ b/run.sh @@ -1 +1 @@ -uvicorn app.main:app --factory --reload +uvicorn app.main:run --factory --reload diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/files/test1/test.mo.yaml b/tests/files/test1/test.mo.yaml new file mode 100644 index 0000000..572b953 --- /dev/null +++ b/tests/files/test1/test.mo.yaml @@ -0,0 +1,3 @@ +value: ${sample} +nested: ${some.nested.value} +custom: @{custom.field} diff --git a/tests/test_mo.py b/tests/test_mo.py new file mode 100644 index 0000000..f685e34 --- /dev/null +++ b/tests/test_mo.py @@ -0,0 +1,23 @@ +import os +from pathlib import Path +from unittest import TestCase + +import yaml + +from app.services import Passwords +from app.services.mo import Mo + + +class TestMo(TestCase): + def test_process(self): + target_path = Path('tests/files/test1/test.yaml') + mo = Mo(Passwords()) + mo.process(Path('tests/files/test1/test.mo.yaml').absolute()) + self.assertTrue(os.path.exists(target_path)) + with open(target_path, 'r') as f: + content = f.read() + self.assertFalse(content.__contains__('${')) + parsed = yaml.load(content, Loader=yaml.FullLoader) + self.assertEqual('some_pass', parsed['value']) + self.assertEqual('nested_pass', parsed['nested']) + self.assertEqual('custom_content', parsed['custom'])