Merge pull request 'webhook' (#9) from webhook into develop

Reviewed-on: https://hattori.ztsh.eu/paas/karl/pulls/9

.gitignore

@@ -6,3 +6,4 @@ uv.lock
 
 __pycache__/
 **/dist/
+**/*.log

app/api/models.py

@@ -4,11 +4,11 @@ from typing import List
 
 @dataclass
 class Request:
-  build_id: str
+  _id: str
-  build_url: str
+  commit: str
-  commit_id: str
+  message: str
-  commit_url: str
+  started: str
-  changelist: List[str]
+  files: List[str]
 
 @dataclass
 class Response:

app/api/v1.py

@@ -1,6 +1,12 @@
-from fastapi import APIRouter
+from automapper import mapper, exceptions
+from fastapi import APIRouter, Depends
+from fastapi_utils.cbv import cbv
+from starlette.responses import JSONResponse, Response
 
-from app.api.models import Request, Response
+from app.api.models import Request
+from app.core.injects import AutowireSupport
+from app.core.woodpecker import Woodpecker
+from app.model.webhook import WoodpeckerEvent
 
 router = APIRouter()
 
@@ -10,10 +16,23 @@ async def root():
   return {"message": "Witaj w API v1"}
 
 
-@router.get("/health", summary="Health check")
+@cbv(router)
-async def health():
+class APIv1:
-  return {"status": "ok"}
+  woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker)
+  logger = __import__('logging').getLogger(__name__)
 
-@router.post("/ci", summary="CI Webhook")
+  def __init__(self):
-async def ci(request: Request):
+    try:  # TODO: rejestracja w innym miejscu: klasa jest przeładowywana co żądanie
-  return Response(200)
+      mapper.add(Request, WoodpeckerEvent)
+    except exceptions.DuplicatedRegistrationError:
+      pass
+
+  @router.get("/health", summary="Health check")
+  async def health(self) -> JSONResponse:
+    # TODO: JSON serialize
+    return JSONResponse({"status": "unknown"})
+
+  @router.post("/ci", summary="CI Webhook")
+  async def ci(self, request: Request):
+    self.woodpecker.on_ci_event(mapper.map(request))
+    return Response(status_code=201)

app/config/settings.py

@@ -6,6 +6,11 @@ from pydantic import BaseModel
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
+class LoggingConfig(BaseModel):
+  level: str = "INFO"
+  path: Path = Path("logs/karl.log")
+
+
 class AppConfig(BaseModel):
   host: str = "127.0.0.1"
   port: int = 8081
@@ -26,6 +31,7 @@ class KeePassConfig(BaseModel):
 
 class Settings(BaseSettings):
   model_config = SettingsConfigDict(env_prefix="KARL_", env_nested_delimiter="__")
+  logging: LoggingConfig = LoggingConfig()
   app: AppConfig = AppConfig()
   git: GitConfig = GitConfig()
   kp: KeePassConfig = KeePassConfig()

app/core/core.py

@@ -1,24 +0,0 @@
-from typing import Annotated
-
-from injectable import injectable, autowired, Autowired
-
-from app.model.healthcheck import HealthCheck
-from app.services import DockerService, GitService, Passwords
-
-
-# @injectable
-class WebhookProcessor:
-    @autowired
-    def __init__(self, docker: Annotated[DockerService, Autowired],
-                 git: Annotated[GitService, Autowired],
-                 keepass: Annotated[Passwords, Autowired]):
-      self._docker = docker
-      self._git = git
-      self._keepass = keepass
-
-    @property
-    def health(self) -> HealthCheck:
-      return HealthCheck(
-        self._docker is not None and self._git is not None and self._keepass is not None,
-        f"Docker: {self._docker is not None}, Git: {self._git is not None}, KeePass: {self._keepass is not None}"
-      )

app/core/injects.py

@@ -0,0 +1,10 @@
+from injectable import inject
+
+from app.core.woodpecker import Woodpecker
+
+
+class AutowireSupport:
+
+  @staticmethod
+  def woodpecker():
+    return inject(Woodpecker)

app/core/woodpecker.py

@@ -0,0 +1,107 @@
+import logging
+from collections import deque
+from multiprocessing import Process, Lock
+from pathlib import Path
+from typing import Annotated
+
+from injectable import injectable, Autowired, autowired
+
+from app.config import get_settings
+from app.model.webhook import WoodpeckerEvent
+from app.services import GitService, DockerService
+from app.services.mo import Mo
+
+logger = logging.getLogger(__name__)
+
+
+class WoodpeckerRunner(Process):
+  def __init__(self, git: GitService, docker: DockerService, mo: Mo,
+               success_callback=None, error_callback=None):
+    super().__init__(daemon=True)
+    self._git = git
+    self._docker = docker
+    self._mo = mo
+    self._success_callback = success_callback
+    self._error_callback = error_callback
+    self._event: WoodpeckerEvent | None = None
+    self._root = get_settings().git.path
+
+  def process_event(self, event: WoodpeckerEvent):
+    self._event = event
+    self.start()
+
+  def run(self):
+    try:
+      service = self.get_service(self._event.files)
+      if service is None:
+        logger.info("No service found.")
+        return self._success_callback()
+      service_path = f"{self._root}/compose/{service}/docker-compose.yml"
+      self._git.checkout(self._event.commit)
+      for file in self._event.files:
+        if file.__contains__('.mo.'):
+          self._mo.process(Path(f"{self._root}{file}").absolute())
+      self._docker.reload(Path(service_path).absolute())
+
+      return self._success_callback()
+    except Exception as e:
+      return self._error_callback(e)
+
+  def get_service(self, files: list[str]) -> str | None:
+    supported_files = []
+    for f in files:
+      f_parts = f.split("/")
+      if f_parts[0] in ["compose", "files"]:
+        supported_files.append(f[1])
+    match len(set(supported_files)):
+      case 0:
+        return None
+      case 1:
+        return supported_files[0]
+      case _:
+        raise Exception("Multiple services are not supported.")
+
+
+@injectable(singleton=True)
+class Woodpecker:
+  @autowired
+  def __init__(self, mo: Annotated[Mo, Autowired]):
+    self._mo = mo
+    self._git = GitService()
+    self._docker = DockerService()
+    self._runner: WoodpeckerRunner | None = None
+    self._pending = deque()
+    self._lock = Lock()
+    logger.info("Woodpecker initialized.")
+
+  def on_ci_event(self, event: WoodpeckerEvent):
+    logger.info(f"Received event: {event}")
+    with self._lock:
+      if len(self._pending) > 0 or self._runner is not None:
+        self._pending.append(event)
+        return
+      self._start_runner(event)
+
+  def _start_runner(self, event: WoodpeckerEvent):
+    with self._lock:
+      self._runner = WoodpeckerRunner(self._git, self._docker, self._mo,
+                                      self._on_runner_completed, self._on_runner_error)
+      self._runner.process_event(event)
+
+  def _on_runner_completed(self):
+    logger.info("Runner completed.")
+    self._runner.join()
+    with self._lock:
+      self._runner = None
+      if len(self._pending) > 0:
+        event = self._pending.popleft()
+        self._start_runner(event)
+
+  def _on_runner_error(self, t: Exception):
+    logger.error(f"Runner error: {t}", exc_info=True)
+    self._runner.join()
+    with self._lock:
+      self._runner = None
+      if len(self._pending) > 0:
+        event = self._pending.popleft()
+        self._start_runner(event)

app/main.py

@@ -4,18 +4,18 @@ from fastapi import FastAPI
 from injectable import load_injection_container
 
 from app.config import get_settings
-from app.core.core import WebhookProcessor
+from app.util.logging import HandlerFactory
-from app.util.logging import LoggingHandler, ExternalLoggingHandler
 
 
 class KarlApplication:
   from starlette.types import Receive, Scope, Send
   def __init__(self) -> None:
     self._set_logging()
+    load_injection_container()
     _app = FastAPI(title="Karl", version="0.1.0")
+    self._set_middlewares(_app)
     self._set_routes(_app)
     self._set_events(_app)
-    self._init_services()
 
     self._app = _app
 
@@ -23,7 +23,12 @@ class KarlApplication:
     await self._app.__call__(scope, receive, send)
 
   def _set_logging(self):
-    logging.basicConfig(level=logging.INFO, handlers=[LoggingHandler()])
+    settings = get_settings()
+    logging.addLevelName(5, "TRACE")
+    logging.Logger.trace = lambda s, msg, *args, **kwargs: s.log(5, msg, *args, **kwargs)
+    logging.basicConfig(level=settings.logging.level,
+                        handlers=HandlerFactory.create(HandlerFactory.Target.ALL, handler_prefix='karl.',
+                                                       file_path=settings.logging.path))
 
     loggers = (
       "uvicorn",
@@ -33,12 +38,16 @@ class KarlApplication:
       "asyncio",
       "starlette",
     )
-    external_handler = ExternalLoggingHandler()
+    external_handlers = HandlerFactory.create(HandlerFactory.Target.ALL, file_path=settings.logging.path)
     for logger_name in loggers:
       logging_logger = logging.getLogger(logger_name)
-      logging_logger.handlers = [external_handler]
+      logging_logger.handlers = external_handlers
       logging_logger.propagate = False
 
+  def _set_middlewares(self, app: FastAPI):
+    from app.web.middlewares import LoggingMiddleware
+    app.add_middleware(LoggingMiddleware)
+
   def _set_routes(self, app: FastAPI):
     from app.core.router import router as core_router
     app.include_router(core_router)
@@ -49,14 +58,8 @@ class KarlApplication:
   def _set_events(self, app: FastAPI):
     pass
 
-  def _init_services(self):
-    logger = logging.getLogger(__name__)
-    load_injection_container()
-    webhook_service = WebhookProcessor()
-    logger.info(webhook_service.health)
 
-
+def run():
-def app():
   return KarlApplication()
 
 
@@ -65,7 +68,7 @@ if __name__ == "__main__":
 
   settings = get_settings()
   uvicorn.run(
-    "app.main:app",
+    "app.main:run",
     factory=True,
     host=settings.app.host,
     port=settings.app.port,

app/model/webhook.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import List
+
+@dataclass
+class WoodpeckerEvent:
+  _id: str
+  commit: str
+  message: str
+  started: str
+  files: List[str]

app/services/containers.py

@@ -1,14 +1,20 @@
+import logging
+from pathlib import Path
+
 import docker
 from docker.models.containers import Container
 from injectable import injectable
 
 from app.model.containers import Tree, Compose, SimpleContainer
 
+logger = logging.getLogger(__name__)
+
 
 @injectable(singleton=True)
 class DockerService:
   def __init__(self):
     self._client = docker.from_env()
+    # logger.info(f"Docker client initialized. Plugins: {self._client.plugins()}")
     self._tree = self._init_tree()
 
   def _init_tree(self) -> Tree:
@@ -28,3 +34,8 @@ class DockerService:
   @property
   def tree(self) -> Tree:
     return self._tree
+
+  def reload(self, compose_path: Path):
+    cmd = ["sudo", "docker", "compose", "-f", str(compose_path), "up", "-d"]
+    # TODO: subprocess
+

app/services/mo.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+from string import Template
+from typing import Annotated
+
+from injectable import injectable, autowired, Autowired
+
+from app.services import Passwords
+
+
+class SimpleValueTemplate(Template):
+  # Pozwala na kropki w nazwach placeholderów, np. ${user.name.first}
+  idpattern = r'[_a-zA-Z][_a-zA-Z0-9.]*'
+
+
+class ComplexValueTemplate(SimpleValueTemplate):
+  delimiter = '@'
+
+
+@injectable
+class Mo:
+  @autowired
+  def __init__(self, passwords: Annotated[Passwords, Autowired]):
+    self._passwords = passwords
+
+  def process(self, mo_file: Path):
+    raw = ''
+    with open(mo_file, "r") as mo:
+      raw = mo.read()
+    cmp = ComplexValueTemplate(raw)
+    rendered = cmp.substitute(self._passwords.get_values(cmp.get_identifiers()))
+    smp = SimpleValueTemplate(rendered)
+    ids = [_id + '.password' for _id in smp.get_identifiers()]
+    mappings = {k.replace('.password', ''): v for k, v in self._passwords.get_values(ids).items()}
+    rendered = smp.substitute(mappings)
+    de_mo_ified = str(mo_file).replace(".mo", "")
+    with open(de_mo_ified, "w") as mo:
+      mo.write(rendered)

app/services/passwords.py

@@ -1,7 +1,8 @@
 import os.path
+import shutil
 
 from injectable import injectable
-from pykeepass import PyKeePass, create_database, Group
+from pykeepass import PyKeePass, create_database
 
 
 @injectable(singleton=True)
@@ -11,30 +12,49 @@ class Passwords:
     settings = get_settings()
 
     with open(settings.kp.secret, "r") as fh:
-      secret = fh.read()
+      secret = fh.read().splitlines()[0]
-
+    self._path = settings.kp.file
-    self._kp_org = self.__get_or_create_store(settings.kp.file, secret)
+    self._kp_org = self._open_or_create(self._path, secret)
-    self._kp = self.__get_lock(settings.kp.file, secret)
+    self._kp = self._open_lock(self._path, secret)
 
   @staticmethod
-  def __get_or_create_store(path, passwd) -> PyKeePass:
+  def _open_or_create(path, password) -> PyKeePass:
     if os.path.exists(path):
-      return PyKeePass(
+      return PyKeePass(path, password=password)
-        path,
+    return create_database(path, password)
-        password=passwd,
-      )
-    return create_database(path, passwd)
 
   @staticmethod
-  def __get_lock(path, passwd) -> PyKeePass:
+  def _open_lock(path, password) -> PyKeePass:
     lock_path = path + ".lock"
-    import shutil
     shutil.copyfile(path, lock_path)
-    return Passwords.__get_or_create_store(lock_path, passwd)
+    return Passwords._open_or_create(lock_path, password)
 
-  @property
+  def get_values(self, keys: list[str]) -> dict[str, str]:
-  def store(self):
+    output = {}
-    return self._kp.root_group
+    for k in keys:
+      key_parts = k.split(".")
+      path = key_parts[:-1] if len(key_parts) > 2 else None
+      entry_name = key_parts[-2]
+      field_name = key_parts[-1]
+      kp_entry = self._kp_org.find_entries(path=path, first=True, title=entry_name)
+      output[k] = self._get_field_value(kp_entry, field_name)
+    return output
 
-  def save(self, group: Group):
+  @staticmethod
-    pass
+  def _get_field_value(kp_entry, field_name):
+    if kp_entry is None:
+      return None
+    match field_name:
+      case "username":
+        return kp_entry.username
+      case "password":
+        return kp_entry.password
+      case "url":
+        return kp_entry.url
+      case _:
+        return kp_entry.get_custom_property(field_name)
+
+  def save(self):
+    # nadpisz plik źródłowy zmianami z lock
+    self._kp.save()
+    shutil.copyfile(self._path + ".lock", self._path)

app/services/vcs.py

@@ -13,15 +13,11 @@ class GitService:
       self._repo.git.checkout(self._settings.git.branch)
     self._origin: Remote = self._repo.remotes.origin
 
-
-  def get_modified_compose(self) -> str | None:
-    self._update()
-    return self._diff()
-
   @staticmethod
   def _check_preconditions(config: GitConfig) -> Repo:
     def clone():
       return Repo.clone_from(config.url, config.path, branch=config.branch)
+
     import os
     if not config.path.exists():
       return clone()
@@ -30,16 +26,6 @@ class GitService:
       return clone()
     return Repo(config.path)
 
-  def _update(self):
+  def checkout(self, sha: str):
-    self._origin.pull()
+    self._origin.fetch()
-
+    self._repo.git.checkout(sha)
-  def _diff(self) -> str | None:
-    diff = self._repo.head.commit.diff("HEAD~1")
-    composes = [f for f in diff if f.a_path.endswith("docker-compose.yml")]
-    match len(composes):
-      case 0:
-        return None
-      case 1:
-        return composes[0].a_path
-      case _:
-        raise Exception("Multiple compose files modified")

app/util/logging.py

@@ -1,4 +1,8 @@
-from logging import Formatter, StreamHandler
+from enum import Enum, auto
+from logging import Formatter, StreamHandler, Handler
+from logging.handlers import TimedRotatingFileHandler
+from pathlib import Path
+from typing import List
 
 
 class NamingCache:
@@ -37,7 +41,7 @@ class ApplicationFormatter(Formatter):
     from datetime import datetime
     timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds')
     level = record.levelname.replace('WARNING', 'WARN').rjust(5)
-    thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache?
+    thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:]  # TODO: NamingCache?
     logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"]
     message = record.getMessage()
     formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}"
@@ -48,12 +52,35 @@ class ApplicationFormatter(Formatter):
     return formatted
 
 
-class LoggingHandler(StreamHandler):
+class HandlerFactory:
-  def __init__(self):
+  class Target(Enum):
-    super().__init__()
+    CONSOLE = auto()
-    self.setFormatter(ApplicationFormatter(handler_prefix='karl.'))
+    FILE = auto()
+    ALL = auto()
 
-class ExternalLoggingHandler(StreamHandler):
+  @staticmethod
-  def __init__(self):
+  def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]:
-    super().__init__()
+    def console_handler(prefix: str = ''):
-    self.setFormatter(ApplicationFormatter())
+      handler = StreamHandler()
+      handler.setFormatter(ApplicationFormatter(prefix))
+      handler.setLevel('INFO')
+      return handler
+
+    def file_handler(prefix: str = ''):
+      handler = TimedRotatingFileHandler(file_path, when='midnight', backupCount=30)
+      handler.setFormatter(ApplicationFormatter(prefix))
+      handler.setLevel('TRACE')
+      return handler
+
+    handlers = []
+    match target:
+      case HandlerFactory.Target.CONSOLE:
+        handlers.append(console_handler(handler_prefix))
+      case HandlerFactory.Target.FILE:
+        handlers.append(file_handler(handler_prefix))
+      case HandlerFactory.Target.ALL:
+        handlers.append(file_handler(handler_prefix))
+        handlers.append(console_handler(handler_prefix))
+      case _:
+        raise ValueError(f"Unknown target: {target}")
+    return handlers

app/web/middlewares.py

@@ -0,0 +1,24 @@
+import logging
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+
+logger = logging.getLogger(__name__)
+
+
+class LoggingMiddleware(BaseHTTPMiddleware):
+  async def dispatch(self, request: Request, call_next):
+    client = f"{request.client.host}:{request.client.port}"
+    match request.method:
+      case "POST" | "PUT" | "DELETE" if request.headers.get("Content-Type") == "application/json":
+        body = await request.body()
+        logger.trace(f"Request from {client}: {body.decode()}")
+      case "GET":
+        logger.trace(f"Request from {client}")
+      case _:
+        logger.trace(f"Request from {client} (content-type:{request.headers.get("Content-Type")})")
+
+    response = await call_next(request)
+    logger.trace(f"Respone: {response.status_code} {type(response)}")
+
+    return response

config/config.yaml

@@ -1,7 +1,10 @@
+logging:
+    level: "TRACE"
+    path: "logs/karl.log"
 app:
     host: "127.0.0.1"
     port: 8081
-    reload: true
+    reload: false
 git:
     path: "F:/IdeaProjects/paas/karl/.compose_repository"
     branch: "main"

pyproject.toml

@@ -15,6 +15,8 @@ dependencies = [
     "pykeepass>=4.1.1.post1",
     "docker>=7.1.0",
     "injectable==4.0.1",
+    "py-automapper>=2.2.0",
+    "fastapi-utils>=0.8.0",
 ]
 
 [project.optional-dependencies]

run.sh

@@ -1 +1 @@
-uvicorn app.main:app --factory --reload
+uvicorn app.main:run --factory --reload

tests/files/test1/test.mo.yaml

@@ -0,0 +1,3 @@
+value: ${sample}
+nested: ${some.nested.value}
+custom: @{custom.field}

tests/test_mo.py

@@ -0,0 +1,23 @@
+import os
+from pathlib import Path
+from unittest import TestCase
+
+import yaml
+
+from app.services import Passwords
+from app.services.mo import Mo
+
+
+class TestMo(TestCase):
+  def test_process(self):
+    target_path = Path('tests/files/test1/test.yaml')
+    mo = Mo(Passwords())
+    mo.process(Path('tests/files/test1/test.mo.yaml').absolute())
+    self.assertTrue(os.path.exists(target_path))
+    with open(target_path, 'r') as f:
+      content = f.read()
+    self.assertFalse(content.__contains__('${'))
+    parsed = yaml.load(content, Loader=yaml.FullLoader)
+    self.assertEqual('some_pass', parsed['value'])
+    self.assertEqual('nested_pass', parsed['nested'])
+    self.assertEqual('custom_content', parsed['custom'])