diff --git a/.gitignore b/.gitignore index 52364cd..06c7873 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,6 @@ uv.lock **/*.kdbx* .compose_repository -__pycache__/ +deployment/ **/dist/ **/*.log diff --git a/src/karl/api/v1.py b/src/karl/api/v1.py index de85719..8a529ab 100644 --- a/src/karl/api/v1.py +++ b/src/karl/api/v1.py @@ -8,8 +8,7 @@ from starlette.responses import JSONResponse, Response from karl.api.models import Request from karl.core.injects import AutowireSupport -from karl.core.woodpecker import Woodpecker -from karl.model.webhook import WoodpeckerEvent +from karl.model.webhook import WoodpeckerEvent, ReloadEvent router = APIRouter() logger = logging.getLogger(__name__) @@ -21,7 +20,6 @@ async def root(): @cbv(router) class APIv1: - woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) bus: EventBus = Depends(AutowireSupport.bus) def __init__(self): @@ -39,3 +37,10 @@ class APIv1: async def ci(self, request: Request): await self.bus.dispatch(mapper.map(request)) return Response(status_code=201) + + @router.get("/reload", summary="Manual service reload") + async def reload(self, service: str = None) -> Response: + if service is None: + return Response(status_code=400) + await self.bus.dispatch(ReloadEvent(service=service)) + return Response(status_code=201) diff --git a/src/karl/config/settings.py b/src/karl/config/settings.py index 79e66c9..8e20b51 100644 --- a/src/karl/config/settings.py +++ b/src/karl/config/settings.py @@ -51,4 +51,8 @@ class Settings(BaseSettings): @lru_cache def get_settings() -> Settings: - return Settings.from_yaml() + paths = ['deployment/config.yaml', 'config/config.yaml'] + for path in paths: + if Path(path).exists(): + return Settings.from_yaml(path) + raise Exception("Config file not found") diff --git a/src/karl/core/injects.py b/src/karl/core/injects.py index 34a6280..b52451b 100644 --- a/src/karl/core/injects.py +++ b/src/karl/core/injects.py @@ -1,6 +1,7 @@ from bubus import EventBus from injectable import inject +from karl.core.reload import ReloadService from karl.core.woodpecker import Woodpecker @@ -10,6 +11,10 @@ class AutowireSupport: def woodpecker(): return inject(Woodpecker) + @staticmethod + def reload(): + return inject(ReloadService) + @staticmethod def bus(): return inject(EventBus) diff --git a/src/karl/core/reload.py b/src/karl/core/reload.py new file mode 100644 index 0000000..f3975c4 --- /dev/null +++ b/src/karl/core/reload.py @@ -0,0 +1,48 @@ +import logging +from datetime import datetime +from pathlib import Path +from typing import Annotated + +from bubus import EventBus +from injectable import injectable, autowired, Autowired + +from karl import get_settings +from model.webhook import ReloadEvent, WoodpeckerEvent +from services import GitService + +logger = logging.getLogger(__name__) + + +@injectable(singleton=True) +class ReloadService: + + @autowired + def __init__(self, bus: Annotated[EventBus, Autowired]): + self._bus = bus + self._git = GitService() + bus.on(ReloadEvent, self.on_reload) + self.root_path = get_settings().git.path + logger.info("ReloadService initialized.") + + async def on_reload(self, event: ReloadEvent): + try: + logger.info(f"Received ReloadEvent: {event.service}") + head = self._git.get_head() + file_path = Path(self.root_path) / f"files/{event.service}" + if not file_path.exists(): + raise Exception(f"Service {event.service} not found: {file_path.absolute()} does not exist.") + logger.debug(f"Found service files at {file_path}: {', '.join([str(f) for f in list(file_path.iterdir())])}") + mos = list(file_path.glob('*.mo.*')) + logger.debug(f"Found {len(mos)} .mo files") + we = WoodpeckerEvent( + _id=-1, + commit=head.sha, + ref=head.branch, + message=f"Manual reload of {event.service}", + started=int(datetime.now().timestamp()), + files=[f"compose/{event.service}/docker-compose.yml"] + [str(pp).replace(str(self.root_path), '') for pp in mos] + ) + logger.debug(f"Sending str | None: + def get_service(files: list[str]) -> list[str]: supported_files = [] for f in files: f_parts = f.split("/") if f_parts[0] in ["compose", "files"]: supported_files.append(f_parts[1]) - match len(set(supported_files)): - case 0: - return None - case 1: - return supported_files[0] - case _: - logger.error(f"Multiple services were provided: {', '.join(supported_files)}") - raise Exception("Multiple services are not supported.") + # TODO: check service priorities (NGINX last) + return list(dict.fromkeys(supported_files)) @injectable(singleton=True) diff --git a/src/karl/main.py b/src/karl/main.py index fc75ec3..25029ac 100644 --- a/src/karl/main.py +++ b/src/karl/main.py @@ -3,6 +3,7 @@ import logging from fastapi import FastAPI from injectable import load_injection_container +from core.injects import AutowireSupport from karl.config import get_settings from karl.util.logging import HandlerFactory @@ -15,7 +16,7 @@ class KarlApplication: _app = FastAPI(title="Karl", version="0.1.0") self._set_middlewares(_app) self._set_routes(_app) - self._set_events(_app) + self._init_services(_app) self._app = _app @@ -55,8 +56,9 @@ class KarlApplication: app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) pass - def _set_events(self, app: FastAPI): - pass + def _init_services(self, app: FastAPI): + AutowireSupport.reload() + AutowireSupport.woodpecker() def run(): diff --git a/src/karl/model/vcs.py b/src/karl/model/vcs.py new file mode 100644 index 0000000..80800bc --- /dev/null +++ b/src/karl/model/vcs.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass +class Head: + sha: str + branch: str diff --git a/src/karl/model/webhook.py b/src/karl/model/webhook.py index 6c941ae..5b44d46 100644 --- a/src/karl/model/webhook.py +++ b/src/karl/model/webhook.py @@ -10,3 +10,6 @@ class WoodpeckerEvent(BaseEvent): message: str started: int files: List[str] + +class ReloadEvent(BaseEvent): + service: str diff --git a/src/karl/services/containers.py b/src/karl/services/containers.py index cc7cf56..dce9c81 100644 --- a/src/karl/services/containers.py +++ b/src/karl/services/containers.py @@ -15,6 +15,8 @@ class DockerService: def reload(self, compose_path: Path): os.chdir(compose_path.parent) - self._client.compose.ps() - self._client.compose.down(remove_orphans=True) - self._client.compose.up() + # TODO: Check if container needs to be reloaded from scratch + # if len(self._client.compose.ps()) > 0: + # self._client.compose.restart() + # return + self._client.compose.up(detach=True) diff --git a/src/karl/services/vcs.py b/src/karl/services/vcs.py index e89836d..0d70139 100644 --- a/src/karl/services/vcs.py +++ b/src/karl/services/vcs.py @@ -2,6 +2,7 @@ from git import Repo, Remote from injectable import injectable from karl.config import GitConfig, get_settings +from model.vcs import Head @injectable(singleton=True) @@ -27,3 +28,11 @@ class GitService: def checkout(self, sha: str): self._origin.fetch() self._repo.git.checkout(sha) + + def get_head(self) -> Head: + if self._repo.head.is_detached: + return Head(self._repo.head.object.hexsha, "detached") + return Head( + self._repo.active_branch.commit.hexsha, + self._repo.active_branch.name + ) diff --git a/src/karl/util/logging.py b/src/karl/util/logging.py index f25ed4e..27a2423 100644 --- a/src/karl/util/logging.py +++ b/src/karl/util/logging.py @@ -32,19 +32,23 @@ class NamingCache: class ApplicationFormatter(Formatter): - def __init__(self, handler_prefix: str = ''): + def __init__(self, handler_prefix: str = '', include_timestamp: bool = True): super().__init__() self._logger_names = NamingCache() self._handler_prefix = handler_prefix + self._include_timestamp = include_timestamp def format(self, record): from datetime import datetime - timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') + if self._include_timestamp: + timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') + ' ' + else: + timestamp = '' level = record.levelname.replace('WARNING', 'WARN').rjust(5) thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"] message = record.getMessage() - formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}" + formatted = f"{timestamp}{level} [{thread_name}] {logger_name} : {message}" if record.exc_info: formatted += "\n" + self.formatException(record.exc_info) @@ -62,7 +66,7 @@ class HandlerFactory: def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]: def console_handler(prefix: str = ''): handler = StreamHandler() - handler.setFormatter(ApplicationFormatter(prefix)) + handler.setFormatter(ApplicationFormatter(prefix, include_timestamp=False)) handler.setLevel('INFO') return handler diff --git a/tests/test_woodpecker.py b/tests/test_woodpecker.py index 7521d88..9bb4616 100644 --- a/tests/test_woodpecker.py +++ b/tests/test_woodpecker.py @@ -4,8 +4,13 @@ files = [".gitignore", "compose/nginx/docker-compose.yaml", "config/heimdall.kdb "files/nginx/cert/key.mo.pem", "files/nginx/conf.d/default.conf", "files/nginx/conf.d/forge.conf", "files/nginx/conf.d/mqtt.conf", "files/nginx/conf.d/nextcloud.conf", "files/nginx/conf.d/zitadel.conf", "files/nginx/nginx.conf"] +files2 = [".gitignore", "compose/nginx/docker-compose.yaml", "compose/nginx2/docker-compose.yaml"] -def test_get_service(): +def test_get_service_single(): service = woodpecker.WoodpeckerRunner.get_service(files) - assert service == 'nginx' + assert service == ['nginx'] + +def test_get_service_multi(): + services = woodpecker.WoodpeckerRunner.get_service(files2) + assert services == ['nginx', 'nginx2']