diff --git a/.gitignore b/.gitignore index 52364cd..c6460ce 100644 --- a/.gitignore +++ b/.gitignore @@ -10,10 +10,9 @@ wheels/ .venv .idea *.iml -uv.lock -**/*.kdbx* +**/*.lock .compose_repository -__pycache__/ +deployment/ **/dist/ **/*.log diff --git a/.woodpecker/dev.yaml b/.woodpecker/dev.yaml index f837559..5f86119 100644 --- a/.woodpecker/dev.yaml +++ b/.woodpecker/dev.yaml @@ -13,3 +13,5 @@ steps: - event: pull_request evaluate: 'CI_COMMIT_PULL_REQUEST_LABELS contains "ci-ready"' - event: manual +depends_on: + - name: python diff --git a/.woodpecker/latest.yaml b/.woodpecker/latest.yaml index 8309154..d43e184 100644 --- a/.woodpecker/latest.yaml +++ b/.woodpecker/latest.yaml @@ -12,3 +12,5 @@ steps: when: - event: [ tag, push, manual ] branch: master +depends_on: + - name: python diff --git a/.woodpecker/python.yaml b/.woodpecker/python.yaml new file mode 100644 index 0000000..ed0d43d --- /dev/null +++ b/.woodpecker/python.yaml @@ -0,0 +1,24 @@ +steps: + - name: prepare + image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine + commands: + - apk add --no-cache gcc python3-dev musl-dev linux-headers + - uv sync + - name: test + image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine + commands: + - uv sync --group dev + - uv run pytest + - name: test-run + image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine + commands: + - | + python src/karl/__init__.py 2>&1 | while IFS= read -r line; do + echo "$line" + if echo "$line" | grep -q "Uvicorn running on"; then + pkill -f "uvicorn" + break + fi + done +when: + - event: [ push, manual ] diff --git a/config/kp.kdbx b/config/kp.kdbx new file mode 100644 index 0000000..1734db9 Binary files /dev/null and b/config/kp.kdbx differ diff --git a/src/karl/api/v1.py b/src/karl/api/v1.py index de85719..8a529ab 100644 --- a/src/karl/api/v1.py +++ b/src/karl/api/v1.py @@ -8,8 +8,7 @@ from starlette.responses import JSONResponse, Response from karl.api.models import Request from karl.core.injects import AutowireSupport -from karl.core.woodpecker import Woodpecker -from karl.model.webhook import WoodpeckerEvent +from karl.model.webhook import WoodpeckerEvent, ReloadEvent router = APIRouter() logger = logging.getLogger(__name__) @@ -21,7 +20,6 @@ async def root(): @cbv(router) class APIv1: - woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) bus: EventBus = Depends(AutowireSupport.bus) def __init__(self): @@ -39,3 +37,10 @@ class APIv1: async def ci(self, request: Request): await self.bus.dispatch(mapper.map(request)) return Response(status_code=201) + + @router.get("/reload", summary="Manual service reload") + async def reload(self, service: str = None) -> Response: + if service is None: + return Response(status_code=400) + await self.bus.dispatch(ReloadEvent(service=service)) + return Response(status_code=201) diff --git a/src/karl/config/settings.py b/src/karl/config/settings.py index 79e66c9..8e20b51 100644 --- a/src/karl/config/settings.py +++ b/src/karl/config/settings.py @@ -51,4 +51,8 @@ class Settings(BaseSettings): @lru_cache def get_settings() -> Settings: - return Settings.from_yaml() + paths = ['deployment/config.yaml', 'config/config.yaml'] + for path in paths: + if Path(path).exists(): + return Settings.from_yaml(path) + raise Exception("Config file not found") diff --git a/src/karl/core/injects.py b/src/karl/core/injects.py index 34a6280..b52451b 100644 --- a/src/karl/core/injects.py +++ b/src/karl/core/injects.py @@ -1,6 +1,7 @@ from bubus import EventBus from injectable import inject +from karl.core.reload import ReloadService from karl.core.woodpecker import Woodpecker @@ -10,6 +11,10 @@ class AutowireSupport: def woodpecker(): return inject(Woodpecker) + @staticmethod + def reload(): + return inject(ReloadService) + @staticmethod def bus(): return inject(EventBus) diff --git a/src/karl/core/reload.py b/src/karl/core/reload.py new file mode 100644 index 0000000..f3975c4 --- /dev/null +++ b/src/karl/core/reload.py @@ -0,0 +1,48 @@ +import logging +from datetime import datetime +from pathlib import Path +from typing import Annotated + +from bubus import EventBus +from injectable import injectable, autowired, Autowired + +from karl import get_settings +from model.webhook import ReloadEvent, WoodpeckerEvent +from services import GitService + +logger = logging.getLogger(__name__) + + +@injectable(singleton=True) +class ReloadService: + + @autowired + def __init__(self, bus: Annotated[EventBus, Autowired]): + self._bus = bus + self._git = GitService() + bus.on(ReloadEvent, self.on_reload) + self.root_path = get_settings().git.path + logger.info("ReloadService initialized.") + + async def on_reload(self, event: ReloadEvent): + try: + logger.info(f"Received ReloadEvent: {event.service}") + head = self._git.get_head() + file_path = Path(self.root_path) / f"files/{event.service}" + if not file_path.exists(): + raise Exception(f"Service {event.service} not found: {file_path.absolute()} does not exist.") + logger.debug(f"Found service files at {file_path}: {', '.join([str(f) for f in list(file_path.iterdir())])}") + mos = list(file_path.glob('*.mo.*')) + logger.debug(f"Found {len(mos)} .mo files") + we = WoodpeckerEvent( + _id=-1, + commit=head.sha, + ref=head.branch, + message=f"Manual reload of {event.service}", + started=int(datetime.now().timestamp()), + files=[f"compose/{event.service}/docker-compose.yml"] + [str(pp).replace(str(self.root_path), '') for pp in mos] + ) + logger.debug(f"Sending str | None: + def get_service(files: list[str]) -> list[str]: supported_files = [] for f in files: f_parts = f.split("/") if f_parts[0] in ["compose", "files"]: supported_files.append(f_parts[1]) - match len(set(supported_files)): - case 0: - return None - case 1: - return supported_files[0] - case _: - logger.error(f"Multiple services were provided: {', '.join(supported_files)}") - raise Exception("Multiple services are not supported.") + # TODO: check service priorities (NGINX last) + return list(dict.fromkeys(supported_files)) @injectable(singleton=True) @@ -94,7 +97,7 @@ class Woodpecker: logger.info("Woodpecker initialized.") async def on_ci_event(self, event: WoodpeckerEvent): - logger.debug(f"Received WoodpeckerEvent: {event.event_id}") + logger.info(f"Received WoodpeckerEvent: {event.event_id}") with self._lock: logger.debug("Lock acquired [on-ci-event]") if len(self._pending) > 0 or self._runner is not None: diff --git a/src/karl/main.py b/src/karl/main.py index fc75ec3..25029ac 100644 --- a/src/karl/main.py +++ b/src/karl/main.py @@ -3,6 +3,7 @@ import logging from fastapi import FastAPI from injectable import load_injection_container +from core.injects import AutowireSupport from karl.config import get_settings from karl.util.logging import HandlerFactory @@ -15,7 +16,7 @@ class KarlApplication: _app = FastAPI(title="Karl", version="0.1.0") self._set_middlewares(_app) self._set_routes(_app) - self._set_events(_app) + self._init_services(_app) self._app = _app @@ -55,8 +56,9 @@ class KarlApplication: app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) pass - def _set_events(self, app: FastAPI): - pass + def _init_services(self, app: FastAPI): + AutowireSupport.reload() + AutowireSupport.woodpecker() def run(): diff --git a/src/karl/model/vcs.py b/src/karl/model/vcs.py new file mode 100644 index 0000000..80800bc --- /dev/null +++ b/src/karl/model/vcs.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass +class Head: + sha: str + branch: str diff --git a/src/karl/model/webhook.py b/src/karl/model/webhook.py index 6c941ae..5b44d46 100644 --- a/src/karl/model/webhook.py +++ b/src/karl/model/webhook.py @@ -10,3 +10,6 @@ class WoodpeckerEvent(BaseEvent): message: str started: int files: List[str] + +class ReloadEvent(BaseEvent): + service: str diff --git a/src/karl/services/containers.py b/src/karl/services/containers.py index cc7cf56..dce9c81 100644 --- a/src/karl/services/containers.py +++ b/src/karl/services/containers.py @@ -15,6 +15,8 @@ class DockerService: def reload(self, compose_path: Path): os.chdir(compose_path.parent) - self._client.compose.ps() - self._client.compose.down(remove_orphans=True) - self._client.compose.up() + # TODO: Check if container needs to be reloaded from scratch + # if len(self._client.compose.ps()) > 0: + # self._client.compose.restart() + # return + self._client.compose.up(detach=True) diff --git a/src/karl/services/mo.py b/src/karl/services/mo.py index c063d62..e0b1388 100644 --- a/src/karl/services/mo.py +++ b/src/karl/services/mo.py @@ -9,7 +9,7 @@ from karl.services import Passwords class ValueTemplate(Template): # Pozwala na kropki i ukośniki w nazwach placeholderów, np. ${user.name/first} - idpattern = r'[_a-zA-Z][_a-zA-Z0-9.\/]*' + idpattern = r'[_a-zA-Z][_a-zA-Z0-9.\/\-]*' @injectable diff --git a/src/karl/services/passwords.py b/src/karl/services/passwords.py index bf2cda0..aafcf52 100644 --- a/src/karl/services/passwords.py +++ b/src/karl/services/passwords.py @@ -6,6 +6,10 @@ import keyring from injectable import injectable from pykeepass import PyKeePass, create_database +import logging + +logger = logging.getLogger(__name__) + class KeyRequest: def __init__(self, prompt: str): self.field_name = None @@ -14,7 +18,8 @@ class KeyRequest: self._parse_prompt(prompt) def _parse_prompt(self, prompt: str): - prompt_parts = prompt.split(".") + logger.debug(f"Got prompt: {prompt}") + prompt_parts = prompt.replace('\n', '').split(".") key = None match len(prompt_parts): case 1: @@ -26,6 +31,7 @@ class KeyRequest: case _: key = None if key is None: + logger.warning(f"Prompt {prompt} cannot be parsed") return key_parts = key.split("/") self.path = key_parts[:] if len(key_parts) > 1 else None @@ -39,9 +45,13 @@ class Passwords: import keyring_backend keyring.set_keyring(keyring=keyring_backend.Backend()) + logger.info(f"Reading secret file at {settings.kp.secret}") with open(settings.kp.secret, "r") as fh: - keyring.set_password("karl", "kp", fh.read().splitlines()[0]) + keyring.set_password("karl", "kp", fh.read().splitlines()[0].replace('\r', '').strip()) + logger.info(f"Database file: {settings.kp.file}") self._path = settings.kp.file + if not os.path.exists(self._path): + logger.warning(f"Database file {self._path} does not exist") @contextmanager def open(self, mode: str = "r") -> Generator[PyKeePass | Any, Any, None]: @@ -57,6 +67,8 @@ class Passwords: request = KeyRequest(k) with self.open() as kp: kp_entry = kp.find_entries(path=request.path, first=True, title=request.entry_name) + if kp_entry is None: + logger.warning(f"No value found for key {k}") output[k] = self._get_field_value(kp_entry, request.field_name) return output diff --git a/src/karl/services/vcs.py b/src/karl/services/vcs.py index e89836d..c47a7ce 100644 --- a/src/karl/services/vcs.py +++ b/src/karl/services/vcs.py @@ -1,7 +1,25 @@ -from git import Repo, Remote +try: + from git import Repo, Remote +except ImportError: + import sys + + sys.stderr.write("Cannot import Git\n") + sys.stderr.flush() + + + class Repo: + @staticmethod + def clone_from(*args): + raise RuntimeError("git lib not imported") + + + class Remote: + pass + from injectable import injectable from karl.config import GitConfig, get_settings +from karl.model.vcs import Head @injectable(singleton=True) @@ -27,3 +45,11 @@ class GitService: def checkout(self, sha: str): self._origin.fetch() self._repo.git.checkout(sha) + + def get_head(self) -> Head: + if self._repo.head.is_detached: + return Head(self._repo.head.object.hexsha, "detached") + return Head( + self._repo.active_branch.commit.hexsha, + self._repo.active_branch.name + ) diff --git a/src/karl/util/logging.py b/src/karl/util/logging.py index f25ed4e..27a2423 100644 --- a/src/karl/util/logging.py +++ b/src/karl/util/logging.py @@ -32,19 +32,23 @@ class NamingCache: class ApplicationFormatter(Formatter): - def __init__(self, handler_prefix: str = ''): + def __init__(self, handler_prefix: str = '', include_timestamp: bool = True): super().__init__() self._logger_names = NamingCache() self._handler_prefix = handler_prefix + self._include_timestamp = include_timestamp def format(self, record): from datetime import datetime - timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') + if self._include_timestamp: + timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') + ' ' + else: + timestamp = '' level = record.levelname.replace('WARNING', 'WARN').rjust(5) thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"] message = record.getMessage() - formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}" + formatted = f"{timestamp}{level} [{thread_name}] {logger_name} : {message}" if record.exc_info: formatted += "\n" + self.formatException(record.exc_info) @@ -62,7 +66,7 @@ class HandlerFactory: def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]: def console_handler(prefix: str = ''): handler = StreamHandler() - handler.setFormatter(ApplicationFormatter(prefix)) + handler.setFormatter(ApplicationFormatter(prefix, include_timestamp=False)) handler.setLevel('INFO') return handler diff --git a/tests/files/test1/test.mo.env b/tests/files/test1/test.mo.env new file mode 100644 index 0000000..cbdde2f --- /dev/null +++ b/tests/files/test1/test.mo.env @@ -0,0 +1,10 @@ +value=${sample} +nested=${some/nested/value} +mixed=${some/nested/value.username} +custom=${custom.field} + +uname=${sample.username} +invalid=${double.dot.example} + +#some comment +dashed=${some/dashed} diff --git a/tests/test_mo_env.py b/tests/test_mo_env.py new file mode 100644 index 0000000..5e245b9 --- /dev/null +++ b/tests/test_mo_env.py @@ -0,0 +1,63 @@ +from pathlib import Path + +import pytest + +from karl.services import Passwords +from karl.services.mo import Mo + + +@pytest.fixture(scope='class') +def target_path(): + p = Path('tests/files/test1/test.env') + # posprzątaj przed testem, gdyby plik istniał z poprzednich uruchomień + if p.exists(): + p.unlink() + yield p + # sprzątanie po teście + if p.exists(): + p.unlink() + + +@pytest.fixture(scope='class') +def test1_content(target_path: Path): + mo = Mo(Passwords()) + mo.process(Path('tests/files/test1/test.mo.env').absolute()) + + assert target_path.absolute().exists() + + content = target_path.read_text() + assert '${' not in content + + props = {} + prop_lines = content.split('\n') + for line in prop_lines: + if len(line) > 1 and not line.startswith("#"): + splitted = line.split('=') + props[splitted[0]] = splitted[1] + # props = {prop[0]:prop[1] for prop in content.split('\n')} + print(props) + yield props + + +class TestEnvParsing: + + def test_simple(self, test1_content: dict): + assert test1_content['value'] == 'some_pass' + + def test_nested(self, test1_content: dict): + assert test1_content['nested'] == 'nested_pass' + + def test_mixed(self, test1_content: dict): + assert test1_content['mixed'] == 'nested_username' + + def test_dashed(self, test1_content: dict): + assert test1_content['dashed'] == 'dd-value' + + def test_custom_field(self, test1_content: dict): + assert test1_content['custom'] == 'custom_content' + + def test_username_field(self, test1_content: dict): + assert test1_content['uname'] == 'sample_username' + + def test_invalid_key(self, test1_content: dict): + assert test1_content.get('invalid') == 'None' diff --git a/tests/test_woodpecker.py b/tests/test_woodpecker.py index 7521d88..f4104db 100644 --- a/tests/test_woodpecker.py +++ b/tests/test_woodpecker.py @@ -1,11 +1,16 @@ -from core import woodpecker +from karl.core import woodpecker files = [".gitignore", "compose/nginx/docker-compose.yaml", "config/heimdall.kdbx", "files/nginx/cert/fullchain.mo.pem", "files/nginx/cert/key.mo.pem", "files/nginx/conf.d/default.conf", "files/nginx/conf.d/forge.conf", "files/nginx/conf.d/mqtt.conf", "files/nginx/conf.d/nextcloud.conf", "files/nginx/conf.d/zitadel.conf", "files/nginx/nginx.conf"] +files2 = [".gitignore", "compose/nginx/docker-compose.yaml", "compose/nginx2/docker-compose.yaml"] -def test_get_service(): +def test_get_service_single(): service = woodpecker.WoodpeckerRunner.get_service(files) - assert service == 'nginx' + assert service == ['nginx'] + +def test_get_service_multi(): + services = woodpecker.WoodpeckerRunner.get_service(files2) + assert services == ['nginx', 'nginx2']