diff --git a/.gitignore b/.gitignore index c6460ce..52364cd 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,10 @@ wheels/ .venv .idea *.iml -**/*.lock +uv.lock +**/*.kdbx* .compose_repository -deployment/ +__pycache__/ **/dist/ **/*.log diff --git a/.woodpecker/dev.yaml b/.woodpecker/dev.yaml index 5f86119..f837559 100644 --- a/.woodpecker/dev.yaml +++ b/.woodpecker/dev.yaml @@ -13,5 +13,3 @@ steps: - event: pull_request evaluate: 'CI_COMMIT_PULL_REQUEST_LABELS contains "ci-ready"' - event: manual -depends_on: - - name: python diff --git a/.woodpecker/latest.yaml b/.woodpecker/latest.yaml index d43e184..8309154 100644 --- a/.woodpecker/latest.yaml +++ b/.woodpecker/latest.yaml @@ -12,5 +12,3 @@ steps: when: - event: [ tag, push, manual ] branch: master -depends_on: - - name: python diff --git a/.woodpecker/python.yaml b/.woodpecker/python.yaml deleted file mode 100644 index ed0d43d..0000000 --- a/.woodpecker/python.yaml +++ /dev/null @@ -1,24 +0,0 @@ -steps: - - name: prepare - image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine - commands: - - apk add --no-cache gcc python3-dev musl-dev linux-headers - - uv sync - - name: test - image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine - commands: - - uv sync --group dev - - uv run pytest - - name: test-run - image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine - commands: - - | - python src/karl/__init__.py 2>&1 | while IFS= read -r line; do - echo "$line" - if echo "$line" | grep -q "Uvicorn running on"; then - pkill -f "uvicorn" - break - fi - done -when: - - event: [ push, manual ] diff --git a/config/kp.kdbx b/config/kp.kdbx deleted file mode 100644 index 1734db9..0000000 Binary files a/config/kp.kdbx and /dev/null differ diff --git a/src/karl/api/v1.py b/src/karl/api/v1.py index 8a529ab..de85719 100644 --- a/src/karl/api/v1.py +++ b/src/karl/api/v1.py @@ -8,7 +8,8 @@ from starlette.responses import JSONResponse, Response from karl.api.models import Request from karl.core.injects import AutowireSupport -from karl.model.webhook import WoodpeckerEvent, ReloadEvent +from karl.core.woodpecker import Woodpecker +from karl.model.webhook import WoodpeckerEvent router = APIRouter() logger = logging.getLogger(__name__) @@ -20,6 +21,7 @@ async def root(): @cbv(router) class APIv1: + woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker) bus: EventBus = Depends(AutowireSupport.bus) def __init__(self): @@ -37,10 +39,3 @@ class APIv1: async def ci(self, request: Request): await self.bus.dispatch(mapper.map(request)) return Response(status_code=201) - - @router.get("/reload", summary="Manual service reload") - async def reload(self, service: str = None) -> Response: - if service is None: - return Response(status_code=400) - await self.bus.dispatch(ReloadEvent(service=service)) - return Response(status_code=201) diff --git a/src/karl/config/settings.py b/src/karl/config/settings.py index 8e20b51..79e66c9 100644 --- a/src/karl/config/settings.py +++ b/src/karl/config/settings.py @@ -51,8 +51,4 @@ class Settings(BaseSettings): @lru_cache def get_settings() -> Settings: - paths = ['deployment/config.yaml', 'config/config.yaml'] - for path in paths: - if Path(path).exists(): - return Settings.from_yaml(path) - raise Exception("Config file not found") + return Settings.from_yaml() diff --git a/src/karl/core/injects.py b/src/karl/core/injects.py index b52451b..34a6280 100644 --- a/src/karl/core/injects.py +++ b/src/karl/core/injects.py @@ -1,7 +1,6 @@ from bubus import EventBus from injectable import inject -from karl.core.reload import ReloadService from karl.core.woodpecker import Woodpecker @@ -11,10 +10,6 @@ class AutowireSupport: def woodpecker(): return inject(Woodpecker) - @staticmethod - def reload(): - return inject(ReloadService) - @staticmethod def bus(): return inject(EventBus) diff --git a/src/karl/core/reload.py b/src/karl/core/reload.py deleted file mode 100644 index f3975c4..0000000 --- a/src/karl/core/reload.py +++ /dev/null @@ -1,48 +0,0 @@ -import logging -from datetime import datetime -from pathlib import Path -from typing import Annotated - -from bubus import EventBus -from injectable import injectable, autowired, Autowired - -from karl import get_settings -from model.webhook import ReloadEvent, WoodpeckerEvent -from services import GitService - -logger = logging.getLogger(__name__) - - -@injectable(singleton=True) -class ReloadService: - - @autowired - def __init__(self, bus: Annotated[EventBus, Autowired]): - self._bus = bus - self._git = GitService() - bus.on(ReloadEvent, self.on_reload) - self.root_path = get_settings().git.path - logger.info("ReloadService initialized.") - - async def on_reload(self, event: ReloadEvent): - try: - logger.info(f"Received ReloadEvent: {event.service}") - head = self._git.get_head() - file_path = Path(self.root_path) / f"files/{event.service}" - if not file_path.exists(): - raise Exception(f"Service {event.service} not found: {file_path.absolute()} does not exist.") - logger.debug(f"Found service files at {file_path}: {', '.join([str(f) for f in list(file_path.iterdir())])}") - mos = list(file_path.glob('*.mo.*')) - logger.debug(f"Found {len(mos)} .mo files") - we = WoodpeckerEvent( - _id=-1, - commit=head.sha, - ref=head.branch, - message=f"Manual reload of {event.service}", - started=int(datetime.now().timestamp()), - files=[f"compose/{event.service}/docker-compose.yml"] + [str(pp).replace(str(self.root_path), '') for pp in mos] - ) - logger.debug(f"Sending list[str]: + def get_service(files: list[str]) -> str | None: supported_files = [] for f in files: f_parts = f.split("/") if f_parts[0] in ["compose", "files"]: supported_files.append(f_parts[1]) - # TODO: check service priorities (NGINX last) - return list(dict.fromkeys(supported_files)) + match len(set(supported_files)): + case 0: + return None + case 1: + return supported_files[0] + case _: + logger.error(f"Multiple services were provided: {', '.join(supported_files)}") + raise Exception("Multiple services are not supported.") @injectable(singleton=True) @@ -97,7 +94,7 @@ class Woodpecker: logger.info("Woodpecker initialized.") async def on_ci_event(self, event: WoodpeckerEvent): - logger.info(f"Received WoodpeckerEvent: {event.event_id}") + logger.debug(f"Received WoodpeckerEvent: {event.event_id}") with self._lock: logger.debug("Lock acquired [on-ci-event]") if len(self._pending) > 0 or self._runner is not None: diff --git a/src/karl/main.py b/src/karl/main.py index 25029ac..fc75ec3 100644 --- a/src/karl/main.py +++ b/src/karl/main.py @@ -3,7 +3,6 @@ import logging from fastapi import FastAPI from injectable import load_injection_container -from core.injects import AutowireSupport from karl.config import get_settings from karl.util.logging import HandlerFactory @@ -16,7 +15,7 @@ class KarlApplication: _app = FastAPI(title="Karl", version="0.1.0") self._set_middlewares(_app) self._set_routes(_app) - self._init_services(_app) + self._set_events(_app) self._app = _app @@ -56,9 +55,8 @@ class KarlApplication: app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"]) pass - def _init_services(self, app: FastAPI): - AutowireSupport.reload() - AutowireSupport.woodpecker() + def _set_events(self, app: FastAPI): + pass def run(): diff --git a/src/karl/model/vcs.py b/src/karl/model/vcs.py deleted file mode 100644 index 80800bc..0000000 --- a/src/karl/model/vcs.py +++ /dev/null @@ -1,7 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class Head: - sha: str - branch: str diff --git a/src/karl/model/webhook.py b/src/karl/model/webhook.py index 5b44d46..6c941ae 100644 --- a/src/karl/model/webhook.py +++ b/src/karl/model/webhook.py @@ -10,6 +10,3 @@ class WoodpeckerEvent(BaseEvent): message: str started: int files: List[str] - -class ReloadEvent(BaseEvent): - service: str diff --git a/src/karl/services/containers.py b/src/karl/services/containers.py index dce9c81..cc7cf56 100644 --- a/src/karl/services/containers.py +++ b/src/karl/services/containers.py @@ -15,8 +15,6 @@ class DockerService: def reload(self, compose_path: Path): os.chdir(compose_path.parent) - # TODO: Check if container needs to be reloaded from scratch - # if len(self._client.compose.ps()) > 0: - # self._client.compose.restart() - # return - self._client.compose.up(detach=True) + self._client.compose.ps() + self._client.compose.down(remove_orphans=True) + self._client.compose.up() diff --git a/src/karl/services/mo.py b/src/karl/services/mo.py index e0b1388..c063d62 100644 --- a/src/karl/services/mo.py +++ b/src/karl/services/mo.py @@ -9,7 +9,7 @@ from karl.services import Passwords class ValueTemplate(Template): # Pozwala na kropki i ukośniki w nazwach placeholderów, np. ${user.name/first} - idpattern = r'[_a-zA-Z][_a-zA-Z0-9.\/\-]*' + idpattern = r'[_a-zA-Z][_a-zA-Z0-9.\/]*' @injectable diff --git a/src/karl/services/passwords.py b/src/karl/services/passwords.py index aafcf52..bf2cda0 100644 --- a/src/karl/services/passwords.py +++ b/src/karl/services/passwords.py @@ -6,10 +6,6 @@ import keyring from injectable import injectable from pykeepass import PyKeePass, create_database -import logging - -logger = logging.getLogger(__name__) - class KeyRequest: def __init__(self, prompt: str): self.field_name = None @@ -18,8 +14,7 @@ class KeyRequest: self._parse_prompt(prompt) def _parse_prompt(self, prompt: str): - logger.debug(f"Got prompt: {prompt}") - prompt_parts = prompt.replace('\n', '').split(".") + prompt_parts = prompt.split(".") key = None match len(prompt_parts): case 1: @@ -31,7 +26,6 @@ class KeyRequest: case _: key = None if key is None: - logger.warning(f"Prompt {prompt} cannot be parsed") return key_parts = key.split("/") self.path = key_parts[:] if len(key_parts) > 1 else None @@ -45,13 +39,9 @@ class Passwords: import keyring_backend keyring.set_keyring(keyring=keyring_backend.Backend()) - logger.info(f"Reading secret file at {settings.kp.secret}") with open(settings.kp.secret, "r") as fh: - keyring.set_password("karl", "kp", fh.read().splitlines()[0].replace('\r', '').strip()) - logger.info(f"Database file: {settings.kp.file}") + keyring.set_password("karl", "kp", fh.read().splitlines()[0]) self._path = settings.kp.file - if not os.path.exists(self._path): - logger.warning(f"Database file {self._path} does not exist") @contextmanager def open(self, mode: str = "r") -> Generator[PyKeePass | Any, Any, None]: @@ -67,8 +57,6 @@ class Passwords: request = KeyRequest(k) with self.open() as kp: kp_entry = kp.find_entries(path=request.path, first=True, title=request.entry_name) - if kp_entry is None: - logger.warning(f"No value found for key {k}") output[k] = self._get_field_value(kp_entry, request.field_name) return output diff --git a/src/karl/services/vcs.py b/src/karl/services/vcs.py index c47a7ce..e89836d 100644 --- a/src/karl/services/vcs.py +++ b/src/karl/services/vcs.py @@ -1,25 +1,7 @@ -try: - from git import Repo, Remote -except ImportError: - import sys - - sys.stderr.write("Cannot import Git\n") - sys.stderr.flush() - - - class Repo: - @staticmethod - def clone_from(*args): - raise RuntimeError("git lib not imported") - - - class Remote: - pass - +from git import Repo, Remote from injectable import injectable from karl.config import GitConfig, get_settings -from karl.model.vcs import Head @injectable(singleton=True) @@ -45,11 +27,3 @@ class GitService: def checkout(self, sha: str): self._origin.fetch() self._repo.git.checkout(sha) - - def get_head(self) -> Head: - if self._repo.head.is_detached: - return Head(self._repo.head.object.hexsha, "detached") - return Head( - self._repo.active_branch.commit.hexsha, - self._repo.active_branch.name - ) diff --git a/src/karl/util/logging.py b/src/karl/util/logging.py index 27a2423..f25ed4e 100644 --- a/src/karl/util/logging.py +++ b/src/karl/util/logging.py @@ -32,23 +32,19 @@ class NamingCache: class ApplicationFormatter(Formatter): - def __init__(self, handler_prefix: str = '', include_timestamp: bool = True): + def __init__(self, handler_prefix: str = ''): super().__init__() self._logger_names = NamingCache() self._handler_prefix = handler_prefix - self._include_timestamp = include_timestamp def format(self, record): from datetime import datetime - if self._include_timestamp: - timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') + ' ' - else: - timestamp = '' + timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') level = record.levelname.replace('WARNING', 'WARN').rjust(5) thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:] # TODO: NamingCache? logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"] message = record.getMessage() - formatted = f"{timestamp}{level} [{thread_name}] {logger_name} : {message}" + formatted = f"{timestamp} {level} [{thread_name}] {logger_name} : {message}" if record.exc_info: formatted += "\n" + self.formatException(record.exc_info) @@ -66,7 +62,7 @@ class HandlerFactory: def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]: def console_handler(prefix: str = ''): handler = StreamHandler() - handler.setFormatter(ApplicationFormatter(prefix, include_timestamp=False)) + handler.setFormatter(ApplicationFormatter(prefix)) handler.setLevel('INFO') return handler diff --git a/tests/files/test1/test.mo.env b/tests/files/test1/test.mo.env deleted file mode 100644 index cbdde2f..0000000 --- a/tests/files/test1/test.mo.env +++ /dev/null @@ -1,10 +0,0 @@ -value=${sample} -nested=${some/nested/value} -mixed=${some/nested/value.username} -custom=${custom.field} - -uname=${sample.username} -invalid=${double.dot.example} - -#some comment -dashed=${some/dashed} diff --git a/tests/test_mo_env.py b/tests/test_mo_env.py deleted file mode 100644 index 5e245b9..0000000 --- a/tests/test_mo_env.py +++ /dev/null @@ -1,63 +0,0 @@ -from pathlib import Path - -import pytest - -from karl.services import Passwords -from karl.services.mo import Mo - - -@pytest.fixture(scope='class') -def target_path(): - p = Path('tests/files/test1/test.env') - # posprzątaj przed testem, gdyby plik istniał z poprzednich uruchomień - if p.exists(): - p.unlink() - yield p - # sprzątanie po teście - if p.exists(): - p.unlink() - - -@pytest.fixture(scope='class') -def test1_content(target_path: Path): - mo = Mo(Passwords()) - mo.process(Path('tests/files/test1/test.mo.env').absolute()) - - assert target_path.absolute().exists() - - content = target_path.read_text() - assert '${' not in content - - props = {} - prop_lines = content.split('\n') - for line in prop_lines: - if len(line) > 1 and not line.startswith("#"): - splitted = line.split('=') - props[splitted[0]] = splitted[1] - # props = {prop[0]:prop[1] for prop in content.split('\n')} - print(props) - yield props - - -class TestEnvParsing: - - def test_simple(self, test1_content: dict): - assert test1_content['value'] == 'some_pass' - - def test_nested(self, test1_content: dict): - assert test1_content['nested'] == 'nested_pass' - - def test_mixed(self, test1_content: dict): - assert test1_content['mixed'] == 'nested_username' - - def test_dashed(self, test1_content: dict): - assert test1_content['dashed'] == 'dd-value' - - def test_custom_field(self, test1_content: dict): - assert test1_content['custom'] == 'custom_content' - - def test_username_field(self, test1_content: dict): - assert test1_content['uname'] == 'sample_username' - - def test_invalid_key(self, test1_content: dict): - assert test1_content.get('invalid') == 'None' diff --git a/tests/test_woodpecker.py b/tests/test_woodpecker.py index f4104db..7521d88 100644 --- a/tests/test_woodpecker.py +++ b/tests/test_woodpecker.py @@ -1,16 +1,11 @@ -from karl.core import woodpecker +from core import woodpecker files = [".gitignore", "compose/nginx/docker-compose.yaml", "config/heimdall.kdbx", "files/nginx/cert/fullchain.mo.pem", "files/nginx/cert/key.mo.pem", "files/nginx/conf.d/default.conf", "files/nginx/conf.d/forge.conf", "files/nginx/conf.d/mqtt.conf", "files/nginx/conf.d/nextcloud.conf", "files/nginx/conf.d/zitadel.conf", "files/nginx/nginx.conf"] -files2 = [".gitignore", "compose/nginx/docker-compose.yaml", "compose/nginx2/docker-compose.yaml"] -def test_get_service_single(): +def test_get_service(): service = woodpecker.WoodpeckerRunner.get_service(files) - assert service == ['nginx'] - -def test_get_service_multi(): - services = woodpecker.WoodpeckerRunner.get_service(files2) - assert services == ['nginx', 'nginx2'] + assert service == 'nginx'