test: password file read fix?

* Possibly fixes issues with fail2ban

.gitignore

@@ -10,10 +10,9 @@ wheels/
 .venv
 .idea
 *.iml
-uv.lock
-**/*.kdbx*
+**/*.lock
 .compose_repository
 
-__pycache__/
+deployment/
 **/dist/
 **/*.log

.woodpecker/dev.yaml

@@ -13,3 +13,5 @@ steps:
       - event: pull_request
         evaluate: 'CI_COMMIT_PULL_REQUEST_LABELS contains "ci-ready"'
       - event: manual
+depends_on:
+  - name: python

.woodpecker/latest.yaml

@@ -12,3 +12,5 @@ steps:
     when:
       - event: [ tag, push, manual ]
         branch: master
+depends_on:
+  - name: python

.woodpecker/python.yaml

@@ -0,0 +1,24 @@
+steps:
+  - name: prepare
+    image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine
+    commands:
+      - apk add --no-cache gcc python3-dev musl-dev linux-headers
+      - uv sync
+  - name: test
+    image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine
+    commands:
+      - uv sync --group dev
+      - uv run pytest
+  - name: test-run
+    image: ghcr.io/astral-sh/uv:0.9-python3.12-alpine
+    commands:
+      - |
+        python src/karl/__init__.py 2>&1 | while IFS= read -r line; do
+          echo "$line"
+          if echo "$line" | grep -q "Uvicorn running on"; then
+            pkill -f "uvicorn"
+            break
+          fi
+        done
+when:
+  - event: [ push, manual ]

src/karl/api/v1.py

@@ -8,8 +8,7 @@ from starlette.responses import JSONResponse, Response
 
 from karl.api.models import Request
 from karl.core.injects import AutowireSupport
-from karl.core.woodpecker import Woodpecker
-from karl.model.webhook import WoodpeckerEvent
+from karl.model.webhook import WoodpeckerEvent, ReloadEvent
 
 router = APIRouter()
 logger = logging.getLogger(__name__)
@@ -21,7 +20,6 @@ async def root():
 
 @cbv(router)
 class APIv1:
-  woodpecker: Woodpecker = Depends(AutowireSupport.woodpecker)
   bus: EventBus = Depends(AutowireSupport.bus)
 
   def __init__(self):
@@ -39,3 +37,10 @@ class APIv1:
   async def ci(self, request: Request):
     await self.bus.dispatch(mapper.map(request))
     return Response(status_code=201)
+
+  @router.get("/reload", summary="Manual service reload")
+  async def reload(self, service: str = None) -> Response:
+    if service is None:
+      return Response(status_code=400)
+    await self.bus.dispatch(ReloadEvent(service=service))
+    return Response(status_code=201)

src/karl/config/settings.py

@@ -51,4 +51,8 @@ class Settings(BaseSettings):
 
 @lru_cache
 def get_settings() -> Settings:
-  return Settings.from_yaml()
+  paths = ['deployment/config.yaml', 'config/config.yaml']
+  for path in paths:
+    if Path(path).exists():
+      return Settings.from_yaml(path)
+  raise Exception("Config file not found")

src/karl/core/injects.py

@@ -1,6 +1,7 @@
 from bubus import EventBus
 from injectable import inject
 
+from karl.core.reload import ReloadService
 from karl.core.woodpecker import Woodpecker
 
 
@@ -10,6 +11,10 @@ class AutowireSupport:
   def woodpecker():
     return inject(Woodpecker)
 
+  @staticmethod
+  def reload():
+    return inject(ReloadService)
+
   @staticmethod
   def bus():
     return inject(EventBus)

src/karl/core/reload.py

@@ -0,0 +1,48 @@
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Annotated
+
+from bubus import EventBus
+from injectable import injectable, autowired, Autowired
+
+from karl import get_settings
+from model.webhook import ReloadEvent, WoodpeckerEvent
+from services import GitService
+
+logger = logging.getLogger(__name__)
+
+
+@injectable(singleton=True)
+class ReloadService:
+
+  @autowired
+  def __init__(self, bus: Annotated[EventBus, Autowired]):
+    self._bus = bus
+    self._git = GitService()
+    bus.on(ReloadEvent, self.on_reload)
+    self.root_path = get_settings().git.path
+    logger.info("ReloadService initialized.")
+
+  async def on_reload(self, event: ReloadEvent):
+    try:
+      logger.info(f"Received ReloadEvent: {event.service}")
+      head = self._git.get_head()
+      file_path = Path(self.root_path) / f"files/{event.service}"
+      if not file_path.exists():
+        raise Exception(f"Service {event.service} not found: {file_path.absolute()} does not exist.")
+      logger.debug(f"Found service files at {file_path}: {', '.join([str(f) for f in list(file_path.iterdir())])}")
+      mos = list(file_path.glob('*.mo.*'))
+      logger.debug(f"Found {len(mos)} .mo files")
+      we = WoodpeckerEvent(
+        _id=-1,
+        commit=head.sha,
+        ref=head.branch,
+        message=f"Manual reload of {event.service}",
+        started=int(datetime.now().timestamp()),
+        files=[f"compose/{event.service}/docker-compose.yml"] + [str(pp).replace(str(self.root_path), '') for pp in mos]
+      )
+      logger.debug(f"Sending <WoodpeckerEvent commit={we.commit} ref={we.ref} message={we.message} started={we.started} files={we.files}")
+      await self._bus.dispatch(we)
+    except Exception as e:
+      logger.error(f"Reload error: {e}", exc_info=True)

src/karl/core/woodpecker.py

@@ -2,7 +2,7 @@ import asyncio
 import logging
 from collections import deque
 from pathlib import Path
-from threading import RLock, Thread
+from threading import RLock, Thread, current_thread
 from typing import Annotated
 
 from bubus import EventBus, BaseEvent
@@ -32,6 +32,7 @@ class WoodpeckerRunner(Thread):
     self._bus = bus
     self._event: WoodpeckerEvent | None = None
     self._root = get_settings().git.path
+    self.thread_logger = logging.getLogger(current_thread().name)
 
   def process_event(self, event: WoodpeckerEvent):
     self._event = event
@@ -42,18 +43,26 @@ class WoodpeckerRunner(Thread):
       await self._bus.dispatch(r)
 
     result = RunnerResult()
+    self.thread_logger.info("Runner started")
     try:
-      service = self.get_service(self._event.files)
-      if service is None:
-        logger.info("No service found.")
+      services = self.get_service(self._event.files)
+      if len(services) == 0:
+        self.thread_logger.info("No service found.")
         result.success = True
       else:
-        service_path = f"{self._root}/compose/{service}/docker-compose.yml"
         self._git.checkout(self._event.commit)
+        self.thread_logger.info(f"Checked out successfully at {self._event.commit}")
+        paths = []
+        for service in services:
+          service_path = f"{self._root}/compose/{service}/docker-compose.yml"
           for file in self._event.files:
             if file.__contains__('.mo.'):
-            self._mo.process(Path(f"{self._root}{file}").absolute())
+              self._mo.process(Path(f"{self._root}/{file}").absolute())
+            paths.append(service_path)
+        self.thread_logger.info(f"{len(paths)} paths set")
+        for service_path in paths:
           self._docker.reload(Path(service_path).absolute())
+        self.thread_logger.info("Reloaded all services")
       result.success = True
     except Exception as e:
       result.throwable = e
@@ -61,20 +70,14 @@ class WoodpeckerRunner(Thread):
     asyncio.run(dispatch(result))
 
   @staticmethod
-  def get_service(files: list[str]) -> str | None:
+  def get_service(files: list[str]) -> list[str]:
     supported_files = []
     for f in files:
       f_parts = f.split("/")
       if f_parts[0] in ["compose", "files"]:
         supported_files.append(f_parts[1])
-    match len(set(supported_files)):
-      case 0:
-        return None
-      case 1:
-        return supported_files[0]
-      case _:
-        logger.error(f"Multiple services were provided: {', '.join(supported_files)}")
-        raise Exception("Multiple services are not supported.")
+    # TODO: check service priorities (NGINX last)
+    return list(dict.fromkeys(supported_files))
 
 
 @injectable(singleton=True)
@@ -94,7 +97,7 @@ class Woodpecker:
     logger.info("Woodpecker initialized.")
 
   async def on_ci_event(self, event: WoodpeckerEvent):
-    logger.debug(f"Received WoodpeckerEvent: {event.event_id}")
+    logger.info(f"Received WoodpeckerEvent: {event.event_id}")
     with self._lock:
       logger.debug("Lock acquired [on-ci-event]")
       if len(self._pending) > 0 or self._runner is not None:

src/karl/main.py

@@ -3,6 +3,7 @@ import logging
 from fastapi import FastAPI
 from injectable import load_injection_container
 
+from core.injects import AutowireSupport
 from karl.config import get_settings
 from karl.util.logging import HandlerFactory
 
@@ -15,7 +16,7 @@ class KarlApplication:
     _app = FastAPI(title="Karl", version="0.1.0")
     self._set_middlewares(_app)
     self._set_routes(_app)
-    self._set_events(_app)
+    self._init_services(_app)
 
     self._app = _app
 
@@ -55,8 +56,9 @@ class KarlApplication:
     app.include_router(api_v1_router, prefix="/api/v1", tags=["v1"])
     pass
 
-  def _set_events(self, app: FastAPI):
-    pass
+  def _init_services(self, app: FastAPI):
+    AutowireSupport.reload()
+    AutowireSupport.woodpecker()
 
 
 def run():

src/karl/model/vcs.py

@@ -0,0 +1,7 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Head:
+  sha: str
+  branch: str

src/karl/model/webhook.py

@@ -10,3 +10,6 @@ class WoodpeckerEvent(BaseEvent):
   message: str
   started: int
   files: List[str]
+
+class ReloadEvent(BaseEvent):
+  service: str

src/karl/services/containers.py

@@ -15,6 +15,8 @@ class DockerService:
 
   def reload(self, compose_path: Path):
     os.chdir(compose_path.parent)
-    self._client.compose.ps()
-    self._client.compose.down(remove_orphans=True)
-    self._client.compose.up()
+    # TODO: Check if container needs to be reloaded from scratch
+    # if len(self._client.compose.ps()) > 0:
+    #   self._client.compose.restart()
+    #   return
+    self._client.compose.up(detach=True)

src/karl/services/mo.py

@@ -9,7 +9,7 @@ from karl.services import Passwords
 
 class ValueTemplate(Template):
   # Pozwala na kropki i ukośniki w nazwach placeholderów, np. ${user.name/first}
-  idpattern = r'[_a-zA-Z][_a-zA-Z0-9.\/]*'
+  idpattern = r'[_a-zA-Z][_a-zA-Z0-9.\/\-]*'
 
 
 @injectable

src/karl/services/passwords.py

@@ -6,6 +6,10 @@ import keyring
 from injectable import injectable
 from pykeepass import PyKeePass, create_database
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 class KeyRequest:
   def __init__(self, prompt: str):
     self.field_name = None
@@ -14,7 +18,8 @@ class KeyRequest:
     self._parse_prompt(prompt)
 
   def _parse_prompt(self, prompt: str):
-    prompt_parts = prompt.split(".")
+    logger.debug(f"Got prompt: {prompt}")
+    prompt_parts = prompt.replace('\n', '').split(".")
     key = None
     match len(prompt_parts):
       case 1:
@@ -26,6 +31,7 @@ class KeyRequest:
       case _:
         key = None
     if key is None:
+      logger.warning(f"Prompt {prompt} cannot be parsed")
       return
     key_parts = key.split("/")
     self.path = key_parts[:] if len(key_parts) > 1 else None
@@ -39,9 +45,13 @@ class Passwords:
     import keyring_backend
     keyring.set_keyring(keyring=keyring_backend.Backend())
 
+    logger.info(f"Reading secret file at {settings.kp.secret}")
     with open(settings.kp.secret, "r") as fh:
-      keyring.set_password("karl", "kp", fh.read().splitlines()[0])
+      keyring.set_password("karl", "kp", fh.read().splitlines()[0].replace('\r', '').strip())
+    logger.info(f"Database file: {settings.kp.file}")
     self._path = settings.kp.file
+    if not os.path.exists(self._path):
+      logger.warning(f"Database file {self._path} does not exist")
 
   @contextmanager
   def open(self, mode: str = "r") -> Generator[PyKeePass | Any, Any, None]:
@@ -57,6 +67,8 @@ class Passwords:
       request = KeyRequest(k)
       with self.open() as kp:
         kp_entry = kp.find_entries(path=request.path, first=True, title=request.entry_name)
+        if kp_entry is None:
+          logger.warning(f"No value found for key {k}")
       output[k] = self._get_field_value(kp_entry, request.field_name)
     return output
 

src/karl/services/vcs.py

@@ -1,7 +1,25 @@
+try:
   from git import Repo, Remote
+except ImportError:
+  import sys
+
+  sys.stderr.write("Cannot import Git\n")
+  sys.stderr.flush()
+
+
+  class Repo:
+    @staticmethod
+    def clone_from(*args):
+      raise RuntimeError("git lib not imported")
+
+
+  class Remote:
+    pass
+
 from injectable import injectable
 
 from karl.config import GitConfig, get_settings
+from karl.model.vcs import Head
 
 
 @injectable(singleton=True)
@@ -27,3 +45,11 @@ class GitService:
   def checkout(self, sha: str):
     self._origin.fetch()
     self._repo.git.checkout(sha)
+
+  def get_head(self) -> Head:
+    if self._repo.head.is_detached:
+      return Head(self._repo.head.object.hexsha, "detached")
+    return Head(
+      self._repo.active_branch.commit.hexsha,
+      self._repo.active_branch.name
+    )

src/karl/util/logging.py

@@ -32,14 +32,18 @@ class NamingCache:
 
 
 class ApplicationFormatter(Formatter):
-  def __init__(self, handler_prefix: str = ''):
+  def __init__(self, handler_prefix: str = '', include_timestamp: bool = True):
     super().__init__()
     self._logger_names = NamingCache()
     self._handler_prefix = handler_prefix
+    self._include_timestamp = include_timestamp
 
   def format(self, record):
     from datetime import datetime
-    timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds')
+    if self._include_timestamp:
+      timestamp = datetime.fromtimestamp(record.created).isoformat(sep=' ', timespec='milliseconds') + ' '
+    else:
+      timestamp = ''
     level = record.levelname.replace('WARNING', 'WARN').rjust(5)
     thread_name = record.threadName.replace(' (', '#').replace(')', '').rjust(16)[-16:]  # TODO: NamingCache?
     logger_name = self._logger_names[f"{self._handler_prefix}{record.name}"]
@@ -62,7 +66,7 @@ class HandlerFactory:
   def create(target: Target, handler_prefix: str = '', file_path: Path = None) -> List[Handler]:
     def console_handler(prefix: str = ''):
       handler = StreamHandler()
-      handler.setFormatter(ApplicationFormatter(prefix))
+      handler.setFormatter(ApplicationFormatter(prefix, include_timestamp=False))
       handler.setLevel('INFO')
       return handler
 

tests/files/test1/test.mo.env

@@ -0,0 +1,10 @@
+value=${sample}
+nested=${some/nested/value}
+mixed=${some/nested/value.username}
+custom=${custom.field}
+
+uname=${sample.username}
+invalid=${double.dot.example}
+
+#some comment
+dashed=${some/dashed}

tests/test_mo_env.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+
+import pytest
+
+from karl.services import Passwords
+from karl.services.mo import Mo
+
+
+@pytest.fixture(scope='class')
+def target_path():
+  p = Path('tests/files/test1/test.env')
+  # posprzątaj przed testem, gdyby plik istniał z poprzednich uruchomień
+  if p.exists():
+    p.unlink()
+  yield p
+  # sprzątanie po teście
+  if p.exists():
+    p.unlink()
+
+
+@pytest.fixture(scope='class')
+def test1_content(target_path: Path):
+  mo = Mo(Passwords())
+  mo.process(Path('tests/files/test1/test.mo.env').absolute())
+
+  assert target_path.absolute().exists()
+
+  content = target_path.read_text()
+  assert '${' not in content
+
+  props = {}
+  prop_lines = content.split('\n')
+  for line in prop_lines:
+    if len(line) > 1 and not line.startswith("#"):
+      splitted = line.split('=')
+      props[splitted[0]] = splitted[1]
+  # props = {prop[0]:prop[1] for prop in content.split('\n')}
+  print(props)
+  yield props
+
+
+class TestEnvParsing:
+
+  def test_simple(self, test1_content: dict):
+    assert test1_content['value'] == 'some_pass'
+
+  def test_nested(self, test1_content: dict):
+    assert test1_content['nested'] == 'nested_pass'
+
+  def test_mixed(self, test1_content: dict):
+    assert test1_content['mixed'] == 'nested_username'
+
+  def test_dashed(self, test1_content: dict):
+    assert test1_content['dashed'] == 'dd-value'
+
+  def test_custom_field(self, test1_content: dict):
+    assert test1_content['custom'] == 'custom_content'
+
+  def test_username_field(self, test1_content: dict):
+    assert test1_content['uname'] == 'sample_username'
+
+  def test_invalid_key(self, test1_content: dict):
+    assert test1_content.get('invalid') == 'None'

tests/test_woodpecker.py

@@ -1,11 +1,16 @@
-from core import woodpecker
+from karl.core import woodpecker
 
 files = [".gitignore", "compose/nginx/docker-compose.yaml", "config/heimdall.kdbx", "files/nginx/cert/fullchain.mo.pem",
          "files/nginx/cert/key.mo.pem", "files/nginx/conf.d/default.conf", "files/nginx/conf.d/forge.conf",
          "files/nginx/conf.d/mqtt.conf", "files/nginx/conf.d/nextcloud.conf", "files/nginx/conf.d/zitadel.conf",
          "files/nginx/nginx.conf"]
+files2 = [".gitignore", "compose/nginx/docker-compose.yaml", "compose/nginx2/docker-compose.yaml"]
 
 
-def test_get_service():
+def test_get_service_single():
   service = woodpecker.WoodpeckerRunner.get_service(files)
-  assert service == 'nginx'
+  assert service == ['nginx']
+
+def test_get_service_multi():
+  services = woodpecker.WoodpeckerRunner.get_service(files2)
+  assert services == ['nginx', 'nginx2']