diff --git a/.woodpecker/latest.yaml b/.woodpecker/latest.yaml deleted file mode 100644 index 92d447d..0000000 --- a/.woodpecker/latest.yaml +++ /dev/null @@ -1,13 +0,0 @@ -steps: - - name: build - image: woodpeckerci/plugin-docker-buildx:5.2.2 - settings: - platforms: linux/amd64 - repo: git.ztsh.eu/stawros/slack-exporter - registry: git.ztsh.eu - tags: latest - username: stawros - password: - from_secret: git_pat - when: - - event: [tag, push, manual] diff --git a/Dockerfile b/Dockerfile index 944b773..b265373 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,10 @@ LABEL authors="stawros" RUN mkdir /app -COPY requirements.txt /app/ -COPY exporter_ng.py /app/ -COPY tui.py /app/ +COPY slack-exporter/requirements.txt /app/ +COPY slack-exporter/exporter.py /app/ RUN pip3 uninstall urllib3 RUN pip3 install -r /app/requirements.txt -CMD ["python3", "/app/tui.py"] +CMD ["python3", "/app/exporter.py"] diff --git a/exporter_ng.py b/exporter-ng.py similarity index 81% rename from exporter_ng.py rename to exporter-ng.py index dad9d00..16a7ecc 100644 --- a/exporter_ng.py +++ b/exporter-ng.py @@ -1,15 +1,16 @@ -import json import os import sys -from dataclasses import dataclass -from datetime import datetime -from time import sleep -from typing import List, Optional, Dict, Any - import requests +import json +from timeit import default_timer +from datetime import datetime +import argparse from dotenv import load_dotenv from pathvalidate import sanitize_filename - +from time import sleep +from dataclasses import dataclass +from typing import List, Optional, Dict, Any, Iterator, Tuple +from abc import ABC, abstractmethod @dataclass class SlackConfig: @@ -29,10 +30,8 @@ class SlackConfig: except KeyError: raise ValueError("Brak SLACK_USER_TOKEN w zmiennych środowiskowych") - class SlackAPI: """Klasa do komunikacji z API Slacka""" - def __init__(self, config: SlackConfig): self.config = config self.headers = {"Authorization": f"Bearer {config.user_token}"} @@ -131,7 +130,6 @@ class SlackAPI: params["channel"] = channel return self.paginated_get("files.list", params, "files") - @dataclass class SlackUser: """Reprezentacja użytkownika Slack""" @@ -150,11 +148,12 @@ class SlackUser: @classmethod def from_dict(cls, data: Dict) -> 'SlackUser': + profile = data.get("profile", {}) return cls( id=data["id"], name=data.get("name", ""), - real_name=data.get("real_name"), - display_name=data.get("display_name"), + real_name=profile.get("real_name"), + display_name=profile.get("display_name"), is_admin=data.get("is_admin", False), is_owner=data.get("is_owner", False), is_primary_owner=data.get("is_primary_owner", False), @@ -190,22 +189,17 @@ class SlackUser: return ", ".join(parts) - class SlackChannel: """Reprezentacja kanału Slack""" - - def __init__(self, data: Dict, users: Dict[str, SlackUser]): + def __init__(self, data: Dict): self.id = data["id"] self.name = data.get("name", "") self.is_private = data.get("is_private", False) self.is_im = data.get("is_im", False) self.is_mpim = data.get("is_mpim", False) self.is_group = data.get("is_group", False) - self._creator_id = data.get("creator") - self._user_id = data.get("user") - self.user = users[self._user_id].get_display_name() if self._user_id and self._user_id in users else "(nieznany)" - self._mpim_users = self._list_mpim({u.name: u for u in users.values()}) if self.is_mpim else [] - self.label = self._create_label(users) + self.creator_id = data.get("creator") + self.user_id = data.get("user") @property def type(self) -> str: @@ -218,24 +212,7 @@ class SlackChannel: else: return "channel" - @property - def short_label(self) -> str: - if self.is_im: - return f"(DM) {self.user}" - elif self.is_mpim: - return f"(MPDM) {', '.join(self._mpim_users)}" - return self.name - - def _list_mpim(self, users: Dict[str, SlackUser]) -> List[str]: - result = [] - for part in self.name.split("-"): - name = users.get(part) - if name: - result.append(name.get_display_name()) - return result - - - def _create_label(self, users: Dict[str, SlackUser]) -> str: + def format(self, users: Dict[str, SlackUser]) -> str: """Formatuje informacje o kanale""" parts = [f"[{self.id}]"] @@ -247,14 +224,13 @@ class SlackChannel: parts.append(self.type) - if self._creator_id and self._creator_id in users: - parts.append(f"created by {users[self._creator_id].get_display_name()}") - elif self._user_id and self._user_id in users: - parts.append(f"with {users[self._user_id].get_display_name()}") + if self.creator_id and self.creator_id in users: + parts.append(f"created by {users[self.creator_id].name}") + elif self.user_id and self.user_id in users: + parts.append(f"with {users[self.user_id].name}") return " ".join(parts) - @dataclass class SlackFile: """Reprezentacja pliku Slack""" @@ -270,10 +246,8 @@ class SlackFile: url_private=data.get("url_private", "") ) - class SlackMessage: """Reprezentacja wiadomości Slack""" - def __init__(self, data: Dict, users: Dict[str, SlackUser]): self.timestamp = float(data["ts"]) self.text = data.get("text", "[no message content]") @@ -321,11 +295,9 @@ class SlackMessage: return message + "\n\n" + "*" * 24 + "\n\n" - class SlackExporter: """Główna klasa eksportera""" - - def __init__(self, config: SlackConfig, output_dir: str = 'out'): + def __init__(self, config: SlackConfig, output_dir: str): self.api = SlackAPI(config) self.timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") self.output_dir = self._set_output_dir(output_dir) @@ -342,14 +314,12 @@ class SlackExporter: def _load_users(self) -> Dict[str, SlackUser]: """Ładuje użytkowników""" users_data = self.api.get_users() - # users_data = json.load(open("out/user_list.json", "r", encoding="utf-8")) return {u["id"]: SlackUser.from_dict(u) for u in users_data} def _load_channels(self) -> List[SlackChannel]: """Ładuje kanały""" channels_data = self.api.get_channels() - # channels_data = json.load(open("out/channel_list.json", "r", encoding="utf-8")) - return [SlackChannel(ch, self.users) for ch in channels_data] + return [SlackChannel(ch) for ch in channels_data] def _save_data(self, data: Any, filename: str, as_json: bool = False): """Zapisuje dane do pliku""" @@ -367,25 +337,12 @@ class SlackExporter: else: f.write(data) - def export_channels(self, channels: List[str]): - channels_map = {ch.id: ch for ch in self.channels} - self.export_channel_list(True) - self.export_user_list(True) - for channel_id in channels: - print(f"Eksport {channels_map.get(channel_id).label}...") - print(f"[{channel_id}] Historia") - self.export_channel_history(channel_id) - print(f"[{channel_id}] Odpowiedzi") - self.export_channel_replies(channel_id) - print(f"[{channel_id}] Pliki") - self.export_channel_files(channel_id) - def export_channel_list(self, as_json: bool = False): """Eksportuje listę kanałów""" if as_json: data = [vars(ch) for ch in self.channels] else: - data = "\n".join(ch.label for ch in self.channels) + data = "\n".join(ch.format(self.users) for ch in self.channels) self._save_data(data, "channel_list", as_json) def export_user_list(self, as_json: bool = False): @@ -397,13 +354,10 @@ class SlackExporter: self._save_data(data, "user_list", as_json) def export_channel_history(self, channel_id: str, oldest: Optional[str] = None, - latest: Optional[str] = None): - history = self.api.get_channel_history(channel_id, oldest, latest) - self._export_channel_history(channel_id, history, True) - self._export_channel_history(channel_id, history, False) - - def _export_channel_history(self, channel_id: str, history: List[Dict], as_json: bool): + latest: Optional[str] = None, as_json: bool = False): """Eksportuje historię kanału""" + history = self.api.get_channel_history(channel_id, oldest, latest) + if as_json: data = history else: @@ -420,18 +374,16 @@ class SlackExporter: self._save_data(data, f"channel_{channel_id}", as_json) def export_channel_replies(self, channel_id: str, oldest: Optional[str] = None, - latest: Optional[str] = None): + latest: Optional[str] = None, as_json: bool = False): + """Eksportuje wątki w kanale""" history = self.api.get_channel_history(channel_id, oldest, latest) thread_messages = [msg for msg in history if "reply_count" in msg] + all_replies = [] for msg in thread_messages: replies = self.api.get_replies(channel_id, msg["ts"]) all_replies.extend(replies) - self._export_channel_replies(channel_id, all_replies, True) - self._export_channel_replies(channel_id, all_replies, False) - def _export_channel_replies(self, channel_id: str, all_replies: List[Dict], as_json: bool): - """Eksportuje wątki w kanale""" if as_json: data = all_replies else: @@ -445,14 +397,14 @@ class SlackExporter: data = header + "".join(msg.format(True) for msg in messages) self._save_data(data, f"channel-replies_{channel_id}", as_json) - + def export_channel_files(self, channel_id: Optional[str] = None): """Eksportuje pliki w kanale""" files = [SlackFile.from_dict(f) for f in self.api.get_files(channel_id)] for file in files: filename = f"{file.id}-{sanitize_filename(file.name)}" self.download_file(filename, file.url_private) - + def download_file(self, filename: str, url: str, attempts: int = 10) -> bool: if attempts == 0: return False @@ -465,13 +417,63 @@ class SlackExporter: with open(target, 'wb') as f: for chunk in response.iter_content(chunk_size=8192): f.write(chunk) - print(f"Zapisano plik {target}") return True except requests.exceptions.RequestException as e: print(f"Error downloading file {filename}: {e}. {attempts} attempts left.") return self.download_file(filename, url, attempts - 1) +def main(): + parser = argparse.ArgumentParser(description="Eksporter danych ze Slacka") + parser.add_argument("-o", help="Katalog wyjściowy (jeśli pusty, wyświetla na stdout)") + parser.add_argument("--lc", action="store_true", help="Lista wszystkich konwersacji") + parser.add_argument("--lu", action="store_true", help="Lista wszystkich użytkowników") + parser.add_argument("--json", action="store_true", help="Wynik w formacie JSON") + parser.add_argument("-c", action="store_true", help="Historia wszystkich dostępnych konwersacji") + parser.add_argument("--ch", help="Z -c, ogranicza eksport do podanego ID kanału") + parser.add_argument("--fr", help="Z -c, timestamp początku zakresu (Unix)") + parser.add_argument("--to", help="Z -c, timestamp końca zakresu (Unix)") + parser.add_argument("-r", action="store_true", help="Pobierz wątki ze wszystkich konwersacji") + parser.add_argument("--files", action="store_true", help="Pobierz wszystkie pliki") + + args = parser.parse_args() + + if args.files and not args.o: + print("Opcja --files wymaga określenia katalogu wyjściowego (-o)") + sys.exit(1) + + try: + config = SlackConfig.from_env() + exporter = SlackExporter(config) + + if args.o: + exporter.set_output_dir(args.o) + + exporter.load_users() + exporter.load_channels() + + if args.lc: + exporter.export_channel_list(args.json) + + if args.lu: + exporter.export_user_list(args.json) + + if args.c or args.r: + channel_ids = [args.ch] if args.ch else [ch.id for ch in exporter.channels] + for channel_id in channel_ids: + if args.c: + exporter.export_channel_history(channel_id, args.fr, args.to, args.json) + if args.r: + exporter.export_channel_replies(channel_id, args.fr, args.to, args.json) + + if args.files and args.o: + # TODO: Implementacja pobierania plików + print("Funkcja pobierania plików jeszcze nie zaimplementowana") + + except Exception as e: + print(f"Błąd: {e}") + sys.exit(1) if __name__ == "__main__": + # main() config = SlackConfig.from_env() - exporter = SlackExporter(config) + exporter = SlackExporter(config, 'out') diff --git a/requirements.txt b/requirements.txt index 03d29b3..19d4264 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,3 @@ Flask~=1.1.2 requests~=2.24.0 python-dotenv~=0.15.0 pathvalidate~=2.5.2 - -textual~=3.1.1 diff --git a/tui.py b/tui.py deleted file mode 100644 index d146443..0000000 --- a/tui.py +++ /dev/null @@ -1,119 +0,0 @@ -import sys - -from textual import work -from textual.app import App, ComposeResult -from textual.containers import Container -from textual.screen import Screen -from textual.widgets import Header, Button, ListView, ListItem, Footer, Label - -from exporter_ng import SlackExporter, SlackConfig - - -class SlackExporterScreen(Screen): - """Ekran główny eksportera Slack""" - - def __init__(self, exporter: SlackExporter, *args, **kwargs): - super().__init__(*args, **kwargs) - self.exporter = exporter - self.selected_channels = set() - - def compose(self) -> ComposeResult: - """Komponuje widgety na ekranie""" - yield Header(show_clock=True) - yield Container( - ListView(*[ - ListItem(Label(ch.short_label, id=ch.id)) - for ch in self.exporter.channels - ], id="channel-list"), - Button("Eksportuj zaznaczone", variant="primary", id="export-btn"), - id="main-container" - ) - yield Footer() - - def on_list_view_selected(self, event: ListView.Selected) -> None: - """Obsługuje zaznaczenie elementu listy""" - item = event.item - item_id = item.children[0].id - if item_id in self.selected_channels: - self.selected_channels.remove(item_id) - item.remove_class("selected") - else: - self.selected_channels.add(item_id) - item.add_class("selected") - - @work(exclusive=True) - async def export_channels(self): - """Eksportuje zaznaczone kanały""" - if not self.selected_channels: - self.notify("Nie wybrano żadnych kanałów") - return - - # self.notify("Rozpoczynam eksport...") - # self.exporter.export_channels(list(self.selected_channels)) - # # for channel_id in self.selected_channels: - # # self.notify(f"Eksportuję kanał {channel_id}...") - # # self.exporter.export_channel_history(channel_id) - # self.notify("Eksport zakończony") - self.app.exit(return_code=8080) - - - async def on_button_pressed(self, event: Button.Pressed) -> None: - """Obsługuje kliknięcie przycisku""" - if event.button.id == "export-btn": - self.export_channels() - -class SlackTUI(App): - """Główna klasa interfejsu użytkownika""" - CSS = """ - #main-container { - layout: vertical; - height: 100%; - padding: 1; - } - - ListView { - height: 1fr; - border: solid green; - } - - .selected { - background: $accent; - color: $text; - } - - Button { - margin: 1; - width: 100%; - } - """ - - def __init__(self, exporter: SlackExporter): - super().__init__() - self.exporter = exporter - self._screen = SlackExporterScreen(self.exporter) - - def on_mount(self) -> None: - """Wywoływane przy montowaniu aplikacji""" - self.push_screen(self._screen) - - def get_selection(self): - return self._screen.selected_channels - - def get_return_code(self): - return self._return_code - -def run_tui(exporter: SlackExporter): - """Uruchamia interfejs użytkownika""" - app = SlackTUI(exporter) - app.run() - if app.get_return_code() == 8080: - exporter.export_channels(list(app.get_selection())) - -if __name__ == "__main__": - try: - config = SlackConfig.from_env() - exporter = SlackExporter(config) - run_tui(exporter) - except Exception as e: - print(f"Błąd: {e}") - sys.exit(1)