Added exporter.py and documentation
This commit is contained in:
Seb Seager
parent
23385c2f01
commit
8c4550f307
5 changed files with 439 additions and 82 deletions
170
bot.py
170
bot.py
|
|
@ -1,42 +1,18 @@
|
|||
import os
|
||||
import ssl
|
||||
import slack
|
||||
from slack.errors import SlackApiError
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
from flask import Flask, request, Response
|
||||
from urllib.parse import urljoin
|
||||
from uuid import uuid4
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
ssl_context = ssl.create_default_context()
|
||||
ssl_context.check_hostname = False
|
||||
ssl_context.verify_mode = ssl.CERT_NONE
|
||||
from dotenv import load_dotenv
|
||||
from exporter import parse_replies, parse_channel_history
|
||||
|
||||
app = Flask(__name__)
|
||||
load_dotenv(os.path.join(app.root_path, '.env'))
|
||||
client = slack.WebClient(token=os.environ['SLACK_BOT_TOKEN'], ssl=ssl_context)
|
||||
|
||||
|
||||
# chat write interactions
|
||||
|
||||
def send_to_channel(channel_id, text):
|
||||
try:
|
||||
client.chat_postMessage(channel=channel_id, text=text)
|
||||
except SlackApiError as e:
|
||||
print(e)
|
||||
pass
|
||||
|
||||
|
||||
def send_to_user(user_id, text):
|
||||
try:
|
||||
client.chat_postMessage(channel=user_id, text=text, as_user=True)
|
||||
except SlackApiError as e:
|
||||
print(e)
|
||||
pass
|
||||
|
||||
|
||||
def post_response(response_url, text):
|
||||
requests.post(response_url, json={'text': text})
|
||||
|
||||
|
|
@ -72,7 +48,10 @@ def paginated_get(url, params, response_url, combine_key=None):
|
|||
result = []
|
||||
while True:
|
||||
next_cursor, data = get_at_cursor(url, params, response_url, cursor=next_cursor)
|
||||
result.extend(data) if combine_key is None else result.extend(data[combine_key])
|
||||
try:
|
||||
result.extend(data) if combine_key is None else result.extend(data[combine_key])
|
||||
except KeyError:
|
||||
post_response(response_url, "Sorry! I got an unexpected response (KeyError).")
|
||||
if next_cursor is None:
|
||||
break
|
||||
|
||||
|
|
@ -101,37 +80,19 @@ def user_list(team_id, response_url):
|
|||
return paginated_get('https://slack.com/api/users.list', params, response_url, combine_key='members')
|
||||
|
||||
|
||||
# parsing
|
||||
def channel_replies(timestamps, channel_id, response_url):
|
||||
replies = []
|
||||
for timestamp in timestamps:
|
||||
params = {
|
||||
'token': os.environ['SLACK_USER_TOKEN'],
|
||||
'channel': channel_id,
|
||||
'ts': timestamp,
|
||||
'limit': 200
|
||||
}
|
||||
r = paginated_get('https://slack.com/api/conversations.replies', params, response_url, combine_key='messages')
|
||||
replies.append(r)
|
||||
|
||||
def user_list_to_names(user_dict):
|
||||
return {x['id']: {'name': x['name'], 'real_name': x['real_name']} for x in user_dict}
|
||||
|
||||
|
||||
def channel_history_to_text(msgs_dict, users):
|
||||
messages = [x for x in msgs_dict['messages'] if x['type'] == 'message'] # files are also messages
|
||||
body = 'Team ID: %s\nTeam Domain: %s\nChannel ID: %s\nChannel Name: %s\n\n' % \
|
||||
(msgs_dict['team_id'], msgs_dict['team_domain'], msgs_dict['channel_id'], msgs_dict['channel_name'])
|
||||
body += '%s\n %s Messages\n%s\n\n' % ('=' * 16, len(messages), '=' * 16)
|
||||
for msg in messages:
|
||||
usr = users[msg['user']] if 'user' in msg else {'name': '', 'real_name': 'none'}
|
||||
ts = datetime.fromtimestamp(round(float(msg['ts']))).strftime('%m-%d-%Y %H:%M:%S')
|
||||
text = msg['text'] if msg['text'].strip() != "" else "[no message content]"
|
||||
for u in users.keys():
|
||||
# if u in text:
|
||||
# print(u)
|
||||
text = str(text).replace('<@%s>' % u, '<@%s> (%s)' % (u, users[u]['name']))
|
||||
entry = "Message at %s\nUser: %s (%s)\n%s" % (ts, usr['name'], usr['real_name'], text)
|
||||
if 'reactions' in msg:
|
||||
rxns = msg['reactions']
|
||||
entry += "\nReactions: " + ', '.join('%s (%s)' % (x['name'], ', '.join(
|
||||
users[u]['name'] for u in x['users'])) for x in rxns)
|
||||
if 'files' in msg:
|
||||
files = msg['files']
|
||||
entry += "\nFiles:\n" + '\n'.join(' - %s, %s' % (f['name'], f['url_private_download']) for f in files)
|
||||
|
||||
body += entry.strip() + '\n\n%s\n\n' % ('=' * 16)
|
||||
|
||||
return body
|
||||
return replies
|
||||
|
||||
|
||||
# Flask routes
|
||||
|
|
@ -143,32 +104,37 @@ def export_channel():
|
|||
try:
|
||||
team_id = data['team_id']
|
||||
team_domain = data['team_domain']
|
||||
channel_id = data['channel_id']
|
||||
channel_name = data['channel_name']
|
||||
ch_id = data['channel_id']
|
||||
ch_name = data['channel_name']
|
||||
response_url = data['response_url']
|
||||
command_args = data['text']
|
||||
except KeyError:
|
||||
return Response("Sorry! I got an unexpected response from Slack (KeyError)."), 200
|
||||
return Response("Sorry! I got an unexpected response (KeyError)."), 200
|
||||
|
||||
post_response(response_url, "Retrieving history for this channel...")
|
||||
all_messages = {
|
||||
'team_id': team_id,
|
||||
'team_domain': team_domain,
|
||||
'channel_id': channel_id,
|
||||
'channel_name': channel_name,
|
||||
'messages': channel_history(channel_id, response_url)
|
||||
}
|
||||
ch_hist = channel_history(ch_id, response_url)
|
||||
|
||||
filename = "%s-%s-%s.json" % (team_domain, channel_id, str(uuid4().hex)[:6])
|
||||
filepath = os.path.join(app.root_path, 'exports', filename)
|
||||
export_mode = str(command_args).lower()
|
||||
|
||||
exports_subdir = 'exports'
|
||||
exports_dir = os.path.join(app.root_path, exports_subdir)
|
||||
file_ext = '.txt' if export_mode == 'text' else '.json'
|
||||
filename = "%s-ch_%s-%s%s" % (team_domain, ch_id, str(uuid4().hex)[:6], file_ext)
|
||||
filepath = os.path.join(exports_dir, filename)
|
||||
loc = urljoin(request.url_root, 'download/%s' % filename)
|
||||
|
||||
if not os.path.isdir(exports_dir):
|
||||
os.makedirs(exports_dir, exist_ok=True)
|
||||
|
||||
with open(filepath, mode='w') as f:
|
||||
if str(command_args).lower() == 'text':
|
||||
users = user_list_to_names(user_list(team_id, response_url))
|
||||
f.write(channel_history_to_text(all_messages, users))
|
||||
if export_mode == 'text':
|
||||
num_msgs = len(ch_hist)
|
||||
sep = '=' * 24
|
||||
header_str = 'Channel Name: %s\nChannel ID: %s\n%s Messages\n%s\n\n' % (ch_name, ch_id, num_msgs, sep)
|
||||
data_ch = header_str + parse_channel_history(ch_hist, user_list(team_id, response_url))
|
||||
f.write(data_ch)
|
||||
else:
|
||||
json.dump(all_messages, f, indent=4)
|
||||
json.dump(ch_hist, f, indent=4)
|
||||
|
||||
post_response(response_url, "Done! This channel's history is available for download here (note that this link "
|
||||
"is single-use): %s" % loc)
|
||||
|
|
@ -176,8 +142,60 @@ def export_channel():
|
|||
return Response(), 200
|
||||
|
||||
|
||||
@app.route('/slack/export-replies', methods=['POST'])
|
||||
def export_replies():
|
||||
data = request.form
|
||||
|
||||
try:
|
||||
team_id = data['team_id']
|
||||
team_domain = data['team_domain']
|
||||
ch_id = data['channel_id']
|
||||
ch_name = data['channel_name']
|
||||
response_url = data['response_url']
|
||||
command_args = data['text']
|
||||
except KeyError:
|
||||
return Response("Sorry! I got an unexpected response (KeyError)."), 200
|
||||
|
||||
post_response(response_url, "Retrieving reply threads for this channel...")
|
||||
print(ch_id)
|
||||
ch_hist = channel_history(ch_id, response_url)
|
||||
print(ch_hist)
|
||||
ch_replies = channel_replies([x['ts'] for x in ch_hist if 'reply_count' in x], ch_id, response_url)
|
||||
|
||||
export_mode = str(command_args).lower()
|
||||
|
||||
exports_subdir = 'exports'
|
||||
exports_dir = os.path.join(app.root_path, exports_subdir)
|
||||
file_ext = '.txt' if export_mode == 'text' else '.json'
|
||||
filename = "%s-re_%s-%s%s" % (team_domain, ch_id, str(uuid4().hex)[:6], file_ext)
|
||||
filepath = os.path.join(exports_dir, filename)
|
||||
loc = urljoin(request.url_root, 'download/%s' % filename)
|
||||
|
||||
if export_mode == 'text':
|
||||
header_str = 'Threads in: %s\n%s Messages' % (ch_name, len(ch_replies))
|
||||
data_replies = parse_replies(ch_replies, user_list(team_id, response_url))
|
||||
sep = '=' * 24
|
||||
data_replies = '%s\n%s\n\n%s' % (header_str, sep, data_replies)
|
||||
else:
|
||||
data_replies = ch_replies
|
||||
|
||||
if not os.path.isdir(exports_dir):
|
||||
os.makedirs(exports_dir, exist_ok=True)
|
||||
|
||||
with open(filepath, mode='w') as f:
|
||||
if export_mode == 'text':
|
||||
f.write(data_replies)
|
||||
else:
|
||||
json.dump(data_replies, f, indent=4)
|
||||
|
||||
post_response(response_url, "Done! This channel's reply threads are available for download here (note that this "
|
||||
"link is single-use): %s" % loc)
|
||||
|
||||
return Response(), 200
|
||||
|
||||
|
||||
@app.route('/download/<filename>')
|
||||
def download(filename, mimetype='application/json'):
|
||||
def download(filename):
|
||||
path = os.path.join(app.root_path, 'exports', filename)
|
||||
|
||||
def generate():
|
||||
|
|
@ -185,10 +203,12 @@ def download(filename, mimetype='application/json'):
|
|||
yield from f
|
||||
os.remove(path)
|
||||
|
||||
mimetype = 'text/plain' if os.path.splitext(filename)[-1] == '.txt' else 'application/json'
|
||||
|
||||
r = app.response_class(generate(), mimetype=mimetype)
|
||||
r.headers.set('Content-Disposition', 'attachment', filename=filename)
|
||||
return r
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(debug=True)
|
||||
app.run(debug=False)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue