From 8cfb87eaf624ca93bcb249188224b1e4c2888e7a Mon Sep 17 00:00:00 2001 From: Arash Date: Tue, 4 Jun 2024 20:58:08 +0200 Subject: [PATCH] Rewritten the bluesky and slack plugins and add some fixes. This commit updates the dependencies in the `requirements.txt` file. It also updates the GitHub Actions workflows (`preview.yml` and `publish_content.yml`) to use the latest versions of the `actions/checkout` and `actions/setup-python` actions. The Python version in the workflows is updated to `3.12` (this was necessary for using `asyncio.Runner()`) . These changes aim to improve the overall functionality and compatibility of the project. --- .github/workflows/preview.yml | 6 +-- .github/workflows/publish_content.yml | 6 +-- github_run.py | 16 +++---- lib/galaxy_social.py | 16 ++++--- lib/plugins/bluesky.py | 66 +++++++++++++++++++++------ lib/plugins/markdown.py | 14 ++---- lib/plugins/mastodon.py | 40 +++++++++++----- lib/plugins/matrix.py | 22 ++++++--- lib/plugins/slack.py | 58 ++++++++++++++++++----- requirements.txt | 3 +- 10 files changed, 171 insertions(+), 76 deletions(-) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 0e64922..9369ffb 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -13,7 +13,7 @@ jobs: pull-requests: write steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get changed files in posts folder id: get_changed_files @@ -30,9 +30,9 @@ jobs: - name: Set up Python if: steps.get_changed_files.outputs.any_changed == 'true' - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: '3.12' - name: Install dependencies if: steps.get_changed_files.outputs.any_changed == 'true' diff --git a/.github/workflows/publish_content.yml b/.github/workflows/publish_content.yml index 86d19f8..9af8f3c 100644 --- a/.github/workflows/publish_content.yml +++ b/.github/workflows/publish_content.yml @@ -14,7 +14,7 @@ jobs: pull-requests: write steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get changed files in posts folder id: get_changed_files @@ -31,9 +31,9 @@ jobs: - name: Set up Python if: steps.get_changed_files.outputs.any_changed == 'true' - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: '3.12' - name: Install dependencies if: steps.get_changed_files.outputs.any_changed == 'true' diff --git a/github_run.py b/github_run.py index c89c1f5..5c5ebf1 100644 --- a/github_run.py +++ b/github_run.py @@ -31,14 +31,14 @@ def comment(self, comment_text): url = ( f"https://api.github.com/repos/{self.repo}/issues/{self.pr_number}/comments" ) - data = {"body": str(comment_text)} - response = requests.post(url, headers=headers, json=data) - if response.status_code == 201: - return True - else: - raise Exception( - f"Failed to create github comment!, {response.json().get('message')}" - ) + for comment_body in comment_text.split("\n\n---\n"): + data = {"body": str(comment_body)} + response = requests.post(url, headers=headers, json=data) + if response.status_code != 201: + raise Exception( + f"Failed to create github comment!, {response.json().get('message')}" + ) + return True def get_files(self): url = f"https://api.github.com/repos/{self.repo}/pulls/{self.pr_number}/files" diff --git a/lib/galaxy_social.py b/lib/galaxy_social.py index 70285dc..eb50e8e 100644 --- a/lib/galaxy_social.py +++ b/lib/galaxy_social.py @@ -121,14 +121,14 @@ def process_markdown_file(self, file_path, processed_files): except Exception as e: raise Exception(f"Failed to format post for {file_path}.\n{e}") if self.preview: - message = f"File: {file_path}" + message = f'Hi, I\'m your friendly social media assistant. In the following, you will see a preview of this post "{file_path}"' for media in metadata["media"]: formatted_content, preview, warning = formatting_results[media] - message += f"\n\nThis is a preview of what will be posted to {media}:\n\n" + message += f"\n\n## {media}\n\n" message += preview if warning: message += f"\nWARNING: {warning}" - return processed_files, message + return processed_files, message.strip() stats = {} url = {} @@ -143,7 +143,11 @@ def process_markdown_file(self, file_path, processed_files): formatted_content, file_path=file_path ) url_text = "\n".join( - [f"- [{media}]({link})" if link else f"- {media}" for media, link in url.items() if stats[media]] + [ + f"- [{media}]({link})" if link else f"- {media}" + for media, link in url.items() + if stats[media] + ] ) message = f"Posted to:\n\n{url_text}" if url_text else "No posts created." @@ -153,7 +157,7 @@ def process_markdown_file(self, file_path, processed_files): def process_files(self, files_to_process): processed_files = {} - messages = "---\n" + messages = "" processed_files_path = self.json_out if os.path.exists(processed_files_path): with open(processed_files_path, "r") as file: @@ -162,7 +166,7 @@ def process_files(self, files_to_process): processed_files, message = self.process_markdown_file( file_path, processed_files ) - messages += f"{message}\n---\n" + messages += f"{message}\n\n---\n" if not self.preview: with open(processed_files_path, "w") as file: json.dump(processed_files, file) diff --git a/lib/plugins/bluesky.py b/lib/plugins/bluesky.py index 13680ea..215c642 100644 --- a/lib/plugins/bluesky.py +++ b/lib/plugins/bluesky.py @@ -149,11 +149,52 @@ def handle_url_card( ) return embed_external - def create_post( - self, content, mentions, hashtags, images, **kwargs - ) -> Tuple[bool, Optional[str]]: + def wrap_text_with_index(self, content): + if len(content) <= self.max_content_length: + return [content] + urls = re.findall(r"https?://\S+", content) + placeholder_content = re.sub( + r"https?://\S+", lambda m: "~" * len(m.group()), content + ) + wrapped_lines = textwrap.wrap( + placeholder_content, self.max_content_length - 8, replace_whitespace=False + ) + final_lines = [] + url_index = 0 + for i, line in enumerate(wrapped_lines, 1): + while "~~~~~~~~~~" in line and url_index < len(urls): + placeholder = "~" * len(urls[url_index]) + line = line.replace(placeholder, urls[url_index], 1) + url_index += 1 + final_lines.append(f"{line} ({i}/{len(wrapped_lines)})") + return final_lines + + def format_content(self, content, mentions, hashtags, images, **kwargs): + mentions = " ".join([f"@{v}" for v in mentions]) + hashtags = " ".join([f"#{v}" for v in hashtags]) + if len(images) > 4: + warnings = f"A maximum of four images, not {len(images)}, can be included in a single bluesky post." + images = images[:4] + else: + warnings = "" + + chunks = self.wrap_text_with_index(f"{content}\n\n{mentions}\n{hashtags}") + + formatted_content = { + "body": "\n\n".join(chunks), + "images": images, + "chunks": chunks, + } + preview = formatted_content["body"] + images_preview = "\n".join( + [f'![{image.get("alt_text", "")}]({image["url"]})' for image in images] + ) + preview += "\n\n" + images_preview + return formatted_content, preview, warnings + + def create_post(self, content, **kwargs) -> Tuple[bool, Optional[str]]: embed_images = [] - for image in images[:4]: + for image in content["images"][:4]: response = requests.get(image["url"]) if response.status_code == 200 and response.headers.get( "Content-Type", "" @@ -172,17 +213,11 @@ def create_post( else None ) - status = [] reply_to = None - mentions = " ".join([f"@{v}" for v in mentions]) - hashtags = " ".join([f"#{v}" for v in hashtags]) - for text in textwrap.wrap( - content + "\n" + mentions + "\n" + hashtags, - self.max_content_length, - replace_whitespace=False, - ): + + for text in content["chunks"]: facets, last_url = self.parse_facets(text) - if not images or reply_to: + if not content["images"] or reply_to: embed = self.handle_url_card(cast(str, last_url)) post = self.blueskysocial.send_post( @@ -192,8 +227,9 @@ def create_post( for _ in range(5): data = self.blueskysocial.get_posts([post.uri]).posts if data: - status.append(data[0].record.text == text) break + else: + return False, None if reply_to is None: link = f"https://bsky.app/profile/{self.blueskysocial.me.handle}/post/{post.uri.split('/')[-1]}" @@ -201,4 +237,4 @@ def create_post( parent = atproto.models.create_strong_ref(post) reply_to = atproto.models.AppBskyFeedPost.ReplyRef(parent=parent, root=root) - return all(status), link + return True, link diff --git a/lib/plugins/markdown.py b/lib/plugins/markdown.py index f707f9f..7c24bc5 100644 --- a/lib/plugins/markdown.py +++ b/lib/plugins/markdown.py @@ -12,10 +12,7 @@ def __init__(self, **kwargs): def format_content(self, content, mentions, hashtags, images, **kwargs): _images = "\n".join( - [ - f'![{image.get("alt_text", "")}]({image["url"]})' - for image in images - ] + [f'![{image.get("alt_text", "")}]({image["url"]})' for image in images] ) mentions = " ".join([f"@{v}" for v in mentions]) hashtags = " ".join([f"#{v}" for v in hashtags]) @@ -29,13 +26,10 @@ def create_post(self, formatted_content, **kwargs): if self.save_path: os.makedirs(self.save_path, exist_ok=True) prefix = kwargs.get("file_path", "").replace(".md", "") - file_name = ( - f"{self.save_path}/{prefix.replace('/', '-')}_{time.strftime('%Y%m%d-%H%M%S')}.md" - ) + file_name = f"{self.save_path}/{prefix.replace('/', '-')}_{time.strftime('%Y%m%d-%H%M%S')}.md" with open(file_name, "w") as f: f.write(formatted_content) return True, None except Exception as e: - print(e) - return False, None - + print(e) + return False, None diff --git a/lib/plugins/mastodon.py b/lib/plugins/mastodon.py index 50d07fc..3af2196 100644 --- a/lib/plugins/mastodon.py +++ b/lib/plugins/mastodon.py @@ -1,8 +1,8 @@ +import re import tempfile import textwrap import requests -from bs4 import BeautifulSoup from mastodon import Mastodon @@ -14,6 +14,26 @@ def __init__(self, **kwargs): ) self.max_content_length = kwargs.get("max_content_length", 500) + def wrap_text_with_index(self, content): + if len(content) <= self.max_content_length: + return [content] + urls = re.findall(r"https?://\S+", content) + placeholder_content = re.sub( + r"https?://\S+", lambda m: "~" * len(m.group()), content + ) + wrapped_lines = textwrap.wrap( + placeholder_content, self.max_content_length - 8, replace_whitespace=False + ) + final_lines = [] + url_index = 0 + for i, line in enumerate(wrapped_lines, 1): + while "~~~~~~~~~~" in line and url_index < len(urls): + placeholder = "~" * len(urls[url_index]) + line = line.replace(placeholder, urls[url_index], 1) + url_index += 1 + final_lines.append(f"{line} ({i}/{len(wrapped_lines)})") + return final_lines + def format_content(self, content, mentions, hashtags, images, **kwargs): mentions = " ".join([f"@{v}" for v in mentions]) hashtags = " ".join([f"#{v}" for v in hashtags]) @@ -22,16 +42,17 @@ def format_content(self, content, mentions, hashtags, images, **kwargs): images = images[:4] else: warnings = "" + + chunks = self.wrap_text_with_index(f"{content}\n\n{mentions}\n{hashtags}") + formatted_content = { - "body": f"{content}\n\n{mentions}\n{hashtags}", + "body": "\n\n".join(chunks), "images": images, + "chunks": chunks, } preview = formatted_content["body"] images_preview = "\n".join( - [ - f'![{image.get("alt_text", "")}]({image["url"]})' - for image in images - ] + [f'![{image.get("alt_text", "")}]({image["url"]})' for image in images] ) preview += "\n\n" + images_preview return formatted_content, preview, warnings @@ -53,12 +74,7 @@ def create_post(self, content, **kwargs): media_ids.append(media_uploaded["id"]) toot_id = link = None - status = [] - for text in textwrap.wrap( - content["body"], - self.max_content_length, - replace_whitespace=False, - ): + for text in content["chunks"]: toot = self.mastodon_handle.status_post( status=text, in_reply_to_id=toot_id, diff --git a/lib/plugins/matrix.py b/lib/plugins/matrix.py index 3897a55..a443ab6 100644 --- a/lib/plugins/matrix.py +++ b/lib/plugins/matrix.py @@ -49,19 +49,28 @@ async def async_format_content(self, content, mentions, hashtags, images, **kwar # try to get the display name of the mentioned matrix user response = await self.client.get_displayname(f"@{mention}") mention_name = getattr(response, "displayname", mention) - mention_links.append(f"[{mention_name}](https://matrix.to/#/@{mention})") + mention_links.append( + f"[{mention_name}](https://matrix.to/#/@{mention})" + ) message_content["m.mentions"]["user_ids"].append(f"@{mention}") mentions_string = " ".join(mention_links) content = f"{mentions_string}: {content}" if hashtags: content += "\n\n" + " ".join([f"\\#{h}" for h in hashtags]) formatted_body = markdown(content) - body = BeautifulSoup(formatted_body, features="html.parser").get_text("\n", strip=True) + body = BeautifulSoup(formatted_body, features="html.parser").get_text( + "\n", strip=True + ) message_content["body"] = body message_content["formatted_body"] = formatted_body formatted_content.append(message_content) warnings = "" - return formatted_content, preview + "\n" + message_content["formatted_body"], warnings + await self.client.close() + return ( + formatted_content, + preview + "\n" + message_content["formatted_body"], + warnings, + ) async def async_create_post(self, content): for msg in content: @@ -113,9 +122,10 @@ async def async_create_post(self, content): return True, event_link def format_content(self, *args, **kwargs): - return self.runner.run(self.async_format_content(*args, **kwargs)) + result = self.runner.run(self.async_format_content(*args, **kwargs)) + return result def create_post(self, content, **kwargs): # hashtags and alt_texts are not used in this function - return self.runner.run(self.async_create_post(content)) - + result = self.runner.run(self.async_create_post(content)) + return result diff --git a/lib/plugins/slack.py b/lib/plugins/slack.py index 7619523..cf2f2ec 100644 --- a/lib/plugins/slack.py +++ b/lib/plugins/slack.py @@ -1,3 +1,4 @@ +import re import textwrap import requests @@ -10,6 +11,42 @@ def __init__(self, **kwargs): self.channel_id = kwargs.get("channel_id") self.max_content_length = kwargs.get("max_content_length", 40000) + def wrap_text_with_index(self, content): + if len(content) <= self.max_content_length: + return [content] + urls = re.findall(r"https?://\S+", content) + placeholder_content = re.sub( + r"https?://\S+", lambda m: "~" * len(m.group()), content + ) + wrapped_lines = textwrap.wrap( + placeholder_content, self.max_content_length - 8, replace_whitespace=False + ) + final_lines = [] + url_index = 0 + for i, line in enumerate(wrapped_lines, 1): + while "~~~~~~~~~~" in line and url_index < len(urls): + placeholder = "~" * len(urls[url_index]) + line = line.replace(placeholder, urls[url_index], 1) + url_index += 1 + final_lines.append(f"{line} ({i}/{len(wrapped_lines)})") + return final_lines + + def format_content(self, content, mentions, hashtags, images, **kwargs): + warnings = "" + chunks = self.wrap_text_with_index(content) + + formatted_content = { + "body": "\n\n".join(chunks), + "images": images, + "chunks": chunks, + } + preview = formatted_content["body"] + images_preview = "\n".join( + [f'![{image.get("alt_text", "")}]({image["url"]})' for image in images] + ) + preview += "\n\n" + images_preview + return formatted_content, preview, warnings + def upload_images(self, images): uploaded_files = [] for image in images: @@ -40,15 +77,10 @@ def upload_images(self, images): ) return response - def create_post(self, text, mentions, hashtags, images, **kwargs): - status = [] + def create_post(self, content, **kwargs): link = None parent_ts = None - for text in textwrap.wrap( - text, - self.max_content_length, - replace_whitespace=False, - ): + for text in content["chunks"]: response = self.client.chat_postMessage( channel=self.channel_id, text=text, @@ -59,8 +91,10 @@ def create_post(self, text, mentions, hashtags, images, **kwargs): link = self.client.chat_getPermalink( channel=self.channel_id, message_ts=parent_ts )["permalink"] - status.append(response["ok"]) - if images: - response = self.upload_images(images) - status.append(response["ok"]) - return all(status), link + if not response["ok"]: + return False, None + if content["images"]: + response = self.upload_images(content["images"]) + if not response["ok"]: + return False, None + return True, link diff --git a/requirements.txt b/requirements.txt index b8340b6..6adcef0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ matrix-nio==0.24.0 Pillow==10.3.0 PyYAML==6.0.1 slack_sdk==3.27.1 -jsonschema==4.21.1 \ No newline at end of file +jsonschema==4.21.1 +Markdown==3.6 \ No newline at end of file