From 79cf8a6b4a5a327a69baa5379f1ce026706e04d7 Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 19 Aug 2024 01:30:08 +1200 Subject: [PATCH 01/19] feat: email templates & SMTP --- .env.example | 13 +++++++ email_templates/_base.html | 23 ++++++++++++ email_templates/_base.txt | 11 ++++++ email_templates/_lockdown.html | 9 +++++ email_templates/_lockdown.txt | 3 ++ email_templates/email_changed.html | 4 +++ email_templates/email_changed.txt | 4 +++ email_templates/locked.html | 6 ++++ email_templates/locked.txt | 8 +++++ email_templates/mfa_added.html | 4 +++ email_templates/mfa_added.txt | 4 +++ email_templates/mfa_removed.html | 4 +++ email_templates/mfa_removed.txt | 4 +++ email_templates/password_changed.html | 4 +++ email_templates/password_changed.txt | 4 +++ email_templates/recovery.html | 12 +++++++ email_templates/recovery.txt | 6 ++++ email_templates/verify.html | 12 +++++++ email_templates/verify.txt | 6 ++++ security.py | 50 ++++++++++++++++++++++++++- test.html | 26 ++++++++++++++ 21 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 email_templates/_base.html create mode 100644 email_templates/_base.txt create mode 100644 email_templates/_lockdown.html create mode 100644 email_templates/_lockdown.txt create mode 100644 email_templates/email_changed.html create mode 100644 email_templates/email_changed.txt create mode 100644 email_templates/locked.html create mode 100644 email_templates/locked.txt create mode 100644 email_templates/mfa_added.html create mode 100644 email_templates/mfa_added.txt create mode 100644 email_templates/mfa_removed.html create mode 100644 email_templates/mfa_removed.txt create mode 100644 email_templates/password_changed.html create mode 100644 email_templates/password_changed.txt create mode 100644 email_templates/recovery.html create mode 100644 email_templates/recovery.txt create mode 100644 email_templates/verify.html create mode 100644 email_templates/verify.txt create mode 100644 test.html diff --git a/.env.example b/.env.example index 9c74d34..de92fd4 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,19 @@ INTERNAL_API_TOKEN="" # used for authenticating internal API requests (gives ac CAPTCHA_SITEKEY= CAPTCHA_SECRET= +EMAIL_SMTP_HOST= +EMAIL_SMTP_PORT= +EMAIL_SMTP_TLS= +EMAIL_SMTP_USERNAME= +EMAIL_SMTP_PASSWORD= +EMAIL_FROM_NAME= +EMAIL_FROM_ADDRESS= +EMAIL_PLATFORM_NAME="Meower" +EMAIL_PLATFORM_LOGO="" +EMAIL_PLATFORM_BRAND="Meower Media" +EMAIL_PLATFORM_FRONTEND="https://meower.org" +EMAIL_PLATFORM_SUPPORT="support@meower.org" + GRPC_AUTH_ADDRESS="0.0.0.0:5000" GRPC_AUTH_TOKEN= diff --git a/email_templates/_base.html b/email_templates/_base.html new file mode 100644 index 0000000..d9740d1 --- /dev/null +++ b/email_templates/_base.html @@ -0,0 +1,23 @@ + + + +
+
+ {{ env['EMAIL_PLATFORM_NAME'] }} Logo +
+
+ {{ subject }} + Hey {{ name }}! + {% block body %}{% endblock %} + + {# We include the platform brand in the _lockdown.html file as well #} + {# because doing extends seems to cut-off the rest of the file. #} + {% if include_lockdown %} + {% extends "_lockdown.html" %} + {% else %} + - {{ env['EMAIL_PLATFORM_BRAND'] }} + {% endif %} +
+
+ + \ No newline at end of file diff --git a/email_templates/_base.txt b/email_templates/_base.txt new file mode 100644 index 0000000..18a9958 --- /dev/null +++ b/email_templates/_base.txt @@ -0,0 +1,11 @@ +Hey {{ name }}! + +{% block body %}{% endblock %} + +{# We include the platform brand in the _lockdown.txt file as well #} +{# because doing extends seems to cut-off the rest of the file. #} +{% if include_lockdown %} +{% extends "_lockdown.txt" %} +{% else %} +- {{ env['EMAIL_PLATFORM_BRAND'] }} +{% endif %} \ No newline at end of file diff --git a/email_templates/_lockdown.html b/email_templates/_lockdown.html new file mode 100644 index 0000000..05dfde2 --- /dev/null +++ b/email_templates/_lockdown.html @@ -0,0 +1,9 @@ +If you didn't request this, please click the button below within the next 24 hours to revert this change and secure your account. + + This wasn't me! + +- {{ env['EMAIL_PLATFORM_BRAND'] }} \ No newline at end of file diff --git a/email_templates/_lockdown.txt b/email_templates/_lockdown.txt new file mode 100644 index 0000000..0f46b00 --- /dev/null +++ b/email_templates/_lockdown.txt @@ -0,0 +1,3 @@ +If you didn't request this, please follow this link within the next 24 hours to revert this change and secure your account: {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/lockdown#{{ token }} + +- {{ env['EMAIL_PLATFORM_BRAND'] }} \ No newline at end of file diff --git a/email_templates/email_changed.html b/email_templates/email_changed.html new file mode 100644 index 0000000..48292a3 --- /dev/null +++ b/email_templates/email_changed.html @@ -0,0 +1,4 @@ +{% extends "_base.html" %} +{% block body %} + The email on your {{ env['EMAIL_PLATFORM_NAME'] }} account was recently changed. +{% endblock %} \ No newline at end of file diff --git a/email_templates/email_changed.txt b/email_templates/email_changed.txt new file mode 100644 index 0000000..8e0a969 --- /dev/null +++ b/email_templates/email_changed.txt @@ -0,0 +1,4 @@ +{% extends "_base.txt" %} +{% block body %} +The email on your {{ env['EMAIL_PLATFORM_NAME'] }} account was recently changed. +{% endblock %} \ No newline at end of file diff --git a/email_templates/locked.html b/email_templates/locked.html new file mode 100644 index 0000000..13613af --- /dev/null +++ b/email_templates/locked.html @@ -0,0 +1,6 @@ +{% extends "_base.html" %} +{% block body %} + Your {{ env['EMAIL_PLATFORM_NAME'] }} account has been locked because we believe it may have been compromised. This can happen if your {{ env['EMAIL_PLATFORM_NAME'] }} password is weak, you used the same password on another website and that website was hacked, or you accidentally gave an access token to someone else. + You will be required to reset your password using this email address ({{ address }}) before logging back in to {{ env['EMAIL_PLATFORM_NAME'] }}. + If you have any questions, please reach out to {{ env['EMAIL_PLATFORM_SUPPORT'] }}. +{% endblock %} \ No newline at end of file diff --git a/email_templates/locked.txt b/email_templates/locked.txt new file mode 100644 index 0000000..52dcec3 --- /dev/null +++ b/email_templates/locked.txt @@ -0,0 +1,8 @@ +{% extends "_base.txt" %} +{% block body %} +Your {{ env['EMAIL_PLATFORM_NAME'] }} account has been locked because we believe it may have been compromised. This can happen if your {{ env['EMAIL_PLATFORM_NAME'] }} password is weak, you used the same password on another website and that website was hacked, or you accidentally gave an access token to someone else. + +You will be required to reset your password using this email address ({{ address }}) before logging back in to {{ env['EMAIL_PLATFORM_NAME'] }}. + +If you have any questions, please reach out to {{ env['EMAIL_PLATFORM_SUPPORT'] }}. +{% endblock %} \ No newline at end of file diff --git a/email_templates/mfa_added.html b/email_templates/mfa_added.html new file mode 100644 index 0000000..2b3d011 --- /dev/null +++ b/email_templates/mfa_added.html @@ -0,0 +1,4 @@ +{% extends "_base.html" %} +{% block body %} + A multi-factor authenticator was recently added to your {{ env['EMAIL_PLATFORM_NAME'] }} account. +{% endblock %} \ No newline at end of file diff --git a/email_templates/mfa_added.txt b/email_templates/mfa_added.txt new file mode 100644 index 0000000..619d8c9 --- /dev/null +++ b/email_templates/mfa_added.txt @@ -0,0 +1,4 @@ +{% extends "_base.txt" %} +{% block body %} +A multi-factor authenticator was recently added to your {{ env['EMAIL_PLATFORM_NAME'] }} account. +{% endblock %} \ No newline at end of file diff --git a/email_templates/mfa_removed.html b/email_templates/mfa_removed.html new file mode 100644 index 0000000..e2f2e51 --- /dev/null +++ b/email_templates/mfa_removed.html @@ -0,0 +1,4 @@ +{% extends "_base.html" %} +{% block body %} + A multi-factor authenticator was recently removed from your {{ env['EMAIL_PLATFORM_NAME'] }} account. +{% endblock %} \ No newline at end of file diff --git a/email_templates/mfa_removed.txt b/email_templates/mfa_removed.txt new file mode 100644 index 0000000..735ad0b --- /dev/null +++ b/email_templates/mfa_removed.txt @@ -0,0 +1,4 @@ +{% extends "_base.txt" %} +{% block body %} +A multi-factor authenticator was recently removed from your {{ env['EMAIL_PLATFORM_NAME'] }} account. +{% endblock %} \ No newline at end of file diff --git a/email_templates/password_changed.html b/email_templates/password_changed.html new file mode 100644 index 0000000..ce19c8c --- /dev/null +++ b/email_templates/password_changed.html @@ -0,0 +1,4 @@ +{% extends "_base.html" %} +{% block body %} + The password to your {{ env['EMAIL_PLATFORM_NAME'] }} account was recently changed. +{% endblock %} \ No newline at end of file diff --git a/email_templates/password_changed.txt b/email_templates/password_changed.txt new file mode 100644 index 0000000..a9b4cb4 --- /dev/null +++ b/email_templates/password_changed.txt @@ -0,0 +1,4 @@ +{% extends "_base.txt" %} +{% block body %} +The password to your {{ env['EMAIL_PLATFORM_NAME'] }} account was recently changed. +{% endblock %} \ No newline at end of file diff --git a/email_templates/recovery.html b/email_templates/recovery.html new file mode 100644 index 0000000..8be3de8 --- /dev/null +++ b/email_templates/recovery.html @@ -0,0 +1,12 @@ +{% extends "_base.html" %} +{% block body %} + To reset your {{ env['EMAIL_PLATFORM_NAME'] }} account password, please click the button below. + If you didn't request this, please ignore this email, no further action is required. + + Reset Password + +{% endblock %} \ No newline at end of file diff --git a/email_templates/recovery.txt b/email_templates/recovery.txt new file mode 100644 index 0000000..79fe129 --- /dev/null +++ b/email_templates/recovery.txt @@ -0,0 +1,6 @@ +{% extends "_base.txt" %} +{% block body %} +To reset your {{ env['EMAIL_PLATFORM_NAME'] }} account password, please follow this link: {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/recovery#{{ token }} + +If you didn't request this, please ignore this email, no further action is required. +{% endblock %} \ No newline at end of file diff --git a/email_templates/verify.html b/email_templates/verify.html new file mode 100644 index 0000000..d698b37 --- /dev/null +++ b/email_templates/verify.html @@ -0,0 +1,12 @@ +{% extends "_base.html" %} +{% block body %} + To confirm adding your email address ({{ email }}) to your {{ env['EMAIL_PLATFORM_NAME'] }} account, please click the button below. + If you didn't request this, please ignore this email, no further action is required. + + Verify Email Address + +{% endblock %} \ No newline at end of file diff --git a/email_templates/verify.txt b/email_templates/verify.txt new file mode 100644 index 0000000..e7a8fca --- /dev/null +++ b/email_templates/verify.txt @@ -0,0 +1,6 @@ +{% extends "_base.txt" %} +{% block body %} +To confirm adding your email address ({{ email }}) to your {{ env['EMAIL_PLATFORM_NAME'] }} account, please follow this link: {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/verify#{{ token }} + +If you didn't request this, please ignore this email, no further action is required. +{% endblock %} \ No newline at end of file diff --git a/security.py b/security.py index 6f5db22..ba425f6 100644 --- a/security.py +++ b/security.py @@ -1,6 +1,9 @@ from hashlib import sha256 from typing import Optional -import time, requests, os, uuid, secrets, bcrypt, msgpack +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formataddr +import time, requests, os, uuid, secrets, bcrypt, msgpack, jinja2, smtplib from database import db, rdb from utils import log @@ -43,6 +46,10 @@ TOKEN_BYTES = 64 +email_file_loader = jinja2.FileSystemLoader("email_templates") +email_env = jinja2.Environment(loader=email_file_loader) + + class UserFlags: SYSTEM = 1 DELETED = 2 @@ -536,3 +543,44 @@ def hash_password(password: str) -> str: def check_password_hash(password: str, hashed_password: str) -> bool: return bcrypt.checkpw(password.encode(), hashed_password.encode()) + + +def send_email(template: str, to_name: str, to_address: str, token: Optional[str] = ""): + txt_tmpl = email_env.get_template(f"{template}.txt") + html_tmpl = email_env.get_template(f"{template}.html") + + message = MIMEMultipart("alternative") + message["From"] = formataddr((os.environ["EMAIL_FROM_NAME"], os.environ["EMAIL_FROM_ADDRESS"])) + message["To"] = formataddr((to_name, to_address)) + + match template: + case "verify": + message["Subject"] = "Verify your email address" + case "recovery": + message["Subject"] = "Reset your password" + case "email_changed": + message["Subject"] = "Your email has been changed" + case "password_changed": + message["Subject"] = "Your password has been changed" + case "mfa_added": + message["Subject"] = "Multi-factor authenticator added" + case "mfa_removed": + message["Subject"] = "Multi-factor authenticator removed" + case "locked": + message["Subject"] = "Your account has been locked" + + data = { + "subject": message["Subject"], + "name": to_name, + "address": to_address, + "token": token, + "env": os.environ + } + message.attach(MIMEText(txt_tmpl.render(data), "plain")) + message.attach(MIMEText(html_tmpl.render(data), "html")) + + with smtplib.SMTP(os.environ["EMAIL_SMTP_HOST"], int(os.environ["EMAIL_SMTP_PORT"])) as server: + if os.getenv("EMAIL_SMTP_TLS"): + server.starttls() + server.login(os.environ["EMAIL_SMTP_USERNAME"], os.environ["EMAIL_SMTP_PASSWORD"]) + server.sendmail(os.environ["EMAIL_FROM_ADDRESS"], to_address, message.as_string()) diff --git a/test.html b/test.html new file mode 100644 index 0000000..3d62a0c --- /dev/null +++ b/test.html @@ -0,0 +1,26 @@ + + + +
+
+ Logo +
+
+ Your email has been changed + Hey ! + + The email on your account was recently changed. + + + + + + If you didn't request this, please click the button below within the next 24 hours to revert this change and secure your account. + + This wasn't me! + +- Meower Media \ No newline at end of file From 27181fc89b5561409426a1796b0d17c6ea9783ad Mon Sep 17 00:00:00 2001 From: Tnix Date: Wed, 21 Aug 2024 01:45:01 +1200 Subject: [PATCH 02/19] add ed25519 signing (not used for anything yet) --- database.py | 33 +++++++++++++++++++++++++++++---- security.py | 28 ++++++++++++++++++++++++++-- signing.py | 35 +++++++++++++++++++++++++++++++++++ test.html | 26 -------------------------- utils.py | 2 -- 5 files changed, 90 insertions(+), 34 deletions(-) create mode 100644 signing.py delete mode 100644 test.html diff --git a/database.py b/database.py index c1d8e14..f9301ea 100644 --- a/database.py +++ b/database.py @@ -1,10 +1,14 @@ import pymongo +import pymongo.errors import redis import os import secrets +import time from radix import Radix +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey -from utils import log +from utils import log, create_ed25519_keys, import_priv_ed25519_key, import_pub_ed25519_key +from signing import SigningKeys CURRENT_DB_VERSION = 9 @@ -198,20 +202,41 @@ "last_seen": None, "delete_after": None }) - except: pass + except pymongo.errors.DuplicateKeyError: pass try: db.config.insert_one({ "_id": "migration", "database": 1 }) -except: pass +except pymongo.errors.DuplicateKeyError: pass try: db.config.insert_one({ "_id": "status", "repair_mode": False, "registration": True }) -except: pass +except pymongo.errors.DuplicateKeyError: pass + + +# Load existing signing keys or create new ones +# The active private key should be rotated every 10 days by the background thread +# and public keys older than 90 days should be invalidated. +_priv_key = Ed25519PrivateKey.generate() +if db.config.count_documents({"_id": "signing_key"}, limit=1): + _priv_key = Ed25519PrivateKey.from_private_bytes(db.config.find_one({"_id": "signing_key"})["raw"]) +else: + db.config.update_one({"_id": "signing_key"}, {"$set": { + "raw": _priv_key.private_bytes_raw(), + "rotated_at": int(time.time()) + }}, upsert=True) + db.pub_signing_keys.insert_one({ + "raw": _priv_key.public_key().public_bytes_raw(), + "created_at": int(time.time()) + }) +signing_keys = SigningKeys(_priv_key, [ + Ed25519PublicKey.from_public_bytes(pub_signing_key["raw"]) + for pub_signing_key in db.pub_signing_keys.find({}) +]) # Load netblocks diff --git a/security.py b/security.py index ba425f6..6d59202 100644 --- a/security.py +++ b/security.py @@ -1,11 +1,11 @@ from hashlib import sha256 -from typing import Optional +from typing import Optional, Any from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formataddr import time, requests, os, uuid, secrets, bcrypt, msgpack, jinja2, smtplib -from database import db, rdb +from database import db, rdb, signing_keys from utils import log from uploads import clear_files @@ -15,6 +15,7 @@ """ SENSITIVE_ACCOUNT_FIELDS = { + "email", "pswd", "mfa_recovery_code", "tokens", @@ -261,6 +262,14 @@ def create_user_token(username: str, ip: str, used_token: Optional[str] = None) return new_token +def create_token(ttype: int, subject: Any, scopes: int, expires_in: Optional[int] = None) -> str: + pass + + +def extract_token(token: str) -> tuple[int, Any, int]: + pass + + def update_settings(username, newdata): # Check datatype if not isinstance(username, str): @@ -497,6 +506,21 @@ def background_tasks_loop(): log("Running background tasks...") + # Rotate signing key (every 10 days) + if db.config.count_documents({"_id": "signing_key", "rotated_at": {"$lt": int(time.time())-864000}}, limit=1): + new_priv_bytes, new_pub_bytes = signing_keys.rotate() + db.pub_signing_keys.insert_one({ + "raw": new_pub_bytes, + "created_at": int(time.time()) + }) + db.config.update_one({"_id": "signing_key"}, {"$set": { + "raw": new_priv_bytes, + "rotated_at": int(time.time()) + }}, upsert=True) + + # Delete public signing keys that are older than 90 days + db.pub_signing_keys.delete_many({"created_at": {"$lt": int(time.time())-7776000}}) + # Delete accounts scheduled for deletion for user in db.usersv0.find({"delete_after": {"$lt": int(time.time())}}, projection={"_id": 1}): try: diff --git a/signing.py b/signing.py new file mode 100644 index 0000000..7ff511e --- /dev/null +++ b/signing.py @@ -0,0 +1,35 @@ +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from typing import List + +class SigningKeys: + def __init__(self, priv_bytes: bytes, pub_bytes: bytes|List[bytes]): + self.priv_key = Ed25519PrivateKey.from_private_bytes(priv_bytes) + if isinstance(pub_bytes, bytes): + pub_bytes = [pub_bytes] + self.pub_keys = [ + Ed25519PublicKey.from_public_bytes(_pub_bytes) + for _pub_bytes in pub_bytes + ] + + def sign(self, data: bytes) -> bytes: + return self.priv_key.sign(data) + + def validate(self, signature: bytes, data: bytes) -> bool: + for pub_key in self.pub_keys: + try: + pub_key.verify(signature, data) + except: + continue + else: + return True + + return False + + def rotate(self) -> tuple[bytes, bytes]: + new_priv = Ed25519PrivateKey.generate() + new_pub = new_priv.public_key() + + self.priv_key = new_priv + self.pub_keys.append(new_pub) + + return new_priv.private_bytes_raw(), new_pub.public_bytes_raw() diff --git a/test.html b/test.html deleted file mode 100644 index 3d62a0c..0000000 --- a/test.html +++ /dev/null @@ -1,26 +0,0 @@ - - - -
-
- Logo -
-
- Your email has been changed - Hey ! - - The email on your account was recently changed. - - - - - - If you didn't request this, please click the button below within the next 24 hours to revert this change and secure your account. - - This wasn't me! - -- Meower Media \ No newline at end of file diff --git a/utils.py b/utils.py index 1dacd93..b359b6b 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,4 @@ -from typing import Literal from datetime import datetime -import time import sys import traceback From d54ec93f386559ca905ddd709d378a41cb0751f1 Mon Sep 17 00:00:00 2001 From: Tnix Date: Sat, 24 Aug 2024 02:53:41 +1200 Subject: [PATCH 03/19] feat: simpler signed tokens implementation --- database.py | 52 +++++++++++++++++++++++++++++------------------ rest_api/v0/me.py | 29 ++++++++++++++++++++++++++ security.py | 42 +++++++++++++++++++++++++++++++++----- 3 files changed, 98 insertions(+), 25 deletions(-) diff --git a/database.py b/database.py index f9301ea..8ee5b78 100644 --- a/database.py +++ b/database.py @@ -3,12 +3,10 @@ import redis import os import secrets -import time from radix import Radix -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from utils import log, create_ed25519_keys, import_priv_ed25519_key, import_pub_ed25519_key -from signing import SigningKeys +from utils import log CURRENT_DB_VERSION = 9 @@ -219,24 +217,38 @@ # Load existing signing keys or create new ones -# The active private key should be rotated every 10 days by the background thread -# and public keys older than 90 days should be invalidated. -_priv_key = Ed25519PrivateKey.generate() -if db.config.count_documents({"_id": "signing_key"}, limit=1): - _priv_key = Ed25519PrivateKey.from_private_bytes(db.config.find_one({"_id": "signing_key"})["raw"]) +signing_keys = {} +if db.config.count_documents({"_id": "signing_keys"}, limit=1): + data = db.config.count_documents({"_id": "signing_keys"}, limit=1) + + acc_priv = Ed25519PrivateKey.from_private_bytes(data["acc_priv"]) + email_priv = Ed25519PrivateKey.from_private_bytes(data["email_priv"]) + + signing_keys.update({ + "acc_priv": acc_priv, + "acc_pub": acc_priv.public_key(), + + "email_priv": email_priv, + "email_pub": email_priv.public_key() + }) else: - db.config.update_one({"_id": "signing_key"}, {"$set": { - "raw": _priv_key.private_bytes_raw(), - "rotated_at": int(time.time()) - }}, upsert=True) - db.pub_signing_keys.insert_one({ - "raw": _priv_key.public_key().public_bytes_raw(), - "created_at": int(time.time()) + acc_priv = Ed25519PrivateKey.generate() + email_priv = Ed25519PrivateKey.generate() + + signing_keys.update({ + "acc_priv": acc_priv, + "acc_pub": acc_priv.public_key(), + + "email_priv": email_priv, + "email_pub": email_priv.public_key() }) -signing_keys = SigningKeys(_priv_key, [ - Ed25519PublicKey.from_public_bytes(pub_signing_key["raw"]) - for pub_signing_key in db.pub_signing_keys.find({}) -]) + + data = { + "_id": "signing_keys", + "acc_priv": acc_priv.private_bytes_raw(), + "email_priv": email_priv.private_bytes_raw() + } + db.confing.insert_one(signing_keys) # Load netblocks diff --git a/rest_api/v0/me.py b/rest_api/v0/me.py index b98ac62..a2ead10 100644 --- a/rest_api/v0/me.py +++ b/rest_api/v0/me.py @@ -12,6 +12,7 @@ import qrcode, qrcode.image.svg import uuid import secrets +import os import security from database import db, rdb, get_total_pages @@ -45,6 +46,10 @@ class Config: validate_assignment = True str_strip_whitespace = True +class UpdateEmailBody(BaseModel): + password: str = Field(min_length=1, max_length=255) # change in API v1 + email: Optional[str] = Field(default=None, max_length=255) + class ChangePasswordBody(BaseModel): old: str = Field(min_length=1, max_length=255) # change in API v1 new: str = Field(min_length=8, max_length=72) @@ -200,6 +205,30 @@ async def get_relationships(): }, 200 +@me_bp.patch("/email") +@validate_request(UpdateEmailBody) +async def update_email(data: UpdateEmailBody): + # Make sure email is enabled + if not os.getenv("EMAIL_SMTP_HOST"): + return {"error": True, "type": "featureDisabled"}, 503 + + # Check authorization + if not request.user: + abort(401) + + # Check ratelimits + if security.ratelimited(f"login:u:{request.user}") or security.ratelimited(f"emailch:{request.user}"): + abort(429) + + # Check password + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) + if not security.check_password_hash(data.old, account["pswd"]): + security.ratelimit(f"login:u:{request.user}", 5, 60) + return {"error": True, "type": "invalidCredentials"}, 401 + + # Send email + + @me_bp.patch("/password") @validate_request(ChangePasswordBody) async def change_password(data: ChangePasswordBody): diff --git a/security.py b/security.py index 6d59202..78279cc 100644 --- a/security.py +++ b/security.py @@ -1,5 +1,6 @@ from hashlib import sha256 -from typing import Optional, Any +from typing import Optional, Any, Literal +from base64 import urlsafe_b64encode, urlsafe_b64decode from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formataddr @@ -47,6 +48,12 @@ TOKEN_BYTES = 64 +TOKEN_TYPES = Literal[ + "acc", # account authorization + "email", # email actions (such as email verification or account recovery) +] + + email_file_loader = jinja2.FileSystemLoader("email_templates") email_env = jinja2.Environment(loader=email_file_loader) @@ -142,6 +149,7 @@ def create_account(username: str, password: str, ip: str): "avatar": "", "avatar_color": "000000", "quote": "", + "email": "", "pswd": hash_password(password), "mfa_recovery_code": secrets.token_hex(5), "tokens": [], @@ -262,12 +270,36 @@ def create_user_token(username: str, ip: str, used_token: Optional[str] = None) return new_token -def create_token(ttype: int, subject: Any, scopes: int, expires_in: Optional[int] = None) -> str: - pass +def create_token(ttype: TOKEN_TYPES, claims: Any, expires_in: Optional[int] = None) -> str: + token = b"miau_" + ttype.encode() + + # Add claims + token += b"." + urlsafe_b64encode(msgpack(claims)) + + # Add expiration + token += b"." + urlsafe_b64encode(str(int(time.time())+expires_in).encode()) + + # Sign token and add signature to token + token += b"." + urlsafe_b64encode(signing_keys[ttype + "_priv"].sign(token)) + + return token.decode() + + +def extract_token(token: str, expected_type: TOKEN_TYPES) -> Optional[Any]: + # Extract data from the token + ttype, claims, expires_at, signature = token.split(".") + + # Check type + if ttype.replace("miau_", "") != expected_type: + return None + # Check signature + signing_keys[ttype.replace("miau_", "") + "_pub"].verify( + urlsafe_b64decode(signature), + (ttype.encode() + b"." + claims.encode() + b"." + expires_at.encode()) + ) -def extract_token(token: str) -> tuple[int, Any, int]: - pass + return msgpack.unpack(urlsafe_b64decode(claims)) def update_settings(username, newdata): From 5a1657e23feebf86b87f98dddc3cb28c2ce63427 Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 26 Aug 2024 03:42:11 +1200 Subject: [PATCH 04/19] feat: sessions and hmac tokens --- .env.example | 2 + cloudlink.py | 12 +++-- database.py | 57 ++++++++-------------- errors.py | 3 ++ grpc_auth/service.py | 23 +++++---- main.py | 4 ++ requirements.txt | 3 +- rest_api/__init__.py | 22 ++++++--- rest_api/admin.py | 32 ++++++------- rest_api/v0/auth.py | 41 ++++++++++++---- rest_api/v0/me.py | 17 +++---- security.py | 83 ++++++++------------------------ sessions.py | 112 +++++++++++++++++++++++++++++++++++++++++++ supporter.py | 9 +++- 14 files changed, 262 insertions(+), 158 deletions(-) create mode 100644 errors.py create mode 100644 sessions.py diff --git a/.env.example b/.env.example index de92fd4..3e06e5c 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,8 @@ API_ROOT= INTERNAL_API_ENDPOINT="http://127.0.0.1:3001" # used for proxying CL3 commands INTERNAL_API_TOKEN="" # used for authenticating internal API requests (gives access to any account, meant to be used by CL3) +SENTRY_DSN= + CAPTCHA_SITEKEY= CAPTCHA_SECRET= diff --git a/cloudlink.py b/cloudlink.py index 65ea0a7..feae122 100755 --- a/cloudlink.py +++ b/cloudlink.py @@ -226,7 +226,8 @@ def __init__( self.server = server self.websocket = websocket - # Set username, protocol version, IP, and trusted status + # Set account session ID, username, protocol version, IP, and trusted status + self.acc_session_id: Optional[str] = None self.username: Optional[str] = None try: self.proto_version: int = int(self.req_params.get("v")[0]) @@ -255,7 +256,7 @@ def ip(self): else: return self.websocket.remote_address - def authenticate(self, account: dict[str, Any], token: str, listener: Optional[str] = None): + def authenticate(self, acc_session: dict[str, Any], token: str, account: dict[str, Any], listener: Optional[str] = None): if self.username: self.logout() @@ -265,6 +266,7 @@ def authenticate(self, account: dict[str, Any], token: str, listener: Optional[s return self.send_statuscode("Banned", listener) # Authenticate + self.acc_session_id = acc_session["_id"] self.username = account["_id"] if self.username in self.server.usernames: self.server.usernames[self.username].append(self) @@ -275,6 +277,7 @@ def authenticate(self, account: dict[str, Any], token: str, listener: Optional[s # Send auth payload self.send("auth", { "username": self.username, + "session": acc_session, "token": token, "account": account, "relationships": self.proxy_api_request("/me/relationships", "get")["autoget"], @@ -307,6 +310,7 @@ def proxy_api_request( headers.update({ "X-Internal-Token": os.environ["INTERNAL_API_TOKEN"], "X-Internal-Ip": self.ip, + "X-Internal-UA": self.websocket.request_headers.get("User-Agent"), }) if self.username: headers["X-Internal-Username"] = self.username @@ -356,7 +360,7 @@ def send_statuscode(self, statuscode: str, listener: Optional[str] = None): def kick(self): async def _kick(): await self.websocket.close() - asyncio.create_task(_kick()) + asyncio.run(_kick()) class CloudlinkCommands: @staticmethod @@ -389,7 +393,7 @@ async def authpswd(client: CloudlinkClient, val, listener: Optional[str] = None) else: if resp and not resp["error"]: # Authenticate client - client.authenticate(resp["account"], resp["token"], listener=listener) + client.authenticate(resp["session"], resp["token"], resp["account"], listener=listener) # Tell the client it is authenticated client.send_statuscode("OK", listener) diff --git a/database.py b/database.py index 8ee5b78..5b8e057 100644 --- a/database.py +++ b/database.py @@ -4,11 +4,12 @@ import os import secrets from radix import Radix -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from hashlib import sha256 +from base64 import urlsafe_b64encode from utils import log -CURRENT_DB_VERSION = 9 +CURRENT_DB_VERSION = 10 # Create Redis connection log("Connecting to Redis...") @@ -43,8 +44,6 @@ # Create usersv0 indexes try: db.usersv0.create_index([("lower_username", pymongo.ASCENDING)], name="lower_username", unique=True) except: pass -try: db.usersv0.create_index([("tokens", pymongo.ASCENDING)], name="tokens", unique=True) -except: pass try: db.usersv0.create_index([("created", pymongo.DESCENDING)], name="recent_users") except: pass try: @@ -193,7 +192,6 @@ "avatar_color": None, "quote": None, "pswd": None, - "tokens": None, "flags": 1, "permissions": None, "ban": None, @@ -214,41 +212,17 @@ "registration": True }) except pymongo.errors.DuplicateKeyError: pass - - -# Load existing signing keys or create new ones -signing_keys = {} -if db.config.count_documents({"_id": "signing_keys"}, limit=1): - data = db.config.count_documents({"_id": "signing_keys"}, limit=1) - - acc_priv = Ed25519PrivateKey.from_private_bytes(data["acc_priv"]) - email_priv = Ed25519PrivateKey.from_private_bytes(data["email_priv"]) - - signing_keys.update({ - "acc_priv": acc_priv, - "acc_pub": acc_priv.public_key(), - - "email_priv": email_priv, - "email_pub": email_priv.public_key() +try: + db.config.insert_one({ + "_id": "signing_keys", + "acc": secrets.token_bytes(64), + "email": secrets.token_bytes(64) }) -else: - acc_priv = Ed25519PrivateKey.generate() - email_priv = Ed25519PrivateKey.generate() +except pymongo.errors.DuplicateKeyError: pass - signing_keys.update({ - "acc_priv": acc_priv, - "acc_pub": acc_priv.public_key(), - "email_priv": email_priv, - "email_pub": email_priv.public_key() - }) - - data = { - "_id": "signing_keys", - "acc_priv": acc_priv.private_bytes_raw(), - "email_priv": email_priv.private_bytes_raw() - } - db.confing.insert_one(signing_keys) +# Load signing keys +signing_keys = db.config.find_one({"_id": "signing_keys"}) # Load netblocks @@ -343,6 +317,15 @@ def get_total_pages(collection: str, query: dict, page_size: int = 25) -> int: "mfa_recovery_code": user["mfa_recovery_code"][:10] }}) + # New sessions + log("[Migrator] Adding new sessions") + from sessions import AccSession + for user in db.usersv0.find({"tokens": {"$exists": True}}, projection={"_id": 1, "tokens": 1}): + if user["tokens"]: + for token in user["tokens"]: + rdb.set(urlsafe_b64encode(sha256(token.encode()).digest()), user["_id"], ex=1209600) # 14 days + db.usersv0.update_one({"_id": user["_id"]}, {"$set": {"tokens": []}}) + db.config.update_one({"_id": "migration"}, {"$set": {"database": CURRENT_DB_VERSION}}) log(f"[Migrator] Finished Migrating DB to version {CURRENT_DB_VERSION}") diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..6bb0c3e --- /dev/null +++ b/errors.py @@ -0,0 +1,3 @@ +class InvalidTokenSignature(Exception): pass + +class SessionNotFound(Exception): pass \ No newline at end of file diff --git a/grpc_auth/service.py b/grpc_auth/service.py index e4c9181..9a433bd 100644 --- a/grpc_auth/service.py +++ b/grpc_auth/service.py @@ -6,7 +6,10 @@ auth_service_pb2 as pb2 ) +from sentry_sdk import capture_exception + from database import db +from sessions import AccSession class AuthService(pb2_grpc.AuthServicer): @@ -22,15 +25,19 @@ def CheckToken(self, request, context): if not authed: context.abort(grpc.StatusCode.UNAUTHENTICATED, "Invalid or missing token") - account = db.usersv0.find_one({"tokens": request.token}, projection={ - "_id": 1, - "ban.state": 1, - "ban.expires": 1 - }) - if account: + try: + username = AccSession.get_username_by_token(request.token) + except Exception as e: + capture_exception(e) + else: + account = db.usersv0.find_one({"_id": username}, projection={ + "_id": 1, + "ban.state": 1, + "ban.expires": 1 + }) if account and \ - (account["ban"]["state"] == "perm_ban" or \ - (account["ban"]["state"] == "temp_ban" and account["ban"]["expires"] > time.time())): + (account["ban"]["state"] == "perm_ban" or \ + (account["ban"]["state"] == "temp_ban" and account["ban"]["expires"] > time.time())): account = None return pb2.CheckTokenResp( diff --git a/main.py b/main.py index b453bb1..a2a1955 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ import asyncio import os import uvicorn +import sentry_sdk from threading import Thread @@ -16,6 +17,9 @@ if __name__ == "__main__": + # Initialise Sentry (uses SENTRY_DSN env var) + sentry_sdk.init() + # Create Cloudlink server cl = CloudlinkServer() diff --git a/requirements.txt b/requirements.txt index 5c8dca2..11e0c6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ protobuf pyotp emoji websockets -qrcode \ No newline at end of file +qrcode +sentry-sdk \ No newline at end of file diff --git a/rest_api/__init__.py b/rest_api/__init__.py index bd76c2d..47bddad 100755 --- a/rest_api/__init__.py +++ b/rest_api/__init__.py @@ -2,6 +2,7 @@ from quart_cors import cors from quart_schema import QuartSchema, RequestSchemaValidationError, validate_headers, hide from pydantic import BaseModel +from sentry_sdk import capture_exception import time, os from .v0 import v0 @@ -10,6 +11,7 @@ from .admin import admin_bp from database import db, blocked_ips, registration_blocked_ips +from sessions import AccSession import security @@ -41,6 +43,7 @@ async def internal_auth(): abort(401) request.internal_ip = request.headers.get("X-Internal-Ip") + request.headers["User-Agent"] = request.headers.get("X-Internal-UA") request.internal_username = request.headers.get("X-Internal-Username") request.bypass_captcha = True @@ -74,13 +77,18 @@ async def check_auth(headers: TokenHeader): "ban.expires": 1 }) elif headers.token: # external auth - account = db.usersv0.find_one({"tokens": headers.token}, projection={ - "_id": 1, - "flags": 1, - "permissions": 1, - "ban.state": 1, - "ban.expires": 1 - }) + try: + username = AccSession.get_username_by_token(headers.token) + except Exception as e: + capture_exception(e) + else: + account = db.usersv0.find_one({"_id": username}, projection={ + "_id": 1, + "flags": 1, + "permissions": 1, + "ban.state": 1, + "ban.expires": 1 + }) if account: if account["ban"]["state"] == "perm_ban" or (account["ban"]["state"] == "temp_ban" and account["ban"]["expires"] > time.time()): diff --git a/rest_api/admin.py b/rest_api/admin.py index 5240fdc..6da89b6 100644 --- a/rest_api/admin.py +++ b/rest_api/admin.py @@ -8,6 +8,7 @@ import security from database import db, get_total_pages, blocked_ips, registration_blocked_ips +from sessions import AccSession admin_bp = Blueprint("admin_bp", __name__, url_prefix="/admin") @@ -651,17 +652,17 @@ async def delete_user(username, query_args: DeleteUserQueryArgs): {"_id": username}, {"$set": {"delete_after": None}} ) elif deletion_mode in ["schedule", "immediate", "purge"]: - db.usersv0.update_one( - {"_id": username}, - { - "$set": { - "tokens": [], - "delete_after": int(time.time()) + (604800 if deletion_mode == "schedule" else 0), - } - }, - ) - for client in app.cl.usernames.get(username, []): - client.kick() + if deletion_mode == "schedule": + db.usersv0.update_one( + {"_id": username}, + { + "$set": { + "delete_after": int(time.time()) + (604800 if deletion_mode == "schedule" else 0), + } + }, + ) + for session in AccSession.get_all(username): + session.revoke() if deletion_mode in ["immediate", "purge"]: security.delete_account(username, purge=(deletion_mode == "purge")) else: @@ -828,12 +829,9 @@ async def kick_user(username): if not security.has_permission(request.permissions, security.AdminPermissions.KICK_USERS): abort(401) - # Revoke tokens - db.usersv0.update_one({"_id": username}, {"$set": {"tokens": []}}) - - # Kick clients - for client in app.cl.usernames.get(username, []): - client.kick() + # Revoke sessions + for session in AccSession.get_all(username): + session.revoke() # Add log security.add_audit_log( diff --git a/rest_api/v0/auth.py b/rest_api/v0/auth.py index ead9fc3..e1aadd7 100644 --- a/rest_api/v0/auth.py +++ b/rest_api/v0/auth.py @@ -4,7 +4,10 @@ from quart_schema import validate_request from pydantic import Field from typing import Optional -from database import db, registration_blocked_ips +from base64 import urlsafe_b64encode +from hashlib import sha256 +from database import db, rdb, registration_blocked_ips +from sessions import AccSession import security auth_bp = Blueprint("auth_bp", __name__, url_prefix="/auth") @@ -29,7 +32,6 @@ async def login(data: AuthRequest): account = db.usersv0.find_one({"lower_username": data.username.lower()}, projection={ "_id": 1, "flags": 1, - "tokens": 1, "pswd": 1, "mfa_recovery_code": 1 }) @@ -44,8 +46,19 @@ async def login(data: AuthRequest): if security.ratelimited(f"login:u:{account['_id']}"): abort(429) - # Check credentials - if data.password not in account["tokens"]: + # Legacy tokens (remove in the future at some point) + if len(data.password) == 86: + encoded_token = urlsafe_b64encode(sha256(data.password.encode()).digest()) + username = rdb.get(encoded_token) + if username and username.decode() == account["_id"]: + data.password = AccSession.create(username.decode(), request.ip, request.headers.get("User-Agent")).token + rdb.delete(encoded_token) + + # Check credentials & get session + try: # token for already existing session + session = AccSession.get_by_token(data.password) + session.refresh(request.ip, request.headers.get("User-Agent"), check_token=data.password) + except: # no error capturing here, as it's probably just a password rather than a token, and we don't want to capture passwords # Check password password_valid = security.check_password_hash(data.password, account["pswd"]) @@ -106,11 +119,15 @@ async def login(data: AuthRequest): "mfa_methods": list(mfa_methods) }, 401 - # Return account and token + # Create session + session = AccSession.create(account["_id"], request.ip, request.headers.get("User-Agent")) + + # Return session and account details return { "error": False, - "account": security.get_account(account['_id'], True), - "token": security.create_user_token(account['_id'], request.ip, used_token=data.password) + "session": session.v0, + "token": session.token, + "account": security.get_account(account['_id'], True) }, 200 @auth_bp.post("/register") @@ -156,9 +173,13 @@ async def register(data: AuthRequest): # Ratelimit security.ratelimit(f"register:{request.ip}:s", 5, 900) - # Return account and token + # Create session + session, token = security.create_session(data.username, request.ip, request.headers.get("User-Agent")) + + # Return session and account details return { "error": False, - "account": security.get_account(data.username, True), - "token": security.create_user_token(data.username, request.ip) + "session": session, + "token": token, + "account": security.get_account(data.username, True) }, 200 diff --git a/rest_api/v0/me.py b/rest_api/v0/me.py index a2ead10..793ff06 100644 --- a/rest_api/v0/me.py +++ b/rest_api/v0/me.py @@ -17,6 +17,7 @@ import security from database import db, rdb, get_total_pages from uploads import claim_file, delete_file +from sessions import AccSession from utils import log @@ -106,13 +107,12 @@ async def delete_account(data: DeleteAccountBody): # Schedule account for deletion db.usersv0.update_one({"_id": request.user}, {"$set": { - "tokens": [], "delete_after": int(time.time())+604800 # 7 days }}) - # Disconnect clients - for client in app.cl.usernames.get(request.user, []): - client.kick() + # Revoke sessions + for session in AccSession.get_all(request.user): + session.revoke() return {"error": False}, 200 @@ -432,12 +432,9 @@ async def delete_tokens(): if not request.user: abort(401) - # Revoke tokens - db.usersv0.update_one({"_id": request.user}, {"$set": {"tokens": []}}) - - # Disconnect clients - for client in app.cl.usernames.get(request.user, []): - client.kick() + # Revoke sessions + for session in AccSession.get_all(request.user): + session.revoke() return {"error": False}, 200 diff --git a/security.py b/security.py index 78279cc..414409d 100644 --- a/security.py +++ b/security.py @@ -1,14 +1,15 @@ -from hashlib import sha256 from typing import Optional, Any, Literal +from hashlib import sha256 from base64 import urlsafe_b64encode, urlsafe_b64decode from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formataddr -import time, requests, os, uuid, secrets, bcrypt, msgpack, jinja2, smtplib +import time, requests, os, uuid, secrets, bcrypt, hmac, msgpack, jinja2, smtplib from database import db, rdb, signing_keys from utils import log from uploads import clear_files +import errors """ Meower Security Module @@ -19,7 +20,6 @@ "email", "pswd", "mfa_recovery_code", - "tokens", "delete_after" } @@ -152,7 +152,6 @@ def create_account(username: str, password: str, ip: str): "email": "", "pswd": hash_password(password), "mfa_recovery_code": secrets.token_hex(5), - "tokens": [], "flags": 0, "permissions": 0, "ban": { @@ -233,73 +232,34 @@ def get_account(username, include_config=False): return account -def create_user_token(username: str, ip: str, used_token: Optional[str] = None) -> str: - # Get required account details - account = db.usersv0.find_one({"_id": username}, projection={ - "_id": 1, - "tokens": 1, - "delete_after": 1 - }) - - # Update netlog - db.netlog.update_one({"_id": { - "ip": ip, - "user": username, - }}, {"$set": {"last_used": int(time.time())}}, upsert=True) - - # Restore account - if account["delete_after"]: - db.usersv0.update_one({"_id": account["_id"]}, {"$set": {"delete_after": None}}) - rdb.publish("admin", msgpack.packb({ - "op": "alert_user", - "user": account["_id"], - "content": "Your account was scheduled for deletion but you logged back in. Your account is no longer scheduled for deletion! If you didn't request for your account to be deleted, please change your password immediately." - })) - - # Generate new token, revoke used token, and update last seen timestamp - new_token = secrets.token_urlsafe(TOKEN_BYTES) - account["tokens"].append(new_token) - if used_token in account["tokens"]: - account["tokens"].remove(used_token) - db.usersv0.update_one({"_id": account["_id"]}, {"$set": { - "tokens": account["tokens"], - "last_seen": int(time.time()) - }}) - - # Return new token - return new_token +def create_token(ttype: TOKEN_TYPES, claims: Any) -> str: + # Encode claims + encoded_claims = msgpack.packb(claims) + # Sign encoded claims + signature = hmac.digest(signing_keys[ttype], encoded_claims, digest=sha256) -def create_token(ttype: TOKEN_TYPES, claims: Any, expires_in: Optional[int] = None) -> str: - token = b"miau_" + ttype.encode() - - # Add claims - token += b"." + urlsafe_b64encode(msgpack(claims)) - - # Add expiration - token += b"." + urlsafe_b64encode(str(int(time.time())+expires_in).encode()) - - # Sign token and add signature to token - token += b"." + urlsafe_b64encode(signing_keys[ttype + "_priv"].sign(token)) + # Construct token + token = b".".join([urlsafe_b64encode(encoded_claims), urlsafe_b64encode(signature)]) return token.decode() def extract_token(token: str, expected_type: TOKEN_TYPES) -> Optional[Any]: - # Extract data from the token - ttype, claims, expires_at, signature = token.split(".") - - # Check type - if ttype.replace("miau_", "") != expected_type: - return None + # Extract data and signature + encoded_claims, signature = token.split(".") + encoded_claims = urlsafe_b64decode(encoded_claims) + signature = urlsafe_b64decode(signature) # Check signature - signing_keys[ttype.replace("miau_", "") + "_pub"].verify( - urlsafe_b64decode(signature), - (ttype.encode() + b"." + claims.encode() + b"." + expires_at.encode()) - ) + expected_signature = hmac.digest(signing_keys[expected_type], encoded_claims, digest=sha256) + if not hmac.compare_digest(signature, expected_signature): + raise errors.InvalidTokenSignature - return msgpack.unpack(urlsafe_b64decode(claims)) + # Decode claims + claims = msgpack.unpackb(encoded_claims) + + return claims def update_settings(username, newdata): @@ -412,7 +372,6 @@ def delete_account(username, purge=False): "quote": None, "pswd": None, "mfa_recovery_code": None, - "tokens": None, "flags": account["flags"], "permissions": None, "ban": None, diff --git a/sessions.py b/sessions.py new file mode 100644 index 0000000..c107c47 --- /dev/null +++ b/sessions.py @@ -0,0 +1,112 @@ +from typing import Optional, TypedDict +import uuid, time, msgpack + +from database import db, rdb +import security, errors + + +class AccSessionDB(TypedDict): + _id: str + user: str + ip: str + user_agent: str + created_at: int + refreshed_at: int + + +class AccSessionV0(TypedDict): + _id: str + ip: str + location: str + user_agent: str + created_at: int + refreshed_at: int + + +class AccSession: + def __init__(self, data: AccSessionDB): + self._db = data + + @classmethod + def create(cls: "AccSession", user: str, ip: str, user_agent: str) -> "AccSession": + data: AccSessionDB = { + "_id": str(uuid.uuid4()), + "user": user, + "ip": ip, + "user_agent": user_agent, + "created_at": int(time.time()), + "refreshed_at": int(time.time()) + } + db.acc_sessions.insert_one(data) + return cls(data) + + @classmethod + def get_by_id(cls: "AccSession", session_id: str) -> "AccSession": + data: Optional[AccSessionDB] = db.acc_sessions.find_one({"_id": session_id}) + if not data: + raise errors.SessionNotFound + + return cls(data) + + @classmethod + def get_by_token(cls: "AccSession", token: str) -> "AccSession": + session_id, _ = security.extract_token(token, "acc") + return cls.get_by_id(session_id) + + @classmethod + def get_username_by_token(cls: "AccSession", token: str) -> str: + session_id, _ = security.extract_token(token, "acc") + username = rdb.get(f"u{session_id}") + if username: + return username.decode() + else: + session = cls.get_by_id(session_id) + username = session.username + rdb.set(f"u{session_id}", username, ex=300) + return username + + @classmethod + def get_all(cls: "AccSession", user: str) -> list["AccSession"]: + return [cls(data) for data in db.acc_sessions.find({"user": user})] + + @property + def token(self) -> str: + return security.create_token("acc", [self._db["_id"], self._db["refreshed_at"]]) + + @property + def username(self): + return self._db["user"] + + @property + def v0(self) -> AccSessionV0: + return { + "_id": self._db["_id"], + "ip": self._db["ip"], + "location": "", + "user_agent": self._db["user_agent"], + "created_at": self._db["created_at"], + "refreshed_at": self._db["refreshed_at"] + } + + def refresh(self, ip: str, user_agent: str, check_token: Optional[str] = None): + if check_token: + # token re-use prevention + _, refreshed_at = security.extract_token(check_token, "acc") + if refreshed_at != self._db["refreshed_at"]: + return self.revoke() + + self._db.update({ + "ip": ip, + "user_agent": user_agent, + "refreshed_at": int(time.time()) + }) + db.acc_sessions.update_one({"_id": self._db["_id"]}, {"$set": self._db}) + + def revoke(self): + db.acc_sessions.delete_one({"_id": self._db["_id"]}) + rdb.delete(f"u{self._db['_id']}") + rdb.publish("admin", msgpack.packb({ + "op": "revoke_acc_session", + "user": self._db["user"], + "sid": self._db["_id"] + })) diff --git a/supporter.py b/supporter.py index bd220fa..9dacc86 100644 --- a/supporter.py +++ b/supporter.py @@ -128,8 +128,13 @@ def listen_for_admin_pubsub(self): pubsub.subscribe("admin") for msg in pubsub.listen(): try: - msg = msgpack.loads(msg["data"]) + msg = msgpack.unpackb(msg["data"]) match msg.pop("op"): + case "revoke_acc_session": + for c in self.cl.usernames.get(msg["user"], []): + if "sid" in msg and msg["sid"] != c.acc_session_id: + continue + c.kick() case "alert_user": self.create_post("inbox", msg["user"], msg["content"]) case "ban_user": @@ -163,7 +168,7 @@ def listen_for_admin_pubsub(self): # Logout user (can't kick because of async stuff) for c in self.cl.usernames.get(username, []): - c.logout() + c.kick() except: continue From b1a7446d4971e897f960ff622b99b0f1cf0b3757 Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 26 Aug 2024 16:02:07 +1200 Subject: [PATCH 05/19] set expiration for sessions and fix some other problems with sessions --- database.py | 11 +++++++++-- security.py | 21 ++++++--------------- signing.py | 35 ----------------------------------- 3 files changed, 15 insertions(+), 52 deletions(-) delete mode 100644 signing.py diff --git a/database.py b/database.py index 5b8e057..7cef238 100644 --- a/database.py +++ b/database.py @@ -61,6 +61,12 @@ try: db.authenticators.create_index([("user", pymongo.ASCENDING)], name="user") except: pass +# Create account sessions indexes +try: db.acc_sessions.create_index([("user", pymongo.ASCENDING)], name="user") +except: pass +try: db.acc_sessions.create_index([("refreshed_at", pymongo.ASCENDING)], name="refreshed_at") +except: pass + # Create data exports indexes try: db.data_exports.create_index([("user", pymongo.ASCENDING)], name="user") except: pass @@ -319,12 +325,13 @@ def get_total_pages(collection: str, query: dict, page_size: int = 25) -> int: # New sessions log("[Migrator] Adding new sessions") - from sessions import AccSession for user in db.usersv0.find({"tokens": {"$exists": True}}, projection={"_id": 1, "tokens": 1}): if user["tokens"]: for token in user["tokens"]: rdb.set(urlsafe_b64encode(sha256(token.encode()).digest()), user["_id"], ex=1209600) # 14 days - db.usersv0.update_one({"_id": user["_id"]}, {"$set": {"tokens": []}}) + db.usersv0.update_many({}, {"$unset": {"tokens": ""}}) + try: db.usersv0.drop_index("tokens") + except: pass db.config.update_one({"_id": "migration"}, {"$set": {"database": CURRENT_DB_VERSION}}) log(f"[Migrator] Finished Migrating DB to version {CURRENT_DB_VERSION}") diff --git a/security.py b/security.py index 414409d..194dc97 100644 --- a/security.py +++ b/security.py @@ -382,6 +382,9 @@ def delete_account(username, purge=False): # Delete authenticators db.authenticators.delete_many({"user": username}) + # Delete sessions + db.acc_sessions.delete_many({"user": username}) + # Delete uploaded files clear_files(username) @@ -497,21 +500,6 @@ def background_tasks_loop(): log("Running background tasks...") - # Rotate signing key (every 10 days) - if db.config.count_documents({"_id": "signing_key", "rotated_at": {"$lt": int(time.time())-864000}}, limit=1): - new_priv_bytes, new_pub_bytes = signing_keys.rotate() - db.pub_signing_keys.insert_one({ - "raw": new_pub_bytes, - "created_at": int(time.time()) - }) - db.config.update_one({"_id": "signing_key"}, {"$set": { - "raw": new_priv_bytes, - "rotated_at": int(time.time()) - }}, upsert=True) - - # Delete public signing keys that are older than 90 days - db.pub_signing_keys.delete_many({"created_at": {"$lt": int(time.time())-7776000}}) - # Delete accounts scheduled for deletion for user in db.usersv0.find({"delete_after": {"$lt": int(time.time())}}, projection={"_id": 1}): try: @@ -519,6 +507,9 @@ def background_tasks_loop(): except Exception as e: log(f"Failed to delete account {user['_id']}: {e}") + # Revoke old sessions (60 days) + db.acc_sessions.delete_many({"refreshed_at": {"$lt": int(time.time())-5184000}}) + # Purge old netinfo db.netinfo.delete_many({"last_refreshed": {"$lt": int(time.time())-2419200}}) diff --git a/signing.py b/signing.py deleted file mode 100644 index 7ff511e..0000000 --- a/signing.py +++ /dev/null @@ -1,35 +0,0 @@ -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey -from typing import List - -class SigningKeys: - def __init__(self, priv_bytes: bytes, pub_bytes: bytes|List[bytes]): - self.priv_key = Ed25519PrivateKey.from_private_bytes(priv_bytes) - if isinstance(pub_bytes, bytes): - pub_bytes = [pub_bytes] - self.pub_keys = [ - Ed25519PublicKey.from_public_bytes(_pub_bytes) - for _pub_bytes in pub_bytes - ] - - def sign(self, data: bytes) -> bytes: - return self.priv_key.sign(data) - - def validate(self, signature: bytes, data: bytes) -> bool: - for pub_key in self.pub_keys: - try: - pub_key.verify(signature, data) - except: - continue - else: - return True - - return False - - def rotate(self) -> tuple[bytes, bytes]: - new_priv = Ed25519PrivateKey.generate() - new_pub = new_priv.public_key() - - self.priv_key = new_priv - self.pub_keys.append(new_pub) - - return new_priv.private_bytes_raw(), new_pub.public_bytes_raw() From b8932d2249a2ed51f124ce03c5cd26ec23c4fe06 Mon Sep 17 00:00:00 2001 From: Tnix Date: Fri, 23 Aug 2024 22:50:13 +1200 Subject: [PATCH 06/19] fix: register /emojis blueprint --- rest_api/v0/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest_api/v0/__init__.py b/rest_api/v0/__init__.py index 3cce37f..3d9fc16 100644 --- a/rest_api/v0/__init__.py +++ b/rest_api/v0/__init__.py @@ -8,6 +8,7 @@ from .auth import auth_bp from .home import home_bp from .me import me_bp +from .emojis import emojis_bp v0 = Blueprint("v0", __name__) @@ -19,4 +20,4 @@ v0.register_blueprint(users_bp) v0.register_blueprint(chats_bp) v0.register_blueprint(search_bp) - +v0.register_blueprint(emojis_bp) From 544719c1251e4d2d2fd10bd32a6ae3d71c0eb5de Mon Sep 17 00:00:00 2001 From: Tnix Date: Fri, 23 Aug 2024 22:54:19 +1200 Subject: [PATCH 07/19] fix: don't add an extra .json --- rest_api/v0/emojis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest_api/v0/emojis.py b/rest_api/v0/emojis.py index 1536a6d..d7ac037 100644 --- a/rest_api/v0/emojis.py +++ b/rest_api/v0/emojis.py @@ -8,7 +8,7 @@ DEFAULT_EMOJIS = {} # {lang: [...]} for filename in os.listdir("emojis"): if filename.endswith(".json"): - f = open(f"emojis/{filename}.json", "r") + f = open(f"emojis/{filename}", "r") DEFAULT_EMOJIS[filename.replace(".json", "")] = json.load(f) f.close() From 3cf8adf6d67b10796a687460de530dff5b4936e2 Mon Sep 17 00:00:00 2001 From: Tnix Date: Sat, 24 Aug 2024 01:10:16 +1200 Subject: [PATCH 08/19] fix: add 'error: false' to /emojis (#297) --- rest_api/v0/emojis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest_api/v0/emojis.py b/rest_api/v0/emojis.py index d7ac037..da0bd97 100644 --- a/rest_api/v0/emojis.py +++ b/rest_api/v0/emojis.py @@ -16,6 +16,6 @@ @emojis_bp.get("/") async def get_default_emojis(lang: str): if lang in DEFAULT_EMOJIS: - return DEFAULT_EMOJIS[lang], 200 + return {"error": False, "groups": DEFAULT_EMOJIS[lang]}, 200 else: abort(404) From 5c436bd3ef99dc541b64b8841d7582d99f08d743 Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 26 Aug 2024 11:07:34 +1200 Subject: [PATCH 09/19] switch from IPHub to IP-API --- .env.example | 1 - security.py | 32 +++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 3e06e5c..f8ea401 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,6 @@ MONGO_URI=mongodb://127.0.0.1:27017 MONGO_DB=meowerserver REDIS_URI=redis://127.0.0.1:6379/0 REAL_IP_HEADER= -IPHUB_KEY= CL3_HOST="0.0.0.0" CL3_PORT=3000 API_HOST="0.0.0.0" diff --git a/security.py b/security.py index 194dc97..718e4e2 100644 --- a/security.py +++ b/security.py @@ -433,7 +433,7 @@ def delete_account(username, purge=False): def get_netinfo(ip_address): """ - Get IP info from IPHub. + Get IP info from IP-API. Returns: ```json @@ -451,21 +451,23 @@ def get_netinfo(ip_address): # Get IP hash ip_hash = sha256(ip_address.encode()).hexdigest() - # Get from database or IPHub if not cached + # Get from database or IP-API if not cached netinfo = db.netinfo.find_one({"_id": ip_hash}) if not netinfo: - iphub_key = os.getenv("IPHUB_KEY") - if iphub_key: - iphub_info = requests.get(f"http://v2.api.iphub.info/ip/{ip_address}", headers={ - "X-Key": iphub_key - }).json() + resp = requests.get(f"http://ip-api.com/json/{ip_address}?fields=25349915") + if resp.ok: + resp_json = resp.json() netinfo = { "_id": ip_hash, - "country_code": iphub_info["countryCode"], - "country_name": iphub_info["countryName"], - "asn": iphub_info["asn"], - "isp": iphub_info["isp"], - "vpn": (iphub_info["block"] == 1), + "country_code": resp_json["countryCode"], + "country_name": resp_json["country"], + "region": resp_json["regionName"], + "city": resp_json["city"], + "timezone": resp_json["timezone"], + "currency": resp_json["currency"], + "as": resp_json["as"], + "isp": resp_json["isp"], + "vpn": (resp_json.get("hosting") or resp_json.get("proxy")), "last_refreshed": int(time.time()) } db.netinfo.update_one({"_id": ip_hash}, {"$set": netinfo}, upsert=True) @@ -474,7 +476,11 @@ def get_netinfo(ip_address): "_id": ip_hash, "country_code": "Unknown", "country_name": "Unknown", - "asn": "Unknown", + "region": "Unknown", + "city": "Unknown", + "timezone": "Unknown", + "currency": "Unknown", + "as": "Unknown", "isp": "Unknown", "vpn": False, "last_refreshed": int(time.time()) From 987f962963e57ec03370f1460bbe154c7cea6ade Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 26 Aug 2024 16:15:09 +1200 Subject: [PATCH 10/19] add email tickets --- errors.py | 4 +++- sessions.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/errors.py b/errors.py index 6bb0c3e..22d59fa 100644 --- a/errors.py +++ b/errors.py @@ -1,3 +1,5 @@ class InvalidTokenSignature(Exception): pass -class SessionNotFound(Exception): pass \ No newline at end of file +class SessionNotFound(Exception): pass + +class EmailTicketExpired(Exception): pass \ No newline at end of file diff --git a/sessions.py b/sessions.py index c107c47..1ab62cb 100644 --- a/sessions.py +++ b/sessions.py @@ -1,4 +1,4 @@ -from typing import Optional, TypedDict +from typing import Optional, TypedDict, Literal import uuid, time, msgpack from database import db, rdb @@ -110,3 +110,33 @@ def revoke(self): "user": self._db["user"], "sid": self._db["_id"] })) + + +class EmailTicket: + def __init__( + self, + email_address: str, + username: str, + action: Literal["verify", "recover", "lockdown"], + expires_at: int + ): + self.email_address = email_address + self.username = username + self.action = action + self.expires_at = expires_at + + if self.expires_at < int(time.time()): + raise errors.EmailTicketExpired + + @classmethod + def get_by_token(cls: "EmailTicket", token: str) -> "EmailTicket": + return cls(*security.extract_token(token, "email")) + + @property + def token(self) -> str: + return security.create_token("email", [ + self.email_address, + self.username, + self.action, + self.expires_at + ]) From a23e1dccad50a06a73f86aff82a07db0281897fe Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 9 Sep 2024 21:15:23 +1200 Subject: [PATCH 11/19] Update email templates --- email_templates/_base.html | 24 +++++++------- email_templates/_base.txt | 8 +---- email_templates/_lockdown.html | 9 ------ email_templates/_lockdown.txt | 3 -- email_templates/email_changed.html | 4 --- email_templates/email_changed.txt | 4 --- email_templates/locked.html | 26 +++++++++++++-- email_templates/locked.txt | 2 ++ email_templates/mfa_added.html | 4 --- email_templates/mfa_added.txt | 4 --- email_templates/mfa_removed.html | 4 --- email_templates/mfa_removed.txt | 4 --- email_templates/password_changed.html | 4 --- email_templates/password_changed.txt | 4 --- email_templates/recover.html | 30 +++++++++++++++++ email_templates/{recovery.txt => recover.txt} | 2 +- email_templates/recovery.html | 12 ------- email_templates/security_alert.html | 26 +++++++++++++++ email_templates/security_alert.txt | 6 ++++ email_templates/verify.html | 32 +++++++++++++------ email_templates/verify.txt | 4 +-- 21 files changed, 127 insertions(+), 89 deletions(-) delete mode 100644 email_templates/_lockdown.html delete mode 100644 email_templates/_lockdown.txt delete mode 100644 email_templates/email_changed.html delete mode 100644 email_templates/email_changed.txt delete mode 100644 email_templates/mfa_added.html delete mode 100644 email_templates/mfa_added.txt delete mode 100644 email_templates/mfa_removed.html delete mode 100644 email_templates/mfa_removed.txt delete mode 100644 email_templates/password_changed.html delete mode 100644 email_templates/password_changed.txt create mode 100644 email_templates/recover.html rename email_templates/{recovery.txt => recover.txt} (61%) delete mode 100644 email_templates/recovery.html create mode 100644 email_templates/security_alert.html create mode 100644 email_templates/security_alert.txt diff --git a/email_templates/_base.html b/email_templates/_base.html index d9740d1..b1539e9 100644 --- a/email_templates/_base.html +++ b/email_templates/_base.html @@ -5,19 +5,21 @@
{{ env['EMAIL_PLATFORM_NAME'] }} Logo
-
- {{ subject }} - Hey {{ name }}! + + + + + + + + + {% block body %}{% endblock %} - {# We include the platform brand in the _lockdown.html file as well #} - {# because doing extends seems to cut-off the rest of the file. #} - {% if include_lockdown %} - {% extends "_lockdown.html" %} - {% else %} - - {{ env['EMAIL_PLATFORM_BRAND'] }} - {% endif %} - + + + +
{{ subject }}
Hey {{ name }}!
- {{ env['EMAIL_PLATFORM_BRAND'] }}
\ No newline at end of file diff --git a/email_templates/_base.txt b/email_templates/_base.txt index 18a9958..7cd79e3 100644 --- a/email_templates/_base.txt +++ b/email_templates/_base.txt @@ -2,10 +2,4 @@ Hey {{ name }}! {% block body %}{% endblock %} -{# We include the platform brand in the _lockdown.txt file as well #} -{# because doing extends seems to cut-off the rest of the file. #} -{% if include_lockdown %} -{% extends "_lockdown.txt" %} -{% else %} -- {{ env['EMAIL_PLATFORM_BRAND'] }} -{% endif %} \ No newline at end of file +- {{ env['EMAIL_PLATFORM_BRAND'] }} \ No newline at end of file diff --git a/email_templates/_lockdown.html b/email_templates/_lockdown.html deleted file mode 100644 index 05dfde2..0000000 --- a/email_templates/_lockdown.html +++ /dev/null @@ -1,9 +0,0 @@ -If you didn't request this, please click the button below within the next 24 hours to revert this change and secure your account. - - This wasn't me! - -- {{ env['EMAIL_PLATFORM_BRAND'] }} \ No newline at end of file diff --git a/email_templates/_lockdown.txt b/email_templates/_lockdown.txt deleted file mode 100644 index 0f46b00..0000000 --- a/email_templates/_lockdown.txt +++ /dev/null @@ -1,3 +0,0 @@ -If you didn't request this, please follow this link within the next 24 hours to revert this change and secure your account: {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/lockdown#{{ token }} - -- {{ env['EMAIL_PLATFORM_BRAND'] }} \ No newline at end of file diff --git a/email_templates/email_changed.html b/email_templates/email_changed.html deleted file mode 100644 index 48292a3..0000000 --- a/email_templates/email_changed.html +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "_base.html" %} -{% block body %} - The email on your {{ env['EMAIL_PLATFORM_NAME'] }} account was recently changed. -{% endblock %} \ No newline at end of file diff --git a/email_templates/email_changed.txt b/email_templates/email_changed.txt deleted file mode 100644 index 8e0a969..0000000 --- a/email_templates/email_changed.txt +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "_base.txt" %} -{% block body %} -The email on your {{ env['EMAIL_PLATFORM_NAME'] }} account was recently changed. -{% endblock %} \ No newline at end of file diff --git a/email_templates/locked.html b/email_templates/locked.html index 13613af..d9ece28 100644 --- a/email_templates/locked.html +++ b/email_templates/locked.html @@ -1,6 +1,26 @@ {% extends "_base.html" %} {% block body %} - Your {{ env['EMAIL_PLATFORM_NAME'] }} account has been locked because we believe it may have been compromised. This can happen if your {{ env['EMAIL_PLATFORM_NAME'] }} password is weak, you used the same password on another website and that website was hacked, or you accidentally gave an access token to someone else. - You will be required to reset your password using this email address ({{ address }}) before logging back in to {{ env['EMAIL_PLATFORM_NAME'] }}. - If you have any questions, please reach out to {{ env['EMAIL_PLATFORM_SUPPORT'] }}. + + + Your {{ env['EMAIL_PLATFORM_NAME'] }} account has been locked because we believe it may have been compromised. This can happen if your {{ env['EMAIL_PLATFORM_NAME'] }} password is weak, you used the same password on another website and that website was hacked, or you accidentally gave an access token to someone else. + + + + + + You will be required to reset your password using this email address ({{ address }}) before logging back in to {{ env['EMAIL_PLATFORM_NAME'] }}. + + + + + + If you had multi-factor authentication enabled, it has been temporarily disabled as a precaution, in case it was modified by someone attempting to lock you out of your account. + + + + + + If you have any questions, please reach out to {{ env['EMAIL_PLATFORM_SUPPORT'] }}. + + {% endblock %} \ No newline at end of file diff --git a/email_templates/locked.txt b/email_templates/locked.txt index 52dcec3..7650678 100644 --- a/email_templates/locked.txt +++ b/email_templates/locked.txt @@ -4,5 +4,7 @@ Your {{ env['EMAIL_PLATFORM_NAME'] }} account has been locked because we believe You will be required to reset your password using this email address ({{ address }}) before logging back in to {{ env['EMAIL_PLATFORM_NAME'] }}. +If you had multi-factor authentication enabled, it has been temporarily disabled as a precaution, in case it was modified by someone attempting to lock you out of your account. + If you have any questions, please reach out to {{ env['EMAIL_PLATFORM_SUPPORT'] }}. {% endblock %} \ No newline at end of file diff --git a/email_templates/mfa_added.html b/email_templates/mfa_added.html deleted file mode 100644 index 2b3d011..0000000 --- a/email_templates/mfa_added.html +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "_base.html" %} -{% block body %} - A multi-factor authenticator was recently added to your {{ env['EMAIL_PLATFORM_NAME'] }} account. -{% endblock %} \ No newline at end of file diff --git a/email_templates/mfa_added.txt b/email_templates/mfa_added.txt deleted file mode 100644 index 619d8c9..0000000 --- a/email_templates/mfa_added.txt +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "_base.txt" %} -{% block body %} -A multi-factor authenticator was recently added to your {{ env['EMAIL_PLATFORM_NAME'] }} account. -{% endblock %} \ No newline at end of file diff --git a/email_templates/mfa_removed.html b/email_templates/mfa_removed.html deleted file mode 100644 index e2f2e51..0000000 --- a/email_templates/mfa_removed.html +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "_base.html" %} -{% block body %} - A multi-factor authenticator was recently removed from your {{ env['EMAIL_PLATFORM_NAME'] }} account. -{% endblock %} \ No newline at end of file diff --git a/email_templates/mfa_removed.txt b/email_templates/mfa_removed.txt deleted file mode 100644 index 735ad0b..0000000 --- a/email_templates/mfa_removed.txt +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "_base.txt" %} -{% block body %} -A multi-factor authenticator was recently removed from your {{ env['EMAIL_PLATFORM_NAME'] }} account. -{% endblock %} \ No newline at end of file diff --git a/email_templates/password_changed.html b/email_templates/password_changed.html deleted file mode 100644 index ce19c8c..0000000 --- a/email_templates/password_changed.html +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "_base.html" %} -{% block body %} - The password to your {{ env['EMAIL_PLATFORM_NAME'] }} account was recently changed. -{% endblock %} \ No newline at end of file diff --git a/email_templates/password_changed.txt b/email_templates/password_changed.txt deleted file mode 100644 index a9b4cb4..0000000 --- a/email_templates/password_changed.txt +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "_base.txt" %} -{% block body %} -The password to your {{ env['EMAIL_PLATFORM_NAME'] }} account was recently changed. -{% endblock %} \ No newline at end of file diff --git a/email_templates/recover.html b/email_templates/recover.html new file mode 100644 index 0000000..6a33174 --- /dev/null +++ b/email_templates/recover.html @@ -0,0 +1,30 @@ +{% extends "_base.html" %} +{% block body %} + + + To reset your {{ env['EMAIL_PLATFORM_NAME'] }} account password, please click the button below. + + + + + + If you didn't request this, please ignore this email, no further action is required. + + + + + This link will expire in 30 minutes. + + + + + + Reset Password + + + +{% endblock %} \ No newline at end of file diff --git a/email_templates/recovery.txt b/email_templates/recover.txt similarity index 61% rename from email_templates/recovery.txt rename to email_templates/recover.txt index 79fe129..5035e13 100644 --- a/email_templates/recovery.txt +++ b/email_templates/recover.txt @@ -1,6 +1,6 @@ {% extends "_base.txt" %} {% block body %} -To reset your {{ env['EMAIL_PLATFORM_NAME'] }} account password, please follow this link: {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/recovery#{{ token }} +To reset your {{ env['EMAIL_PLATFORM_NAME'] }} account password, please follow this link (this link will expire in 30 minutes): {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/recover#{{ token }} If you didn't request this, please ignore this email, no further action is required. {% endblock %} \ No newline at end of file diff --git a/email_templates/recovery.html b/email_templates/recovery.html deleted file mode 100644 index 8be3de8..0000000 --- a/email_templates/recovery.html +++ /dev/null @@ -1,12 +0,0 @@ -{% extends "_base.html" %} -{% block body %} - To reset your {{ env['EMAIL_PLATFORM_NAME'] }} account password, please click the button below. - If you didn't request this, please ignore this email, no further action is required. - - Reset Password - -{% endblock %} \ No newline at end of file diff --git a/email_templates/security_alert.html b/email_templates/security_alert.html new file mode 100644 index 0000000..3e24c6d --- /dev/null +++ b/email_templates/security_alert.html @@ -0,0 +1,26 @@ +{% extends "_base.html" %} +{% block body %} + + {{ msg }} + + + + If this wasn't you, please click the button below to secure your account. + + + + This link will expire in 24 hours. + + + + + + This wasn't me! + + + +{% endblock %} \ No newline at end of file diff --git a/email_templates/security_alert.txt b/email_templates/security_alert.txt new file mode 100644 index 0000000..81a2101 --- /dev/null +++ b/email_templates/security_alert.txt @@ -0,0 +1,6 @@ +{% extends "_base.txt" %} +{% block body %} +{{ msg }} + +If this wasn't you, please follow this link to secure your account (this link will expire in 24 hours): {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/lockdown#{{ token }} +{% endblock %} \ No newline at end of file diff --git a/email_templates/verify.html b/email_templates/verify.html index d698b37..12b168c 100644 --- a/email_templates/verify.html +++ b/email_templates/verify.html @@ -1,12 +1,26 @@ {% extends "_base.html" %} {% block body %} - To confirm adding your email address ({{ email }}) to your {{ env['EMAIL_PLATFORM_NAME'] }} account, please click the button below. - If you didn't request this, please ignore this email, no further action is required. - - Verify Email Address - + + To confirm adding your email address ({{ address }}) to your {{ env['EMAIL_PLATFORM_NAME'] }} account, please click the button below. + + + + If this wasn't you, please ignore this email, no further action is required. + + + + This link will expire in 30 minutes. + + + + + + Verify Email Address + + + {% endblock %} \ No newline at end of file diff --git a/email_templates/verify.txt b/email_templates/verify.txt index e7a8fca..786ecd3 100644 --- a/email_templates/verify.txt +++ b/email_templates/verify.txt @@ -1,6 +1,6 @@ {% extends "_base.txt" %} {% block body %} -To confirm adding your email address ({{ email }}) to your {{ env['EMAIL_PLATFORM_NAME'] }} account, please follow this link: {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/verify#{{ token }} +To confirm adding your email address ({{ address }}) to your {{ env['EMAIL_PLATFORM_NAME'] }} account, please follow this link (this link will expire in 30 minutes): {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/verify#{{ token }} -If you didn't request this, please ignore this email, no further action is required. +If this wasn't you, please ignore this email, no further action is required. {% endblock %} \ No newline at end of file From 16a92d855baeaec4bae0b34fa702a3d1e91b0f64 Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 9 Sep 2024 23:00:29 +1200 Subject: [PATCH 12/19] update security alert email template --- email_templates/security_alert.html | 46 ++++++++++++++++------------- email_templates/security_alert.txt | 3 +- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/email_templates/security_alert.html b/email_templates/security_alert.html index 3e24c6d..38c755b 100644 --- a/email_templates/security_alert.html +++ b/email_templates/security_alert.html @@ -1,26 +1,32 @@ {% extends "_base.html" %} {% block body %} - - {{ msg }} - + {% if token %} + + {{ msg }} + - - If this wasn't you, please click the button below to secure your account. - + + If this wasn't you, please click the button below to secure your account. + - - This link will expire in 24 hours. - + + This link will expire in 24 hours. + - - - - This wasn't me! - - - + + + + This wasn't me! + + + + {% else %} + + {{ msg }} + + {% endif %} {% endblock %} \ No newline at end of file diff --git a/email_templates/security_alert.txt b/email_templates/security_alert.txt index 81a2101..edea1c4 100644 --- a/email_templates/security_alert.txt +++ b/email_templates/security_alert.txt @@ -1,6 +1,7 @@ {% extends "_base.txt" %} {% block body %} {{ msg }} - +{% if token %} If this wasn't you, please follow this link to secure your account (this link will expire in 24 hours): {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/lockdown#{{ token }} +{% endif %} {% endblock %} \ No newline at end of file From 86e8d377ac1f76a6e09d8fb5d97b7556082bcd06 Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 9 Sep 2024 23:00:41 +1200 Subject: [PATCH 13/19] update kick function in cl3 --- cloudlink.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cloudlink.py b/cloudlink.py index feae122..ce7c14e 100755 --- a/cloudlink.py +++ b/cloudlink.py @@ -325,8 +325,6 @@ def proxy_api_request( return resp else: match resp["type"]: - case "repairModeEnabled": - self.kick() case "ipBlocked"|"registrationBlocked": self.send_statuscode("Blocked", listener) case "badRequest": @@ -357,10 +355,8 @@ def send(self, cmd: str, val: Any, extra: Optional[dict] = None, listener: Optio def send_statuscode(self, statuscode: str, listener: Optional[str] = None): return self.send("statuscode", self.server.statuscodes[statuscode], listener=listener) - def kick(self): - async def _kick(): - await self.websocket.close() - asyncio.run(_kick()) + async def kick(self): + await self.websocket.close() class CloudlinkCommands: @staticmethod From aa7c986b50b1fced23ef01b1f91da38d63e75a17 Mon Sep 17 00:00:00 2001 From: Tnix Date: Mon, 9 Sep 2024 23:03:10 +1200 Subject: [PATCH 14/19] update cl3 kick to async --- rest_api/admin.py | 8 ++++---- supporter.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rest_api/admin.py b/rest_api/admin.py index 6da89b6..3dca416 100644 --- a/rest_api/admin.py +++ b/rest_api/admin.py @@ -710,7 +710,7 @@ async def ban_user(username, data: UpdateUserBanBody): data.state == "temp_ban" and data.expires > time.time() ): for client in app.cl.usernames.get(username, []): - client.kick() + await client.websocket.close() else: app.cl.send_event("update_config", {"ban": data.model_dump()}, usernames=[username]) @@ -1217,7 +1217,7 @@ async def create_netblock(cidr, data: NetblockBody): if data.type == 0: for client in copy(app.cl.clients): if blocked_ips.search_best(client.ip): - client.kick() + await client.websocket.close() # Add log security.add_audit_log( @@ -1319,7 +1319,7 @@ async def kick_all_clients(): # Kick all clients for client in copy(app.cl.clients): - client.kick() + await client.websocket.close() # Add log security.add_audit_log("kicked_all", request.user, request.ip, {}) @@ -1341,7 +1341,7 @@ async def enable_repair_mode(): # Kick all clients for client in copy(app.cl.clients): - client.kick() + await client.websocket.close() # Add log security.add_audit_log("enabled_repair_mode", request.user, request.ip, {}) diff --git a/supporter.py b/supporter.py index 9dacc86..d79d713 100644 --- a/supporter.py +++ b/supporter.py @@ -1,6 +1,6 @@ from threading import Thread from typing import Optional, Iterable, Any -import uuid, time, msgpack, pymongo, re, copy +import uuid, time, msgpack, pymongo, re, copy, asyncio from cloudlink import CloudlinkServer from database import db, rdb @@ -134,7 +134,7 @@ def listen_for_admin_pubsub(self): for c in self.cl.usernames.get(msg["user"], []): if "sid" in msg and msg["sid"] != c.acc_session_id: continue - c.kick() + asyncio.run(c.kick()) case "alert_user": self.create_post("inbox", msg["user"], msg["content"]) case "ban_user": @@ -168,7 +168,7 @@ def listen_for_admin_pubsub(self): # Logout user (can't kick because of async stuff) for c in self.cl.usernames.get(username, []): - c.kick() + asyncio.run(c.kick()) except: continue From 6c9d4c8e2c852049e3dd46d765eb9d0599d28ba3 Mon Sep 17 00:00:00 2001 From: Tnix Date: Tue, 10 Sep 2024 01:10:13 +1200 Subject: [PATCH 15/19] finish emails --- cloudlink.py | 5 +- database.py | 65 +++++----- errors.py | 4 +- rest_api/__init__.py | 4 +- rest_api/admin.py | 46 +++---- rest_api/v0/__init__.py | 2 + rest_api/v0/auth.py | 106 +++++++++++++-- rest_api/v0/emails.py | 187 +++++++++++++++++++++++++++ rest_api/v0/me.py | 95 ++++++++++---- security.py | 276 ++++++++++++++++++++++++++-------------- sessions.py | 53 ++++++-- 11 files changed, 639 insertions(+), 204 deletions(-) create mode 100644 rest_api/v0/emails.py diff --git a/cloudlink.py b/cloudlink.py index ce7c14e..e2cd568 100755 --- a/cloudlink.py +++ b/cloudlink.py @@ -51,7 +51,8 @@ def __init__(self): #"Kicked": "E:020 | Kicked", -- deprecated #"ChatFull": "E:023 | Chat full", -- deprecated #"LoggedOut": "I:024 | Logged out", -- deprecated - "Deleted": "E:025 | Deleted" + "Deleted": "E:025 | Deleted", + "AccountLocked": "E:026 | Account Locked" } self.commands: dict[str, function] = { # Core commands @@ -337,6 +338,8 @@ def proxy_api_request( self.send_statuscode("2FARequired", listener) case "accountDeleted": self.send_statuscode("Deleted", listener) + case "accountLocked": + self.send_statuscode("AccountLocked", listener) case "accountBanned": self.send_statuscode("Banned", listener) case "tooManyRequests": diff --git a/database.py b/database.py index 7cef238..cae9e44 100644 --- a/database.py +++ b/database.py @@ -3,6 +3,7 @@ import redis import os import secrets +import time from radix import Radix from hashlib import sha256 from base64 import urlsafe_b64encode @@ -64,9 +65,15 @@ # Create account sessions indexes try: db.acc_sessions.create_index([("user", pymongo.ASCENDING)], name="user") except: pass +try: db.acc_sessions.create_index([("ip", pymongo.ASCENDING)], name="ip") +except: pass try: db.acc_sessions.create_index([("refreshed_at", pymongo.ASCENDING)], name="refreshed_at") except: pass +# Create security log indexes +try: db.security_log.create_index([("user", pymongo.ASCENDING)], name="user") +except: pass + # Create data exports indexes try: db.data_exports.create_index([("user", pymongo.ASCENDING)], name="user") except: pass @@ -75,18 +82,6 @@ try: db.relationships.create_index([("_id.from", pymongo.ASCENDING)], name="from") except: pass -# Create netinfo indexes -try: db.netinfo.create_index([("last_refreshed", pymongo.ASCENDING)], name="last_refreshed") -except: pass - -# Create netlog indexes -try: db.netlog.create_index([("_id.ip", pymongo.ASCENDING)], name="ip") -except: pass -try: db.netlog.create_index([("_id.user", pymongo.ASCENDING)], name="user") -except: pass -try: db.netlog.create_index([("last_used", pymongo.ASCENDING)], name="last_used") -except: pass - # Create posts indexes try: db.posts.create_index([ @@ -186,25 +181,6 @@ # Create default database items -for username in ["Server", "Deleted", "Meower", "Admin", "username"]: - try: - db.usersv0.insert_one({ - "_id": username, - "lower_username": username.lower(), - "uuid": None, - "created": None, - "pfp_data": None, - "avatar": None, - "avatar_color": None, - "quote": None, - "pswd": None, - "flags": 1, - "permissions": None, - "ban": None, - "last_seen": None, - "delete_after": None - }) - except pymongo.errors.DuplicateKeyError: pass try: db.config.insert_one({ "_id": "migration", @@ -323,16 +299,39 @@ def get_total_pages(collection: str, query: dict, page_size: int = 25) -> int: "mfa_recovery_code": user["mfa_recovery_code"][:10] }}) + # Delete system users from DB + log("[Migrator] Deleting system users from DB") + db.usersv0.delete_many({"_id": {"$in": ["Server", "Deleted", "Meower", "Admin", "username"]}}) + + # Emails + log("[Migrator] Adding email addresses") + db.usersv0.update_many({"email": {"$exists": False}}, {"$set": { + "email": "", + "normalized_email_hash": "" + }}) + # New sessions log("[Migrator] Adding new sessions") - for user in db.usersv0.find({"tokens": {"$exists": True}}, projection={"_id": 1, "tokens": 1}): + for user in db.usersv0.find({ + "tokens": {"$exists": True}, + "last_seen": {"$ne": None, "$gt": int(time.time())-(86400*21)}, + }, projection={"_id": 1, "tokens": 1}): if user["tokens"]: for token in user["tokens"]: - rdb.set(urlsafe_b64encode(sha256(token.encode()).digest()), user["_id"], ex=1209600) # 14 days + rdb.set( + urlsafe_b64encode(sha256(token.encode()).digest()), + user["_id"], + ex=86400*21 # 21 days + ) db.usersv0.update_many({}, {"$unset": {"tokens": ""}}) try: db.usersv0.drop_index("tokens") except: pass + # No more netinfo and netlog + log("[Migrator] Removing netinfo and netlog") + db.netinfo.drop() + db.netlog.drop() + db.config.update_one({"_id": "migration"}, {"$set": {"database": CURRENT_DB_VERSION}}) log(f"[Migrator] Finished Migrating DB to version {CURRENT_DB_VERSION}") diff --git a/errors.py b/errors.py index 22d59fa..92f4d9b 100644 --- a/errors.py +++ b/errors.py @@ -1,5 +1,7 @@ class InvalidTokenSignature(Exception): pass -class SessionNotFound(Exception): pass +class AccSessionTokenExpired(Exception): pass + +class AccSessionNotFound(Exception): pass class EmailTicketExpired(Exception): pass \ No newline at end of file diff --git a/rest_api/__init__.py b/rest_api/__init__.py index 47bddad..6153be7 100755 --- a/rest_api/__init__.py +++ b/rest_api/__init__.py @@ -49,13 +49,11 @@ async def internal_auth(): @app.before_request -async def check_ip(): +async def get_ip(): if hasattr(request, "internal_ip") and request.internal_ip: # internal IP forwarding request.ip = request.internal_ip else: request.ip = (request.headers.get("Cf-Connecting-Ip", request.remote_addr)) - if request.path != "/status" and blocked_ips.search_best(request.ip): - return {"error": True, "type": "ipBlocked"}, 403 @app.before_request diff --git a/rest_api/admin.py b/rest_api/admin.py index 3dca416..d4c8c0d 100644 --- a/rest_api/admin.py +++ b/rest_api/admin.py @@ -532,21 +532,19 @@ async def get_user(username): # Get netlogs netlogs = [ { - "ip": netlog["_id"]["ip"], - "user": netlog["_id"]["user"], - "last_used": netlog["last_used"], + "ip": session._db["ip"], + "user": session.username, + "last_used": session._db["refreshed_at"], } - for netlog in db.netlog.find( - {"_id.user": username}, sort=[("last_used", pymongo.DESCENDING)] - ) + for session in AccSession.get_all(username) ] # Get alts alts = [ - netlog["_id"]["user"] - for netlog in db.netlog.find( - {"_id.ip": {"$in": [netlog["ip"] for netlog in netlogs]}} - ) + session["user"] + for session in db.acc_sessions.find({ + "ip": {"$in": [netlog["ip"] for netlog in netlogs]} + }) ] if username in alts: alts.remove(username) @@ -557,7 +555,7 @@ async def get_user(username): payload["recent_ips"] = [ { "ip": netlog["ip"], - "netinfo": security.get_netinfo(netlog["ip"]), + "netinfo": security.get_ip_info(netlog["ip"]), "last_used": netlog["last_used"], "blocked": ( blocked_ips.search_best(netlog["ip"]) @@ -1096,9 +1094,6 @@ async def get_netinfo(ip): if not security.has_permission(request.permissions, security.AdminPermissions.VIEW_IPS): abort(403) - # Get netinfo - netinfo = security.get_netinfo(ip) - # Get netblocks netblocks = [] for radix_node in blocked_ips.search_covering(ip): @@ -1107,16 +1102,17 @@ async def get_netinfo(ip): netblocks.append(db.netblock.find_one({"_id": radix_node.prefix})) # Get netlogs - netlogs = [ - { - "ip": netlog["_id"]["ip"], - "user": netlog["_id"]["user"], - "last_used": netlog["last_used"], - } - for netlog in db.netlog.find( - {"_id.ip": ip}, sort=[("last_used", pymongo.DESCENDING)] - ) - ] + hit_users = set() + netlogs = [] + for session in db.acc_sessions.find({"ip": ip}, sort=[("refreshed_at", pymongo.DESCENDING)]): + if session["user"] in hit_users: + continue + netlogs.append({ + "ip": session["ip"], + "user": session["user"], + "last_used": session["refreshed_at"] + }) + hit_users.add(session["user"]) # Add log security.add_audit_log("got_netinfo", request.user, request.ip, {"ip": ip}) @@ -1124,7 +1120,7 @@ async def get_netinfo(ip): # Return netinfo, netblocks, and netlogs return { "error": False, - "netinfo": netinfo, + "netinfo": security.get_ip_info(ip), "netblocks": netblocks, "netlogs": netlogs, }, 200 diff --git a/rest_api/v0/__init__.py b/rest_api/v0/__init__.py index 3d9fc16..7fb6162 100644 --- a/rest_api/v0/__init__.py +++ b/rest_api/v0/__init__.py @@ -9,10 +9,12 @@ from .home import home_bp from .me import me_bp from .emojis import emojis_bp +from .emails import emails_bp v0 = Blueprint("v0", __name__) v0.register_blueprint(auth_bp) +v0.register_blueprint(emails_bp) v0.register_blueprint(me_bp) v0.register_blueprint(home_bp) v0.register_blueprint(inbox_bp) diff --git a/rest_api/v0/auth.py b/rest_api/v0/auth.py index e1aadd7..1b35b5b 100644 --- a/rest_api/v0/auth.py +++ b/rest_api/v0/auth.py @@ -1,4 +1,4 @@ -import re, os, requests, pyotp, secrets +import re, os, requests, pyotp, secrets, time from pydantic import BaseModel from quart import Blueprint, request, abort, current_app as app from quart_schema import validate_request @@ -6,8 +6,10 @@ from typing import Optional from base64 import urlsafe_b64encode from hashlib import sha256 -from database import db, rdb, registration_blocked_ips -from sessions import AccSession +from threading import Thread + +from database import db, rdb, blocked_ips, registration_blocked_ips +from sessions import AccSession, EmailTicket import security auth_bp = Blueprint("auth_bp", __name__, url_prefix="/auth") @@ -19,6 +21,16 @@ class AuthRequest(BaseModel): mfa_recovery_code: Optional[str] = Field(default="", min_length=10, max_length=10) captcha: Optional[str] = Field(default="", max_length=2000) +class RecoverAccountBody(BaseModel): + email: str = Field(min_length=1, max_length=255, pattern=security.EMAIL_REGEX) + captcha: Optional[str] = Field(default="", max_length=2000) + + +@auth_bp.before_request +async def ip_block_check(): + if blocked_ips.search_best(request.ip): + return {"error": True, "type": "ipBlocked"}, 403 + @auth_bp.post("/login") @validate_request(AuthRequest) @@ -29,7 +41,11 @@ async def login(data: AuthRequest): security.ratelimit(f"login:i:{request.ip}", 50, 900) # Get basic account details - account = db.usersv0.find_one({"lower_username": data.username.lower()}, projection={ + account = db.usersv0.find_one({ + "email": data.username + } if "@" in data.username else { + "lower_username": data.username.lower() + }, projection={ "_id": 1, "flags": 1, "pswd": 1, @@ -42,6 +58,10 @@ async def login(data: AuthRequest): if account["flags"] & security.UserFlags.DELETED: return {"error": True, "type": "accountDeleted"}, 401 + # Make sure account isn't locked + if account["flags"] & security.UserFlags.LOCKED: + return {"error": True, "type": "accountLocked"}, 401 + # Make sure account isn't ratelimited if security.ratelimited(f"login:u:{account['_id']}"): abort(429) @@ -51,7 +71,11 @@ async def login(data: AuthRequest): encoded_token = urlsafe_b64encode(sha256(data.password.encode()).digest()) username = rdb.get(encoded_token) if username and username.decode() == account["_id"]: - data.password = AccSession.create(username.decode(), request.ip, request.headers.get("User-Agent")).token + data.password = AccSession.create( + username.decode(), + request.ip, + request.headers.get("User-Agent") + ).token rdb.delete(encoded_token) # Check credentials & get session @@ -83,6 +107,11 @@ async def login(data: AuthRequest): # Abort if password is invalid if not password_valid: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_password", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) # Check MFA @@ -98,21 +127,43 @@ async def login(data: AuthRequest): break if not passed: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_totp_code", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) elif data.mfa_recovery_code: if data.mfa_recovery_code == account["mfa_recovery_code"]: db.authenticators.delete_many({"user": account["_id"]}) + + new_recovery_code = secrets.token_hex(5) db.usersv0.update_one({"_id": account["_id"]}, {"$set": { - "mfa_recovery_code": secrets.token_hex(5) + "mfa_recovery_code": new_recovery_code }}) - app.supporter.create_post("inbox", account["_id"], "All multi-factor authenticators have been removed from your account by someone who used your multi-factor authentication recovery code. If this wasn't you, please secure your account immediately.") + security.log_security_action("mfa_recovery_used", account["_id"], { + "old_recovery_code_hash": urlsafe_b64encode(sha256(data.mfa_recovery_code.encode()).digest()).decode(), + "new_recovery_code_hash": urlsafe_b64encode(sha256(new_recovery_code.encode()).digest()).decode(), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) else: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_recovery_code", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) else: mfa_methods = set() for authenticator in authenticators: mfa_methods.add(authenticator["type"]) + security.log_security_action("auth_fail", account["_id"], { + "status": "mfa_required", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return { "error": True, "type": "mfaRequired", @@ -130,6 +181,7 @@ async def login(data: AuthRequest): "account": security.get_account(account['_id'], True) }, 200 + @auth_bp.post("/register") @validate_request(AuthRequest) async def register(data: AuthRequest): @@ -174,12 +226,46 @@ async def register(data: AuthRequest): security.ratelimit(f"register:{request.ip}:s", 5, 900) # Create session - session, token = security.create_session(data.username, request.ip, request.headers.get("User-Agent")) + session = AccSession.create(data.username, request.ip, request.headers.get("User-Agent")) # Return session and account details return { "error": False, - "session": session, - "token": token, + "session": session.v0, + "token": session.token, "account": security.get_account(data.username, True) }, 200 + + +@auth_bp.post("/recover") +@validate_request(RecoverAccountBody) +async def recover_account(data: RecoverAccountBody): + # Check ratelimits + if security.ratelimited(f"recover:{request.ip}"): + abort(429) + security.ratelimit(f"recover:{request.ip}", 3, 2700) + + # Check captcha + if os.getenv("CAPTCHA_SECRET") and not (hasattr(request, "bypass_captcha") and request.bypass_captcha): + if not requests.post("https://api.hcaptcha.com/siteverify", data={ + "secret": os.getenv("CAPTCHA_SECRET"), + "response": data.captcha, + }).json()["success"]: + return {"error": True, "type": "invalidCaptcha"}, 403 + + # Get account + account = db.usersv0.find_one({"email": data.email}, projection={"_id": 1, "email": 1, "flags": 1}) + if not account: + return {"error": False}, 200 + + # Create recovery email ticket + ticket = EmailTicket(data.email, account["_id"], "recover", expires_at=int(time.time())+1800) + + # Send email + txt_tmpl, html_tmpl = security.render_email_tmpl("recover", account["_id"], account["email"], {"token": ticket.token}) + Thread( + target=security.send_email, + args=[security.EMAIL_SUBJECTS["recover"], account["_id"], account["email"], txt_tmpl, html_tmpl] + ).start() + + return {"error": False}, 200 diff --git a/rest_api/v0/emails.py b/rest_api/v0/emails.py new file mode 100644 index 0000000..7d8365c --- /dev/null +++ b/rest_api/v0/emails.py @@ -0,0 +1,187 @@ +from quart import Blueprint, current_app as app, request, abort +from quart_schema import validate_request +from pydantic import BaseModel, Field +from hashlib import sha256 +from base64 import urlsafe_b64encode +import time, secrets + +from sessions import EmailTicket, AccSession +from database import db, rdb +import security, errors + + +emails_bp = Blueprint("emails_bp", __name__, url_prefix="/emails") + + +class VerifyEmailBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + +class RecoverAccountBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + password: str = Field(min_length=8, max_length=72) + +class LockAccountBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + + +@emails_bp.post("/verify") +@validate_request(VerifyEmailBody) +async def verify_email(data: VerifyEmailBody): + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "verify": + abort(401) + + # Make sure the email address matches the user's pending email address + pending_email = rdb.get(f"pe{ticket.username}") + if (not pending_email) or (pending_email.decode() != ticket.email_address): + abort(401) + rdb.delete(f"pe{ticket.username}") + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "email": 1 + }) + + # Log action + security.log_security_action("email_changed", account["_id"], { + "old_email_hash": security.get_normalized_email_hash(account["email"]) if account.get("email") else None, + "new_email_hash": security.get_normalized_email_hash(ticket.email_address), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Update user's email address + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "email": ticket.email_address, + "normalized_email_hash": security.get_normalized_email_hash(ticket.email_address) + }}) + app.cl.send_event("update_config", {"email": ticket.email_address}, usernames=[account["_id"]]) + + return {"error": False}, 200 + + +@emails_bp.post("/recover") +@validate_request(RecoverAccountBody) +async def recover_account(data: RecoverAccountBody): + # Make sure ticket hasn't already been used + if rdb.exists(urlsafe_b64encode(sha256(data.token.encode()).digest()).decode()): + abort(401) + + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "recover": + abort(401) + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "email": 1, + "pswd": 1, + "flags": 1 + }) + + # Make sure the ticket email matches the user's current email + if ticket.email_address != account["email"]: + abort(401) + + # Revoke ticket + rdb.set( + urlsafe_b64encode(sha256(data.token.encode()).digest()).decode(), + "", + ex=(ticket.expires_at-int(time.time()))+1 + ) + + # Update password (and remove locked flag) + new_hash = security.hash_password(data.password) + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "pswd": new_hash, + "flags": account["flags"] ^ security.UserFlags.LOCKED + }}) + + # Log action + security.log_security_action("password_changed", account["_id"], { + "method": "email", + "old_pswd_hash": account["pswd"], + "new_pswd_hash": new_hash, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Revoke sessions + for session in AccSession.get_all(account["_id"]): + session.revoke() + + return {"error": False}, 200 + + +@emails_bp.post("/lockdown") +@validate_request(LockAccountBody) +async def lock_account(data: LockAccountBody): + # Make sure ticket hasn't already been used + if rdb.exists(urlsafe_b64encode(sha256(data.token.encode()).digest()).decode()): + abort(401) + + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "lockdown": + abort(401) + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "flags": 1 + }) + + # Revoke ticket + rdb.set( + urlsafe_b64encode(sha256(data.token.encode()).digest()).decode(), + "", + ex=(ticket.expires_at-int(time.time()))+1 + ) + + # Make sure the account hasn't already been locked in the last 24 hours (lockdown tickets last for 24 hours) + # This is to stop multiple identity/credential rotations by an attacker to keep access via lockdown tickets. + if security.ratelimited(f"lock:{account['_id']}"): + abort(429) + security.ratelimit(f"lock:{account['_id']}", 1, 86400) + + # Update account + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "email": ticket.email_address, + "normalized_email_hash": security.get_normalized_email_hash(ticket.email_address), + "flags": account["flags"] | security.UserFlags.LOCKED, + "mfa_recovery_code": secrets.token_hex(5) + }}) + + # Remove authenticators + db.authenticators.delete_many({"user": account["_id"]}) + + # Log event + security.log_security_action("locked", account["_id"], { + "method": "email", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Revoke sessions + for session in AccSession.get_all(account["_id"]): + session.revoke() + + return {"error": False}, 200 diff --git a/rest_api/v0/me.py b/rest_api/v0/me.py index 793ff06..118ed49 100644 --- a/rest_api/v0/me.py +++ b/rest_api/v0/me.py @@ -3,8 +3,9 @@ from pydantic import BaseModel, Field from typing import Optional, List, Literal from copy import copy -from base64 import b64encode -from io import BytesIO +from base64 import urlsafe_b64encode +from hashlib import sha256 +from threading import Thread import pymongo import uuid import time @@ -13,11 +14,12 @@ import uuid import secrets import os +import requests import security from database import db, rdb, get_total_pages from uploads import claim_file, delete_file -from sessions import AccSession +from sessions import AccSession, EmailTicket from utils import log @@ -49,7 +51,8 @@ class Config: class UpdateEmailBody(BaseModel): password: str = Field(min_length=1, max_length=255) # change in API v1 - email: Optional[str] = Field(default=None, max_length=255) + email: Optional[str] = Field(default=None, max_length=255, pattern=security.EMAIL_REGEX) + captcha: Optional[str] = Field(default="", max_length=2000) class ChangePasswordBody(BaseModel): old: str = Field(min_length=1, max_length=255) # change in API v1 @@ -221,12 +224,36 @@ async def update_email(data: UpdateEmailBody): abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) - if not security.check_password_hash(data.old, account["pswd"]): + account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 + # Ratelimit + security.ratelimit(f"emailch:{request.user}", 3, 2700) + + # Check captcha + if os.getenv("CAPTCHA_SECRET") and not (hasattr(request, "bypass_captcha") and request.bypass_captcha): + if not requests.post("https://api.hcaptcha.com/siteverify", data={ + "secret": os.getenv("CAPTCHA_SECRET"), + "response": data.captcha, + }).json()["success"]: + return {"error": True, "type": "invalidCaptcha"}, 403 + + # Create email verification ticket + ticket = EmailTicket(data.email, request.user, "verify", expires_at=int(time.time())+1800) + + # Set pending email address + rdb.set(f"pe{request.user}", data.email, ex=1800) + # Send email + txt_tmpl, html_tmpl = security.render_email_tmpl("verify", request.user, data.email, {"token": ticket.token}) + Thread( + target=security.send_email, + args=[security.EMAIL_SUBJECTS["verify"], request.user, data.email, txt_tmpl, html_tmpl] + ).start() + + return {"error": False}, 200 @me_bp.patch("/password") @@ -241,16 +268,23 @@ async def change_password(data: ChangePasswordBody): abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) if not security.check_password_hash(data.old, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 # Update password - db.usersv0.update_one({"_id": request.user}, {"$set": {"pswd": security.hash_password(data.new)}}) - - # Send alert - app.supporter.create_post("inbox", account["_id"], "Your account password has been changed. If this wasn't requested by you, please secure your account immediately.") + new_hash = security.hash_password(data.new) + db.usersv0.update_one({"_id": request.user}, {"$set": {"pswd": new_hash}}) + + # Log event + security.log_security_action("password_changed", account["_id"], { + "method": "self", + "old_pswd_hash": account["pswd"], + "new_pswd_hash": new_hash, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return {"error": False}, 200 @@ -289,7 +323,7 @@ async def add_authenticator(data: AddAuthenticatorBody): abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1, "mfa_recovery_code": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 @@ -305,8 +339,12 @@ async def add_authenticator(data: AddAuthenticatorBody): } db.authenticators.insert_one(authenticator) - # Send alert - app.supporter.create_post("inbox", account["_id"], "A multi-factor authenticator has been added to your account. If this wasn't requested by you, please secure your account immediately.") + # Log action + security.log_security_action("mfa_added", account["_id"], { + "authenticator_id": authenticator["_id"], + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) # Return authenticator and MFA recovery code del authenticator["user"] @@ -363,7 +401,7 @@ async def remove_authenticator(authenticator_id: str, data: RemoveAuthenticatorB abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 @@ -376,8 +414,12 @@ async def remove_authenticator(authenticator_id: str, data: RemoveAuthenticatorB if result.deleted_count < 1: abort(404) - # Send alert - app.supporter.create_post("inbox", account["_id"], "A multi-factor authenticator has been removed from your account. If this wasn't requested by you, please secure your account immediately.") + # Log action + security.log_security_action("mfa_removed", account["_id"], { + "authenticator_id": authenticator_id, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return {"error": False} @@ -411,19 +453,24 @@ async def reset_mfa_recovery_code(data: ResetMFARecoveryCodeBody): abort(401) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 # Reset MFA recovery code - mfa_recovery_code = secrets.token_hex(5) - db.usersv0.update_one({"_id": request.user}, {"$set": {"mfa_recovery_code": mfa_recovery_code}}) - - # Send alert - app.supporter.create_post("inbox", account["_id"], "Your multi-factor authentication recovery code has been reset. If this wasn't requested by you, please secure your account immediately.") + new_recovery_code = secrets.token_hex(5) + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "mfa_recovery_code": new_recovery_code + }}) + security.log_security_action("mfa_recovery_reset", account["_id"], { + "old_recovery_code_hash": urlsafe_b64encode(sha256(account["mfa_recovery_code"]).digest()).decode(), + "new_recovery_code_hash": urlsafe_b64encode(sha256(new_recovery_code.encode()).digest()).decode(), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) - return {"error": False, "mfa_recovery_code": mfa_recovery_code} + return {"error": False, "mfa_recovery_code": new_recovery_code} @me_bp.delete("/tokens") diff --git a/security.py b/security.py index 718e4e2..d6f242a 100644 --- a/security.py +++ b/security.py @@ -1,10 +1,11 @@ from typing import Optional, Any, Literal from hashlib import sha256 from base64 import urlsafe_b64encode, urlsafe_b64decode +from threading import Thread from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formataddr -import time, requests, os, uuid, secrets, bcrypt, hmac, msgpack, jinja2, smtplib +import time, requests, os, uuid, secrets, bcrypt, hmac, msgpack, jinja2, smtplib, re from database import db, rdb, signing_keys from utils import log @@ -17,7 +18,7 @@ """ SENSITIVE_ACCOUNT_FIELDS = { - "email", + "normalized_email_hash", "pswd", "mfa_recovery_code", "delete_after" @@ -27,6 +28,9 @@ for key in SENSITIVE_ACCOUNT_FIELDS: SENSITIVE_ACCOUNT_FIELDS_DB_PROJECTION[key] = 0 +SYSTEM_USER_USERNAMES = {"server", "deleted", "meower", "admin", "username"} +SYSTEM_USER = {} + DEFAULT_USER_SETTINGS = { "unread_inbox": True, "theme": "orange", @@ -43,6 +47,8 @@ USERNAME_REGEX = "[a-zA-Z0-9-_]{1,20}" +# I hate this. But, thanks https://stackoverflow.com/a/201378 +EMAIL_REGEX = r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])""" TOTP_REGEX = "[0-9]{6}" BCRYPT_SALT_ROUNDS = 14 TOKEN_BYTES = 64 @@ -54,6 +60,14 @@ ] +EMAIL_SUBJECTS = { + "verify": "Verify your email address", + "recover": "Reset your password", + "security_alert": "Security alert", + "locked": "Your account has been locked" +} + + email_file_loader = jinja2.FileSystemLoader("email_templates") email_env = jinja2.Environment(loader=email_file_loader) @@ -63,6 +77,8 @@ class UserFlags: DELETED = 2 PROTECTED = 4 POST_RATELIMIT_BYPASS = 8 + REQUIRE_EMAIL = 16 # not used yet + LOCKED = 32 class AdminPermissions: @@ -134,6 +150,9 @@ def account_exists(username, ignore_case=False): log(f"Error on account_exists: Expected str for username, got {type(username)}") return False + if (ignore_case or username == username.lower()) and username.lower() in SYSTEM_USER_USERNAMES: + return True + query = ({"lower_username": username.lower()} if ignore_case else {"_id": username}) return (db.usersv0.count_documents(query, limit=1) > 0) @@ -150,6 +169,7 @@ def create_account(username: str, password: str, ip: str): "avatar_color": "000000", "quote": "", "email": "", + "normalized_email_hash": "", "pswd": hash_password(password), "mfa_recovery_code": secrets.token_hex(5), "flags": 0, @@ -173,7 +193,7 @@ def create_account(username: str, password: str, ip: str): })) # Automatically report if VPN is detected - if get_netinfo(ip)["vpn"]: + if get_ip_info(ip)["vpn"]: db.reports.insert_one({ "_id": str(uuid.uuid4()), "type": "user", @@ -183,7 +203,7 @@ def create_account(username: str, password: str, ip: str): "reports": [{ "user": "Server", "ip": ip, - "reason": "User registered while using a VPN.", + "reason": f"User registered while using a VPN ({ip}).", "comment": "", "time": int(time()) }] @@ -196,6 +216,24 @@ def get_account(username, include_config=False): log(f"Error on get_account: Expected str for username, got {type(username)}") return None + # System users + if username.lower() in SYSTEM_USER_USERNAMES: + return { + "_id": username.title(), + "lower_username": username.lower(), + "uuid": None, + "created": None, + "pfp_data": None, + "avatar": None, + "avatar_color": None, + "quote": None, + "email": None, + "flags": 1, + "permissions": None, + "ban": None, + "last_seen": None + } + # Get account account = db.usersv0.find_one({"lower_username": username.lower()}, projection=SENSITIVE_ACCOUNT_FIELDS_DB_PROJECTION) if not account: @@ -226,7 +264,8 @@ def get_account(username, include_config=False): del user_settings["_id"] account.update(user_settings) else: - # Remove ban if not including config + # Remove email and ban if not including config + del account["email"] del account["ban"] return account @@ -370,6 +409,8 @@ def delete_account(username, purge=False): "avatar": None, "avatar_color": None, "quote": None, + "email": None, + "normalized_email_hash": None, "pswd": None, "mfa_recovery_code": None, "flags": account["flags"], @@ -379,21 +420,24 @@ def delete_account(username, purge=False): "delete_after": None }}) + # Delete pending email + rdb.delete(f"pe{username}") + # Delete authenticators db.authenticators.delete_many({"user": username}) # Delete sessions db.acc_sessions.delete_many({"user": username}) + # Delete security logs + db.security_log.delete_many({"user": username}) + # Delete uploaded files clear_files(username) # Delete user settings db.user_settings.delete_one({"_id": username}) - # Delete netlogs - db.netlog.delete_many({"_id.user": username}) - # Remove from reports db.reports.update_many({"reports.user": username}, {"$pull": { "reports": {"user": username} @@ -431,62 +475,100 @@ def delete_account(username, purge=False): db.usersv0.delete_one({"_id": username}) -def get_netinfo(ip_address): - """ - Get IP info from IP-API. - - Returns: - ```json - { - "_id": str, - "country_code": str, - "country_name": str, - "asn": int, - "isp": str, - "vpn": bool +def get_ip_info(ip_address): + # Get IP hash + ip_hash = urlsafe_b64encode(sha256(ip_address.encode()).digest()).decode() + + # Get from cache + ip_info = rdb.get(f"ip{ip_hash}") + if ip_info: + return msgpack.unpackb(ip_info) + + # Get from IP-API + resp = requests.get(f"http://ip-api.com/json/{ip_address}?fields=25349915") + if resp.ok and resp.json()["status"] == "success": + resp_json = resp.json() + ip_info = { + "country_code": resp_json["countryCode"], + "country_name": resp_json["country"], + "region": resp_json["regionName"], + "city": resp_json["city"], + "timezone": resp_json["timezone"], + "currency": resp_json["currency"], + "as": resp_json["as"], + "isp": resp_json["isp"], + "vpn": (resp_json.get("hosting") or resp_json.get("proxy")) + } + rdb.set(f"ip{ip_hash}", msgpack.packb(ip_info), ex=int(time.time())+(86400*21)) # cache for 3 weeks + return ip_info + + # Fallback + return { + "country_code": "Unknown", + "country_name": "Unknown", + "region": "Unknown", + "city": "Unknown", + "timezone": "Unknown", + "currency": "Unknown", + "as": "Unknown", + "isp": "Unknown", + "vpn": False } - ``` - """ - # Get IP hash - ip_hash = sha256(ip_address.encode()).hexdigest() - - # Get from database or IP-API if not cached - netinfo = db.netinfo.find_one({"_id": ip_hash}) - if not netinfo: - resp = requests.get(f"http://ip-api.com/json/{ip_address}?fields=25349915") - if resp.ok: - resp_json = resp.json() - netinfo = { - "_id": ip_hash, - "country_code": resp_json["countryCode"], - "country_name": resp_json["country"], - "region": resp_json["regionName"], - "city": resp_json["city"], - "timezone": resp_json["timezone"], - "currency": resp_json["currency"], - "as": resp_json["as"], - "isp": resp_json["isp"], - "vpn": (resp_json.get("hosting") or resp_json.get("proxy")), - "last_refreshed": int(time.time()) - } - db.netinfo.update_one({"_id": ip_hash}, {"$set": netinfo}, upsert=True) - else: - netinfo = { - "_id": ip_hash, - "country_code": "Unknown", - "country_name": "Unknown", - "region": "Unknown", - "city": "Unknown", - "timezone": "Unknown", - "currency": "Unknown", - "as": "Unknown", - "isp": "Unknown", - "vpn": False, - "last_refreshed": int(time.time()) - } - - return netinfo + +def log_security_action(action_type: str, user: str, data: dict): + db.security_log.insert_one({ + "_id": str(uuid.uuid4()), + "type": action_type, + "user": user, + "time": int(time.time()), + "data": data + }) + + if action_type in { + "email_changed", + "password_changed", + "mfa_added", + "mfa_removed", + "mfa_recovery_reset", + "mfa_recovery_used", + "locked" + }: + tmpl_name = "locked" if action_type == "locked" else "security_alert" + platform_name = os.environ["EMAIL_PLATFORM_NAME"] + + account = db.usersv0.find_one({"_id": user}, projection={"_id": 1, "email": 1}) + + txt_tmpl, html_tmpl = render_email_tmpl(tmpl_name, account["_id"], account.get("email", ""), { + "msg": { + "email_changed": f"The email address on your {platform_name} account has been changed.", + "password_changed": f"The password on your {platform_name} account has been changed.", + "mfa_added": f"A multi-factor authenticator has been added to your {platform_name} account.", + "mfa_removed": f"A multi-factor authenticator has been removed from your {platform_name} account.", + "mfa_recovery_reset": f"The multi-factor authentication recovery code on your {platform_name} account has been reset.", + "mfa_recovery_used": f"Your multi-factor authentication recovery code has been used to reset multi-factor authentication on your {platform_name} account." + }[action_type] if action_type != "locked" else None, + "token": create_token("email", [ # this doesn't use EmailTicket in sessions.py because it'd be a recursive import + account["email"], + account["_id"], + "lockdown", + int(time.time())+86400 + ]) if account.get("email") and action_type != "locked" else None + }) + + # Email + if account.get('email'): + Thread( + target=send_email, + args=[EMAIL_SUBJECTS[tmpl_name], account["_id"], account["email"], txt_tmpl, html_tmpl] + ).start() + + # Inbox + rdb.publish("admin", msgpack.packb({ + "op": "alert_user", + "user": account["_id"], + "content": txt_tmpl + })) def add_audit_log(action_type, mod_username, mod_ip, data): @@ -513,14 +595,8 @@ def background_tasks_loop(): except Exception as e: log(f"Failed to delete account {user['_id']}: {e}") - # Revoke old sessions (60 days) - db.acc_sessions.delete_many({"refreshed_at": {"$lt": int(time.time())-5184000}}) - - # Purge old netinfo - db.netinfo.delete_many({"last_refreshed": {"$lt": int(time.time())-2419200}}) - - # Purge old netlogs - db.netlog.delete_many({"last_used": {"$lt": int(time.time())-2419200}}) + # Revoke inactive sessions (3 weeks of inactivity) + db.acc_sessions.delete_many({"refreshed_at": {"$lt": int(time.time())-(86400*21)}}) # Purge old deleted posts db.posts.delete_many({"deleted_at": {"$lt": int(time.time())-2419200}}) @@ -528,7 +604,8 @@ def background_tasks_loop(): # Purge old post revisions db.post_revisions.delete_many({"time": {"$lt": int(time.time())-2419200}}) - # Purge old admin audit logs + """ we should probably not be getting rid of audit logs... + # Purge old "get" admin audit logs db.audit_log.delete_many({ "time": {"$lt": int(time.time())-2419200}, "type": {"$in": [ @@ -545,6 +622,7 @@ def background_tasks_loop(): "got_announcements" ]} }) + """ log("Finished background tasks!") @@ -557,39 +635,41 @@ def check_password_hash(password: str, hashed_password: str) -> bool: return bcrypt.checkpw(password.encode(), hashed_password.encode()) -def send_email(template: str, to_name: str, to_address: str, token: Optional[str] = ""): +def get_normalized_email_hash(address: str) -> str: + """ + Get a hash of an email address with aliases and dots stripped. + This is to allow using address aliases, but to still detect ban evasion. + Also, Gmail ignores dots in addresses. Thanks Google. + """ + + identifier, domain = address.split("@") + identifier = re.split(r'\+|\%', identifier)[0] + identifier = identifier.replace(".", "") + + return urlsafe_b64encode(sha256(f"{identifier}@{domain}".encode()).digest()).decode() + + +def render_email_tmpl(template: str, to_name: str, to_address: str, data: Optional[dict[str, str]] = {}) -> tuple[str, str]: + data.update({ + "subject": EMAIL_SUBJECTS[template], + "name": to_name, + "address": to_address, + "env": os.environ + }) + txt_tmpl = email_env.get_template(f"{template}.txt") html_tmpl = email_env.get_template(f"{template}.html") + return txt_tmpl.render(data), html_tmpl.render(data) + + +def send_email(subject: str, to_name: str, to_address: str, txt_tmpl: str, html_tmpl: str): message = MIMEMultipart("alternative") message["From"] = formataddr((os.environ["EMAIL_FROM_NAME"], os.environ["EMAIL_FROM_ADDRESS"])) message["To"] = formataddr((to_name, to_address)) - - match template: - case "verify": - message["Subject"] = "Verify your email address" - case "recovery": - message["Subject"] = "Reset your password" - case "email_changed": - message["Subject"] = "Your email has been changed" - case "password_changed": - message["Subject"] = "Your password has been changed" - case "mfa_added": - message["Subject"] = "Multi-factor authenticator added" - case "mfa_removed": - message["Subject"] = "Multi-factor authenticator removed" - case "locked": - message["Subject"] = "Your account has been locked" - - data = { - "subject": message["Subject"], - "name": to_name, - "address": to_address, - "token": token, - "env": os.environ - } - message.attach(MIMEText(txt_tmpl.render(data), "plain")) - message.attach(MIMEText(html_tmpl.render(data), "html")) + message["Subject"] = subject + message.attach(MIMEText(txt_tmpl, "plain")) + message.attach(MIMEText(html_tmpl, "html")) with smtplib.SMTP(os.environ["EMAIL_SMTP_HOST"], int(os.environ["EMAIL_SMTP_PORT"])) as server: if os.getenv("EMAIL_SMTP_TLS"): diff --git a/sessions.py b/sessions.py index 1ab62cb..259db64 100644 --- a/sessions.py +++ b/sessions.py @@ -1,5 +1,5 @@ from typing import Optional, TypedDict, Literal -import uuid, time, msgpack +import uuid, time, msgpack, pymongo from database import db, rdb import security, errors @@ -38,40 +38,65 @@ def create(cls: "AccSession", user: str, ip: str, user_agent: str) -> "AccSessio "refreshed_at": int(time.time()) } db.acc_sessions.insert_one(data) + + security.log_security_action("session_create", user, { + "session_id": data["_id"], + "ip": ip, + "user_agent": user_agent + }) + return cls(data) @classmethod def get_by_id(cls: "AccSession", session_id: str) -> "AccSession": data: Optional[AccSessionDB] = db.acc_sessions.find_one({"_id": session_id}) if not data: - raise errors.SessionNotFound + raise errors.AccSessionNotFound return cls(data) @classmethod def get_by_token(cls: "AccSession", token: str) -> "AccSession": - session_id, _ = security.extract_token(token, "acc") + session_id, _, expires_at = security.extract_token(token, "acc") + if expires_at < int(time.time()): + raise errors.AccSessionTokenExpired return cls.get_by_id(session_id) @classmethod def get_username_by_token(cls: "AccSession", token: str) -> str: - session_id, _ = security.extract_token(token, "acc") - username = rdb.get(f"u{session_id}") + session_id, _, expires_at = security.extract_token(token, "acc") + if expires_at < int(time.time()): + raise errors.AccSessionTokenExpired + username = rdb.get(session_id) if username: return username.decode() else: session = cls.get_by_id(session_id) username = session.username - rdb.set(f"u{session_id}", username, ex=300) + rdb.set(session_id, username, ex=300) return username @classmethod def get_all(cls: "AccSession", user: str) -> list["AccSession"]: - return [cls(data) for data in db.acc_sessions.find({"user": user})] + return [ + cls(data) + for data in db.acc_sessions.find( + {"user": user}, + sort=[("refreshed_at", pymongo.DESCENDING)] + ) + ] + + @property + def id(self) -> str: + return self._db["_id"] @property def token(self) -> str: - return security.create_token("acc", [self._db["_id"], self._db["refreshed_at"]]) + return security.create_token("acc", [ + self._db["_id"], + self._db["refreshed_at"], + self._db["refreshed_at"]+(86400*21) # expire token after 3 weeks + ]) @property def username(self): @@ -91,7 +116,7 @@ def v0(self) -> AccSessionV0: def refresh(self, ip: str, user_agent: str, check_token: Optional[str] = None): if check_token: # token re-use prevention - _, refreshed_at = security.extract_token(check_token, "acc") + _, refreshed_at, _ = security.extract_token(check_token, "acc") if refreshed_at != self._db["refreshed_at"]: return self.revoke() @@ -102,6 +127,12 @@ def refresh(self, ip: str, user_agent: str, check_token: Optional[str] = None): }) db.acc_sessions.update_one({"_id": self._db["_id"]}, {"$set": self._db}) + security.log_security_action("session_refresh", self._db["user"], { + "session_id": self._db["_id"], + "ip": ip, + "user_agent": user_agent + }) + def revoke(self): db.acc_sessions.delete_one({"_id": self._db["_id"]}) rdb.delete(f"u{self._db['_id']}") @@ -111,6 +142,10 @@ def revoke(self): "sid": self._db["_id"] })) + security.log_security_action("session_revoke", self._db["user"], { + "session_id": self._db["_id"] + }) + class EmailTicket: def __init__( From 7d010063a985835b3109697d71b0fa5b223a4601 Mon Sep 17 00:00:00 2001 From: Tnix Date: Tue, 10 Sep 2024 01:13:45 +1200 Subject: [PATCH 16/19] add back account restoration --- sessions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sessions.py b/sessions.py index 259db64..a5e9b5d 100644 --- a/sessions.py +++ b/sessions.py @@ -29,6 +29,15 @@ def __init__(self, data: AccSessionDB): @classmethod def create(cls: "AccSession", user: str, ip: str, user_agent: str) -> "AccSession": + # restore account if it is pending deletion + result = db.usersv0.update_one({"_id": user}, {"$set": {"delete_after": None}}) + if result.modified_count: + rdb.publish("admin", msgpack.packb({ + "op": "alert_user", + "user": user, + "content": "Your account was scheduled for deletion but you logged back in. Your account is no longer scheduled for deletion! If you didn't request for your account to be deleted, please change your password immediately." + })) + data: AccSessionDB = { "_id": str(uuid.uuid4()), "user": user, From 2f8ebca52737ae60c73ba83ccc5035a2093ee634 Mon Sep 17 00:00:00 2001 From: Tnix Date: Tue, 10 Sep 2024 01:21:33 +1200 Subject: [PATCH 17/19] make sure an email address cannot be used more than once --- rest_api/v0/me.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rest_api/v0/me.py b/rest_api/v0/me.py index 118ed49..ae78be9 100644 --- a/rest_api/v0/me.py +++ b/rest_api/v0/me.py @@ -228,7 +228,11 @@ async def update_email(data: UpdateEmailBody): if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 - + + # Make sure the email address hasn't been used before + if db.usersv0.count_documents({"normalized_email_hash": security.get_normalized_email_hash(data.email)}, limit=1): + return {"error": True, "type": "emailExists"}, 409 + # Ratelimit security.ratelimit(f"emailch:{request.user}", 3, 2700) From d316337e873380f165215e063bb76845037424e2 Mon Sep 17 00:00:00 2001 From: Tnix Date: Tue, 10 Sep 2024 01:22:22 +1200 Subject: [PATCH 18/19] add usersv0 indexes for emails --- database.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/database.py b/database.py index cae9e44..e66ba1a 100644 --- a/database.py +++ b/database.py @@ -45,6 +45,10 @@ # Create usersv0 indexes try: db.usersv0.create_index([("lower_username", pymongo.ASCENDING)], name="lower_username", unique=True) except: pass +try: db.usersv0.create_index([("email", pymongo.ASCENDING)], name="email", unique=True) +except: pass +try: db.usersv0.create_index([("normalized_email_hash", pymongo.ASCENDING)], name="normalized_email_hash", unique=True) +except: pass try: db.usersv0.create_index([("created", pymongo.DESCENDING)], name="recent_users") except: pass try: From 792c26f98790260f0ffa826083f6681a7790c955 Mon Sep 17 00:00:00 2001 From: Tnix Date: Wed, 11 Sep 2024 00:31:03 +1200 Subject: [PATCH 19/19] add the ability to remove email addresses --- rest_api/v0/me.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/rest_api/v0/me.py b/rest_api/v0/me.py index ae78be9..8e7b0e9 100644 --- a/rest_api/v0/me.py +++ b/rest_api/v0/me.py @@ -51,9 +51,12 @@ class Config: class UpdateEmailBody(BaseModel): password: str = Field(min_length=1, max_length=255) # change in API v1 - email: Optional[str] = Field(default=None, max_length=255, pattern=security.EMAIL_REGEX) + email: str = Field(max_length=255, pattern=security.EMAIL_REGEX) captcha: Optional[str] = Field(default="", max_length=2000) +class RemoveEmailBody(BaseModel): + password: str = Field(min_length=1, max_length=255) # change in API v1 + class ChangePasswordBody(BaseModel): old: str = Field(min_length=1, max_length=255) # change in API v1 new: str = Field(min_length=8, max_length=72) @@ -260,6 +263,41 @@ async def update_email(data: UpdateEmailBody): return {"error": False}, 200 +@me_bp.delete("/email") +@validate_request(RemoveEmailBody) +async def remove_email(data: RemoveEmailBody): + # Check authorization + if not request.user: + abort(401) + + # Check ratelimits + if security.ratelimited(f"login:u:{request.user}"): + abort(429) + + # Check password + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) + if not security.check_password_hash(data.password, account["pswd"]): + security.ratelimit(f"login:u:{request.user}", 5, 60) + return {"error": True, "type": "invalidCredentials"}, 401 + + # Log action + security.log_security_action("email_changed", account["_id"], { + "old_email_hash": security.get_normalized_email_hash(account["email"]) if account.get("email") else None, + "new_email_hash": None, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Update user's email address + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "email": "", + "normalized_email_hash": "" + }}) + app.cl.send_event("update_config", {"email": ""}, usernames=[account["_id"]]) + + return {"error": False}, 200 + + @me_bp.patch("/password") @validate_request(ChangePasswordBody) async def change_password(data: ChangePasswordBody):