From 6c9d4c8e2c852049e3dd46d765eb9d0599d28ba3 Mon Sep 17 00:00:00 2001 From: Tnix Date: Tue, 10 Sep 2024 01:10:13 +1200 Subject: [PATCH] finish emails --- cloudlink.py | 5 +- database.py | 65 +++++----- errors.py | 4 +- rest_api/__init__.py | 4 +- rest_api/admin.py | 46 +++---- rest_api/v0/__init__.py | 2 + rest_api/v0/auth.py | 106 +++++++++++++-- rest_api/v0/emails.py | 187 +++++++++++++++++++++++++++ rest_api/v0/me.py | 95 ++++++++++---- security.py | 276 ++++++++++++++++++++++++++-------------- sessions.py | 53 ++++++-- 11 files changed, 639 insertions(+), 204 deletions(-) create mode 100644 rest_api/v0/emails.py diff --git a/cloudlink.py b/cloudlink.py index ce7c14e..e2cd568 100755 --- a/cloudlink.py +++ b/cloudlink.py @@ -51,7 +51,8 @@ def __init__(self): #"Kicked": "E:020 | Kicked", -- deprecated #"ChatFull": "E:023 | Chat full", -- deprecated #"LoggedOut": "I:024 | Logged out", -- deprecated - "Deleted": "E:025 | Deleted" + "Deleted": "E:025 | Deleted", + "AccountLocked": "E:026 | Account Locked" } self.commands: dict[str, function] = { # Core commands @@ -337,6 +338,8 @@ def proxy_api_request( self.send_statuscode("2FARequired", listener) case "accountDeleted": self.send_statuscode("Deleted", listener) + case "accountLocked": + self.send_statuscode("AccountLocked", listener) case "accountBanned": self.send_statuscode("Banned", listener) case "tooManyRequests": diff --git a/database.py b/database.py index 7cef238..cae9e44 100644 --- a/database.py +++ b/database.py @@ -3,6 +3,7 @@ import redis import os import secrets +import time from radix import Radix from hashlib import sha256 from base64 import urlsafe_b64encode @@ -64,9 +65,15 @@ # Create account sessions indexes try: db.acc_sessions.create_index([("user", pymongo.ASCENDING)], name="user") except: pass +try: db.acc_sessions.create_index([("ip", pymongo.ASCENDING)], name="ip") +except: pass try: db.acc_sessions.create_index([("refreshed_at", pymongo.ASCENDING)], name="refreshed_at") except: pass +# Create security log indexes +try: db.security_log.create_index([("user", pymongo.ASCENDING)], name="user") +except: pass + # Create data exports indexes try: db.data_exports.create_index([("user", pymongo.ASCENDING)], name="user") except: pass @@ -75,18 +82,6 @@ try: db.relationships.create_index([("_id.from", pymongo.ASCENDING)], name="from") except: pass -# Create netinfo indexes -try: db.netinfo.create_index([("last_refreshed", pymongo.ASCENDING)], name="last_refreshed") -except: pass - -# Create netlog indexes -try: db.netlog.create_index([("_id.ip", pymongo.ASCENDING)], name="ip") -except: pass -try: db.netlog.create_index([("_id.user", pymongo.ASCENDING)], name="user") -except: pass -try: db.netlog.create_index([("last_used", pymongo.ASCENDING)], name="last_used") -except: pass - # Create posts indexes try: db.posts.create_index([ @@ -186,25 +181,6 @@ # Create default database items -for username in ["Server", "Deleted", "Meower", "Admin", "username"]: - try: - db.usersv0.insert_one({ - "_id": username, - "lower_username": username.lower(), - "uuid": None, - "created": None, - "pfp_data": None, - "avatar": None, - "avatar_color": None, - "quote": None, - "pswd": None, - "flags": 1, - "permissions": None, - "ban": None, - "last_seen": None, - "delete_after": None - }) - except pymongo.errors.DuplicateKeyError: pass try: db.config.insert_one({ "_id": "migration", @@ -323,16 +299,39 @@ def get_total_pages(collection: str, query: dict, page_size: int = 25) -> int: "mfa_recovery_code": user["mfa_recovery_code"][:10] }}) + # Delete system users from DB + log("[Migrator] Deleting system users from DB") + db.usersv0.delete_many({"_id": {"$in": ["Server", "Deleted", "Meower", "Admin", "username"]}}) + + # Emails + log("[Migrator] Adding email addresses") + db.usersv0.update_many({"email": {"$exists": False}}, {"$set": { + "email": "", + "normalized_email_hash": "" + }}) + # New sessions log("[Migrator] Adding new sessions") - for user in db.usersv0.find({"tokens": {"$exists": True}}, projection={"_id": 1, "tokens": 1}): + for user in db.usersv0.find({ + "tokens": {"$exists": True}, + "last_seen": {"$ne": None, "$gt": int(time.time())-(86400*21)}, + }, projection={"_id": 1, "tokens": 1}): if user["tokens"]: for token in user["tokens"]: - rdb.set(urlsafe_b64encode(sha256(token.encode()).digest()), user["_id"], ex=1209600) # 14 days + rdb.set( + urlsafe_b64encode(sha256(token.encode()).digest()), + user["_id"], + ex=86400*21 # 21 days + ) db.usersv0.update_many({}, {"$unset": {"tokens": ""}}) try: db.usersv0.drop_index("tokens") except: pass + # No more netinfo and netlog + log("[Migrator] Removing netinfo and netlog") + db.netinfo.drop() + db.netlog.drop() + db.config.update_one({"_id": "migration"}, {"$set": {"database": CURRENT_DB_VERSION}}) log(f"[Migrator] Finished Migrating DB to version {CURRENT_DB_VERSION}") diff --git a/errors.py b/errors.py index 22d59fa..92f4d9b 100644 --- a/errors.py +++ b/errors.py @@ -1,5 +1,7 @@ class InvalidTokenSignature(Exception): pass -class SessionNotFound(Exception): pass +class AccSessionTokenExpired(Exception): pass + +class AccSessionNotFound(Exception): pass class EmailTicketExpired(Exception): pass \ No newline at end of file diff --git a/rest_api/__init__.py b/rest_api/__init__.py index 47bddad..6153be7 100755 --- a/rest_api/__init__.py +++ b/rest_api/__init__.py @@ -49,13 +49,11 @@ async def internal_auth(): @app.before_request -async def check_ip(): +async def get_ip(): if hasattr(request, "internal_ip") and request.internal_ip: # internal IP forwarding request.ip = request.internal_ip else: request.ip = (request.headers.get("Cf-Connecting-Ip", request.remote_addr)) - if request.path != "/status" and blocked_ips.search_best(request.ip): - return {"error": True, "type": "ipBlocked"}, 403 @app.before_request diff --git a/rest_api/admin.py b/rest_api/admin.py index 3dca416..d4c8c0d 100644 --- a/rest_api/admin.py +++ b/rest_api/admin.py @@ -532,21 +532,19 @@ async def get_user(username): # Get netlogs netlogs = [ { - "ip": netlog["_id"]["ip"], - "user": netlog["_id"]["user"], - "last_used": netlog["last_used"], + "ip": session._db["ip"], + "user": session.username, + "last_used": session._db["refreshed_at"], } - for netlog in db.netlog.find( - {"_id.user": username}, sort=[("last_used", pymongo.DESCENDING)] - ) + for session in AccSession.get_all(username) ] # Get alts alts = [ - netlog["_id"]["user"] - for netlog in db.netlog.find( - {"_id.ip": {"$in": [netlog["ip"] for netlog in netlogs]}} - ) + session["user"] + for session in db.acc_sessions.find({ + "ip": {"$in": [netlog["ip"] for netlog in netlogs]} + }) ] if username in alts: alts.remove(username) @@ -557,7 +555,7 @@ async def get_user(username): payload["recent_ips"] = [ { "ip": netlog["ip"], - "netinfo": security.get_netinfo(netlog["ip"]), + "netinfo": security.get_ip_info(netlog["ip"]), "last_used": netlog["last_used"], "blocked": ( blocked_ips.search_best(netlog["ip"]) @@ -1096,9 +1094,6 @@ async def get_netinfo(ip): if not security.has_permission(request.permissions, security.AdminPermissions.VIEW_IPS): abort(403) - # Get netinfo - netinfo = security.get_netinfo(ip) - # Get netblocks netblocks = [] for radix_node in blocked_ips.search_covering(ip): @@ -1107,16 +1102,17 @@ async def get_netinfo(ip): netblocks.append(db.netblock.find_one({"_id": radix_node.prefix})) # Get netlogs - netlogs = [ - { - "ip": netlog["_id"]["ip"], - "user": netlog["_id"]["user"], - "last_used": netlog["last_used"], - } - for netlog in db.netlog.find( - {"_id.ip": ip}, sort=[("last_used", pymongo.DESCENDING)] - ) - ] + hit_users = set() + netlogs = [] + for session in db.acc_sessions.find({"ip": ip}, sort=[("refreshed_at", pymongo.DESCENDING)]): + if session["user"] in hit_users: + continue + netlogs.append({ + "ip": session["ip"], + "user": session["user"], + "last_used": session["refreshed_at"] + }) + hit_users.add(session["user"]) # Add log security.add_audit_log("got_netinfo", request.user, request.ip, {"ip": ip}) @@ -1124,7 +1120,7 @@ async def get_netinfo(ip): # Return netinfo, netblocks, and netlogs return { "error": False, - "netinfo": netinfo, + "netinfo": security.get_ip_info(ip), "netblocks": netblocks, "netlogs": netlogs, }, 200 diff --git a/rest_api/v0/__init__.py b/rest_api/v0/__init__.py index 3d9fc16..7fb6162 100644 --- a/rest_api/v0/__init__.py +++ b/rest_api/v0/__init__.py @@ -9,10 +9,12 @@ from .home import home_bp from .me import me_bp from .emojis import emojis_bp +from .emails import emails_bp v0 = Blueprint("v0", __name__) v0.register_blueprint(auth_bp) +v0.register_blueprint(emails_bp) v0.register_blueprint(me_bp) v0.register_blueprint(home_bp) v0.register_blueprint(inbox_bp) diff --git a/rest_api/v0/auth.py b/rest_api/v0/auth.py index e1aadd7..1b35b5b 100644 --- a/rest_api/v0/auth.py +++ b/rest_api/v0/auth.py @@ -1,4 +1,4 @@ -import re, os, requests, pyotp, secrets +import re, os, requests, pyotp, secrets, time from pydantic import BaseModel from quart import Blueprint, request, abort, current_app as app from quart_schema import validate_request @@ -6,8 +6,10 @@ from typing import Optional from base64 import urlsafe_b64encode from hashlib import sha256 -from database import db, rdb, registration_blocked_ips -from sessions import AccSession +from threading import Thread + +from database import db, rdb, blocked_ips, registration_blocked_ips +from sessions import AccSession, EmailTicket import security auth_bp = Blueprint("auth_bp", __name__, url_prefix="/auth") @@ -19,6 +21,16 @@ class AuthRequest(BaseModel): mfa_recovery_code: Optional[str] = Field(default="", min_length=10, max_length=10) captcha: Optional[str] = Field(default="", max_length=2000) +class RecoverAccountBody(BaseModel): + email: str = Field(min_length=1, max_length=255, pattern=security.EMAIL_REGEX) + captcha: Optional[str] = Field(default="", max_length=2000) + + +@auth_bp.before_request +async def ip_block_check(): + if blocked_ips.search_best(request.ip): + return {"error": True, "type": "ipBlocked"}, 403 + @auth_bp.post("/login") @validate_request(AuthRequest) @@ -29,7 +41,11 @@ async def login(data: AuthRequest): security.ratelimit(f"login:i:{request.ip}", 50, 900) # Get basic account details - account = db.usersv0.find_one({"lower_username": data.username.lower()}, projection={ + account = db.usersv0.find_one({ + "email": data.username + } if "@" in data.username else { + "lower_username": data.username.lower() + }, projection={ "_id": 1, "flags": 1, "pswd": 1, @@ -42,6 +58,10 @@ async def login(data: AuthRequest): if account["flags"] & security.UserFlags.DELETED: return {"error": True, "type": "accountDeleted"}, 401 + # Make sure account isn't locked + if account["flags"] & security.UserFlags.LOCKED: + return {"error": True, "type": "accountLocked"}, 401 + # Make sure account isn't ratelimited if security.ratelimited(f"login:u:{account['_id']}"): abort(429) @@ -51,7 +71,11 @@ async def login(data: AuthRequest): encoded_token = urlsafe_b64encode(sha256(data.password.encode()).digest()) username = rdb.get(encoded_token) if username and username.decode() == account["_id"]: - data.password = AccSession.create(username.decode(), request.ip, request.headers.get("User-Agent")).token + data.password = AccSession.create( + username.decode(), + request.ip, + request.headers.get("User-Agent") + ).token rdb.delete(encoded_token) # Check credentials & get session @@ -83,6 +107,11 @@ async def login(data: AuthRequest): # Abort if password is invalid if not password_valid: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_password", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) # Check MFA @@ -98,21 +127,43 @@ async def login(data: AuthRequest): break if not passed: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_totp_code", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) elif data.mfa_recovery_code: if data.mfa_recovery_code == account["mfa_recovery_code"]: db.authenticators.delete_many({"user": account["_id"]}) + + new_recovery_code = secrets.token_hex(5) db.usersv0.update_one({"_id": account["_id"]}, {"$set": { - "mfa_recovery_code": secrets.token_hex(5) + "mfa_recovery_code": new_recovery_code }}) - app.supporter.create_post("inbox", account["_id"], "All multi-factor authenticators have been removed from your account by someone who used your multi-factor authentication recovery code. If this wasn't you, please secure your account immediately.") + security.log_security_action("mfa_recovery_used", account["_id"], { + "old_recovery_code_hash": urlsafe_b64encode(sha256(data.mfa_recovery_code.encode()).digest()).decode(), + "new_recovery_code_hash": urlsafe_b64encode(sha256(new_recovery_code.encode()).digest()).decode(), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) else: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_recovery_code", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) else: mfa_methods = set() for authenticator in authenticators: mfa_methods.add(authenticator["type"]) + security.log_security_action("auth_fail", account["_id"], { + "status": "mfa_required", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return { "error": True, "type": "mfaRequired", @@ -130,6 +181,7 @@ async def login(data: AuthRequest): "account": security.get_account(account['_id'], True) }, 200 + @auth_bp.post("/register") @validate_request(AuthRequest) async def register(data: AuthRequest): @@ -174,12 +226,46 @@ async def register(data: AuthRequest): security.ratelimit(f"register:{request.ip}:s", 5, 900) # Create session - session, token = security.create_session(data.username, request.ip, request.headers.get("User-Agent")) + session = AccSession.create(data.username, request.ip, request.headers.get("User-Agent")) # Return session and account details return { "error": False, - "session": session, - "token": token, + "session": session.v0, + "token": session.token, "account": security.get_account(data.username, True) }, 200 + + +@auth_bp.post("/recover") +@validate_request(RecoverAccountBody) +async def recover_account(data: RecoverAccountBody): + # Check ratelimits + if security.ratelimited(f"recover:{request.ip}"): + abort(429) + security.ratelimit(f"recover:{request.ip}", 3, 2700) + + # Check captcha + if os.getenv("CAPTCHA_SECRET") and not (hasattr(request, "bypass_captcha") and request.bypass_captcha): + if not requests.post("https://api.hcaptcha.com/siteverify", data={ + "secret": os.getenv("CAPTCHA_SECRET"), + "response": data.captcha, + }).json()["success"]: + return {"error": True, "type": "invalidCaptcha"}, 403 + + # Get account + account = db.usersv0.find_one({"email": data.email}, projection={"_id": 1, "email": 1, "flags": 1}) + if not account: + return {"error": False}, 200 + + # Create recovery email ticket + ticket = EmailTicket(data.email, account["_id"], "recover", expires_at=int(time.time())+1800) + + # Send email + txt_tmpl, html_tmpl = security.render_email_tmpl("recover", account["_id"], account["email"], {"token": ticket.token}) + Thread( + target=security.send_email, + args=[security.EMAIL_SUBJECTS["recover"], account["_id"], account["email"], txt_tmpl, html_tmpl] + ).start() + + return {"error": False}, 200 diff --git a/rest_api/v0/emails.py b/rest_api/v0/emails.py new file mode 100644 index 0000000..7d8365c --- /dev/null +++ b/rest_api/v0/emails.py @@ -0,0 +1,187 @@ +from quart import Blueprint, current_app as app, request, abort +from quart_schema import validate_request +from pydantic import BaseModel, Field +from hashlib import sha256 +from base64 import urlsafe_b64encode +import time, secrets + +from sessions import EmailTicket, AccSession +from database import db, rdb +import security, errors + + +emails_bp = Blueprint("emails_bp", __name__, url_prefix="/emails") + + +class VerifyEmailBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + +class RecoverAccountBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + password: str = Field(min_length=8, max_length=72) + +class LockAccountBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + + +@emails_bp.post("/verify") +@validate_request(VerifyEmailBody) +async def verify_email(data: VerifyEmailBody): + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "verify": + abort(401) + + # Make sure the email address matches the user's pending email address + pending_email = rdb.get(f"pe{ticket.username}") + if (not pending_email) or (pending_email.decode() != ticket.email_address): + abort(401) + rdb.delete(f"pe{ticket.username}") + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "email": 1 + }) + + # Log action + security.log_security_action("email_changed", account["_id"], { + "old_email_hash": security.get_normalized_email_hash(account["email"]) if account.get("email") else None, + "new_email_hash": security.get_normalized_email_hash(ticket.email_address), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Update user's email address + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "email": ticket.email_address, + "normalized_email_hash": security.get_normalized_email_hash(ticket.email_address) + }}) + app.cl.send_event("update_config", {"email": ticket.email_address}, usernames=[account["_id"]]) + + return {"error": False}, 200 + + +@emails_bp.post("/recover") +@validate_request(RecoverAccountBody) +async def recover_account(data: RecoverAccountBody): + # Make sure ticket hasn't already been used + if rdb.exists(urlsafe_b64encode(sha256(data.token.encode()).digest()).decode()): + abort(401) + + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "recover": + abort(401) + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "email": 1, + "pswd": 1, + "flags": 1 + }) + + # Make sure the ticket email matches the user's current email + if ticket.email_address != account["email"]: + abort(401) + + # Revoke ticket + rdb.set( + urlsafe_b64encode(sha256(data.token.encode()).digest()).decode(), + "", + ex=(ticket.expires_at-int(time.time()))+1 + ) + + # Update password (and remove locked flag) + new_hash = security.hash_password(data.password) + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "pswd": new_hash, + "flags": account["flags"] ^ security.UserFlags.LOCKED + }}) + + # Log action + security.log_security_action("password_changed", account["_id"], { + "method": "email", + "old_pswd_hash": account["pswd"], + "new_pswd_hash": new_hash, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Revoke sessions + for session in AccSession.get_all(account["_id"]): + session.revoke() + + return {"error": False}, 200 + + +@emails_bp.post("/lockdown") +@validate_request(LockAccountBody) +async def lock_account(data: LockAccountBody): + # Make sure ticket hasn't already been used + if rdb.exists(urlsafe_b64encode(sha256(data.token.encode()).digest()).decode()): + abort(401) + + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "lockdown": + abort(401) + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "flags": 1 + }) + + # Revoke ticket + rdb.set( + urlsafe_b64encode(sha256(data.token.encode()).digest()).decode(), + "", + ex=(ticket.expires_at-int(time.time()))+1 + ) + + # Make sure the account hasn't already been locked in the last 24 hours (lockdown tickets last for 24 hours) + # This is to stop multiple identity/credential rotations by an attacker to keep access via lockdown tickets. + if security.ratelimited(f"lock:{account['_id']}"): + abort(429) + security.ratelimit(f"lock:{account['_id']}", 1, 86400) + + # Update account + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "email": ticket.email_address, + "normalized_email_hash": security.get_normalized_email_hash(ticket.email_address), + "flags": account["flags"] | security.UserFlags.LOCKED, + "mfa_recovery_code": secrets.token_hex(5) + }}) + + # Remove authenticators + db.authenticators.delete_many({"user": account["_id"]}) + + # Log event + security.log_security_action("locked", account["_id"], { + "method": "email", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Revoke sessions + for session in AccSession.get_all(account["_id"]): + session.revoke() + + return {"error": False}, 200 diff --git a/rest_api/v0/me.py b/rest_api/v0/me.py index 793ff06..118ed49 100644 --- a/rest_api/v0/me.py +++ b/rest_api/v0/me.py @@ -3,8 +3,9 @@ from pydantic import BaseModel, Field from typing import Optional, List, Literal from copy import copy -from base64 import b64encode -from io import BytesIO +from base64 import urlsafe_b64encode +from hashlib import sha256 +from threading import Thread import pymongo import uuid import time @@ -13,11 +14,12 @@ import uuid import secrets import os +import requests import security from database import db, rdb, get_total_pages from uploads import claim_file, delete_file -from sessions import AccSession +from sessions import AccSession, EmailTicket from utils import log @@ -49,7 +51,8 @@ class Config: class UpdateEmailBody(BaseModel): password: str = Field(min_length=1, max_length=255) # change in API v1 - email: Optional[str] = Field(default=None, max_length=255) + email: Optional[str] = Field(default=None, max_length=255, pattern=security.EMAIL_REGEX) + captcha: Optional[str] = Field(default="", max_length=2000) class ChangePasswordBody(BaseModel): old: str = Field(min_length=1, max_length=255) # change in API v1 @@ -221,12 +224,36 @@ async def update_email(data: UpdateEmailBody): abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) - if not security.check_password_hash(data.old, account["pswd"]): + account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 + # Ratelimit + security.ratelimit(f"emailch:{request.user}", 3, 2700) + + # Check captcha + if os.getenv("CAPTCHA_SECRET") and not (hasattr(request, "bypass_captcha") and request.bypass_captcha): + if not requests.post("https://api.hcaptcha.com/siteverify", data={ + "secret": os.getenv("CAPTCHA_SECRET"), + "response": data.captcha, + }).json()["success"]: + return {"error": True, "type": "invalidCaptcha"}, 403 + + # Create email verification ticket + ticket = EmailTicket(data.email, request.user, "verify", expires_at=int(time.time())+1800) + + # Set pending email address + rdb.set(f"pe{request.user}", data.email, ex=1800) + # Send email + txt_tmpl, html_tmpl = security.render_email_tmpl("verify", request.user, data.email, {"token": ticket.token}) + Thread( + target=security.send_email, + args=[security.EMAIL_SUBJECTS["verify"], request.user, data.email, txt_tmpl, html_tmpl] + ).start() + + return {"error": False}, 200 @me_bp.patch("/password") @@ -241,16 +268,23 @@ async def change_password(data: ChangePasswordBody): abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) if not security.check_password_hash(data.old, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 # Update password - db.usersv0.update_one({"_id": request.user}, {"$set": {"pswd": security.hash_password(data.new)}}) - - # Send alert - app.supporter.create_post("inbox", account["_id"], "Your account password has been changed. If this wasn't requested by you, please secure your account immediately.") + new_hash = security.hash_password(data.new) + db.usersv0.update_one({"_id": request.user}, {"$set": {"pswd": new_hash}}) + + # Log event + security.log_security_action("password_changed", account["_id"], { + "method": "self", + "old_pswd_hash": account["pswd"], + "new_pswd_hash": new_hash, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return {"error": False}, 200 @@ -289,7 +323,7 @@ async def add_authenticator(data: AddAuthenticatorBody): abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1, "mfa_recovery_code": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 @@ -305,8 +339,12 @@ async def add_authenticator(data: AddAuthenticatorBody): } db.authenticators.insert_one(authenticator) - # Send alert - app.supporter.create_post("inbox", account["_id"], "A multi-factor authenticator has been added to your account. If this wasn't requested by you, please secure your account immediately.") + # Log action + security.log_security_action("mfa_added", account["_id"], { + "authenticator_id": authenticator["_id"], + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) # Return authenticator and MFA recovery code del authenticator["user"] @@ -363,7 +401,7 @@ async def remove_authenticator(authenticator_id: str, data: RemoveAuthenticatorB abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 @@ -376,8 +414,12 @@ async def remove_authenticator(authenticator_id: str, data: RemoveAuthenticatorB if result.deleted_count < 1: abort(404) - # Send alert - app.supporter.create_post("inbox", account["_id"], "A multi-factor authenticator has been removed from your account. If this wasn't requested by you, please secure your account immediately.") + # Log action + security.log_security_action("mfa_removed", account["_id"], { + "authenticator_id": authenticator_id, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return {"error": False} @@ -411,19 +453,24 @@ async def reset_mfa_recovery_code(data: ResetMFARecoveryCodeBody): abort(401) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 # Reset MFA recovery code - mfa_recovery_code = secrets.token_hex(5) - db.usersv0.update_one({"_id": request.user}, {"$set": {"mfa_recovery_code": mfa_recovery_code}}) - - # Send alert - app.supporter.create_post("inbox", account["_id"], "Your multi-factor authentication recovery code has been reset. If this wasn't requested by you, please secure your account immediately.") + new_recovery_code = secrets.token_hex(5) + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "mfa_recovery_code": new_recovery_code + }}) + security.log_security_action("mfa_recovery_reset", account["_id"], { + "old_recovery_code_hash": urlsafe_b64encode(sha256(account["mfa_recovery_code"]).digest()).decode(), + "new_recovery_code_hash": urlsafe_b64encode(sha256(new_recovery_code.encode()).digest()).decode(), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) - return {"error": False, "mfa_recovery_code": mfa_recovery_code} + return {"error": False, "mfa_recovery_code": new_recovery_code} @me_bp.delete("/tokens") diff --git a/security.py b/security.py index 718e4e2..d6f242a 100644 --- a/security.py +++ b/security.py @@ -1,10 +1,11 @@ from typing import Optional, Any, Literal from hashlib import sha256 from base64 import urlsafe_b64encode, urlsafe_b64decode +from threading import Thread from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formataddr -import time, requests, os, uuid, secrets, bcrypt, hmac, msgpack, jinja2, smtplib +import time, requests, os, uuid, secrets, bcrypt, hmac, msgpack, jinja2, smtplib, re from database import db, rdb, signing_keys from utils import log @@ -17,7 +18,7 @@ """ SENSITIVE_ACCOUNT_FIELDS = { - "email", + "normalized_email_hash", "pswd", "mfa_recovery_code", "delete_after" @@ -27,6 +28,9 @@ for key in SENSITIVE_ACCOUNT_FIELDS: SENSITIVE_ACCOUNT_FIELDS_DB_PROJECTION[key] = 0 +SYSTEM_USER_USERNAMES = {"server", "deleted", "meower", "admin", "username"} +SYSTEM_USER = {} + DEFAULT_USER_SETTINGS = { "unread_inbox": True, "theme": "orange", @@ -43,6 +47,8 @@ USERNAME_REGEX = "[a-zA-Z0-9-_]{1,20}" +# I hate this. But, thanks https://stackoverflow.com/a/201378 +EMAIL_REGEX = r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])""" TOTP_REGEX = "[0-9]{6}" BCRYPT_SALT_ROUNDS = 14 TOKEN_BYTES = 64 @@ -54,6 +60,14 @@ ] +EMAIL_SUBJECTS = { + "verify": "Verify your email address", + "recover": "Reset your password", + "security_alert": "Security alert", + "locked": "Your account has been locked" +} + + email_file_loader = jinja2.FileSystemLoader("email_templates") email_env = jinja2.Environment(loader=email_file_loader) @@ -63,6 +77,8 @@ class UserFlags: DELETED = 2 PROTECTED = 4 POST_RATELIMIT_BYPASS = 8 + REQUIRE_EMAIL = 16 # not used yet + LOCKED = 32 class AdminPermissions: @@ -134,6 +150,9 @@ def account_exists(username, ignore_case=False): log(f"Error on account_exists: Expected str for username, got {type(username)}") return False + if (ignore_case or username == username.lower()) and username.lower() in SYSTEM_USER_USERNAMES: + return True + query = ({"lower_username": username.lower()} if ignore_case else {"_id": username}) return (db.usersv0.count_documents(query, limit=1) > 0) @@ -150,6 +169,7 @@ def create_account(username: str, password: str, ip: str): "avatar_color": "000000", "quote": "", "email": "", + "normalized_email_hash": "", "pswd": hash_password(password), "mfa_recovery_code": secrets.token_hex(5), "flags": 0, @@ -173,7 +193,7 @@ def create_account(username: str, password: str, ip: str): })) # Automatically report if VPN is detected - if get_netinfo(ip)["vpn"]: + if get_ip_info(ip)["vpn"]: db.reports.insert_one({ "_id": str(uuid.uuid4()), "type": "user", @@ -183,7 +203,7 @@ def create_account(username: str, password: str, ip: str): "reports": [{ "user": "Server", "ip": ip, - "reason": "User registered while using a VPN.", + "reason": f"User registered while using a VPN ({ip}).", "comment": "", "time": int(time()) }] @@ -196,6 +216,24 @@ def get_account(username, include_config=False): log(f"Error on get_account: Expected str for username, got {type(username)}") return None + # System users + if username.lower() in SYSTEM_USER_USERNAMES: + return { + "_id": username.title(), + "lower_username": username.lower(), + "uuid": None, + "created": None, + "pfp_data": None, + "avatar": None, + "avatar_color": None, + "quote": None, + "email": None, + "flags": 1, + "permissions": None, + "ban": None, + "last_seen": None + } + # Get account account = db.usersv0.find_one({"lower_username": username.lower()}, projection=SENSITIVE_ACCOUNT_FIELDS_DB_PROJECTION) if not account: @@ -226,7 +264,8 @@ def get_account(username, include_config=False): del user_settings["_id"] account.update(user_settings) else: - # Remove ban if not including config + # Remove email and ban if not including config + del account["email"] del account["ban"] return account @@ -370,6 +409,8 @@ def delete_account(username, purge=False): "avatar": None, "avatar_color": None, "quote": None, + "email": None, + "normalized_email_hash": None, "pswd": None, "mfa_recovery_code": None, "flags": account["flags"], @@ -379,21 +420,24 @@ def delete_account(username, purge=False): "delete_after": None }}) + # Delete pending email + rdb.delete(f"pe{username}") + # Delete authenticators db.authenticators.delete_many({"user": username}) # Delete sessions db.acc_sessions.delete_many({"user": username}) + # Delete security logs + db.security_log.delete_many({"user": username}) + # Delete uploaded files clear_files(username) # Delete user settings db.user_settings.delete_one({"_id": username}) - # Delete netlogs - db.netlog.delete_many({"_id.user": username}) - # Remove from reports db.reports.update_many({"reports.user": username}, {"$pull": { "reports": {"user": username} @@ -431,62 +475,100 @@ def delete_account(username, purge=False): db.usersv0.delete_one({"_id": username}) -def get_netinfo(ip_address): - """ - Get IP info from IP-API. - - Returns: - ```json - { - "_id": str, - "country_code": str, - "country_name": str, - "asn": int, - "isp": str, - "vpn": bool +def get_ip_info(ip_address): + # Get IP hash + ip_hash = urlsafe_b64encode(sha256(ip_address.encode()).digest()).decode() + + # Get from cache + ip_info = rdb.get(f"ip{ip_hash}") + if ip_info: + return msgpack.unpackb(ip_info) + + # Get from IP-API + resp = requests.get(f"http://ip-api.com/json/{ip_address}?fields=25349915") + if resp.ok and resp.json()["status"] == "success": + resp_json = resp.json() + ip_info = { + "country_code": resp_json["countryCode"], + "country_name": resp_json["country"], + "region": resp_json["regionName"], + "city": resp_json["city"], + "timezone": resp_json["timezone"], + "currency": resp_json["currency"], + "as": resp_json["as"], + "isp": resp_json["isp"], + "vpn": (resp_json.get("hosting") or resp_json.get("proxy")) + } + rdb.set(f"ip{ip_hash}", msgpack.packb(ip_info), ex=int(time.time())+(86400*21)) # cache for 3 weeks + return ip_info + + # Fallback + return { + "country_code": "Unknown", + "country_name": "Unknown", + "region": "Unknown", + "city": "Unknown", + "timezone": "Unknown", + "currency": "Unknown", + "as": "Unknown", + "isp": "Unknown", + "vpn": False } - ``` - """ - # Get IP hash - ip_hash = sha256(ip_address.encode()).hexdigest() - - # Get from database or IP-API if not cached - netinfo = db.netinfo.find_one({"_id": ip_hash}) - if not netinfo: - resp = requests.get(f"http://ip-api.com/json/{ip_address}?fields=25349915") - if resp.ok: - resp_json = resp.json() - netinfo = { - "_id": ip_hash, - "country_code": resp_json["countryCode"], - "country_name": resp_json["country"], - "region": resp_json["regionName"], - "city": resp_json["city"], - "timezone": resp_json["timezone"], - "currency": resp_json["currency"], - "as": resp_json["as"], - "isp": resp_json["isp"], - "vpn": (resp_json.get("hosting") or resp_json.get("proxy")), - "last_refreshed": int(time.time()) - } - db.netinfo.update_one({"_id": ip_hash}, {"$set": netinfo}, upsert=True) - else: - netinfo = { - "_id": ip_hash, - "country_code": "Unknown", - "country_name": "Unknown", - "region": "Unknown", - "city": "Unknown", - "timezone": "Unknown", - "currency": "Unknown", - "as": "Unknown", - "isp": "Unknown", - "vpn": False, - "last_refreshed": int(time.time()) - } - - return netinfo + +def log_security_action(action_type: str, user: str, data: dict): + db.security_log.insert_one({ + "_id": str(uuid.uuid4()), + "type": action_type, + "user": user, + "time": int(time.time()), + "data": data + }) + + if action_type in { + "email_changed", + "password_changed", + "mfa_added", + "mfa_removed", + "mfa_recovery_reset", + "mfa_recovery_used", + "locked" + }: + tmpl_name = "locked" if action_type == "locked" else "security_alert" + platform_name = os.environ["EMAIL_PLATFORM_NAME"] + + account = db.usersv0.find_one({"_id": user}, projection={"_id": 1, "email": 1}) + + txt_tmpl, html_tmpl = render_email_tmpl(tmpl_name, account["_id"], account.get("email", ""), { + "msg": { + "email_changed": f"The email address on your {platform_name} account has been changed.", + "password_changed": f"The password on your {platform_name} account has been changed.", + "mfa_added": f"A multi-factor authenticator has been added to your {platform_name} account.", + "mfa_removed": f"A multi-factor authenticator has been removed from your {platform_name} account.", + "mfa_recovery_reset": f"The multi-factor authentication recovery code on your {platform_name} account has been reset.", + "mfa_recovery_used": f"Your multi-factor authentication recovery code has been used to reset multi-factor authentication on your {platform_name} account." + }[action_type] if action_type != "locked" else None, + "token": create_token("email", [ # this doesn't use EmailTicket in sessions.py because it'd be a recursive import + account["email"], + account["_id"], + "lockdown", + int(time.time())+86400 + ]) if account.get("email") and action_type != "locked" else None + }) + + # Email + if account.get('email'): + Thread( + target=send_email, + args=[EMAIL_SUBJECTS[tmpl_name], account["_id"], account["email"], txt_tmpl, html_tmpl] + ).start() + + # Inbox + rdb.publish("admin", msgpack.packb({ + "op": "alert_user", + "user": account["_id"], + "content": txt_tmpl + })) def add_audit_log(action_type, mod_username, mod_ip, data): @@ -513,14 +595,8 @@ def background_tasks_loop(): except Exception as e: log(f"Failed to delete account {user['_id']}: {e}") - # Revoke old sessions (60 days) - db.acc_sessions.delete_many({"refreshed_at": {"$lt": int(time.time())-5184000}}) - - # Purge old netinfo - db.netinfo.delete_many({"last_refreshed": {"$lt": int(time.time())-2419200}}) - - # Purge old netlogs - db.netlog.delete_many({"last_used": {"$lt": int(time.time())-2419200}}) + # Revoke inactive sessions (3 weeks of inactivity) + db.acc_sessions.delete_many({"refreshed_at": {"$lt": int(time.time())-(86400*21)}}) # Purge old deleted posts db.posts.delete_many({"deleted_at": {"$lt": int(time.time())-2419200}}) @@ -528,7 +604,8 @@ def background_tasks_loop(): # Purge old post revisions db.post_revisions.delete_many({"time": {"$lt": int(time.time())-2419200}}) - # Purge old admin audit logs + """ we should probably not be getting rid of audit logs... + # Purge old "get" admin audit logs db.audit_log.delete_many({ "time": {"$lt": int(time.time())-2419200}, "type": {"$in": [ @@ -545,6 +622,7 @@ def background_tasks_loop(): "got_announcements" ]} }) + """ log("Finished background tasks!") @@ -557,39 +635,41 @@ def check_password_hash(password: str, hashed_password: str) -> bool: return bcrypt.checkpw(password.encode(), hashed_password.encode()) -def send_email(template: str, to_name: str, to_address: str, token: Optional[str] = ""): +def get_normalized_email_hash(address: str) -> str: + """ + Get a hash of an email address with aliases and dots stripped. + This is to allow using address aliases, but to still detect ban evasion. + Also, Gmail ignores dots in addresses. Thanks Google. + """ + + identifier, domain = address.split("@") + identifier = re.split(r'\+|\%', identifier)[0] + identifier = identifier.replace(".", "") + + return urlsafe_b64encode(sha256(f"{identifier}@{domain}".encode()).digest()).decode() + + +def render_email_tmpl(template: str, to_name: str, to_address: str, data: Optional[dict[str, str]] = {}) -> tuple[str, str]: + data.update({ + "subject": EMAIL_SUBJECTS[template], + "name": to_name, + "address": to_address, + "env": os.environ + }) + txt_tmpl = email_env.get_template(f"{template}.txt") html_tmpl = email_env.get_template(f"{template}.html") + return txt_tmpl.render(data), html_tmpl.render(data) + + +def send_email(subject: str, to_name: str, to_address: str, txt_tmpl: str, html_tmpl: str): message = MIMEMultipart("alternative") message["From"] = formataddr((os.environ["EMAIL_FROM_NAME"], os.environ["EMAIL_FROM_ADDRESS"])) message["To"] = formataddr((to_name, to_address)) - - match template: - case "verify": - message["Subject"] = "Verify your email address" - case "recovery": - message["Subject"] = "Reset your password" - case "email_changed": - message["Subject"] = "Your email has been changed" - case "password_changed": - message["Subject"] = "Your password has been changed" - case "mfa_added": - message["Subject"] = "Multi-factor authenticator added" - case "mfa_removed": - message["Subject"] = "Multi-factor authenticator removed" - case "locked": - message["Subject"] = "Your account has been locked" - - data = { - "subject": message["Subject"], - "name": to_name, - "address": to_address, - "token": token, - "env": os.environ - } - message.attach(MIMEText(txt_tmpl.render(data), "plain")) - message.attach(MIMEText(html_tmpl.render(data), "html")) + message["Subject"] = subject + message.attach(MIMEText(txt_tmpl, "plain")) + message.attach(MIMEText(html_tmpl, "html")) with smtplib.SMTP(os.environ["EMAIL_SMTP_HOST"], int(os.environ["EMAIL_SMTP_PORT"])) as server: if os.getenv("EMAIL_SMTP_TLS"): diff --git a/sessions.py b/sessions.py index 1ab62cb..259db64 100644 --- a/sessions.py +++ b/sessions.py @@ -1,5 +1,5 @@ from typing import Optional, TypedDict, Literal -import uuid, time, msgpack +import uuid, time, msgpack, pymongo from database import db, rdb import security, errors @@ -38,40 +38,65 @@ def create(cls: "AccSession", user: str, ip: str, user_agent: str) -> "AccSessio "refreshed_at": int(time.time()) } db.acc_sessions.insert_one(data) + + security.log_security_action("session_create", user, { + "session_id": data["_id"], + "ip": ip, + "user_agent": user_agent + }) + return cls(data) @classmethod def get_by_id(cls: "AccSession", session_id: str) -> "AccSession": data: Optional[AccSessionDB] = db.acc_sessions.find_one({"_id": session_id}) if not data: - raise errors.SessionNotFound + raise errors.AccSessionNotFound return cls(data) @classmethod def get_by_token(cls: "AccSession", token: str) -> "AccSession": - session_id, _ = security.extract_token(token, "acc") + session_id, _, expires_at = security.extract_token(token, "acc") + if expires_at < int(time.time()): + raise errors.AccSessionTokenExpired return cls.get_by_id(session_id) @classmethod def get_username_by_token(cls: "AccSession", token: str) -> str: - session_id, _ = security.extract_token(token, "acc") - username = rdb.get(f"u{session_id}") + session_id, _, expires_at = security.extract_token(token, "acc") + if expires_at < int(time.time()): + raise errors.AccSessionTokenExpired + username = rdb.get(session_id) if username: return username.decode() else: session = cls.get_by_id(session_id) username = session.username - rdb.set(f"u{session_id}", username, ex=300) + rdb.set(session_id, username, ex=300) return username @classmethod def get_all(cls: "AccSession", user: str) -> list["AccSession"]: - return [cls(data) for data in db.acc_sessions.find({"user": user})] + return [ + cls(data) + for data in db.acc_sessions.find( + {"user": user}, + sort=[("refreshed_at", pymongo.DESCENDING)] + ) + ] + + @property + def id(self) -> str: + return self._db["_id"] @property def token(self) -> str: - return security.create_token("acc", [self._db["_id"], self._db["refreshed_at"]]) + return security.create_token("acc", [ + self._db["_id"], + self._db["refreshed_at"], + self._db["refreshed_at"]+(86400*21) # expire token after 3 weeks + ]) @property def username(self): @@ -91,7 +116,7 @@ def v0(self) -> AccSessionV0: def refresh(self, ip: str, user_agent: str, check_token: Optional[str] = None): if check_token: # token re-use prevention - _, refreshed_at = security.extract_token(check_token, "acc") + _, refreshed_at, _ = security.extract_token(check_token, "acc") if refreshed_at != self._db["refreshed_at"]: return self.revoke() @@ -102,6 +127,12 @@ def refresh(self, ip: str, user_agent: str, check_token: Optional[str] = None): }) db.acc_sessions.update_one({"_id": self._db["_id"]}, {"$set": self._db}) + security.log_security_action("session_refresh", self._db["user"], { + "session_id": self._db["_id"], + "ip": ip, + "user_agent": user_agent + }) + def revoke(self): db.acc_sessions.delete_one({"_id": self._db["_id"]}) rdb.delete(f"u{self._db['_id']}") @@ -111,6 +142,10 @@ def revoke(self): "sid": self._db["_id"] })) + security.log_security_action("session_revoke", self._db["user"], { + "session_id": self._db["_id"] + }) + class EmailTicket: def __init__(