Replace Logz.io with Datadog

.env.example

@@ -43,10 +43,13 @@ SERVER_STATELESS=false
 
 # Logging Configuration
 LOG_LEVEL=INFO
-
-LOGZIO_ENABLED=false
-LOGZIO_TOKEN=
-LOGZIO_URL=https://listener.logz.io:8071
+DD_HOST=datadog
+DD_PORT=8126
+DD_STATS_HOST=datadog
+DD_STATS_PORT=8125
+DD_ENABLED=0
+DD_API_KEY=
+DD_SITE=
 
 # Utility Configuration
 PHPMYADMIN_PORT=8080

.gitignore

@@ -138,3 +138,6 @@ mysql_data/
 meilisearch_data/
 redis_data/
 rgdps_data/
+
+# Datadog
+logs.log

Makefile

@@ -7,13 +7,15 @@ run:
 		redis \
 		mysql \
 		meilisearch \
+		datadog \
 		realistikgdps
 
 run-bg:
 	docker-compose up -d \
 		redis \
 		mysql \
 		meilisearch \
+		datadog \
 		realistikgdps
 
 stop:
@@ -28,6 +30,9 @@ shell:
 pma:
 	docker-compose up phpmyadmin
 
+datadog:
+	docker-compose up datadog
+
 converter:
 	APP_COMPONENT=converter docker-compose up \
 		redis \

README.md

@@ -12,7 +12,6 @@ It is written in asynchronous, modern Python and is meant as a replacement for o
 - MeiliSearch, allowing for typo tolerance
 - S3 support, allowing for flexible storage solutions
 - Proper ratelimiting
-- Logz.io logging support
 - Flexible command framework
 
 ## How to set up?

datadog/conf.d/realistikgdps.yaml

@@ -0,0 +1,4 @@
+logs:
+  - type: file
+    path: /data/logs.log
+    service: "realistikgdps"

docker-compose.yml

@@ -33,6 +33,7 @@ services:
       - mysql
       - redis
       - meilisearch
+      - datadog
     restart: always
     environment:
       - APP_PORT=${APP_PORT}
@@ -65,10 +66,16 @@ services:
       - SERVER_GD_URL=${SERVER_GD_URL}
       - SERVER_STATELESS=${SERVER_STATELESS}
 
-      - LOG_LEVEL=${LOG_LEVEL}
-      - LOGZIO_ENABLED=${LOGZIO_ENABLED}
-      - LOGZIO_TOKEN=${LOGZIO_TOKEN}
-      - LOGZIO_URL=${LOGZIO_URL}
+      - DD_ENABLED=${DD_ENABLED}
+      - DD_HOST=${DD_HOST}
+      - DD_PORT=${DD_PORT}
+      - DD_STATS_HOST=${DD_STATS_HOST}
+      - DD_STATS_PORT=${DD_STATS_PORT}
+      - DD_API_KEY=${DD_API_KEY}
+      - DD_SITE=${DD_SITE}
+      - DD_LOGS_INJECTION=true
+      - DD_SERVICE=realistikgdps
+      - DD_LOGS_ENABLED=true
 
       # Internal docker specific variables
       - INTERNAL_RGDPS_DIRECTORY=/data # NOTE: Ensure this matches the volume mount below.
@@ -98,3 +105,22 @@ services:
       - UPLOAD_LIMIT=500M
     depends_on:
       - mysql
+
+  datadog:
+    image: gcr.io/datadoghq/agent:7
+    ports:
+      - "8125:8125/udp"
+      - "8126:8126"
+    environment:
+      - DD_API_KEY=${DD_API_KEY}
+      - DD_SITE=${DD_SITE}
+      - DD_APM_ENABLED=true
+      - DD_LOGS_ENABLED=true
+      - DD_TRACE_ENABLED=true
+      - DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
+      - .:/data
+      - ./datadog/conf.d:/etc/datadog-agent/conf.d

logging.yaml

@@ -0,0 +1,28 @@
+version: 1
+disable_existing_loggers: true
+loggers:
+  httpx:
+    level: WARNING
+    handlers: [console, file]
+    propagate: no
+  httpcore:
+    level: WARNING
+    handlers: [console, file]
+    propagate: no
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: plaintext
+    stream: ext://sys.stdout
+  file:
+    class: logging.FileHandler
+    formatter: json
+    filename: logs.log
+formatters:
+  plaintext:
+    format: '%(asctime)s %(name)s %(levelname)s %(message)s'
+  json:
+    class: pythonjsonlogger.jsonlogger.JsonFormatter
+    format: '%(asctime)s %(name)s %(levelname)s %(message)s'
+root:
+  handlers: [console, file]

requirements/main.txt

@@ -2,11 +2,12 @@ aiobotocore == 2.9.0
 bcrypt == 4.1.2
 cryptography
 databases[asyncmy] == 0.8.0
+datadog == 0.49.1
+ddtrace == 2.7.6
 email-validator == 2.0.0
 fastapi == 0.108.0
 fastapi-limiter == 0.1.5
 httpx == 0.26.0
-logzio-python-handler == 4.1.0
 meilisearch-python-sdk == 2.0.1
 orjson == 3.9.15
 python-dotenv == 1.0.1
@@ -17,3 +18,5 @@ uvicorn == 0.19.0
 uvloop == 0.19.0; sys_platform != "win32"
 winloop == 0.1.0; sys_platform == "win32"
 xor-cipher == 3.0.1
+python-json-logger == 2.0.7
+PyYAML == 6.0.1

rgdps/api/__init__.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import urllib.parse
+import logging
 import uuid
 
 from databases import DatabaseURL
@@ -32,14 +33,7 @@
 
 
 def init_logging() -> None:
-    if settings.LOGZIO_ENABLED:
-        logger.init_logzio_logging(
-            settings.LOGZIO_TOKEN,
-            settings.LOG_LEVEL,
-            settings.LOGZIO_URL,
-        )
-    else:
-        logger.init_basic_logging(settings.LOG_LEVEL)
+    logger.init_basic_logging(settings.LOG_LEVEL)
 
 
 def init_events(app: FastAPI) -> None:
@@ -48,7 +42,7 @@ async def on_validation_error(
         request: Request,
         e: RequestValidationError,
     ) -> Response:
-        logger.exception(
+        logging.exception(
             f"A validation error has occured while parsing a request.",
             extra={
                 "url": str(request.url),
@@ -84,7 +78,7 @@ def init_mysql(app: FastAPI) -> None:
     @app.on_event("startup")
     async def on_startup() -> None:
         await app.state.mysql.connect()
-        logger.info(
+        logging.info(
             "Connected to the MySQL database.",
             extra={
                 "host": settings.SQL_HOST,
@@ -118,7 +112,7 @@ async def on_startup() -> None:
             prefix="rgdps:ratelimit",
         )
 
-        logger.info(
+        logging.info(
             "Connected to the Redis database.",
             extra={
                 "host": settings.REDIS_HOST,
@@ -141,7 +135,7 @@ def init_meili(app: FastAPI) -> None:
     @app.on_event("startup")
     async def startup() -> None:
         await app.state.meili.health()
-        logger.info(
+        logging.info(
             "Connected to the MeiliSearch database.",
             extra={
                 "host": settings.MEILI_HOST,
@@ -163,7 +157,7 @@ def init_s3_storage(app: FastAPI) -> None:
     @app.on_event("startup")
     async def startup() -> None:
         app.state.storage = await app.state.storage.connect()
-        logger.info(
+        logging.info(
             "Connected to S3 storage.",
             extra={
                 "bucket": settings.S3_BUCKET,
@@ -183,15 +177,15 @@ def init_local_storage(app: FastAPI) -> None:
 
     @app.on_event("startup")
     async def startup() -> None:
-        logger.info("Connected to the local storage.")
+        logging.info("Connected to the local storage.")
 
 
 def init_gd(app: FastAPI) -> None:
     app.state.gd = GeometryDashClient(
         settings.SERVER_GD_URL,
     )
 
-    logger.info(
+    logging.info(
         "Initialised the main Geometry Dash client.",
         extra={
             "server_url": settings.SERVER_GD_URL,
@@ -203,7 +197,7 @@ def init_cache_stateful(app: FastAPI) -> None:
     app.state.user_cache = SimpleAsyncMemoryCache()
     app.state.password_cache = SimpleAsyncMemoryCache()
 
-    logger.info("Initialised stateful caching.")
+    logging.info("Initialised stateful caching.")
 
 
 def init_cache_stateless(app: FastAPI) -> None:
@@ -218,7 +212,7 @@ def init_cache_stateless(app: FastAPI) -> None:
         serialise=lambda x: x.encode(),
     )
 
-    logger.info("Initialised stateless caching.")
+    logging.info("Initialised stateless caching.")
 
 
 def init_routers(app: FastAPI) -> None:
@@ -230,7 +224,7 @@ def init_routers(app: FastAPI) -> None:
 def init_middlewares(app: FastAPI) -> None:
     @app.middleware("http")
     async def mysql_transaction(request: Request, call_next):
-        logger.debug(
+        logging.debug(
             "Opened a new MySQL transaction for request.",
             extra={
                 "uuid": request.state.uuid,
@@ -250,7 +244,7 @@ async def enforce_user_agent(
             # GD sends an empty User-Agent header.
             user_agent = request.headers.get("User-Agent")
             if user_agent != "":
-                logger.info(
+                logging.info(
                     "Client request stopped due to invalid User-Agent header.",
                     extra={
                         "url": str(request.url),
@@ -270,7 +264,7 @@ async def exception_logging(
         try:
             return await call_next(request)
         except Exception as e:
-            logger.exception(
+            logging.exception(
                 f"An exception has occured while processing a request!",
                 extra={
                     "url": str(request.url),
@@ -288,6 +282,7 @@ async def assign_uuid(request: Request, call_next):
 
 def init_api() -> FastAPI:
     init_logging()
+
     app = FastAPI(
         title="RealistikGDPS",
         openapi_url=None,

rgdps/api/commands/framework.py

@@ -14,7 +14,7 @@
 from typing import get_origin
 from typing import get_type_hints
 
-from rgdps import logger
+import logging
 from rgdps import repositories
 from rgdps.common.context import Context
 from rgdps.constants.errors import ServiceError
@@ -107,7 +107,7 @@ async def _resolve_from_type[T](ctx: CommandContext, value: str, cast: type[T])
     elif issubclass(cast, Enum):
         return cast(value)
 
-    logger.error(
+    logging.error(
         "Command parser tried to parse an unsupported type!",
         extra={
             "value": value,
@@ -323,7 +323,7 @@ def merge(self, router: CommandRouter) -> None:
 
         for key, value in router._routes.items():
             if key in self._routes:
-                logger.warning(
+                logging.warning(
                     "Command router merge has overwritten an existing command!",
                     extra={
                         "command": key,
@@ -390,7 +390,7 @@ async def entrypoint(
         if level_id is not None:
             level = await repositories.level.from_id(base_ctx, level_id)
             if level is None:
-                logger.error(
+                logging.error(
                     "Failed to resolve the command level!",
                     extra={"level_id": level_id},
                 )
@@ -400,7 +400,7 @@ async def entrypoint(
         if target_user_id is not None:
             target_user = await repositories.user.from_id(base_ctx, target_user_id)
             if target_user is None:
-                logger.error(
+                logging.error(
                     "Failed to resolve the command target user!",
                     extra={"target_user_id": target_user_id},
                 )
@@ -511,7 +511,7 @@ def decorator(func: CommandConditional) -> CommandConditional:
 
 # Command specific event handlers
 async def _event_on_exception(ctx: CommandContext, exception: Exception) -> str:
-    logger.exception(
+    logging.exception(
         "An exception has occurred while executing command!",
         extra={
             "command_name": ctx.layer.name,
@@ -599,7 +599,7 @@ async def execute(self, ctx: CommandContext) -> str:
 
         try:
             result = await self.handle(ctx, *params)
-            logger.info(
+            logging.info(
                 "Successfully executed command!",
                 extra={
                     "command_name": self.name,
@@ -610,7 +610,7 @@ async def execute(self, ctx: CommandContext) -> str:
         except CommandException as e:
             return await self._event_interruption(ctx, e)
         except Exception as e:
-            logger.exception(
+            logging.exception(
                 "Failed to run command handler!",
                 extra={
                     "command_name": self.name,