task/WP-271 Upgrade Django and 3rd party dependancies (#895)

* updated django version and dependencies * upgraded channels to version 3 and fixed tests * upgraded channels to version 4 * formatting fix * Added nginx conf * add pytest filterwarning * removed added nginx conf, added CSRF trusted origin for cep.test * upgrade postgres to v14.9 * linting, added CSRF setting to settings_default * update settings --------- Co-authored-by: Sal Tijerina <[email protected]> Co-authored-by: Chandra Y <[email protected]>
TACC · Dec 8, 2023 · 1337741 · 1337741
1 parent da23c0e
commit 1337741
Show file tree

Hide file tree

Showing 21 changed files with 1,279 additions and 1,005 deletions.
diff --git a/server/conf/docker/docker-compose-dev.all.debug.yml b/server/conf/docker/docker-compose-dev.all.debug.yml
@@ -50,7 +50,7 @@ services:
       - 127.0.0.1:9200:9200
 
   postgres:
-    image: postgres:11.5
+    image: postgres:14.9
     volumes:
       - core_portal_postgres_data:/var/lib/postgresql/data/portal
     container_name: core_portal_postgres
@@ -61,7 +61,7 @@ services:
       - PGDATA=/var/lib/postgresql/data/portal
 
   postgrescms:
-    image: postgres:11.5
+    image: postgres:14.9
     volumes:
       - core_cms_postgres_data:/var/lib/postgresql/data/portal
     container_name: core_cms_postgres

diff --git a/server/poetry.lock b/server/poetry.lock
diff --git a/server/portal/apps/accounts/api/urls.py b/server/portal/apps/accounts/api/urls.py
@@ -2,7 +2,7 @@
 .. :module:: apps.accounts.api.urls
    :synopsis: Manager handling anything pertaining to accounts
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from portal.apps.accounts.api.views.systems import SystemKeysView
 
 app_name = 'portal_accounts_api'

diff --git a/server/portal/apps/accounts/urls.py b/server/portal/apps/accounts/urls.py
@@ -2,7 +2,7 @@
 .. module:: portal.apps.accounts.urls
    :synopsis: Accounts URLs
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from django.contrib.auth.views import LogoutView
 from portal.apps.accounts.views import accounts
 from portal.apps.accounts import views

diff --git a/server/portal/apps/auth/urls.py b/server/portal/apps/auth/urls.py
@@ -2,7 +2,7 @@
 .. module:: portal.apps.auth.urls
    :synopsis: Auth URls
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from portal.apps.auth import views
 
 app_name = 'portal_auth'

diff --git a/server/portal/apps/notifications/consumers_unit_test.py b/server/portal/apps/notifications/consumers_unit_test.py
@@ -46,8 +46,8 @@ async def auth_connect(user):
 
     # Pass session ID in headers to authenticate.
     communicator = WebsocketCommunicator(
-        NotificationsConsumer,
-        path='/ws/notifications/'
+        NotificationsConsumer.as_asgi(),
+        path='/ws/notifications/',
     )
     communicator.scope['user'] = user
     connected, _ = await communicator.connect()

diff --git a/server/portal/apps/notifications/routing.py b/server/portal/apps/notifications/routing.py
@@ -7,5 +7,5 @@
 from portal.apps.notifications.consumers import NotificationsConsumer
 
 websocket_urlpatterns = [
-    re_path(r'ws/notifications/$', NotificationsConsumer),
+    re_path(r'ws/notifications/$', NotificationsConsumer.as_asgi()),
 ]
diff --git a/server/portal/apps/notifications/urls.py b/server/portal/apps/notifications/urls.py
@@ -1,4 +1,4 @@
-from django.conf.urls import re_path
+from django.urls import re_path
 from portal.apps.notifications.views import ManageNotificationsView
 
 

diff --git a/server/portal/apps/onboarding/api/views_unit_test.py b/server/portal/apps/onboarding/api/views_unit_test.py
@@ -199,7 +199,7 @@ def test_admin_route(client, authenticated_staff):
     # If the user is authenticated and is_staff, then the route should
     # return a JsonResponse
     response = client.get("/api/onboarding/admin/")
-    assert type(response) == JsonResponse
+    assert isinstance(response, JsonResponse)
 
 
 def test_admin_route_is_protected(authenticated_user, client):

diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py
@@ -14,8 +14,8 @@
 from portal.views.base import BaseApiView
 from portal.apps.projects.managers.base import ProjectsManager
 from portal.apps.projects.workspace_operations.shared_workspace_operations import \
-        list_projects, get_project, create_shared_workspace,\
-        update_project, get_workspace_role, change_user_role, add_user_to_workspace,\
+        list_projects, get_project, create_shared_workspace, \
+        update_project, get_workspace_role, change_user_role, add_user_to_workspace, \
         remove_user, transfer_ownership
 from portal.apps.search.tasks import tapis_project_listing_indexer
 from portal.libs.elasticsearch.indexes import IndexedProject

diff --git a/server/portal/apps/users/urls.py b/server/portal/apps/users/urls.py
@@ -1,5 +1,4 @@
-from django.conf.urls import re_path
-from django.urls import path
+from django.urls import path, re_path
 from portal.apps.users.views import (SearchView, AuthenticatedView, UsageView, AllocationsView, TeamView,
                                      UserDataView, TasUsersView, AllocationUsageView, AllocationManagementView)
 

diff --git a/server/portal/apps/webhooks/models.py b/server/portal/apps/webhooks/models.py
@@ -1,6 +1,6 @@
 from django.db import models
 from django.conf import settings
-from datetime import datetime
+from django.utils import timezone
 
 
 class ExternalCall(models.Model):
@@ -21,7 +21,7 @@ class ExternalCall(models.Model):
     )
 
     # Timestamp for outbound external call
-    time = models.DateTimeField(default=datetime.now)
+    time = models.DateTimeField(default=timezone.now)
 
     # Callback class to be executed when a webhook is received
     callback = models.CharField(max_length=300, null=True)

diff --git a/server/portal/apps/workspace/api/urls.py b/server/portal/apps/workspace/api/urls.py
@@ -1,6 +1,6 @@
 """Workpace API Urls
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from portal.apps.workspace.api import views
 
 

diff --git a/server/portal/apps/workspace/urls.py b/server/portal/apps/workspace/urls.py
@@ -1,6 +1,6 @@
 """Workspace URLs
 """
-from django.conf.urls import re_path
+from django.urls import re_path
 from portal.apps.workspace import views
 
 urlpatterns = [

diff --git a/server/portal/asgi.py b/server/portal/asgi.py
@@ -5,8 +5,12 @@
 
 import os
 import django
-from channels.routing import get_default_application
+from channels.routing import ProtocolTypeRouter
+from django.core.asgi import get_asgi_application
+
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings.settings')
 django.setup()
-application = get_default_application()
+application = ProtocolTypeRouter({
+    "http": get_asgi_application(),
+})
diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py
@@ -44,6 +44,8 @@
 # whether the session cookie should be secure (https:// only)
 SESSION_COOKIE_SECURE = True
 CSRF_COOKIE_SAMESITE = 'Strict'
+# for local testing
+CSRF_TRUSTED_ORIGINS = getattr(settings_custom, '_CSRF_TRUSTED_ORIGINS', [])
 
 SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 ALLOWED_HOSTS = ['*']
@@ -64,6 +66,10 @@
 
 INSTALLED_APPS = [
 
+    # Django Channels
+    'channels',
+    'daphne',
+
     # Core Django.
     'django.contrib.admin',
     'django.contrib.auth',
@@ -75,9 +81,7 @@
     'django.contrib.sitemaps',
     'django.contrib.sessions.middleware',
 
-    # Django Channels
-    'channels',
-
+    # Pipeline.
     'termsandconditions',
     'impersonate',
 

diff --git a/server/portal/settings/settings_custom.example.py b/server/portal/settings/settings_custom.example.py
@@ -28,6 +28,7 @@
 
 _RT_QUEUE = 'Web & Mobile Apps'
 _RT_TAG = 'core_portal'
+_CSRF_TRUSTED_ORIGINS = ['https://cep.test']
 
 ########################
 # AGAVE SETTINGS

diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py
@@ -28,6 +28,7 @@
 
 _RT_QUEUE = 'Web & Mobile Apps'
 _RT_TAG = 'core_portal'
+_CSRF_TRUSTED_ORIGINS = ['https://cep.test']
 
 ########################
 # TAPIS SETTINGS

diff --git a/server/portal/settings/unit_test_settings.py b/server/portal/settings/unit_test_settings.py
@@ -47,6 +47,10 @@
 
 INSTALLED_APPS = [
 
+    # Django Channels
+    'channels',
+    'daphne',
+
     # Core Django.
     'django.contrib.admin',
     'django.contrib.auth',
@@ -58,9 +62,6 @@
     'django.contrib.sitemaps',
     'django.contrib.sessions.middleware',
 
-    # Django Channels
-    'channels',
-
     # Pipeline.
     'termsandconditions',
     'impersonate',

diff --git a/server/pyproject.toml b/server/pyproject.toml
@@ -6,7 +6,7 @@ authors = ["TACC-WMA <[email protected]>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-Django = "^3.2"
+Django = "^4.2"
 python = "^3.11"
 elasticsearch-dsl = "^7.2.1"
 paramiko = "^2.10.1"
@@ -26,8 +26,9 @@ elasticsearch = "^7.7.1"
 uwsgi = "^2.0.22"
 requests = "^2.31.0"
 django-impersonate = "^1.5"
-channels = "^2.4.0"
-channels-redis = "^3.1.0"
+channels = "^4.0.0"
+channels-redis = "^4.1.0"
+daphne = "^4.0.0"
 pytas = {git = "https://bitbucket.org/taccaci/pytas.git", tag = "v1.6.0"}
 google-auth = "^1.23.0"
 google-auth-oauthlib = "^0.4.2"
@@ -46,6 +47,7 @@ pytest-mock = "^3.10.0"
 mock = "^5.0.2"
 pytest-cov = "^4.0.0"
 pytest-django = "^4.5.2"
+pytest-asyncio = "^0.21.1"
 flake8 = "^6.0.0"
 coverage = "^7.2.5"
 requests-mock = "^1.10.0"

diff --git a/server/pytest.ini b/server/pytest.ini
@@ -2,15 +2,9 @@
 DJANGO_SETTINGS_MODULE = portal.settings.unit_test_settings
 python_files = tests.py *unit_test.py
 
-filterwarnings =
-    ignore::DeprecationWarning:django.*:
-    ignore::DeprecationWarning:openapi_schema_validator.*:
-    ignore::DeprecationWarning:openapi_spec_validator.*:
-    ignore::DeprecationWarning:openapi_core.*:
-    ignore::DeprecationWarning:kombu.*:
-    ignore::DeprecationWarning:pkg_resources.*:
-    ignore:.*Django now detects this configuration.*:django.utils.deprecation.RemovedInDjango41Warning
-
+filterwarnings = 
+    ignore::DeprecationWarning:twisted.*:
+    ignore::django.utils.deprecation.RemovedInDjango50Warning
 markers =
     asyncio: mark a test as asyncio.