Merge branch 'main' into task/WP-288-QueueFilter

TACC · Oct 24, 2023 · 96e4a2e · 96e4a2e
2 parents fb3c323 + e427c97
commit 96e4a2e
Show file tree

Hide file tree

Showing 12 changed files with 113 additions and 69 deletions.
diff --git a/client/src/components/ManageAccount/ManageAccountTables.jsx b/client/src/components/ManageAccount/ManageAccountTables.jsx
@@ -65,8 +65,6 @@ export const ProfileInformation = () => {
       { Header: 'Institution', accessor: 'institution' },
       { Header: 'Country of Residence', accessor: 'country' },
       { Header: 'Country of Citizenship', accessor: 'citizenship' },
-      { Header: 'Ethnicity', accessor: 'ethnicity' },
-      { Header: 'Gender', accessor: 'gender' },
     ],
     []
   );

diff --git a/client/src/components/ManageAccount/tests/ManageAccountTables.test.js b/client/src/components/ManageAccount/tests/ManageAccountTables.test.js
@@ -21,12 +21,6 @@ const dummyState = {
   },
   data: {
     demographics: {
-      ethnicity: 'Asian',
-      gender: 'Male',
-      bio: '',
-      website: 'http://owais.io',
-      orcid_id: 'test',
-      professional_level: 'Staff (support, administration, etc)',
       username: 'ojamil',
       email: '[email protected]',
       firstName: 'Owais',
@@ -77,8 +71,6 @@ describe('Profile Information Component', () => {
       'Institution',
       'Country of Residence',
       'Country of Citizenship',
-      'Ethnicity',
-      'Gender',
     ];
     headings.forEach((heading) => {
       expect(getByText(heading)).toBeInTheDocument();

diff --git a/server/portal/apps/accounts/migrations/0006_auto_20231018_1927.py b/server/portal/apps/accounts/migrations/0006_auto_20231018_1927.py
@@ -0,0 +1,37 @@
+# Generated by Django 3.2.20 on 2023-10-18 19:27
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0005_auto_20210316_1950'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='portalprofile',
+            name='bio',
+        ),
+        migrations.RemoveField(
+            model_name='portalprofile',
+            name='ethnicity',
+        ),
+        migrations.RemoveField(
+            model_name='portalprofile',
+            name='gender',
+        ),
+        migrations.RemoveField(
+            model_name='portalprofile',
+            name='orcid_id',
+        ),
+        migrations.RemoveField(
+            model_name='portalprofile',
+            name='professional_level',
+        ),
+        migrations.RemoveField(
+            model_name='portalprofile',
+            name='website',
+        ),
+    ]
diff --git a/server/portal/apps/accounts/models.py b/server/portal/apps/accounts/models.py
@@ -26,13 +26,6 @@ class PortalProfile(models.Model):
         related_name='profile',
         on_delete=models.CASCADE
     )
-    ethnicity = models.CharField(max_length=255)
-    gender = models.CharField(max_length=255)
-    bio = models.CharField(max_length=4096, default=None, null=True, blank=True)
-    website = models.CharField(max_length=256, default=None, null=True, blank=True)
-    orcid_id = models.CharField(max_length=256, default=None, null=True, blank=True)
-    professional_level = models.CharField(max_length=256, default=None, null=True)
-
     # Default to False. If PORTAL_USER_ACCOUNT_SETUP_STEPS is empty,
     # setup_complete will be set to True on first login
     setup_complete = models.BooleanField(default=False)

diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py
@@ -17,7 +17,9 @@
         list_projects, get_project, create_shared_workspace,\
         update_project, get_workspace_role, change_user_role, add_user_to_workspace,\
         remove_user, transfer_ownership
-
+from portal.apps.search.tasks import tapis_project_listing_indexer
+from portal.libs.elasticsearch.indexes import IndexedProject
+from elasticsearch_dsl import Q
 
 LOGGER = logging.getLogger(__name__)
 
@@ -63,12 +65,35 @@ def get(self, request):
         }
         ```
         """
-        # TODOv3: Support Elasticsearch queries for V3 projects https://jira.tacc.utexas.edu/browse/TV3-160
-        # query_string = request.GET.get('query_string')
-        # offset = int(request.GET.get('offset', 0))
-        # limit = int(request.GET.get('limit', 100))
-        client = request.user.tapis_oauth.client
-        listing = list_projects(client)
+
+        query_string = request.GET.get('query_string')
+        offset = int(request.GET.get('offset', 0))
+        limit = int(request.GET.get('limit', 100))
+
+        listing = []
+
+        if query_string:
+            search = IndexedProject.search()
+
+            ngram_query = Q("query_string", query=query_string,
+                            fields=["title", "id"],
+                            minimum_should_match='100%',
+                            default_operator='or')
+
+            wildcard_query = Q("wildcard", title=f'*{query_string}*') | Q("wildcard", id=f'*{query_string}*')
+
+            search = search.query(ngram_query | wildcard_query)
+            search = search.extra(from_=int(offset), size=int(limit))
+
+            res = search.execute()
+            hits = [hit.to_dict() for hit in res]
+            listing = hits
+        else:
+            client = request.user.tapis_oauth.client
+            listing = list_projects(client)
+
+        tapis_project_listing_indexer.delay(listing)
+
         return JsonResponse({"status": 200, "response": listing})
 
     def post(self, request):  # pylint: disable=no-self-use

diff --git a/server/portal/apps/search/tasks.py b/server/portal/apps/search/tasks.py
@@ -3,7 +3,7 @@
 from django.conf import settings
 from celery import shared_task
 from portal.libs.agave.utils import user_account, service_account
-from portal.libs.elasticsearch.utils import index_listing
+from portal.libs.elasticsearch.utils import index_listing, index_project_listing
 from portal.apps.users.utils import get_tas_allocations
 from portal.apps.projects.models.metadata import ProjectMetadata
 from portal.libs.elasticsearch.docs.base import (IndexedAllocation,
@@ -78,3 +78,8 @@ def index_project(self, project_id):
     project_doc = IndexedProject(**project_dict)
     project_doc.meta.id = project_id
     project_doc.save()
+
+
+@shared_task(bind=True, max_retries=3, queue='default')
+def tapis_project_listing_indexer(self, projects):
+    index_project_listing(projects)
diff --git a/server/portal/fixtures/accounts.json b/server/portal/fixtures/accounts.json
@@ -3,27 +3,21 @@
   "model": "accounts.portalprofile",
   "pk": 1,
   "fields": {
-    "user": 1,
-    "ethnicity": "",
-    "gender": ""
+    "user": 1
   }
 },
 {
   "model": "accounts.portalprofile",
   "pk": 2,
   "fields": {
-    "user": 2,
-    "ethnicity": "",
-    "gender": ""
+    "user": 2
   }
 },
 {
   "model": "accounts.portalprofile",
   "pk": 3,
   "fields": {
-    "user": 3,
-    "ethnicity": "",
-    "gender": ""
+    "user": 3
   }
 },
 {

diff --git a/server/portal/fixtures/users.json b/server/portal/fixtures/users.json
@@ -58,12 +58,6 @@
     "pk": 1,
     "fields": {
       "user": 1,
-      "ethnicity": "",
-      "gender": "",
-      "bio": null,
-      "website": null,
-      "orcid_id": null,
-      "professional_level": null,
       "setup_complete": true
     }
   }

diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py
@@ -139,7 +139,7 @@ def search(client, system, path='', offset=0, limit=100, query_string='', filter
     if filter:
         search = search.filter(filter_query)
 
-    search = search.filter('prefix', **{'path._exact': path})
+    search = search.filter('prefix', **{'path._exact': path.strip('/')})
     search = search.filter('term', **{'system._exact': system})
     search = search.extra(from_=int(offset), size=int(limit))
     res = search.execute()

diff --git a/server/portal/libs/agave/operations_unit_test.py b/server/portal/libs/agave/operations_unit_test.py
@@ -74,7 +74,7 @@ def test_search(self, mock_search, mock_listing):
                                                      "name._exact, name._pattern"],
                                                  default_operator='and'))
 
-        mock_search().query().filter.assert_called_with('prefix', **{'path._exact': '/path'})
+        mock_search().query().filter.assert_called_with('prefix', **{'path._exact': 'path'})
         mock_search().query().filter().filter.assert_called_with('term', **{'system._exact': 'test.system'})
         mock_search().query().filter().filter().extra.assert_called_with(from_=int(0), size=int(100))
         self.assertEqual(search_res, {'listing':

diff --git a/server/portal/libs/elasticsearch/docs/base.py b/server/portal/libs/elasticsearch/docs/base.py
@@ -17,37 +17,21 @@
 
 
 class IndexedProject(Document):
+    id = Keyword(fields={'_exact': Keyword()})
     title = Text(fields={'_exact': Keyword()})
     description = Text()
-    created = Date()
-    lastModified = Date()
-    projectId = Keyword()
+    path = Text()
+    name = Text()
+    host = Text()
     owner = Object(
-        properties={
-            'username': Keyword(),
-            'fullName': Text()
-        }
-    )
-    pi = Object(
-        properties={
-            'username': Keyword(),
-            'fullName': Text()
-        }
-    )
-    coPIs = Object(
-        multi=True,
-        properties={
-            'username': Keyword(),
-            'fullName': Text()
-        }
-    )
-    teamMembers = Object(
-        multi=True,
-        properties={
-            'username': Keyword(),
-            'fullName': Text()
-        }
+            properties={
+                'username': Keyword(),
+                'firstName': Text(),
+                'lastName': Text(),
+                'email': Text()
+            }
     )
+    updated = Date()
 
     @classmethod
     def from_id(cls, projectId):

diff --git a/server/portal/libs/elasticsearch/utils.py b/server/portal/libs/elasticsearch/utils.py
@@ -219,3 +219,25 @@ def index_listing(files):
             })
 
     bulk(client, ops)
+
+
+def index_project_listing(projects):
+    from portal.libs.elasticsearch.docs.base import IndexedProject
+
+    idx = IndexedProject.Index.name
+    client = get_connection('default')
+    ops = []
+
+    for _project in projects:
+        project_dict = dict(_project)
+        project_dict['updated'] = current_time()
+        project_uuid = get_sha256_hash(project_dict['id'])
+        ops.append({
+            '_index': idx,
+            '_id': project_uuid,
+            'doc': project_dict,
+            '_op_type': 'update',
+            'doc_as_upsert': True
+        })
+
+    bulk(client, ops)