refactor: large refactoring of filtering functionality and performance

.vscode/settings.json

@@ -1,7 +1,8 @@
 {
   "i18n-ally.localesPaths": [
     "packages/locales/lib/human",
-    "packages/locales/lib/generated"
+    "packages/locales/lib/generated",
+    "packages/locales/lib/data"
   ],
   "i18n-ally.keystyle": "flat",
   "[javascript]": {
@@ -14,4 +15,4 @@
   },
   "editor.formatOnSave": true,
   "docwriter.style": "JSDoc"
-}
+}

package.json

@@ -168,9 +168,7 @@
     "react-i18next": "^11.16.7",
     "react-leaflet": "4.2.1",
     "react-router-dom": "^6.15.0",
-    "react-virtualized-auto-sizer": "^1.0.20",
-    "react-virtuoso": "^4.5.0",
-    "react-window": "^1.8.9",
+    "react-virtuoso": "^4.6.2",
     "rtree": "^1.4.2",
     "source-map": "^0.7.4",
     "suncalc": "^1.9.0",

packages/locales/lib/human/en.json

@@ -713,5 +713,7 @@
   "expert": "Expert",
   "basic_description": "Easily select Pokémon and apply a global filter",
   "intermediate_description": "Set individual filters globally and per Pokémon (traditional)",
-  "expert_description": "Manual input queries for the most customization"
+  "expert_description": "Manual input queries for the most customization",
+  "icon_size": "Icon Size",
+  "developer": "Developer"
 }

packages/types/lib/client.d.ts

@@ -1,7 +1,8 @@
 import * as React from 'react'
 import { Config } from './config'
 import UAssets from '@services/Icons'
-import { ButtonProps } from '@mui/material'
+import { ButtonProps, SxProps, Theme } from '@mui/material'
+import { SystemStyleObject } from '@mui/system'
 
 declare global {
   declare const CONFIG: Config<true>
@@ -15,3 +16,38 @@ declare global {
 export interface CustomI extends React.HTMLProps<HTMLLIElement> {
   size?: ButtonProps['size']
 }
+
+export type TimesOfDay = 'day' | 'night' | 'dawn' | 'dusk'
+
+export type MarginProps = {
+  [Key in
+    | 'm'
+    | 'mt'
+    | 'mb'
+    | 'ml'
+    | 'mr'
+    | 'mx'
+    | 'my']?: React.CSSProperties['margin']
+}
+
+export type PaddingProps = {
+  [Key in
+    | 'p'
+    | 'pt'
+    | 'pb'
+    | 'pl'
+    | 'pr'
+    | 'px'
+    | 'py']?: React.CSSProperties['padding']
+}
+
+export interface MultiSelectorProps<V> {
+  value: V
+  items: readonly V[]
+  tKey?: string
+  disabled?: boolean
+  onClick?: (
+    oldValue: V,
+    newValue: V,
+  ) => (e?: React.MouseEvent<HTMLButtonElement, MouseEvent>) => void
+}

packages/types/lib/config.d.ts

@@ -107,8 +107,8 @@ export interface Config<Client extends boolean = false>
   database: {
     schemas: Schema[]
     settings: {
-      extraUserFields: string[]
-    } & BaseConfig['database']['settings']
+      extraUserFields: (ExtraField | string)[]
+    } & Omit<BaseConfig['database']['settings'], 'extraUserFields'>
   } & BaseConfig['database']
   scanner: {
     scanNext: {
@@ -130,6 +130,12 @@ export interface Config<Client extends boolean = false>
   manualAreas: ExampleConfig['manualAreas'][number][]
 }
 
+export interface ExtraField {
+  name: string
+  database: string
+  disabled: boolean
+}
+
 export interface Webhook {
   enabled: boolean
   provider: 'poracle'
@@ -163,7 +169,7 @@ export type DeepKeys<T, P extends string = ''> = {
     : never
 }[keyof T]
 
-export type ConfigPaths = DeepKeys<Config>
+export type ConfigPaths<T extends object> = DeepKeys<T>
 
 export type PathValue<T, P> = P extends `${infer K}.${infer Rest}`
   ? K extends keyof T
@@ -175,7 +181,10 @@ export type PathValue<T, P> = P extends `${infer K}.${infer Rest}`
   ? T[P]
   : never
 
-export type ConfigPathValue<P extends ConfigPaths> = PathValue<Config, P>
+export type ConfigPathValue<
+  T extends object,
+  P extends ConfigPaths<T>,
+> = PathValue<T, P>
 
 export type Join<K, P> = K extends string | number
   ? P extends string | number
@@ -223,4 +232,4 @@ export type NestedObjectPaths = Paths<Config>
 
 export type GetSafeConfig = <P extends NestedObjectPaths>(
   path: P,
-) => ConfigPathValue<P>
+) => ConfigPathValue<Config, P>

packages/types/lib/general.d.ts

@@ -26,6 +26,7 @@ export type RMGeoJSON = {
 
 import masterfile = require('packages/masterfile/lib/data/masterfile.json')
 import { Config } from './config'
+import { SliderProps } from '@mui/material'
 
 export type Masterfile = typeof masterfile
 
@@ -99,3 +100,28 @@ export type UAssetsClient = Config['icons']['styles'][number] & { data: UICONS }
 export type FullClientIcons = Omit<Config['icons'], 'styles'> & {
   styles: (Config['icons']['styles'][number] & { data: UICONS })[]
 }
+
+export interface RMSlider extends SliderProps {
+  label?: string
+  perm?: string
+  step?: number
+  i18nKey?: string
+  disabled?: boolean
+  low?: number
+  high?: number
+  i18nKey?: string
+  markI18n?: string
+  noTextInput?: boolean
+  marks?: number[]
+}
+
+export type RMSliderHandleChange<N extends string = string> = (
+  name: N,
+  values: number | number[],
+) => void
+
+export interface RMSliderProps {
+  slide: RMSlider
+  values: number[]
+  handleChange: RMSliderHandleChange
+}

packages/types/lib/scanner.d.ts

@@ -62,6 +62,8 @@ export interface Gym {
   power_up_level: number
   power_up_points: number
   power_up_end_timestamp: number
+  deleted: boolean
+  enabled: boolean
 }
 
 export type FullGym = FullModel<Gym, GymModel>

packages/types/lib/server.d.ts

@@ -21,7 +21,7 @@ import Backup = require('server/src/models/Backup')
 import Nest = require('server/src/models/Nest')
 import NestSubmission = require('server/src/models/NestSubmission')
 import Pokestop = require('server/src/models/Pokestop')
-import { ModelReturn } from './utility'
+import { ModelReturn, OnlyType } from './utility'
 import { Profile } from 'passport-discord'
 import { User } from './models'
 
@@ -235,3 +235,20 @@ export type DiscordVerifyFunction = (
   profile: Profile,
   done: VerifyCallback,
 ) => void
+
+export type BaseFilter = import('server/src/services/filters/Base')
+
+export type PokemonFilter =
+  import('server/src/services/filters/pokemon/Frontend')
+
+export type AllFilters = ReturnType<
+  typeof import('server/src/services/filters/builder/base')
+>
+
+export type Categories = keyof AllFilters
+
+export type AdvCategories = 'pokemon' | 'gyms' | 'pokestops' | 'nests'
+
+export type UIObject = ReturnType<
+  typeof import('server/src/services/ui/primary')
+>

packages/types/package.json

@@ -14,6 +14,7 @@
   "devDependencies": {
     "@apollo/client": "^3.7.15",
     "@mui/material": "^5.14.0",
+    "@mui/system": "^5.15.2",
     "@sentry/node": "^7.48.0",
     "@types/config": "^3.3.0",
     "@types/node": "^20.5.1",

server/src/graphql/server.js

@@ -81,6 +81,7 @@ async function startApollo(httpServer) {
         async requestDidStart(requestContext) {
           requestContext.contextValue.startTime = Date.now()
 
+          log.debug(requestContext.request?.variables?.filters)
           return {
             async willSendResponse(context) {
               const filterCount = Object.keys(
@@ -98,7 +99,6 @@ async function startApollo(httpServer) {
 
                 const data = response.body.singleResult.data?.[endpoint]
                 const returned = Array.isArray(data) ? data.length : 0
-
                 log.info(
                   HELPERS[endpoint] || `[${endpoint?.toUpperCase()}]`,
                   '|',

server/src/models/Gym.js

@@ -187,13 +187,16 @@ class Gym extends Model {
     )
 
     teams.forEach((team) => {
-      let slotCount = 0
-      slots.forEach((slot) => {
-        if (slot.team === team) {
-          slotCount += 1
-          finalSlots[team].push(+slot.slots)
-        }
-      })
+      const all = args.filters[`t${team}-0`]?.all
+      let slotCount = all ? baseGymSlotAmounts.length : 0
+      if (!all) {
+        slots.forEach((slot) => {
+          if (slot.team === team) {
+            slotCount += 1
+            finalSlots[team].push(+slot.slots)
+          }
+        })
+      }
       if (slotCount === baseGymSlotAmounts.length || team == 0) {
         delete finalSlots[team]
         finalTeams.push(+team)
@@ -436,12 +439,9 @@ class Gym extends Model {
       .then((r) => {
         const unique = new Set()
         r.forEach((result) => {
-          if (result.team) {
-            if (result.slots) {
-              unique.add(`g${result.team}-${result.slots}`)
-            } else {
-              unique.add(`t${result.team}-0`)
-            }
+          if (result.team !== null && result.slots !== null) {
+            unique.add(`t${result.team}-0`)
+            unique.add(`g${result.team}-${6 - result.slots}`)
           }
         })
         return [...unique]

server/src/models/Pokestop.js

@@ -870,10 +870,9 @@ class Pokestop extends Model {
             if (
               quest.quest_timestamp >= midnight &&
               (filters.onlyAllPokestops ||
-                (filters[newQuest.key] &&
-                  (filters[newQuest.key].adv
-                    ? filters[newQuest.key].adv.includes(quest.quest_title)
-                    : true)) ||
+                (filters[newQuest.key]?.adv && !filters[newQuest.key].all
+                  ? filters[newQuest.key].adv.includes(quest.quest_title)
+                  : true) ||
                 filters[`u${quest.quest_reward_type}`])
             ) {
               this.fieldAssigner(newQuest, quest, fields)

server/src/services/filters/builder/base.js

@@ -54,6 +54,7 @@ function buildDefaultFilters(perms, database) {
               : undefined,
             badge: perms.gymBadges ? 'all' : undefined,
             raidTier: perms.raids ? 'all' : undefined,
+            standard: new BaseFilter(),
             filter: {
               ...buildGyms(perms, defaultFilters.gyms),
               ...pokemon.raids,
@@ -67,6 +68,7 @@ function buildDefaultFilters(perms, database) {
             pokemon: defaultFilters.nests.pokemon,
             polygons: defaultFilters.nests.polygons,
             avgFilter: defaultFilters.nests.avgFilter,
+            standard: new BaseFilter(),
             filter: pokemon.nests,
           }
         : undefined,
@@ -93,6 +95,7 @@ function buildDefaultFilters(perms, database) {
               ? defaultFilters.pokestops.invasions
               : undefined,
             arEligible: perms.pokestops ? false : undefined,
+            standard: new BaseFilter(),
             filter: {
               ...pokemon.rocket,
               ...buildPokestops(perms, defaultFilters.pokestops),
@@ -130,6 +133,7 @@ function buildDefaultFilters(perms, database) {
               0,
               Math.ceil(database.filterContext.Route.maxDistance / 1000) + 1,
             ],
+            standard: new BaseFilter(),
             filter: {
               global: new BaseFilter(),
             },
@@ -139,6 +143,7 @@ function buildDefaultFilters(perms, database) {
       perms.portals && database.models.Portal
         ? {
             enabled: defaultFilters.portals.enabled,
+            standard: new BaseFilter(),
             filter: {
               global: new BaseFilter(),
               old: new BaseFilter(),
@@ -149,6 +154,7 @@ function buildDefaultFilters(perms, database) {
     scanAreas: perms.scanAreas
       ? {
           enabled: defaultFilters.scanAreas.enabled,
+          standard: new BaseFilter(),
           filterByAreas: false,
           filter: { areas: [], search: '' },
         }
@@ -161,27 +167,31 @@ function buildDefaultFilters(perms, database) {
             s17Cells: defaultFilters.submissionCells.s17Cells,
             s14Cells: defaultFilters.submissionCells.s14Cells,
             includeSponsored: defaultFilters.submissionCells.includeSponsored,
+            standard: new BaseFilter(),
             filter: { global: new BaseFilter() },
           }
         : undefined,
     s2cells: perms.s2cells
       ? {
           enabled: defaultFilters.s2cells.enabled,
           cells: defaultFilters.s2cells.cells,
+          standard: new BaseFilter(),
           filter: { global: new BaseFilter() },
         }
       : undefined,
     weather:
       perms.weather && database.models.Weather
         ? {
             enabled: defaultFilters.weather.enabled,
+            standard: new BaseFilter(),
             filter: { global: new BaseFilter() },
           }
         : undefined,
     spawnpoints:
       perms.spawnpoints && database.models.Spawnpoint
         ? {
             enabled: defaultFilters.spawnpoints.enabled,
+            standard: new BaseFilter(),
             tth: defaultFilters.spawnpoints.tth,
             filter: {
               global: new BaseFilter(),
@@ -194,13 +204,15 @@ function buildDefaultFilters(perms, database) {
       perms.scanCells && database.models.ScanCell
         ? {
             enabled: defaultFilters.scanCells.enabled,
+            standard: new BaseFilter(),
             filter: { global: new BaseFilter() },
           }
         : undefined,
     devices:
       perms.devices && database.models.Device
         ? {
             enabled: defaultFilters.devices.enabled,
+            standard: new BaseFilter(),
             filter: {
               online: new BaseFilter(),
               offline: new BaseFilter(),

server/src/services/filters/builder/gym.js

@@ -6,10 +6,9 @@ const BaseFilter = require('../Base')
  *
  * @param {import("@rm/types").Permissions} perms
  * @param {import("@rm/types").Config['defaultFilters']['gyms']} defaults
- * @returns
  */
 function buildGyms(perms, defaults) {
-  const gymFilters = {}
+  const gymFilters = /** @type {Record<string, BaseFilter>} */ ({})
 
   if (perms.gyms) {
     Object.keys(Event.masterfile.teams).forEach((team, i) => {