diff --git a/client/src/components/Applications/AppForm/AppForm.jsx b/client/src/components/Applications/AppForm/AppForm.jsx index e90801b4a9..9ea5e81491 100644 --- a/client/src/components/Applications/AppForm/AppForm.jsx +++ b/client/src/components/Applications/AppForm/AppForm.jsx @@ -26,8 +26,10 @@ import { getMaxMinutesValidation, getNodeCountValidation, getCoresPerNodeValidation, - getTargetPathFieldName, updateValuesForQueue, + getQueueValueForExecSystem, + getAppQueueValues, + getExecSystemFromId, } from './AppFormUtils'; import DataFilesSelectModal from '../../DataFiles/DataFilesModals/DataFilesSelectModal'; import * as ROUTES from '../../../constants/routes'; @@ -47,6 +49,13 @@ const appShape = PropTypes.shape({ }), systemNeedsKeys: PropTypes.bool, pushKeysSystem: PropTypes.shape({}), + availableExecSystems: PropTypes.arrayOf( + PropTypes.shape({ + host: PropTypes.string, + scheduler: PropTypes.string, + batchLogicalQueues: PropTypes.arrayOf(PropTypes.shape({})), + }) + ), exec_sys: PropTypes.shape({ host: PropTypes.string, scheduler: PropTypes.string, @@ -127,25 +136,46 @@ export const AppDetail = () => { }; /** - * AdjustValuesWhenQueueChanges is a component that makes uses of + * HandleDependentFieldChanges is a component that makes uses of * useFormikContext to ensure that when users switch queues, some * variables are updated to match the queue specifications (i.e. * correct node count, runtime etc) */ -const AdjustValuesWhenQueueChanges = ({ app }) => { +const HandleDependentFieldChanges = ({ app, updateFormState }) => { const [previousValues, setPreviousValues] = useState(); // Grab values and update if queue changes const { values, setValues } = useFormikContext(); React.useEffect(() => { - if ( - previousValues && - previousValues.execSystemLogicalQueue !== values.execSystemLogicalQueue - ) { - setValues(updateValuesForQueue(app, values)); + // Update exec system values first. + if (previousValues) { + let valueUpdated = false; + let updatedValues = { ...values }; + if (previousValues.execSystemId !== values.execSystemId) { + const exec_sys = getExecSystemFromId(app, values.execSystemId); + updatedValues.execSystemLogicalQueue = getQueueValueForExecSystem( + app, + exec_sys + ).name; + updatedValues = updateValuesForQueue(app, updatedValues); + valueUpdated = true; + + // Update form state, used outside the form. + updateFormState.setExecSys(exec_sys); + updateFormState.setAppQueueValues(getAppQueueValues(app, exec_sys.batchLogicalQueues)); + updateFormState.setAllocationsForExecSys(exec_sys); + } + + if ( + previousValues.execSystemLogicalQueue !== values.execSystemLogicalQueue + ) { + updatedValues = updateValuesForQueue(app, values); + valueUpdated = true; + } + if (valueUpdated) setValues(updatedValues); } setPreviousValues(values); - }, [app, values, setValues]); + }, [app, values, setValues, updateFormState]); return null; }; @@ -193,7 +223,7 @@ export const AppSchemaForm = ({ app }) => { dispatch({ type: 'GET_SYSTEM_MONITOR' }); }, [dispatch]); const { - allocations, + execSystemAllocationsMap, portalAlloc, jobSubmission, hasDefaultAllocation, @@ -203,11 +233,23 @@ export const AppSchemaForm = ({ app }) => { execSystem, defaultSystem, keyService, + execSystemsWithAllocation, } = useSelector((state) => { - const matchingExecutionHost = Object.keys(state.allocations.hosts).find( - (host) => - app.exec_sys.host === host || app.exec_sys.host.endsWith(`.${host}`) - ); + const matchingExecutionHostsMap = app.availableExecSystems.reduce((map, exec_sys) => { + const matchingExecutionHost = Object.keys(state.allocations.hosts).find( + host => + exec_sys.host === host || exec_sys.host.endsWith(`.${host}`) + ); + + if (matchingExecutionHost) { + map.set(exec_sys.id, state.allocations.hosts[matchingExecutionHost]); + } + + return map; + }, new Map()); + + const execSystemsWithAllocation = [...matchingExecutionHostsMap.keys()]; + const { defaultHost, configuration, defaultSystem } = state.systems.storage; const keyService = state.systems.storage.configuration.find( @@ -220,9 +262,7 @@ export const AppSchemaForm = ({ app }) => { defaultHost?.endsWith(s) ); return { - allocations: matchingExecutionHost - ? state.allocations.hosts[matchingExecutionHost] - : [], + execSystemAllocationsMap: matchingExecutionHostsMap, portalAlloc: state.allocations.portal_alloc, jobSubmission: state.jobs.submit, hasDefaultAllocation: @@ -240,6 +280,7 @@ export const AppSchemaForm = ({ app }) => { execSystem: state.app ? state.app.exec_sys.host : '', defaultSystem, keyService, + execSystemsWithAllocation, }; }, shallowEqual); const hideManageAccount = useSelector( @@ -263,6 +304,25 @@ export const AppSchemaForm = ({ app }) => { }; const appFields = FormSchema(app); + const [currentValues, setCurrentValues] = useState({ + execSys: app.exec_sys, + allocations: execSystemAllocationsMap.get(app.exec_sys.id)??[], + appQueueValues: getAppQueueValues(app, app.exec_sys.batchLogicalQueues), + }); + + const updateFormState = { + setExecSys: (newValue) => { + setCurrentValues(prevState => ({ ...prevState, execSys: newValue })); + }, + setAllocationsForExecSys: (execSys) => { + setCurrentValues(prevState => ({ ...prevState, allocations: execSystemAllocationsMap.get(execSys?.id)??[] })); + }, + setAppQueueValues: (newValue) => { + setCurrentValues(prevState => ({ ...prevState, appQueueValues: newValue })); + }, + + }; + // initial form values const initialValues = { @@ -284,20 +344,14 @@ export const AppSchemaForm = ({ app }) => { let missingAllocation = false; if (app.definition.jobType === 'BATCH') { - initialValues.execSystemLogicalQueue = ( - (app.definition.jobAttributes.execSystemLogicalQueue - ? app.exec_sys.batchLogicalQueues.find( - (q) => - q.name === app.definition.jobAttributes.execSystemLogicalQueue - ) - : app.exec_sys.batchLogicalQueues.find( - (q) => q.name === app.exec_sys.batchDefaultLogicalQueue - )) || app.exec_sys.batchLogicalQueues[0] + initialValues.execSystemLogicalQueue = getQueueValueForExecSystem( + app, + app.exec_sys ).name; - if (allocations.includes(portalAlloc)) { + if (currentValues.allocations.includes(portalAlloc)) { initialValues.allocation = portalAlloc; } else { - initialValues.allocation = allocations.length === 1 ? allocations[0] : ''; + initialValues.allocation = currentValues.allocations.length === 1 ? currentValues.allocations[0] : ''; } if (!hasDefaultAllocation && hasStorageSystems) { jobSubmission.error = true; @@ -307,11 +361,11 @@ export const AppSchemaForm = ({ app }) => { )} to run this application.`, }; missingAllocation = true; - } else if (!allocations.length) { + } else if (!currentValues.allocations.length) { jobSubmission.error = true; jobSubmission.response = { message: `You need an allocation on ${getSystemName( - app.exec_sys.host + currentValues.execSys.host )} to run this application.`, }; missingAllocation = true; @@ -435,8 +489,11 @@ export const AppSchemaForm = ({ app }) => { return Yup.mixed().notRequired(); } return Yup.lazy((values) => { - const queue = app.exec_sys.batchLogicalQueues.find( - (q) => q.name === values.execSystemLogicalQueue + const exec_sys = getExecSystemFromId(app, values.execSystemId); + const queue = getQueueValueForExecSystem( + app, + exec_sys, + values.execSystemLogicalQueue ); const schema = Yup.object({ parameterSet: Yup.object({ @@ -456,7 +513,7 @@ export const AppSchemaForm = ({ app }) => { .required('Required'), execSystemLogicalQueue: Yup.string() .required('Required') - .oneOf(app.exec_sys.batchLogicalQueues.map((q) => q.name)), + .oneOf(exec_sys.batchLogicalQueues.map((q) => q.name)), nodeCount: getNodeCountValidation(queue, app), coresPerNode: getCoresPerNodeValidation(queue), maxMinutes: getMaxMinutesValidation(queue).required('Required'), @@ -465,7 +522,7 @@ export const AppSchemaForm = ({ app }) => { allocation: Yup.string() .required('Required') .oneOf( - allocations, + currentValues.allocations, 'Please select an allocation from the dropdown.' ), }); @@ -602,7 +659,10 @@ export const AppSchemaForm = ({ app }) => { (app.definition.jobType === 'BATCH' && missingAllocation); return (
- + {Object.keys(appFields.fileInputs).length > 0 && (
@@ -680,7 +740,28 @@ export const AppSchemaForm = ({ app }) => {
Configuration
+ {app.availableExecSystems && + Object.keys(execSystemsWithAllocation).length > 0 && ( + + {execSystemsWithAllocation + .map((exec_system_id) => ( + + )) + .sort()} + + )} {app.definition.jobType === 'BATCH' && ( + // TODO: Add option for exec system. form field. + // + { type="select" required > - {app.exec_sys.batchLogicalQueues - /* - Hide queues for which the app default nodeCount does not meet the minimum or maximum requirements - while hideNodeCountAndCoresPerNode is true - */ - .filter( - (q) => - !app.definition.notes - .hideNodeCountAndCoresPerNode || - (app.definition.jobAttributes.nodeCount >= - q.minNodeCount && - app.definition.jobAttributes.nodeCount <= - q.maxNodeCount) - ) - .map((q) => q.name) - .sort() - .map((queueName) => - app.definition.notes.queueFilter ? ( - app.definition.notes.queueFilter.includes( - queueName - ) && ( - - ) - ) : ( - - ) - ) - .sort()} + {currentValues.appQueueValues.map((queueName) => ( + + ))} )} { - {allocations.sort().map((projectId) => ( + {currentValues.allocations.sort().map((projectId) => ( diff --git a/client/src/components/Applications/AppForm/AppFormUtils.js b/client/src/components/Applications/AppForm/AppFormUtils.js index fcaa7910ce..e5d507a6bc 100644 --- a/client/src/components/Applications/AppForm/AppFormUtils.js +++ b/client/src/components/Applications/AppForm/AppFormUtils.js @@ -3,9 +3,9 @@ import { getSystemName } from 'utils/systems'; export const TARGET_PATH_FIELD_PREFIX = '_TargetPath_'; -export const getQueueMaxMinutes = (app, queueName) => { - return app.exec_sys.batchLogicalQueues.find((q) => q.name === queueName) - .maxMinutes; +export const getQueueMaxMinutes = (exec_sys, queueName) => { + return exec_sys.batchLogicalQueues.find((q) => q.name === queueName) + ?.maxMinutes; }; /** @@ -123,8 +123,9 @@ export const getCoresPerNodeValidation = (queue) => { * @returns {Object} updated/fixed values */ export const updateValuesForQueue = (app, values) => { + const exec_sys = getExecSystemFromId(app, values.execSystemId); const updatedValues = { ...values }; - const queue = app.exec_sys.batchLogicalQueues.find( + const queue = exec_sys.batchLogicalQueues.find( (q) => q.name === values.execSystemLogicalQueue ); @@ -168,6 +169,59 @@ export const updateValuesForQueue = (app, values) => { return updatedValues; }; +/** + * Get the field name used for target path in AppForm + * + * @function + * @param {String} inputFieldName + * @returns {String} field Name prefixed with target path + */ +export const getQueueValueForExecSystem = (app, exec_sys, queue_name) => { + const queueName = + queue_name ?? + app.definition.jobAttributes.execSystemLogicalQueue ?? + exec_sys?.batchDefaultLogicalQueue; + return ( + exec_sys.batchLogicalQueues.find((q) => q.name === queueName) || + exec_sys.batchLogicalQueues[0] + ); +}; + +export const getAppQueueValues = (app, queues) => { + /* + Hide queues for which the app default nodeCount does not meet the minimum or maximum requirements + while hideNodeCountAndCoresPerNode is true + */ + return queues + .filter( + (q) => + !app.definition.notes.hideNodeCountAndCoresPerNode || + (app.definition.jobAttributes.nodeCount >= q.minNodeCount && + app.definition.jobAttributes.nodeCount <= q.maxNodeCount) + ) + .map((q) => q.name) + .filter((queueName) => + app.definition.notes.queueFilter + ? app.definition.notes.queueFilter.includes(queueName) + : true + ) + .sort(); +}; + +export const getExecSystemFromId = (app, execSystemId) => { + if ( + app.availableExecSystems && + Object.keys(app.availableExecSystems).length > 0 + ) { + return app.availableExecSystems.find( + (exec_sys) => exec_sys.id === execSystemId + ); + } + if (app.exec_sys.id === execSystemId) return app.exec_sys; + + return null; +}; + /** * Get the field name used for target path in AppForm * diff --git a/client/src/components/Applications/AppForm/fixtures/AppForm.app.fixture.js b/client/src/components/Applications/AppForm/fixtures/AppForm.app.fixture.js index 1458448ea9..c6af8fd98c 100644 --- a/client/src/components/Applications/AppForm/fixtures/AppForm.app.fixture.js +++ b/client/src/components/Applications/AppForm/fixtures/AppForm.app.fixture.js @@ -1,3 +1,5 @@ +import availableExecSystemsFixture from './fixtures/AppForm.executionsystems.fixture'; + export const helloWorldAppFixture = { definition: { sharedAppCtx: true, @@ -126,6 +128,7 @@ export const helloWorldAppFixture = { created: '2022-12-12T23:40:16.158241Z', updated: '2023-02-14T21:45:08.818354Z', }, + availableExecSystems: availableExecSystemsFixture, exec_sys: { isPublic: true, isDynamicEffectiveUser: true, diff --git a/client/src/redux/reducers/apps.reducers.js b/client/src/redux/reducers/apps.reducers.js index ddcc332256..4a2ba0d332 100644 --- a/client/src/redux/reducers/apps.reducers.js +++ b/client/src/redux/reducers/apps.reducers.js @@ -65,6 +65,7 @@ export const initialAppState = { loading: false, systemNeedsKeys: false, pushKeysSystem: {}, + availableExecSystems: {}, exec_sys: {}, license: {}, appListing: [], @@ -77,6 +78,7 @@ export function app(state = initialAppState, action) { definition: action.payload.definition, systemNeedsKeys: action.payload.systemNeedsKeys, pushKeysSystem: action.payload.pushKeysSystem, + availableExecSystems: action.payload.availableExecSystems, exec_sys: action.payload.exec_sys, license: action.payload.license, appListing: action.payload.appListing, @@ -90,6 +92,7 @@ export function app(state = initialAppState, action) { definition: {}, systemNeedsKeys: false, pushKeysSystem: {}, + availableExecSystems: {}, exec_sys: {}, license: {}, appListing: [], diff --git a/client/src/redux/sagas/fixtures/executionsystems.fixture.js b/client/src/redux/sagas/fixtures/executionsystems.fixture.js new file mode 100644 index 0000000000..620e8da1dd --- /dev/null +++ b/client/src/redux/sagas/fixtures/executionsystems.fixture.js @@ -0,0 +1,160 @@ +export const availableExecSystemsFixture = [ + { + isPublic: true, + isDynamicEffectiveUser: true, + tenant: 'portals', + id: 'frontera', + description: 'System for running jobs on the Frontera HPC system.', + systemType: 'LINUX', + owner: 'wma_prtl', + host: 'frontera.tacc.utexas.edu', + enabled: true, + effectiveUserId: 'sal', + defaultAuthnMethod: 'PKI_KEYS', + authnCredential: null, + bucketName: null, + rootDir: '/', + port: 22, + useProxy: false, + proxyHost: null, + proxyPort: -1, + dtnSystemId: null, + dtnMountPoint: null, + dtnMountSourcePath: null, + isDtn: false, + canExec: true, + canRunBatch: true, + enableCmdPrefix: false, + mpiCmd: null, + jobRuntimes: [ + { + runtimeType: 'SINGULARITY', + version: null, + }, + ], + jobWorkingDir: 'HOST_EVAL($SCRATCH)/tapis/${JobUUID}', + jobEnvVariables: [], + jobMaxJobs: 2147483647, + jobMaxJobsPerUser: 2147483647, + batchScheduler: 'SLURM', + batchLogicalQueues: [ + { + name: 'normal', + hpcQueueName: 'normal', + maxJobs: -1, + maxJobsPerUser: 100, + minNodeCount: 3, + maxNodeCount: 512, + minCoresPerNode: 1, + maxCoresPerNode: 56, + minMemoryMB: 1, + maxMemoryMB: 192000, + minMinutes: 1, + maxMinutes: 2880, + }, + { + name: 'development', + hpcQueueName: 'development', + maxJobs: -1, + maxJobsPerUser: 1, + minNodeCount: 1, + maxNodeCount: 40, + minCoresPerNode: 1, + maxCoresPerNode: 56, + minMemoryMB: 1, + maxMemoryMB: 192000, + minMinutes: 1, + maxMinutes: 120, + }, + ], + batchDefaultLogicalQueue: 'normal', + batchSchedulerProfile: 'tacc', + jobCapabilities: [], + tags: [], + notes: {}, + importRefId: null, + uuid: 'ca524f9e-6574-4a40-a075-0429c807d6c8', + deleted: false, + created: '2022-12-06T22:52:06.300829Z', + updated: '2022-12-06T22:52:06.300829Z', + }, + { + isPublic: true, + isDynamicEffectiveUser: true, + tenant: 'portals', + id: 'ls6', + description: 'System for running jobs on the Frontera HPC system.', + systemType: 'LINUX', + owner: 'wma_prtl', + host: 'ls6.tacc.utexas.edu', + enabled: true, + effectiveUserId: 'sal', + defaultAuthnMethod: 'PKI_KEYS', + authnCredential: null, + bucketName: null, + rootDir: '/', + port: 22, + useProxy: false, + proxyHost: null, + proxyPort: -1, + dtnSystemId: null, + dtnMountPoint: null, + dtnMountSourcePath: null, + isDtn: false, + canExec: true, + canRunBatch: true, + enableCmdPrefix: false, + mpiCmd: null, + jobRuntimes: [ + { + runtimeType: 'SINGULARITY', + version: null, + }, + ], + jobWorkingDir: 'HOST_EVAL($SCRATCH)/tapis/${JobUUID}', + jobEnvVariables: [], + jobMaxJobs: 2147483647, + jobMaxJobsPerUser: 2147483647, + batchScheduler: 'SLURM', + batchLogicalQueues: [ + { + name: 'normal', + hpcQueueName: 'normal', + maxJobs: -1, + maxJobsPerUser: 100, + minNodeCount: 3, + maxNodeCount: 512, + minCoresPerNode: 1, + maxCoresPerNode: 56, + minMemoryMB: 1, + maxMemoryMB: 192000, + minMinutes: 1, + maxMinutes: 2880, + }, + { + name: 'development', + hpcQueueName: 'development', + maxJobs: -1, + maxJobsPerUser: 1, + minNodeCount: 1, + maxNodeCount: 40, + minCoresPerNode: 1, + maxCoresPerNode: 56, + minMemoryMB: 1, + maxMemoryMB: 192000, + minMinutes: 1, + maxMinutes: 120, + }, + ], + batchDefaultLogicalQueue: 'normal', + batchSchedulerProfile: 'tacc', + jobCapabilities: [], + tags: [], + notes: {}, + importRefId: null, + uuid: 'ca524f9e-6574-4a40-a075-0429c807d6c8', + deleted: false, + created: '2022-12-06T22:52:06.300829Z', + updated: '2022-12-06T22:52:06.300829Z', + }, +]; diff --git a/client/src/redux/sagas/fixtures/extract.fixture.js b/client/src/redux/sagas/fixtures/extract.fixture.js index a008862b6a..c03221f9fb 100644 --- a/client/src/redux/sagas/fixtures/extract.fixture.js +++ b/client/src/redux/sagas/fixtures/extract.fixture.js @@ -87,6 +87,7 @@ export const extractApp = { created: '2023-03-08T22:38:29.409242Z', updated: '2023-03-16T22:23:22.822513Z', }, + availableExecSystems: availableExecSystemsFixture, exec_sys: { isPublic: true, isDynamicEffectiveUser: true, diff --git a/server/portal/apps/workspace/api/views.py b/server/portal/apps/workspace/api/views.py index bec3f5b357..7324ee21e6 100644 --- a/server/portal/apps/workspace/api/views.py +++ b/server/portal/apps/workspace/api/views.py @@ -43,6 +43,16 @@ def _get_user_app_license(license_type, user): lic = license_model.objects.filter(user=user).first() return lic +# List of all exec systems available. +# Number of systems available today is low, so info control on +# results is not needed at this time. +def _get_all_exec_systems(user): + tapis = user.tapis_oauth.client + # Intentionally not using search='(canExec.eq.true)'), because + # is it really slow and make the UI look unresponsive. + return tapis.systems.getSystems(listType='ALL', select='allAttributes', search='(canExec.eq.true)') + + def _get_app(app_id, app_version, user): tapis = user.tapis_oauth.client @@ -52,8 +62,13 @@ def _get_app(app_id, app_version, user): app_def = tapis.apps.getAppLatestVersion(appId=app_id) data = {'definition': app_def} - # GET EXECUTION SYSTEM INFO TO PROCESS SPECIFIC SYSTEM DATA E.G. QUEUE INFORMATION - data['exec_sys'] = tapis.systems.getSystem(systemId=app_def.jobAttributes.execSystemId) + has_dynamic_exec_system = getattr(app_def.notes, 'dynamicExecSystem', False) + if (has_dynamic_exec_system): + data['availableExecSystems'] = _get_all_exec_systems(user) + data['exec_sys'] = next((exec_sys for exec_sys in data['availableExecSystems'] if exec_sys.id == app_def.jobAttributes.execSystemId), data['availableExecSystems'][0] if data['availableExecSystems'] else None) + else: + # GET EXECUTION SYSTEM INFO TO PROCESS SPECIFIC SYSTEM DATA E.G. QUEUE INFORMATION + data['exec_sys'] = tapis.systems.getSystem(systemId=app_def.jobAttributes.execSystemId) lic_type = _app_license_type(app_def) data['license'] = {