task/TV3-82: v3 Push Keys (#739)

* add more sys onboarding logs * remove non key service systems from onboarding * implement listType when getting apps * tapis v3 push keys methods * add serializer to job submit response * fix test * enable archiving * fix merge error * change systemHasKeys to systemNeedsKeys * logging and adjustments to push keys methods * reenable empty tab filtering * task/TV3-82--perms: Set system permissions during onboarding (#749) * Add permission util and listing/mkdir operations * use user JWT for pushing credentials * linting Co-authored-by: Sal Tijerina <[email protected]> * update settings custom onboarding * include execSystemId in job request; handle version in app id * fix selected app highlighting * conditionally render Inputs section * Update server/portal/apps/workspace/api/views.py Co-authored-by: Nathan Franklin <[email protected]> * fix job archive macros * fix spacing * include both tacc token prompts * turn off tacc token field autocomplete * skip credential step if system is functional * handle system permissions and system credentials in onboarding step * app form fixes; default sys for archiving * remove logs * remove logs * fix linting * remove redundant comment * task/TV3-82: Cleanup unused/redundant files and methods (#740) * remove unused files and methods * fix test * fix tests * fix tests * fix tests * fix tests Co-authored-by: Jake Rosenberg <[email protected]> Co-authored-by: Nathan Franklin <[email protected]>
TACC · Jan 5, 2023 · 6bfc782 · 6bfc782
1 parent 76df297
commit 6bfc782
Show file tree

Hide file tree

Showing 52 changed files with 619 additions and 1,749 deletions.
diff --git a/client/src/components/Applications/AppBrowser/AppBrowser.jsx b/client/src/components/Applications/AppBrowser/AppBrowser.jsx
@@ -1,7 +1,11 @@
 import React, { useState } from 'react';
-import { NavLink as RRNavLink, useRouteMatch } from 'react-router-dom';
+import {
+  NavLink as RRNavLink,
+  useRouteMatch,
+  useLocation,
+} from 'react-router-dom';
+import queryString from 'query-string';
 import { useSelector, shallowEqual } from 'react-redux';
-// import PropTypes from 'prop-types';
 import { Nav, NavItem, NavLink, TabContent, TabPane } from 'reactstrap';
 import { AppIcon, Icon, Message } from '_common';
 import './AppBrowser.scss';
@@ -14,6 +18,8 @@ const findAppTab = (categoryDict, appId) => {
 };
 
 const AppBrowser = () => {
+  const location = useLocation();
+  const { appVersion } = queryString.parse(location.search);
   const { params } = useRouteMatch();
   const [activeTab, setActiveTab] = useState();
 
@@ -80,6 +86,10 @@ const AppBrowser = () => {
                       (app.version ? `?appVersion=${app.version}` : '')
                     }
                     activeClassName="active"
+                    isActive={() =>
+                      `${params.appId}-${appVersion}` ===
+                      `${app.appId}-${app.version}`
+                    }
                   >
                     <span className="nav-content">
                       <AppIcon appId={app.appId} />

diff --git a/client/src/components/Applications/AppForm/AppForm.jsx b/client/src/components/Applications/AppForm/AppForm.jsx
@@ -42,7 +42,7 @@ const appShape = PropTypes.shape({
     maxMinutes: PropTypes.number,
     tags: PropTypes.arrayOf(PropTypes.string),
   }),
-  systemHasKeys: PropTypes.bool,
+  systemNeedsKeys: PropTypes.bool,
   pushKeysSystem: PropTypes.shape({}),
   exec_sys: PropTypes.shape({
     host: PropTypes.string,
@@ -186,17 +186,21 @@ export const AppSchemaForm = ({ app }) => {
     hasStorageSystems,
     downSystems,
     execSystem,
+    defaultSystem,
   } = useSelector((state) => {
     const matchingExecutionHost = Object.keys(state.allocations.hosts).find(
       (host) =>
         app.exec_sys.host === host || app.exec_sys.host.endsWith(`.${host}`)
     );
-    const { defaultHost, configuration } = state.systems.storage;
+    const { defaultHost, configuration, defaultSystem } = state.systems.storage;
+
     const hasCorral =
       configuration.length &&
-      ['cloud.corral.tacc.utexas.edu', 'data.tacc.utexas.edu'].some((s) =>
-        defaultHost.endsWith(s)
-      );
+      [
+        'cloud.corral.tacc.utexas.edu',
+        'data.tacc.utexas.edu',
+        'cloud.data.tacc.utexas.edu',
+      ].some((s) => defaultHost.endsWith(s));
     return {
       allocations: matchingExecutionHost
         ? state.allocations.hosts[matchingExecutionHost]
@@ -216,14 +220,14 @@ export const AppSchemaForm = ({ app }) => {
             .map((downSys) => downSys.hostname)
         : [],
       execSystem: state.app ? state.app.exec_sys.host : '',
+      defaultSystem,
     };
   }, shallowEqual);
-
   const hideManageAccount = useSelector(
     (state) => state.workbench.config.hideManageAccount
   );
 
-  const { systemHasKeys, pushKeysSystem } = app;
+  const { systemNeedsKeys, pushKeysSystem } = app;
 
   const missingLicense = app.license.type && !app.license.enabled;
   const pushKeys = (e) => {
@@ -258,9 +262,12 @@ export const AppSchemaForm = ({ app }) => {
     nodeCount: app.definition.jobAttributes.nodeCount,
     coresPerNode: app.definition.jobAttributes.coresPerNode,
     maxMinutes: app.definition.jobAttributes.maxMinutes,
-    archiveSystemDir: '',
+    archiveSystemId:
+      defaultSystem || app.definition.jobAttributes.archiveSystemId,
+    archiveSystemDir: app.definition.jobAttributes.archiveSystemDir,
     archiveOnAppError: true,
     appId: app.definition.id,
+    execSystemId: app.definition.jobAttributes.execSystemId,
   };
   let missingAllocation = false;
   if (app.exec_sys.batchScheduler === 'SLURM') {
@@ -293,11 +300,11 @@ export const AppSchemaForm = ({ app }) => {
     <div id="appForm-wrapper">
       {/* The !! is needed because the second value of this shorthand
           is interpreted as a literal 0 if not. */}
-      {!!(!systemHasKeys && hasStorageSystems) && (
+      {!!(systemNeedsKeys && hasStorageSystems) && (
         <div className="appDetail-error">
           <SectionMessage type="warning">
             There was a problem accessing your default My Data file system. If
-            this is your first time logging in, you may need to &nbsp;
+            this is your first time logging in, you may need to&nbsp;
             <a
               className="data-files-nav-link"
               type="button"
@@ -404,6 +411,7 @@ export const AppSchemaForm = ({ app }) => {
               nodeCount: getNodeCountValidation(queue, app),
               coresPerNode: getCoresPerNodeValidation(queue),
               maxMinutes: getMaxMinutesValidation(queue).required('Required'),
+              archiveSystemId: Yup.string(),
               archiveSystemDir: Yup.string(),
               allocation: Yup.string()
                 .required('Required')
@@ -497,51 +505,57 @@ export const AppSchemaForm = ({ app }) => {
           return (
             <Form>
               <AdjustValuesWhenQueueChanges app={app} />
-              <FormGroup tag="fieldset" disabled={readOnly || !systemHasKeys}>
-                <div className="appSchema-section">
-                  <div className="appSchema-header">
-                    <span>Inputs</span>
+              <FormGroup tag="fieldset" disabled={readOnly || systemNeedsKeys}>
+                {Object.keys(appFields.fileInputs).length > 0 && (
+                  <div className="appSchema-section">
+                    <div className="appSchema-header">
+                      <span>Inputs</span>
+                    </div>
+                    {Object.entries(appFields.fileInputs).map(
+                      ([name, field]) => {
+                        // TODOv3 handle fileInputArrays https://jira.tacc.utexas.edu/browse/TV3-8
+                        return (
+                          <FormField
+                            {...field}
+                            name={`fileInputs.${name}`}
+                            agaveFile
+                            SelectModal={DataFilesSelectModal}
+                            placeholder="Browse Data Files"
+                            key={`fileInputs.${name}`}
+                          />
+                        );
+                      }
+                    )}
+                    {Object.entries(appFields.appArgs).map(([name, field]) => {
+                      return (
+                        <FormField
+                          {...field}
+                          name={`appArgs.${name}`}
+                          key={`appArgs.${name}`}
+                        >
+                          {field.options
+                            ? field.options.map((item) => {
+                                let val = item;
+                                if (val instanceof String) {
+                                  const tmp = {};
+                                  tmp[val] = val;
+                                  val = tmp;
+                                }
+                                return Object.entries(val).map(
+                                  ([key, value]) => (
+                                    <option key={key} value={key}>
+                                      {value}
+                                    </option>
+                                  )
+                                );
+                              })
+                            : null}
+                        </FormField>
+                      );
+                    })}
+                    {/* TODOv3 handle parameterSet.envVariables */}
                   </div>
-                  {Object.entries(appFields.fileInputs).map(([name, field]) => {
-                    // TODOv3 handle fileInputArrays https://jira.tacc.utexas.edu/browse/TV3-8
-                    return (
-                      <FormField
-                        {...field}
-                        name={`fileInputs.${name}`}
-                        agaveFile
-                        SelectModal={DataFilesSelectModal}
-                        placeholder="Browse Data Files"
-                        key={`fileInputs.${name}`}
-                      />
-                    );
-                  })}
-                  {Object.entries(appFields.appArgs).map(([name, field]) => {
-                    return (
-                      <FormField
-                        {...field}
-                        name={`appArgs.${name}`}
-                        key={`appArgs.${name}`}
-                      >
-                        {field.options
-                          ? field.options.map((item) => {
-                              let val = item;
-                              if (val instanceof String) {
-                                const tmp = {};
-                                tmp[val] = val;
-                                val = tmp;
-                              }
-                              return Object.entries(val).map(([key, value]) => (
-                                <option key={key} value={key}>
-                                  {value}
-                                </option>
-                              ));
-                            })
-                          : null}
-                      </FormField>
-                    );
-                  })}
-                  {/* TODOv3 handle parameterSet.envVariables */}
-                </div>
+                )}
                 <div className="appSchema-section">
                   <div className="appSchema-header">
                     <span>Configuration</span>
@@ -631,15 +645,22 @@ export const AppSchemaForm = ({ app }) => {
                     required
                   />
                   {!app.definition.notes.isInteractive ? (
-                    <FormField
-                      label="Output Location"
-                      description={parse(
-                        'Specify a location where the job output should be archived. By default, job output will be archived at: <code>archive/jobs/${YYYY-MM-DD}/${JOB_NAME}-${JOB_ID}</code>.' // eslint-disable-line no-template-curly-in-string
-                      )}
-                      name="archivePath"
-                      type="text"
-                      placeholder="archive/jobs/${YYYY-MM-DD}/${JOB_NAME}-${JOB_ID}" // eslint-disable-line no-template-curly-in-string
-                    />
+                    <>
+                      <FormField
+                        label="Archive System"
+                        description="System into which output files are archived after application execution."
+                        name="archiveSystemId"
+                        type="text"
+                        placeholder={defaultSystem}
+                      />
+                      <FormField
+                        label="Archive Directory"
+                        description="Directory into which output files are archived after application execution."
+                        name="archiveSystemDir"
+                        type="text"
+                        placeholder="HOST_EVAL($WORK)/tapis-jobs-archive/${JobCreateDate}/${JobName}-${JobUUID}" // TODOv3: How do we know if portal supports HOME vs WORK?
+                      />
+                    </>
                   ) : null}
                 </div>
                 <Button

diff --git a/client/src/components/Applications/AppForm/AppForm.test.js b/client/src/components/Applications/AppForm/AppForm.test.js
@@ -211,6 +211,7 @@ describe.skip('AppSchemaForm', () => {
           errorMessage: null,
           loading: false,
           defaultHost: '',
+          defaultSystem: '',
         },
         definitions: {
           list: [],

diff --git a/client/src/components/Applications/AppForm/AppFormSchema.js b/client/src/components/Applications/AppForm/AppFormSchema.js
@@ -75,7 +75,7 @@ const FormSchema = (app) => {
     const field = {
       label: input.name,
       description: input.description,
-      required: input.inputMode === 'REQUIRED ',
+      required: input.inputMode === 'REQUIRED',
     };
 
     field.type = 'text';
@@ -91,8 +91,8 @@ const FormSchema = (app) => {
     */
 
     if (field.required) {
-      appFields.schema.inputs[input.name] =
-        appFields.schema.inputs[input.name].required('Required');
+      appFields.schema.fileInputs[input.name] =
+        appFields.schema.fileInputs[input.name].required('Required');
     }
 
     appFields.fileInputs[input.name] = field;

diff --git a/client/src/components/Applications/AppLayout/AppLayout.jsx b/client/src/components/Applications/AppLayout/AppLayout.jsx
@@ -44,7 +44,7 @@ const AppsHeader = (categoryDict) => {
   const appMeta = Object.values(categoryDict.categoryDict)
     .flatMap((e) => e)
     .find((app) => app.appId === params.appId);
-  const path = appMeta ? ` / ${appMeta.label}` : '';
+  const path = appMeta ? ` / ${appMeta.label || appMeta.appId}` : '';
   return `Applications ${path}`;
 };
 

diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx
@@ -5,7 +5,6 @@ import DataFilesMoveModal from './DataFilesMoveModal';
 import DataFilesUploadModal from './DataFilesUploadModal';
 import DataFilesMkdirModal from './DataFilesMkdirModal';
 import DataFilesRenameModal from './DataFilesRenameModal';
-import DataFilesPushKeysModal from './DataFilesPushKeysModal';
 import DataFilesCopyModal from './DataFilesCopyModal';
 import DataFilesEmptyModal from './DataFilesEmptyModal';
 import DataFilesCompressModal from './DataFilesCompressModal';
@@ -28,7 +27,6 @@ export default function DataFilesModals() {
       <DataFilesUploadModal layout="default" />
       <DataFilesMkdirModal />
       <DataFilesRenameModal />
-      <DataFilesPushKeysModal />
       <DataFilesEmptyModal />
       <DataFilesLinkModal />
       <DataFilesShowPathModal />