Skip to content

Commit

Permalink
This is a combination of 2 commits.
Browse files Browse the repository at this point in the history 
feat: allow FE features in browser

feat: allow `generateViewScript` and `isDropInStatements` FE feature in browser

feat: allow `generateContainerScript` FE feature in browser; eliminated circular dependencies
  • Loading branch information
@chulanovskyi-bs
chulanovskyi-bs committed Jan 21, 2025
1 parent 628f0b0 commit 19dfb23
Show file tree
Hide file tree
Showing 21 changed files with 673 additions and 640 deletions.
11 changes: 11 additions & 0 deletions api/fe.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
const { generateScript } = require('../forward_engineering/generateScript');
const { generateViewScript } = require('../forward_engineering/generateViewScript');
const { isDropInStatements } = require('../forward_engineering/isDropInStatements');
const { generateContainerScript } = require('../forward_engineering/generateContainerScript');

module.exports = {
generateScript,
generateViewScript,
generateContainerScript,
isDropInStatements,
};
1 change: 1 addition & 0 deletions esbuild.package.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const RELEASE_FOLDER_PATH = path.join(DEFAULT_RELEASE_FOLDER_PATH, `${packageDat
esbuild
.build({
entryPoints: [
path.resolve(__dirname, 'api', 'fe.js'),
path.resolve(__dirname, 'api', 're.js'),
path.resolve(__dirname, 'forward_engineering', 'api.js'),
path.resolve(__dirname, 'reverse_engineering', 'api.js'),
Expand Down
338 changes: 14 additions & 324 deletions forward_engineering/api.js
View file
Original file line number Diff line number Diff line change
@@ -1,313 +1,37 @@
'use strict';

const _ = require('lodash');
const { getDatabaseStatement, getUseCatalogStatement } = require('./helpers/databaseHelper');
const { getCleanedUrl, buildScript, isSupportUnityCatalog, getDBVersionNumber } = require('./utils/general');
const fetchRequestHelper = require('../reverse_engineering/helpers/fetchRequestHelper');
const databricksHelper = require('../reverse_engineering/helpers/databricksHelper');

const logHelper = require('../reverse_engineering/logHelper');
const {
buildEntityLevelFEScript,
buildContainerLevelFEScriptDto,
buildContainerLevelFEScript,
} = require('./helpers/feScriptBuilder');
const {
buildEntityLevelAlterScript,
buildContainerLevelAlterScript,
doesContainerLevelAlterScriptContainDropStatements,
doesEntityLevelAlterScriptContainDropStatements,
} = require('./alterScript/alterScriptBuilder');
const {
ModelDefinitions,
InternalDefinitions,
ExternalDefinitions,
ContainerJsonSchema,
ContainerStyles,
EntityJsonSchema,
} = require('./types/coreApplicationDataTypes');
const { getCleanedUrl } = require('./utils/general');
const { generateScript } = require('./generateScript');
const { generateContainerScript } = require('./generateContainerScript');
const { generateViewScript } = require('./generateViewScript');
const { isDropInStatements } = require('./isDropInStatements');
const { App, CoreData, Logger, PluginError } = require('./types/coreApplicationTypes');
const {
getSampleGenerationOptions,
parseJsonData,
generateSampleForDemonstration,
generateSamplesForEntity,
} = require('./sampleGeneration/sampleGenerationService');
const { getDataForSampleGeneration } = require('./sampleGeneration/sampleGenerationService');
const { ContainerJsonSchema, ContainerStyles, EntityJsonSchema } = require('./types/coreApplicationDataTypes');

/**
* @typedef {(error?: PluginError | null, result?: any | null) => void} PluginCallback
* @typedef {import('./sampleGeneration/sampleGenerationTypes').EntitiesData} EntitiesData
* */

/**
* @typedef {[ContainerJsonSchema, ContainerStyles]} ContainerData
* */

/**
* @typedef {{
* [id: string]: EntityJsonSchema
* }} EntitiesJsonSchema
*/

const parseEntities = (entities, serializedItems) => {
return (
entities?.reduce((result, entityId) => {
try {
return Object.assign({}, result, {
[entityId]: JSON.parse(serializedItems[entityId]),
});
} catch (e) {
return result;
}
}, {}) ?? {}
);
};

/**
* @param {CoreData} data
* @return {{
* jsonSchema: unknown,
* modelDefinitions: ModelDefinitions | unknown,
* internalDefinitions: InternalDefinitions | unknown,
* externalDefinitions: ExternalDefinitions | unknown,
* containerData: ContainerData | unknown,
* entityData: unknown,
* jsonData: ParsedJsonData
* }}
* */
const parseDataForEntityLevelScript = data => {
const jsonSchema = JSON.parse(data.jsonSchema);
const modelDefinitions = JSON.parse(data.modelDefinitions);
const internalDefinitions = JSON.parse(data.internalDefinitions);
const externalDefinitions = JSON.parse(data.externalDefinitions);
const containerData = data.containerData;
const modelData = data.modelData;
const entityData = data.entityData;
const jsonData = parseJsonData(data.jsonData);

return {
jsonSchema,
modelDefinitions,
internalDefinitions,
externalDefinitions,
containerData,
entityData,
modelData,
jsonData,
};
};

/**
* @param {CoreData} data
* @return {{
* modelDefinitions: ModelDefinitions | unknown,
* internalDefinitions: InternalDefinitions | unknown,
* externalDefinitions: ExternalDefinitions | unknown,
* containerData: ContainerData | unknown,
* entitiesJsonSchema: EntitiesJsonSchema | unknown,
* jsonData: Record<string, Object>,
* entitiesData: EntitiesData | undefined,
* }}
* */
const parseDataForContainerLevelScript = data => {
const modelData = data.modelData;
const containerData = data.containerData;
const modelDefinitions = JSON.parse(data.modelDefinitions);
const externalDefinitions = JSON.parse(data.externalDefinitions);
const entitiesJsonSchema = parseEntities(data.entities, data.jsonSchema);
const internalDefinitions = parseEntities(data.entities, data.internalDefinitions);
const relatedSchemas = parseEntities(data.relatedEntities, data.relatedSchemas);
const { jsonData, entitiesData } = getDataForSampleGeneration(data, entitiesJsonSchema);

return {
modelData,
modelDefinitions,
internalDefinitions,
externalDefinitions,
containerData,
entitiesJsonSchema,
jsonData,
entitiesData,
relatedSchemas,
};
};

/**
* @param {string} script
* @param {string} sample
* @return {Array<{ title: string, script: string, mode: string }>}
* */
const getScriptAndSampleResponse = (script, sample) => {
const mode = 'sql';
return [
{
title: 'DDL script',
script,
mode,
},
{
title: 'Sample data',
script: sample,
mode,
},
];
};

/**
* @param {CoreData} data
* @param {App} app
* @return {Promise<{
* container: string,
* entities: Array<{ name: string, script: string }>,
* views: Array<{ name: string, script: string }>,
* }>}
* */
const getContainerScriptWithSeparateBuckets = async (app, data) => {
const parsedData = parseDataForContainerLevelScript(data);
const sampleGenerationOptions = getSampleGenerationOptions(data);

const scriptData = await buildContainerLevelFEScriptDto(
data,
app,
)({
...parsedData,
includeRelationshipsInEntityScripts: true,
includeSamplesInEntityScripts: sampleGenerationOptions.isSampleGenerationRequired,
});

const useCatalogStatement = scriptData.catalog ? scriptData.catalog + '\n\n' : '';
return {
container: useCatalogStatement + scriptData.container,
entities: scriptData.entities,
views: scriptData.views,
};
};

/**
* @param {CoreData} data
* @param {App} app
* @return {Promise<string | Array<{ title: string, script: string, mode: string }>>}
* */
const getContainerScriptWithNotSeparateBuckets = async (app, data) => {
const parsedData = parseDataForContainerLevelScript(data);
const sampleGenerationOptions = getSampleGenerationOptions(data);
const scriptData = await buildContainerLevelFEScriptDto(
data,
app,
)({
...parsedData,
includeRelationshipsInEntityScripts: false,
includeSamplesInEntityScripts: false,
});
const scripts = buildContainerLevelFEScript(scriptData);
if (!sampleGenerationOptions.isSampleGenerationRequired) {
return scripts;
}

if (parsedData.jsonData) {
const demoSample = generateSampleForDemonstration(parsedData, 'container');

return getScriptAndSampleResponse(scripts, demoSample);
}

const sampleScripts = [];

for (const entityData of Object.values(parsedData.entitiesData || {})) {
const samples = await generateSamplesForEntity(entityData);
sampleScripts.push(...samples);
}

return getScriptAndSampleResponse(scripts, sampleScripts.join('\n\n'));
const logInfo = (step, connectionInfo, logger) => {
logger.clear();
logger.log('info', logHelper.getSystemInfo(connectionInfo), step);
logger.log('info', connectionInfo, 'connectionInfo', connectionInfo.hiddenKeys);
};

module.exports = {
/**
* @param {CoreData} data
* @param {Logger} logger
* @param {PluginCallback} callback
* @param {App} app
* */
generateScript(data, logger, callback, app) {
try {
const parsedData = parseDataForEntityLevelScript(data);

if (data.isUpdateScript) {
const scripts = buildEntityLevelAlterScript(data, app)(parsedData);
callback(null, scripts);
} else {
const scripts = buildEntityLevelFEScript(data, app)(parsedData);
const sampleGenerationOptions = getSampleGenerationOptions(data);
if (!sampleGenerationOptions.isSampleGenerationRequired) {
return callback(null, scripts);
}
const demoSample = generateSampleForDemonstration(parsedData, 'entity');
return callback(null, getScriptAndSampleResponse(scripts, demoSample));
}
} catch (e) {
logger.log('error', { message: e.message, stack: e.stack }, 'DeltaLake Forward-Engineering Error');
callback({ message: e.message, stack: e.stack });
}
},

/**
* @param {CoreData} data
* @param {Logger} logger
* @param {PluginCallback} callback
* @param {App} app
* */
generateViewScript(data, logger, callback, app) {
try {
const provider = require('./ddlProvider/ddlProvider')(app);
const viewSchema = JSON.parse(data.jsonSchema || '{}');
const dbVersion = data.modelData[0].dbVersion;
const isUnityCatalogSupports = isSupportUnityCatalog(dbVersion);
generateScript,

const useCatalogStatement = isUnityCatalogSupports ? getUseCatalogStatement(data.containerData) : '';
const databaseStatement = getDatabaseStatement(data.containerData, isUnityCatalogSupports, dbVersion);
generateViewScript,

const script = provider.createView({
schema: viewSchema,
viewData: data.viewData,
containerData: data.containerData,
collectionRefsDefinitionsMap: data.collectionRefsDefinitionsMap,
isKeyspaceActivated: true,
});

callback(null, buildScript([useCatalogStatement, databaseStatement, script]));
} catch (e) {
logger.log('error', { message: e.message, stack: e.stack }, 'DeltaLake Forward-Engineering Error');

callback({ message: e.message, stack: e.stack });
}
},

/**
* @param {CoreData} data
* @param {Logger} logger
* @param {PluginCallback} callback
* @param {App} app
* */
async generateContainerScript(data, logger, callback, app) {
try {
const parsedData = parseDataForContainerLevelScript(data);
if (data.isUpdateScript) {
const script = buildContainerLevelAlterScript(data, app)(parsedData);
callback(null, script);
} else {
if (data.options.separateBucket) {
const scripts = await getContainerScriptWithSeparateBuckets(app, data);
return callback(null, scripts);
}
const scripts = await getContainerScriptWithNotSeparateBuckets(app, data);
return callback(null, scripts);
}
} catch (e) {
logger.log('error', { message: e.message, stack: e.stack }, 'DeltaLake Forward-Engineering Error');
generateContainerScript,

callback({ message: e.message, stack: e.stack });
}
},
isDropInStatements,

/**
* @param {CoreData} data
Expand Down Expand Up @@ -361,38 +85,4 @@ module.exports = {
cb({ message: err.message, stack: err.stack });
}
},

/**
* @param {CoreData} data
* @param {Logger} logger
* @param {PluginCallback} callback
* @param {App} app
* */
isDropInStatements(data, logger, callback, app) {
try {
if (data.level === 'container') {
const parsedData = parseDataForContainerLevelScript(data);
const doesContainDropStatements = doesContainerLevelAlterScriptContainDropStatements(
data,
app,
)(parsedData);
callback(null, doesContainDropStatements);
} else if (data.level === 'entity') {
const parsedData = parseDataForEntityLevelScript(data);
const doesContainDropStatements = doesEntityLevelAlterScriptContainDropStatements(
data,
app,
)(parsedData);
callback(null, doesContainDropStatements);
}
} catch (e) {
callback({ message: e.message, stack: e.stack });
}
},
};

const logInfo = (step, connectionInfo, logger) => {
logger.clear();
logger.log('info', logHelper.getSystemInfo(connectionInfo), step);
logger.log('info', connectionInfo, 'connectionInfo', connectionInfo.hiddenKeys);
};
Loading

0 comments on commit 19dfb23

Please sign in to comment.