Skip to content

Commit

Permalink
Fix/hck 5595 replace ssh tunnel (#20)
Browse files Browse the repository at this point in the history 
* Replace ssh-tunnel with internal sshService

* delete ssh-tunnel npm deps

* delete commented code
  • Loading branch information
@VitaliiBedletskyi
VitaliiBedletskyi authored Apr 30, 2024
1 parent 2309445 commit c388568
Show file tree
Hide file tree
Showing 238 changed files with 80 additions and 42,575 deletions.
47 changes: 27 additions & 20 deletions reverse_engineering/api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,17 @@ BigInt.prototype.toJSON = function () {
const ACCESS_DENIED_ERROR = 1045;

module.exports = {
async connect(connectionInfo) {

const connection = await connectionHelper.connect(connectionInfo);
async connect(connectionInfo, sshService) {
const connection = await connectionHelper.connect(connectionInfo, sshService);

return connection;
},

disconnect(connectionInfo, logger, callback, app) {
connectionHelper.close();

const sshService = app.require('@hackolade/ssh-service');

connectionHelper.close(sshService);

callback();
},

Expand All @@ -34,7 +35,9 @@ module.exports = {
logger.clear();
logger.log('info', connectionInfo, 'connectionInfo', connectionInfo.hiddenKeys);

const connection = await this.connect(connectionInfo);
const sshService = app.require('@hackolade/ssh-service');

const connection = await this.connect(connectionInfo, sshService);
const instance = connectionHelper.createInstance(connection, logger);

await instance.ping();
Expand Down Expand Up @@ -64,10 +67,12 @@ module.exports = {
logger.log('info', connectionInfo, 'connectionInfo', connectionInfo.hiddenKeys);
const systemDatabases = connectionInfo.includeSystemCollection ? [] : ['information_schema', 'mysql', 'performance_schema'];

const connection = await this.connect(connectionInfo);
const sshService = app.require('@hackolade/ssh-service');

const connection = await this.connect(connectionInfo, sshService);
const instance = connectionHelper.createInstance(connection, logger);
const databases = connectionInfo.databaseName ? [connectionInfo.databaseName] : await instance.getDatabases(systemDatabases);

const collections = await databases.reduce(async (next, dbName) => {
const result = await next;
try {
Expand Down Expand Up @@ -113,18 +118,20 @@ module.exports = {
try {
logger.log('info', data, 'data', data.hiddenKeys);

const sshService = app.require('@hackolade/ssh-service');

const collections = data.collectionData.collections;
const dataBaseNames = data.collectionData.dataBaseNames;
const connection = await this.connect(data);
const connection = await this.connect(data, sshService);
const instance = await connectionHelper.createInstance(connection, logger);
const dbVersion = await instance.serverVersion();

log.info('MySQL version: ' + dbVersion);
log.progress('Start reverse engineering ...');
log.progress('Start reverse engineering ...');
const isVersion8 = getMajorVersionNumber(dbVersion) >= 8;

let tablespaces = {};

if (isVersion8) {
tablespaces = mysqlHelper.getTablespaces({
innoDb: await instance.getInnoDBTablespaces(),
Expand All @@ -135,16 +142,16 @@ module.exports = {
const result = await async.mapSeries(dataBaseNames, async (dbName) => {
const tables = (collections[dbName] || []).filter(name => !isViewName(name));
const views = (collections[dbName] || []).filter(isViewName).map(getViewName);

log.info(`Parsing database "${dbName}"`);
log.progress(`Parsing database "${dbName}"`, dbName);
log.progress(`Parsing database "${dbName}"`, dbName);

const containerData = mysqlHelper.parseDatabaseStatement(
await instance.describeDatabase(dbName)
);

log.info(`Parsing functions`);
log.progress(`Parsing functions`, dbName);
log.progress(`Parsing functions`, dbName);

const UDFs = mysqlHelper.parseFunctions(
await instance.getFunctions(dbName), log
Expand All @@ -164,15 +171,15 @@ module.exports = {

const columns = await instance.getColumns(dbName, tableName);
let records = [];

if (containsJson(columns)) {
log.info(`Sampling table "${tableName}"`);
log.progress(`Sampling table`, dbName, tableName);

const count = await instance.getCount(dbName, tableName);
records = await instance.getRecords(dbName, tableName, getSampleDocSize(count, data.recordSamplingSettings));
}

log.info(`Get create table statement "${tableName}"`);
log.progress(`Get create table statement`, dbName, tableName);

Expand Down Expand Up @@ -213,7 +220,7 @@ module.exports = {
},
};
});

const viewData = await async.mapSeries(views, async (viewName) => {
log.info(`Getting data from view "${viewName}"`);
log.progress(`Getting data from view`, dbName, viewName);
Expand All @@ -239,7 +246,7 @@ module.exports = {
},
];
}

return result;
});

Expand Down
98 changes: 36 additions & 62 deletions reverse_engineering/helpers/connectionHelper.js
View file
Original file line number Diff line number Diff line change
@@ -1,48 +1,8 @@
const mysql2 = require('mysql2/promise');
const fs = require('fs');
const ssh = require('tunnel-ssh');

let connection;
let sshTunnel;

const getSshConfig = (info) => {
const config = {
username: info.ssh_user,
host: info.ssh_host,
port: info.ssh_port,
dstHost: info.host,
dstPort: info.port,
localHost: '127.0.0.1',
localPort: info.port,
keepAlive: true
};

if (info.ssh_method === 'privateKey') {
return Object.assign({}, config, {
privateKey: fs.readFileSync(info.ssh_key_file),
passphrase: info.ssh_key_passphrase
});
} else {
return Object.assign({}, config, {
password: info.ssh_password
});
}
};

const connectViaSsh = (info) => new Promise((resolve, reject) => {
ssh(getSshConfig(info), (err, tunnel) => {
if (err) {
reject(err);
} else {
resolve({
tunnel,
info: Object.assign({}, info, {
host: '127.0.0.1',
})
});
}
});
});
let useSshTunnel;

const getSslOptions = (connectionInfo) => {
if (connectionInfo.sslType === 'Off') {
Expand Down Expand Up @@ -70,17 +30,31 @@ const getSslOptions = (connectionInfo) => {
}
};

const createConnection = async (connectionInfo) => {
const createConnection = async (connectionInfo, sshService) => {
if (connectionInfo.ssh) {
const { info, tunnel } = await connectViaSsh(connectionInfo);
sshTunnel = tunnel;
connectionInfo = info;
useSshTunnel = true;
const { options } = await sshService.openTunnel({
sshAuthMethod: connectionInfo.ssh_method === 'privateKey' ? 'IDENTITY_FILE' : 'USER_PASSWORD',
sshTunnelHostname: connectionInfo.ssh_host,
sshTunnelPort: connectionInfo.ssh_port,
sshTunnelUsername: connectionInfo.ssh_user,
sshTunnelPassword: connectionInfo.ssh_password,
sshTunnelIdentityFile: connectionInfo.ssh_key_file,
sshTunnelPassphrase: connectionInfo.ssh_key_passphrase,
host: connectionInfo.host,
port: connectionInfo.port,
});
connectionInfo = {
...connectionInfo,
host: options.host,
port: options.port.toString(),
};
}

return await mysql2.createConnection({
return await mysql2.createConnection({
host: connectionInfo.host,
user: connectionInfo.userName,
password: connectionInfo.userPassword,
user: connectionInfo.userName,
password: connectionInfo.userPassword,
port: connectionInfo.port,
metaAsArray: false,
ssl: getSslOptions(connectionInfo),
Expand All @@ -92,12 +66,12 @@ const createConnection = async (connectionInfo) => {
});
};

const connect = async (connectionInfo) => {
const connect = async (connectionInfo, sshService) => {
if (connection) {
return connection;
}
connection = await createConnection(connectionInfo);

connection = await createConnection(connectionInfo, sshService);

return connection;
};
Expand All @@ -109,13 +83,13 @@ const createInstance = (connection, logger) => {

const getDatabases = async (systemDatabases) => {
const databases = await query('show databases;');

return databases.map(item => item.Database).filter(dbName => !systemDatabases.includes(dbName));
};

const getTables = async (dbName) => {
const tables = await query(`show full tables from \`${dbName}\`;`);

return tables;
};

Expand All @@ -124,7 +98,7 @@ const createInstance = (connection, logger) => {

return Number(count[0]?.count || 0);
};

const getRecords = async (dbName, tableName, limit) => {
const result = await query({
sql: `SELECT * FROM \`${dbName}\`.\`${tableName}\` LIMIT ${limit};`
Expand All @@ -143,7 +117,7 @@ const createInstance = (connection, logger) => {
const data = await query(`show create database \`${dbName}\`;`);

return data[0]['Create Database'];
};
};

const getFunctions = async (dbName) => {
const functions = await query(`show function status WHERE Db = '${dbName}'`);
Expand Down Expand Up @@ -180,7 +154,7 @@ const createInstance = (connection, logger) => {
const getConstraints = async (dbName, tableName) => {
try {
const result = await query(`select * from information_schema.TABLE_CONSTRAINTS where CONSTRAINT_SCHEMA='${dbName}' AND TABLE_NAME='${tableName}';`);

return result;
} catch (error) {
logger.log('error', {
Expand Down Expand Up @@ -220,7 +194,7 @@ const createInstance = (connection, logger) => {

const serverVersion = async () => {
const result = await query('select VERSION() as version;');

return result[0]?.version || '';
};

Expand Down Expand Up @@ -275,15 +249,15 @@ const createInstance = (connection, logger) => {
};
};

const close = () => {
const close = async (sshService) => {
if (connection) {
connection.end();
connection = null;
}

if (sshTunnel) {
sshTunnel.close();
sshTunnel = null;
if (useSshTunnel) {
useSshTunnel = false;
await sshService.closeConsumer();
}
};

Expand Down
Loading

0 comments on commit c388568

Please sign in to comment.