Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/hck 5595 replace ssh tunnel #20

Merged
merged 3 commits into from
Apr 30, 2024
Merged

Fix/hck 5595 replace ssh tunnel #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d5c36c5
Replace ssh-tunnel with internal sshService
VitaliiBedletskyi Apr 30, 2024
d0f564c
delete ssh-tunnel npm deps
VitaliiBedletskyi Apr 30, 2024
58a939e
delete commented code
VitaliiBedletskyi Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 27 additions & 20 deletions reverse_engineering/api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,17 @@ BigInt.prototype.toJSON = function () {
const ACCESS_DENIED_ERROR = 1045;

module.exports = {
async connect(connectionInfo) {

const connection = await connectionHelper.connect(connectionInfo);
async connect(connectionInfo, sshService) {
const connection = await connectionHelper.connect(connectionInfo, sshService);

return connection;
},

disconnect(connectionInfo, logger, callback, app) {
connectionHelper.close();

const sshService = app.require('@hackolade/ssh-service');

connectionHelper.close(sshService);

callback();
},

Expand All @@ -34,7 +35,9 @@ module.exports = {
logger.clear();
logger.log('info', connectionInfo, 'connectionInfo', connectionInfo.hiddenKeys);

const connection = await this.connect(connectionInfo);
const sshService = app.require('@hackolade/ssh-service');

const connection = await this.connect(connectionInfo, sshService);
const instance = connectionHelper.createInstance(connection, logger);

await instance.ping();
Expand Down Expand Up @@ -64,10 +67,12 @@ module.exports = {
logger.log('info', connectionInfo, 'connectionInfo', connectionInfo.hiddenKeys);
const systemDatabases = connectionInfo.includeSystemCollection ? [] : ['information_schema', 'mysql', 'performance_schema'];

const connection = await this.connect(connectionInfo);
const sshService = app.require('@hackolade/ssh-service');

const connection = await this.connect(connectionInfo, sshService);
const instance = connectionHelper.createInstance(connection, logger);
const databases = connectionInfo.databaseName ? [connectionInfo.databaseName] : await instance.getDatabases(systemDatabases);

const collections = await databases.reduce(async (next, dbName) => {
const result = await next;
try {
Expand Down Expand Up @@ -113,18 +118,20 @@ module.exports = {
try {
logger.log('info', data, 'data', data.hiddenKeys);

const sshService = app.require('@hackolade/ssh-service');

const collections = data.collectionData.collections;
const dataBaseNames = data.collectionData.dataBaseNames;
const connection = await this.connect(data);
const connection = await this.connect(data, sshService);
const instance = await connectionHelper.createInstance(connection, logger);
const dbVersion = await instance.serverVersion();

log.info('MySQL version: ' + dbVersion);
log.progress('Start reverse engineering ...');
log.progress('Start reverse engineering ...');
const isVersion8 = getMajorVersionNumber(dbVersion) >= 8;

let tablespaces = {};

if (isVersion8) {
tablespaces = mysqlHelper.getTablespaces({
innoDb: await instance.getInnoDBTablespaces(),
Expand All @@ -135,16 +142,16 @@ module.exports = {
const result = await async.mapSeries(dataBaseNames, async (dbName) => {
const tables = (collections[dbName] || []).filter(name => !isViewName(name));
const views = (collections[dbName] || []).filter(isViewName).map(getViewName);

log.info(`Parsing database "${dbName}"`);
log.progress(`Parsing database "${dbName}"`, dbName);
log.progress(`Parsing database "${dbName}"`, dbName);

const containerData = mysqlHelper.parseDatabaseStatement(
await instance.describeDatabase(dbName)
);

log.info(`Parsing functions`);
log.progress(`Parsing functions`, dbName);
log.progress(`Parsing functions`, dbName);

const UDFs = mysqlHelper.parseFunctions(
await instance.getFunctions(dbName), log
Expand All @@ -164,15 +171,15 @@ module.exports = {

const columns = await instance.getColumns(dbName, tableName);
let records = [];

if (containsJson(columns)) {
log.info(`Sampling table "${tableName}"`);
log.progress(`Sampling table`, dbName, tableName);

const count = await instance.getCount(dbName, tableName);
records = await instance.getRecords(dbName, tableName, getSampleDocSize(count, data.recordSamplingSettings));
}

log.info(`Get create table statement "${tableName}"`);
log.progress(`Get create table statement`, dbName, tableName);

Expand Down Expand Up @@ -213,7 +220,7 @@ module.exports = {
},
};
});

const viewData = await async.mapSeries(views, async (viewName) => {
log.info(`Getting data from view "${viewName}"`);
log.progress(`Getting data from view`, dbName, viewName);
Expand All @@ -239,7 +246,7 @@ module.exports = {
},
];
}

return result;
});

Expand Down
98 changes: 36 additions & 62 deletions reverse_engineering/helpers/connectionHelper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,48 +1,8 @@
const mysql2 = require('mysql2/promise');
const fs = require('fs');
const ssh = require('tunnel-ssh');

let connection;
let sshTunnel;

const getSshConfig = (info) => {
const config = {
username: info.ssh_user,
host: info.ssh_host,
port: info.ssh_port,
dstHost: info.host,
dstPort: info.port,
localHost: '127.0.0.1',
localPort: info.port,
keepAlive: true
};

if (info.ssh_method === 'privateKey') {
return Object.assign({}, config, {
privateKey: fs.readFileSync(info.ssh_key_file),
passphrase: info.ssh_key_passphrase
});
} else {
return Object.assign({}, config, {
password: info.ssh_password
});
}
};

const connectViaSsh = (info) => new Promise((resolve, reject) => {
ssh(getSshConfig(info), (err, tunnel) => {
if (err) {
reject(err);
} else {
resolve({
tunnel,
info: Object.assign({}, info, {
host: '127.0.0.1',
})
});
}
});
});
let useSshTunnel;

const getSslOptions = (connectionInfo) => {
if (connectionInfo.sslType === 'Off') {
Expand Down Expand Up @@ -70,17 +30,31 @@ const getSslOptions = (connectionInfo) => {
}
};

const createConnection = async (connectionInfo) => {
const createConnection = async (connectionInfo, sshService) => {
if (connectionInfo.ssh) {
const { info, tunnel } = await connectViaSsh(connectionInfo);
sshTunnel = tunnel;
connectionInfo = info;
useSshTunnel = true;
const { options } = await sshService.openTunnel({
sshAuthMethod: connectionInfo.ssh_method === 'privateKey' ? 'IDENTITY_FILE' : 'USER_PASSWORD',
sshTunnelHostname: connectionInfo.ssh_host,
sshTunnelPort: connectionInfo.ssh_port,
sshTunnelUsername: connectionInfo.ssh_user,
sshTunnelPassword: connectionInfo.ssh_password,
sshTunnelIdentityFile: connectionInfo.ssh_key_file,
sshTunnelPassphrase: connectionInfo.ssh_key_passphrase,
host: connectionInfo.host,
port: connectionInfo.port,
});
connectionInfo = {
...connectionInfo,
host: options.host,
port: options.port.toString(),
};
}

return await mysql2.createConnection({
return await mysql2.createConnection({
host: connectionInfo.host,
user: connectionInfo.userName,
password: connectionInfo.userPassword,
user: connectionInfo.userName,
password: connectionInfo.userPassword,
port: connectionInfo.port,
metaAsArray: false,
ssl: getSslOptions(connectionInfo),
Expand All @@ -92,12 +66,12 @@ const createConnection = async (connectionInfo) => {
});
};

const connect = async (connectionInfo) => {
const connect = async (connectionInfo, sshService) => {
if (connection) {
return connection;
}
connection = await createConnection(connectionInfo);

connection = await createConnection(connectionInfo, sshService);

return connection;
};
Expand All @@ -109,13 +83,13 @@ const createInstance = (connection, logger) => {

const getDatabases = async (systemDatabases) => {
const databases = await query('show databases;');

return databases.map(item => item.Database).filter(dbName => !systemDatabases.includes(dbName));
};

const getTables = async (dbName) => {
const tables = await query(`show full tables from \`${dbName}\`;`);

return tables;
};

Expand All @@ -124,7 +98,7 @@ const createInstance = (connection, logger) => {

return Number(count[0]?.count || 0);
};

const getRecords = async (dbName, tableName, limit) => {
const result = await query({
sql: `SELECT * FROM \`${dbName}\`.\`${tableName}\` LIMIT ${limit};`
Expand All @@ -143,7 +117,7 @@ const createInstance = (connection, logger) => {
const data = await query(`show create database \`${dbName}\`;`);

return data[0]['Create Database'];
};
};

const getFunctions = async (dbName) => {
const functions = await query(`show function status WHERE Db = '${dbName}'`);
Expand Down Expand Up @@ -180,7 +154,7 @@ const createInstance = (connection, logger) => {
const getConstraints = async (dbName, tableName) => {
try {
const result = await query(`select * from information_schema.TABLE_CONSTRAINTS where CONSTRAINT_SCHEMA='${dbName}' AND TABLE_NAME='${tableName}';`);

return result;
} catch (error) {
logger.log('error', {
Expand Down Expand Up @@ -220,7 +194,7 @@ const createInstance = (connection, logger) => {

const serverVersion = async () => {
const result = await query('select VERSION() as version;');

return result[0]?.version || '';
};

Expand Down Expand Up @@ -275,15 +249,15 @@ const createInstance = (connection, logger) => {
};
};

const close = () => {
const close = async (sshService) => {
if (connection) {
connection.end();
connection = null;
}

if (sshTunnel) {
sshTunnel.close();
sshTunnel = null;
if (useSshTunnel) {
useSshTunnel = false;
await sshService.closeConsumer();
}
};

Expand Down
Loading