Skip to content

Commit

Permalink
feat(reports): add avg-rps to reports (#292)
Browse files Browse the repository at this point in the history 
* feat(reports): add avg-rps to reports
  • Loading branch information
@OrFrenkelZooz
OrFrenkelZooz authored Apr 16, 2020
1 parent 33a4c5c commit fa61a39
Show file tree
Hide file tree
Showing 7 changed files with 174 additions and 20 deletions.
3 changes: 3 additions & 0 deletions docs/devguide/docs/swagger-docs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1825,6 +1825,9 @@ components:
last_stats:
type: string
description: The current report metrics.
avg_rps:
type: number
description: The average rps.
notes:
type: string
description: notes about the test
Expand Down
3 changes: 3 additions & 0 deletions docs/openapi3.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2101,6 +2101,9 @@ components:
last_stats:
type: string
description: The current report metrics.
avg_rps:
type: number
description: The average rps.
notes:
type: string
description: notes about the test
Expand Down
8 changes: 6 additions & 2 deletions src/reports/models/reportsManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,18 +72,21 @@ module.exports.postReport = async (testId, reportBody) => {
};

function getReportResponse(summaryRow, config) {
let timeEndOrCurrent = summaryRow.end_time || new Date();
let lastUpdateTime = summaryRow.end_time || summaryRow.last_updated_at;

let testConfiguration = summaryRow.test_configuration ? JSON.parse(summaryRow.test_configuration) : {};
const reportDurationSeconds = (new Date(lastUpdateTime).getTime() - new Date(summaryRow.start_time).getTime()) / 1000;

let rps = 0;
let totalRequests = 0;
let completedRequests = 0;
let successRequests = 0;

summaryRow.subscribers.forEach((subscriber) => {
if (subscriber.last_stats && subscriber.last_stats.rps && subscriber.last_stats.codes) {
completedRequests += subscriber.last_stats.requestsCompleted;
rps += subscriber.last_stats.rps.mean;
totalRequests += subscriber.last_stats.rps.total_count || 0;
Object.keys(subscriber.last_stats.codes).forEach(key => {
if (key[0] === '2') {
successRequests += subscriber.last_stats.codes[key];
Expand All @@ -104,7 +107,7 @@ function getReportResponse(summaryRow, config) {
start_time: summaryRow.start_time,
end_time: summaryRow.end_time || undefined,
phase: summaryRow.phase,
duration_seconds: (new Date(timeEndOrCurrent).getTime() - new Date(summaryRow.start_time).getTime()) / 1000,
duration_seconds: reportDurationSeconds,
arrival_rate: testConfiguration.arrival_rate,
duration: testConfiguration.duration,
ramp_to: testConfiguration.ramp_to,
Expand All @@ -115,6 +118,7 @@ function getReportResponse(summaryRow, config) {
environment: testConfiguration.environment,
subscribers: summaryRow.subscribers,
last_rps: rps,
avg_rps: Number((totalRequests / reportDurationSeconds).toFixed(2)),
last_success_rate: successRate,
score: summaryRow.score ? summaryRow.score : undefined,
benchmark_weights_data: summaryRow.benchmark_weights_data ? JSON.parse(summaryRow.benchmark_weights_data) : undefined
Expand Down
18 changes: 15 additions & 3 deletions src/reports/models/statsManager.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
const uuid = require('uuid');
const _ = require('lodash');
const databaseConnector = require('./databaseConnector'),
notifier = require('./notifier'),
reportsManager = require('./reportsManager'),
Expand All @@ -14,10 +15,10 @@ module.exports.postStats = async (report, stats) => {
const statsParsed = JSON.parse(stats.data);
const statsTime = statsParsed.timestamp;

if (stats.phase_status === constants.SUBSCRIBER_DONE_STAGE) {
if (stats.phase_status === constants.SUBSCRIBER_DONE_STAGE || stats.phase_status === constants.SUBSCRIBER_ABORTED_STAGE) {
await databaseConnector.updateSubscriber(report.test_id, report.report_id, stats.runner_id, stats.phase_status);
} else {
await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, stats.data);
await updateSubscriberWithStatsInternal(report, stats);
}

if (stats.phase_status === constants.SUBSCRIBER_INTERMEDIATE_STAGE || stats.phase_status === constants.SUBSCRIBER_FIRST_INTERMEDIATE_STAGE) {
Expand All @@ -31,6 +32,17 @@ module.exports.postStats = async (report, stats) => {
return stats;
};

async function updateSubscriberWithStatsInternal(report, stats) {
const parseData = JSON.parse(stats.data);
const subscriber = report.subscribers.find(subscriber => subscriber.runner_id === stats.runner_id);
const { last_stats } = subscriber;
if (last_stats && parseData.rps) {
const lastTotalCount = _.get(last_stats, 'rps.total_count', 0);
parseData.rps.total_count = lastTotalCount + parseData.rps.count;
}
await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, JSON.stringify(parseData));
}

async function updateReportBenchmarkIfNeeded(report) {
if (!reportUtil.isAllRunnersInExpectedPhase(report, constants.SUBSCRIBER_DONE_STAGE)) {
return;
Expand Down Expand Up @@ -58,4 +70,4 @@ async function extractBenchmark(testId) {
} catch (e) {
return undefined;
}
}
}
18 changes: 9 additions & 9 deletions tests/integration-tests/reports/helpers/statsGenerator.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict';

module.exports.generateStats = (phaseStatus, runnerId) => {
module.exports.generateStats = (phaseStatus, runnerId, statsTime, rpsCount) => {
let stats;
switch (phaseStatus) {
case 'error':
Expand All @@ -9,13 +9,13 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
runner_id: runnerId,
phase_status: 'error',
stats_time: Date.now().toString(),
data: JSON.stringify({ timestamp: Date.now(), message: error.message}),
data: JSON.stringify({ timestamp: statsTime || Date.now(), message: error.message }),
error
};
break;
case 'started_phase':
const startedPhaseInfo = {
'timestamp': Date.now(),
'timestamp': statsTime || Date.now(),
'duration': 120,
'arrivalRate': 500,
'mode': 'uniform',
Expand All @@ -31,7 +31,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
break;
case 'intermediate':
const intermediatePhaseInfo = {
'timestamp': Date.now(),
'timestamp': statsTime || Date.now(),
'scenariosCreated': 101,
'scenariosCompleted': 101,
'requestsCompleted': 101,
Expand All @@ -43,7 +43,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
'p99': 1059
},
'rps': {
'count': 101,
'count': rpsCount || 101,
'mean': 90.99
},
'scenarioDuration': {
Expand Down Expand Up @@ -76,7 +76,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
break;
case 'done':
const donePhaseInfo = {
'timestamp': Date.now(),
'timestamp': statsTime || Date.now(),
'scenariosCreated': 150,
'scenariosCompleted': 150,
'requestsCompleted': 150,
Expand All @@ -88,7 +88,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
'p99': 1057.6
},
'rps': {
'count': 150,
'count': rpsCount || 150,
'mean': 0.14
},
'scenarioDuration': {
Expand Down Expand Up @@ -121,7 +121,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
break;
case 'aborted':
const abortedPhaseInfo = {
'timestamp': Date.now()
'timestamp': statsTime || Date.now()
};
stats = {
runner_id: runnerId,
Expand All @@ -135,4 +135,4 @@ module.exports.generateStats = (phaseStatus, runnerId) => {
}

return stats;
};
};
39 changes: 35 additions & 4 deletions tests/integration-tests/reports/reportsApi-test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ describe('Integration tests for the reports api', function() {

lastReports.forEach((report) => {
const REPORT_KEYS = ['test_id', 'test_name', 'revision_id', 'report_id', 'job_id', 'test_type', 'start_time',
'phase', 'status'];
'phase', 'status', 'avg_rps'];

REPORT_KEYS.forEach((key) => {
should(report).hasOwnProperty(key);
Expand Down Expand Up @@ -404,6 +404,36 @@ describe('Integration tests for the reports api', function() {
validateFinishedReport(report);
});

it('Post full cycle stats and verify report rps avg', async function () {
const phaseStartedStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('started_phase', runnerId));
should(phaseStartedStatsResponse.statusCode).be.eql(204);

const getReport = await reportsRequestCreator.getReport(testId, reportId);
should(getReport.statusCode).be.eql(200);
const testStartTime = new Date(getReport.body.start_time);
const statDateFirst = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 20);
let intermediateStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('intermediate', runnerId, statDateFirst, 600));
should(intermediateStatsResponse.statusCode).be.eql(204);
let getReportResponse = await reportsRequestCreator.getReport(testId, reportId);
let report = getReportResponse.body;
should(report.avg_rps).eql(30);

const statDateSecond = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 40);
intermediateStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('intermediate', runnerId, statDateSecond, 200));
should(intermediateStatsResponse.statusCode).be.eql(204);
getReportResponse = await reportsRequestCreator.getReport(testId, reportId);
report = getReportResponse.body;
should(report.avg_rps).eql(20);

const statDateThird = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 60);
const doneStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('done', runnerId, statDateThird));
should(doneStatsResponse.statusCode).be.eql(204);
getReportResponse = await reportsRequestCreator.getReport(testId, reportId);
should(getReportResponse.statusCode).be.eql(200);
report = getReportResponse.body;
should(report.avg_rps).eql(13.33);
});

it('Post only "done" phase stats', async function () {
const doneStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('done', runnerId));
should(doneStatsResponse.statusCode).be.eql(204);
Expand Down Expand Up @@ -527,6 +557,7 @@ describe('Integration tests for the reports api', function() {
getReportResponse = await reportsRequestCreator.getReport(testId, reportId);
report = getReportResponse.body;
should(report.status).eql('aborted');
validateFinishedReport(report,undefined,'aborted');
});
});
});
Expand Down Expand Up @@ -854,15 +885,15 @@ describe('Integration tests for the reports api', function() {
});
});

function validateFinishedReport(report, expectedValues = {}) {
function validateFinishedReport(report, expectedValues = {},status) {
const REPORT_KEYS = ['test_id', 'test_name', 'revision_id', 'report_id', 'job_id', 'test_type', 'start_time',
'end_time', 'phase', 'last_updated_at', 'status'];

REPORT_KEYS.forEach((key) => {
should(report).hasOwnProperty(key);
});

should(report.status).eql('finished');
status = status || 'finished';
should(report.status).eql(status);
should(report.test_id).eql(testId);
should(report.report_id).eql(reportId);
should(report.phase).eql('0');
Expand Down
105 changes: 103 additions & 2 deletions tests/unit-tests/reporter/models/reportsManager-test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,45 @@ describe('Reports manager tests', function () {
should.exist(reports);
reports.length.should.eql(0);
});

it('get last report with avg rsp when test running', async () => {
const now = new Date();
const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10);
const subscriber = { last_stats: { rps: { total_count: 200 }, codes: { '200': 10 } } };
const report = Object.assign({}, REPORT, { last_updated_at: now, start_time: tenSecBefore, subscribers: [subscriber] });
databaseGetLastReportsStub.resolves([report]);
const reports = await manager.getLastReports();
reports.length.should.eql(1);
should(reports[0].avg_rps).eql(20);
});
it('get last report with avg rsp when test finished', async () => {
const now = new Date();
const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10);
const subscriber = { last_stats: { rps: { total_count: 300 }, codes: { '200': 10 } } };
const report = Object.assign({}, REPORT, {
end_time: now,
start_time: tenSecBefore,
subscribers: [subscriber]
});
databaseGetLastReportsStub.resolves([report]);
const reports = await manager.getLastReports();
reports.length.should.eql(1);
should(reports[0].avg_rps).eql(30);
});
it('get last report with avg rsp when total_count not exist ', async () => {
const now = new Date();
const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10);
const subscriber = { last_stats: { rps: { test: 'test' }, codes: { '200': 10 } } };
const report = Object.assign({}, REPORT, {
end_time: now,
start_time: tenSecBefore,
subscribers: [subscriber]
});
databaseGetLastReportsStub.resolves([report]);
const reports = await manager.getLastReports();
reports.length.should.eql(1);
should(reports[0].avg_rps).eql(0);
});
});

describe('Create new report', function () {
Expand Down Expand Up @@ -429,9 +468,9 @@ describe('Reports manager tests', function () {
databasePostStatsStub.resolves();
getJobStub.resolves(JOB);
notifierStub.resolves();
const stats = { phase_status: 'intermediate', data: JSON.stringify({ median: 4 }) };
const stats = { phase_status: 'intermediate', data: JSON.stringify({ median: 4 }), runner_id: 123 };

const statsResponse = await statsManager.postStats('test_id', stats);
const statsResponse = await statsManager.postStats({ subscribers: [{ runner_id: 123 }] }, stats);

databaseUpdateSubscriberStub.callCount.should.eql(0);
databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1);
Expand All @@ -440,6 +479,52 @@ describe('Reports manager tests', function () {
statsResponse.should.eql(stats);
});

it('Stats intermediate and verify update subscriber with total_count in first time', async () => {
configStub.resolves({});
databaseGetReportStub.resolves([REPORT]);
databasePostStatsStub.resolves();
getJobStub.resolves(JOB);
notifierStub.resolves();
const stats = {
phase_status: 'intermediate',
data: JSON.stringify({ rps: { count: 10 } }),
runner_id: 123
};
const statsResponse = await statsManager.postStats({ subscribers: [{ runner_id: 123, last_stats: {} }] }, stats);

databaseUpdateSubscriberStub.callCount.should.eql(0);
databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1);
const data = JSON.parse(databaseUpdateSubscriberWithStatsStub.args[0][4]);
should(data.rps.total_count).eql(10);
should.exist(statsResponse);
statsResponse.should.eql(stats);
});
it('Stats intermediate and verify update subscriber second time with total_count', async () => {
configStub.resolves({});
databaseGetReportStub.resolves([REPORT]);
databasePostStatsStub.resolves();
getJobStub.resolves(JOB);
notifierStub.resolves();
const stats = {
phase_status: 'intermediate',
data: JSON.stringify({ rps: { count: 10 } }),
runner_id: 123
};
const statsResponse = await statsManager.postStats({
subscribers: [{
runner_id: 123,
last_stats: { rps: { total_count: 18 } }
}]
}, stats);

databaseUpdateSubscriberStub.callCount.should.eql(0);
databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1);
const data = JSON.parse(databaseUpdateSubscriberWithStatsStub.args[0][4]);
should(data.rps.total_count).eql(28);
should.exist(statsResponse);
statsResponse.should.eql(stats);
});

it('Stats consumer handles message with status done', async () => {
configStub.resolves({});
databaseGetReportStub.resolves([REPORT]);
Expand All @@ -456,6 +541,22 @@ describe('Reports manager tests', function () {
should.exist(statsResponse);
statsResponse.should.eql(stats);
});
it('Stats consumer handles message with status aborted', async () => {
configStub.resolves({});
databaseGetReportStub.resolves([REPORT]);
databasePostStatsStub.resolves();
getJobStub.resolves(JOB);
notifierStub.resolves();
const stats = { phase_status: 'aborted', data: JSON.stringify({ median: 4 }) };

const statsResponse = await statsManager.postStats('test_id', stats);

databaseUpdateSubscriberStub.callCount.should.eql(1);
databaseUpdateSubscriberWithStatsStub.callCount.should.eql(0);

should.exist(statsResponse);
statsResponse.should.eql(stats);
});

it('when report done and have benchmark data ', async () => {
databaseGetReportStub.resolves([REPORT_DONE]);
Expand Down

0 comments on commit fa61a39

Please sign in to comment.