diff --git a/docs/devguide/docs/swagger-docs.yaml b/docs/devguide/docs/swagger-docs.yaml index b62676d86..2047461bd 100644 --- a/docs/devguide/docs/swagger-docs.yaml +++ b/docs/devguide/docs/swagger-docs.yaml @@ -1825,6 +1825,9 @@ components: last_stats: type: string description: The current report metrics. + avg_rps: + type: number + description: The average rps. notes: type: string description: notes about the test diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index c36252305..8e13017da 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -2101,6 +2101,9 @@ components: last_stats: type: string description: The current report metrics. + avg_rps: + type: number + description: The average rps. notes: type: string description: notes about the test diff --git a/src/reports/models/reportsManager.js b/src/reports/models/reportsManager.js index 9fb5836fc..e6fb88e27 100644 --- a/src/reports/models/reportsManager.js +++ b/src/reports/models/reportsManager.js @@ -72,11 +72,13 @@ module.exports.postReport = async (testId, reportBody) => { }; function getReportResponse(summaryRow, config) { - let timeEndOrCurrent = summaryRow.end_time || new Date(); + let lastUpdateTime = summaryRow.end_time || summaryRow.last_updated_at; let testConfiguration = summaryRow.test_configuration ? JSON.parse(summaryRow.test_configuration) : {}; + const reportDurationSeconds = (new Date(lastUpdateTime).getTime() - new Date(summaryRow.start_time).getTime()) / 1000; let rps = 0; + let totalRequests = 0; let completedRequests = 0; let successRequests = 0; @@ -84,6 +86,7 @@ function getReportResponse(summaryRow, config) { if (subscriber.last_stats && subscriber.last_stats.rps && subscriber.last_stats.codes) { completedRequests += subscriber.last_stats.requestsCompleted; rps += subscriber.last_stats.rps.mean; + totalRequests += subscriber.last_stats.rps.total_count || 0; Object.keys(subscriber.last_stats.codes).forEach(key => { if (key[0] === '2') { successRequests += subscriber.last_stats.codes[key]; @@ -104,7 +107,7 @@ function getReportResponse(summaryRow, config) { start_time: summaryRow.start_time, end_time: summaryRow.end_time || undefined, phase: summaryRow.phase, - duration_seconds: (new Date(timeEndOrCurrent).getTime() - new Date(summaryRow.start_time).getTime()) / 1000, + duration_seconds: reportDurationSeconds, arrival_rate: testConfiguration.arrival_rate, duration: testConfiguration.duration, ramp_to: testConfiguration.ramp_to, @@ -115,6 +118,7 @@ function getReportResponse(summaryRow, config) { environment: testConfiguration.environment, subscribers: summaryRow.subscribers, last_rps: rps, + avg_rps: Number((totalRequests / reportDurationSeconds).toFixed(2)), last_success_rate: successRate, score: summaryRow.score ? summaryRow.score : undefined, benchmark_weights_data: summaryRow.benchmark_weights_data ? JSON.parse(summaryRow.benchmark_weights_data) : undefined diff --git a/src/reports/models/statsManager.js b/src/reports/models/statsManager.js index 05d46a90d..5149c5e63 100644 --- a/src/reports/models/statsManager.js +++ b/src/reports/models/statsManager.js @@ -1,4 +1,5 @@ const uuid = require('uuid'); +const _ = require('lodash'); const databaseConnector = require('./databaseConnector'), notifier = require('./notifier'), reportsManager = require('./reportsManager'), @@ -14,10 +15,10 @@ module.exports.postStats = async (report, stats) => { const statsParsed = JSON.parse(stats.data); const statsTime = statsParsed.timestamp; - if (stats.phase_status === constants.SUBSCRIBER_DONE_STAGE) { + if (stats.phase_status === constants.SUBSCRIBER_DONE_STAGE || stats.phase_status === constants.SUBSCRIBER_ABORTED_STAGE) { await databaseConnector.updateSubscriber(report.test_id, report.report_id, stats.runner_id, stats.phase_status); } else { - await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, stats.data); + await updateSubscriberWithStatsInternal(report, stats); } if (stats.phase_status === constants.SUBSCRIBER_INTERMEDIATE_STAGE || stats.phase_status === constants.SUBSCRIBER_FIRST_INTERMEDIATE_STAGE) { @@ -31,6 +32,17 @@ module.exports.postStats = async (report, stats) => { return stats; }; +async function updateSubscriberWithStatsInternal(report, stats) { + const parseData = JSON.parse(stats.data); + const subscriber = report.subscribers.find(subscriber => subscriber.runner_id === stats.runner_id); + const { last_stats } = subscriber; + if (last_stats && parseData.rps) { + const lastTotalCount = _.get(last_stats, 'rps.total_count', 0); + parseData.rps.total_count = lastTotalCount + parseData.rps.count; + } + await databaseConnector.updateSubscriberWithStats(report.test_id, report.report_id, stats.runner_id, stats.phase_status, JSON.stringify(parseData)); +} + async function updateReportBenchmarkIfNeeded(report) { if (!reportUtil.isAllRunnersInExpectedPhase(report, constants.SUBSCRIBER_DONE_STAGE)) { return; @@ -58,4 +70,4 @@ async function extractBenchmark(testId) { } catch (e) { return undefined; } -} \ No newline at end of file +} diff --git a/tests/integration-tests/reports/helpers/statsGenerator.js b/tests/integration-tests/reports/helpers/statsGenerator.js index 5c3dfd6ae..bbae6f45c 100644 --- a/tests/integration-tests/reports/helpers/statsGenerator.js +++ b/tests/integration-tests/reports/helpers/statsGenerator.js @@ -1,6 +1,6 @@ 'use strict'; -module.exports.generateStats = (phaseStatus, runnerId) => { +module.exports.generateStats = (phaseStatus, runnerId, statsTime, rpsCount) => { let stats; switch (phaseStatus) { case 'error': @@ -9,13 +9,13 @@ module.exports.generateStats = (phaseStatus, runnerId) => { runner_id: runnerId, phase_status: 'error', stats_time: Date.now().toString(), - data: JSON.stringify({ timestamp: Date.now(), message: error.message}), + data: JSON.stringify({ timestamp: statsTime || Date.now(), message: error.message }), error }; break; case 'started_phase': const startedPhaseInfo = { - 'timestamp': Date.now(), + 'timestamp': statsTime || Date.now(), 'duration': 120, 'arrivalRate': 500, 'mode': 'uniform', @@ -31,7 +31,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => { break; case 'intermediate': const intermediatePhaseInfo = { - 'timestamp': Date.now(), + 'timestamp': statsTime || Date.now(), 'scenariosCreated': 101, 'scenariosCompleted': 101, 'requestsCompleted': 101, @@ -43,7 +43,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => { 'p99': 1059 }, 'rps': { - 'count': 101, + 'count': rpsCount || 101, 'mean': 90.99 }, 'scenarioDuration': { @@ -76,7 +76,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => { break; case 'done': const donePhaseInfo = { - 'timestamp': Date.now(), + 'timestamp': statsTime || Date.now(), 'scenariosCreated': 150, 'scenariosCompleted': 150, 'requestsCompleted': 150, @@ -88,7 +88,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => { 'p99': 1057.6 }, 'rps': { - 'count': 150, + 'count': rpsCount || 150, 'mean': 0.14 }, 'scenarioDuration': { @@ -121,7 +121,7 @@ module.exports.generateStats = (phaseStatus, runnerId) => { break; case 'aborted': const abortedPhaseInfo = { - 'timestamp': Date.now() + 'timestamp': statsTime || Date.now() }; stats = { runner_id: runnerId, @@ -135,4 +135,4 @@ module.exports.generateStats = (phaseStatus, runnerId) => { } return stats; -}; \ No newline at end of file +}; diff --git a/tests/integration-tests/reports/reportsApi-test.js b/tests/integration-tests/reports/reportsApi-test.js index 015cee034..8edca462d 100644 --- a/tests/integration-tests/reports/reportsApi-test.js +++ b/tests/integration-tests/reports/reportsApi-test.js @@ -291,7 +291,7 @@ describe('Integration tests for the reports api', function() { lastReports.forEach((report) => { const REPORT_KEYS = ['test_id', 'test_name', 'revision_id', 'report_id', 'job_id', 'test_type', 'start_time', - 'phase', 'status']; + 'phase', 'status', 'avg_rps']; REPORT_KEYS.forEach((key) => { should(report).hasOwnProperty(key); @@ -404,6 +404,36 @@ describe('Integration tests for the reports api', function() { validateFinishedReport(report); }); + it('Post full cycle stats and verify report rps avg', async function () { + const phaseStartedStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('started_phase', runnerId)); + should(phaseStartedStatsResponse.statusCode).be.eql(204); + + const getReport = await reportsRequestCreator.getReport(testId, reportId); + should(getReport.statusCode).be.eql(200); + const testStartTime = new Date(getReport.body.start_time); + const statDateFirst = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 20); + let intermediateStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('intermediate', runnerId, statDateFirst, 600)); + should(intermediateStatsResponse.statusCode).be.eql(204); + let getReportResponse = await reportsRequestCreator.getReport(testId, reportId); + let report = getReportResponse.body; + should(report.avg_rps).eql(30); + + const statDateSecond = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 40); + intermediateStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('intermediate', runnerId, statDateSecond, 200)); + should(intermediateStatsResponse.statusCode).be.eql(204); + getReportResponse = await reportsRequestCreator.getReport(testId, reportId); + report = getReportResponse.body; + should(report.avg_rps).eql(20); + + const statDateThird = new Date(testStartTime).setSeconds(testStartTime.getSeconds() + 60); + const doneStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('done', runnerId, statDateThird)); + should(doneStatsResponse.statusCode).be.eql(204); + getReportResponse = await reportsRequestCreator.getReport(testId, reportId); + should(getReportResponse.statusCode).be.eql(200); + report = getReportResponse.body; + should(report.avg_rps).eql(13.33); + }); + it('Post only "done" phase stats', async function () { const doneStatsResponse = await reportsRequestCreator.postStats(testId, reportId, statsGenerator.generateStats('done', runnerId)); should(doneStatsResponse.statusCode).be.eql(204); @@ -527,6 +557,7 @@ describe('Integration tests for the reports api', function() { getReportResponse = await reportsRequestCreator.getReport(testId, reportId); report = getReportResponse.body; should(report.status).eql('aborted'); + validateFinishedReport(report,undefined,'aborted'); }); }); }); @@ -854,15 +885,15 @@ describe('Integration tests for the reports api', function() { }); }); -function validateFinishedReport(report, expectedValues = {}) { +function validateFinishedReport(report, expectedValues = {},status) { const REPORT_KEYS = ['test_id', 'test_name', 'revision_id', 'report_id', 'job_id', 'test_type', 'start_time', 'end_time', 'phase', 'last_updated_at', 'status']; REPORT_KEYS.forEach((key) => { should(report).hasOwnProperty(key); }); - - should(report.status).eql('finished'); + status = status || 'finished'; + should(report.status).eql(status); should(report.test_id).eql(testId); should(report.report_id).eql(reportId); should(report.phase).eql('0'); diff --git a/tests/unit-tests/reporter/models/reportsManager-test.js b/tests/unit-tests/reporter/models/reportsManager-test.js index f79a449f1..9077ef9e7 100644 --- a/tests/unit-tests/reporter/models/reportsManager-test.js +++ b/tests/unit-tests/reporter/models/reportsManager-test.js @@ -379,6 +379,45 @@ describe('Reports manager tests', function () { should.exist(reports); reports.length.should.eql(0); }); + + it('get last report with avg rsp when test running', async () => { + const now = new Date(); + const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10); + const subscriber = { last_stats: { rps: { total_count: 200 }, codes: { '200': 10 } } }; + const report = Object.assign({}, REPORT, { last_updated_at: now, start_time: tenSecBefore, subscribers: [subscriber] }); + databaseGetLastReportsStub.resolves([report]); + const reports = await manager.getLastReports(); + reports.length.should.eql(1); + should(reports[0].avg_rps).eql(20); + }); + it('get last report with avg rsp when test finished', async () => { + const now = new Date(); + const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10); + const subscriber = { last_stats: { rps: { total_count: 300 }, codes: { '200': 10 } } }; + const report = Object.assign({}, REPORT, { + end_time: now, + start_time: tenSecBefore, + subscribers: [subscriber] + }); + databaseGetLastReportsStub.resolves([report]); + const reports = await manager.getLastReports(); + reports.length.should.eql(1); + should(reports[0].avg_rps).eql(30); + }); + it('get last report with avg rsp when total_count not exist ', async () => { + const now = new Date(); + const tenSecBefore = new Date(now).setSeconds(now.getSeconds() - 10); + const subscriber = { last_stats: { rps: { test: 'test' }, codes: { '200': 10 } } }; + const report = Object.assign({}, REPORT, { + end_time: now, + start_time: tenSecBefore, + subscribers: [subscriber] + }); + databaseGetLastReportsStub.resolves([report]); + const reports = await manager.getLastReports(); + reports.length.should.eql(1); + should(reports[0].avg_rps).eql(0); + }); }); describe('Create new report', function () { @@ -429,9 +468,9 @@ describe('Reports manager tests', function () { databasePostStatsStub.resolves(); getJobStub.resolves(JOB); notifierStub.resolves(); - const stats = { phase_status: 'intermediate', data: JSON.stringify({ median: 4 }) }; + const stats = { phase_status: 'intermediate', data: JSON.stringify({ median: 4 }), runner_id: 123 }; - const statsResponse = await statsManager.postStats('test_id', stats); + const statsResponse = await statsManager.postStats({ subscribers: [{ runner_id: 123 }] }, stats); databaseUpdateSubscriberStub.callCount.should.eql(0); databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1); @@ -440,6 +479,52 @@ describe('Reports manager tests', function () { statsResponse.should.eql(stats); }); + it('Stats intermediate and verify update subscriber with total_count in first time', async () => { + configStub.resolves({}); + databaseGetReportStub.resolves([REPORT]); + databasePostStatsStub.resolves(); + getJobStub.resolves(JOB); + notifierStub.resolves(); + const stats = { + phase_status: 'intermediate', + data: JSON.stringify({ rps: { count: 10 } }), + runner_id: 123 + }; + const statsResponse = await statsManager.postStats({ subscribers: [{ runner_id: 123, last_stats: {} }] }, stats); + + databaseUpdateSubscriberStub.callCount.should.eql(0); + databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1); + const data = JSON.parse(databaseUpdateSubscriberWithStatsStub.args[0][4]); + should(data.rps.total_count).eql(10); + should.exist(statsResponse); + statsResponse.should.eql(stats); + }); + it('Stats intermediate and verify update subscriber second time with total_count', async () => { + configStub.resolves({}); + databaseGetReportStub.resolves([REPORT]); + databasePostStatsStub.resolves(); + getJobStub.resolves(JOB); + notifierStub.resolves(); + const stats = { + phase_status: 'intermediate', + data: JSON.stringify({ rps: { count: 10 } }), + runner_id: 123 + }; + const statsResponse = await statsManager.postStats({ + subscribers: [{ + runner_id: 123, + last_stats: { rps: { total_count: 18 } } + }] + }, stats); + + databaseUpdateSubscriberStub.callCount.should.eql(0); + databaseUpdateSubscriberWithStatsStub.callCount.should.eql(1); + const data = JSON.parse(databaseUpdateSubscriberWithStatsStub.args[0][4]); + should(data.rps.total_count).eql(28); + should.exist(statsResponse); + statsResponse.should.eql(stats); + }); + it('Stats consumer handles message with status done', async () => { configStub.resolves({}); databaseGetReportStub.resolves([REPORT]); @@ -456,6 +541,22 @@ describe('Reports manager tests', function () { should.exist(statsResponse); statsResponse.should.eql(stats); }); + it('Stats consumer handles message with status aborted', async () => { + configStub.resolves({}); + databaseGetReportStub.resolves([REPORT]); + databasePostStatsStub.resolves(); + getJobStub.resolves(JOB); + notifierStub.resolves(); + const stats = { phase_status: 'aborted', data: JSON.stringify({ median: 4 }) }; + + const statsResponse = await statsManager.postStats('test_id', stats); + + databaseUpdateSubscriberStub.callCount.should.eql(1); + databaseUpdateSubscriberWithStatsStub.callCount.should.eql(0); + + should.exist(statsResponse); + statsResponse.should.eql(stats); + }); it('when report done and have benchmark data ', async () => { databaseGetReportStub.resolves([REPORT_DONE]);