diff options
Diffstat (limited to 'src/catch2/reporters')
20 files changed, 654 insertions, 117 deletions
diff --git a/src/catch2/reporters/catch_reporter_automake.cpp b/src/catch2/reporters/catch_reporter_automake.cpp index 0660d092..5e506a6b 100644 --- a/src/catch2/reporters/catch_reporter_automake.cpp +++ b/src/catch2/reporters/catch_reporter_automake.cpp @@ -12,12 +12,14 @@ namespace Catch { - AutomakeReporter::~AutomakeReporter() {} + AutomakeReporter::~AutomakeReporter() = default; void AutomakeReporter::testCaseEnded(TestCaseStats const& _testCaseStats) { // Possible values to emit are PASS, XFAIL, SKIP, FAIL, XPASS and ERROR. m_stream << ":test-result: "; - if (_testCaseStats.totals.assertions.allPassed()) { + if ( _testCaseStats.totals.testCases.skipped > 0 ) { + m_stream << "SKIP"; + } else if (_testCaseStats.totals.assertions.allPassed()) { m_stream << "PASS"; } else if (_testCaseStats.totals.assertions.allOk()) { m_stream << "XFAIL"; diff --git a/src/catch2/reporters/catch_reporter_automake.hpp b/src/catch2/reporters/catch_reporter_automake.hpp index 3475a1fd..a639428c 100644 --- a/src/catch2/reporters/catch_reporter_automake.hpp +++ b/src/catch2/reporters/catch_reporter_automake.hpp @@ -8,7 +8,6 @@ #ifndef CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED #define CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED -#include <catch2/interfaces/catch_interfaces_reporter.hpp> #include <catch2/reporters/catch_reporter_streaming_base.hpp> #include <catch2/internal/catch_move_and_forward.hpp> diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp index d8088457..0f855944 100644 --- a/src/catch2/reporters/catch_reporter_compact.cpp +++ b/src/catch2/reporters/catch_reporter_compact.cpp @@ -105,6 +105,11 @@ public: printIssue("explicitly"); printRemainingMessages(Colour::None); break; + case ResultWas::ExplicitSkip: + printResultType(Colour::Skip, "skipped"_sr); + printMessage(); + printRemainingMessages(); + break; // These cases are here to prevent compiler warnings case ResultWas::Unknown: case ResultWas::FailureBit: @@ -166,7 +171,7 @@ private: return; const auto itEnd = messages.cend(); - const auto N = static_cast<std::size_t>(std::distance(itMessage, itEnd)); + const auto N = static_cast<std::size_t>(itEnd - itMessage); stream << colourImpl->guardColour( colour ) << " with " << pluralise( N, "message"_sr ) << ':'; @@ -187,7 +192,7 @@ private: private: std::ostream& stream; AssertionResult const& result; - std::vector<MessageInfo> messages; + std::vector<MessageInfo> const& messages; std::vector<MessageInfo>::const_iterator itMessage; bool printInfoMessages; ColourImpl* colourImpl; @@ -220,7 +225,7 @@ private: // Drop out if result was successful and we're not printing those if( !m_config->includeSuccessfulResults() && result.isOk() ) { - if( result.getResultType() != ResultWas::Warning ) + if( result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip ) return; printInfoMessages = false; } @@ -244,6 +249,6 @@ private: StreamingReporterBase::testRunEnded( _testRunStats ); } - CompactReporter::~CompactReporter() {} + CompactReporter::~CompactReporter() = default; } // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp index 0edb121e..bbde5ec9 100644 --- a/src/catch2/reporters/catch_reporter_console.cpp +++ b/src/catch2/reporters/catch_reporter_console.cpp @@ -51,7 +51,6 @@ public: stats(_stats), result(_stats.assertionResult), colour(Colour::None), - message(result.getMessage()), messages(_stats.infoMessages), colourImpl(colourImpl_), printInfoMessages(_printInfoMessages) { @@ -60,10 +59,10 @@ public: colour = Colour::Success; passOrFail = "PASSED"_sr; //if( result.hasMessage() ) - if (_stats.infoMessages.size() == 1) - messageLabel = "with message"; - if (_stats.infoMessages.size() > 1) - messageLabel = "with messages"; + if (messages.size() == 1) + messageLabel = "with message"_sr; + if (messages.size() > 1) + messageLabel = "with messages"_sr; break; case ResultWas::ExpressionFailed: if (result.isOk()) { @@ -73,43 +72,57 @@ public: colour = Colour::Error; passOrFail = "FAILED"_sr; } - if (_stats.infoMessages.size() == 1) - messageLabel = "with message"; - if (_stats.infoMessages.size() > 1) - messageLabel = "with messages"; + if (messages.size() == 1) + messageLabel = "with message"_sr; + if (messages.size() > 1) + messageLabel = "with messages"_sr; break; case ResultWas::ThrewException: colour = Colour::Error; passOrFail = "FAILED"_sr; - messageLabel = "due to unexpected exception with "; - if (_stats.infoMessages.size() == 1) - messageLabel += "message"; - if (_stats.infoMessages.size() > 1) - messageLabel += "messages"; + // todo switch + switch (messages.size()) { case 0: + messageLabel = "due to unexpected exception with "_sr; + break; + case 1: + messageLabel = "due to unexpected exception with message"_sr; + break; + default: + messageLabel = "due to unexpected exception with messages"_sr; + break; + } break; case ResultWas::FatalErrorCondition: colour = Colour::Error; passOrFail = "FAILED"_sr; - messageLabel = "due to a fatal error condition"; + messageLabel = "due to a fatal error condition"_sr; break; case ResultWas::DidntThrowException: colour = Colour::Error; passOrFail = "FAILED"_sr; - messageLabel = "because no exception was thrown where one was expected"; + messageLabel = "because no exception was thrown where one was expected"_sr; break; case ResultWas::Info: - messageLabel = "info"; + messageLabel = "info"_sr; break; case ResultWas::Warning: - messageLabel = "warning"; + messageLabel = "warning"_sr; break; case ResultWas::ExplicitFailure: passOrFail = "FAILED"_sr; colour = Colour::Error; - if (_stats.infoMessages.size() == 1) - messageLabel = "explicitly with message"; - if (_stats.infoMessages.size() > 1) - messageLabel = "explicitly with messages"; + if (messages.size() == 1) + messageLabel = "explicitly with message"_sr; + if (messages.size() > 1) + messageLabel = "explicitly with messages"_sr; + break; + case ResultWas::ExplicitSkip: + colour = Colour::Skip; + passOrFail = "SKIPPED"_sr; + if (messages.size() == 1) + messageLabel = "explicitly with message"_sr; + if (messages.size() > 1) + messageLabel = "explicitly with messages"_sr; break; // These cases are here to prevent compiler warnings case ResultWas::Unknown: @@ -173,9 +186,8 @@ private: AssertionResult const& result; Colour::Code colour; StringRef passOrFail; - std::string messageLabel; - std::string message; - std::vector<MessageInfo> messages; + StringRef messageLabel; + std::vector<MessageInfo> const& messages; ColourImpl* colourImpl; bool printInfoMessages; }; @@ -185,13 +197,16 @@ std::size_t makeRatio( std::uint64_t number, std::uint64_t total ) { return (ratio == 0 && number > 0) ? 1 : static_cast<std::size_t>(ratio); } -std::size_t& findMax( std::size_t& i, std::size_t& j, std::size_t& k ) { - if (i > j && i > k) +std::size_t& +findMax( std::size_t& i, std::size_t& j, std::size_t& k, std::size_t& l ) { + if (i > j && i > k && i > l) return i; - else if (j > k) + else if (j > k && j > l) return j; - else + else if (k > l) return k; + else + return l; } enum class Justification { Left, Right }; @@ -203,6 +218,7 @@ struct ColumnInfo { }; struct ColumnBreak {}; struct RowBreak {}; +struct OutputFlush {}; class Duration { enum class Unit { @@ -319,12 +335,12 @@ public: } template<typename T> - friend TablePrinter& operator << (TablePrinter& tp, T const& value) { + friend TablePrinter& operator<< (TablePrinter& tp, T const& value) { tp.m_oss << value; return tp; } - friend TablePrinter& operator << (TablePrinter& tp, ColumnBreak) { + friend TablePrinter& operator<< (TablePrinter& tp, ColumnBreak) { auto colStr = tp.m_oss.str(); const auto strSize = colStr.size(); tp.m_oss.str(""); @@ -346,13 +362,18 @@ public: return tp; } - friend TablePrinter& operator << (TablePrinter& tp, RowBreak) { + friend TablePrinter& operator<< (TablePrinter& tp, RowBreak) { if (tp.m_currentColumn > 0) { tp.m_os << '\n'; tp.m_currentColumn = -1; } return tp; } + + friend TablePrinter& operator<<(TablePrinter& tp, OutputFlush) { + tp.m_os << std::flush; + return tp; + } }; ConsoleReporter::ConsoleReporter(ReporterConfig&& config): @@ -374,7 +395,7 @@ ConsoleReporter::ConsoleReporter(ReporterConfig&& config): { "benchmark name", CATCH_CONFIG_CONSOLE_WIDTH - 43, Justification::Left }, { "samples mean std dev", 14, Justification::Right }, { "iterations low mean low std dev", 14, Justification::Right }, - { "estimated high mean high std dev", 14, Justification::Right } + { "est run time high mean high std dev", 14, Justification::Right } }; } }())) {} @@ -400,7 +421,8 @@ void ConsoleReporter::assertionEnded(AssertionStats const& _assertionStats) { bool includeResults = m_config->includeSuccessfulResults() || !result.isOk(); // Drop out if result was successful but we're not printing them. - if (!includeResults && result.getResultType() != ResultWas::Warning) + // TODO: Make configurable whether skips should be printed + if (!includeResults && result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip) return; lazyPrint(); @@ -457,8 +479,11 @@ void ConsoleReporter::benchmarkPreparing( StringRef name ) { void ConsoleReporter::benchmarkStarting(BenchmarkInfo const& info) { (*m_tablePrinter) << info.samples << ColumnBreak() << info.iterations << ColumnBreak(); - if (!m_config->benchmarkNoAnalysis()) - (*m_tablePrinter) << Duration(info.estimatedDuration) << ColumnBreak(); + if ( !m_config->benchmarkNoAnalysis() ) { + ( *m_tablePrinter ) + << Duration( info.estimatedDuration ) << ColumnBreak(); + } + ( *m_tablePrinter ) << OutputFlush{}; } void ConsoleReporter::benchmarkEnded(BenchmarkStats<> const& stats) { if (m_config->benchmarkNoAnalysis()) @@ -603,10 +628,11 @@ void ConsoleReporter::printTotalsDivider(Totals const& totals) { std::size_t failedRatio = makeRatio(totals.testCases.failed, totals.testCases.total()); std::size_t failedButOkRatio = makeRatio(totals.testCases.failedButOk, totals.testCases.total()); std::size_t passedRatio = makeRatio(totals.testCases.passed, totals.testCases.total()); - while (failedRatio + failedButOkRatio + passedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) - findMax(failedRatio, failedButOkRatio, passedRatio)++; + std::size_t skippedRatio = makeRatio(totals.testCases.skipped, totals.testCases.total()); + while (failedRatio + failedButOkRatio + passedRatio + skippedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)++; while (failedRatio + failedButOkRatio + passedRatio > CATCH_CONFIG_CONSOLE_WIDTH - 1) - findMax(failedRatio, failedButOkRatio, passedRatio)--; + findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)--; m_stream << m_colour->guardColour( Colour::Error ) << std::string( failedRatio, '=' ) @@ -619,6 +645,8 @@ void ConsoleReporter::printTotalsDivider(Totals const& totals) { m_stream << m_colour->guardColour( Colour::Success ) << std::string( passedRatio, '=' ); } + m_stream << m_colour->guardColour( Colour::Skip ) + << std::string( skippedRatio, '=' ); } else { m_stream << m_colour->guardColour( Colour::Warning ) << std::string( CATCH_CONFIG_CONSOLE_WIDTH - 1, '=' ); diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.cpp b/src/catch2/reporters/catch_reporter_cumulative_base.cpp index effd98b7..5e106326 100644 --- a/src/catch2/reporters/catch_reporter_cumulative_base.cpp +++ b/src/catch2/reporters/catch_reporter_cumulative_base.cpp @@ -70,7 +70,8 @@ namespace Catch { void CumulativeReporterBase::sectionStarting( SectionInfo const& sectionInfo ) { - SectionStats incompleteStats( sectionInfo, Counts(), 0, false ); + // We need a copy, because SectionStats expect to take ownership + SectionStats incompleteStats( SectionInfo(sectionInfo), Counts(), 0, false ); SectionNode* node; if ( m_sectionStack.empty() ) { if ( !m_rootSection ) { diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.hpp b/src/catch2/reporters/catch_reporter_cumulative_base.hpp index cdff9991..267b39fd 100644 --- a/src/catch2/reporters/catch_reporter_cumulative_base.hpp +++ b/src/catch2/reporters/catch_reporter_cumulative_base.hpp @@ -8,7 +8,6 @@ #ifndef CATCH_REPORTER_CUMULATIVE_BASE_HPP_INCLUDED #define CATCH_REPORTER_CUMULATIVE_BASE_HPP_INCLUDED -#include <catch2/interfaces/catch_interfaces_reporter.hpp> #include <catch2/reporters/catch_reporter_common_base.hpp> #include <catch2/internal/catch_move_and_forward.hpp> #include <catch2/internal/catch_unique_ptr.hpp> @@ -125,7 +124,7 @@ namespace Catch { void skipTest(TestCaseInfo const&) override {} protected: - //! Should the cumulative base store the assertion expansion for succesful assertions? + //! Should the cumulative base store the assertion expansion for successful assertions? bool m_shouldStoreSuccesfulAssertions = true; //! Should the cumulative base store the assertion expansion for failed assertions? bool m_shouldStoreFailedAssertions = true; diff --git a/src/catch2/reporters/catch_reporter_helpers.cpp b/src/catch2/reporters/catch_reporter_helpers.cpp index f6aa6fc5..ffb32ffb 100644 --- a/src/catch2/reporters/catch_reporter_helpers.cpp +++ b/src/catch2/reporters/catch_reporter_helpers.cpp @@ -316,15 +316,22 @@ namespace Catch { } std::vector<SummaryColumn> columns; + // Don't include "skipped assertions" in total count + const auto totalAssertionCount = + totals.assertions.total() - totals.assertions.skipped; columns.push_back( SummaryColumn( "", Colour::None ) .addRow( totals.testCases.total() ) - .addRow( totals.assertions.total() ) ); + .addRow( totalAssertionCount ) ); columns.push_back( SummaryColumn( "passed", Colour::Success ) .addRow( totals.testCases.passed ) .addRow( totals.assertions.passed ) ); columns.push_back( SummaryColumn( "failed", Colour::ResultError ) .addRow( totals.testCases.failed ) .addRow( totals.assertions.failed ) ); + columns.push_back( SummaryColumn( "skipped", Colour::Skip ) + .addRow( totals.testCases.skipped ) + // Don't print "skipped assertions" + .addRow( 0 ) ); columns.push_back( SummaryColumn( "failed as expected", Colour::ResultExpectedFailure ) .addRow( totals.testCases.failedButOk ) diff --git a/src/catch2/reporters/catch_reporter_json.cpp b/src/catch2/reporters/catch_reporter_json.cpp new file mode 100644 index 00000000..1f0db8b0 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_json.cpp @@ -0,0 +1,372 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +// +#include <catch2/catch_test_case_info.hpp> +#include <catch2/catch_test_spec.hpp> +#include <catch2/catch_version.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/internal/catch_list.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/reporters/catch_reporter_json.hpp> + +namespace Catch { + namespace { + void writeSourceInfo( JsonObjectWriter& writer, + SourceLineInfo const& sourceInfo ) { + auto source_location_writer = + writer.write( "source-location"_sr ).writeObject(); + source_location_writer.write( "filename"_sr ) + .write( sourceInfo.file ); + source_location_writer.write( "line"_sr ).write( sourceInfo.line ); + } + + void writeTags( JsonArrayWriter writer, std::vector<Tag> const& tags ) { + for ( auto const& tag : tags ) { + writer.write( tag.original ); + } + } + + void writeProperties( JsonArrayWriter writer, + TestCaseInfo const& info ) { + if ( info.isHidden() ) { writer.write( "is-hidden"_sr ); } + if ( info.okToFail() ) { writer.write( "ok-to-fail"_sr ); } + if ( info.expectedToFail() ) { + writer.write( "expected-to-fail"_sr ); + } + if ( info.throws() ) { writer.write( "throws"_sr ); } + } + + } // namespace + + JsonReporter::JsonReporter( ReporterConfig&& config ): + StreamingReporterBase{ CATCH_MOVE( config ) } { + + m_preferences.shouldRedirectStdOut = true; + // TBD: Do we want to report all assertions? XML reporter does + // not, but for machine-parseable reporters I think the answer + // should be yes. + m_preferences.shouldReportAllAssertions = true; + + m_objectWriters.emplace( m_stream ); + m_writers.emplace( Writer::Object ); + auto& writer = m_objectWriters.top(); + + writer.write( "version"_sr ).write( 1 ); + + { + auto metadata_writer = writer.write( "metadata"_sr ).writeObject(); + metadata_writer.write( "name"_sr ).write( m_config->name() ); + metadata_writer.write( "rng-seed"_sr ).write( m_config->rngSeed() ); + metadata_writer.write( "catch2-version"_sr ) + .write( libraryVersion() ); + if ( m_config->testSpec().hasFilters() ) { + metadata_writer.write( "filters"_sr ) + .write( m_config->testSpec() ); + } + } + } + + JsonReporter::~JsonReporter() { + endListing(); + // TODO: Ensure this closes the top level object, add asserts + assert( m_writers.size() == 1 && "Only the top level object should be open" ); + assert( m_writers.top() == Writer::Object ); + endObject(); + m_stream << '\n' << std::flush; + assert( m_writers.empty() ); + } + + JsonArrayWriter& JsonReporter::startArray() { + m_arrayWriters.emplace( m_arrayWriters.top().writeArray() ); + m_writers.emplace( Writer::Array ); + return m_arrayWriters.top(); + } + JsonArrayWriter& JsonReporter::startArray( StringRef key ) { + m_arrayWriters.emplace( + m_objectWriters.top().write( key ).writeArray() ); + m_writers.emplace( Writer::Array ); + return m_arrayWriters.top(); + } + + JsonObjectWriter& JsonReporter::startObject() { + m_objectWriters.emplace( m_arrayWriters.top().writeObject() ); + m_writers.emplace( Writer::Object ); + return m_objectWriters.top(); + } + JsonObjectWriter& JsonReporter::startObject( StringRef key ) { + m_objectWriters.emplace( + m_objectWriters.top().write( key ).writeObject() ); + m_writers.emplace( Writer::Object ); + return m_objectWriters.top(); + } + + void JsonReporter::endObject() { + assert( isInside( Writer::Object ) ); + m_objectWriters.pop(); + m_writers.pop(); + } + void JsonReporter::endArray() { + assert( isInside( Writer::Array ) ); + m_arrayWriters.pop(); + m_writers.pop(); + } + + bool JsonReporter::isInside( Writer writer ) { + return !m_writers.empty() && m_writers.top() == writer; + } + + void JsonReporter::startListing() { + if ( !m_startedListing ) { startObject( "listings"_sr ); } + m_startedListing = true; + } + void JsonReporter::endListing() { + if ( m_startedListing ) { endObject(); } + m_startedListing = false; + } + + std::string JsonReporter::getDescription() { + return "Outputs listings as JSON. Test listing is Work-in-Progress!"; + } + + void JsonReporter::testRunStarting( TestRunInfo const& testInfo ) { + StreamingReporterBase::testRunStarting( testInfo ); + endListing(); + + assert( isInside( Writer::Object ) ); + startObject( "test-run"_sr ); + startArray( "test-cases"_sr ); + } + + static void writeCounts( JsonObjectWriter&& writer, Counts const& counts ) { + writer.write( "passed"_sr ).write( counts.passed ); + writer.write( "failed"_sr ).write( counts.failed ); + writer.write( "fail-but-ok"_sr ).write( counts.failedButOk ); + writer.write( "skipped"_sr ).write( counts.skipped ); + } + + void JsonReporter::testRunEnded(TestRunStats const& runStats) { + assert( isInside( Writer::Array ) ); + // End "test-cases" + endArray(); + + { + auto totals = + m_objectWriters.top().write( "totals"_sr ).writeObject(); + writeCounts( totals.write( "assertions"_sr ).writeObject(), + runStats.totals.assertions ); + writeCounts( totals.write( "test-cases"_sr ).writeObject(), + runStats.totals.testCases ); + } + + // End the "test-run" object + endObject(); + } + + void JsonReporter::testCaseStarting( TestCaseInfo const& tcInfo ) { + StreamingReporterBase::testCaseStarting( tcInfo ); + + assert( isInside( Writer::Array ) && + "We should be in the 'test-cases' array" ); + startObject(); + // "test-info" prelude + { + auto testInfo = + m_objectWriters.top().write( "test-info"_sr ).writeObject(); + // TODO: handle testName vs className!! + testInfo.write( "name"_sr ).write( tcInfo.name ); + writeSourceInfo(testInfo, tcInfo.lineInfo); + writeTags( testInfo.write( "tags"_sr ).writeArray(), tcInfo.tags ); + writeProperties( testInfo.write( "properties"_sr ).writeArray(), + tcInfo ); + } + + + // Start the array for individual test runs (testCasePartial pairs) + startArray( "runs"_sr ); + } + + void JsonReporter::testCaseEnded( TestCaseStats const& tcStats ) { + StreamingReporterBase::testCaseEnded( tcStats ); + + // We need to close the 'runs' array before finishing the test case + assert( isInside( Writer::Array ) ); + endArray(); + + { + auto totals = + m_objectWriters.top().write( "totals"_sr ).writeObject(); + writeCounts( totals.write( "assertions"_sr ).writeObject(), + tcStats.totals.assertions ); + // We do not write the test case totals, because there will always be just one test case here. + // TODO: overall "result" -> success, skip, fail here? Or in partial result? + } + // We do not write out stderr/stdout, because we instead wrote those out in partial runs + + // TODO: aborting? + + // And we also close this test case's object + assert( isInside( Writer::Object ) ); + endObject(); + } + + void JsonReporter::testCasePartialStarting( TestCaseInfo const& /*tcInfo*/, + uint64_t index ) { + startObject(); + m_objectWriters.top().write( "run-idx"_sr ).write( index ); + startArray( "path"_sr ); + // TODO: we want to delay most of the printing to the 'root' section + // TODO: childSection key name? + } + + void JsonReporter::testCasePartialEnded( TestCaseStats const& tcStats, + uint64_t /*index*/ ) { + // Fixme: the top level section handles this. + //// path object + endArray(); + if ( !tcStats.stdOut.empty() ) { + m_objectWriters.top() + .write( "captured-stdout"_sr ) + .write( tcStats.stdOut ); + } + if ( !tcStats.stdErr.empty() ) { + m_objectWriters.top() + .write( "captured-stderr"_sr ) + .write( tcStats.stdErr ); + } + { + auto totals = + m_objectWriters.top().write( "totals"_sr ).writeObject(); + writeCounts( totals.write( "assertions"_sr ).writeObject(), + tcStats.totals.assertions ); + // We do not write the test case totals, because there will + // always be just one test case here. + // TODO: overall "result" -> success, skip, fail here? Or in + // partial result? + } + // TODO: aborting? + // run object + endObject(); + } + + void JsonReporter::sectionStarting( SectionInfo const& sectionInfo ) { + assert( isInside( Writer::Array ) && + "Section should always start inside an object" ); + // We want to nest top level sections, even though it shares name + // and source loc with the TEST_CASE + auto& sectionObject = startObject(); + sectionObject.write( "kind"_sr ).write( "section"_sr ); + sectionObject.write( "name"_sr ).write( sectionInfo.name ); + writeSourceInfo( m_objectWriters.top(), sectionInfo.lineInfo ); + + + // TBD: Do we want to create this event lazily? It would become + // rather complex, but we could do it, and it would look + // better for empty sections. OTOH, empty sections should + // be rare. + startArray( "path"_sr ); + } + void JsonReporter::sectionEnded( SectionStats const& /*sectionStats */) { + // End the subpath array + endArray(); + // TODO: metadata + // TODO: what info do we have here? + + // End the section object + endObject(); + } + + void JsonReporter::assertionStarting( AssertionInfo const& /*assertionInfo*/ ) {} + void JsonReporter::assertionEnded( AssertionStats const& assertionStats ) { + // TODO: There is lot of different things to handle here, but + // we can fill it in later, after we show that the basic + // outline and streaming reporter impl works well enough. + //if ( !m_config->includeSuccessfulResults() + // && assertionStats.assertionResult.isOk() ) { + // return; + //} + assert( isInside( Writer::Array ) ); + auto assertionObject = m_arrayWriters.top().writeObject(); + + assertionObject.write( "kind"_sr ).write( "assertion"_sr ); + writeSourceInfo( assertionObject, + assertionStats.assertionResult.getSourceInfo() ); + assertionObject.write( "status"_sr ) + .write( assertionStats.assertionResult.isOk() ); + // TODO: handling of result. + // TODO: messages + // TODO: totals? + } + + + void JsonReporter::benchmarkPreparing( StringRef name ) { (void)name; } + void JsonReporter::benchmarkStarting( BenchmarkInfo const& ) {} + void JsonReporter::benchmarkEnded( BenchmarkStats<> const& ) {} + void JsonReporter::benchmarkFailed( StringRef error ) { (void)error; } + + void JsonReporter::listReporters( + std::vector<ReporterDescription> const& descriptions ) { + startListing(); + + auto writer = + m_objectWriters.top().write( "reporters"_sr ).writeArray(); + for ( auto const& desc : descriptions ) { + auto desc_writer = writer.writeObject(); + desc_writer.write( "name"_sr ).write( desc.name ); + desc_writer.write( "description"_sr ).write( desc.description ); + } + } + void JsonReporter::listListeners( + std::vector<ListenerDescription> const& descriptions ) { + startListing(); + + auto writer = + m_objectWriters.top().write( "listeners"_sr ).writeArray(); + + for ( auto const& desc : descriptions ) { + auto desc_writer = writer.writeObject(); + desc_writer.write( "name"_sr ).write( desc.name ); + desc_writer.write( "description"_sr ).write( desc.description ); + } + } + void JsonReporter::listTests( std::vector<TestCaseHandle> const& tests ) { + startListing(); + + auto writer = m_objectWriters.top().write( "tests"_sr ).writeArray(); + + for ( auto const& test : tests ) { + auto desc_writer = writer.writeObject(); + auto const& info = test.getTestCaseInfo(); + + desc_writer.write( "name"_sr ).write( info.name ); + desc_writer.write( "class-name"_sr ).write( info.className ); + { + auto tag_writer = desc_writer.write( "tags"_sr ).writeArray(); + for ( auto const& tag : info.tags ) { + tag_writer.write( tag.original ); + } + } + writeSourceInfo( desc_writer, info.lineInfo ); + } + } + void JsonReporter::listTags( std::vector<TagInfo> const& tags ) { + startListing(); + + auto writer = m_objectWriters.top().write( "tags"_sr ).writeArray(); + for ( auto const& tag : tags ) { + auto tag_writer = writer.writeObject(); + { + auto aliases_writer = + tag_writer.write( "aliases"_sr ).writeArray(); + for ( auto alias : tag.spellings ) { + aliases_writer.write( alias ); + } + } + tag_writer.write( "count"_sr ).write( tag.count ); + } + } +} // namespace Catch diff --git a/src/catch2/reporters/catch_reporter_json.hpp b/src/catch2/reporters/catch_reporter_json.hpp new file mode 100644 index 00000000..c938ca39 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_json.hpp @@ -0,0 +1,95 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#ifndef CATCH_REPORTER_JSON_HPP_INCLUDED +#define CATCH_REPORTER_JSON_HPP_INCLUDED + +#include <catch2/catch_timer.hpp> +#include <catch2/internal/catch_jsonwriter.hpp> +#include <catch2/reporters/catch_reporter_streaming_base.hpp> + +#include <stack> + +namespace Catch { + class JsonReporter : public StreamingReporterBase { + public: + JsonReporter( ReporterConfig&& config ); + + ~JsonReporter() override; + + static std::string getDescription(); + + public: // StreamingReporterBase + void testRunStarting( TestRunInfo const& runInfo ) override; + void testRunEnded( TestRunStats const& runStats ) override; + + void testCaseStarting( TestCaseInfo const& tcInfo ) override; + void testCaseEnded( TestCaseStats const& tcStats ) override; + + void testCasePartialStarting( TestCaseInfo const& tcInfo, + uint64_t index ) override; + void testCasePartialEnded( TestCaseStats const& tcStats, + uint64_t index ) override; + + void sectionStarting( SectionInfo const& sectionInfo ) override; + void sectionEnded( SectionStats const& sectionStats ) override; + + void assertionStarting( AssertionInfo const& assertionInfo ) override; + void assertionEnded( AssertionStats const& assertionStats ) override; + + //void testRunEndedCumulative() override; + + void benchmarkPreparing( StringRef name ) override; + void benchmarkStarting( BenchmarkInfo const& ) override; + void benchmarkEnded( BenchmarkStats<> const& ) override; + void benchmarkFailed( StringRef error ) override; + + void listReporters( + std::vector<ReporterDescription> const& descriptions ) override; + void listListeners( + std::vector<ListenerDescription> const& descriptions ) override; + void listTests( std::vector<TestCaseHandle> const& tests ) override; + void listTags( std::vector<TagInfo> const& tags ) override; + + private: + Timer m_testCaseTimer; + enum class Writer { + Object, + Array + }; + + JsonArrayWriter& startArray(); + JsonArrayWriter& startArray( StringRef key ); + + JsonObjectWriter& startObject(); + JsonObjectWriter& startObject( StringRef key ); + + void endObject(); + void endArray(); + + bool isInside( Writer writer ); + + void startListing(); + void endListing(); + + // Invariant: + // When m_writers is not empty and its top element is + // - Writer::Object, then m_objectWriters is not be empty + // - Writer::Array, then m_arrayWriters shall not be empty + std::stack<JsonObjectWriter> m_objectWriters{}; + std::stack<JsonArrayWriter> m_arrayWriters{}; + std::stack<Writer> m_writers{}; + + bool m_startedListing = false; + + // std::size_t m_sectionDepth = 0; + // std::size_t m_sectionStarted = 0; + }; +} // namespace Catch + +#endif // CATCH_REPORTER_JSON_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index 837d0489..fc5cae34 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -33,6 +33,8 @@ namespace Catch { gmtime_s(&timeInfo, &rawtime); #elif defined (CATCH_PLATFORM_PLAYSTATION) gmtime_s(&rawtime, &timeInfo); +#elif defined (__IAR_SYSTEMS_ICC__) + timeInfo = *std::gmtime(&rawtime); #else gmtime_r(&rawtime, &timeInfo); #endif @@ -132,6 +134,7 @@ namespace Catch { xml.writeAttribute( "name"_sr, stats.runInfo.name ); xml.writeAttribute( "errors"_sr, unexpectedExceptions ); xml.writeAttribute( "failures"_sr, stats.totals.assertions.failed-unexpectedExceptions ); + xml.writeAttribute( "skipped"_sr, stats.totals.assertions.skipped ); xml.writeAttribute( "tests"_sr, stats.totals.assertions.total() ); xml.writeAttribute( "hostname"_sr, "tbd"_sr ); // !TBD if( m_config->showDurations() == ShowDurations::Never ) @@ -244,7 +247,8 @@ namespace Catch { void JunitReporter::writeAssertion( AssertionStats const& stats ) { AssertionResult const& result = stats.assertionResult; - if( !result.isOk() ) { + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { std::string elementName; switch( result.getResultType() ) { case ResultWas::ThrewException: @@ -256,7 +260,9 @@ namespace Catch { case ResultWas::DidntThrowException: elementName = "failure"; break; - + case ResultWas::ExplicitSkip: + elementName = "skipped"; + break; // We should never see these here: case ResultWas::Info: case ResultWas::Warning: @@ -274,7 +280,9 @@ namespace Catch { xml.writeAttribute( "type"_sr, result.getTestMacroName() ); ReusableStringStream rss; - if (stats.totals.assertions.total() > 0) { + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + rss << "SKIPPED\n"; + } else { rss << "FAILED" << ":\n"; if (result.hasExpression()) { rss << " "; @@ -285,11 +293,9 @@ namespace Catch { rss << "with expansion:\n"; rss << TextFlow::Column(result.getExpandedExpression()).indent(2) << '\n'; } - } else { - rss << '\n'; } - if( !result.getMessage().empty() ) + if( result.hasMessage() ) rss << result.getMessage() << '\n'; for( auto const& msg : stats.infoMessages ) if( msg.type == ResultWas::Info ) diff --git a/src/catch2/reporters/catch_reporter_multi.cpp b/src/catch2/reporters/catch_reporter_multi.cpp index ebf28b64..531902be 100644 --- a/src/catch2/reporters/catch_reporter_multi.cpp +++ b/src/catch2/reporters/catch_reporter_multi.cpp @@ -114,7 +114,6 @@ namespace Catch { } } - // The return value indicates if the messages buffer should be cleared: void MultiReporter::assertionEnded( AssertionStats const& assertionStats ) { const bool reportByDefault = assertionStats.assertionResult.getResultType() != ResultWas::Ok || diff --git a/src/catch2/reporters/catch_reporter_registrars.cpp b/src/catch2/reporters/catch_reporter_registrars.cpp index a9787ce5..2a3ac957 100644 --- a/src/catch2/reporters/catch_reporter_registrars.cpp +++ b/src/catch2/reporters/catch_reporter_registrars.cpp @@ -8,6 +8,7 @@ #include <catch2/reporters/catch_reporter_registrars.hpp> +#include <catch2/interfaces/catch_interfaces_registry_hub.hpp> #include <catch2/internal/catch_compiler_capabilities.hpp> namespace Catch { @@ -26,5 +27,10 @@ namespace Catch { } } + void registerListenerImpl( Detail::unique_ptr<EventListenerFactory> listenerFactory ) { + getMutableRegistryHub().registerListener( CATCH_MOVE(listenerFactory) ); + } + + } // namespace Detail } // namespace Catch diff --git a/src/catch2/reporters/catch_reporter_registrars.hpp b/src/catch2/reporters/catch_reporter_registrars.hpp index fe8cf1ec..a93963f0 100644 --- a/src/catch2/reporters/catch_reporter_registrars.hpp +++ b/src/catch2/reporters/catch_reporter_registrars.hpp @@ -8,8 +8,6 @@ #ifndef CATCH_REPORTER_REGISTRARS_HPP_INCLUDED #define CATCH_REPORTER_REGISTRARS_HPP_INCLUDED -#include <catch2/interfaces/catch_interfaces_registry_hub.hpp> -#include <catch2/interfaces/catch_interfaces_reporter.hpp> #include <catch2/interfaces/catch_interfaces_reporter_factory.hpp> #include <catch2/internal/catch_compiler_capabilities.hpp> #include <catch2/internal/catch_unique_name.hpp> @@ -36,7 +34,8 @@ namespace Catch { //! independent on the reporter's concrete type void registerReporterImpl( std::string const& name, IReporterFactoryPtr reporterPtr ); - + //! Actually registers the factory, independent on listener's concrete type + void registerListenerImpl( Detail::unique_ptr<EventListenerFactory> listenerFactory ); } // namespace Detail class IEventListener; @@ -97,7 +96,7 @@ namespace Catch { public: ListenerRegistrar(StringRef listenerName) { - getMutableRegistryHub().registerListener( Detail::make_unique<TypedListenerFactory>(listenerName) ); + registerListenerImpl( Detail::make_unique<TypedListenerFactory>(listenerName) ); } }; } @@ -118,7 +117,7 @@ namespace Catch { CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ namespace { \ Catch::ListenerRegistrar<listenerType> INTERNAL_CATCH_UNIQUE_NAME( \ - catch_internal_RegistrarFor )( #listenerType ); \ + catch_internal_RegistrarFor )( #listenerType##_catch_sr ); \ } \ CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp index 365979f4..9c391b1f 100644 --- a/src/catch2/reporters/catch_reporter_sonarqube.cpp +++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp @@ -40,7 +40,7 @@ namespace Catch { } void SonarQubeReporter::writeRun( TestRunNode const& runNode ) { - std::map<std::string, std::vector<TestCaseNode const*>> testsPerFile; + std::map<StringRef, std::vector<TestCaseNode const*>> testsPerFile; for ( auto const& child : runNode.children ) { testsPerFile[child->value.testInfo->lineInfo.file].push_back( @@ -52,7 +52,7 @@ namespace Catch { } } - void SonarQubeReporter::writeTestFile(std::string const& filename, std::vector<TestCaseNode const*> const& testCaseNodes) { + void SonarQubeReporter::writeTestFile(StringRef filename, std::vector<TestCaseNode const*> const& testCaseNodes) { XmlWriter::ScopedElement e = xml.scopedElement("file"); xml.writeAttribute("path"_sr, filename); @@ -97,7 +97,8 @@ namespace Catch { void SonarQubeReporter::writeAssertion(AssertionStats const& stats, bool okToFail) { AssertionResult const& result = stats.assertionResult; - if (!result.isOk()) { + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { std::string elementName; if (okToFail) { elementName = "skipped"; @@ -108,15 +109,13 @@ namespace Catch { elementName = "error"; break; case ResultWas::ExplicitFailure: - elementName = "failure"; - break; case ResultWas::ExpressionFailed: - elementName = "failure"; - break; case ResultWas::DidntThrowException: elementName = "failure"; break; - + case ResultWas::ExplicitSkip: + elementName = "skipped"; + break; // We should never see these here: case ResultWas::Info: case ResultWas::Warning: @@ -136,7 +135,9 @@ namespace Catch { xml.writeAttribute("message"_sr, messageRss.str()); ReusableStringStream textRss; - if (stats.totals.assertions.total() > 0) { + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + textRss << "SKIPPED\n"; + } else { textRss << "FAILED:\n"; if (result.hasExpression()) { textRss << '\t' << result.getExpressionInMacro() << '\n'; @@ -146,7 +147,7 @@ namespace Catch { } } - if (!result.getMessage().empty()) + if (result.hasMessage()) textRss << result.getMessage() << '\n'; for (auto const& msg : stats.infoMessages) diff --git a/src/catch2/reporters/catch_reporter_sonarqube.hpp b/src/catch2/reporters/catch_reporter_sonarqube.hpp index bc8f0332..cad6deec 100644 --- a/src/catch2/reporters/catch_reporter_sonarqube.hpp +++ b/src/catch2/reporters/catch_reporter_sonarqube.hpp @@ -41,7 +41,7 @@ namespace Catch { void writeRun( TestRunNode const& groupNode ); - void writeTestFile(std::string const& filename, std::vector<TestCaseNode const*> const& testCaseNodes); + void writeTestFile(StringRef filename, std::vector<TestCaseNode const*> const& testCaseNodes); void writeTestCase(TestCaseNode const& testCaseNode); diff --git a/src/catch2/reporters/catch_reporter_streaming_base.hpp b/src/catch2/reporters/catch_reporter_streaming_base.hpp index 13672a28..5448000c 100644 --- a/src/catch2/reporters/catch_reporter_streaming_base.hpp +++ b/src/catch2/reporters/catch_reporter_streaming_base.hpp @@ -8,7 +8,6 @@ #ifndef CATCH_REPORTER_STREAMING_BASE_HPP_INCLUDED #define CATCH_REPORTER_STREAMING_BASE_HPP_INCLUDED -#include <catch2/interfaces/catch_interfaces_reporter.hpp> #include <catch2/reporters/catch_reporter_common_base.hpp> #include <catch2/internal/catch_move_and_forward.hpp> diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp index 59f8fb8b..67d406fb 100644 --- a/src/catch2/reporters/catch_reporter_tap.cpp +++ b/src/catch2/reporters/catch_reporter_tap.cpp @@ -14,7 +14,6 @@ #include <catch2/reporters/catch_reporter_helpers.hpp> #include <algorithm> -#include <iterator> #include <ostream> namespace Catch { @@ -100,6 +99,12 @@ namespace Catch { printIssue("explicitly"_sr); printRemainingMessages(Colour::None); break; + case ResultWas::ExplicitSkip: + printResultType(tapPassedString); + printIssue(" # SKIP"_sr); + printMessage(); + printRemainingMessages(); + break; // These cases are here to prevent compiler warnings case ResultWas::Unknown: case ResultWas::FailureBit: @@ -159,7 +164,7 @@ namespace Catch { // using messages.end() directly (or auto) yields compilation error: std::vector<MessageInfo>::const_iterator itEnd = messages.end(); - const std::size_t N = static_cast<std::size_t>(std::distance(itMessage, itEnd)); + const std::size_t N = static_cast<std::size_t>(itEnd - itMessage); stream << colourImpl->guardColour( colour ) << " with " << pluralise( N, "message"_sr ) << ':'; @@ -178,7 +183,7 @@ namespace Catch { private: std::ostream& stream; AssertionResult const& result; - std::vector<MessageInfo> messages; + std::vector<MessageInfo> const& messages; std::vector<MessageInfo>::const_iterator itMessage; bool printInfoMessages; std::size_t counter; diff --git a/src/catch2/reporters/catch_reporter_teamcity.cpp b/src/catch2/reporters/catch_reporter_teamcity.cpp index 1d002c27..38aa55a6 100644 --- a/src/catch2/reporters/catch_reporter_teamcity.cpp +++ b/src/catch2/reporters/catch_reporter_teamcity.cpp @@ -45,7 +45,7 @@ namespace Catch { } // end anonymous namespace - TeamCityReporter::~TeamCityReporter() {} + TeamCityReporter::~TeamCityReporter() = default; void TeamCityReporter::testRunStarting( TestRunInfo const& runInfo ) { m_stream << "##teamcity[testSuiteStarted name='" << escape( runInfo.name ) @@ -59,7 +59,8 @@ namespace Catch { void TeamCityReporter::assertionEnded(AssertionStats const& assertionStats) { AssertionResult const& result = assertionStats.assertionResult; - if (!result.isOk()) { + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { ReusableStringStream msg; if (!m_headerPrintedForThisSection) @@ -84,6 +85,9 @@ namespace Catch { case ResultWas::ExplicitFailure: msg << "explicit failure"; break; + case ResultWas::ExplicitSkip: + msg << "explicit skip"; + break; // We shouldn't get here because of the isOk() test case ResultWas::Ok: @@ -111,18 +115,16 @@ namespace Catch { " " << result.getExpandedExpression() << '\n'; } - if (currentTestCaseInfo->okToFail()) { + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + m_stream << "##teamcity[testIgnored"; + } else if ( currentTestCaseInfo->okToFail() ) { msg << "- failure ignore as test marked as 'ok to fail'\n"; - m_stream << "##teamcity[testIgnored" - << " name='" << escape(currentTestCaseInfo->name) << '\'' - << " message='" << escape(msg.str()) << '\'' - << "]\n"; + m_stream << "##teamcity[testIgnored"; } else { - m_stream << "##teamcity[testFailed" - << " name='" << escape(currentTestCaseInfo->name) << '\'' - << " message='" << escape(msg.str()) << '\'' - << "]\n"; + m_stream << "##teamcity[testFailed"; } + m_stream << " name='" << escape( currentTestCaseInfo->name ) << '\'' + << " message='" << escape( msg.str() ) << '\'' << "]\n"; } m_stream.flush(); } diff --git a/src/catch2/reporters/catch_reporter_xml.cpp b/src/catch2/reporters/catch_reporter_xml.cpp index 57fa1cab..35a3028e 100644 --- a/src/catch2/reporters/catch_reporter_xml.cpp +++ b/src/catch2/reporters/catch_reporter_xml.cpp @@ -56,7 +56,7 @@ namespace Catch { m_xml.startElement("Catch2TestRun") .writeAttribute("name"_sr, m_config->name()) .writeAttribute("rng-seed"_sr, m_config->rngSeed()) - .writeAttribute("xml-format-version"_sr, 2) + .writeAttribute("xml-format-version"_sr, 3) .writeAttribute("catch2-version"_sr, libraryVersion()); if ( m_config->testSpec().hasFilters() ) { m_xml.writeAttribute( "filters"_sr, m_config->testSpec() ); @@ -66,7 +66,7 @@ namespace Catch { void XmlReporter::testCaseStarting( TestCaseInfo const& testInfo ) { StreamingReporterBase::testCaseStarting(testInfo); m_xml.startElement( "TestCase" ) - .writeAttribute( "name"_sr, trim( testInfo.name ) ) + .writeAttribute( "name"_sr, trim( StringRef(testInfo.name) ) ) .writeAttribute( "tags"_sr, testInfo.tagsAsString() ); writeSourceInfo( testInfo.lineInfo ); @@ -80,7 +80,7 @@ namespace Catch { StreamingReporterBase::sectionStarting( sectionInfo ); if( m_sectionDepth++ > 0 ) { m_xml.startElement( "Section" ) - .writeAttribute( "name"_sr, trim( sectionInfo.name ) ); + .writeAttribute( "name"_sr, trim( StringRef(sectionInfo.name) ) ); writeSourceInfo( sectionInfo.lineInfo ); m_xml.ensureTagClosed(); } @@ -98,19 +98,22 @@ namespace Catch { // Print any info messages in <Info> tags. for( auto const& msg : assertionStats.infoMessages ) { if( msg.type == ResultWas::Info && includeResults ) { - m_xml.scopedElement( "Info" ) - .writeText( msg.message ); + auto t = m_xml.scopedElement( "Info" ); + writeSourceInfo( msg.lineInfo ); + t.writeText( msg.message ); } else if ( msg.type == ResultWas::Warning ) { - m_xml.scopedElement( "Warning" ) - .writeText( msg.message ); + auto t = m_xml.scopedElement( "Warning" ); + writeSourceInfo( msg.lineInfo ); + t.writeText( msg.message ); } } } // Drop out if result was successful but we're not printing them. - if( !includeResults && result.getResultType() != ResultWas::Warning ) + if ( !includeResults && result.getResultType() != ResultWas::Warning && + result.getResultType() != ResultWas::ExplicitSkip ) { return; - + } // Print the expression if there is one. if( result.hasExpression() ) { @@ -153,6 +156,12 @@ namespace Catch { m_xml.writeText( result.getMessage() ); m_xml.endElement(); break; + case ResultWas::ExplicitSkip: + m_xml.startElement( "Skip" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; default: break; } @@ -163,15 +172,18 @@ namespace Catch { void XmlReporter::sectionEnded( SectionStats const& sectionStats ) { StreamingReporterBase::sectionEnded( sectionStats ); - if( --m_sectionDepth > 0 ) { - XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); - e.writeAttribute( "successes"_sr, sectionStats.assertions.passed ); - e.writeAttribute( "failures"_sr, sectionStats.assertions.failed ); - e.writeAttribute( "expectedFailures"_sr, sectionStats.assertions.failedButOk ); - - if ( m_config->showDurations() == ShowDurations::Always ) - e.writeAttribute( "durationInSeconds"_sr, sectionStats.durationInSeconds ); - + if ( --m_sectionDepth > 0 ) { + { + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); + e.writeAttribute( "successes"_sr, sectionStats.assertions.passed ); + e.writeAttribute( "failures"_sr, sectionStats.assertions.failed ); + e.writeAttribute( "expectedFailures"_sr, sectionStats.assertions.failedButOk ); + e.writeAttribute( "skipped"_sr, sectionStats.assertions.skipped > 0 ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds"_sr, sectionStats.durationInSeconds ); + } + // Ends assertion tag m_xml.endElement(); } } @@ -180,14 +192,14 @@ namespace Catch { StreamingReporterBase::testCaseEnded( testCaseStats ); XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" ); e.writeAttribute( "success"_sr, testCaseStats.totals.assertions.allOk() ); + e.writeAttribute( "skips"_sr, testCaseStats.totals.assertions.skipped ); if ( m_config->showDurations() == ShowDurations::Always ) e.writeAttribute( "durationInSeconds"_sr, m_testCaseTimer.getElapsedSeconds() ); - if( !testCaseStats.stdOut.empty() ) - m_xml.scopedElement( "StdOut" ).writeText( trim( testCaseStats.stdOut ), XmlFormatting::Newline ); + m_xml.scopedElement( "StdOut" ).writeText( trim( StringRef(testCaseStats.stdOut) ), XmlFormatting::Newline ); if( !testCaseStats.stdErr.empty() ) - m_xml.scopedElement( "StdErr" ).writeText( trim( testCaseStats.stdErr ), XmlFormatting::Newline ); + m_xml.scopedElement( "StdErr" ).writeText( trim( StringRef(testCaseStats.stdErr) ), XmlFormatting::Newline ); m_xml.endElement(); } @@ -197,11 +209,13 @@ namespace Catch { m_xml.scopedElement( "OverallResults" ) .writeAttribute( "successes"_sr, testRunStats.totals.assertions.passed ) .writeAttribute( "failures"_sr, testRunStats.totals.assertions.failed ) - .writeAttribute( "expectedFailures"_sr, testRunStats.totals.assertions.failedButOk ); + .writeAttribute( "expectedFailures"_sr, testRunStats.totals.assertions.failedButOk ) + .writeAttribute( "skips"_sr, testRunStats.totals.assertions.skipped ); m_xml.scopedElement( "OverallResultsCases") .writeAttribute( "successes"_sr, testRunStats.totals.testCases.passed ) .writeAttribute( "failures"_sr, testRunStats.totals.testCases.failed ) - .writeAttribute( "expectedFailures"_sr, testRunStats.totals.testCases.failedButOk ); + .writeAttribute( "expectedFailures"_sr, testRunStats.totals.testCases.failedButOk ) + .writeAttribute( "skips"_sr, testRunStats.totals.testCases.skipped ); m_xml.endElement(); } @@ -220,26 +234,23 @@ namespace Catch { } void XmlReporter::benchmarkEnded(BenchmarkStats<> const& benchmarkStats) { - m_xml.startElement("mean") + m_xml.scopedElement("mean") .writeAttribute("value"_sr, benchmarkStats.mean.point.count()) .writeAttribute("lowerBound"_sr, benchmarkStats.mean.lower_bound.count()) .writeAttribute("upperBound"_sr, benchmarkStats.mean.upper_bound.count()) .writeAttribute("ci"_sr, benchmarkStats.mean.confidence_interval); - m_xml.endElement(); - m_xml.startElement("standardDeviation") + m_xml.scopedElement("standardDeviation") .writeAttribute("value"_sr, benchmarkStats.standardDeviation.point.count()) .writeAttribute("lowerBound"_sr, benchmarkStats.standardDeviation.lower_bound.count()) .writeAttribute("upperBound"_sr, benchmarkStats.standardDeviation.upper_bound.count()) .writeAttribute("ci"_sr, benchmarkStats.standardDeviation.confidence_interval); - m_xml.endElement(); - m_xml.startElement("outliers") + m_xml.scopedElement("outliers") .writeAttribute("variance"_sr, benchmarkStats.outlierVariance) .writeAttribute("lowMild"_sr, benchmarkStats.outliers.low_mild) .writeAttribute("lowSevere"_sr, benchmarkStats.outliers.low_severe) .writeAttribute("highMild"_sr, benchmarkStats.outliers.high_mild) .writeAttribute("highSevere"_sr, benchmarkStats.outliers.high_severe); m_xml.endElement(); - m_xml.endElement(); } void XmlReporter::benchmarkFailed(StringRef error) { diff --git a/src/catch2/reporters/catch_reporters_all.hpp b/src/catch2/reporters/catch_reporters_all.hpp index 16f7bd70..5c713fe1 100644 --- a/src/catch2/reporters/catch_reporters_all.hpp +++ b/src/catch2/reporters/catch_reporters_all.hpp @@ -28,6 +28,7 @@ #include <catch2/reporters/catch_reporter_cumulative_base.hpp> #include <catch2/reporters/catch_reporter_event_listener.hpp> #include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/reporters/catch_reporter_json.hpp> #include <catch2/reporters/catch_reporter_junit.hpp> #include <catch2/reporters/catch_reporter_multi.hpp> #include <catch2/reporters/catch_reporter_registrars.hpp> |