Files
Catch2/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
Martin Hořeňovský 3a0cf7e75f Support filtering on both sections and generators (#3069)
Not being able to filter generators to specific element has been regularly
causing problems. It was possible to use a dynamic section to run tests
for specific element in a generator, at least if the element had a nice
string representation, but the test case would still run once per element
in the generator.

With this change, it is possible to have the generator return only one
specific element, and do so based on the index, rather than the string
representation of the element. This enables simple debugging of tests
that fail for specific generator element.
2026-02-15 20:27:15 +01:00

563 lines
20 KiB
C++

// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
#include <catch2/catch_config.hpp>
#include <catch2/catch_approx.hpp>
#include <catch2/catch_test_macros.hpp>
#include <catch2/matchers/catch_matchers_string.hpp>
#include <catch2/internal/catch_test_spec_parser.hpp>
#include <catch2/catch_user_config.hpp>
#include <catch2/catch_test_case_info.hpp>
#include <catch2/internal/catch_commandline.hpp>
#include <catch2/generators/catch_generators.hpp>
#include <catch2/internal/catch_compiler_capabilities.hpp>
namespace {
auto fakeTestCase(const char* name, const char* desc = "") { return Catch::makeTestCaseInfo("", { name, desc }, CATCH_INTERNAL_LINEINFO); }
}
TEST_CASE( "Process can be configured on command line", "[config][command-line]" ) {
using namespace Catch::Matchers;
Catch::ConfigData config;
auto cli = Catch::makeCommandLineParser(config);
SECTION("empty args don't cause a crash") {
auto result = cli.parse({""});
CHECK(result);
CHECK(config.processName == "");
}
SECTION("default - no arguments") {
auto result = cli.parse({"test"});
CHECK(result);
CHECK(config.processName == "test");
CHECK(config.shouldDebugBreak == false);
CHECK(config.abortAfter == -1);
CHECK(config.noThrow == false);
CHECK( config.reporterSpecifications.empty() );
Catch::Config cfg(config);
CHECK_FALSE(cfg.hasTestFilters());
// The Config is responsible for mixing in the default reporter
auto expectedReporter =
#if defined( CATCH_CONFIG_DEFAULT_REPORTER )
CATCH_CONFIG_DEFAULT_REPORTER
#else
"console"
#endif
;
CHECK( cfg.getReporterSpecs().size() == 1 );
CHECK( cfg.getReporterSpecs()[0] ==
Catch::ReporterSpec{ expectedReporter, {}, {}, {} } );
CHECK( cfg.getProcessedReporterSpecs().size() == 1 );
CHECK( cfg.getProcessedReporterSpecs()[0] ==
Catch::ProcessedReporterSpec{ expectedReporter,
std::string{},
Catch::ColourMode::PlatformDefault,
{} } );
}
SECTION("test lists") {
SECTION("Specify one test case using") {
auto result = cli.parse({"test", "test1"});
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(*fakeTestCase("notIncluded")) == false);
REQUIRE(cfg.testSpec().matches(*fakeTestCase("test1")));
}
SECTION("Specify one test case exclusion using exclude:") {
auto result = cli.parse({"test", "exclude:test1"});
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(*fakeTestCase("test1")) == false);
REQUIRE(cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")));
}
SECTION("Specify one test case exclusion using ~") {
auto result = cli.parse({"test", "~test1"});
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(*fakeTestCase("test1")) == false);
REQUIRE(cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")));
}
}
SECTION("reporter") {
using vec_Specs = std::vector<Catch::ReporterSpec>;
using namespace std::string_literals;
SECTION("-r/console") {
auto result = cli.parse({"test", "-r", "console"});
CAPTURE(result.errorMessage());
CHECK(result);
REQUIRE( config.reporterSpecifications ==
vec_Specs{ { "console", {}, {}, {} } } );
}
SECTION("-r/xml") {
auto result = cli.parse({"test", "-r", "xml"});
CAPTURE(result.errorMessage());
CHECK(result);
REQUIRE( config.reporterSpecifications ==
vec_Specs{ { "xml", {}, {}, {} } } );
}
SECTION("--reporter/junit") {
auto result = cli.parse({"test", "--reporter", "junit"});
CAPTURE(result.errorMessage());
CHECK(result);
REQUIRE( config.reporterSpecifications ==
vec_Specs{ { "junit", {}, {}, {} } } );
}
SECTION("must match one of the available ones") {
auto result = cli.parse({"test", "--reporter", "unsupported"});
CHECK(!result);
REQUIRE_THAT(result.errorMessage(), ContainsSubstring("Unrecognized reporter"));
}
SECTION("With output file") {
auto result = cli.parse({ "test", "-r", "console::out=out.txt" });
CAPTURE(result.errorMessage());
CHECK(result);
REQUIRE( config.reporterSpecifications ==
vec_Specs{ { "console", "out.txt"s, {}, {} } } );
}
SECTION("With Windows-like absolute path as output file") {
auto result = cli.parse({ "test", "-r", "console::out=C:\\Temp\\out.txt" });
CAPTURE(result.errorMessage());
CHECK(result);
REQUIRE( config.reporterSpecifications ==
vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } );
}
SECTION("Multiple reporters") {
SECTION("All with output files") {
CHECK(cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }));
REQUIRE( config.reporterSpecifications ==
vec_Specs{ { "xml", "output.xml"s, {}, {} },
{ "junit", "output-junit.xml"s, {}, {} } } );
}
SECTION("Mixed output files and default output") {
CHECK(cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }));
REQUIRE( config.reporterSpecifications ==
vec_Specs{ { "xml", "output.xml"s, {}, {} },
{ "console", {}, {}, {} } } );
}
SECTION("cannot have multiple reporters with default output") {
auto result = cli.parse({ "test", "-r", "console", "-r", "xml::out=output.xml", "-r", "junit" });
CHECK(!result);
REQUIRE_THAT(result.errorMessage(), ContainsSubstring("Only one reporter may have unspecified output file."));
}
}
}
SECTION("debugger") {
SECTION("-b") {
CHECK(cli.parse({"test", "-b"}));
REQUIRE(config.shouldDebugBreak == true);
}
SECTION("--break") {
CHECK(cli.parse({"test", "--break"}));
REQUIRE(config.shouldDebugBreak);
}
}
SECTION("abort") {
SECTION("-a aborts after first failure") {
CHECK(cli.parse({"test", "-a"}));
REQUIRE(config.abortAfter == 1);
}
SECTION("-x 2 aborts after two failures") {
CHECK(cli.parse({"test", "-x", "2"}));
REQUIRE(config.abortAfter == 2);
}
SECTION("-x must be numeric") {
auto result = cli.parse({"test", "-x", "oops"});
CHECK(!result);
REQUIRE_THAT(result.errorMessage(), ContainsSubstring("convert") && ContainsSubstring("oops"));
}
SECTION("wait-for-keypress") {
SECTION("Accepted options") {
using tuple_type = std::tuple<char const*, Catch::WaitForKeypress::When>;
auto input = GENERATE(table<char const*, Catch::WaitForKeypress::When>({
tuple_type{"never", Catch::WaitForKeypress::Never},
tuple_type{"start", Catch::WaitForKeypress::BeforeStart},
tuple_type{"exit", Catch::WaitForKeypress::BeforeExit},
tuple_type{"both", Catch::WaitForKeypress::BeforeStartAndExit},
}));
CHECK(cli.parse({"test", "--wait-for-keypress", std::get<0>(input)}));
REQUIRE(config.waitForKeypress == std::get<1>(input));
}
SECTION("invalid options are reported") {
auto result = cli.parse({"test", "--wait-for-keypress", "sometimes"});
CHECK(!result);
#ifndef CATCH_CONFIG_DISABLE_MATCHERS
REQUIRE_THAT(result.errorMessage(), ContainsSubstring("never") && ContainsSubstring("both"));
#endif
}
}
}
SECTION("nothrow") {
SECTION("-e") {
CHECK(cli.parse({"test", "-e"}));
REQUIRE(config.noThrow);
}
SECTION("--nothrow") {
CHECK(cli.parse({"test", "--nothrow"}));
REQUIRE(config.noThrow);
}
}
SECTION("output filename") {
SECTION("-o filename") {
CHECK(cli.parse({"test", "-o", "filename.ext"}));
REQUIRE(config.defaultOutputFilename == "filename.ext");
}
SECTION("--out") {
CHECK(cli.parse({"test", "--out", "filename.ext"}));
REQUIRE(config.defaultOutputFilename == "filename.ext");
}
}
SECTION("combinations") {
SECTION("Single character flags can be combined") {
CHECK(cli.parse({"test", "-abe"}));
CHECK(config.abortAfter == 1);
CHECK(config.shouldDebugBreak);
CHECK(config.noThrow == true);
}
}
SECTION( "use-colour") {
using Catch::ColourMode;
SECTION( "without option" ) {
CHECK(cli.parse({"test"}));
REQUIRE( config.defaultColourMode == ColourMode::PlatformDefault );
}
SECTION( "auto" ) {
CHECK( cli.parse( { "test", "--colour-mode", "default" } ) );
REQUIRE( config.defaultColourMode == ColourMode::PlatformDefault );
}
SECTION( "yes" ) {
CHECK(cli.parse({"test", "--colour-mode", "ansi"}));
REQUIRE( config.defaultColourMode == ColourMode::ANSI );
}
SECTION( "no" ) {
CHECK(cli.parse({"test", "--colour-mode", "none"}));
REQUIRE( config.defaultColourMode == ColourMode::None );
}
SECTION( "error" ) {
auto result = cli.parse({"test", "--colour-mode", "wrong"});
CHECK( !result );
CHECK_THAT( result.errorMessage(), ContainsSubstring( "colour mode must be one of" ) );
}
}
SECTION("Benchmark options") {
SECTION("samples") {
CHECK(cli.parse({ "test", "--benchmark-samples=200" }));
REQUIRE(config.benchmarkSamples == 200);
}
SECTION("samples must be greater than zero"){
auto result = cli.parse({"test", "--benchmark-samples=0"});
CHECK_FALSE(result);
REQUIRE_THAT(
result.errorMessage(),
ContainsSubstring("Benchmark samples must be greater than 0"));
}
SECTION("samples must be parseable") {
auto result = cli.parse({"test", "--benchmark-samples=abc"});
CHECK_FALSE(result);
REQUIRE_THAT(
result.errorMessage(),
ContainsSubstring("Could not parse 'abc' as benchmark samples"));
}
SECTION("resamples") {
CHECK(cli.parse({ "test", "--benchmark-resamples=20000" }));
REQUIRE(config.benchmarkResamples == 20000);
}
SECTION("confidence-interval") {
CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
REQUIRE(config.benchmarkConfidenceInterval == Catch::Approx(0.99));
}
SECTION("no-analysis") {
CHECK(cli.parse({ "test", "--benchmark-no-analysis" }));
REQUIRE(config.benchmarkNoAnalysis);
}
SECTION("warmup-time") {
CHECK(cli.parse({ "test", "--benchmark-warmup-time=10" }));
REQUIRE(config.benchmarkWarmupTime == 10);
}
}
}
TEST_CASE("Parsing sharding-related cli flags", "[sharding]") {
using namespace Catch::Matchers;
Catch::ConfigData config;
auto cli = Catch::makeCommandLineParser(config);
SECTION("shard-count") {
CHECK(cli.parse({ "test", "--shard-count=8" }));
REQUIRE(config.shardCount == 8);
}
SECTION("Negative shard count reports error") {
auto result = cli.parse({ "test", "--shard-count=-1" });
CHECK_FALSE(result);
REQUIRE_THAT(
result.errorMessage(),
ContainsSubstring( "Could not parse '-1' as shard count" ) );
}
SECTION("Zero shard count reports error") {
auto result = cli.parse({ "test", "--shard-count=0" });
CHECK_FALSE(result);
REQUIRE_THAT(
result.errorMessage(),
ContainsSubstring( "Shard count must be positive" ) );
}
SECTION("shard-index") {
CHECK(cli.parse({ "test", "--shard-index=2" }));
REQUIRE(config.shardIndex == 2);
}
SECTION("Negative shard index reports error") {
auto result = cli.parse({ "test", "--shard-index=-12" });
CHECK_FALSE(result);
REQUIRE_THAT(
result.errorMessage(),
ContainsSubstring( "Could not parse '-12' as shard index" ) );
}
SECTION("Shard index 0 is accepted") {
CHECK(cli.parse({ "test", "--shard-index=0" }));
REQUIRE(config.shardIndex == 0);
}
}
TEST_CASE( "Parsing warnings", "[cli][warnings]" ) {
using Catch::WarnAbout;
Catch::ConfigData config;
auto cli = Catch::makeCommandLineParser( config );
SECTION( "NoAssertions" ) {
REQUIRE(cli.parse( { "test", "-w", "NoAssertions" } ));
REQUIRE( config.warnings == WarnAbout::NoAssertions );
}
SECTION( "NoTests is no longer supported" ) {
REQUIRE_FALSE(cli.parse( { "test", "-w", "NoTests" } ));
}
SECTION( "Combining multiple warnings" ) {
REQUIRE( cli.parse( { "test",
"--warn", "NoAssertions",
"--warn", "UnmatchedTestSpec",
"--warn", "InfiniteGenerators" } ) );
REQUIRE( config.warnings == ( WarnAbout::NoAssertions | WarnAbout::UnmatchedTestSpec | WarnAbout::InfiniteGenerator ) );
}
}
TEST_CASE("Test with special, characters \"in name", "[cli][regression]") {
// This test case succeeds if we can invoke it from the CLI
SUCCEED();
}
TEST_CASE("Various suspicious reporter specs are rejected",
"[cli][reporter-spec][approvals]") {
Catch::ConfigData config;
auto cli = Catch::makeCommandLineParser( config );
auto spec = GENERATE( as<std::string>{},
"",
"::console",
"console::",
"console::some-file::",
"::console::some-file::" );
CAPTURE( spec );
auto result = cli.parse( { "test", "--reporter", spec } );
REQUIRE_FALSE( result );
}
TEST_CASE("Win32 colour implementation is compile-time optional",
"[approvals][cli][colours]") {
Catch::ConfigData config;
auto cli = Catch::makeCommandLineParser( config );
auto result = cli.parse( { "test", "--colour-mode", "win32" } );
#if defined( CATCH_CONFIG_COLOUR_WIN32 )
REQUIRE( result );
#else
REQUIRE_FALSE( result );
#endif
}
TEST_CASE( "Parse rng seed in different formats", "[approvals][cli][rng-seed]" ) {
Catch::ConfigData config;
auto cli = Catch::makeCommandLineParser( config );
SECTION("well formed cases") {
char const* seed_string;
uint32_t seed_value;
// GCC-5 workaround
using gen_type = std::tuple<char const*, uint32_t>;
std::tie( seed_string, seed_value ) = GENERATE( table<char const*, uint32_t>({
gen_type{ "0xBEEF", 0xBEEF },
gen_type{ "12345678", 12345678 }
} ) );
CAPTURE( seed_string );
auto result = cli.parse( { "tests", "--rng-seed", seed_string } );
REQUIRE( result );
REQUIRE( config.rngSeed == seed_value );
}
SECTION( "Error cases" ) {
auto seed_string =
GENERATE( "0xSEED", "999999999999", "08888", "BEEF", "123 456" );
CAPTURE( seed_string );
REQUIRE_FALSE( cli.parse( { "tests", "--rng-seed", seed_string } ) );
}
}
TEST_CASE( "Parsing path filter specs",
"[cli][section-spec]" ) {
using Catch::PathFilter;
Catch::ConfigData config;
auto cli = Catch::makeCommandLineParser( config );
SECTION( "Only section specs leads to old filter behaviour" ) {
auto result = cli.parse( { "tests", "-c", "1", "--section", "a section" } );
REQUIRE( result );
REQUIRE_FALSE( config.useNewPathFilteringBehaviour );
}
SECTION( "Generator specs enable new filter behaviour" ) {
auto result =
cli.parse( { "tests", "-g", "1", "--generator-index", "2" } );
REQUIRE( result );
REQUIRE( config.useNewPathFilteringBehaviour );
}
SECTION("Generator specs do not accept stringish arguments") {
auto result = cli.parse( { "tests", "--generator-index", "foo-baz" } );
REQUIRE_FALSE( result );
}
SECTION( "Generator specs accept star as argument" ) {
auto result = cli.parse( { "tests", "--generator-index", "*" } );
REQUIRE( result );
REQUIRE( config.pathFilters[0] ==
PathFilter( PathFilter::For::Generator, "*" ) );
}
SECTION( "Generator specs do not accept negative numbers" ) {
auto result = cli.parse( { "tests", "--generator-index", "-2" } );
REQUIRE_FALSE( result );
}
SECTION( "Generic path spec enables new filter behaviour" ) {
auto result =
cli.parse( { "tests", "-p", "g:1", "--path-filter", "c:foobar" } );
REQUIRE( result );
REQUIRE( config.useNewPathFilteringBehaviour );
REQUIRE( config.pathFilters[0] ==
PathFilter( PathFilter::For::Generator, "1" ) );
REQUIRE( config.pathFilters[1] ==
PathFilter( PathFilter::For::Section, "foobar" ) );
}
SECTION( "Generic path spec for generator is validated" ) {
auto result = cli.parse( { "tests", "-p", "g:foo-bar" } );
REQUIRE_FALSE( result );
}
SECTION( "Generic path spec without colon is rejected" ) {
auto result1 = cli.parse( { "tests", "-p", "g123" } );
REQUIRE_FALSE( result1 );
auto result2 = cli.parse( { "tests", "-p", "carp" } );
REQUIRE_FALSE( result2 );
}
SECTION( "Using both section and generator filters creates filter stack" ) {
auto result = cli.parse( { "tests",
"--section", "foo-bar",
"--generator-index", "3",
"-g", "123",
"-c", "baz"
});
REQUIRE( result );
REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) );
REQUIRE( config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) );
REQUIRE( config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) );
REQUIRE( config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) );
}
SECTION( "Section/generator filters are whitespace trimmed" ) {
auto result = cli.parse( { "tests",
"--section", " untrimmed ",
"--generator-index", " 42 "
});
REQUIRE( result );
REQUIRE( config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) );
REQUIRE( config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) );
}
}