From c567ac6c5431d9d2867ace1da1418f4575215516 Mon Sep 17 00:00:00 2001 From: vladlosev Date: Thu, 25 Apr 2013 17:58:52 +0000 Subject: When --gtest_filter is specified, XML report now doesn't contain information about tests that are filtered out (issue 141). git-svn-id: http://googletest.googlecode.com/svn/trunk@654 861a406c-534a-0410-8894-cb66d6ee9925 --- CHANGES | 2 ++ include/gtest/gtest.h | 37 ++++++++++++++++++++-- src/gtest-internal-inl.h | 6 ++++ src/gtest.cc | 59 ++++++++++++++++++++++++++++------- test/gtest_output_test_golden_lin.txt | 5 --- test/gtest_xml_output_unittest.py | 35 +++++++++++++++++---- test/gtest_xml_test_utils.py | 8 +++-- 7 files changed, 123 insertions(+), 29 deletions(-) diff --git a/CHANGES b/CHANGES index f2968d6..1cfe07a 100644 --- a/CHANGES +++ b/CHANGES @@ -12,6 +12,8 @@ Changes for 1.7.0: * Improvement: failure summary in XML reports now includes file and line information. * Improvement: the XML element now has a timestamp attribute. +* Improvement: When --gtest_filter is specified, XML report now doesn't + contain information about tests that are filtered out. * Fixed the bug where long --gtest_filter flag values are truncated in death tests. * Potentially breaking change: RUN_ALL_TESTS() is now implemented as a diff --git a/include/gtest/gtest.h b/include/gtest/gtest.h index 751aa2e..6fa0a39 100644 --- a/include/gtest/gtest.h +++ b/include/gtest/gtest.h @@ -646,9 +646,9 @@ class GTEST_API_ TestInfo { return NULL; } - // Returns true if this test should run, that is if the test is not disabled - // (or it is disabled but the also_run_disabled_tests flag has been specified) - // and its full name matches the user-specified filter. + // Returns true if this test should run, that is if the test is not + // disabled (or it is disabled but the also_run_disabled_tests flag has + // been specified) and its full name matches the user-specified filter. // // Google Test allows the user to filter the tests by their full names. // The full name of a test Bar in test case Foo is defined as @@ -664,6 +664,14 @@ class GTEST_API_ TestInfo { // contains the character 'A' or starts with "Foo.". bool should_run() const { return should_run_; } + // Returns true iff this test will appear in the XML report. + bool is_reportable() const { + // For now, the XML report includes all tests matching the filter. + // In the future, we may trim tests that are excluded because of + // sharding. + return matches_filter_; + } + // Returns the result of the test. const TestResult* result() const { return &result_; } @@ -776,9 +784,15 @@ class GTEST_API_ TestCase { // Gets the number of failed tests in this test case. int failed_test_count() const; + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + // Gets the number of disabled tests in this test case. int disabled_test_count() const; + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + // Get the number of tests in this test case that should run. int test_to_run_count() const; @@ -854,11 +868,22 @@ class GTEST_API_ TestCase { return test_info->should_run() && test_info->result()->Failed(); } + // Returns true iff the test is disabled and will be reported in the XML + // report. + static bool TestReportableDisabled(const TestInfo* test_info) { + return test_info->is_reportable() && test_info->is_disabled_; + } + // Returns true iff test is disabled. static bool TestDisabled(const TestInfo* test_info) { return test_info->is_disabled_; } + // Returns true iff this test will appear in the XML report. + static bool TestReportable(const TestInfo* test_info) { + return test_info->is_reportable(); + } + // Returns true if the given test should run. static bool ShouldRunTest(const TestInfo* test_info) { return test_info->should_run(); @@ -1151,9 +1176,15 @@ class GTEST_API_ UnitTest { // Gets the number of failed tests. int failed_test_count() const; + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + // Gets the number of disabled tests. int disabled_test_count() const; + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + // Gets the number of all tests. int total_test_count() const; diff --git a/src/gtest-internal-inl.h b/src/gtest-internal-inl.h index 302b6cd..35df303 100644 --- a/src/gtest-internal-inl.h +++ b/src/gtest-internal-inl.h @@ -550,9 +550,15 @@ class GTEST_API_ UnitTestImpl { // Gets the number of failed tests. int failed_test_count() const; + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + // Gets the number of disabled tests. int disabled_test_count() const; + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + // Gets the number of all tests. int total_test_count() const; diff --git a/src/gtest.cc b/src/gtest.cc index 6b9d75d..57719aa 100644 --- a/src/gtest.cc +++ b/src/gtest.cc @@ -731,11 +731,22 @@ int UnitTestImpl::failed_test_count() const { return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count); } +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTestImpl::reportable_disabled_test_count() const { + return SumOverTestCaseList(test_cases_, + &TestCase::reportable_disabled_test_count); +} + // Gets the number of disabled tests. int UnitTestImpl::disabled_test_count() const { return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count); } +// Gets the number of tests to be printed in the XML report. +int UnitTestImpl::reportable_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::reportable_test_count); +} + // Gets the number of all tests. int UnitTestImpl::total_test_count() const { return SumOverTestCaseList(test_cases_, &TestCase::total_test_count); @@ -2338,10 +2349,21 @@ int TestCase::failed_test_count() const { return CountIf(test_info_list_, TestFailed); } +// Gets the number of disabled tests that will be reported in the XML report. +int TestCase::reportable_disabled_test_count() const { + return CountIf(test_info_list_, TestReportableDisabled); +} + +// Gets the number of disabled tests in this test case. int TestCase::disabled_test_count() const { return CountIf(test_info_list_, TestDisabled); } +// Gets the number of tests to be printed in the XML report. +int TestCase::reportable_test_count() const { + return CountIf(test_info_list_, TestReportable); +} + // Get the number of tests in this test case that should run. int TestCase::test_to_run_count() const { return CountIf(test_info_list_, ShouldRunTest); @@ -2851,7 +2873,7 @@ void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, num_failures == 1 ? "TEST" : "TESTS"); } - int num_disabled = unit_test.disabled_test_count(); + int num_disabled = unit_test.reportable_disabled_test_count(); if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { if (!num_failures) { printf("\n"); // Add a spacer if no FAILURE banner is displayed. @@ -3310,19 +3332,22 @@ void XmlUnitTestResultPrinter::PrintXmlTestCase(std::ostream* stream, *stream << " <" << kTestsuite; OutputXmlAttribute(stream, kTestsuite, "name", test_case.name()); OutputXmlAttribute(stream, kTestsuite, "tests", - StreamableToString(test_case.total_test_count())); + StreamableToString(test_case.reportable_test_count())); OutputXmlAttribute(stream, kTestsuite, "failures", StreamableToString(test_case.failed_test_count())); - OutputXmlAttribute(stream, kTestsuite, "disabled", - StreamableToString(test_case.disabled_test_count())); + OutputXmlAttribute( + stream, kTestsuite, "disabled", + StreamableToString(test_case.reportable_disabled_test_count())); OutputXmlAttribute(stream, kTestsuite, "errors", "0"); OutputXmlAttribute(stream, kTestsuite, "time", FormatTimeInMillisAsSeconds(test_case.elapsed_time())); *stream << TestPropertiesAsXmlAttributes(test_case.ad_hoc_test_result()) << ">\n"; - for (int i = 0; i < test_case.total_test_count(); ++i) - OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i)); + for (int i = 0; i < test_case.total_test_count(); ++i) { + if (test_case.GetTestInfo(i)->is_reportable()) + OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i)); + } *stream << " \n"; } @@ -3335,11 +3360,12 @@ void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream, *stream << "<" << kTestsuites; OutputXmlAttribute(stream, kTestsuites, "tests", - StreamableToString(unit_test.total_test_count())); + StreamableToString(unit_test.reportable_test_count())); OutputXmlAttribute(stream, kTestsuites, "failures", StreamableToString(unit_test.failed_test_count())); - OutputXmlAttribute(stream, kTestsuites, "disabled", - StreamableToString(unit_test.disabled_test_count())); + OutputXmlAttribute( + stream, kTestsuites, "disabled", + StreamableToString(unit_test.reportable_disabled_test_count())); OutputXmlAttribute(stream, kTestsuites, "errors", "0"); OutputXmlAttribute( stream, kTestsuites, "timestamp", @@ -3357,9 +3383,9 @@ void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream, OutputXmlAttribute(stream, kTestsuites, "name", "AllTests"); *stream << ">\n"; - for (int i = 0; i < unit_test.total_test_case_count(); ++i) { - PrintXmlTestCase(stream, *unit_test.GetTestCase(i)); + if (unit_test.GetTestCase(i)->reportable_test_count() > 0) + PrintXmlTestCase(stream, *unit_test.GetTestCase(i)); } *stream << "\n"; } @@ -3629,11 +3655,21 @@ int UnitTest::successful_test_count() const { // Gets the number of failed tests. int UnitTest::failed_test_count() const { return impl()->failed_test_count(); } +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTest::reportable_disabled_test_count() const { + return impl()->reportable_disabled_test_count(); +} + // Gets the number of disabled tests. int UnitTest::disabled_test_count() const { return impl()->disabled_test_count(); } +// Gets the number of tests to be printed in the XML report. +int UnitTest::reportable_test_count() const { + return impl()->reportable_test_count(); +} + // Gets the number of all tests. int UnitTest::total_test_count() const { return impl()->total_test_count(); } @@ -3929,7 +3965,6 @@ UnitTestImpl::UnitTestImpl(UnitTest* parent) start_timestamp_(0), elapsed_time_(0), #if GTEST_HAS_DEATH_TEST - internal_run_death_test_flag_(NULL), death_test_factory_(new DefaultDeathTestFactory), #endif // Will be overridden by the flag before first use. diff --git a/test/gtest_output_test_golden_lin.txt b/test/gtest_output_test_golden_lin.txt index 0e26d63..960eedc 100644 --- a/test/gtest_output_test_golden_lin.txt +++ b/test/gtest_output_test_golden_lin.txt @@ -699,8 +699,6 @@ Expected: (3) >= (a[i]), actual: 3 vs 6 [ FAILED ] LoggingTest.InterleavingLoggingAndAssertions 4 FAILED TESTS - YOU HAVE 1 DISABLED TEST - Note: Google Test filter = *DISABLED_* [==========] Running 1 test from 1 test case. [----------] Global test environment set-up. @@ -720,6 +718,3 @@ Note: This is test shard 2 of 2. [----------] Global test environment tear-down [==========] 1 test from 1 test case ran. [ PASSED ] 1 test. - - YOU HAVE 1 DISABLED TEST - diff --git a/test/gtest_xml_output_unittest.py b/test/gtest_xml_output_unittest.py index 3d794e6..f605d4e 100755 --- a/test/gtest_xml_output_unittest.py +++ b/test/gtest_xml_output_unittest.py @@ -44,6 +44,7 @@ import gtest_test_utils import gtest_xml_test_utils +GTEST_FILTER_FLAG = '--gtest_filter' GTEST_LIST_TESTS_FLAG = '--gtest_list_tests' GTEST_OUTPUT_FLAG = "--gtest_output" GTEST_DEFAULT_OUTPUT_FILE = "test_detail.xml" @@ -128,9 +129,18 @@ Invalid characters in brackets []%(stack)s]]> """ % {'stack': STACK_TRACE_TEMPLATE} +EXPECTED_FILTERED_TEST_XML = """ + + + + +""" EXPECTED_EMPTY_XML = """ - + """ GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME) @@ -169,7 +179,7 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): Runs a test program that generates an empty XML output, and checks if the timestamp attribute in the testsuites tag is valid. """ - actual = self._GetXmlOutput('gtest_no_test_unittest', 0) + actual = self._GetXmlOutput('gtest_no_test_unittest', [], 0) date_time_str = actual.documentElement.getAttributeNode('timestamp').value # datetime.strptime() is only available in Python 2.5+ so we have to # parse the expected datetime manually. @@ -239,7 +249,17 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): self.assert_(not os.path.isfile(xml_path)) - def _GetXmlOutput(self, gtest_prog_name, expected_exit_code): + def testFilteredTestXmlOutput(self): + """Verifies XML output when a filter is applied. + + Runs a test program that executes only some tests and verifies that + non-selected tests do not show up in the XML output. + """ + + self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED_TEST_XML, 0, + extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG]) + + def _GetXmlOutput(self, gtest_prog_name, extra_args, expected_exit_code): """ Returns the xml output generated by running the program gtest_prog_name. Furthermore, the program's exit code must be expected_exit_code. @@ -248,7 +268,8 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): gtest_prog_name + 'out.xml') gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name) - command = [gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)] + command = ([gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)] + + extra_args) p = gtest_test_utils.Subprocess(command) if p.terminated_by_signal: self.assert_(False, @@ -262,7 +283,8 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): actual = minidom.parse(xml_path) return actual - def _TestXmlOutput(self, gtest_prog_name, expected_xml, expected_exit_code): + def _TestXmlOutput(self, gtest_prog_name, expected_xml, + expected_exit_code, extra_args=None): """ Asserts that the XML document generated by running the program gtest_prog_name matches expected_xml, a string containing another @@ -270,7 +292,8 @@ class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): expected_exit_code. """ - actual = self._GetXmlOutput(gtest_prog_name, expected_exit_code) + actual = self._GetXmlOutput(gtest_prog_name, extra_args or [], + expected_exit_code) expected = minidom.parseString(expected_xml) self.NormalizeXml(actual.documentElement) self.AssertEquivalentNodes(expected.documentElement, diff --git a/test/gtest_xml_test_utils.py b/test/gtest_xml_test_utils.py index 35458f8..3d0c3b2 100755 --- a/test/gtest_xml_test_utils.py +++ b/test/gtest_xml_test_utils.py @@ -90,9 +90,11 @@ class GTestXMLTestCase(gtest_test_utils.TestCase): actual_attr is not None, 'expected attribute %s not found in element %s' % (expected_attr.name, actual_node.tagName)) - self.assertEquals(expected_attr.value, actual_attr.value, - ' values of attribute %s in element %s differ' % - (expected_attr.name, actual_node.tagName)) + self.assertEquals( + expected_attr.value, actual_attr.value, + ' values of attribute %s in element %s differ: %s vs %s' % + (expected_attr.name, actual_node.tagName, + expected_attr.value, actual_attr.value)) expected_children = self._GetChildren(expected_node) actual_children = self._GetChildren(actual_node) -- cgit v1.2.3