From 10f01d5539110c2080b69aff90af6ca0d269f0df Mon Sep 17 00:00:00 2001 From: Antti Kaihola Date: Tue, 7 Aug 2018 10:51:49 +0300 Subject: [PATCH 1/6] Fix #5273: doctest: add :skipif: option for doctest directives This option allows conditional skipping of doctests. --- sphinx/ext/doctest.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py index 68877371c7b..bd1352b98dc 100644 --- a/sphinx/ext/doctest.py +++ b/sphinx/ext/doctest.py @@ -90,6 +90,16 @@ class TestDirective(SphinxDirective): def run(self): # type: () -> List[nodes.Node] + if 'skipif' in self.options: + condition = self.options['skipif'] + context = {} + if self.config.doctest_global_setup: + exec(self.config.doctest_global_setup, context) + should_skip = eval(condition, context) + if self.config.doctest_global_cleanup: + exec(self.config.doctest_global_cleanup, context) + if should_skip: + return [] # use ordinary docutils nodes for test code: they get special attributes # so that our builder recognizes them, and the other builders are happy. code = '\n'.join(self.content) @@ -155,11 +165,11 @@ def run(self): class TestsetupDirective(TestDirective): - option_spec = {} # type: Dict + option_spec = {'skipif': directives.unchanged_required} # type: Dict class TestcleanupDirective(TestDirective): - option_spec = {} # type: Dict + option_spec = {'skipif': directives.unchanged_required} # type: Dict class DoctestDirective(TestDirective): @@ -167,6 +177,7 @@ class DoctestDirective(TestDirective): 'hide': directives.flag, 'options': directives.unchanged, 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, } @@ -174,6 +185,7 @@ class TestcodeDirective(TestDirective): option_spec = { 'hide': directives.flag, 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, } @@ -182,6 +194,7 @@ class TestoutputDirective(TestDirective): 'hide': directives.flag, 'options': directives.unchanged, 'pyversion': directives.unchanged_required, + 'skipif': directives.unchanged_required, } From 9185c2c356afd3b829956513dc2e9cbbe7b44c46 Mon Sep 17 00:00:00 2001 From: Antti Kaihola Date: Thu, 16 Aug 2018 13:22:59 +0300 Subject: [PATCH 2/6] Ref #5273: doctest: add tests for the :skipif: option --- tests/roots/test-ext-doctest-skipif/conf.py | 10 +++ .../roots/test-ext-doctest-skipif/skipif.txt | 81 +++++++++++++++++++ tests/test_ext_doctest.py | 45 +++++++++++ 3 files changed, 136 insertions(+) create mode 100644 tests/roots/test-ext-doctest-skipif/conf.py create mode 100644 tests/roots/test-ext-doctest-skipif/skipif.txt diff --git a/tests/roots/test-ext-doctest-skipif/conf.py b/tests/roots/test-ext-doctest-skipif/conf.py new file mode 100644 index 00000000000..5f3e897861b --- /dev/null +++ b/tests/roots/test-ext-doctest-skipif/conf.py @@ -0,0 +1,10 @@ +extensions = ['sphinx.ext.doctest'] + +project = 'test project for the doctest :skipif: directive' +master_doc = 'skipif' +source_suffix = '.txt' +exclude_patterns = ['_build'] + +doctest_global_setup = ''' +from test_ext_doctest import record +''' \ No newline at end of file diff --git a/tests/roots/test-ext-doctest-skipif/skipif.txt b/tests/roots/test-ext-doctest-skipif/skipif.txt new file mode 100644 index 00000000000..c5bd3982180 --- /dev/null +++ b/tests/roots/test-ext-doctest-skipif/skipif.txt @@ -0,0 +1,81 @@ +Testing the doctest extension's `:skipif:` option +================================================= + +testsetup +--------- + +.. testsetup:: group-skipif + :skipif: record('testsetup', ':skipif:', True) != 'this will be True' + + record('testsetup', 'body', True) + +.. testsetup:: group-skipif + :skipif: record('testsetup', ':skipif:', False) == 'this will be False' + + record('testsetup', 'body', False) + + +doctest +------- +.. doctest:: group-skipif + :skipif: record('doctest', ':skipif:', True) != 'this will be True' + + >>> print(record('doctest', 'body', True)) + The test is skipped, and this expected text is ignored + + +.. doctest:: + :skipif: record('doctest', ':skipif:', False) == 'this will be False' + + >>> print(record('doctest', 'body', False)) + Recorded doctest body False + + +testcode and testoutput +----------------------- + +testcode skipped +~~~~~~~~~~~~~~~~ + +.. testcode:: group-skipif + :skipif: record('testcode', ':skipif:', True) != 'this will be True' + + print(record('testcode', 'body', True)) + +.. testoutput:: group-skipif + :skipif: record('testoutput-1', ':skipif:', True) != 'this will be True' + + The previous testcode is skipped, and the :skipif: condition is True, + so this testoutput is ignored + +testcode executed +~~~~~~~~~~~~~~~~~ + +.. testcode:: group-skipif + :skipif: record('testcode', ':skipif:', False) == 'this will be False' + + print(record('testcode', 'body', False)) + +.. testoutput:: group-skipif + :skipif: record('testoutput-2', ':skipif:', False) == 'this will be False' + + Recorded testcode body False + +.. testoutput:: group-skipif + :skipif: record('testoutput-2', ':skipif:', True) != 'this will be True' + + The :skipif: condition is False, so this testoutput is ignored + + +testcleanup +----------- + +.. testcleanup:: group-skipif + :skipif: record('testcleanup', ':skipif:', True) != 'this will be True' + + record('testcleanup', 'body', True) + +.. testcleanup:: group-skipif + :skipif: record('testcleanup', ':skipif:', False) == 'this will be False' + + record('testcleanup', 'body', False) diff --git a/tests/test_ext_doctest.py b/tests/test_ext_doctest.py index 7137af31bbd..5ae3c0f7aae 100644 --- a/tests/test_ext_doctest.py +++ b/tests/test_ext_doctest.py @@ -61,6 +61,51 @@ def cleanup_call(): cleanup_called += 1 +recorded_calls = set() + + +@pytest.mark.sphinx('doctest', testroot='ext-doctest-skipif') +def test_skipif(app, status, warning): + """Tests for the :skipif: option + + The tests are separated into a different test root directory since the + ``app`` object only evaluates options once in its lifetime. If these tests + were combined with the other doctest tests, the ``:skipif:`` evaluations + would be recorded only on the first ``app.builder.build_all()`` run, i.e. + in ``test_build`` above, and the assertion below would fail. + + """ + global recorded_calls + recorded_calls = set() + app.builder.build_all() + if app.statuscode != 0: + assert False, 'failures in doctests:' + status.getvalue() + # The `:skipif:` expressions are always run. + # Actual tests and setup/cleanup code is only run if the `:skipif:` + # expression evaluates to a False value. + assert recorded_calls == {('testsetup', ':skipif:', True), + ('testsetup', ':skipif:', False), + ('testsetup', 'body', False), + ('doctest', ':skipif:', True), + ('doctest', ':skipif:', False), + ('doctest', 'body', False), + ('testcode', ':skipif:', True), + ('testcode', ':skipif:', False), + ('testcode', 'body', False), + ('testoutput-1', ':skipif:', True), + ('testoutput-2', ':skipif:', True), + ('testoutput-2', ':skipif:', False), + ('testcleanup', ':skipif:', True), + ('testcleanup', ':skipif:', False), + ('testcleanup', 'body', False)} + + +def record(directive, part, should_skip): + global recorded_calls + recorded_calls.add((directive, part, should_skip)) + return 'Recorded {} {} {}'.format(directive, part, should_skip) + + @pytest.mark.xfail( PY2, reason='node.source points to document instead of filename', ) From 43d5810083de3a0c5b595ea4f937e792b101dc16 Mon Sep 17 00:00:00 2001 From: Antti Kaihola Date: Thu, 16 Aug 2018 14:01:57 +0300 Subject: [PATCH 3/6] Ref #5273: doctest: add documentation for the :skipif: option --- doc/usage/extensions/doctest.rst | 75 ++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/doc/usage/extensions/doctest.rst b/doc/usage/extensions/doctest.rst index 341b7a9a705..e321e3a5448 100644 --- a/doc/usage/extensions/doctest.rst +++ b/doc/usage/extensions/doctest.rst @@ -198,6 +198,81 @@ The following is an example for the usage of the directives. The test via This parrot wouldn't voom if you put 3000 volts through it! +Skipping tests conditionally +---------------------------- + +``skipif``, a string option, can be used to skip directives conditionally. This +may be useful e.g. when a different set of tests should be run depending on the +environment (hardware, network/VPN, optional dependencies or different versions +of dependencies). The ``skipif`` option is supported by all of the doctest +directives. Below are typical use cases for ``skipif`` when used for different +directives: + +- :rst:dir:`testsetup` and :rst:dir:`testcleanup` + + - conditionally skip test setup and/or cleanup + - customize setup/cleanup code per environment + +- :rst:dir:`doctest` + + - conditionally skip both a test and its output verification + +- :rst:dir:`testcode` + + - conditionally skip a test + - customize test code per environment + +- :rst:dir:`testoutput` + + - conditionally skip output assertion for a skipped test + - expect different output depending on the environment + +The value of the ``skipif`` option is evaluated as a Python expression. If the +result is a true value, the directive is omitted from the test run just as if +it wasn't present in the file at all. + +Instead of repeating an expression, the :confval:`doctest_global_setup` +configuration option can be used to assign it to a variable which can then be +used instead. + +Here's an example which skips some tests if Pandas is not installed: + +.. code-block:: py + :caption: conf.py + + extensions = ['sphinx.ext.doctest'] + doctest_global_setup = ''' + try: + import pandas as pd + except ImportError: + pd = None + ''' + +.. code-block:: rst + :caption: contents.rst + + .. testsetup:: + :skipif: pd is None + + data = pd.Series([42]) + + .. doctest:: + :skipif: pd is None + + >>> data.iloc[0] + 42 + + .. testcode:: + :skipif: pd is None + + print(data.iloc[-1]) + + .. testoutput:: + :skipif: pd is None + + 42 + + Configuration ------------- From 5ffc924940a6dfbfef84e02cb07691b5630517e2 Mon Sep 17 00:00:00 2001 From: Antti Kaihola Date: Thu, 16 Aug 2018 14:09:06 +0300 Subject: [PATCH 4/6] Ref #5273: doctest: added a missing final newline --- tests/roots/test-ext-doctest-skipif/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/roots/test-ext-doctest-skipif/conf.py b/tests/roots/test-ext-doctest-skipif/conf.py index 5f3e897861b..d6be0585a0a 100644 --- a/tests/roots/test-ext-doctest-skipif/conf.py +++ b/tests/roots/test-ext-doctest-skipif/conf.py @@ -7,4 +7,4 @@ doctest_global_setup = ''' from test_ext_doctest import record -''' \ No newline at end of file +''' From d52488f028a583af0d0c921ef034ebd37b707aa7 Mon Sep 17 00:00:00 2001 From: Antti Kaihola Date: Thu, 16 Aug 2018 14:56:53 +0300 Subject: [PATCH 5/6] Ref #5273: doctest: added a missing type hint --- sphinx/ext/doctest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py index bd1352b98dc..fa6cbed6ddb 100644 --- a/sphinx/ext/doctest.py +++ b/sphinx/ext/doctest.py @@ -92,7 +92,7 @@ def run(self): # type: () -> List[nodes.Node] if 'skipif' in self.options: condition = self.options['skipif'] - context = {} + context = {} # type: Dict[str, Any] if self.config.doctest_global_setup: exec(self.config.doctest_global_setup, context) should_skip = eval(condition, context) From 098f37fddf2f4239a4387e18d5d78df814faf925 Mon Sep 17 00:00:00 2001 From: Antti Kaihola Date: Thu, 16 Aug 2018 17:56:29 +0300 Subject: [PATCH 6/6] Ref #5273: doctest: add test for skipif calling global setup/cleanup Ensure that `doctest_global_setup` and `doctest_global_cleanup` are executed before and after evaluating each `:skipif:` option. --- tests/roots/test-ext-doctest-skipif/conf.py | 6 +++ tests/test_ext_doctest.py | 42 ++++++++++++--------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/tests/roots/test-ext-doctest-skipif/conf.py b/tests/roots/test-ext-doctest-skipif/conf.py index d6be0585a0a..dd94b7049c5 100644 --- a/tests/roots/test-ext-doctest-skipif/conf.py +++ b/tests/roots/test-ext-doctest-skipif/conf.py @@ -7,4 +7,10 @@ doctest_global_setup = ''' from test_ext_doctest import record + +record('doctest_global_setup', 'body', True) ''' + +doctest_global_cleanup = ''' +record('doctest_global_cleanup', 'body', True) +''' \ No newline at end of file diff --git a/tests/test_ext_doctest.py b/tests/test_ext_doctest.py index 5ae3c0f7aae..f7b6ca5f8fb 100644 --- a/tests/test_ext_doctest.py +++ b/tests/test_ext_doctest.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for details. """ import os +from collections import Counter import pytest from packaging.specifiers import InvalidSpecifier @@ -61,7 +62,7 @@ def cleanup_call(): cleanup_called += 1 -recorded_calls = set() +recorded_calls = Counter() @pytest.mark.sphinx('doctest', testroot='ext-doctest-skipif') @@ -76,33 +77,38 @@ def test_skipif(app, status, warning): """ global recorded_calls - recorded_calls = set() + recorded_calls = Counter() app.builder.build_all() if app.statuscode != 0: assert False, 'failures in doctests:' + status.getvalue() # The `:skipif:` expressions are always run. # Actual tests and setup/cleanup code is only run if the `:skipif:` # expression evaluates to a False value. - assert recorded_calls == {('testsetup', ':skipif:', True), - ('testsetup', ':skipif:', False), - ('testsetup', 'body', False), - ('doctest', ':skipif:', True), - ('doctest', ':skipif:', False), - ('doctest', 'body', False), - ('testcode', ':skipif:', True), - ('testcode', ':skipif:', False), - ('testcode', 'body', False), - ('testoutput-1', ':skipif:', True), - ('testoutput-2', ':skipif:', True), - ('testoutput-2', ':skipif:', False), - ('testcleanup', ':skipif:', True), - ('testcleanup', ':skipif:', False), - ('testcleanup', 'body', False)} + # Global setup/cleanup are run before/after evaluating the `:skipif:` + # option in each directive - thus 11 additional invocations for each on top + # of the ones made for the whole test file. + assert recorded_calls == {('doctest_global_setup', 'body', True): 13, + ('testsetup', ':skipif:', True): 1, + ('testsetup', ':skipif:', False): 1, + ('testsetup', 'body', False): 1, + ('doctest', ':skipif:', True): 1, + ('doctest', ':skipif:', False): 1, + ('doctest', 'body', False): 1, + ('testcode', ':skipif:', True): 1, + ('testcode', ':skipif:', False): 1, + ('testcode', 'body', False): 1, + ('testoutput-1', ':skipif:', True): 1, + ('testoutput-2', ':skipif:', True): 1, + ('testoutput-2', ':skipif:', False): 1, + ('testcleanup', ':skipif:', True): 1, + ('testcleanup', ':skipif:', False): 1, + ('testcleanup', 'body', False): 1, + ('doctest_global_cleanup', 'body', True): 13} def record(directive, part, should_skip): global recorded_calls - recorded_calls.add((directive, part, should_skip)) + recorded_calls[(directive, part, should_skip)] += 1 return 'Recorded {} {} {}'.format(directive, part, should_skip)