From 0dc65250fea7d92d66ec6fa91e1d7dd51649cf93 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 08:19:52 -0600 Subject: [PATCH 01/16] more coverage --- .github/workflows/python-tests.yml | 1 - pyproject.toml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 192a698e97..7d8da74b42 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -37,7 +37,6 @@ jobs: sudo apt-get update sudo apt-get install texlive-plain-generic inkscape texlive-xetex sudo apt-get install xvfb x11-utils libxkbcommon-x11-0 - pip install pandoc - name: Run the tests if: ${{ !startsWith(matrix.python-version, 'pypy') && !startsWith(matrix.os, 'windows') }} run: hatch run cov:test -W default || hatch run cov:test -W default --lf diff --git a/pyproject.toml b/pyproject.toml index 9a5ad677e0..b45c352352 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ Tracker = "https://github.com/jupyter-server/jupyter_server/issues" [project.optional-dependencies] test = [ "ipykernel", + "pandoc", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", From a395ea34ba130c0360675a5473fa253cc3f50dd7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 08:21:35 -0600 Subject: [PATCH 02/16] add cov report --- pyproject.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b45c352352..9d463ba7ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,6 +146,20 @@ filterwarnings = [ "module:Jupyter is migrating its paths to use standard platformdirs:DeprecationWarning", ] +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:", + "class .*\bProtocol\\):", +"@(abc\\.)?abstractmethod", +] + [tool.jupyter-releaser.hooks] before-build-python = ["npm install", "npm run build"] From 5b0cee8557ab3ee3e51393aaa4406eed903e26c3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 08:31:17 -0600 Subject: [PATCH 03/16] actually install pandoc --- .github/workflows/python-tests.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 7d8da74b42..b24f9d35f4 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -36,7 +36,7 @@ jobs: run: | sudo apt-get update sudo apt-get install texlive-plain-generic inkscape texlive-xetex - sudo apt-get install xvfb x11-utils libxkbcommon-x11-0 + sudo apt-get install xvfb x11-utils libxkbcommon-x11-0 pandoc - name: Run the tests if: ${{ !startsWith(matrix.python-version, 'pypy') && !startsWith(matrix.os, 'windows') }} run: hatch run cov:test -W default || hatch run cov:test -W default --lf @@ -70,20 +70,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - - uses: pre-commit/action@v3.0.0 - with: - extra_args: --all-files --hook-stage=manual - - name: Help message if pre-commit fail - if: ${{ failure() }} - run: | - echo "You can install pre-commit hooks to automatically run formatting" - echo "on each commit with:" - echo " pre-commit install" - echo "or you can run by hand on staged files with" - echo " pre-commit run" - echo "or after-the-fact on already committed files with" - echo " pre-commit run --all-files --hook-stage=manual" + - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + - uses: jupyterlab/maintainer-tools/.github/actions/pre-commit@v1 test_docs: name: Test Docs From 85dfb36d7edc170c11e3e21da4cbc321eac0c0e3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 08:38:21 -0600 Subject: [PATCH 04/16] bump cov-fail --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9d463ba7ee..747b6aa577 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ Tracker = "https://github.com/jupyter-server/jupyter_server/issues" [project.optional-dependencies] test = [ "ipykernel", - "pandoc", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", @@ -105,7 +104,7 @@ dependencies = ["coverage", "pytest-cov"] [tool.hatch.envs.cov.env-vars] ARGS = "-vv --cov jupyter_server --cov-branch --cov-report term-missing:skip-covered" [tool.hatch.envs.cov.scripts] -test = "python -m pytest $ARGS --cov-fail-under 70 {args}" +test = "python -m pytest $ARGS --cov-fail-under 75 {args}" [tool.hatch.version] path = "jupyter_server/_version.py" From 5fe9f1c01f7219f4d01570e2cf587bac9bb8bf25 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:13:43 -0600 Subject: [PATCH 05/16] more coverage --- .github/workflows/integration-tests.yml | 3 +- jupyter_server/pytest_plugin.py | 5 +- jupyter_server/serverapp.py | 1 - pyproject.toml | 1 + tests/test_utils.py | 62 ++++++++++++++++++- .../test_serverapp_integration.py | 21 +++++++ 6 files changed, 89 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4aa90c158d..57693a166b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -27,7 +27,8 @@ jobs: pip check - name: Run the tests run: | - pytest -vv --integration_tests=true tests + hatch run cov:integration + integration_check: # This job does nothing and is only used for the branch protection if: always() diff --git a/jupyter_server/pytest_plugin.py b/jupyter_server/pytest_plugin.py index b7725e40f2..ec93d9db87 100644 --- a/jupyter_server/pytest_plugin.py +++ b/jupyter_server/pytest_plugin.py @@ -522,7 +522,10 @@ def inner(nbpath): def jp_server_cleanup(asyncio_loop): yield app: ServerApp = ServerApp.instance() - asyncio_loop.run_until_complete(app._cleanup()) + try: + asyncio_loop.run_until_complete(app._cleanup()) + except RuntimeError: + pass ServerApp.clear_instance() diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 865f797b17..b069512195 100644 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -510,7 +510,6 @@ def shutdown_server(server_info, timeout=5, log=None): url = server_info["url"] pid = server_info["pid"] - try: shutdown_url = urljoin(url, "api/shutdown") if log: diff --git a/pyproject.toml b/pyproject.toml index 747b6aa577..38a4f3fc99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,6 +105,7 @@ dependencies = ["coverage", "pytest-cov"] ARGS = "-vv --cov jupyter_server --cov-branch --cov-report term-missing:skip-covered" [tool.hatch.envs.cov.scripts] test = "python -m pytest $ARGS --cov-fail-under 75 {args}" +integration = "python -m pytest $ARGS --integration_tests=true {args}" [tool.hatch.version] path = "jupyter_server/_version.py" diff --git a/tests/test_utils.py b/tests/test_utils.py index 5c5d16e73d..85e38dfd71 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,10 +1,27 @@ +import os +import socket +import subprocess +import sys +import warnings from pathlib import Path from unittest.mock import patch import pytest from traitlets.tests.utils import check_help_all_output -from jupyter_server.utils import is_namespace_package, url_escape, url_unescape +from jupyter_server.utils import ( + check_pid, + check_version, + is_namespace_package, + path2url, + run_sync_in_loop, + samefile_simple, + to_api_path, + unix_socket_in_use, + url2path, + url_escape, + url_unescape, +) def test_help_output(): @@ -59,3 +76,46 @@ def test_is_namespace_package_no_spec(): assert is_namespace_package("dummy") is None mocked_spec.assert_called_once_with("dummy") + + +def test_path_utils(tmp_path): + path = str(tmp_path) + assert os.path.basename(path2url(path)) == os.path.basename(path) + + url = path2url(path) + assert path.endswith(url2path(url)) + + assert samefile_simple(path, path) + + assert to_api_path(path, os.path.dirname(path)) == os.path.basename(path) + + +def test_check_version(): + assert check_version("1.0.2", "1.0.1") + assert not check_version("1.0.0", "1.0.1") + assert check_version(1.0, "1.0.1") + + +def test_check_pid(): + proc = subprocess.Popen([sys.executable]) + proc.kill() + proc.wait() + check_pid(proc.pid) + + +async def test_run_sync_in_loop(): + async def foo(): + pass + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + await run_sync_in_loop(foo()) + + +@pytest.mark.skipif(sys.platform != "linux", reason="Requires unix sockets") +def test_unix_socket_in_use(tmp_path): + server_address = str(tmp_path) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(server_address) + assert unix_socket_in_use(server_address) + sock.close() diff --git a/tests/unix_sockets/test_serverapp_integration.py b/tests/unix_sockets/test_serverapp_integration.py index 9661539d7e..a8bfc8ac64 100644 --- a/tests/unix_sockets/test_serverapp_integration.py +++ b/tests/unix_sockets/test_serverapp_integration.py @@ -3,6 +3,8 @@ import pytest +from jupyter_server.serverapp import list_running_servers, shutdown_server + # Skip this module if on Windows. Unix sockets are not available on Windows. pytestmark = pytest.mark.skipif( sys.platform.startswith("win"), reason="Unix sockets are not available on Windows." @@ -171,3 +173,22 @@ def test_launch_socket_collision(jp_unix_socket_file): _ensure_stopped() _cleanup_process(p1) + + +@pytest.mark.integration_test +def test_shutdown_server(jp_environ): + # Start a server in another process + # Stop that server + import subprocess + + from jupyter_client.connect import LocalPortCache + + port = LocalPortCache().find_available_port("localhost") + p = subprocess.Popen(["jupyter-server", f"--port={port}"]) + servers = [] + while 1: + servers = list(list_running_servers()) + if len(servers): + break + shutdown_server(servers[0]) + p.wait() From ad09347db16fc43478d13ed4c2fda960be89683d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:15:31 -0600 Subject: [PATCH 06/16] undo change to cleanup --- jupyter_server/pytest_plugin.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/jupyter_server/pytest_plugin.py b/jupyter_server/pytest_plugin.py index ec93d9db87..b7725e40f2 100644 --- a/jupyter_server/pytest_plugin.py +++ b/jupyter_server/pytest_plugin.py @@ -522,10 +522,7 @@ def inner(nbpath): def jp_server_cleanup(asyncio_loop): yield app: ServerApp = ServerApp.instance() - try: - asyncio_loop.run_until_complete(app._cleanup()) - except RuntimeError: - pass + asyncio_loop.run_until_complete(app._cleanup()) ServerApp.clear_instance() From d4886ef245ea307cc7d08a47dbe412559810f509 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:20:29 -0600 Subject: [PATCH 07/16] handle race condition --- jupyter_server/serverapp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index b069512195..b39bb097d6 100644 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -2890,7 +2890,11 @@ def list_running_servers(runtime_dir=None, log=None): for file_name in os.listdir(runtime_dir): if re.match("jpserver-(.+).json", file_name): with open(os.path.join(runtime_dir, file_name), encoding="utf-8") as f: - info = json.load(f) + # Handle race condition where file is being written. + try: + info = json.load(f) + except json.JSONDecodeError: + continue # Simple check whether that process is really still running # Also remove leftover files from IPython 2.x without a pid field From a6eed46a53ade54252e3609884c27c6c2c3d405f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:32:48 -0600 Subject: [PATCH 08/16] fixups --- tests/test_utils.py | 1 + tests/unix_sockets/test_serverapp_integration.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 85e38dfd71..16a79e87dd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -114,6 +114,7 @@ async def foo(): @pytest.mark.skipif(sys.platform != "linux", reason="Requires unix sockets") def test_unix_socket_in_use(tmp_path): + os.unlink(str(tmp_path)) server_address = str(tmp_path) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.bind(server_address) diff --git a/tests/unix_sockets/test_serverapp_integration.py b/tests/unix_sockets/test_serverapp_integration.py index a8bfc8ac64..7a66ccac31 100644 --- a/tests/unix_sockets/test_serverapp_integration.py +++ b/tests/unix_sockets/test_serverapp_integration.py @@ -1,5 +1,6 @@ import stat import sys +import time import pytest @@ -190,5 +191,11 @@ def test_shutdown_server(jp_environ): servers = list(list_running_servers()) if len(servers): break - shutdown_server(servers[0]) + time.sleep(0.1) + while 1: + try: + shutdown_server(servers[0]) + break + except ConnectionRefusedError: + time.sleep(0.1) p.wait() From acd207e704781c2e1337dca627d62bb7b64a2c4e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:36:40 -0600 Subject: [PATCH 09/16] cleanup imports --- tests/unix_sockets/test_serverapp_integration.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/unix_sockets/test_serverapp_integration.py b/tests/unix_sockets/test_serverapp_integration.py index 7a66ccac31..d11a99d60c 100644 --- a/tests/unix_sockets/test_serverapp_integration.py +++ b/tests/unix_sockets/test_serverapp_integration.py @@ -1,22 +1,19 @@ +import os import stat +import subprocess import sys import time import pytest from jupyter_server.serverapp import list_running_servers, shutdown_server +from jupyter_server.utils import urlencode_unix_socket, urlencode_unix_socket_path # Skip this module if on Windows. Unix sockets are not available on Windows. pytestmark = pytest.mark.skipif( sys.platform.startswith("win"), reason="Unix sockets are not available on Windows." ) -import os -import subprocess -import time - -from jupyter_server.utils import urlencode_unix_socket, urlencode_unix_socket_path - def _cleanup_process(proc): proc.wait() From 34f3e3e7917146dace470ea87e23155c78bc76f0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:42:01 -0600 Subject: [PATCH 10/16] fix socket file --- tests/test_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 16a79e87dd..d721e6921e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -114,8 +114,7 @@ async def foo(): @pytest.mark.skipif(sys.platform != "linux", reason="Requires unix sockets") def test_unix_socket_in_use(tmp_path): - os.unlink(str(tmp_path)) - server_address = str(tmp_path) + server_address = os.path.join(tmp_path, "socket_file") sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.bind(server_address) assert unix_socket_in_use(server_address) From caf0ae62c8d12405416ab1872865d0148b3d1abe Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:42:25 -0600 Subject: [PATCH 11/16] try a 1% target --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index eb9b9dff30..643aadce70 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,4 +6,4 @@ coverage: threshold: 10 patch: default: - target: 0% + target: 1% From 9087a489d8082338b85c53b02144ae1d927c8a63 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:47:20 -0600 Subject: [PATCH 12/16] try again --- codecov.yml | 4 ++-- tests/test_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codecov.yml b/codecov.yml index 643aadce70..b75c3e2dbc 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,7 +3,7 @@ coverage: project: default: target: auto - threshold: 10 + threshold: 1 patch: default: - target: 1% + target: 0% diff --git a/tests/test_utils.py b/tests/test_utils.py index d721e6921e..c97fb6f9fa 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -112,10 +112,10 @@ async def foo(): await run_sync_in_loop(foo()) -@pytest.mark.skipif(sys.platform != "linux", reason="Requires unix sockets") def test_unix_socket_in_use(tmp_path): server_address = os.path.join(tmp_path, "socket_file") sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.bind(server_address) + sock.listen(0) assert unix_socket_in_use(server_address) sock.close() From beeb582e48d6efe083c35822ba17dadcac074f86 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 10:55:44 -0600 Subject: [PATCH 13/16] fix fallback --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index b24f9d35f4..3cdea3eec0 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -39,7 +39,7 @@ jobs: sudo apt-get install xvfb x11-utils libxkbcommon-x11-0 pandoc - name: Run the tests if: ${{ !startsWith(matrix.python-version, 'pypy') && !startsWith(matrix.os, 'windows') }} - run: hatch run cov:test -W default || hatch run cov:test -W default --lf + run: hatch run cov:test -W default || hatch run test:test -W default --lf - name: Run the tests on pypy and windows if: ${{ startsWith(matrix.python-version, 'pypy') || startsWith(matrix.os, 'windows') }} run: hatch run test:test -W default || hatch run test:test -W default --lf From b4ff75ec5cf4ba6259ede08ad8bff6afee3e0f79 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 11:01:35 -0600 Subject: [PATCH 14/16] skip test on windows --- tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index c97fb6f9fa..dcec12f5f9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -112,6 +112,7 @@ async def foo(): await run_sync_in_loop(foo()) +@pytest.mark.skipif(os.name != "posix", reason="Requires unix sockets") def test_unix_socket_in_use(tmp_path): server_address = os.path.join(tmp_path, "socket_file") sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) From f6f8bd24072665089450ce05f1475cc567a7db95 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 11:13:09 -0600 Subject: [PATCH 15/16] fix path len --- tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index dcec12f5f9..777b633ee4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -114,7 +114,8 @@ async def foo(): @pytest.mark.skipif(os.name != "posix", reason="Requires unix sockets") def test_unix_socket_in_use(tmp_path): - server_address = os.path.join(tmp_path, "socket_file") + root_tmp_dir = Path("/tmp").resolve() + server_address = os.path.join(root_tmp_dir, os.path.basename(tmp_path)) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.bind(server_address) sock.listen(0) From dd46f62209d5d48742d173dfd7042b05224f54ea Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Nov 2022 11:14:21 -0600 Subject: [PATCH 16/16] skip test on windows --- tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 777b633ee4..d3e168ff60 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -78,6 +78,7 @@ def test_is_namespace_package_no_spec(): mocked_spec.assert_called_once_with("dummy") +@pytest.mark.skipif(os.name == "nt", reason="Paths are annoying on Windows") def test_path_utils(tmp_path): path = str(tmp_path) assert os.path.basename(path2url(path)) == os.path.basename(path)