From c83eaadc718ec32d03e851ed956253e8633b8362 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 14 Mar 2025 11:57:42 +0100 Subject: [PATCH 1/8] Fix liftover --- CHANGELOG.md | 1 + scout/utils/ensembl_rest_clients.py | 28 +++++++----------------- tests/utils/conftest.py | 14 ++++-------- tests/utils/test_ensembl_rest_clients.py | 24 ++++++++++---------- 4 files changed, 25 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d104df81cb..a1370afdc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/) - Updated color scheme for variant assessment badges that were hard to see in light mode, notably Risk Factor (#5318) - Avoid page timeout by skipping HGVS validations in ClinVar multistep submission for non-MANE transcripts from variants in build 38 (#5302) - Sashimi view page displaying an error message when Ensembl REST API (LiftOver) is not available (#5322) +- Refactored the liftover functionality to avoid using the old Ensembl REST API () ## [4.98] ### Added diff --git a/scout/utils/ensembl_rest_clients.py b/scout/utils/ensembl_rest_clients.py index 5d4a31da16..3d29c63306 100644 --- a/scout/utils/ensembl_rest_clients.py +++ b/scout/utils/ensembl_rest_clients.py @@ -10,23 +10,18 @@ LOG = logging.getLogger(__name__) HEADERS = {"Content-type": "application/json"} -RESTAPI_37 = "https://grch37.rest.ensembl.org" -RESTAPI_38 = "https://rest.ensembl.org" +RESTAPI_URL = "https://rest.ensembl.org" class EnsemblRestApiClient: """A class handling requests and responses to and from the Ensembl REST APIs. - Endpoints for human build 37: https://grch37.rest.ensembl.org - Endpoints for human build 38: http://rest.ensembl.org/ + Endpoints: http://rest.ensembl.org/ Documentation: https://github.com/Ensembl/ensembl-rest/wiki doi:10.1093/bioinformatics/btu613 """ - def __init__(self, build="37"): - if build == "38": - self.server = RESTAPI_38 - else: - self.server = RESTAPI_37 + def __init__(self): + self.server = RESTAPI_URL def build_url(self, endpoint, params=None): """Build an url to query ensembml""" @@ -63,18 +58,11 @@ def send_request(url) -> Optional[dict]: flash(error) return data - def liftover(self, build, chrom, start, end=None): + def liftover( + self, build: str, chrom: str, start: int, end: Optional[int] = None + ) -> Optional[dict]: """Perform variant liftover using Ensembl REST API - - Args: - build(str): genome build: "37" or "38" - chrom(str): 1-22,X,Y,MT,M - start(int): start coordinate - stop(int): stop coordinate or None - - Returns: - mappings(list of dict): example: - example: https://rest.ensembl.org/map/human/GRCh37/X:1000000..1000100:1/GRCh38?content-type=application/json + example: https://rest.ensembl.org/map/human/GRCh37/X:1000000..1000100:1/GRCh38?content-type=application/json """ build = "GRCh38" if "38" in str(build) else "GRCh37" diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py index a59f401edd..837bb6f996 100644 --- a/tests/utils/conftest.py +++ b/tests/utils/conftest.py @@ -49,21 +49,15 @@ def fixture_ensembl_biomart_xml_query(): @pytest.fixture -def ensembl_biomart_client_37(): +def ensembl_biomart_client(): """Return a client to the ensembl biomart, build 37""" return ensembl_rest_clients.EnsemblBiomartClient("37") @pytest.fixture -def ensembl_rest_client_37(): - """Return a client to the ensembl rest api, build 37""" - return ensembl_rest_clients.EnsemblRestApiClient("37") - - -@pytest.fixture -def ensembl_rest_client_38(): - """Return a client to the ensembl rest api, build 38""" - return ensembl_rest_clients.EnsemblRestApiClient("38") +def ensembl_rest_client(): + """Return a client to the ensembl rest api.""" + return ensembl_rest_clients.EnsemblRestApiClient() @pytest.fixture diff --git a/tests/utils/test_ensembl_rest_clients.py b/tests/utils/test_ensembl_rest_clients.py index 83f4c00cb6..68a2b57f90 100644 --- a/tests/utils/test_ensembl_rest_clients.py +++ b/tests/utils/test_ensembl_rest_clients.py @@ -4,31 +4,31 @@ from requests.exceptions import MissingSchema from requests.models import Response -from scout.utils.ensembl_rest_clients import RESTAPI_37 +from scout.utils.ensembl_rest_clients import RESTAPI_URL @responses.activate -def test_liftover(ensembl_rest_client_37, ensembl_liftover_response): +def test_liftover(ensembl_rest_client, ensembl_liftover_response): """Test send request for coordinates liftover""" # GIVEN a patched response from Ensembl - url = f"{RESTAPI_37}/map/human/GRCh37/X:1000000..1000100/GRCh38?content-type=application/json" + url = f"{RESTAPI_URL}/map/human/GRCh37/X:1000000..1000100/GRCh38?content-type=application/json" responses.add( responses.GET, url, json=ensembl_liftover_response, status=200, ) - client = ensembl_rest_client_37 + client = ensembl_rest_client # WHEN sending the liftover request the function should return a mapped locus mapped_coords = client.liftover("37", "X", 1000000, 1000100) assert mapped_coords[0]["mapped"] @responses.activate -def test_send_gene_request(ensembl_gene_response, ensembl_rest_client_37): +def test_send_gene_request(ensembl_gene_response, ensembl_rest_client): """Test send request with correct params and endpoint""" - url = f"{RESTAPI_37}/overlap/id/ENSG00000103591?feature=gene" - client = ensembl_rest_client_37 + url = f"{RESTAPI_URL}/overlap/id/ENSG00000103591?feature=gene" + client = ensembl_rest_client responses.add( responses.GET, url, @@ -44,13 +44,13 @@ def test_send_gene_request(ensembl_gene_response, ensembl_rest_client_37): assert data[0]["end"] -def test_send_request_fakey_url(mock_app, ensembl_rest_client_37, mocker): +def test_send_request_fakey_url(mock_app, ensembl_rest_client, mocker): """Test the Ensembl REST client with an URL that is raising missing schema error.""" # GIVEN a completely invalid URL url = "fakeyurl" # GIVEN a patched Ensembl client - client = ensembl_rest_client_37 + client = ensembl_rest_client mocker.patch("requests.get", side_effect=MissingSchema("Invalid URL")) # THEN the client should return no content @@ -59,12 +59,12 @@ def test_send_request_fakey_url(mock_app, ensembl_rest_client_37, mocker): assert data is None -def test_send_request_unavaailable(mock_app, ensembl_rest_client_37, mocker): +def test_send_request_unavaailable(mock_app, ensembl_rest_client, mocker): """Test the Ensembl REST client with an URL that is not available (500 error).""" - url = f"{RESTAPI_37}/fakeyurl" + url = f"{RESTAPI_URL}/fakeyurl" # GIVEN a patched Ensembl client - client = ensembl_rest_client_37 + client = ensembl_rest_client # GIVEN a mocked 550 response from Ensembl mock_response = Response() From 6d3d2c7ea3037cf02f87ffe4883a0abdd88bb823 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 14 Mar 2025 12:00:17 +0100 Subject: [PATCH 2/8] PR ref on changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1370afdc9..d4ef1bb36c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/) - Updated color scheme for variant assessment badges that were hard to see in light mode, notably Risk Factor (#5318) - Avoid page timeout by skipping HGVS validations in ClinVar multistep submission for non-MANE transcripts from variants in build 38 (#5302) - Sashimi view page displaying an error message when Ensembl REST API (LiftOver) is not available (#5322) -- Refactored the liftover functionality to avoid using the old Ensembl REST API () +- Refactored the liftover functionality to avoid using the old Ensembl REST API (#5326) ## [4.98] ### Added From 9065748c834ed953da032db0fa8b54ba48346cea Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 14 Mar 2025 12:09:07 +0100 Subject: [PATCH 3/8] Replace url in 2 other tests --- .../blueprints/alignviewers/test_alignviewers_controllers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/server/blueprints/alignviewers/test_alignviewers_controllers.py b/tests/server/blueprints/alignviewers/test_alignviewers_controllers.py index ff3d065a10..bdced76f17 100644 --- a/tests/server/blueprints/alignviewers/test_alignviewers_controllers.py +++ b/tests/server/blueprints/alignviewers/test_alignviewers_controllers.py @@ -5,6 +5,7 @@ from scout.server.blueprints.alignviewers import controllers from scout.server.extensions import config_igv_tracks, store +from scout.utils.ensembl_rest_clients import RESTAPI_URL @responses.activate @@ -15,7 +16,7 @@ def test_make_sashimi_tracks_variant_38(app, case_obj, ensembl_liftover_response test_variant = store.variant_collection.find_one({"hgnc_symbols": ["POT1"]}) # GIVEN a patched response from Ensembl liftover API - url = f'https://grch37.rest.ensembl.org/map/human/GRCh37/{test_variant["chromosome"]}:{test_variant["position"]}..{test_variant["end"]}/GRCh38?content-type=application/json' + url = f'{RESTAPI_URL}/map/human/GRCh37/{test_variant["chromosome"]}:{test_variant["position"]}..{test_variant["end"]}/GRCh38?content-type=application/json' responses.add( responses.GET, url, @@ -56,7 +57,7 @@ def test_make_sashimi_tracks_variant_37(app, case_obj, ensembl_liftover_response test_variant = store.variant_collection.find_one({"hgnc_symbols": ["POT1"]}) # GIVEN a patched response from Ensembl liftover API - url = f'https://grch37.rest.ensembl.org/map/human/GRCh37/{test_variant["chromosome"]}:{test_variant["position"]}..{test_variant["end"]}/GRCh38?content-type=application/json' + url = f'{RESTAPI_URL}/map/human/GRCh37/{test_variant["chromosome"]}:{test_variant["position"]}..{test_variant["end"]}/GRCh38?content-type=application/json' responses.add( responses.GET, url, From 223a10a2df1f4dc4d7fffb82d36f189871f71fd2 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 14 Mar 2025 12:16:17 +0100 Subject: [PATCH 4/8] One more fix in a test --- tests/server/blueprints/variant/test_variant_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/server/blueprints/variant/test_variant_views.py b/tests/server/blueprints/variant/test_variant_views.py index c1306e10a4..0d99281880 100644 --- a/tests/server/blueprints/variant/test_variant_views.py +++ b/tests/server/blueprints/variant/test_variant_views.py @@ -3,9 +3,9 @@ import responses from flask import url_for -from flask_login import current_user from scout.server.extensions import store +from scout.utils.ensembl_rest_clients import RESTAPI_URL @responses.activate @@ -24,7 +24,7 @@ def test_marrvel_link_38(app, case_obj): store.variant_collection.insert_one(test_variant) # GIVEN that the variant can be lifted over to build 37 - url = f"https://grch37.rest.ensembl.org/map/human/GRCh38/{test_variant['chromosome']}:{test_variant['position']}..{test_variant['position']}/GRCh37?content-type=application/json" + url = f"{RESTAPI_URL}/map/human/GRCh38/{test_variant['chromosome']}:{test_variant['position']}..{test_variant['position']}/GRCh37?content-type=application/json" liftover_mappings = { "mappings": [ From 5fedc8dbb5831343f0a69669dd457a911af4875f Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 14 Mar 2025 13:26:02 +0100 Subject: [PATCH 5/8] Replace one more --- .../server/blueprints/alignviewers/test_alignviewers_views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/server/blueprints/alignviewers/test_alignviewers_views.py b/tests/server/blueprints/alignviewers/test_alignviewers_views.py index c6dc9896bb..6bbdf4c48f 100644 --- a/tests/server/blueprints/alignviewers/test_alignviewers_views.py +++ b/tests/server/blueprints/alignviewers/test_alignviewers_views.py @@ -2,6 +2,8 @@ import responses from flask import session, url_for +from scout.utils.ensembl_rest_clients import RESTAPI_URL + def test_remote_static_no_auth(app): """Test endpoint that serves alignment files as non-logged user""" @@ -152,7 +154,7 @@ def test_sashimi_igv(app, user_obj, case_obj, variant_obj, ensembl_liftover_resp # GIVEN a mocked response from the Ensembl liftover service chromosome = variant_obj["chromosome"] position = variant_obj["position"] - mocked_liftover_url = f"https://grch37.rest.ensembl.org/map/human/GRCh37/{chromosome}:{position}..{position}/GRCh38?content-type=application/json" + mocked_liftover_url = f"{RESTAPI_URL}map/human/GRCh37/{chromosome}:{position}..{position}/GRCh38?content-type=application/json" responses.add( responses.GET, mocked_liftover_url, From a6d3feefa97a9b7b7216482c49ab0cfdc17ca983 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 14 Mar 2025 13:37:36 +0100 Subject: [PATCH 6/8] Typo --- tests/server/blueprints/alignviewers/test_alignviewers_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server/blueprints/alignviewers/test_alignviewers_views.py b/tests/server/blueprints/alignviewers/test_alignviewers_views.py index 6bbdf4c48f..3170c615d9 100644 --- a/tests/server/blueprints/alignviewers/test_alignviewers_views.py +++ b/tests/server/blueprints/alignviewers/test_alignviewers_views.py @@ -154,7 +154,7 @@ def test_sashimi_igv(app, user_obj, case_obj, variant_obj, ensembl_liftover_resp # GIVEN a mocked response from the Ensembl liftover service chromosome = variant_obj["chromosome"] position = variant_obj["position"] - mocked_liftover_url = f"{RESTAPI_URL}map/human/GRCh37/{chromosome}:{position}..{position}/GRCh38?content-type=application/json" + mocked_liftover_url = f"{RESTAPI_URL}/map/human/GRCh37/{chromosome}:{position}..{position}/GRCh38?content-type=application/json" responses.add( responses.GET, mocked_liftover_url, From f69feb0cdf931438ae9d5e9ee606fbc6375ed109 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 14 Mar 2025 13:46:06 +0100 Subject: [PATCH 7/8] Small fixes --- scout/utils/ensembl_rest_clients.py | 2 +- tests/utils/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scout/utils/ensembl_rest_clients.py b/scout/utils/ensembl_rest_clients.py index 3d29c63306..50a84b550d 100644 --- a/scout/utils/ensembl_rest_clients.py +++ b/scout/utils/ensembl_rest_clients.py @@ -15,7 +15,7 @@ class EnsemblRestApiClient: """A class handling requests and responses to and from the Ensembl REST APIs. - Endpoints: http://rest.ensembl.org/ + Endpoint: http://rest.ensembl.org/ Documentation: https://github.com/Ensembl/ensembl-rest/wiki doi:10.1093/bioinformatics/btu613 """ diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py index 837bb6f996..51a59fc7b5 100644 --- a/tests/utils/conftest.py +++ b/tests/utils/conftest.py @@ -56,7 +56,7 @@ def ensembl_biomart_client(): @pytest.fixture def ensembl_rest_client(): - """Return a client to the ensembl rest api.""" + """Return a client to the Ensembl REST API.""" return ensembl_rest_clients.EnsemblRestApiClient() From a74869df39fe223187b439af293c9d24b7e2ace6 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 14 Mar 2025 14:10:04 +0100 Subject: [PATCH 8/8] Remove pointless test --- tests/utils/test_ensembl_rest_clients.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/utils/test_ensembl_rest_clients.py b/tests/utils/test_ensembl_rest_clients.py index 68a2b57f90..c3a1660d2c 100644 --- a/tests/utils/test_ensembl_rest_clients.py +++ b/tests/utils/test_ensembl_rest_clients.py @@ -24,26 +24,6 @@ def test_liftover(ensembl_rest_client, ensembl_liftover_response): assert mapped_coords[0]["mapped"] -@responses.activate -def test_send_gene_request(ensembl_gene_response, ensembl_rest_client): - """Test send request with correct params and endpoint""" - url = f"{RESTAPI_URL}/overlap/id/ENSG00000103591?feature=gene" - client = ensembl_rest_client - responses.add( - responses.GET, - url, - json=ensembl_gene_response, - status=200, - ) - data = client.send_request(url) - - # get all gene for the ensembl gene, They should be a list of items - assert data[0]["assembly_name"] == "GRCh37" - assert data[0]["external_name"] == "AAGAB" - assert data[0]["start"] - assert data[0]["end"] - - def test_send_request_fakey_url(mock_app, ensembl_rest_client, mocker): """Test the Ensembl REST client with an URL that is raising missing schema error."""