diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index ddfbb87..bf0f6b1 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -39,7 +39,31 @@ jobs: BRANCH=dev fi echo "BUILD_BRANCH=$BRANCH" >> $GITHUB_ENV + + - name: Clone beacon 2 + uses: actions/checkout@v2 + with: + repository: 'CSCfi/beacon-2.x' + ref: 'test' + path: beacon2 + + - name: Download beacon 2 data + env: + BEACON_2_DATA_URL: ${{ secrets.BEACON_2_DATA_URL }} + run: | + cd beacon2/deploy/db + rm data.sql.gz + wget $BEACON_2_DATA_URL + - name: Run beacon 2 + run: | + cd beacon2/deploy + docker network create beacon-network_apps + docker-compose up -d db + sleep 30 + docker-compose up -d + sleep 30 + - name: Build uses: docker/build-push-action@v2 with: @@ -56,12 +80,12 @@ jobs: run: | docker-compose -f docker-compose-test.yml up -d sleep 30 - + - name: Set up external services for integration run: ./tests/test_files/add_fixtures.sh - name: Install integration tests dependencies - run: pip install asyncio httpx + run: pip install asyncio httpx ujson - name: Run Integration tests run: python tests/integration/run_tests.py diff --git a/.github/workflows/unit-agg.yml b/.github/workflows/unit-agg.yml index ebd108a..e6131ba 100644 --- a/.github/workflows/unit-agg.yml +++ b/.github/workflows/unit-agg.yml @@ -22,7 +22,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install libcurl-devel - run: sudo apt-get install libcurl4-openssl-dev + run: | + sudo apt-get update + sudo apt-get install libcurl4-openssl-dev - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/unit-reg.yml b/.github/workflows/unit-reg.yml index dd6965f..54d5f17 100644 --- a/.github/workflows/unit-reg.yml +++ b/.github/workflows/unit-reg.yml @@ -22,7 +22,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install libcurl-devel - run: sudo apt-get install libcurl4-openssl-dev + run: | + sudo apt-get update + sudo apt-get install libcurl4-openssl-dev - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/Dockerfile b/Dockerfile index e61d396..60eb171 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer "CSC Developers" RUN apk add --update \ && apk add --no-cache build-base curl-dev linux-headers bash git \ - && apk add --no-cache libressl-dev libffi-dev \ + && apk add --no-cache libressl-dev libffi-dev libstdc++ \ && apk add --no-cache supervisor \ && rm -rf /var/cache/apk/* @@ -20,7 +20,7 @@ RUN pip install --upgrade pip && \ FROM python:3.8-alpine3.13 -RUN apk add --no-cache --update bash +RUN apk add --no-cache --update libstdc++ LABEL maintainer "CSC Developers" LABEL org.label-schema.schema-version="1.0" diff --git a/aggregator/config/__init__.py b/aggregator/config/__init__.py index f7659ff..fc20bc2 100644 --- a/aggregator/config/__init__.py +++ b/aggregator/config/__init__.py @@ -1,7 +1,7 @@ """Aggregator Configuration.""" import os -import json +import ujson from configparser import ConfigParser from collections import namedtuple @@ -16,7 +16,7 @@ def load_json(json_file): data = {} if os.path.isfile(json_file): with open(json_file, "r") as contents: - data = json.loads(contents.read()) + data = ujson.loads(contents.read()) return data diff --git a/aggregator/config/registries.json b/aggregator/config/registries.json index 919de5a..f8d3a87 100644 --- a/aggregator/config/registries.json +++ b/aggregator/config/registries.json @@ -1,6 +1,6 @@ [ { - "url": "https://localhost:8080/services", + "url": "http://localhost:9090/services", "key": "secret" } ] \ No newline at end of file diff --git a/aggregator/utils/utils.py b/aggregator/utils/utils.py index 90948a3..95f2a20 100644 --- a/aggregator/utils/utils.py +++ b/aggregator/utils/utils.py @@ -2,7 +2,7 @@ import os import sys -import json +import ujson import ssl from urllib import parse @@ -97,32 +97,36 @@ async def process_url(url): New in Beacon 2.0: `/g_variants` endpoint replaces the 1.0 `/query` endpoint. """ LOG.debug("Processing URLs.") - # convert tuple to list for processing url = list(url) - # Check which endpoint to use, Beacon 1.0 or 2.0 - query_endpoint = "query" + query_endpoints = ["query"] if url[1] == 2: - query_endpoint = "g_variants" - LOG.debug(f"Using endpoint {query_endpoint}") + query_endpoints = ["individuals", "g_variants", "biosamples", "runs", "analyses", "interactors", "cohorts"] + LOG.debug(f"Using endpoint {query_endpoints}") + urls = [] # Add endpoint if url[0].endswith("/"): - url[0] += query_endpoint + for endpoint in query_endpoints: + urls.append([url[0] + endpoint, url[1]]) elif url[0].endswith("/service-info"): - url[0] = url[0].replace("service-info", query_endpoint) + for endpoint in query_endpoints: + urls.append([url[0].replace("service-info", endpoint), url[1]]) else: # Unknown case # One case is observed, where URL was similar to https://service.institution.org/beacon # For URLs where the info endpoint is /, but / is not present, let's add /query - url[0] += "/" + query_endpoint + for endpoint in query_endpoints: + urls.append([url[0] + "/" + endpoint, url[1]]) + pass # convert back to tuple after processing - url = tuple(url) - - return url + urlTuples = [] + for url in urls: + urlTuples.append(tuple(url)) + return urlTuples async def remove_self(url_self, urls): @@ -134,10 +138,12 @@ async def remove_self(url_self, urls): LOG.debug("Look for self from service URLs.") for url in urls: - url_split = url[0].split("/") - if url_self in url_split: - urls.remove(url) - LOG.debug("Found and removed self from service URLs.") + url = list(url) + for u in url[0]: + url_split = str(u).split("/") + if url_self in url_split: + urls.remove(url) + LOG.debug("Found and removed self from service URLs.") return urls @@ -180,32 +186,40 @@ async def pre_process_payload(version, params): # parse the query string into a dict raw_data = dict(parse.parse_qsl(params)) - if version == 2: - # default data which is always present - data = {"assemblyId": raw_data.get("assemblyId", "GRCh38"), "includeDatasetResponses": raw_data.get("includeDatasetResponses", "ALL")} - - # optionals - if (rn := raw_data.get("referenceName")) is not None: - data["referenceName"] = rn - if (vt := raw_data.get("variantType")) is not None: - data["variantType"] = vt - if (rb := raw_data.get("referenceBases")) is not None: - data["referenceBases"] = rb - if (ab := raw_data.get("alternateBases")) is not None: - data["alternateBases"] = ab - - # exact coordinates - if (s := raw_data.get("start")) is not None: - data["start"] = s - if (e := raw_data.get("end")) is not None: - data["end"] = e - - # range coordinates - if (smin := raw_data.get("startMin")) is not None and (smax := raw_data.get("startMax")) is not None: - data["start"] = ",".join([smin, smax]) - if (emin := raw_data.get("endMin")) is not None and (emax := raw_data.get("endMax")) is not None: - data["end"] = ",".join([emin, emax]) + # checks if a query is a listing search + if (raw_data.get("referenceName")) is not None: + # default data which is always present + data = { + "assemblyId": raw_data.get("assemblyId"), + "includeDatasetResponses": raw_data.get("includeDatasetResponses"), + } + + # optionals + if (rn := raw_data.get("referenceName")) is not None: + data["referenceName"] = rn + if (vt := raw_data.get("variantType")) is not None: + data["variantType"] = vt + if (rb := raw_data.get("referenceBases")) is not None: + data["referenceBases"] = rb + if (ab := raw_data.get("alternateBases")) is not None: + data["alternateBases"] = ab + + # exact coordinates + if (s := raw_data.get("start")) is not None: + data["start"] = s + if (e := raw_data.get("end")) is not None: + data["end"] = e + + # range coordinates + if (smin := raw_data.get("startMin")) is not None and (smax := raw_data.get("startMax")) is not None: + data["start"] = ",".join([smin, smax]) + if (emin := raw_data.get("endMin")) is not None and (emax := raw_data.get("endMax")) is not None: + data["end"] = ",".join([emin, emax]) + else: + # beaconV2 expects some data but in listing search these are not needed thus they are empty + data = {"assemblyId": "", "includeDatasetResponses": ""} + else: # convert string digits into integers # Beacon 1.0 uses integer coordinates, while Beacon 2.0 uses string coordinates (ignore referenceName, it should stay as a string) @@ -217,6 +231,33 @@ async def pre_process_payload(version, params): return data +async def find_query_endpoint(service, params): + """Find endpoint for queries by parameters.""" + # since beaconV2 has multiple endpoints this method is used to define those endpoints from parameters + endpoints = service + # if lenght is 1 then beacon is v1 + raw_data = dict(parse.parse_qsl(params)) + if len(endpoints) <= 1 and raw_data.get("searchInInput") is None: + return service[0] + else: + for endpoint in endpoints: + if raw_data.get("searchInInput") is not None: + if raw_data.get("searchInInput") in endpoint[0]: + if raw_data.get("id") != "0" and raw_data.get("id") is not None: + if raw_data.get("searchByInput") != "" and raw_data.get("searchByInput") is not None: + + url = list(endpoint) + url[0] += "/" + raw_data.get("id") + "/" + raw_data.get("searchByInput") + endpoint = tuple(url) + return endpoint + + url = list(endpoint) + url[0] += "/" + raw_data.get("id") + endpoint = tuple(url) + return endpoint + return endpoint + + async def _service_response(response, ws): """Process response to web socket or HTTP.""" result = await response.json() @@ -234,7 +275,7 @@ async def _service_response(response, ws): else: # The response came from a beacon and is a single object (dict {}) # Send result to websocket - return await ws.send_str(json.dumps(result)) + return await ws.send_str(ujson.dumps(result, escape_forward_slashes=False)) else: # Standard response return result @@ -253,7 +294,7 @@ async def _get_request(session, service, params, headers, ws): error = {"service": service[0], "queryParams": params, "responseStatus": response.status, "exists": None} LOG.error(f"Query to {service} failed: {response}.") if ws is not None: - return await ws.send_str(json.dumps(error)) + return await ws.send_str(ujson.dumps(error, escape_forward_slashes=False)) else: return error @@ -262,37 +303,40 @@ async def query_service(service, params, access_token, ws=None): """Query service with params.""" LOG.debug("Querying service.") headers = {} - if access_token: headers.update({"Authorization": f"Bearer {access_token}"}) - + endpoint = await find_query_endpoint(service, params) # Pre-process query string into payload format - data = await pre_process_payload(service[1], params) - - # Query service in a session - async with aiohttp.ClientSession() as session: - try: - async with session.post(service[0], json=data, headers=headers, ssl=await request_security()) as response: - LOG.info(f"POST query to service: {service[0]}") - # On successful response, forward response - if response.status == 200: - return await _service_response(response, ws) - - elif response.status == 405: - return await _get_request(session, service, params, headers, ws) - - else: - # HTTP errors - error = {"service": service[0], "queryParams": params, "responseStatus": response.status, "exists": None} - LOG.error(f"Query to {service} failed: {response}.") - if ws is not None: - return await ws.send_str(json.dumps(error)) + if endpoint is not None: + data = await pre_process_payload(endpoint[1], params) + + # Query service in a session + async with aiohttp.ClientSession() as session: + try: + async with session.post(endpoint[0], json=data, headers=headers, ssl=await request_security()) as response: + LOG.info(f"POST query to service: {endpoint}") + # On successful response, forward response + if response.status == 200: + return await _service_response(response, ws) + elif response.status == 405: + return await _get_request(session, endpoint, params, headers, ws) else: - return error - - except Exception as e: - LOG.debug(f"Query error {e}.") - web.HTTPInternalServerError(text="An error occurred while attempting to query services.") + # HTTP errors + error = { + "service": endpoint[0], + "queryParams": params, + "responseStatus": response.status, + "exists": None, + } + + LOG.error(f"Query to {service} failed: {response}.") + if ws is not None: + return await ws.send_str(ujson.dumps(error, escape_forward_slashes=False)) + else: + return error + except Exception as e: + LOG.debug(f"Query error {e}.") + web.HTTPInternalServerError(text="An error occurred while attempting to query services.") async def ws_bundle_return(result, ws): @@ -301,7 +345,7 @@ async def ws_bundle_return(result, ws): # A simple function to bundle up websocket returns # when broken down from an aggregator response list - return await ws.send_str(json.dumps(result)) + return await ws.send_str(ujson.dumps(result, escape_forward_slashes=False)) async def validate_service_key(key): @@ -366,7 +410,8 @@ def load_certs(ssl_context): try: ssl_context.load_cert_chain( - os.environ.get("PATH_SSL_CERT_FILE", "/etc/ssl/certs/cert.pem"), keyfile=os.environ.get("PATH_SSL_KEY_FILE", "/etc/ssl/certs/key.pem") + os.environ.get("PATH_SSL_CERT_FILE", "/etc/ssl/certs/cert.pem"), + keyfile=os.environ.get("PATH_SSL_KEY_FILE", "/etc/ssl/certs/key.pem"), ) ssl_context.load_verify_locations(cafile=os.environ.get("PATH_SSL_CA_FILE", "/etc/ssl/certs/ca.pem")) except Exception as e: diff --git a/deploy/app.sh b/deploy/app.sh index e157437..2203366 100755 --- a/deploy/app.sh +++ b/deploy/app.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh THE_HOST=${APP_HOST:="0.0.0.0"} THE_PORT=${APP_PORT:="8080"} diff --git a/docker-compose-test.yml b/docker-compose-test.yml index c0f2dbc..9980661 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -36,7 +36,7 @@ services: - ./tests/test_files:/testconfig environment: APP_HOST: "0.0.0.0" - APP_PORT: 8080 + APP_PORT: 8083 DB_HOST: db_registry DB_PORT: 5432 BEACON_RUN_APP: registry @@ -45,7 +45,7 @@ services: links: - db_registry ports: - - 8080:8080 + - 8083:8083 networks: - net_registry - apps @@ -59,12 +59,12 @@ services: - ./tests/test_files:/testconfig environment: APP_HOST: "0.0.0.0" - APP_PORT: 5050 + APP_PORT: 5054 BEACON_RUN_APP: aggregator DEBUG: "true" CONFIG_FILE: /testconfig/aggregator.ini ports: - - 5050:5050 + - 5054:5054 networks: - apps @@ -114,12 +114,12 @@ services: - ./tests/test_files:/testconfig environment: APP_HOST: "0.0.0.0" - APP_PORT: 5051 + APP_PORT: 5055 BEACON_RUN_APP: aggregator DEBUG: "true" CONFIG_FILE: /testconfig/other_aggregator.ini ports: - - 5051:5051 + - 5055:5055 networks: - apps @@ -138,3 +138,20 @@ services: networks: - other_net_registry - apps + + extra_bad_beacon: + hostname: extra_bad_beacon + image: cscfi/beacon-python + environment: + HOST: "0.0.0.0" + PORT: 5053 + DATABASE_URL: other_db_registry + DATABASE_USER: user + DATABASE_PASSWORD: pass + DATABASE_NAME: registry + ports: + - 5053:5053 + networks: + - other_net_registry + - apps + diff --git a/docker-compose.yml b/docker-compose.yml index 7a44dd2..3779eaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: db_registry: hostname: db_registry - image: postgres:9.6 + image: postgres:13 environment: POSTGRES_USER: user POSTGRES_PASSWORD: pass diff --git a/registry/endpoints/services.py b/registry/endpoints/services.py index d069212..3493f2d 100644 --- a/registry/endpoints/services.py +++ b/registry/endpoints/services.py @@ -1,6 +1,6 @@ """Common Services Endpoint.""" -import json +import ujson from aiohttp import web @@ -105,7 +105,7 @@ async def update_service(request, db_pool): if new_id_found_service and service_id != new_service_id: response["message"] = "Service update failed, see error.." response["error"] = "Another service has already been registered with the new service id." - raise web.HTTPConflict(body=json.dumps(response), content_type="application/json") + raise web.HTTPConflict(text=ujson.dumps(response, escape_forward_slashes=False), content_type="application/json") # Request service info from given url service_info = await http_request_info(url) # Parse and validate service info object diff --git a/registry/registry.py b/registry/registry.py index fd71e09..6bcacdf 100644 --- a/registry/registry.py +++ b/registry/registry.py @@ -1,7 +1,7 @@ """Beacon Registry API.""" import sys -import json +import ujson import aiohttp_cors @@ -68,7 +68,7 @@ async def services_post(request): await invalidate_aggregator_caches(request, db_pool) # Return confirmation and service key if no problems occurred during processing - return web.HTTPCreated(body=json.dumps(response), content_type="application/json") + return web.HTTPCreated(body=ujson.dumps(response, escape_forward_slashes=False), content_type="application/json") @routes.get("/services") diff --git a/registry/schemas/__init__.py b/registry/schemas/__init__.py index 57737d4..e236fd7 100644 --- a/registry/schemas/__init__.py +++ b/registry/schemas/__init__.py @@ -1,7 +1,7 @@ """Load JSON Schemas.""" import os -import json +import ujson def load_schema(name): @@ -10,4 +10,4 @@ def load_schema(name): path = os.path.join(module_path, f"{name}.json") with open(os.path.abspath(path), "r") as fp: data = fp.read() - return json.loads(data) + return ujson.loads(data) diff --git a/requirements.txt b/requirements.txt index db7e06c..8eff95f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,9 @@ aiohttp==3.7.4.post0 aiohttp-cors==0.7.0 aiocache==0.11.1 aiomcache==0.6.0 -ujson==4.0.2 +ujson==4.2.0 uvloop==0.14.0; python_version < '3.7' -uvloop==0.15.3; python_version >= '3.7' -asyncpg==0.23.0 +uvloop==0.16.0; python_version >= '3.7' +asyncpg==0.24.0 jsonschema==3.2.0 gunicorn==20.1.0 diff --git a/setup.py b/setup.py index 4751e18..c477199 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="beacon_network", - version="0.2.dev", + version="1.2.0", description="Beacon Network services", long_description_content_type="text/markdown", project_urls={ @@ -10,7 +10,7 @@ }, author="CSC - IT Center for Science", classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 3 - Beta", "Intended Audience :: Developers", "Topic :: Internet :: WWW/HTTP :: HTTP Servers", "License :: OSI Approved :: Apache Software License", @@ -34,10 +34,10 @@ "aiohttp-cors==0.7.0", "aiocache==0.11.1", "aiomcache==0.6.0", - "ujson==4.0.2", + "ujson==4.2.0", "uvloop==0.14.0; python_version < '3.7'", - "uvloop==0.15.3; python_version >= '3.7'", - "asyncpg==0.23.0", + "uvloop==0.16.0; python_version >= '3.7'", + "asyncpg==0.24.0", "jsonschema==3.2.0", "gunicorn==20.1.0", ], @@ -46,16 +46,16 @@ "coverage==5.5", "pytest<6.3", "pytest-cov==2.12.1", - "coveralls==3.1.0", - "testfixtures==6.18.0", - "tox==3.24.0", + "coveralls==3.2.0", + "testfixtures==6.18.2", + "tox==3.24.4", "flake8==3.9.2", "flake8-docstrings==1.6.0", "asynctest==0.13.0", "aioresponses==0.7.2", - "black==21.7b0", + "black==21.9b0", ], - "docs": ["sphinx >= 1.4", "sphinx_rtd_theme==0.5.2"], + "docs": ["sphinx >= 1.4", "sphinx_rtd_theme==1.0.0"], }, entry_points={ "console_scripts": ["beacon_registry=registry.registry:main", "beacon_aggregator=aggregator.aggregator:main"], diff --git a/tests/aggregator/test_aggregator_utils.py b/tests/aggregator/test_aggregator_utils.py index ae71c92..fba4bea 100644 --- a/tests/aggregator/test_aggregator_utils.py +++ b/tests/aggregator/test_aggregator_utils.py @@ -3,7 +3,7 @@ from aioresponses import aioresponses from aiohttp import web -from aggregator.utils.utils import http_get_service_urls, get_services, process_url +from aggregator.utils.utils import http_get_service_urls, get_services, process_url, find_query_endpoint from aggregator.utils.utils import remove_self, get_access_token, parse_results, query_service from aggregator.utils.utils import validate_service_key, clear_cache, ws_bundle_return from aggregator.utils.utils import parse_version, pre_process_payload @@ -102,22 +102,25 @@ async def test_get_services(self, remo, proc, http): async def test_process_url_1(self): """Test url processing type 1.""" processed = await process_url(("https://beacon.fi/", 1)) - self.assertEqual(("https://beacon.fi/query", 1), processed) + self.assertEqual(("https://beacon.fi/query", 1), processed[0]) async def test_process_url_2(self): """Test url processing type 2.""" processed = await process_url(("https://beacon.fi/service-info", 1)) - self.assertEqual(("https://beacon.fi/query", 1), processed) + self.assertEqual(("https://beacon.fi/query", 1), processed[0]) async def test_process_url_3(self): """Test url processing type 3.""" processed = await process_url(("https://beacon.fi", 1)) - self.assertEqual(("https://beacon.fi/query", 1), processed) + self.assertEqual(("https://beacon.fi/query", 1), processed[0]) async def test_process_url_4(self): """Test url processing type 4.""" + params = "searchInInput=g_variants&id=0&searchByInput=" processed = await process_url(("https://beacon.fi", 2)) - self.assertEqual(("https://beacon.fi/g_variants", 2), processed) + findQuery = await find_query_endpoint(processed, params) + print("\x1b[6;30;42m" + str(findQuery) + "\x1b[0m") + self.assertEqual("https://beacon.fi/g_variants", findQuery[0]) async def test_remove_self(self): """Test removal of host from list of urls.""" @@ -161,8 +164,9 @@ async def test_query_service_ws_success_aggregator(self, m): data = [{"important": "stuff"}] m.post("https://beacon.fi/query", status=200, payload=data) ws = MockWebsocket() - await query_service(("https://beacon.fi/query", 1, "beacon"), "", None, ws=ws) - self.assertEqual(ws.data, '{"important": "stuff"}') + processed = await process_url(("https://beacon.fi/", 1)) + await query_service(processed, "", None, ws=ws) + self.assertEqual(ws.data, '{"important":"stuff"}') @aioresponses() async def test_query_service_ws_success_aggregator_get_request(self, m): @@ -172,8 +176,9 @@ async def test_query_service_ws_success_aggregator_get_request(self, m): m.post("https://beacon.fi/query", status=405) m.get("https://beacon.fi/query", status=200, payload=data) ws = MockWebsocket() - await query_service(("https://beacon.fi/query", 1, "beacon"), "", None, ws=ws) - self.assertEqual(ws.data, '{"important": "stuff"}') + processed = await process_url(("https://beacon.fi/", 1)) + await query_service(processed, "", None, ws=ws) + self.assertEqual(ws.data, '{"important":"stuff"}') @aioresponses() async def test_query_service_ws_success_beacon(self, m): @@ -182,30 +187,34 @@ async def test_query_service_ws_success_beacon(self, m): data = {"important": "stuff"} m.post("https://beacon.fi/query", status=200, payload=data) ws = MockWebsocket() - await query_service(("https://beacon.fi/query", 1, "beacon"), "", None, ws=ws) - self.assertEqual(ws.data, '{"important": "stuff"}') + processed = await process_url(("https://beacon.fi/", 1)) + await query_service(processed, "", None, ws=ws) + self.assertEqual(ws.data, '{"important":"stuff"}') @aioresponses() async def test_query_service_ws_fail(self, m): """Test querying of service: websocket fail.""" m.post("https://beacon.fi/query", status=400) ws = MockWebsocket() - await query_service(("https://beacon.fi/query", 1, "beacon"), "", None, ws=ws) - self.assertEqual(ws.data, '{"service": "https://beacon.fi/query", "queryParams": "", "responseStatus": 400, "exists": null}') + processed = await process_url(("https://beacon.fi/", 1)) + await query_service(processed, "", None, ws=ws) + self.assertEqual(ws.data, '{"service":"https://beacon.fi/query","queryParams":"","responseStatus":400,"exists":null}') @aioresponses() async def test_query_service_http_success(self, m): """Test querying of service: http success.""" data = {"response": "from beacon"} m.post("https://beacon.fi/query", status=200, payload=data) - response = await query_service(("https://beacon.fi/query", 1, "beacon"), "", "token") + processed = await process_url(("https://beacon.fi/", 1)) + response = await query_service(processed, "", "token") self.assertEqual(response, data) @aioresponses() async def test_query_service_http_fail(self, m): """Test querying of service: http fail.""" m.post("https://beacon.fi/query", status=400) - response = await query_service(("https://beacon.fi/query", 1, "beacon"), "", None) + processed = await process_url(("https://beacon.fi/", 1)) + response = await query_service(processed, "", None) self.assertEqual(response["responseStatus"], 400) async def test_validate_service_key_success(self): @@ -258,7 +267,7 @@ async def test_ws_bundle_return(self): """Test websocket return function.""" m_ws = MockWebsocket() await ws_bundle_return({"something": "here"}, m_ws) - self.assertEqual('{"something": "here"}', m_ws.data) + self.assertEqual('{"something":"here"}', m_ws.data) async def test_parse_version(self): """Test semver parsing.""" diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 0e8150a..9ba3b9a 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -2,26 +2,123 @@ import asyncio import httpx import logging +import re +import ujson FORMAT = "[%(asctime)s][%(name)s][%(process)d %(processName)s][%(levelname)-8s](L:%(lineno)s) %(funcName)s: %(message)s" logging.basicConfig(format=FORMAT, datefmt="%Y-%m-%d %H:%M:%S") LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) -SESSION = httpx.AsyncClient() - -AGGREGATOR = "http://localhost:5050" -REGISTRY = "http://localhost:8080" +AGGREGATOR = "http://localhost:5054" +REGISTRY = "http://localhost:8083" +REGISTRY_KEY = "07b4e8ed58a6f97897b03843474c8cc981d154ffe45b10ef88a9f127b15c5c56" async def test_service_info(endpoint, name, artifact): """Test service info endpoint.""" LOG.debug(f"Checking service info endpoint for: {artifact}") - response = await SESSION.get(f"{endpoint}/service-info") - assert response.status_code == 200, "HTTP status code error service info" - data = response.json() - assert data["name"] == name, "Wrong endpoint service name" - assert data["type"]["artifact"] == artifact, "Wrong service artifact" + async with httpx.AsyncClient() as client: + response = await client.get(f"{endpoint}/service-info") + assert response.status_code == 200, "HTTP status code error service info" + data = response.json() + assert data["name"] == name, "Wrong endpoint service name" + assert data["type"]["artifact"] == artifact, "Wrong service artifact" + + +async def test_get_services(endpoint, expected_nb, expected_beacon, version): + """Test GET services registry.""" + LOG.debug("Checking registry listing services") + async with httpx.AsyncClient() as client: + response = await client.get(f"{endpoint}/services") + assert response.status_code == 200, "HTTP status code error service info" + data = response.json() + assert len(data) == expected_nb, "We did not find the expected number of services" + if version == 1: + + assert re.search(f'"id":"{expected_beacon}"', ujson.dumps(data, escape_forward_slashes=False), re.M), "We did not find the expected beacon" + + # we don't fail this test as running on localhost this might be problematic + if version == 2: + beacon2 = re.search(f'"id":"{expected_beacon}"', ujson.dumps(data, escape_forward_slashes=False), re.M) + if beacon2: + data = next((x for x in data if x.get("id") == expected_beacon), None) + if data["type"]["version"] not in ["2.0.0"]: + print("We did not find the expected 2.0. beacon") + + LOG.debug("Checking registry listing services types") + async with httpx.AsyncClient() as client: + server_types = await client.get(f"{endpoint}/services/types") + data = server_types.json() + assert data == [ + "service-registry", + "beacon-aggregator", + "beacon", + ], "We did not find the expected services types" + + +async def test_service_operations(endpoint): + """Test Registry services operations.""" + extra_beacon = {"type": "beacon", "url": "http://extra_bad_beacon:5053/service-info"} + update_beacon = {"type": "beacon", "url": "http://bad_beacon:5052/service-info"} + + LOG.debug("Add new service to the registry") + async with httpx.AsyncClient() as client: + response = await client.post( + f"{endpoint}/services", + data=ujson.dumps(extra_beacon, escape_forward_slashes=False), + headers={"Authorization": REGISTRY_KEY}, + ) + assert response.status_code == 201, "HTTP status code error service add" + data = response.json() + assert data["serviceId"] == "extra_bad_beacon:5053", "Wrong beacon id obtained" + + LOG.debug("Update service from the registry with existing one") + async with httpx.AsyncClient() as client: + conflict_response = await client.put( + f"{endpoint}/services/{data['serviceId']}", + data=ujson.dumps(update_beacon, escape_forward_slashes=False), + headers={"Beacon-Service-Key": data["serviceKey"]}, + ) + + conflict_data = conflict_response.json() + assert conflict_response.status_code == 409, "HTTP status code error service update" + assert conflict_data["error"] == "Another service has already been registered with the new service id.", "Conflict error mismatched" + + LOG.debug("Update service from the registry correctly") + async with httpx.AsyncClient() as client: + update_response = await client.put( + f"{endpoint}/services/{data['serviceId']}", + data=ujson.dumps(extra_beacon, escape_forward_slashes=False), + headers={"Beacon-Service-Key": data["serviceKey"]}, + ) + + updated_data = update_response.json() + assert update_response.status_code == 200, "HTTP status code error service update" + + LOG.debug("Remove service from the registry") + async with httpx.AsyncClient() as client: + await client.delete(f"{endpoint}/services/{updated_data['newServiceId']}", headers={"Beacon-Service-Key": data["serviceKey"]}) + assert update_response.status_code == 200, "HTTP status code error service delete" + + +async def test_query_aggregator(endpoint, expected_nb, expected_beacon): + """Test Aggregator query operation.""" + LOG.debug("make a query over the aggregator") + params = { + "includeDatasetResponses": "HIT", + "assemblyId": "GRCh38", + "referenceName": "MT", + "start": "9", + "referenceBases": "T", + "alternateBases": "C", + } + async with httpx.AsyncClient() as client: + response = await client.get(f"{endpoint}/query", params=params) + data = response.json() + assert response.status_code == 200, "HTTP status code error aggregator query" + print(f" Number of responses: {len(data)}") + assert re.search(f'"service":"{expected_beacon}"', ujson.dumps(data, escape_forward_slashes=False), re.M), "We did not find the expected beacon" async def main(): @@ -31,7 +128,14 @@ async def main(): await test_service_info(AGGREGATOR, "ELIXIR-FI Beacon Aggregator", "beacon-aggregator") await test_service_info(REGISTRY, "ELIXIR-FI Beacon Registry", "service-registry") - await SESSION.aclose() + LOG.debug("=== Test Registry Endpoint ===") + + await test_get_services(REGISTRY, 5, "bad_beacon:5052", 1) + await test_get_services(REGISTRY, 5, "beacon:5050", 2) + await test_service_operations(REGISTRY) + + LOG.debug("=== Test Aggregator Endpoint ===") + await test_query_aggregator(AGGREGATOR, 8, "http://bad_beacon:5052/query") if __name__ == "__main__": diff --git a/tests/test_files/add_fixtures.sh b/tests/test_files/add_fixtures.sh index d00272b..343fc00 100755 --- a/tests/test_files/add_fixtures.sh +++ b/tests/test_files/add_fixtures.sh @@ -1,7 +1,7 @@ #!/bin/bash OTHER_REGISTRY="http://localhost:8082" -REGISTRY="http://localhost:8080" +REGISTRY="http://localhost:8083" docker exec -i beacon-network_db_registry_1 psql -U user -d registry \ -c "INSERT INTO api_keys VALUES ('07b4e8ed58a6f97897b03843474c8cc981d154ffe45b10ef88a9f127b15c5c56', 'test key');" @@ -26,7 +26,7 @@ curl -X 'POST' \ -H 'Content-Type: application/json' \ -d '{ "type": "beacon-aggregator", - "url": "http://app_aggregator:5050/service-info" + "url": "http://app_aggregator:5054/service-info" }' @@ -47,7 +47,7 @@ curl -X 'POST' \ -H 'Content-Type: application/json' \ -d '{ "type": "beacon-aggregator", - "url": "http://other_aggregator:5051/service-info" + "url": "http://other_aggregator:5055/service-info" }' curl -X 'POST' \ @@ -59,3 +59,13 @@ curl -X 'POST' \ "type": "beacon", "url": "http://bad_beacon:5052/service-info" }' + +curl -X 'POST' \ + "${REGISTRY}/services" \ + -H 'Authorization: 07b4e8ed58a6f97897b03843474c8cc981d154ffe45b10ef88a9f127b15c5c56' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "type": "beacon", + "url": "http://beacon:5050/api/service-info" + }' diff --git a/tests/test_files/registries.json b/tests/test_files/registries.json index b5563da..16f8fa0 100644 --- a/tests/test_files/registries.json +++ b/tests/test_files/registries.json @@ -1,6 +1,6 @@ [ { - "url": "http://app_registry:8080/services", + "url": "http://app_registry:8083/services", "key": "secret" } ] \ No newline at end of file