From 28de310c8f0bbe255c4441b5264676b9d3c7936f Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Fri, 23 Jul 2021 12:43:59 +0300 Subject: [PATCH 01/59] proper package version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4751e18..c030440 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="beacon_network", - version="0.2.dev", + version="1.2.0", description="Beacon Network services", long_description_content_type="text/markdown", project_urls={ @@ -10,7 +10,7 @@ }, author="CSC - IT Center for Science", classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 3 - Beta", "Intended Audience :: Developers", "Topic :: Internet :: WWW/HTTP :: HTTP Servers", "License :: OSI Approved :: Apache Software License", From 26ccdec8ab7810191029c4fe4404c76bec6fc8d5 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Fri, 23 Jul 2021 14:34:45 +0300 Subject: [PATCH 02/59] add more integration tests --- docker-compose-test.yml | 16 ++++++ tests/integration/run_tests.py | 90 +++++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index c0f2dbc..828e9e7 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -138,3 +138,19 @@ services: networks: - other_net_registry - apps + + extra_bad_beacon: + hostname: extra_bad_beacon + image: cscfi/beacon-python + environment: + HOST: "0.0.0.0" + PORT: 5053 + DATABASE_URL: other_db_registry + DATABASE_USER: user + DATABASE_PASSWORD: pass + DATABASE_NAME: registry + ports: + - 5053:5053 + networks: + - other_net_registry + - apps diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 0e8150a..7321219 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -2,26 +2,94 @@ import asyncio import httpx import logging +import re +import json FORMAT = "[%(asctime)s][%(name)s][%(process)d %(processName)s][%(levelname)-8s](L:%(lineno)s) %(funcName)s: %(message)s" logging.basicConfig(format=FORMAT, datefmt="%Y-%m-%d %H:%M:%S") LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) -SESSION = httpx.AsyncClient() - AGGREGATOR = "http://localhost:5050" REGISTRY = "http://localhost:8080" +REGISTRY_KEY = "07b4e8ed58a6f97897b03843474c8cc981d154ffe45b10ef88a9f127b15c5c56" async def test_service_info(endpoint, name, artifact): """Test service info endpoint.""" LOG.debug(f"Checking service info endpoint for: {artifact}") - response = await SESSION.get(f"{endpoint}/service-info") - assert response.status_code == 200, "HTTP status code error service info" - data = response.json() - assert data["name"] == name, "Wrong endpoint service name" - assert data["type"]["artifact"] == artifact, "Wrong service artifact" + async with httpx.AsyncClient() as client: + response = await client.get(f"{endpoint}/service-info") + assert response.status_code == 200, "HTTP status code error service info" + data = response.json() + assert data["name"] == name, "Wrong endpoint service name" + assert data["type"]["artifact"] == artifact, "Wrong service artifact" + + +async def test_get_services(endpoint, expected_nb, expected_beacon): + """Test GET services registry.""" + LOG.debug("Checking registry listing services") + async with httpx.AsyncClient() as client: + response = await client.get(f"{endpoint}/services") + assert response.status_code == 200, "HTTP status code error service info" + data = response.json() + assert len(data) == expected_nb, "We did not find the expected number of services" + assert re.search(f'"id": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" + + LOG.debug("Checking registry listing services types") + async with httpx.AsyncClient() as client: + server_types = await client.get(f"{endpoint}/services/types") + data = server_types.json() + assert data == ["service-registry", "beacon-aggregator", "beacon"], "We did not find the expected services types" + + +async def test_service_operations(endpoint): + """Test Registry services operations.""" + extra_beacon = {"type": "beacon", "url": "http://extra_bad_beacon:5053/service-info"} + update_beacon = {"type": "beacon", "url": "http://bad_beacon:5052/service-info"} + + LOG.debug("Add new service to the registry") + async with httpx.AsyncClient() as client: + response = await client.post(f"{endpoint}/services", data=json.dumps(extra_beacon), headers={"Authorization": REGISTRY_KEY}) + assert response.status_code == 201, "HTTP status code error service add" + data = response.json() + assert data["serviceId"] == "extra_bad_beacon:5053", "Wrong beacon id obtained" + + LOG.debug("Update service from the registry with existing one") + async with httpx.AsyncClient() as client: + conflict_response = await client.put( + f"{endpoint}/services/{data['serviceId']}", data=json.dumps(update_beacon), headers={"Beacon-Service-Key": data["serviceKey"]} + ) + + conflict_data = conflict_response.json() + assert conflict_response.status_code == 409, "HTTP status code error service update" + assert conflict_data["error"] == "Another service has already been registered with the new service id.", "Conflict error mismatched" + + LOG.debug("Update service from the registry correctly") + async with httpx.AsyncClient() as client: + update_response = await client.put( + f"{endpoint}/services/{data['serviceId']}", data=json.dumps(extra_beacon), headers={"Beacon-Service-Key": data["serviceKey"]} + ) + + updated_data = update_response.json() + assert update_response.status_code == 200, "HTTP status code error service update" + + LOG.debug("Remove service from the registry") + async with httpx.AsyncClient() as client: + await client.delete(f"{endpoint}/services/{updated_data['newServiceId']}", headers={"Beacon-Service-Key": data["serviceKey"]}) + assert update_response.status_code == 200, "HTTP status code error service delete" + + +async def test_query_aggregator(endpoint, expected_nb, expected_beacon): + """Test Aggregatpr query operation.""" + LOG.debug("make a query over the aggregator") + params = {"includeDatasetResponses": "HIT", "assemblyId": "GRCh38", "referenceName": "MT", "start": "9", "referenceBases": "T", "alternateBases": "C"} + async with httpx.AsyncClient() as client: + response = await client.get(f"{endpoint}/query", params=params) + data = response.json() + assert response.status_code == 200, "HTTP status code error aggregator query" + assert len(data) == expected_nb, "We did not find the expected number of responses" + assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" async def main(): @@ -31,7 +99,13 @@ async def main(): await test_service_info(AGGREGATOR, "ELIXIR-FI Beacon Aggregator", "beacon-aggregator") await test_service_info(REGISTRY, "ELIXIR-FI Beacon Registry", "service-registry") - await SESSION.aclose() + LOG.debug("=== Test Registry Endpoint ===") + + await test_get_services(REGISTRY, 4, "bad_beacon:5052") + await test_service_operations(REGISTRY) + + LOG.debug("=== Test Aggregator Endpoint ===") + await test_query_aggregator(AGGREGATOR, 6, "http://bad_beacon:5052/query") if __name__ == "__main__": From 88040ae31922369ced08beb097805e034c84e9b7 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Fri, 23 Jul 2021 14:34:51 +0300 Subject: [PATCH 03/59] fix issue with 409 conglict response in registry --- registry/endpoints/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/endpoints/services.py b/registry/endpoints/services.py index d069212..c7c42d5 100644 --- a/registry/endpoints/services.py +++ b/registry/endpoints/services.py @@ -105,7 +105,7 @@ async def update_service(request, db_pool): if new_id_found_service and service_id != new_service_id: response["message"] = "Service update failed, see error.." response["error"] = "Another service has already been registered with the new service id." - raise web.HTTPConflict(body=json.dumps(response), content_type="application/json") + raise web.HTTPConflict(text=json.dumps(response), content_type="application/json") # Request service info from given url service_info = await http_request_info(url) # Parse and validate service info object From b9aff01ce06e2042c582e553af20db343c1ec24a Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Mon, 26 Jul 2021 11:00:58 +0300 Subject: [PATCH 04/59] fix apt install --- .github/workflows/unit-agg.yml | 4 +++- .github/workflows/unit-reg.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-agg.yml b/.github/workflows/unit-agg.yml index ebd108a..e6131ba 100644 --- a/.github/workflows/unit-agg.yml +++ b/.github/workflows/unit-agg.yml @@ -22,7 +22,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install libcurl-devel - run: sudo apt-get install libcurl4-openssl-dev + run: | + sudo apt-get update + sudo apt-get install libcurl4-openssl-dev - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/unit-reg.yml b/.github/workflows/unit-reg.yml index dd6965f..54d5f17 100644 --- a/.github/workflows/unit-reg.yml +++ b/.github/workflows/unit-reg.yml @@ -22,7 +22,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install libcurl-devel - run: sudo apt-get install libcurl4-openssl-dev + run: | + sudo apt-get update + sudo apt-get install libcurl4-openssl-dev - name: Install dependencies run: | python -m pip install --upgrade pip From bbecbb49430ec2f35a5f78ddedebffd5cfd50158 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 08:28:25 +0000 Subject: [PATCH 05/59] Bump coveralls from 3.1.0 to 3.2.0 Bumps [coveralls](https://github.com/TheKevJames/coveralls-python) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/TheKevJames/coveralls-python/releases) - [Changelog](https://github.com/TheKevJames/coveralls-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/TheKevJames/coveralls-python/compare/3.1.0...3.2.0) --- updated-dependencies: - dependency-name: coveralls dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c030440..ba79637 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ "coverage==5.5", "pytest<6.3", "pytest-cov==2.12.1", - "coveralls==3.1.0", + "coveralls==3.2.0", "testfixtures==6.18.0", "tox==3.24.0", "flake8==3.9.2", From 115c5423f9602a1514824bb5161166258d7313e2 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 30 Jul 2021 15:24:33 +0300 Subject: [PATCH 06/59] added beaconV2 queries and updated some tests --- aggregator/utils/utils.py | 169 +++++++++++++--------- tests/aggregator/test_aggregator_utils.py | 33 +++-- 2 files changed, 123 insertions(+), 79 deletions(-) diff --git a/aggregator/utils/utils.py b/aggregator/utils/utils.py index 90948a3..eabd47c 100644 --- a/aggregator/utils/utils.py +++ b/aggregator/utils/utils.py @@ -97,32 +97,36 @@ async def process_url(url): New in Beacon 2.0: `/g_variants` endpoint replaces the 1.0 `/query` endpoint. """ LOG.debug("Processing URLs.") - # convert tuple to list for processing url = list(url) - # Check which endpoint to use, Beacon 1.0 or 2.0 - query_endpoint = "query" + query_endpoints = ["query"] if url[1] == 2: - query_endpoint = "g_variants" - LOG.debug(f"Using endpoint {query_endpoint}") + query_endpoints = ["individuals", "g_variants", "biosamples", "runs", "analyses", "interactors", "cohorts"] + LOG.debug(f"Using endpoint {query_endpoints}") + urls = [] # Add endpoint if url[0].endswith("/"): - url[0] += query_endpoint + for endpoint in query_endpoints: + urls.append([url[0] + endpoint, url[1]]) elif url[0].endswith("/service-info"): - url[0] = url[0].replace("service-info", query_endpoint) + for endpoint in query_endpoints: + urls.append([url[0].replace("service-info", endpoint), url[1]]) else: # Unknown case # One case is observed, where URL was similar to https://service.institution.org/beacon # For URLs where the info endpoint is /, but / is not present, let's add /query - url[0] += "/" + query_endpoint + for endpoint in query_endpoints: + urls.append([url[0] + "/" + endpoint, url[1]]) + pass # convert back to tuple after processing - url = tuple(url) - - return url + urlTuples = [] + for url in urls: + urlTuples.append(tuple(url)) + return urlTuples async def remove_self(url_self, urls): @@ -134,10 +138,12 @@ async def remove_self(url_self, urls): LOG.debug("Look for self from service URLs.") for url in urls: - url_split = url[0].split("/") - if url_self in url_split: - urls.remove(url) - LOG.debug("Found and removed self from service URLs.") + url = list(url) + for u in url[0]: + url_split = str(u).split("/") + if url_self in url_split: + urls.remove(url) + LOG.debug("Found and removed self from service URLs.") return urls @@ -180,32 +186,37 @@ async def pre_process_payload(version, params): # parse the query string into a dict raw_data = dict(parse.parse_qsl(params)) - if version == 2: - # default data which is always present - data = {"assemblyId": raw_data.get("assemblyId", "GRCh38"), "includeDatasetResponses": raw_data.get("includeDatasetResponses", "ALL")} - - # optionals - if (rn := raw_data.get("referenceName")) is not None: - data["referenceName"] = rn - if (vt := raw_data.get("variantType")) is not None: - data["variantType"] = vt - if (rb := raw_data.get("referenceBases")) is not None: - data["referenceBases"] = rb - if (ab := raw_data.get("alternateBases")) is not None: - data["alternateBases"] = ab - - # exact coordinates - if (s := raw_data.get("start")) is not None: - data["start"] = s - if (e := raw_data.get("end")) is not None: - data["end"] = e - - # range coordinates - if (smin := raw_data.get("startMin")) is not None and (smax := raw_data.get("startMax")) is not None: - data["start"] = ",".join([smin, smax]) - if (emin := raw_data.get("endMin")) is not None and (emax := raw_data.get("endMax")) is not None: - data["end"] = ",".join([emin, emax]) + # checks if a query is a listing search + if (raw_data.get("referenceName")) is not None: + # default data which is always present + data = {"assemblyId": raw_data.get("assemblyId"), "includeDatasetResponses": raw_data.get("includeDatasetResponses")} + + # optionals + if (rn := raw_data.get("referenceName")) is not None: + data["referenceName"] = rn + if (vt := raw_data.get("variantType")) is not None: + data["variantType"] = vt + if (rb := raw_data.get("referenceBases")) is not None: + data["referenceBases"] = rb + if (ab := raw_data.get("alternateBases")) is not None: + data["alternateBases"] = ab + + # exact coordinates + if (s := raw_data.get("start")) is not None: + data["start"] = s + if (e := raw_data.get("end")) is not None: + data["end"] = e + + # range coordinates + if (smin := raw_data.get("startMin")) is not None and (smax := raw_data.get("startMax")) is not None: + data["start"] = ",".join([smin, smax]) + if (emin := raw_data.get("endMin")) is not None and (emax := raw_data.get("endMax")) is not None: + data["end"] = ",".join([emin, emax]) + else: + # beaconV2 expects some data but in listing search these are not needed thus they are empty + data = {"assemblyId": "", "includeDatasetResponses": ""} + else: # convert string digits into integers # Beacon 1.0 uses integer coordinates, while Beacon 2.0 uses string coordinates (ignore referenceName, it should stay as a string) @@ -217,6 +228,33 @@ async def pre_process_payload(version, params): return data +async def find_query_endpoint(service, params): + """Find endpoint for queries by parameters.""" + # since beaconV2 has multiple endpoints this method is used to define those endpoints from parameters + endpoints = service + # if lenght is 1 then beacon is v1 + raw_data = dict(parse.parse_qsl(params)) + if len(endpoints) <= 1 and raw_data.get("searchInInput") is None: + return service[0] + else: + for endpoint in endpoints: + if raw_data.get("searchInInput") is not None: + if raw_data.get("searchInInput") in endpoint[0]: + if raw_data.get("id") != "0" and raw_data.get("id") is not None: + if raw_data.get("searchByInput") != "" and raw_data.get("searchByInput") is not None: + + url = list(endpoint) + url[0] += "/" + raw_data.get("id") + "/" + raw_data.get("searchByInput") + endpoint = tuple(url) + return endpoint + + url = list(endpoint) + url[0] += "/" + raw_data.get("id") + endpoint = tuple(url) + return endpoint + return endpoint + + async def _service_response(response, ws): """Process response to web socket or HTTP.""" result = await response.json() @@ -262,37 +300,34 @@ async def query_service(service, params, access_token, ws=None): """Query service with params.""" LOG.debug("Querying service.") headers = {} - if access_token: headers.update({"Authorization": f"Bearer {access_token}"}) - + endpoint = await find_query_endpoint(service, params) # Pre-process query string into payload format - data = await pre_process_payload(service[1], params) - - # Query service in a session - async with aiohttp.ClientSession() as session: - try: - async with session.post(service[0], json=data, headers=headers, ssl=await request_security()) as response: - LOG.info(f"POST query to service: {service[0]}") - # On successful response, forward response - if response.status == 200: - return await _service_response(response, ws) - - elif response.status == 405: - return await _get_request(session, service, params, headers, ws) - - else: - # HTTP errors - error = {"service": service[0], "queryParams": params, "responseStatus": response.status, "exists": None} - LOG.error(f"Query to {service} failed: {response}.") - if ws is not None: - return await ws.send_str(json.dumps(error)) + if endpoint is not None: + data = await pre_process_payload(endpoint[1], params) + + # Query service in a session + async with aiohttp.ClientSession() as session: + try: + async with session.post(endpoint[0], json=data, headers=headers, ssl=await request_security()) as response: + LOG.info(f"POST query to service: {endpoint}") + # On successful response, forward response + if response.status == 200: + return await _service_response(response, ws) + elif response.status == 405: + return await _get_request(session, endpoint, params, headers, ws) else: - return error - - except Exception as e: - LOG.debug(f"Query error {e}.") - web.HTTPInternalServerError(text="An error occurred while attempting to query services.") + # HTTP errors + error = {"service": endpoint, "queryParams": params, "responseStatus": response.status, "exists": None} + LOG.error(f"Query to {service} failed: {response}.") + if ws is not None: + return await ws.send_str(json.dumps(error)) + else: + return error + except Exception as e: + LOG.debug(f"Query error {e}.") + web.HTTPInternalServerError(text="An error occurred while attempting to query services.") async def ws_bundle_return(result, ws): diff --git a/tests/aggregator/test_aggregator_utils.py b/tests/aggregator/test_aggregator_utils.py index ae71c92..40b1b79 100644 --- a/tests/aggregator/test_aggregator_utils.py +++ b/tests/aggregator/test_aggregator_utils.py @@ -3,7 +3,7 @@ from aioresponses import aioresponses from aiohttp import web -from aggregator.utils.utils import http_get_service_urls, get_services, process_url +from aggregator.utils.utils import http_get_service_urls, get_services, process_url, find_query_endpoint from aggregator.utils.utils import remove_self, get_access_token, parse_results, query_service from aggregator.utils.utils import validate_service_key, clear_cache, ws_bundle_return from aggregator.utils.utils import parse_version, pre_process_payload @@ -102,22 +102,25 @@ async def test_get_services(self, remo, proc, http): async def test_process_url_1(self): """Test url processing type 1.""" processed = await process_url(("https://beacon.fi/", 1)) - self.assertEqual(("https://beacon.fi/query", 1), processed) + self.assertEqual(("https://beacon.fi/query", 1), processed[0]) async def test_process_url_2(self): """Test url processing type 2.""" processed = await process_url(("https://beacon.fi/service-info", 1)) - self.assertEqual(("https://beacon.fi/query", 1), processed) + self.assertEqual(("https://beacon.fi/query", 1), processed[0]) async def test_process_url_3(self): """Test url processing type 3.""" processed = await process_url(("https://beacon.fi", 1)) - self.assertEqual(("https://beacon.fi/query", 1), processed) + self.assertEqual(("https://beacon.fi/query", 1), processed[0]) async def test_process_url_4(self): """Test url processing type 4.""" + params = "searchInInput=g_variants&id=0&searchByInput=" processed = await process_url(("https://beacon.fi", 2)) - self.assertEqual(("https://beacon.fi/g_variants", 2), processed) + findQuery = await find_query_endpoint(processed, params) + print("\x1b[6;30;42m" + str(findQuery) + "\x1b[0m") + self.assertEqual("https://beacon.fi/g_variants", findQuery[0]) async def test_remove_self(self): """Test removal of host from list of urls.""" @@ -161,7 +164,8 @@ async def test_query_service_ws_success_aggregator(self, m): data = [{"important": "stuff"}] m.post("https://beacon.fi/query", status=200, payload=data) ws = MockWebsocket() - await query_service(("https://beacon.fi/query", 1, "beacon"), "", None, ws=ws) + processed = await process_url(("https://beacon.fi/", 1)) + await query_service(processed, "", None, ws=ws) self.assertEqual(ws.data, '{"important": "stuff"}') @aioresponses() @@ -172,7 +176,8 @@ async def test_query_service_ws_success_aggregator_get_request(self, m): m.post("https://beacon.fi/query", status=405) m.get("https://beacon.fi/query", status=200, payload=data) ws = MockWebsocket() - await query_service(("https://beacon.fi/query", 1, "beacon"), "", None, ws=ws) + processed = await process_url(("https://beacon.fi/", 1)) + await query_service(processed, "", None, ws=ws) self.assertEqual(ws.data, '{"important": "stuff"}') @aioresponses() @@ -182,7 +187,8 @@ async def test_query_service_ws_success_beacon(self, m): data = {"important": "stuff"} m.post("https://beacon.fi/query", status=200, payload=data) ws = MockWebsocket() - await query_service(("https://beacon.fi/query", 1, "beacon"), "", None, ws=ws) + processed = await process_url(("https://beacon.fi/", 1)) + await query_service(processed, "", None, ws=ws) self.assertEqual(ws.data, '{"important": "stuff"}') @aioresponses() @@ -190,22 +196,25 @@ async def test_query_service_ws_fail(self, m): """Test querying of service: websocket fail.""" m.post("https://beacon.fi/query", status=400) ws = MockWebsocket() - await query_service(("https://beacon.fi/query", 1, "beacon"), "", None, ws=ws) - self.assertEqual(ws.data, '{"service": "https://beacon.fi/query", "queryParams": "", "responseStatus": 400, "exists": null}') + processed = await process_url(("https://beacon.fi/", 1)) + await query_service(processed, "", None, ws=ws) + self.assertEqual(ws.data, '{"service": ["https://beacon.fi/query", 1], "queryParams": "", "responseStatus": 400, "exists": null}') @aioresponses() async def test_query_service_http_success(self, m): """Test querying of service: http success.""" data = {"response": "from beacon"} m.post("https://beacon.fi/query", status=200, payload=data) - response = await query_service(("https://beacon.fi/query", 1, "beacon"), "", "token") + processed = await process_url(("https://beacon.fi/", 1)) + response = await query_service(processed, "", "token") self.assertEqual(response, data) @aioresponses() async def test_query_service_http_fail(self, m): """Test querying of service: http fail.""" m.post("https://beacon.fi/query", status=400) - response = await query_service(("https://beacon.fi/query", 1, "beacon"), "", None) + processed = await process_url(("https://beacon.fi/", 1)) + response = await query_service(processed, "", None) self.assertEqual(response["responseStatus"], 400) async def test_validate_service_key_success(self): From 3f7b85f2379ea8ba03ddbdd0bf4864f47c6a28dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Aug 2021 08:04:23 +0000 Subject: [PATCH 07/59] Bump tox from 3.24.0 to 3.24.1 Bumps [tox](https://github.com/tox-dev/tox) from 3.24.0 to 3.24.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/master/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/3.24.0...3.24.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ba79637..2b863e1 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ "pytest-cov==2.12.1", "coveralls==3.2.0", "testfixtures==6.18.0", - "tox==3.24.0", + "tox==3.24.1", "flake8==3.9.2", "flake8-docstrings==1.6.0", "asynctest==0.13.0", From 5d90c5a5da9a366ff5f8915f29578242fb59d20f Mon Sep 17 00:00:00 2001 From: Ville Muilu <48563638+ViMuilu@users.noreply.github.com> Date: Tue, 3 Aug 2021 10:33:34 +0300 Subject: [PATCH 08/59] Update aggregator/utils/utils.py Co-authored-by: Stefan Negru --- aggregator/utils/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aggregator/utils/utils.py b/aggregator/utils/utils.py index eabd47c..22c7af8 100644 --- a/aggregator/utils/utils.py +++ b/aggregator/utils/utils.py @@ -319,7 +319,8 @@ async def query_service(service, params, access_token, ws=None): return await _get_request(session, endpoint, params, headers, ws) else: # HTTP errors - error = {"service": endpoint, "queryParams": params, "responseStatus": response.status, "exists": None} + error = {"service": endpoint[0], "queryParams": params, "responseStatus": response.status, "exists": None} + LOG.error(f"Query to {service} failed: {response}.") if ws is not None: return await ws.send_str(json.dumps(error)) From 0ecd64dcfa8e6f0beb2f9e3c3f61c317030f50f4 Mon Sep 17 00:00:00 2001 From: Ville Muilu <48563638+ViMuilu@users.noreply.github.com> Date: Tue, 3 Aug 2021 10:54:31 +0300 Subject: [PATCH 09/59] Update tests/aggregator/test_aggregator_utils.py Co-authored-by: Stefan Negru --- tests/aggregator/test_aggregator_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aggregator/test_aggregator_utils.py b/tests/aggregator/test_aggregator_utils.py index 40b1b79..f6c14cb 100644 --- a/tests/aggregator/test_aggregator_utils.py +++ b/tests/aggregator/test_aggregator_utils.py @@ -198,7 +198,7 @@ async def test_query_service_ws_fail(self, m): ws = MockWebsocket() processed = await process_url(("https://beacon.fi/", 1)) await query_service(processed, "", None, ws=ws) - self.assertEqual(ws.data, '{"service": ["https://beacon.fi/query", 1], "queryParams": "", "responseStatus": 400, "exists": null}') + self.assertEqual(ws.data, '{"service": "https://beacon.fi/query", "queryParams": "", "responseStatus": 400, "exists": null}') @aioresponses() async def test_query_service_http_success(self, m): From a630db8bbd884abdb3f75f9698e85b1d5361c701 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 10:31:47 +0300 Subject: [PATCH 10/59] added beacon2.0 --- .github/workflows/int.yml | 27 +++++++++++++++++++++++++++ aggregator/config/registries.json | 2 +- docker-compose-test.yml | 9 +++++---- tests/test_files/add_fixtures.sh | 14 ++++++++++++-- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index ddfbb87..c386614 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -56,7 +56,34 @@ jobs: run: | docker-compose -f docker-compose-test.yml up -d sleep 30 + + - name: Clone beacon 2.0 + uses: actions/checkout@v2 + with: + repository: 'EGA-archive/beacon-2.x' + ref: 'master' + path: beacon2.0 + - name: Build beacon 2.0 + uses: docker/build-push-action@v2 + with: + context: ./beacon2.0 + push: true + file: ./beacon2.0/Dockerfile + tags: localhost:5001/beacon2.0:latest + cache-from: localhost:5001/beacon2.0:latest + cache-to: type=local,dest=/tmp/.buildx-cache + + -name: Run beacon 2.0 database + run: | + cd beacon2.0/deploy + docker-compose up -d db + + -name: Run beacon2.0 + run: | + cd beacon2.0 + docker-compose up -d + - name: Set up external services for integration run: ./tests/test_files/add_fixtures.sh diff --git a/aggregator/config/registries.json b/aggregator/config/registries.json index 919de5a..f8d3a87 100644 --- a/aggregator/config/registries.json +++ b/aggregator/config/registries.json @@ -1,6 +1,6 @@ [ { - "url": "https://localhost:8080/services", + "url": "http://localhost:9090/services", "key": "secret" } ] \ No newline at end of file diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 828e9e7..9e14879 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -36,7 +36,7 @@ services: - ./tests/test_files:/testconfig environment: APP_HOST: "0.0.0.0" - APP_PORT: 8080 + APP_PORT: 8083 DB_HOST: db_registry DB_PORT: 5432 BEACON_RUN_APP: registry @@ -45,7 +45,7 @@ services: links: - db_registry ports: - - 8080:8080 + - 8083:8083 networks: - net_registry - apps @@ -59,12 +59,12 @@ services: - ./tests/test_files:/testconfig environment: APP_HOST: "0.0.0.0" - APP_PORT: 5050 + APP_PORT: 5054 BEACON_RUN_APP: aggregator DEBUG: "true" CONFIG_FILE: /testconfig/aggregator.ini ports: - - 5050:5050 + - 5054:5054 networks: - apps @@ -154,3 +154,4 @@ services: networks: - other_net_registry - apps + diff --git a/tests/test_files/add_fixtures.sh b/tests/test_files/add_fixtures.sh index d00272b..7b7686a 100755 --- a/tests/test_files/add_fixtures.sh +++ b/tests/test_files/add_fixtures.sh @@ -1,7 +1,7 @@ #!/bin/bash OTHER_REGISTRY="http://localhost:8082" -REGISTRY="http://localhost:8080" +REGISTRY="http://localhost:8083" docker exec -i beacon-network_db_registry_1 psql -U user -d registry \ -c "INSERT INTO api_keys VALUES ('07b4e8ed58a6f97897b03843474c8cc981d154ffe45b10ef88a9f127b15c5c56', 'test key');" @@ -26,7 +26,7 @@ curl -X 'POST' \ -H 'Content-Type: application/json' \ -d '{ "type": "beacon-aggregator", - "url": "http://app_aggregator:5050/service-info" + "url": "http://app_aggregator:5054/service-info" }' @@ -59,3 +59,13 @@ curl -X 'POST' \ "type": "beacon", "url": "http://bad_beacon:5052/service-info" }' + +curl -X 'POST' \ + "${REGISTRY}/services" \ + -H 'Authorization: 07b4e8ed58a6f97897b03843474c8cc981d154ffe45b10ef88a9f127b15c5c56' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "type": "beacon", + "url": "http://localhost:5050/service-info" + }' From 6b35b94cce6d129136dbd3e5ef6acc692cb4dd93 Mon Sep 17 00:00:00 2001 From: Ville Muilu <48563638+ViMuilu@users.noreply.github.com> Date: Fri, 6 Aug 2021 10:40:49 +0300 Subject: [PATCH 11/59] Update int.yml --- .github/workflows/int.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index c386614..1a87211 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -74,13 +74,13 @@ jobs: cache-from: localhost:5001/beacon2.0:latest cache-to: type=local,dest=/tmp/.buildx-cache - -name: Run beacon 2.0 database + - name: Run beacon 2.0 database run: | cd beacon2.0/deploy docker-compose up -d db - -name: Run beacon2.0 - run: | + - name: Run beacon2.0 + run: | cd beacon2.0 docker-compose up -d From 42d51959adbc6a68728a64541c706b8d077793f1 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 10:51:09 +0300 Subject: [PATCH 12/59] changed aggregator url in integration tests --- tests/integration/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 7321219..d3b7ef0 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -10,7 +10,7 @@ LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) -AGGREGATOR = "http://localhost:5050" +AGGREGATOR = "http://localhost:5054" REGISTRY = "http://localhost:8080" REGISTRY_KEY = "07b4e8ed58a6f97897b03843474c8cc981d154ffe45b10ef88a9f127b15c5c56" From bb56636d2b1046cb4d0967ab2fc005ca9b97723c Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 11:30:01 +0300 Subject: [PATCH 13/59] removed cache --- .github/workflows/int.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 1a87211..b1dac2c 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -70,19 +70,18 @@ jobs: context: ./beacon2.0 push: true file: ./beacon2.0/Dockerfile - tags: localhost:5001/beacon2.0:latest - cache-from: localhost:5001/beacon2.0:latest - cache-to: type=local,dest=/tmp/.buildx-cache - name: Run beacon 2.0 database run: | cd beacon2.0/deploy docker-compose up -d db + sleep 30 - name: Run beacon2.0 run: | cd beacon2.0 docker-compose up -d + sleep 30 - name: Set up external services for integration run: ./tests/test_files/add_fixtures.sh From 902c9d7121f3c3a31d9458523f17ee16d25c4723 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 11:35:30 +0300 Subject: [PATCH 14/59] added cache --- .github/workflows/int.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index b1dac2c..673d361 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -70,6 +70,9 @@ jobs: context: ./beacon2.0 push: true file: ./beacon2.0/Dockerfile + tags: localhost:5000/beacon2.0:latest + cache-from: localhost:5000/beacon2.0:latest + cache-to: type=local,dest=/tmp/.buildx-cache - name: Run beacon 2.0 database run: | From a909b58575f442191f2490f9f79fb9d01595333c Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 11:52:12 +0300 Subject: [PATCH 15/59] change to int.yml --- .github/workflows/int.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 673d361..2bab1c4 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -82,7 +82,7 @@ jobs: - name: Run beacon2.0 run: | - cd beacon2.0 + cd .. docker-compose up -d sleep 30 From 31d5d06bbcc618c781d3be815196750886b5a690 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 12:38:19 +0300 Subject: [PATCH 16/59] reverted int.yml change --- .github/workflows/int.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 2bab1c4..673d361 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -82,7 +82,7 @@ jobs: - name: Run beacon2.0 run: | - cd .. + cd beacon2.0 docker-compose up -d sleep 30 From 009a01fbc243afdbfe0d3ce1954ba2ee0661a574 Mon Sep 17 00:00:00 2001 From: Ville Muilu <48563638+ViMuilu@users.noreply.github.com> Date: Fri, 6 Aug 2021 12:48:05 +0300 Subject: [PATCH 17/59] Update int.yml --- .github/workflows/int.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 673d361..d50968c 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -82,7 +82,7 @@ jobs: - name: Run beacon2.0 run: | - cd beacon2.0 + cd beacon2.0/ docker-compose up -d sleep 30 From c3c30425055b5a027a4d3ce947b8252925d7d3a7 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 13:03:54 +0300 Subject: [PATCH 18/59] removed beacon2.0 db --- .github/workflows/int.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 673d361..4acab76 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -74,15 +74,9 @@ jobs: cache-from: localhost:5000/beacon2.0:latest cache-to: type=local,dest=/tmp/.buildx-cache - - name: Run beacon 2.0 database + - name: Run beacon 2.0 run: | cd beacon2.0/deploy - docker-compose up -d db - sleep 30 - - - name: Run beacon2.0 - run: | - cd beacon2.0 docker-compose up -d sleep 30 From 411495a30e31df9d2acb10140dde9c42d4fe2f45 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 13:16:34 +0300 Subject: [PATCH 19/59] changed ports --- docker-compose-test.yml | 4 ++-- tests/integration/run_tests.py | 2 +- tests/test_files/add_fixtures.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 9e14879..9980661 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -114,12 +114,12 @@ services: - ./tests/test_files:/testconfig environment: APP_HOST: "0.0.0.0" - APP_PORT: 5051 + APP_PORT: 5055 BEACON_RUN_APP: aggregator DEBUG: "true" CONFIG_FILE: /testconfig/other_aggregator.ini ports: - - 5051:5051 + - 5055:5055 networks: - apps diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index d3b7ef0..a145edf 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -11,7 +11,7 @@ LOG.setLevel(logging.DEBUG) AGGREGATOR = "http://localhost:5054" -REGISTRY = "http://localhost:8080" +REGISTRY = "http://localhost:8083" REGISTRY_KEY = "07b4e8ed58a6f97897b03843474c8cc981d154ffe45b10ef88a9f127b15c5c56" diff --git a/tests/test_files/add_fixtures.sh b/tests/test_files/add_fixtures.sh index 7b7686a..5cb2fc1 100755 --- a/tests/test_files/add_fixtures.sh +++ b/tests/test_files/add_fixtures.sh @@ -47,7 +47,7 @@ curl -X 'POST' \ -H 'Content-Type: application/json' \ -d '{ "type": "beacon-aggregator", - "url": "http://other_aggregator:5051/service-info" + "url": "http://other_aggregator:5055/service-info" }' curl -X 'POST' \ From 571e3550301c069b28abb464e8de38bd72afd972 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 13:43:07 +0300 Subject: [PATCH 20/59] fix to test --- tests/integration/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index a145edf..eaf0d69 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -105,7 +105,7 @@ async def main(): await test_service_operations(REGISTRY) LOG.debug("=== Test Aggregator Endpoint ===") - await test_query_aggregator(AGGREGATOR, 6, "http://bad_beacon:5052/query") + await test_query_aggregator(AGGREGATOR, 8, "http://bad_beacon:5052/query") if __name__ == "__main__": From 8c36461a00673bd78f28788a7b6b720c804c7a4b Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 14:12:58 +0300 Subject: [PATCH 21/59] changed test expected responses --- tests/integration/run_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index eaf0d69..7ccc4d4 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -88,6 +88,7 @@ async def test_query_aggregator(endpoint, expected_nb, expected_beacon): response = await client.get(f"{endpoint}/query", params=params) data = response.json() assert response.status_code == 200, "HTTP status code error aggregator query" + print('\x1b[6;30;42m' + str(len(data)) + '\x1b[0m') assert len(data) == expected_nb, "We did not find the expected number of responses" assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" @@ -105,7 +106,7 @@ async def main(): await test_service_operations(REGISTRY) LOG.debug("=== Test Aggregator Endpoint ===") - await test_query_aggregator(AGGREGATOR, 8, "http://bad_beacon:5052/query") + await test_query_aggregator(AGGREGATOR, 4, "http://bad_beacon:5052/query") if __name__ == "__main__": From b519bbf3b1173623d004cbb328981e23d73b7164 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 14:22:30 +0300 Subject: [PATCH 22/59] changed regsitry ports --- tests/integration/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 7ccc4d4..a38bfd3 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -88,7 +88,7 @@ async def test_query_aggregator(endpoint, expected_nb, expected_beacon): response = await client.get(f"{endpoint}/query", params=params) data = response.json() assert response.status_code == 200, "HTTP status code error aggregator query" - print('\x1b[6;30;42m' + str(len(data)) + '\x1b[0m') + print("\x1b[6;30;42m" + str(len(data)) + "\x1b[0m") assert len(data) == expected_nb, "We did not find the expected number of responses" assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" From a6f53c248eeeafcd753f3d20d3ea75cc506ad491 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 14:22:59 +0300 Subject: [PATCH 23/59] changed ports --- tests/test_files/registries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_files/registries.json b/tests/test_files/registries.json index b5563da..16f8fa0 100644 --- a/tests/test_files/registries.json +++ b/tests/test_files/registries.json @@ -1,6 +1,6 @@ [ { - "url": "http://app_registry:8080/services", + "url": "http://app_registry:8083/services", "key": "secret" } ] \ No newline at end of file From 6ddc36fe677d29231e588f01e62c35b20df945e6 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 14:38:20 +0300 Subject: [PATCH 24/59] changed test number of responses back --- tests/integration/run_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index a38bfd3..a145edf 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -88,7 +88,6 @@ async def test_query_aggregator(endpoint, expected_nb, expected_beacon): response = await client.get(f"{endpoint}/query", params=params) data = response.json() assert response.status_code == 200, "HTTP status code error aggregator query" - print("\x1b[6;30;42m" + str(len(data)) + "\x1b[0m") assert len(data) == expected_nb, "We did not find the expected number of responses" assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" @@ -106,7 +105,7 @@ async def main(): await test_service_operations(REGISTRY) LOG.debug("=== Test Aggregator Endpoint ===") - await test_query_aggregator(AGGREGATOR, 4, "http://bad_beacon:5052/query") + await test_query_aggregator(AGGREGATOR, 6, "http://bad_beacon:5052/query") if __name__ == "__main__": From e196f55ea1b1f6e5c3c7de6a46886fd654f40d55 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 15:15:54 +0300 Subject: [PATCH 25/59] added beacon2.0 to docker-compose-test --- .github/workflows/int.yml | 38 ++++++++++++++++++++------------------ docker-compose-test.yml | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 4acab76..df7e389 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -40,24 +40,7 @@ jobs: fi echo "BUILD_BRANCH=$BRANCH" >> $GITHUB_ENV - - name: Build - uses: docker/build-push-action@v2 - with: - context: . - push: true - file: ./Dockerfile - tags: localhost:5000/beacon-network:latest - cache-from: localhost:5000/beacon-network:latest - cache-to: type=local,dest=/tmp/.buildx-cache - build-args: | - BRANCH=${{ env.BUILD_BRANCH }} - - - name: Run beacon-network container - run: | - docker-compose -f docker-compose-test.yml up -d - sleep 30 - - - name: Clone beacon 2.0 + - name: Clone beacon 2.0 uses: actions/checkout@v2 with: repository: 'EGA-archive/beacon-2.x' @@ -79,6 +62,25 @@ jobs: cd beacon2.0/deploy docker-compose up -d sleep 30 + + - name: Build + uses: docker/build-push-action@v2 + with: + context: . + push: true + file: ./Dockerfile + tags: localhost:5000/beacon-network:latest + cache-from: localhost:5000/beacon-network:latest + cache-to: type=local,dest=/tmp/.buildx-cache + build-args: | + BRANCH=${{ env.BUILD_BRANCH }} + + - name: Run beacon-network container + run: | + docker-compose -f docker-compose-test.yml up -d + sleep 30 + + - name: Set up external services for integration run: ./tests/test_files/add_fixtures.sh diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 9980661..c0eeea3 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -155,3 +155,19 @@ services: - other_net_registry - apps + beacon2.0: + hostname: beacon2.0 + image: localhost:5000/beacon2.0 + environment: + HOST: "0.0.0.0" + PORT: 5050 + DATABASE_URL: other_db_registry + DATABASE_USER: user + DATABASE_PASSWORD: pass + DATABASE_NAME: registry + ports: + - 5050:5050 + networks: + - other_net_registry + - apps + From 2377570f76cc1242f6926c9971aedbee83fd8d68 Mon Sep 17 00:00:00 2001 From: Ville Muilu <48563638+ViMuilu@users.noreply.github.com> Date: Fri, 6 Aug 2021 15:34:31 +0300 Subject: [PATCH 26/59] Update int.yml --- .github/workflows/int.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index df7e389..413f332 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -39,8 +39,8 @@ jobs: BRANCH=dev fi echo "BUILD_BRANCH=$BRANCH" >> $GITHUB_ENV - - - name: Clone beacon 2.0 + + - name: Clone beacon 2.0 uses: actions/checkout@v2 with: repository: 'EGA-archive/beacon-2.x' From 5e403d47cd8230e6bee2f31e9fe65412b991284c Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Fri, 6 Aug 2021 16:13:11 +0300 Subject: [PATCH 27/59] fixed url in fixtures --- tests/test_files/add_fixtures.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_files/add_fixtures.sh b/tests/test_files/add_fixtures.sh index 5cb2fc1..4d865d1 100755 --- a/tests/test_files/add_fixtures.sh +++ b/tests/test_files/add_fixtures.sh @@ -67,5 +67,5 @@ curl -X 'POST' \ -H 'Content-Type: application/json' \ -d '{ "type": "beacon", - "url": "http://localhost:5050/service-info" + "url": "http://localhost:5050/api/service-info" }' From 3f90eae361e8e3441da436cce6203e1acbebb59c Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 11:05:08 +0300 Subject: [PATCH 28/59] changed beacon2.x repo --- .github/workflows/int.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 413f332..66abcd0 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -43,8 +43,8 @@ jobs: - name: Clone beacon 2.0 uses: actions/checkout@v2 with: - repository: 'EGA-archive/beacon-2.x' - ref: 'master' + repository: 'CSCfi/beacon-2.x' + ref: 'test' path: beacon2.0 - name: Build beacon 2.0 From 164753bc7d6401b8d629ab4da645e63b1dfaa2ed Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 11:22:16 +0300 Subject: [PATCH 29/59] changed hostname --- docker-compose-test.yml | 2 +- tests/integration/run_tests.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index c0eeea3..d59b92b 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -156,7 +156,7 @@ services: - apps beacon2.0: - hostname: beacon2.0 + hostname: beacon image: localhost:5000/beacon2.0 environment: HOST: "0.0.0.0" diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index a145edf..34c83d6 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -81,12 +81,13 @@ async def test_service_operations(endpoint): async def test_query_aggregator(endpoint, expected_nb, expected_beacon): - """Test Aggregatpr query operation.""" + """Test Aggregator query operation.""" LOG.debug("make a query over the aggregator") params = {"includeDatasetResponses": "HIT", "assemblyId": "GRCh38", "referenceName": "MT", "start": "9", "referenceBases": "T", "alternateBases": "C"} async with httpx.AsyncClient() as client: response = await client.get(f"{endpoint}/query", params=params) data = response.json() + print('\x1b[6;30;42m' + str(len(data)) + '\x1b[0m') assert response.status_code == 200, "HTTP status code error aggregator query" assert len(data) == expected_nb, "We did not find the expected number of responses" assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" From cda5b42234cf60717e658c1749127d2006ddfbe4 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 11:24:44 +0300 Subject: [PATCH 30/59] reformat --- tests/integration/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 34c83d6..aab43ba 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -87,7 +87,7 @@ async def test_query_aggregator(endpoint, expected_nb, expected_beacon): async with httpx.AsyncClient() as client: response = await client.get(f"{endpoint}/query", params=params) data = response.json() - print('\x1b[6;30;42m' + str(len(data)) + '\x1b[0m') + print("\x1b[6;30;42m" + str(len(data)) + "\x1b[0m") assert response.status_code == 200, "HTTP status code error aggregator query" assert len(data) == expected_nb, "We did not find the expected number of responses" assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" From 3570c43b77a9a15a22594f90a89f4b4488ba40fc Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 11:41:35 +0300 Subject: [PATCH 31/59] changed beacon url in fixtures --- tests/test_files/add_fixtures.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_files/add_fixtures.sh b/tests/test_files/add_fixtures.sh index 4d865d1..343fc00 100755 --- a/tests/test_files/add_fixtures.sh +++ b/tests/test_files/add_fixtures.sh @@ -67,5 +67,5 @@ curl -X 'POST' \ -H 'Content-Type: application/json' \ -d '{ "type": "beacon", - "url": "http://localhost:5050/api/service-info" + "url": "http://beacon:5050/api/service-info" }' From e04d9270f85d990d7b9251db6d104ed29d99feee Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 11:55:37 +0300 Subject: [PATCH 32/59] changed beacon url in fixtures --- .github/workflows/int.yml | 4 +--- tests/test_files/add_fixtures.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 66abcd0..7b509d8 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -79,9 +79,7 @@ jobs: run: | docker-compose -f docker-compose-test.yml up -d sleep 30 - - - + - name: Set up external services for integration run: ./tests/test_files/add_fixtures.sh diff --git a/tests/test_files/add_fixtures.sh b/tests/test_files/add_fixtures.sh index 343fc00..cbed1f5 100755 --- a/tests/test_files/add_fixtures.sh +++ b/tests/test_files/add_fixtures.sh @@ -67,5 +67,5 @@ curl -X 'POST' \ -H 'Content-Type: application/json' \ -d '{ "type": "beacon", - "url": "http://beacon:5050/api/service-info" + "url": "http://beacon-network_beacon2.0:5050/api/service-info" }' From 20c5dfdf0f63fffe8ff9500da0a41299b161025e Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 13:21:07 +0300 Subject: [PATCH 33/59] remove beacon2 from docker-compose --- docker-compose-test.yml | 16 ---------------- tests/test_files/add_fixtures.sh | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index d59b92b..9980661 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -155,19 +155,3 @@ services: - other_net_registry - apps - beacon2.0: - hostname: beacon - image: localhost:5000/beacon2.0 - environment: - HOST: "0.0.0.0" - PORT: 5050 - DATABASE_URL: other_db_registry - DATABASE_USER: user - DATABASE_PASSWORD: pass - DATABASE_NAME: registry - ports: - - 5050:5050 - networks: - - other_net_registry - - apps - diff --git a/tests/test_files/add_fixtures.sh b/tests/test_files/add_fixtures.sh index cbed1f5..343fc00 100755 --- a/tests/test_files/add_fixtures.sh +++ b/tests/test_files/add_fixtures.sh @@ -67,5 +67,5 @@ curl -X 'POST' \ -H 'Content-Type: application/json' \ -d '{ "type": "beacon", - "url": "http://beacon-network_beacon2.0:5050/api/service-info" + "url": "http://beacon:5050/api/service-info" }' From a6d8c786566134d15cfcdad6f63248bb62f0b53d Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 15:32:16 +0300 Subject: [PATCH 34/59] removed .0 --- .github/workflows/int.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 7b509d8..4b90aec 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -45,15 +45,15 @@ jobs: with: repository: 'CSCfi/beacon-2.x' ref: 'test' - path: beacon2.0 + path: beacon2 - - name: Build beacon 2.0 + - name: Build beacon 2 uses: docker/build-push-action@v2 with: - context: ./beacon2.0 + context: ./beacon2 push: true - file: ./beacon2.0/Dockerfile - tags: localhost:5000/beacon2.0:latest + file: ./beacon2/Dockerfile + tags: localhost:5000/beacon2:latest cache-from: localhost:5000/beacon2.0:latest cache-to: type=local,dest=/tmp/.buildx-cache From f8b150346436c4931e5fecbf6096e64beda7f500 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 15:33:05 +0300 Subject: [PATCH 35/59] removed more .0 --- .github/workflows/int.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 4b90aec..4106d3c 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -40,7 +40,7 @@ jobs: fi echo "BUILD_BRANCH=$BRANCH" >> $GITHUB_ENV - - name: Clone beacon 2.0 + - name: Clone beacon 2 uses: actions/checkout@v2 with: repository: 'CSCfi/beacon-2.x' @@ -54,10 +54,10 @@ jobs: push: true file: ./beacon2/Dockerfile tags: localhost:5000/beacon2:latest - cache-from: localhost:5000/beacon2.0:latest + cache-from: localhost:5000/beacon2:latest cache-to: type=local,dest=/tmp/.buildx-cache - - name: Run beacon 2.0 + - name: Run beacon 2 run: | cd beacon2.0/deploy docker-compose up -d From 8eb61fc3468103f217b8e9c3bf05565a66bc659d Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Mon, 9 Aug 2021 15:36:23 +0300 Subject: [PATCH 36/59] removed .0 --- .github/workflows/int.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 4106d3c..9bf43ef 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -59,7 +59,7 @@ jobs: - name: Run beacon 2 run: | - cd beacon2.0/deploy + cd beacon2/deploy docker-compose up -d sleep 30 From 480d1aebdc7585c97e758df3b4823897f8bd6af0 Mon Sep 17 00:00:00 2001 From: ViMuilu Date: Tue, 10 Aug 2021 11:28:15 +0300 Subject: [PATCH 37/59] build db before beacon2 --- .github/workflows/int.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 9bf43ef..6363079 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -57,6 +57,12 @@ jobs: cache-from: localhost:5000/beacon2:latest cache-to: type=local,dest=/tmp/.buildx-cache + - name: Run beacon 2 db + run: | + cd beacon2/deploy + docker-compose up -d db + sleep 30 + - name: Run beacon 2 run: | cd beacon2/deploy From 054f7fa33a0c98e4e38426c24d8b24e0ec3d2b03 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 10 Aug 2021 11:54:20 +0300 Subject: [PATCH 38/59] Update int.yml --- .github/workflows/int.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 6363079..7be6684 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -46,6 +46,7 @@ jobs: repository: 'CSCfi/beacon-2.x' ref: 'test' path: beacon2 + lfs: true - name: Build beacon 2 uses: docker/build-push-action@v2 @@ -62,6 +63,7 @@ jobs: cd beacon2/deploy docker-compose up -d db sleep 30 + docker-compose logs db - name: Run beacon 2 run: | From 9f1cbb9b8e74d9bd78793580b661c77643763417 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 10 Aug 2021 12:06:19 +0300 Subject: [PATCH 39/59] create network first, not to fail --- .github/workflows/int.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 7be6684..e6a0512 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -61,6 +61,7 @@ jobs: - name: Run beacon 2 db run: | cd beacon2/deploy + docker network create beacon-network_apps docker-compose up -d db sleep 30 docker-compose logs db From 998451d652aeb86d9692e59e75575baba457e42e Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 10 Aug 2021 12:19:19 +0300 Subject: [PATCH 40/59] increase expected nb of services --- tests/integration/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index aab43ba..4249d67 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -102,7 +102,7 @@ async def main(): LOG.debug("=== Test Registry Endpoint ===") - await test_get_services(REGISTRY, 4, "bad_beacon:5052") + await test_get_services(REGISTRY, 5, "bad_beacon:5052") await test_service_operations(REGISTRY) LOG.debug("=== Test Aggregator Endpoint ===") From ff411ac09223a9c984e8665785a345f777dc5691 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 10 Aug 2021 12:23:18 +0300 Subject: [PATCH 41/59] simplify step of running beacon2 --- .github/workflows/int.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index e6a0512..5847377 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -58,18 +58,13 @@ jobs: cache-from: localhost:5000/beacon2:latest cache-to: type=local,dest=/tmp/.buildx-cache - - name: Run beacon 2 db + - name: Run beacon 2 run: | cd beacon2/deploy docker network create beacon-network_apps docker-compose up -d db sleep 30 - docker-compose logs db - - - name: Run beacon 2 - run: | - cd beacon2/deploy - docker-compose up -d + docker-compose up -d --no-build sleep 30 - name: Build From 02fe841513a1fa6c61a0d74f65b1036aab32ca90 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 10 Aug 2021 12:27:30 +0300 Subject: [PATCH 42/59] beacon will be build as part of docker compose --- .github/workflows/int.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 5847377..bffa3a0 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -47,16 +47,6 @@ jobs: ref: 'test' path: beacon2 lfs: true - - - name: Build beacon 2 - uses: docker/build-push-action@v2 - with: - context: ./beacon2 - push: true - file: ./beacon2/Dockerfile - tags: localhost:5000/beacon2:latest - cache-from: localhost:5000/beacon2:latest - cache-to: type=local,dest=/tmp/.buildx-cache - name: Run beacon 2 run: | @@ -64,7 +54,7 @@ jobs: docker network create beacon-network_apps docker-compose up -d db sleep 30 - docker-compose up -d --no-build + docker-compose up -d sleep 30 - name: Build From 1a6ca4f13ef230b34aa1748c2ee38e62f2ee5acc Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 10 Aug 2021 12:37:26 +0300 Subject: [PATCH 43/59] number of responses from aggregator is 8 --- tests/integration/run_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 4249d67..a008ccd 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -87,7 +87,6 @@ async def test_query_aggregator(endpoint, expected_nb, expected_beacon): async with httpx.AsyncClient() as client: response = await client.get(f"{endpoint}/query", params=params) data = response.json() - print("\x1b[6;30;42m" + str(len(data)) + "\x1b[0m") assert response.status_code == 200, "HTTP status code error aggregator query" assert len(data) == expected_nb, "We did not find the expected number of responses" assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" @@ -106,7 +105,7 @@ async def main(): await test_service_operations(REGISTRY) LOG.debug("=== Test Aggregator Endpoint ===") - await test_query_aggregator(AGGREGATOR, 6, "http://bad_beacon:5052/query") + await test_query_aggregator(AGGREGATOR, 8, "http://bad_beacon:5052/query") if __name__ == "__main__": From 6169617a6fa0471438f941ac48392d458ad1ff8d Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 10 Aug 2021 12:37:35 +0300 Subject: [PATCH 44/59] bump postgresql verison --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7a44dd2..3779eaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: db_registry: hostname: db_registry - image: postgres:9.6 + image: postgres:13 environment: POSTGRES_USER: user POSTGRES_PASSWORD: pass From b73b65e167d0877331c1ebdc27119a5ceb8188dc Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 10 Aug 2021 13:43:08 +0300 Subject: [PATCH 45/59] look for beacon 2.x version but don't fail --- tests/integration/run_tests.py | 42 +++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index a008ccd..e270c3e 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -26,7 +26,7 @@ async def test_service_info(endpoint, name, artifact): assert data["type"]["artifact"] == artifact, "Wrong service artifact" -async def test_get_services(endpoint, expected_nb, expected_beacon): +async def test_get_services(endpoint, expected_nb, expected_beacon, version): """Test GET services registry.""" LOG.debug("Checking registry listing services") async with httpx.AsyncClient() as client: @@ -34,13 +34,27 @@ async def test_get_services(endpoint, expected_nb, expected_beacon): assert response.status_code == 200, "HTTP status code error service info" data = response.json() assert len(data) == expected_nb, "We did not find the expected number of services" - assert re.search(f'"id": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" + if version == 1: + + assert re.search(f'"id": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" + + # we don't fail this test as running on localhost this might be problematic + if version == 2: + beacon2 = re.search(f'"id": "{expected_beacon}"', json.dumps(data), re.M) + if beacon2: + data = next((x for x in data if x.get("id") == expected_beacon), None) + if data["type"]["version"] not in ["2.0.0"]: + print("We did not find the expected 2.0. beacon") LOG.debug("Checking registry listing services types") async with httpx.AsyncClient() as client: server_types = await client.get(f"{endpoint}/services/types") data = server_types.json() - assert data == ["service-registry", "beacon-aggregator", "beacon"], "We did not find the expected services types" + assert data == [ + "service-registry", + "beacon-aggregator", + "beacon", + ], "We did not find the expected services types" async def test_service_operations(endpoint): @@ -58,7 +72,9 @@ async def test_service_operations(endpoint): LOG.debug("Update service from the registry with existing one") async with httpx.AsyncClient() as client: conflict_response = await client.put( - f"{endpoint}/services/{data['serviceId']}", data=json.dumps(update_beacon), headers={"Beacon-Service-Key": data["serviceKey"]} + f"{endpoint}/services/{data['serviceId']}", + data=json.dumps(update_beacon), + headers={"Beacon-Service-Key": data["serviceKey"]}, ) conflict_data = conflict_response.json() @@ -68,7 +84,9 @@ async def test_service_operations(endpoint): LOG.debug("Update service from the registry correctly") async with httpx.AsyncClient() as client: update_response = await client.put( - f"{endpoint}/services/{data['serviceId']}", data=json.dumps(extra_beacon), headers={"Beacon-Service-Key": data["serviceKey"]} + f"{endpoint}/services/{data['serviceId']}", + data=json.dumps(extra_beacon), + headers={"Beacon-Service-Key": data["serviceKey"]}, ) updated_data = update_response.json() @@ -83,12 +101,19 @@ async def test_service_operations(endpoint): async def test_query_aggregator(endpoint, expected_nb, expected_beacon): """Test Aggregator query operation.""" LOG.debug("make a query over the aggregator") - params = {"includeDatasetResponses": "HIT", "assemblyId": "GRCh38", "referenceName": "MT", "start": "9", "referenceBases": "T", "alternateBases": "C"} + params = { + "includeDatasetResponses": "HIT", + "assemblyId": "GRCh38", + "referenceName": "MT", + "start": "9", + "referenceBases": "T", + "alternateBases": "C", + } async with httpx.AsyncClient() as client: response = await client.get(f"{endpoint}/query", params=params) data = response.json() assert response.status_code == 200, "HTTP status code error aggregator query" - assert len(data) == expected_nb, "We did not find the expected number of responses" + print(f" Number of responses: {len(data)}") assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" @@ -101,7 +126,8 @@ async def main(): LOG.debug("=== Test Registry Endpoint ===") - await test_get_services(REGISTRY, 5, "bad_beacon:5052") + await test_get_services(REGISTRY, 5, "bad_beacon:5052", 1) + await test_get_services(REGISTRY, 5, "beacon:5050", 2) await test_service_operations(REGISTRY) LOG.debug("=== Test Aggregator Endpoint ===") From 592213773e308ab052d922b4049b839be44d61bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Aug 2021 08:03:49 +0000 Subject: [PATCH 46/59] Bump asyncpg from 0.23.0 to 0.24.0 Bumps [asyncpg](https://github.com/MagicStack/asyncpg) from 0.23.0 to 0.24.0. - [Release notes](https://github.com/MagicStack/asyncpg/releases) - [Commits](https://github.com/MagicStack/asyncpg/compare/v0.23.0...v0.24.0) --- updated-dependencies: - dependency-name: asyncpg dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index db7e06c..d0379bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,6 @@ aiomcache==0.6.0 ujson==4.0.2 uvloop==0.14.0; python_version < '3.7' uvloop==0.15.3; python_version >= '3.7' -asyncpg==0.23.0 +asyncpg==0.24.0 jsonschema==3.2.0 gunicorn==20.1.0 diff --git a/setup.py b/setup.py index 2b863e1..fd471e3 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ "ujson==4.0.2", "uvloop==0.14.0; python_version < '3.7'", "uvloop==0.15.3; python_version >= '3.7'", - "asyncpg==0.23.0", + "asyncpg==0.24.0", "jsonschema==3.2.0", "gunicorn==20.1.0", ], From 4b60d9f3db25331ed800a18e6db5716d8a6a8184 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Aug 2021 09:16:04 +0000 Subject: [PATCH 47/59] Bump uvloop from 0.15.3 to 0.16.0 Bumps [uvloop](https://github.com/MagicStack/uvloop) from 0.15.3 to 0.16.0. - [Release notes](https://github.com/MagicStack/uvloop/releases) - [Commits](https://github.com/MagicStack/uvloop/compare/v0.15.3...v0.16.0) --- updated-dependencies: - dependency-name: uvloop dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d0379bd..e19307f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ aiocache==0.11.1 aiomcache==0.6.0 ujson==4.0.2 uvloop==0.14.0; python_version < '3.7' -uvloop==0.15.3; python_version >= '3.7' +uvloop==0.16.0; python_version >= '3.7' asyncpg==0.24.0 jsonschema==3.2.0 gunicorn==20.1.0 diff --git a/setup.py b/setup.py index fd471e3..9c4c9d2 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "aiomcache==0.6.0", "ujson==4.0.2", "uvloop==0.14.0; python_version < '3.7'", - "uvloop==0.15.3; python_version >= '3.7'", + "uvloop==0.16.0; python_version >= '3.7'", "asyncpg==0.24.0", "jsonschema==3.2.0", "gunicorn==20.1.0", From 541d3eebd5ef074f4ad98f22c7d8081000782a67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 08:04:18 +0000 Subject: [PATCH 48/59] Bump ujson from 4.0.2 to 4.1.0 Bumps [ujson](https://github.com/ultrajson/ultrajson) from 4.0.2 to 4.1.0. - [Release notes](https://github.com/ultrajson/ultrajson/releases) - [Commits](https://github.com/ultrajson/ultrajson/compare/4.0.2...4.1.0) --- updated-dependencies: - dependency-name: ujson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e19307f..a771eb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiohttp==3.7.4.post0 aiohttp-cors==0.7.0 aiocache==0.11.1 aiomcache==0.6.0 -ujson==4.0.2 +ujson==4.1.0 uvloop==0.14.0; python_version < '3.7' uvloop==0.16.0; python_version >= '3.7' asyncpg==0.24.0 diff --git a/setup.py b/setup.py index 9c4c9d2..00f8731 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ "aiohttp-cors==0.7.0", "aiocache==0.11.1", "aiomcache==0.6.0", - "ujson==4.0.2", + "ujson==4.1.0", "uvloop==0.14.0; python_version < '3.7'", "uvloop==0.16.0; python_version >= '3.7'", "asyncpg==0.24.0", From f5e36b50575dd7e3821fcd8249a086a86f7f96e8 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 24 Aug 2021 10:00:40 +0300 Subject: [PATCH 49/59] cache lfs files --- .github/workflows/int.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index bffa3a0..a133796 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -46,7 +46,23 @@ jobs: repository: 'CSCfi/beacon-2.x' ref: 'test' path: beacon2 - lfs: true + + - name: Create LFS file list + run: | + cd beacon2 + git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + + - name: Restore LFS cache + uses: actions/cache@v2 + id: lfs-cache + with: + path: beacon2/.git/lfs + key: ${{ runner.os }}-lfs-${{ hashFiles('.lfs-assets-id') }}-v1 + + - name: Git LFS Pull + run: | + cd beacon2 + git lfs pull - name: Run beacon 2 run: | From 8289183c9039dcb15ca8ea4bd4729969d1c34eb1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 08:02:42 +0000 Subject: [PATCH 50/59] Bump black from 21.7b0 to 21.8b0 Bumps [black](https://github.com/psf/black) from 21.7b0 to 21.8b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c4c9d2..2356bd0 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ "flake8-docstrings==1.6.0", "asynctest==0.13.0", "aioresponses==0.7.2", - "black==21.7b0", + "black==21.8b0", ], "docs": ["sphinx >= 1.4", "sphinx_rtd_theme==0.5.2"], }, From 6175a07f044e4ee7d72fe2a06600f18aafe84a24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Sep 2021 04:56:28 +0000 Subject: [PATCH 51/59] Bump testfixtures from 6.18.0 to 6.18.1 Bumps [testfixtures](https://github.com/Simplistix/testfixtures) from 6.18.0 to 6.18.1. - [Release notes](https://github.com/Simplistix/testfixtures/releases) - [Changelog](https://github.com/Simplistix/testfixtures/blob/master/CHANGELOG.rst) - [Commits](https://github.com/Simplistix/testfixtures/compare/6.18.0...6.18.1) --- updated-dependencies: - dependency-name: testfixtures dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c4c9d2..9cdc30d 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ "pytest<6.3", "pytest-cov==2.12.1", "coveralls==3.2.0", - "testfixtures==6.18.0", + "testfixtures==6.18.1", "tox==3.24.1", "flake8==3.9.2", "flake8-docstrings==1.6.0", From 1e22930d00192bcb9aaeb89ed242a225d3485ccd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Sep 2021 04:57:15 +0000 Subject: [PATCH 52/59] Bump tox from 3.24.1 to 3.24.3 Bumps [tox](https://github.com/tox-dev/tox) from 3.24.1 to 3.24.3. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/master/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/3.24.1...3.24.3) --- updated-dependencies: - dependency-name: tox dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c4c9d2..41c5dcf 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ "pytest-cov==2.12.1", "coveralls==3.2.0", "testfixtures==6.18.0", - "tox==3.24.1", + "tox==3.24.3", "flake8==3.9.2", "flake8-docstrings==1.6.0", "asynctest==0.13.0", From 810f8fc9d6244c9d7ad128ffb5feeee84ab282f6 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Fri, 3 Sep 2021 00:49:26 +0300 Subject: [PATCH 53/59] switch to ujson --- .github/workflows/int.yml | 2 +- Dockerfile | 4 ++-- aggregator/config/__init__.py | 4 ++-- aggregator/utils/utils.py | 25 +++++++++++++++-------- deploy/app.sh | 2 +- registry/endpoints/services.py | 4 ++-- registry/registry.py | 4 ++-- registry/schemas/__init__.py | 4 ++-- tests/aggregator/test_aggregator_utils.py | 10 ++++----- tests/integration/run_tests.py | 18 +++++++++------- 10 files changed, 45 insertions(+), 32 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index a133796..8bedf57 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -94,7 +94,7 @@ jobs: run: ./tests/test_files/add_fixtures.sh - name: Install integration tests dependencies - run: pip install asyncio httpx + run: pip install asyncio httpx ujson - name: Run Integration tests run: python tests/integration/run_tests.py diff --git a/Dockerfile b/Dockerfile index e61d396..60eb171 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer "CSC Developers" RUN apk add --update \ && apk add --no-cache build-base curl-dev linux-headers bash git \ - && apk add --no-cache libressl-dev libffi-dev \ + && apk add --no-cache libressl-dev libffi-dev libstdc++ \ && apk add --no-cache supervisor \ && rm -rf /var/cache/apk/* @@ -20,7 +20,7 @@ RUN pip install --upgrade pip && \ FROM python:3.8-alpine3.13 -RUN apk add --no-cache --update bash +RUN apk add --no-cache --update libstdc++ LABEL maintainer "CSC Developers" LABEL org.label-schema.schema-version="1.0" diff --git a/aggregator/config/__init__.py b/aggregator/config/__init__.py index f7659ff..fc20bc2 100644 --- a/aggregator/config/__init__.py +++ b/aggregator/config/__init__.py @@ -1,7 +1,7 @@ """Aggregator Configuration.""" import os -import json +import ujson from configparser import ConfigParser from collections import namedtuple @@ -16,7 +16,7 @@ def load_json(json_file): data = {} if os.path.isfile(json_file): with open(json_file, "r") as contents: - data = json.loads(contents.read()) + data = ujson.loads(contents.read()) return data diff --git a/aggregator/utils/utils.py b/aggregator/utils/utils.py index 22c7af8..95f2a20 100644 --- a/aggregator/utils/utils.py +++ b/aggregator/utils/utils.py @@ -2,7 +2,7 @@ import os import sys -import json +import ujson import ssl from urllib import parse @@ -190,7 +190,10 @@ async def pre_process_payload(version, params): # checks if a query is a listing search if (raw_data.get("referenceName")) is not None: # default data which is always present - data = {"assemblyId": raw_data.get("assemblyId"), "includeDatasetResponses": raw_data.get("includeDatasetResponses")} + data = { + "assemblyId": raw_data.get("assemblyId"), + "includeDatasetResponses": raw_data.get("includeDatasetResponses"), + } # optionals if (rn := raw_data.get("referenceName")) is not None: @@ -272,7 +275,7 @@ async def _service_response(response, ws): else: # The response came from a beacon and is a single object (dict {}) # Send result to websocket - return await ws.send_str(json.dumps(result)) + return await ws.send_str(ujson.dumps(result, escape_forward_slashes=False)) else: # Standard response return result @@ -291,7 +294,7 @@ async def _get_request(session, service, params, headers, ws): error = {"service": service[0], "queryParams": params, "responseStatus": response.status, "exists": None} LOG.error(f"Query to {service} failed: {response}.") if ws is not None: - return await ws.send_str(json.dumps(error)) + return await ws.send_str(ujson.dumps(error, escape_forward_slashes=False)) else: return error @@ -319,11 +322,16 @@ async def query_service(service, params, access_token, ws=None): return await _get_request(session, endpoint, params, headers, ws) else: # HTTP errors - error = {"service": endpoint[0], "queryParams": params, "responseStatus": response.status, "exists": None} + error = { + "service": endpoint[0], + "queryParams": params, + "responseStatus": response.status, + "exists": None, + } LOG.error(f"Query to {service} failed: {response}.") if ws is not None: - return await ws.send_str(json.dumps(error)) + return await ws.send_str(ujson.dumps(error, escape_forward_slashes=False)) else: return error except Exception as e: @@ -337,7 +345,7 @@ async def ws_bundle_return(result, ws): # A simple function to bundle up websocket returns # when broken down from an aggregator response list - return await ws.send_str(json.dumps(result)) + return await ws.send_str(ujson.dumps(result, escape_forward_slashes=False)) async def validate_service_key(key): @@ -402,7 +410,8 @@ def load_certs(ssl_context): try: ssl_context.load_cert_chain( - os.environ.get("PATH_SSL_CERT_FILE", "/etc/ssl/certs/cert.pem"), keyfile=os.environ.get("PATH_SSL_KEY_FILE", "/etc/ssl/certs/key.pem") + os.environ.get("PATH_SSL_CERT_FILE", "/etc/ssl/certs/cert.pem"), + keyfile=os.environ.get("PATH_SSL_KEY_FILE", "/etc/ssl/certs/key.pem"), ) ssl_context.load_verify_locations(cafile=os.environ.get("PATH_SSL_CA_FILE", "/etc/ssl/certs/ca.pem")) except Exception as e: diff --git a/deploy/app.sh b/deploy/app.sh index e157437..2203366 100755 --- a/deploy/app.sh +++ b/deploy/app.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh THE_HOST=${APP_HOST:="0.0.0.0"} THE_PORT=${APP_PORT:="8080"} diff --git a/registry/endpoints/services.py b/registry/endpoints/services.py index c7c42d5..3493f2d 100644 --- a/registry/endpoints/services.py +++ b/registry/endpoints/services.py @@ -1,6 +1,6 @@ """Common Services Endpoint.""" -import json +import ujson from aiohttp import web @@ -105,7 +105,7 @@ async def update_service(request, db_pool): if new_id_found_service and service_id != new_service_id: response["message"] = "Service update failed, see error.." response["error"] = "Another service has already been registered with the new service id." - raise web.HTTPConflict(text=json.dumps(response), content_type="application/json") + raise web.HTTPConflict(text=ujson.dumps(response, escape_forward_slashes=False), content_type="application/json") # Request service info from given url service_info = await http_request_info(url) # Parse and validate service info object diff --git a/registry/registry.py b/registry/registry.py index fd71e09..6bcacdf 100644 --- a/registry/registry.py +++ b/registry/registry.py @@ -1,7 +1,7 @@ """Beacon Registry API.""" import sys -import json +import ujson import aiohttp_cors @@ -68,7 +68,7 @@ async def services_post(request): await invalidate_aggregator_caches(request, db_pool) # Return confirmation and service key if no problems occurred during processing - return web.HTTPCreated(body=json.dumps(response), content_type="application/json") + return web.HTTPCreated(body=ujson.dumps(response, escape_forward_slashes=False), content_type="application/json") @routes.get("/services") diff --git a/registry/schemas/__init__.py b/registry/schemas/__init__.py index 57737d4..e236fd7 100644 --- a/registry/schemas/__init__.py +++ b/registry/schemas/__init__.py @@ -1,7 +1,7 @@ """Load JSON Schemas.""" import os -import json +import ujson def load_schema(name): @@ -10,4 +10,4 @@ def load_schema(name): path = os.path.join(module_path, f"{name}.json") with open(os.path.abspath(path), "r") as fp: data = fp.read() - return json.loads(data) + return ujson.loads(data) diff --git a/tests/aggregator/test_aggregator_utils.py b/tests/aggregator/test_aggregator_utils.py index f6c14cb..fba4bea 100644 --- a/tests/aggregator/test_aggregator_utils.py +++ b/tests/aggregator/test_aggregator_utils.py @@ -166,7 +166,7 @@ async def test_query_service_ws_success_aggregator(self, m): ws = MockWebsocket() processed = await process_url(("https://beacon.fi/", 1)) await query_service(processed, "", None, ws=ws) - self.assertEqual(ws.data, '{"important": "stuff"}') + self.assertEqual(ws.data, '{"important":"stuff"}') @aioresponses() async def test_query_service_ws_success_aggregator_get_request(self, m): @@ -178,7 +178,7 @@ async def test_query_service_ws_success_aggregator_get_request(self, m): ws = MockWebsocket() processed = await process_url(("https://beacon.fi/", 1)) await query_service(processed, "", None, ws=ws) - self.assertEqual(ws.data, '{"important": "stuff"}') + self.assertEqual(ws.data, '{"important":"stuff"}') @aioresponses() async def test_query_service_ws_success_beacon(self, m): @@ -189,7 +189,7 @@ async def test_query_service_ws_success_beacon(self, m): ws = MockWebsocket() processed = await process_url(("https://beacon.fi/", 1)) await query_service(processed, "", None, ws=ws) - self.assertEqual(ws.data, '{"important": "stuff"}') + self.assertEqual(ws.data, '{"important":"stuff"}') @aioresponses() async def test_query_service_ws_fail(self, m): @@ -198,7 +198,7 @@ async def test_query_service_ws_fail(self, m): ws = MockWebsocket() processed = await process_url(("https://beacon.fi/", 1)) await query_service(processed, "", None, ws=ws) - self.assertEqual(ws.data, '{"service": "https://beacon.fi/query", "queryParams": "", "responseStatus": 400, "exists": null}') + self.assertEqual(ws.data, '{"service":"https://beacon.fi/query","queryParams":"","responseStatus":400,"exists":null}') @aioresponses() async def test_query_service_http_success(self, m): @@ -267,7 +267,7 @@ async def test_ws_bundle_return(self): """Test websocket return function.""" m_ws = MockWebsocket() await ws_bundle_return({"something": "here"}, m_ws) - self.assertEqual('{"something": "here"}', m_ws.data) + self.assertEqual('{"something":"here"}', m_ws.data) async def test_parse_version(self): """Test semver parsing.""" diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index e270c3e..9ba3b9a 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -3,7 +3,7 @@ import httpx import logging import re -import json +import ujson FORMAT = "[%(asctime)s][%(name)s][%(process)d %(processName)s][%(levelname)-8s](L:%(lineno)s) %(funcName)s: %(message)s" logging.basicConfig(format=FORMAT, datefmt="%Y-%m-%d %H:%M:%S") @@ -36,11 +36,11 @@ async def test_get_services(endpoint, expected_nb, expected_beacon, version): assert len(data) == expected_nb, "We did not find the expected number of services" if version == 1: - assert re.search(f'"id": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" + assert re.search(f'"id":"{expected_beacon}"', ujson.dumps(data, escape_forward_slashes=False), re.M), "We did not find the expected beacon" # we don't fail this test as running on localhost this might be problematic if version == 2: - beacon2 = re.search(f'"id": "{expected_beacon}"', json.dumps(data), re.M) + beacon2 = re.search(f'"id":"{expected_beacon}"', ujson.dumps(data, escape_forward_slashes=False), re.M) if beacon2: data = next((x for x in data if x.get("id") == expected_beacon), None) if data["type"]["version"] not in ["2.0.0"]: @@ -64,7 +64,11 @@ async def test_service_operations(endpoint): LOG.debug("Add new service to the registry") async with httpx.AsyncClient() as client: - response = await client.post(f"{endpoint}/services", data=json.dumps(extra_beacon), headers={"Authorization": REGISTRY_KEY}) + response = await client.post( + f"{endpoint}/services", + data=ujson.dumps(extra_beacon, escape_forward_slashes=False), + headers={"Authorization": REGISTRY_KEY}, + ) assert response.status_code == 201, "HTTP status code error service add" data = response.json() assert data["serviceId"] == "extra_bad_beacon:5053", "Wrong beacon id obtained" @@ -73,7 +77,7 @@ async def test_service_operations(endpoint): async with httpx.AsyncClient() as client: conflict_response = await client.put( f"{endpoint}/services/{data['serviceId']}", - data=json.dumps(update_beacon), + data=ujson.dumps(update_beacon, escape_forward_slashes=False), headers={"Beacon-Service-Key": data["serviceKey"]}, ) @@ -85,7 +89,7 @@ async def test_service_operations(endpoint): async with httpx.AsyncClient() as client: update_response = await client.put( f"{endpoint}/services/{data['serviceId']}", - data=json.dumps(extra_beacon), + data=ujson.dumps(extra_beacon, escape_forward_slashes=False), headers={"Beacon-Service-Key": data["serviceKey"]}, ) @@ -114,7 +118,7 @@ async def test_query_aggregator(endpoint, expected_nb, expected_beacon): data = response.json() assert response.status_code == 200, "HTTP status code error aggregator query" print(f" Number of responses: {len(data)}") - assert re.search(f'"service": "{expected_beacon}"', json.dumps(data), re.M), "We did not find the expected beacon" + assert re.search(f'"service":"{expected_beacon}"', ujson.dumps(data, escape_forward_slashes=False), re.M), "We did not find the expected beacon" async def main(): From 963cc722e4af340e03c815261ddf4a47d18ae6b1 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Mon, 27 Sep 2021 11:41:26 +0300 Subject: [PATCH 54/59] use beacon 2 data as url instead of lfs --- .github/workflows/int.yml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml index 8bedf57..bf0f6b1 100644 --- a/.github/workflows/int.yml +++ b/.github/workflows/int.yml @@ -47,22 +47,13 @@ jobs: ref: 'test' path: beacon2 - - name: Create LFS file list + - name: Download beacon 2 data + env: + BEACON_2_DATA_URL: ${{ secrets.BEACON_2_DATA_URL }} run: | - cd beacon2 - git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - - - name: Restore LFS cache - uses: actions/cache@v2 - id: lfs-cache - with: - path: beacon2/.git/lfs - key: ${{ runner.os }}-lfs-${{ hashFiles('.lfs-assets-id') }}-v1 - - - name: Git LFS Pull - run: | - cd beacon2 - git lfs pull + cd beacon2/deploy/db + rm data.sql.gz + wget $BEACON_2_DATA_URL - name: Run beacon 2 run: | From ff34b55b3a6e40215e48e0cce5e86855fa10d1c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:27:50 +0000 Subject: [PATCH 55/59] Bump black from 21.8b0 to 21.9b0 Bumps [black](https://github.com/psf/black) from 21.8b0 to 21.9b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8fbf1e5..9a0bb6b 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ "flake8-docstrings==1.6.0", "asynctest==0.13.0", "aioresponses==0.7.2", - "black==21.8b0", + "black==21.9b0", ], "docs": ["sphinx >= 1.4", "sphinx_rtd_theme==0.5.2"], }, From 34cd2baadd62ec372efabcd76e0d459441d00c65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:28:02 +0000 Subject: [PATCH 56/59] Bump sphinx-rtd-theme from 0.5.2 to 1.0.0 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 0.5.2 to 1.0.0. - [Release notes](https://github.com/readthedocs/sphinx_rtd_theme/releases) - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/0.5.2...1.0.0) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8fbf1e5..7e88cce 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ "aioresponses==0.7.2", "black==21.8b0", ], - "docs": ["sphinx >= 1.4", "sphinx_rtd_theme==0.5.2"], + "docs": ["sphinx >= 1.4", "sphinx_rtd_theme==1.0.0"], }, entry_points={ "console_scripts": ["beacon_registry=registry.registry:main", "beacon_aggregator=aggregator.aggregator:main"], From ff9d8765a6a13ccc7f5c234353af3374768dd7ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:28:27 +0000 Subject: [PATCH 57/59] Bump tox from 3.24.3 to 3.24.4 Bumps [tox](https://github.com/tox-dev/tox) from 3.24.3 to 3.24.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/master/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/3.24.3...3.24.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8fbf1e5..5b90251 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ "pytest-cov==2.12.1", "coveralls==3.2.0", "testfixtures==6.18.1", - "tox==3.24.3", + "tox==3.24.4", "flake8==3.9.2", "flake8-docstrings==1.6.0", "asynctest==0.13.0", From 9a2c2d85efe8ee400d280f72a4d879dcce81055b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:28:41 +0000 Subject: [PATCH 58/59] Bump ujson from 4.1.0 to 4.2.0 Bumps [ujson](https://github.com/ultrajson/ultrajson) from 4.1.0 to 4.2.0. - [Release notes](https://github.com/ultrajson/ultrajson/releases) - [Commits](https://github.com/ultrajson/ultrajson/compare/4.1.0...4.2.0) --- updated-dependencies: - dependency-name: ujson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a771eb5..8eff95f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiohttp==3.7.4.post0 aiohttp-cors==0.7.0 aiocache==0.11.1 aiomcache==0.6.0 -ujson==4.1.0 +ujson==4.2.0 uvloop==0.14.0; python_version < '3.7' uvloop==0.16.0; python_version >= '3.7' asyncpg==0.24.0 diff --git a/setup.py b/setup.py index 8fbf1e5..57cc420 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ "aiohttp-cors==0.7.0", "aiocache==0.11.1", "aiomcache==0.6.0", - "ujson==4.1.0", + "ujson==4.2.0", "uvloop==0.14.0; python_version < '3.7'", "uvloop==0.16.0; python_version >= '3.7'", "asyncpg==0.24.0", From 4ca1c0848debe85b30ef3ab72e76ba68ef3b9026 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:47:48 +0000 Subject: [PATCH 59/59] Bump testfixtures from 6.18.1 to 6.18.2 Bumps [testfixtures](https://github.com/Simplistix/testfixtures) from 6.18.1 to 6.18.2. - [Release notes](https://github.com/Simplistix/testfixtures/releases) - [Changelog](https://github.com/simplistix/testfixtures/blob/master/CHANGELOG.rst) - [Commits](https://github.com/Simplistix/testfixtures/compare/6.18.1...6.18.2) --- updated-dependencies: - dependency-name: testfixtures dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3d8e2e9..c477199 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ "pytest<6.3", "pytest-cov==2.12.1", "coveralls==3.2.0", - "testfixtures==6.18.1", + "testfixtures==6.18.2", "tox==3.24.4", "flake8==3.9.2", "flake8-docstrings==1.6.0",