From 507f5f2be445f8790dd74adf78ef2a9824de8d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20L=C3=B3pez-Doriga?= Date: Wed, 24 Apr 2024 15:53:38 +0200 Subject: [PATCH 1/8] resources filled --- beacon/response/build_response.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beacon/response/build_response.py b/beacon/response/build_response.py index ff2567db..866dd9ac 100644 --- a/beacon/response/build_response.py +++ b/beacon/response/build_response.py @@ -300,6 +300,12 @@ def build_filtering_terms_response(data, # TODO: 'extendedInfo': build_extended_info(), 'response': { 'filteringTerms': data, + 'resources': [{"id":"hp", + "name":"Human Phenotype Ontology", + "url":"https://purl.obolibrary.org/obo/hp.owl", + "version":"27-03-2020", + "namespacePrefix":"HP", + "iriPrefix":"https://purl.obolibrary.org/obo/HP_"}] }, 'beaconHandovers': beacon_handovers(), } From d0e1fac1c3ff082ba86ef9089aec50991588a916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20L=C3=B3pez-Doriga?= Date: Thu, 25 Apr 2024 09:26:59 +0200 Subject: [PATCH 2/8] resources updated --- beacon/response/build_response.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/beacon/response/build_response.py b/beacon/response/build_response.py index 866dd9ac..11e42926 100644 --- a/beacon/response/build_response.py +++ b/beacon/response/build_response.py @@ -300,12 +300,18 @@ def build_filtering_terms_response(data, # TODO: 'extendedInfo': build_extended_info(), 'response': { 'filteringTerms': data, - 'resources': [{"id":"hp", - "name":"Human Phenotype Ontology", - "url":"https://purl.obolibrary.org/obo/hp.owl", - "version":"27-03-2020", - "namespacePrefix":"HP", - "iriPrefix":"https://purl.obolibrary.org/obo/HP_"}] + 'resources': [{"id": "hp","name": "Human Phenotype Ontology","url": "https://purl.obolibrary.org/obo/hp.owl","version": "27-03-2020","namespacePrefix": "HP","iriPrefix": "https://purl.obolibrary.org/obo/HP_"}, + {"id": "icd10","name": "International Classification of Diseaes 10th edition","version": "17-03-2008","namespacePrefix": "ICD10"}, + {"id": "ncit","name": "National Cancer Institute Thesaurus","url": "https://purl.obolibrary.org/obo/ncit.owl","version": "19-10-2023","namespacePrefix": "NCIT","iriPrefix": "https://purl.obolibrary.org/obo/NCIT_"}, + {"id": "loinc","name": "Logical Observation Identifiers Names and Codes","url": "https://loinc.org/download/loinc-complete/","version": "19-10-2023","namespacePrefix": "LOINC"}, + {"id": "gaz","name": "Gazetteer","url": "https://purl.obolibrary.org/obo/gaz.owl","namespacePrefix": "GAZ","iriPrefix": "https://purl.obolibrary.org/obo/GAZ_"}, + {"id": "opcs4","name": "Office of Population Censuses and Surveys 4th revision","version": "01-04-2023","namespacePrefix": "OPCS4"}, + {"id": "genepio","name": "Genomic Epidemiology Ontology","version": "19-08-2023","namespacePrefix": "GENEPIO"}, + {"id": "obi","name": "Ontology for Biomedical Investigations","url": "http://purl.obolibrary.org/obo/obi.owl","version": "19-01-2024","namespacePrefix": "OBI","iriPrefix": "https://purl.obolibrary.org/obo/OBI_"}, + {"id": "efo","name": "Experimental Factor Ontology","version": "15-04-2024","namespacePrefix": "EFO"}, + {"id": "uberon","name": "Uber-anatomy ontology","version": "22-03-2024","namespacePrefix": "UBERON"}, + {"id": "doid","name": "Human Disease Ontology","url": "http://purl.obolibrary.org/obo/doid.owl","version": "28-03-2024","namespacePrefix": "DOID","iriPrefix": "https://purl.obolibrary.org/obo/DOID_"}, + {"id": "geno","name": "GENO ontology","version": "08-10-2023","namespacePrefix": "GENO"}] }, 'beaconHandovers': beacon_handovers(), } From 3e691f40c6a9cacb6ab222549d5b7eae9240113b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20L=C3=B3pez-Doriga?= Date: Thu, 25 Apr 2024 17:53:23 +0200 Subject: [PATCH 3/8] adding error response --- beacon/request/handlers.py | 429 ++++++++++++++++-------------- beacon/request/routes.py | 8 +- beacon/response/build_response.py | 16 ++ beacon/response/error.py | 22 ++ beacon/response/info.py | 81 +++--- beacon/response/service_info.py | 13 +- 6 files changed, 325 insertions(+), 244 deletions(-) create mode 100644 beacon/response/error.py diff --git a/beacon/request/handlers.py b/beacon/request/handlers.py index 0aad25e2..77b61ea0 100644 --- a/beacon/request/handlers.py +++ b/beacon/request/handlers.py @@ -7,6 +7,7 @@ from .. import conf import yaml import jwt +import requests from beacon.request import ontologies from beacon.request.model import Granularity, RequestParams @@ -16,7 +17,8 @@ build_beacon_boolean_response, build_beacon_count_response, build_filtering_terms_response, - build_beacon_resultset_response_by_dataset + build_beacon_resultset_response_by_dataset, + build_beacon_error_response ) from beacon.utils.stream import json_stream from beacon.db.datasets import get_datasets @@ -27,204 +29,70 @@ def collection_handler(db_fn, request=None): async def wrapper(request: Request): - # Get params - json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} - qparams = RequestParams(**json_body).from_request(request) - entry_id = request.match_info["id"] if "id" in request.match_info else None - # Get response - entity_schema, count, records = db_fn(entry_id, qparams) - response_converted = ( - [r for r in records] if records else [] - ) - response = build_beacon_collection_response( - response_converted, count, qparams, lambda x, y: x, entity_schema - ) + try: + # Get params + json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} + qparams = RequestParams(**json_body).from_request(request) + entry_id = request.match_info["id"] if "id" in request.match_info else None + # Get response + entity_schema, count, records = db_fn(entry_id, qparams) + response_converted = ( + [r for r in records] if records else [] + ) + response = build_beacon_collection_response( + response_converted, count, qparams, lambda x, y: x, entity_schema + ) + except Exception as err: + qparams = RequestParams(**json_body).from_request(request) + if str(err) == 'Not Found': + response = build_beacon_error_response(404, qparams, str(err)) + else: + response = build_beacon_error_response(500, qparams, str(err)) return await json_stream(request, response) return wrapper def generic_handler(db_fn, request=None): async def wrapper(request: Request): - # Get params - json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} - qparams = RequestParams(**json_body).from_request(request) - skip = qparams.query.pagination.skip - limit = qparams.query.pagination.limit - #LOG.debug(limit) - LOG.debug(qparams) - search_datasets = [] - authenticated=False - access_token = request.headers.get('Authorization') - - #LOG.debug(access_token) - if access_token is not None: - pass - else: - access_token = 'Bearer public' try: - specific_datasets = qparams.query.request_parameters['datasets'] - except Exception: - specific_datasets = [] - access_token = access_token[7:] # cut out 7 characters: len('Bearer ') - #LOG.debug(access_token) - - - - authorized_datasets, authenticated, username = await resolve_token(access_token, search_datasets) - - #LOG.debug(authorized_datasets) - #LOG.debug(username) - - ##LOG.debug('all datasets: %s', all_datasets) - LOG.info('resolved datasets: %s', authorized_datasets) - #LOG.debug(authenticated) - #LOG.debug(specific_datasets) - - - specific_datasets_unauthorized = [] - search_and_authorized_datasets = [] - # Get response - if specific_datasets != []: - for element in authorized_datasets: - if element in specific_datasets: - search_and_authorized_datasets.append(element) - for elemento in specific_datasets: - if elemento not in search_and_authorized_datasets: - specific_datasets_unauthorized.append(elemento) - qparams.query.request_parameters = {} - qparams.query.request_parameters['datasets'] = '*******' - _, _, datasets = get_datasets(None, qparams) - beacon_datasets = [ r for r in datasets ] - all_datasets = [r['id'] for r in beacon_datasets] + # Get params + json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} + qparams = RequestParams(**json_body).from_request(request) + skip = qparams.query.pagination.skip + limit = qparams.query.pagination.limit + #LOG.debug(limit) + LOG.debug(qparams) + search_datasets = [] + authenticated=False + access_token = request.headers.get('Authorization') - response_datasets = [ r['id'] for r in beacon_datasets if r['id'] in search_and_authorized_datasets] - - else: - qparams.query.request_parameters = {} - qparams.query.request_parameters['datasets'] = '*******' - _, _, datasets = get_datasets(None, qparams) - beacon_datasets = [ r for r in datasets ] - #LOG.debug(authorized_datasets) - specific_datasets = [ r['id'] for r in beacon_datasets if r['id'] not in authorized_datasets] - response_datasets = [ r['id'] for r in beacon_datasets if r['id'] in authorized_datasets] - #LOG.debug(specific_datasets) - #LOG.debug(response_datasets) - specific_datasets_unauthorized.append(specific_datasets) - - - qparams = RequestParams(**json_body).from_request(request) - include = qparams.query.include_resultset_responses - - - if access_token != 'public': - - - with open("/beacon/beacon/request/response_type.yml", 'r') as response_type_file: - response_type_dict = yaml.safe_load(response_type_file) - #LOG.debug(response_type_dict) - try: - response_type = response_type_dict[username] - except Exception: - #LOG.debug(Exception) - response_type = ['boolean'] - if response_type is not None: - for response_typed in response_type: - #LOG.debug(response_typed) - if response_typed == 'boolean': - qparams.query.requested_granularity = Granularity.BOOLEAN - elif response_typed == 'count': - qparams.query.requested_granularity = Granularity.COUNT - elif response_typed == 'record': - qparams.query.requested_granularity = Granularity.RECORD - - - entry_id = request.match_info.get('id', None) - if entry_id == None: - entry_id = request.match_info.get('variantInternalId', None) - datasets_docs={} - datasets_count={} - #LOG.debug(response_datasets) - new_count=0 - for dataset in response_datasets: - #LOG.debug(dataset) - entity_schema, count, dataset_count, records = db_fn(entry_id, qparams, dataset) - - if dataset_count != -1: - new_count+=dataset_count - datasets_docs[dataset]=records - datasets_count[dataset]=dataset_count - - if include != 'NONE': - count=new_count - else: - if limit == 0 or new_count < limit: + #LOG.debug(access_token) + if access_token is not None: pass else: - count = limit - - #LOG.debug(count) - response_converted = records - - - #LOG.debug(qparams.query.requested_granularity) - - if qparams.query.requested_granularity == Granularity.BOOLEAN: - response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - - elif qparams.query.requested_granularity == Granularity.COUNT: - if conf.max_beacon_granularity == Granularity.BOOLEAN: - response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - else: - response = build_beacon_count_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - - # qparams.query.requested_granularity == Granularity.RECORD: - else: - - if conf.max_beacon_granularity == Granularity.BOOLEAN: - response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - elif conf.max_beacon_granularity == Granularity.COUNT: - response = build_beacon_count_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - elif include == 'NONE': - response = build_beacon_resultset_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - else: - response = build_beacon_resultset_response_by_dataset(datasets_docs, datasets_count, count, qparams, lambda x, y: x, entity_schema) - - return await json_stream(request, response) - - return wrapper - -def filtering_terms_handler(db_fn, request=None): - async def wrapper(request: Request): - # Get params - json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} - qparams = RequestParams(**json_body).from_request(request) - - #LOG.debug(qparams) - - search_datasets = [] - authenticated=False - access_token = request.headers.get('Authorization') - #LOG.debug(access_token) - if access_token is not None: + access_token = 'Bearer public' try: specific_datasets = qparams.query.request_parameters['datasets'] except Exception: specific_datasets = [] access_token = access_token[7:] # cut out 7 characters: len('Bearer ') + #LOG.debug(access_token) + + + authorized_datasets, authenticated, username = await resolve_token(access_token, search_datasets) + + #LOG.debug(authorized_datasets) + #LOG.debug(username) - authorized_datasets, authenticated = await resolve_token(access_token, search_datasets) - ##LOG.debug(authorized_datasets) ##LOG.debug('all datasets: %s', all_datasets) LOG.info('resolved datasets: %s', authorized_datasets) - ##LOG.debug(authenticated) - ##LOG.debug(specific_datasets) + #LOG.debug(authenticated) + #LOG.debug(specific_datasets) + specific_datasets_unauthorized = [] - specific_datasets_unauthorized_and_found = [] - bio_list = [] search_and_authorized_datasets = [] - specific_search_datasets = [] # Get response if specific_datasets != []: for element in authorized_datasets: @@ -240,49 +108,204 @@ async def wrapper(request: Request): all_datasets = [r['id'] for r in beacon_datasets] response_datasets = [ r['id'] for r in beacon_datasets if r['id'] in search_and_authorized_datasets] - #LOG.debug(specific_search_datasets) - #LOG.debug(response_datasets) else: qparams.query.request_parameters = {} qparams.query.request_parameters['datasets'] = '*******' _, _, datasets = get_datasets(None, qparams) beacon_datasets = [ r for r in datasets ] + #LOG.debug(authorized_datasets) specific_datasets = [ r['id'] for r in beacon_datasets if r['id'] not in authorized_datasets] response_datasets = [ r['id'] for r in beacon_datasets if r['id'] in authorized_datasets] - ##LOG.debug(specific_datasets) - ##LOG.debug(response_datasets) + #LOG.debug(specific_datasets) + #LOG.debug(response_datasets) + specific_datasets_unauthorized.append(specific_datasets) - else: - list_of_dataset_dicts=[] - qparams = RequestParams(**json_body).from_request(request) - + qparams = RequestParams(**json_body).from_request(request) + include = qparams.query.include_resultset_responses - entry_id = request.match_info.get('id', None) - entity_schema, count, records = db_fn(entry_id, qparams) - - response_converted = records - - if qparams.query.requested_granularity == Granularity.BOOLEAN: - response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - - elif qparams.query.requested_granularity == Granularity.COUNT: - if conf.max_beacon_granularity == Granularity.BOOLEAN: + if access_token != 'public': + + + with open("/beacon/beacon/request/response_type.yml", 'r') as response_type_file: + response_type_dict = yaml.safe_load(response_type_file) + #LOG.debug(response_type_dict) + try: + response_type = response_type_dict[username] + except Exception: + #LOG.debug(Exception) + response_type = ['boolean'] + if response_type is not None: + for response_typed in response_type: + #LOG.debug(response_typed) + if response_typed == 'boolean': + qparams.query.requested_granularity = Granularity.BOOLEAN + elif response_typed == 'count': + qparams.query.requested_granularity = Granularity.COUNT + elif response_typed == 'record': + qparams.query.requested_granularity = Granularity.RECORD + + + entry_id = request.match_info.get('id', None) + if entry_id == None: + entry_id = request.match_info.get('variantInternalId', None) + datasets_docs={} + datasets_count={} + #LOG.debug(response_datasets) + new_count=0 + for dataset in response_datasets: + #LOG.debug(dataset) + entity_schema, count, dataset_count, records = db_fn(entry_id, qparams, dataset) + + if dataset_count != -1: + new_count+=dataset_count + datasets_docs[dataset]=records + datasets_count[dataset]=dataset_count + + if include != 'NONE': + count=new_count + else: + if limit == 0 or new_count < limit: + pass + else: + count = limit + + #LOG.debug(count) + response_converted = records + + + #LOG.debug(qparams.query.requested_granularity) + + if qparams.query.requested_granularity == Granularity.BOOLEAN: response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + + elif qparams.query.requested_granularity == Granularity.COUNT: + if conf.max_beacon_granularity == Granularity.BOOLEAN: + response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + else: + response = build_beacon_count_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + + # qparams.query.requested_granularity == Granularity.RECORD: else: - response = build_beacon_count_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - - # qparams.query.requested_granularity == Granularity.RECORD: - else: - if conf.max_beacon_granularity == Granularity.BOOLEAN: + if conf.max_beacon_granularity == Granularity.BOOLEAN: + response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + elif conf.max_beacon_granularity == Granularity.COUNT: + response = build_beacon_count_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + elif include == 'NONE': + response = build_beacon_resultset_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + else: + response = build_beacon_resultset_response_by_dataset(datasets_docs, datasets_count, count, qparams, lambda x, y: x, entity_schema) + except Exception as err: + qparams = RequestParams(**json_body).from_request(request) + if str(err) == 'Not Found': + response = build_beacon_error_response(404, qparams, str(err)) + else: + response = build_beacon_error_response(500, qparams, str(err)) + + return await json_stream(request, response) + + return wrapper + +def filtering_terms_handler(db_fn, request=None): + async def wrapper(request: Request): + try: + # Get params + json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} + qparams = RequestParams(**json_body).from_request(request) + + #LOG.debug(qparams) + + search_datasets = [] + authenticated=False + access_token = request.headers.get('Authorization') + #LOG.debug(access_token) + if access_token is not None: + try: + specific_datasets = qparams.query.request_parameters['datasets'] + except Exception: + specific_datasets = [] + access_token = access_token[7:] # cut out 7 characters: len('Bearer ') + + + authorized_datasets, authenticated = await resolve_token(access_token, search_datasets) + ##LOG.debug(authorized_datasets) + ##LOG.debug('all datasets: %s', all_datasets) + LOG.info('resolved datasets: %s', authorized_datasets) + ##LOG.debug(authenticated) + ##LOG.debug(specific_datasets) + + specific_datasets_unauthorized = [] + specific_datasets_unauthorized_and_found = [] + bio_list = [] + search_and_authorized_datasets = [] + specific_search_datasets = [] + # Get response + if specific_datasets != []: + for element in authorized_datasets: + if element in specific_datasets: + search_and_authorized_datasets.append(element) + for elemento in specific_datasets: + if elemento not in search_and_authorized_datasets: + specific_datasets_unauthorized.append(elemento) + qparams.query.request_parameters = {} + qparams.query.request_parameters['datasets'] = '*******' + _, _, datasets = get_datasets(None, qparams) + beacon_datasets = [ r for r in datasets ] + all_datasets = [r['id'] for r in beacon_datasets] + + response_datasets = [ r['id'] for r in beacon_datasets if r['id'] in search_and_authorized_datasets] + #LOG.debug(specific_search_datasets) + #LOG.debug(response_datasets) + + else: + qparams.query.request_parameters = {} + qparams.query.request_parameters['datasets'] = '*******' + _, _, datasets = get_datasets(None, qparams) + beacon_datasets = [ r for r in datasets ] + specific_datasets = [ r['id'] for r in beacon_datasets if r['id'] not in authorized_datasets] + response_datasets = [ r['id'] for r in beacon_datasets if r['id'] in authorized_datasets] + ##LOG.debug(specific_datasets) + ##LOG.debug(response_datasets) + + else: + list_of_dataset_dicts=[] + + qparams = RequestParams(**json_body).from_request(request) + + + entry_id = request.match_info.get('id', None) + entity_schema, count, records = db_fn(entry_id, qparams) + + + response_converted = records + + if qparams.query.requested_granularity == Granularity.BOOLEAN: response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) - elif conf.max_beacon_granularity == Granularity.COUNT: - response = build_beacon_count_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + + elif qparams.query.requested_granularity == Granularity.COUNT: + if conf.max_beacon_granularity == Granularity.BOOLEAN: + response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + else: + response = build_beacon_count_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + + # qparams.query.requested_granularity == Granularity.RECORD: + else: + + if conf.max_beacon_granularity == Granularity.BOOLEAN: + response = build_beacon_boolean_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + elif conf.max_beacon_granularity == Granularity.COUNT: + response = build_beacon_count_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + else: + response = build_filtering_terms_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + except Exception as err: + qparams = RequestParams(**json_body).from_request(request) + if str(err) == 'Not Found': + response = build_beacon_error_response(404, qparams, str(err)) else: - response = build_filtering_terms_response(response_converted, count, qparams, lambda x, y: x, entity_schema) + response = build_beacon_error_response(500, qparams, str(err)) return await json_stream(request, response) diff --git a/beacon/request/routes.py b/beacon/request/routes.py index 2b769e15..f8c30452 100644 --- a/beacon/request/routes.py +++ b/beacon/request/routes.py @@ -2,7 +2,7 @@ from beacon.db import analyses, biosamples, cohorts, datasets, g_variants, individuals, runs, filtering_terms from beacon.request.handlers import collection_handler, generic_handler, filtering_terms_handler -from beacon.response import framework, info, service_info +from beacon.response import framework, info, service_info, error routes = [ @@ -76,6 +76,9 @@ web.get('/api/runs/{id}/g_variants', generic_handler(db_fn=runs.get_variants_of_run)), web.get('/api/runs/{id}/analyses', generic_handler(db_fn=runs.get_analyses_of_run)), + web.get('/api/{tail:.*}', error.handler), + web.get('/{tail:.*}', error.handler), + ######################################## @@ -136,4 +139,7 @@ web.post('/api/runs/{id}/g_variants', generic_handler(db_fn=runs.get_variants_of_run)), web.post('/api/runs/{id}/analyses', generic_handler(db_fn=runs.get_analyses_of_run)), + web.post('/api/{tail:.*}', error.handler), + web.post('/{tail:.*}', error.handler) + ] diff --git a/beacon/response/build_response.py b/beacon/response/build_response.py index 11e42926..de0411fa 100644 --- a/beacon/response/build_response.py +++ b/beacon/response/build_response.py @@ -253,6 +253,22 @@ def build_beacon_info_response(data, qparams, func_response_type, authorized_dat return beacon_response +######################################## +# Error Response +######################################## + +def build_beacon_error_response(errorCode, qparams, errorMessage): + + beacon_response = { + 'meta': build_meta(qparams, None, Granularity.RECORD), + 'error': { + 'errorCode': str(errorCode), + 'errorMessage': str(errorMessage) + } + } + + return beacon_response + ######################################## # Service Info Response ######################################## diff --git a/beacon/response/error.py b/beacon/response/error.py new file mode 100644 index 00000000..efa5072e --- /dev/null +++ b/beacon/response/error.py @@ -0,0 +1,22 @@ +import subprocess +import logging +import json +from aiohttp.web_request import Request +from beacon.db.datasets import get_datasets +from beacon.request import RequestParams +from beacon.response.build_response import build_beacon_error_response +from beacon.utils.auth import resolve_token +from beacon.utils.stream import json_stream +from bson import json_util + +LOG = logging.getLogger(__name__) + +async def handler(request: Request): + LOG.error('Running an error request') + + # Fetch datasets info + json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} + qparams = RequestParams(**json_body).from_request(request) + + response_converted= build_beacon_error_response(404, qparams, 'Not Found') + return await json_stream(request, response_converted) diff --git a/beacon/response/info.py b/beacon/response/info.py index 812f299f..7d8f6144 100644 --- a/beacon/response/info.py +++ b/beacon/response/info.py @@ -16,7 +16,7 @@ from aiohttp.web_request import Request from beacon.db.datasets import get_datasets from beacon.request import RequestParams -from beacon.response.build_response import build_beacon_info_response +from beacon.response.build_response import build_beacon_info_response, build_beacon_error_response from beacon.utils.auth import resolve_token from beacon.utils.stream import json_stream from bson import json_util @@ -24,46 +24,53 @@ LOG = logging.getLogger(__name__) async def handler(request: Request): - LOG.info('Running a GET info request') + try: + LOG.info('Running a GET info request') - # Fetch datasets info - json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} - qparams = RequestParams(**json_body).from_request(request) - _, _, datasets = get_datasets(None, qparams) - beacon_datasets = [ r for r in datasets ] - - all_datasets = [ r['_id'] for r in beacon_datasets] - specific_datasets = [ r['id'] for r in beacon_datasets] - LOG.debug(specific_datasets) + # Fetch datasets info + json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} + qparams = RequestParams(**json_body).from_request(request) + _, _, datasets = get_datasets(None, qparams) + beacon_datasets = [ r for r in datasets ] + + all_datasets = [ r['_id'] for r in beacon_datasets] + specific_datasets = [ r['id'] for r in beacon_datasets] + LOG.debug(specific_datasets) - search_datasets = [] - authenticated=False - access_token = request.headers.get('Authorization') - LOG.debug(access_token) - if access_token is not None: - access_token = access_token[7:] # cut out 7 characters: len('Bearer ') - - authorized_datasets, authenticated = await resolve_token(access_token, search_datasets) - LOG.debug(authorized_datasets) - LOG.debug('all datasets: %s', all_datasets) - LOG.info('resolved datasets: %s', authorized_datasets) - LOG.debug(authenticated) + search_datasets = [] + authenticated=False + access_token = request.headers.get('Authorization') + LOG.debug(access_token) + if access_token is not None: + access_token = access_token[7:] # cut out 7 characters: len('Bearer ') + + authorized_datasets, authenticated = await resolve_token(access_token, search_datasets) + LOG.debug(authorized_datasets) + LOG.debug('all datasets: %s', all_datasets) + LOG.info('resolved datasets: %s', authorized_datasets) + LOG.debug(authenticated) - specific_datasets_authorized = [] + specific_datasets_authorized = [] - if not specific_datasets: - for auth_element in authorized_datasets: - specific_datasets_authorized = [ r for r in beacon_datasets if r['id'] == auth_element] + if not specific_datasets: + for auth_element in authorized_datasets: + specific_datasets_authorized = [ r for r in beacon_datasets if r['id'] == auth_element] + else: + for element in specific_datasets: + if element in authorized_datasets: + specific_datasets_authorized = [ r for r in beacon_datasets if r['id'] == element] else: - for element in specific_datasets: - if element in authorized_datasets: - specific_datasets_authorized = [ r for r in beacon_datasets if r['id'] == element] - else: - specific_datasets_authorized = [] + specific_datasets_authorized = [] - - response_converted = build_beacon_info_response(specific_datasets_authorized, - qparams, - lambda x,y,z: x, - authorized_datasets if authenticated else []) + + response_converted = build_beacon_info_response(specific_datasets_authorized, + qparams, + lambda x,y,z: x, + authorized_datasets if authenticated else []) + except Exception as err: + qparams = RequestParams(**json_body).from_request(request) + if str(err) == 'Not Found': + response_converted = build_beacon_error_response(404, qparams, str(err)) + else: + response_converted = build_beacon_error_response(500, qparams, str(err)) return await json_stream(request, response_converted) diff --git a/beacon/response/service_info.py b/beacon/response/service_info.py index 4dadcf83..95f7f3b7 100644 --- a/beacon/response/service_info.py +++ b/beacon/response/service_info.py @@ -1,11 +1,18 @@ import logging from aiohttp.web_request import Request -from beacon.response.build_response import build_beacon_service_info_response +from beacon.response.build_response import build_beacon_service_info_response, build_beacon_error_response from beacon.utils.stream import json_stream LOG = logging.getLogger(__name__) async def handler(request: Request): - LOG.info('Running a GET service info request') - response_converted = build_beacon_service_info_response() + try: + LOG.info('Running a GET service info request') + response_converted = build_beacon_service_info_response() + except Exception as err: + qparams = '' + if str(err) == 'Not Found': + response_converted = build_beacon_error_response(404, qparams, str(err)) + else: + response_converted = build_beacon_error_response(500, qparams, str(err)) return await json_stream(request, response_converted) From 2a7bf457f50613d307b6896c52cb1993967e04fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20L=C3=B3pez-Doriga?= Date: Thu, 25 Apr 2024 19:01:03 +0200 Subject: [PATCH 4/8] adding more error responses --- beacon/request/handlers.py | 10 +++++++--- beacon/request/model.py | 5 ++++- beacon/response/build_response.py | 24 ++++++++++++++++-------- beacon/response/error.py | 3 +-- beacon/response/info.py | 4 +++- beacon/response/service_info.py | 2 ++ 6 files changed, 33 insertions(+), 15 deletions(-) diff --git a/beacon/request/handlers.py b/beacon/request/handlers.py index 77b61ea0..37490d34 100644 --- a/beacon/request/handlers.py +++ b/beacon/request/handlers.py @@ -43,7 +43,7 @@ async def wrapper(request: Request): response_converted, count, qparams, lambda x, y: x, entity_schema ) except Exception as err: - qparams = RequestParams(**json_body).from_request(request) + qparams='' if str(err) == 'Not Found': response = build_beacon_error_response(404, qparams, str(err)) else: @@ -199,9 +199,11 @@ async def wrapper(request: Request): else: response = build_beacon_resultset_response_by_dataset(datasets_docs, datasets_count, count, qparams, lambda x, y: x, entity_schema) except Exception as err: - qparams = RequestParams(**json_body).from_request(request) + qparams = '' if str(err) == 'Not Found': response = build_beacon_error_response(404, qparams, str(err)) + elif str(err) == 'Bad Request': + response = build_beacon_error_response(400, qparams, str(err)) else: response = build_beacon_error_response(500, qparams, str(err)) @@ -301,9 +303,11 @@ async def wrapper(request: Request): else: response = build_filtering_terms_response(response_converted, count, qparams, lambda x, y: x, entity_schema) except Exception as err: - qparams = RequestParams(**json_body).from_request(request) + qparams = '' if str(err) == 'Not Found': response = build_beacon_error_response(404, qparams, str(err)) + elif str(err) == 'Bad Request': + response = build_beacon_error_response(400, qparams, str(err)) else: response = build_beacon_error_response(500, qparams, str(err)) diff --git a/beacon/request/model.py b/beacon/request/model.py index 262628b9..694333d2 100644 --- a/beacon/request/model.py +++ b/beacon/request/model.py @@ -6,6 +6,7 @@ from beacon import conf from humps.main import camelize from aiohttp.web_request import Request +from aiohttp import web LOG = logging.getLogger(__name__) @@ -97,8 +98,10 @@ def from_request(self, request: Request) -> Self: self.query.pagination.limit = int(v) elif k == "includeResultsetResponses": self.query.include_resultset_responses = IncludeResultsetResponses(v) - else: + elif k == 'filters': self.query.request_parameters[k] = v + else: + raise web.HTTPBadRequest return self def summary(self): diff --git a/beacon/response/build_response.py b/beacon/response/build_response.py index de0411fa..ba8dd3fc 100644 --- a/beacon/response/build_response.py +++ b/beacon/response/build_response.py @@ -15,14 +15,22 @@ def build_meta(qparams: RequestParams, entity_schema: Optional[DefaultSchemas], We assume that receivedRequest is the evaluated request (qparams) sent by the user. """ - - meta = { - 'beaconId': conf.beacon_id, - 'apiVersion': conf.api_version, - 'returnedGranularity': returned_granularity, - 'receivedRequestSummary': qparams.summary(), - 'returnedSchemas': [entity_schema.value] if entity_schema is not None else [] - } + try: + meta = { + 'beaconId': conf.beacon_id, + 'apiVersion': conf.api_version, + 'returnedGranularity': returned_granularity, + 'receivedRequestSummary': qparams.summary(), + 'returnedSchemas': [entity_schema.value] if entity_schema is not None else [] + } + except Exception: + meta = { + 'beaconId': conf.beacon_id, + 'apiVersion': conf.api_version, + 'returnedGranularity': returned_granularity, + 'receivedRequestSummary': qparams, + 'returnedSchemas': [entity_schema.value] if entity_schema is not None else [] + } return meta def build_response_summary(exists, num_total_results): diff --git a/beacon/response/error.py b/beacon/response/error.py index efa5072e..5d98385d 100644 --- a/beacon/response/error.py +++ b/beacon/response/error.py @@ -15,8 +15,7 @@ async def handler(request: Request): LOG.error('Running an error request') # Fetch datasets info - json_body = await request.json() if request.method == "POST" and request.has_body and request.can_read_body else {} - qparams = RequestParams(**json_body).from_request(request) + qparams = '' response_converted= build_beacon_error_response(404, qparams, 'Not Found') return await json_stream(request, response_converted) diff --git a/beacon/response/info.py b/beacon/response/info.py index 7d8f6144..4715722e 100644 --- a/beacon/response/info.py +++ b/beacon/response/info.py @@ -68,9 +68,11 @@ async def handler(request: Request): lambda x,y,z: x, authorized_datasets if authenticated else []) except Exception as err: - qparams = RequestParams(**json_body).from_request(request) + qparams = '' if str(err) == 'Not Found': response_converted = build_beacon_error_response(404, qparams, str(err)) + elif str(err) == 'Bad Request': + response_converted = build_beacon_error_response(400, qparams, str(err)) else: response_converted = build_beacon_error_response(500, qparams, str(err)) return await json_stream(request, response_converted) diff --git a/beacon/response/service_info.py b/beacon/response/service_info.py index 95f7f3b7..da1711a8 100644 --- a/beacon/response/service_info.py +++ b/beacon/response/service_info.py @@ -13,6 +13,8 @@ async def handler(request: Request): qparams = '' if str(err) == 'Not Found': response_converted = build_beacon_error_response(404, qparams, str(err)) + elif str(err) == 'Bad Request': + response_converted = build_beacon_error_response(400, qparams, str(err)) else: response_converted = build_beacon_error_response(500, qparams, str(err)) return await json_stream(request, response_converted) From b7751f6f7c8b5b109944501e768f87b65347d5d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20L=C3=B3pez-Doriga?= Date: Fri, 26 Apr 2024 10:06:12 +0200 Subject: [PATCH 5/8] error handling expanded --- beacon/request/handlers.py | 10 ++++++++++ beacon/response/info.py | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/beacon/request/handlers.py b/beacon/request/handlers.py index 37490d34..90b619c9 100644 --- a/beacon/request/handlers.py +++ b/beacon/request/handlers.py @@ -46,6 +46,12 @@ async def wrapper(request: Request): qparams='' if str(err) == 'Not Found': response = build_beacon_error_response(404, qparams, str(err)) + elif str(err) == 'Bad Request': + response = build_beacon_error_response(400, qparams, str(err)) + elif str(err) == 'Bad Gateway': + response = build_beacon_error_response(502, qparams, str(err)) + elif str(err) == 'Method Not Allowed': + response = build_beacon_error_response(405, qparams, str(err)) else: response = build_beacon_error_response(500, qparams, str(err)) return await json_stream(request, response) @@ -308,6 +314,10 @@ async def wrapper(request: Request): response = build_beacon_error_response(404, qparams, str(err)) elif str(err) == 'Bad Request': response = build_beacon_error_response(400, qparams, str(err)) + elif str(err) == 'Bad Gateway': + response = build_beacon_error_response(502, qparams, str(err)) + elif str(err) == 'Method Not Allowed': + response = build_beacon_error_response(405, qparams, str(err)) else: response = build_beacon_error_response(500, qparams, str(err)) diff --git a/beacon/response/info.py b/beacon/response/info.py index 4715722e..e875f250 100644 --- a/beacon/response/info.py +++ b/beacon/response/info.py @@ -73,6 +73,10 @@ async def handler(request: Request): response_converted = build_beacon_error_response(404, qparams, str(err)) elif str(err) == 'Bad Request': response_converted = build_beacon_error_response(400, qparams, str(err)) + elif str(err) == 'Bad Gateway': + response_converted = build_beacon_error_response(502, qparams, str(err)) + elif str(err) == 'Method Not Allowed': + response_converted = build_beacon_error_response(405, qparams, str(err)) else: response_converted = build_beacon_error_response(500, qparams, str(err)) return await json_stream(request, response_converted) From 941149c62d74d296a4020747531d39b7f42735d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20L=C3=B3pez-Doriga?= Date: Fri, 26 Apr 2024 10:20:51 +0200 Subject: [PATCH 6/8] more on error handling --- beacon/request/handlers.py | 10 +++++++--- beacon/request/model.py | 2 +- beacon/response/info.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/beacon/request/handlers.py b/beacon/request/handlers.py index 90b619c9..a9658d8f 100644 --- a/beacon/request/handlers.py +++ b/beacon/request/handlers.py @@ -47,7 +47,7 @@ async def wrapper(request: Request): if str(err) == 'Not Found': response = build_beacon_error_response(404, qparams, str(err)) elif str(err) == 'Bad Request': - response = build_beacon_error_response(400, qparams, str(err)) + response = build_beacon_error_response(400, qparams, str(err)+':'+str(err.text)) elif str(err) == 'Bad Gateway': response = build_beacon_error_response(502, qparams, str(err)) elif str(err) == 'Method Not Allowed': @@ -209,7 +209,11 @@ async def wrapper(request: Request): if str(err) == 'Not Found': response = build_beacon_error_response(404, qparams, str(err)) elif str(err) == 'Bad Request': - response = build_beacon_error_response(400, qparams, str(err)) + response = build_beacon_error_response(400, qparams, str(err)+':'+str(err.text)) + elif str(err) == 'Bad Gateway': + response = build_beacon_error_response(502, qparams, str(err)) + elif str(err) == 'Method Not Allowed': + response = build_beacon_error_response(405, qparams, str(err)) else: response = build_beacon_error_response(500, qparams, str(err)) @@ -313,7 +317,7 @@ async def wrapper(request: Request): if str(err) == 'Not Found': response = build_beacon_error_response(404, qparams, str(err)) elif str(err) == 'Bad Request': - response = build_beacon_error_response(400, qparams, str(err)) + build_beacon_error_response(400, qparams, str(err)+':'+str(err.text)) elif str(err) == 'Bad Gateway': response = build_beacon_error_response(502, qparams, str(err)) elif str(err) == 'Method Not Allowed': diff --git a/beacon/request/model.py b/beacon/request/model.py index 694333d2..1babff23 100644 --- a/beacon/request/model.py +++ b/beacon/request/model.py @@ -101,7 +101,7 @@ def from_request(self, request: Request) -> Self: elif k == 'filters': self.query.request_parameters[k] = v else: - raise web.HTTPBadRequest + raise web.HTTPBadRequest(text='request parameter introduced is not allowed') return self def summary(self): diff --git a/beacon/response/info.py b/beacon/response/info.py index e875f250..673435e5 100644 --- a/beacon/response/info.py +++ b/beacon/response/info.py @@ -72,7 +72,7 @@ async def handler(request: Request): if str(err) == 'Not Found': response_converted = build_beacon_error_response(404, qparams, str(err)) elif str(err) == 'Bad Request': - response_converted = build_beacon_error_response(400, qparams, str(err)) + response_converted = build_beacon_error_response(400, qparams, str(err)+':'+str(err.text)) elif str(err) == 'Bad Gateway': response_converted = build_beacon_error_response(502, qparams, str(err)) elif str(err) == 'Method Not Allowed': From a0439c479d7c6635f05f9a011ed02199ef61b2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20L=C3=B3pez-Doriga?= Date: Mon, 6 May 2024 12:35:16 +0200 Subject: [PATCH 7/8] changing Makefile --- deploy/Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/deploy/Makefile b/deploy/Makefile index a7326b38..fe567133 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -9,14 +9,14 @@ build: docker exec rimongo /bin/bash -c 'mongo beacon -u root -p example --authenticationDatabase admin --eval "db.genomicVariations.deleteMany({})"' docker exec rimongo /bin/bash -c 'mongo beacon -u root -p example --authenticationDatabase admin --eval "db.individuals.deleteMany({})"' docker exec rimongo /bin/bash -c 'mongo beacon -u root -p example --authenticationDatabase admin --eval "db.runs.deleteMany({})"' - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/analyses.json rimongo:tmp/analyses.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/biosamples.json rimongo:tmp/biosamples.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/cohorts.json rimongo:tmp/cohorts.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/datasets.json rimongo:tmp/datasets.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/genomicVariations.json rimongo:tmp/genomicVariations.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/individuals.json rimongo:tmp/individuals.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/runs.json rimongo:tmp/runs.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/filtering_terms.json rimongo:tmp/filtering_terms.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/analyses.json rimongo:tmp/analyses.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/biosamples.json rimongo:tmp/biosamples.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/cohorts.json rimongo:tmp/cohorts.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/datasets.json rimongo:tmp/datasets.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/genomicVariations.json rimongo:tmp/genomicVariations.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/individuals.json rimongo:tmp/individuals.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/runs.json rimongo:tmp/runs.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/filtering_terms.json rimongo:tmp/filtering_terms.json docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/datasets.json --collection datasets docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/individuals.json --collection individuals docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/cohorts.json --collection cohorts From dcef8c4bd0aff1d55bb3f3a21781fb28b427c40f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oriol=20L=C3=B3pez-Doriga?= Date: Tue, 14 May 2024 12:33:17 +0200 Subject: [PATCH 8/8] fixing bad request for request parameters --- beacon/request/model.py | 2 +- deploy/Makefile | 16 ---------------- deploy/README.md | 28 ++++++++++++++-------------- deploy/docker-compose.yml | 2 ++ 4 files changed, 17 insertions(+), 31 deletions(-) diff --git a/beacon/request/model.py b/beacon/request/model.py index 1babff23..14b669b7 100644 --- a/beacon/request/model.py +++ b/beacon/request/model.py @@ -98,7 +98,7 @@ def from_request(self, request: Request) -> Self: self.query.pagination.limit = int(v) elif k == "includeResultsetResponses": self.query.include_resultset_responses = IncludeResultsetResponses(v) - elif k == 'filters': + elif k == 'filters' or k in ["start", "end", "assemblyId", "referenceName", "referenceBases", "alternateBases", "variantType","variantMinLength","variantMaxLength","geneId","genomicAlleleShortForm","aminoacidChange","clinicalRelevance"]: self.query.request_parameters[k] = v else: raise web.HTTPBadRequest(text='request parameter introduced is not allowed') diff --git a/deploy/Makefile b/deploy/Makefile index fe567133..6120a59b 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -9,22 +9,6 @@ build: docker exec rimongo /bin/bash -c 'mongo beacon -u root -p example --authenticationDatabase admin --eval "db.genomicVariations.deleteMany({})"' docker exec rimongo /bin/bash -c 'mongo beacon -u root -p example --authenticationDatabase admin --eval "db.individuals.deleteMany({})"' docker exec rimongo /bin/bash -c 'mongo beacon -u root -p example --authenticationDatabase admin --eval "db.runs.deleteMany({})"' - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/analyses.json rimongo:tmp/analyses.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/biosamples.json rimongo:tmp/biosamples.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/cohorts.json rimongo:tmp/cohorts.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/datasets.json rimongo:tmp/datasets.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/genomicVariations.json rimongo:tmp/genomicVariations.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/individuals.json rimongo:tmp/individuals.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/runs.json rimongo:tmp/runs.json - docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/filtering_terms.json rimongo:tmp/filtering_terms.json - docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/datasets.json --collection datasets - docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/individuals.json --collection individuals - docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/cohorts.json --collection cohorts - docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/analyses.json --collection analyses - docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples - docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/runs.json --collection runs - docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations - docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/filtering_terms.json --collection filtering_terms docker cp data/analyses.json rimongo:tmp/analyses.json docker cp data/biosamples.json rimongo:tmp/biosamples.json docker cp data/cohorts.json rimongo:tmp/cohorts.json diff --git a/deploy/README.md b/deploy/README.md index 5c42c0ad..ced6707c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -40,23 +40,23 @@ With `mongo-express` we can see the contents of the database at [http://localhos To load the database we execute the following commands: ```bash -docker cp /path/to/analyses.json deploy_db_1:tmp/analyses.json -docker cp /path/to/biosamples.json deploy_db_1:tmp/biosamples.json -docker cp /path/to/cohorts.json deploy_db_1:tmp/cohorts.json -docker cp /path/to/datasets.json deploy_db_1:tmp/datasets.json -docker cp /path/to/genomicVariations.json deploy_db_1:tmp/genomicVariations.json -docker cp /path/to/individuals.json deploy_db_1:tmp/individuals.json -docker cp /path/to/runs.json deploy_db_1:tmp/runs.json +docker cp /path/to/analyses.json rimongo:tmp/analyses.json +docker cp /path/to/biosamples.json rimongo:tmp/biosamples.json +docker cp /path/to/cohorts.json rimongo:tmp/cohorts.json +docker cp /path/to/datasets.json rimongo:tmp/datasets.json +docker cp /path/to/genomicVariations.json rimongo:tmp/genomicVariations.json +docker cp /path/to/individuals.json rimongo:tmp/individuals.json +docker cp /path/to/runs.json rimongo:tmp/runs.json ``` ```bash -docker exec deploy_db_1 mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/datasets.json --collection datasets -docker exec deploy_db_1 mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/analyses.json --collection analyses -docker exec deploy_db_1 mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples -docker exec deploy_db_1 mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/cohorts.json --collection cohorts -docker exec deploy_db_1 mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations -docker exec deploy_db_1 mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/individuals.json --collection individuals -docker exec deploy_db_1 mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/runs.json --collection runs +docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/datasets.json --collection datasets +docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/analyses.json --collection analyses +docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples +docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/cohorts.json --collection cohorts +docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations +docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/individuals.json --collection individuals +docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/runs.json --collection runs ``` This loads the JSON files inside of the `data` folder into the MongoDB database container. Each time you import data you will have to create indexes for the queries to run smoothly. Please, check the next point about how to Create the indexes. diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 73f01539..61f8ad83 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -39,6 +39,7 @@ services: db: image: mongo:5 hostname: mongo + container_name: rimongo ports: - 27017:27017 environment: @@ -79,6 +80,7 @@ services: - ./conf.py:/beacon/beacon/conf.py - ./logger.yml:/beacon/beacon/logger.yml - ../beacon/request/datasets.yml:/beacon/beacon/request/datasets.yml + - ../beacon/request/cohorts.yml:/beacon/beacon/request/cohorts.yml - ../beacon/request/response_type.yml:/beacon/beacon/request/response_type.yml ports: - "5050:5050"