DPNG-4948 Creating ES index in parallel won't be a problem.

Test are now using pytest.
Bumpversion is installed by tox.

.bumpversion.cfg

@@ -1,4 +1,4 @@
 [bumpversion]
 files = manifest.yml data_catalog/version.py
-current_version = 0.4.18
+current_version = 0.4.19
 

README.md

@@ -23,12 +23,12 @@ It is used to store, retrieve and to search over metadata describing data sets d
 
 ### Initial setup
 * You need Python (of course).
-* Install Tox: `sudo pip install tox`
+* Install Tox: `sudo pip install --upgrade tox`
 * Necessary for compilation of Python extensions: `sudo apt-get install python-dev`
 
 ### Tests
 * Be in `data-catalog` directory (project's source directory).
-* Run: `tox` (first run will take long) or `tox -r` (if you have added something to requirements.txt)
+* Run: `tox` (first run will take long)
 
 ### Configuration
 Configuration is handled through environment variables. They can be set in the "env" section of the CF (Cloud Foundry) manifest.
@@ -51,7 +51,7 @@ There are few development tools to handle or setup data in data-catalog:
 * **Everything should be done in a Python virtual environment (virtualenv).**
 * To switch the command line to the project's virtualenv run `source .tox/py27/bin/activate`. Run `deactivate` to disable virtualenv.
 * Downloading additional dependencies (libraries): `pip install <library_name>`
-* Install bumpversion tool using `sudo pip install bumpversion` and run `bumpversion patch --allow-dirty` to bump the version before committing code, that will go to master.
+* Run (in virtualenv) `bumpversion patch --allow-dirty` to bump the version before committing code that will go to master.
 
 #### Managing requirements
 * Dependencies need to be put in requirements.txt, requirements-normal.txt and requirements-native.txt.

data_catalog/app.py

@@ -24,7 +24,7 @@
 from flask_restful import Api
 from flask_restful_swagger import swagger
 from elasticsearch import Elasticsearch
-from elasticsearch.exceptions import ConnectionError
+import elasticsearch.exceptions
 
 from data_catalog.auth import Security
 from data_catalog.elastic_admin import ElasticSearchAdminResource
@@ -68,40 +68,6 @@ def handle_error(self, e):
         return self.make_response(response, code)
 
 
-_CONFIG = DCConfig()
-
-app = Flask(__name__)
-# our RESTful API wrapped with Swagger documentation
-api = swagger.docs(
-    ExceptionHandlingApi(app),
-    apiVersion=VERSION,
-    description='Data Catalog - enables search, retrieval and storage of metadata '
-                'describing data sets. ')
-
-api.add_resource(DataSetSearchResource, _CONFIG.app_base_path)
-api.add_resource(MetadataEntryResource, _CONFIG.app_base_path + '/<entry_id>')
-api.add_resource(DataSetCountResource, _CONFIG.app_base_path + '/count')
-api.add_resource(ElasticSearchAdminResource, _CONFIG.app_base_path + '/admin/elastic')
-
-ignore_for_auth = ['/api/spec']
-security = Security(ignore_for_auth)
-app.before_request(security.authenticate)
-
-
-def prepare_environment():
-    """Prepares ElasticSearch index for work if it's not yet ready."""
-    elastic_search = Elasticsearch('{}:{}'.format(
-        _CONFIG.elastic.elastic_hostname,
-        _CONFIG.elastic.elastic_port))
-    try:
-        if not elastic_search.indices.exists(_CONFIG.elastic.elastic_index):
-            elastic_search.indices.create(
-                index=_CONFIG.elastic.elastic_index,
-                body=_CONFIG.elastic.metadata_index_setup)
-    except ConnectionError:
-        sys.exit("Can't start because of no connection to ElasticSearch.")
-
-
 class PositiveMessageFilter(logging.Filter):
 
     """
@@ -113,7 +79,37 @@ def filter(record):
         return record.levelno not in (logging.WARNING, logging.ERROR)
 
 
-def configure_logging():
+def _prepare_environment(config):
+    """
+    Prepares ElasticSearch index for work if it's not yet ready.
+    :param `DCConfig` config:
+    """
+    elastic_search = Elasticsearch('{}:{}'.format(
+        config.elastic.elastic_hostname,
+        config.elastic.elastic_port))
+    try:
+        elastic_search.indices.create(
+            index=config.elastic.elastic_index,
+            body=config.elastic.metadata_index_setup)
+    except elasticsearch.exceptions.RequestError as ex:
+        # Multiple workers can be created at the same time and there's no way
+        # to tell ElasticSearch to create index only if it's not already created,
+        # so we need to attempt to create it and ignore the error that it throws
+        # when attemting to create an existing index.
+        if 'IndexAlreadyExists' in ex.args[1]:
+            print('400 caused by "index already created" - no need to panic.')
+        else:
+            raise ex
+    except elasticsearch.exceptions.TransportError:
+        print("Can't start because of no connection to ElasticSearch.")
+        raise
+
+
+def _configure_logging(config):
+    """
+    Configures proper logging for the application.
+    :param `DCConfig` config:
+    """
     log_formatter = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
 
     positive_handler = logging.StreamHandler(sys.stdout)
@@ -125,16 +121,36 @@ def configure_logging():
     negative_handler.setFormatter(log_formatter)
 
     root_logger = logging.getLogger()
-    root_logger.setLevel(_CONFIG.log_level)
+    root_logger.setLevel(config.log_level)
     root_logger.addHandler(positive_handler)
     root_logger.addHandler(negative_handler)
 
 
+def _create_app(config):
+    app = Flask(__name__)
+    # our RESTful API wrapped with Swagger documentation
+    api = swagger.docs(
+        ExceptionHandlingApi(app),
+        apiVersion=VERSION,
+        description='Data Catalog - enables search, retrieval and storage of metadata '
+                    'describing data sets. ')
+
+    api.add_resource(DataSetSearchResource, config.app_base_path)
+    api.add_resource(MetadataEntryResource, config.app_base_path + '/<entry_id>')
+    api.add_resource(DataSetCountResource, config.app_base_path + '/count')
+    api.add_resource(ElasticSearchAdminResource, config.app_base_path + '/admin/elastic')
+
+    security = Security(auth_exceptions=['/api/spec'])
+    app.before_request(security.authenticate)
+
+    return app
+
+
 def get_app():
     """
     To be used by the WSGI server.
     """
-    configure_logging()
-    prepare_environment()
-    return app
-
+    config = DCConfig()
+    _configure_logging(config)
+    _prepare_environment(config)
+    return _create_app(config)

data_catalog/version.py

@@ -15,4 +15,4 @@
 #
 
 # DO NOT TOUCH - version is changed automatically by Bumpversion
-VERSION = '0.4.18'
+VERSION = '0.4.19'

manifest.yml

@@ -15,4 +15,4 @@ applications:
   env:
     LOG_LEVEL: "INFO"
     # DO NOT TOUCH - version is changed automatically by Bumpversion
-    VERSION: "0.4.18"
+    VERSION: "0.4.19"

tests/base_test.py

@@ -21,16 +21,16 @@
 
 from data_catalog.configuration import (DCConfig, VCAP_APP_PORT, VCAP_SERVICES, VCAP_APPLICATION,
                                         LOG_LEVEL)
+import data_catalog.app
 
 
 class DataCatalogTestCase(unittest.TestCase):
     def setUp(self):
         setup_fake_env()
         self._config = DCConfig()
 
-        from data_catalog import app
-        app.app.config['TESTING'] = True
-        self.app = app.app
+        self.app = data_catalog.app._create_app(self._config)
+        self.app.config['TESTING'] = True
         self.client = self.app.test_client()
 
         self._disable_authentication()

tests/test_app.py

@@ -0,0 +1,62 @@
+#
+# Copyright (c) 2015 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Tests for the app initialization code in app.py.
+"""
+
+import elasticsearch
+from elasticsearch.exceptions import RequestError
+import mock
+import pytest
+
+import base_test
+
+import data_catalog.app as app
+from data_catalog.configuration import DCConfig
+
+
+@pytest.yield_fixture
+def mock_es_create():
+    with mock.patch.object(elasticsearch.client.indices.IndicesClient, 'create') as mock_create:
+        yield mock_create
+
+
+@pytest.fixture
+def test_config():
+    with base_test.fake_env():
+        config = DCConfig()
+    return config
+
+
+def test_first_prepare_environment(mock_es_create, test_config):
+    app._prepare_environment(test_config)
+
+    mock_es_create.assert_called_with(
+        index=test_config.elastic.elastic_index,
+        body=test_config.elastic.metadata_index_setup)
+
+
+def test_subsequent_prepare_environment(mock_es_create, test_config):
+    mock_es_create.side_effect = RequestError(400, 'IndexAlreadyExists errorito')
+    app._prepare_environment(test_config)
+
+
+def test_prepare_environment_err(mock_es_create, test_config):
+    mock_es_create.side_effect = RequestError(123, 'Some other error')
+    with pytest.raises(RequestError):
+        app._prepare_environment(test_config)
+

tools/elastic_migrate_tool.py

@@ -80,7 +80,7 @@ def insert_data(base_url, token):
     if (r.status_code == 200):
         print("data inserted")
     else:
-        print("problem with insert: ", r.status_code, r.text)
+        print("problem with insert:", r.status_code, r.text)
 
 class CheckUrlSchemeAction(argparse.Action):
     def __call__(self, parser, namespace, base_url, option_string=None):
@@ -109,6 +109,7 @@ def is_admin(user_token):
     :return: True if the user is an admin.
     :rtype: bool
     """
+    print('Provided token:', user_token)
     # get token without "bearer"
     token = user_token.split()[1]
     # take the middle part with payload
@@ -120,7 +121,7 @@ def is_admin(user_token):
 
 if __name__ == '__main__':
     args = parse_args()
-    
+
     if not is_admin(args.token):
         sys.exit('ERROR! You must have admin privilages (console.admin scope) to use this tool.')            
 

tox.ini

@@ -6,14 +6,18 @@ envlist = py27
 deps =
     -rrequirements-normal.txt
     -rrequirements-native.txt
+    # test / quality dependencies
+    pytest
+    pytest-cov
     coverage
     pylint
+    # additional tools
+    bumpversion    
     pipdeptree
+
 whitelist_externals =
     /bin/bash
+
 commands =
-    coverage erase
-    coverage run --source=data_catalog -m unittest discover -s tests
-    coverage report
-    coverage xml
+    py.test tests/ --cov-config .coveragerc --cov=data_catalog --cov-report term --cov-report xml
     bash -c "pylint data_catalog --rcfile=.pylintrc | tee pylint_report.txt; exit 0"