Skip to content

Commit

Permalink
Merge branch 'projectcalico-smc-upstream-fixes'
Browse files Browse the repository at this point in the history
  • Loading branch information
lavagetto committed Sep 26, 2015
2 parents 72ce525 + 1b40e52 commit 2b46dca
Show file tree
Hide file tree
Showing 6 changed files with 205 additions and 37 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,13 @@
bin
develop-eggs
eggs
.eggs
.idea
*.egg-info

tmp
build
dist
docs
etcd
.coverage
4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
'pyOpenSSL>=0.14'
]

setup(name='python-etcd',
setup(
name='python-etcd',
version=version,
description="A python client for etcd",
long_description=README + '\n\n' + NEWS,
Expand All @@ -42,5 +43,4 @@
install_requires=install_requires,
tests_require=test_requires,
test_suite='nose.collector',

)
20 changes: 12 additions & 8 deletions src/etcd/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,14 @@ def get_subtree(self, leaves_only=False):
#if the current result is a leaf, return itself
yield self
return
for n in self._children:
node = EtcdResult(None, n)
else:
# node is not a leaf
if not leaves_only:
#Return also dirs, not just value nodes
yield node
for child in node.get_subtree(leaves_only=leaves_only):
yield child
yield self
for n in self._children:
node = EtcdResult(None, n)
for child in node.get_subtree(leaves_only=leaves_only):
yield child
return

@property
Expand Down Expand Up @@ -120,7 +121,7 @@ class EtcdException(Exception):
Generic Etcd Exception.
"""
def __init__(self, message=None, payload=None):
super(Exception, self).__init__(message)
super(EtcdException, self).__init__(message)
self.payload = payload


Expand Down Expand Up @@ -193,7 +194,10 @@ class EtcdConnectionFailed(EtcdException):
"""
Connection to etcd failed.
"""
pass
def __init__(self, message=None, payload=None, cause=None):
super(EtcdConnectionFailed, self).__init__(message=message,
payload=payload)
self.cause = cause


class EtcdWatcherCleared(EtcdException):
Expand Down
69 changes: 42 additions & 27 deletions src/etcd/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -688,26 +688,32 @@ def election(self):

def _result_from_response(self, response):
""" Creates an EtcdResult from json dictionary """
raw_response = response.data
try:
res = json.loads(raw_response.decode('utf-8'))
except (TypeError, ValueError, UnicodeError) as e:
raise etcd.EtcdException(
'Server response was not valid JSON: %r' % e)
try:
res = json.loads(response.data.decode('utf-8'))
r = etcd.EtcdResult(**res)
if response.status == 201:
r.newKey = True
r.parse_headers(response)
return r
except Exception as e:
raise etcd.EtcdException(
'Unable to decode server response: %s' % e)
'Unable to decode server response: %r' % e)

def _next_server(self):
def _next_server(self, cause=None):
""" Selects the next server in the list, refreshes the server list. """
_log.debug("Selection next machine in cache. Available machines: %s",
self._machines_cache)
try:
mach = self._machines_cache.pop()
except IndexError:
_log.error("Machines cache is empty, no machines to try.")
raise etcd.EtcdConnectionFailed('No more machines in the cluster')
raise etcd.EtcdConnectionFailed('No more machines in the cluster',
cause=cause)
else:
_log.info("Selected new etcd server %s", mach)
return mach
Expand Down Expand Up @@ -752,7 +758,15 @@ def api_execute(self, path, method, params=None, timeout=None):
else:
raise etcd.EtcdException(
'HTTP method {} not supported'.format(method))


# Check the cluster ID hasn't changed under us. We use
# preload_content=False above so we can read the headers
# before we wait for the content of a watch.
self._check_cluster_id(response)
# Now force the data to be preloaded in order to trigger any
# IO-related errors in this method rather than when we try to
# access it later.
_ = response.data
# urllib3 doesn't wrap all httplib exceptions and earlier versions
# don't wrap socket errors either.
except (urllib3.exceptions.HTTPError,
Expand All @@ -765,42 +779,43 @@ def api_execute(self, path, method, params=None, timeout=None):
"server.")
# _next_server() raises EtcdException if there are no
# machines left to try, breaking out of the loop.
self._base_uri = self._next_server()
self._base_uri = self._next_server(cause=e)
some_request_failed = True
else:
_log.debug("Reconnection disabled, giving up.")
raise etcd.EtcdConnectionFailed(
"Connection to etcd failed due to %r" % e)
"Connection to etcd failed due to %r" % e,
cause=e
)
except:
_log.exception("Unexpected request failure, re-raising.")
raise

else:
# Check the cluster ID hasn't changed under us. We use
# preload_content=False above so we can read the headers
# before we wait for the content of a long poll.
cluster_id = response.getheader("x-etcd-cluster-id")
id_changed = (self.expected_cluster_id
and cluster_id is not None and
cluster_id != self.expected_cluster_id)
# Update the ID so we only raise the exception once.
old_expected_cluster_id = self.expected_cluster_id
self.expected_cluster_id = cluster_id
if id_changed:
# Defensive: clear the pool so that we connect afresh next
# time.
self.http.clear()
raise etcd.EtcdClusterIdChanged(
'The UUID of the cluster changed from {} to '
'{}.'.format(old_expected_cluster_id, cluster_id))


if some_request_failed:
if not self._use_proxies:
# The cluster may have changed since last invocation
self._machines_cache = self.machines
self._machines_cache.remove(self._base_uri)
return self._handle_server_response(response)

def _check_cluster_id(self, response):
cluster_id = response.getheader("x-etcd-cluster-id")
if not cluster_id:
_log.warning("etcd response did not contain a cluster ID")
return
id_changed = (self.expected_cluster_id and
cluster_id != self.expected_cluster_id)
# Update the ID so we only raise the exception once.
old_expected_cluster_id = self.expected_cluster_id
self.expected_cluster_id = cluster_id
if id_changed:
# Defensive: clear the pool so that we connect afresh next
# time.
self.http.clear()
raise etcd.EtcdClusterIdChanged(
'The UUID of the cluster changed from {} to '
'{}.'.format(old_expected_cluster_id, cluster_id))

def _handle_server_response(self, response):
""" Handles the server response """
if response.status in [200, 201]:
Expand Down
1 change: 1 addition & 0 deletions src/etcd/tests/integration/test_simple.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ class TestSimple(EtcdIntegrationTest):
def test_machines(self):
""" INTEGRATION: retrieve machines """
self.assertEquals(self.client.machines[0], 'http://127.0.0.1:6001')

def test_leader(self):
""" INTEGRATION: retrieve leader """
self.assertEquals(self.client.leader['clientURLs'], ['http://127.0.0.1:6001'])
Expand Down
144 changes: 144 additions & 0 deletions src/etcd/tests/unit/test_result.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
import etcd
import unittest
import json
import urllib3

try:
import mock
except ImportError:
from unittest import mock

class TestEtcdResult(unittest.TestCase):

def test_get_subtree_1_level(self):
"""
Test get_subtree() for a read with tree 1 level deep.
"""
response = {"node": {
'key': "/test",
'value': "hello",
'expiration': None,
'ttl': None,
'modifiedIndex': 5,
'createdIndex': 1,
'newKey': False,
'dir': False,
}}
result = etcd.EtcdResult(**response)
self.assertEqual(result.key, response["node"]["key"])
self.assertEqual(result.value, response["node"]["value"])

# Get subtree returns itself, whether or not leaves_only
subtree = list(result.get_subtree(leaves_only=True))
self.assertListEqual([result], subtree)
subtree = list(result.get_subtree(leaves_only=False))
self.assertListEqual([result], subtree)

def test_get_subtree_2_level(self):
"""
Test get_subtree() for a read with tree 2 levels deep.
"""
leaf0 = {
'key': "/test/leaf0",
'value': "hello1",
'expiration': None,
'ttl': None,
'modifiedIndex': 5,
'createdIndex': 1,
'newKey': False,
'dir': False,
}
leaf1 = {
'key': "/test/leaf1",
'value': "hello2",
'expiration': None,
'ttl': None,
'modifiedIndex': 6,
'createdIndex': 2,
'newKey': False,
'dir': False,
}
testnode = {"node": {
'key': "/test/",
'expiration': None,
'ttl': None,
'modifiedIndex': 6,
'createdIndex': 2,
'newKey': False,
'dir': True,
'nodes': [leaf0, leaf1]
}}
result = etcd.EtcdResult(**testnode)
self.assertEqual(result.key, "/test/")
self.assertTrue(result.dir)

# Get subtree returns just two leaves for leaves only.
subtree = list(result.get_subtree(leaves_only=True))
self.assertEqual(subtree[0].key, "/test/leaf0")
self.assertEqual(subtree[1].key, "/test/leaf1")
self.assertEqual(len(subtree), 2)

# Get subtree returns leaves and directory.
subtree = list(result.get_subtree(leaves_only=False))
self.assertEqual(subtree[0].key, "/test/")
self.assertEqual(subtree[1].key, "/test/leaf0")
self.assertEqual(subtree[2].key, "/test/leaf1")
self.assertEqual(len(subtree), 3)

def test_get_subtree_3_level(self):
"""
Test get_subtree() for a read with tree 3 levels deep.
"""
leaf0 = {
'key': "/test/mid0/leaf0",
'value': "hello1",
}
leaf1 = {
'key': "/test/mid0/leaf1",
'value': "hello2",
}
leaf2 = {
'key': "/test/mid1/leaf2",
'value': "hello1",
}
leaf3 = {
'key': "/test/mid1/leaf3",
'value': "hello2",
}
mid0 = {
'key': "/test/mid0/",
'dir': True,
'nodes': [leaf0, leaf1]
}
mid1 = {
'key': "/test/mid1/",
'dir': True,
'nodes': [leaf2, leaf3]
}
testnode = {"node": {
'key': "/test/",
'dir': True,
'nodes': [mid0, mid1]
}}
result = etcd.EtcdResult(**testnode)
self.assertEqual(result.key, "/test/")
self.assertTrue(result.dir)

# Get subtree returns just two leaves for leaves only.
subtree = list(result.get_subtree(leaves_only=True))
self.assertEqual(subtree[0].key, "/test/mid0/leaf0")
self.assertEqual(subtree[1].key, "/test/mid0/leaf1")
self.assertEqual(subtree[2].key, "/test/mid1/leaf2")
self.assertEqual(subtree[3].key, "/test/mid1/leaf3")
self.assertEqual(len(subtree), 4)

# Get subtree returns leaves and directory.
subtree = list(result.get_subtree(leaves_only=False))
self.assertEqual(subtree[0].key, "/test/")
self.assertEqual(subtree[1].key, "/test/mid0/")
self.assertEqual(subtree[2].key, "/test/mid0/leaf0")
self.assertEqual(subtree[3].key, "/test/mid0/leaf1")
self.assertEqual(subtree[4].key, "/test/mid1/")
self.assertEqual(subtree[5].key, "/test/mid1/leaf2")
self.assertEqual(subtree[6].key, "/test/mid1/leaf3")
self.assertEqual(len(subtree), 7)

0 comments on commit 2b46dca

Please sign in to comment.