diff --git a/flocker/control/_diffing.py b/flocker/control/_diffing.py index 038d628fde..26c1b2833e 100644 --- a/flocker/control/_diffing.py +++ b/flocker/control/_diffing.py @@ -257,9 +257,3 @@ def compose_diffs(iterable_of_diffs): pvector().evolver() ).persistent() ) - - -# Ensure that the representation of a ``Diff`` is entirely serializable: -DIFF_SERIALIZABLE_CLASSES = [ - _Set, _Remove, _Add, Diff -] diff --git a/flocker/control/_generations.py b/flocker/control/_generations.py index 6333a444df..9714f67d04 100644 --- a/flocker/control/_generations.py +++ b/flocker/control/_generations.py @@ -10,7 +10,7 @@ from pyrsistent import PClass, field from ._model import GenerationHash -from ._persistence import make_generation_hash +from ._serialization import make_generation_hash from ._diffing import Diff, create_diff, compose_diffs diff --git a/flocker/control/_model.py b/flocker/control/_model.py index 9579a640cd..0feea61e99 100644 --- a/flocker/control/_model.py +++ b/flocker/control/_model.py @@ -26,8 +26,6 @@ from zope.interface import Interface, implementer -from ._diffing import DIFF_SERIALIZABLE_CLASSES - def _sequence_field(checked_class, suffix, item_type, optional, initial): """ @@ -1270,12 +1268,3 @@ class GenerationHash(PClass): mandatory=True, factory=_generation_hash_value_factory ) - - -# Classes that can be serialized to disk or sent over the network: -SERIALIZABLE_CLASSES = [ - Deployment, Node, DockerImage, Port, Link, RestartNever, RestartAlways, - RestartOnFailure, Application, Dataset, Manifestation, AttachedVolume, - NodeState, DeploymentState, NonManifestDatasets, Configuration, - Lease, Leases, PersistentState, GenerationHash -] + DIFF_SERIALIZABLE_CLASSES diff --git a/flocker/control/_persistence.py b/flocker/control/_persistence.py index f1e06da093..7fc36fb618 100644 --- a/flocker/control/_persistence.py +++ b/flocker/control/_persistence.py @@ -5,44 +5,36 @@ """ from base64 import b16encode -from calendar import timegm from datetime import datetime from json import dumps, loads from mmh3 import hash_bytes as mmh3_hash_bytes -from uuid import UUID -from collections import Set, Mapping, Iterable from eliot import Logger, write_traceback, MessageType, Field, ActionType -from pyrsistent import PRecord, PVector, PMap, PSet, pmap, PClass - from pytz import UTC -from twisted.python.filepath import FilePath from twisted.application.service import Service, MultiService from twisted.internet.defer import succeed from twisted.internet.task import LoopingCall -from weakref import WeakKeyDictionary - from ._model import ( - SERIALIZABLE_CLASSES, Deployment, Configuration, GenerationHash + Deployment, Configuration +) +from ._serialization import ( + wire_encode, + wire_decode, + to_unserialized_json, + _CLASS_MARKER, ) # The class at the root of the configuration tree. ROOT_CLASS = Deployment -# Serialization marker storing the class name: -_CLASS_MARKER = u"$__class__$" - # The latest configuration version. Configuration versions are # always integers. _CONFIG_VERSION = 5 -# Map of serializable class names to classes -_CONFIG_CLASS_MAP = {cls.__name__: cls for cls in SERIALIZABLE_CLASSES} - class ConfigurationMigrationError(Exception): """ @@ -197,278 +189,6 @@ def upgrade_from_v4(cls, config): return dumps(decoded_config) -def _to_serializables(obj): - """ - This function turns assorted types into serializable objects (objects that - can be serialized by the default JSON encoder). Note that this is done - shallowly for containers. For example, ``PClass``es will be turned into - dicts, but the values and keys of the dict might still not be serializable. - - It is up to higher layers to traverse containers recursively to achieve - full serialization. - - :param obj: The object to serialize. - - :returns: An object that is shallowly JSON serializable. - """ - if isinstance(obj, PRecord): - result = dict(obj) - result[_CLASS_MARKER] = obj.__class__.__name__ - return result - elif isinstance(obj, PClass): - result = obj._to_dict() - result[_CLASS_MARKER] = obj.__class__.__name__ - return result - elif isinstance(obj, PMap): - return {_CLASS_MARKER: u"PMap", u"values": dict(obj).items()} - elif isinstance(obj, (PSet, PVector, set)): - return list(obj) - elif isinstance(obj, FilePath): - return {_CLASS_MARKER: u"FilePath", - u"path": obj.path.decode("utf-8")} - elif isinstance(obj, UUID): - return {_CLASS_MARKER: u"UUID", - "hex": unicode(obj)} - elif isinstance(obj, datetime): - if obj.tzinfo is None: - raise ValueError( - "Datetime without a timezone: {}".format(obj)) - return {_CLASS_MARKER: u"datetime", - "seconds": timegm(obj.utctimetuple())} - return obj - - -def _is_pyrsistent(obj): - """ - Boolean check if an object is an instance of a pyrsistent object. - """ - return isinstance(obj, (PRecord, PClass, PMap, PSet, PVector)) - - -_BASIC_JSON_TYPES = frozenset([str, unicode, int, long, float, bool]) -_BASIC_JSON_LISTS = frozenset([list, tuple]) -_BASIC_JSON_COLLECTIONS = frozenset([dict]).union(_BASIC_JSON_LISTS) - - -_UNCACHED_SENTINEL = object() - - -_cached_dfs_serialize_cache = WeakKeyDictionary() - - -def _cached_dfs_serialize(input_object): - """ - This serializes an input object into something that can be serialized by - the python json encoder. - - This caches the serialization of pyrsistent objects in a - ``WeakKeyDictionary``, so the cache should be automatically cleared when - the input object that is cached is destroyed. - - :returns: An entirely serializable version of input_object. - """ - # Ensure this is a quick function for basic types: - if input_object is None: - return None - - # Note that ``type(x) in frozenset([str, int])`` is faster than - # ``isinstance(x, (str, int))``. - input_type = type(input_object) - if input_type in _BASIC_JSON_TYPES: - return input_object - - is_pyrsistent = False - if input_type in _BASIC_JSON_COLLECTIONS: - # Don't send basic collections through shallow object serialization, - # isinstance is not a very cheap operation. - obj = input_object - else: - if _is_pyrsistent(input_object): - is_pyrsistent = True - # Using ``dict.get`` and a sentinel rather than the more pythonic - # try/except KeyError for performance. This function is highly - # recursive and the KeyError is guaranteed to happen the first - # time every object is serialized. We do not want to incur the cost - # of a caught exception for every pyrsistent object ever - # serialized. - cached_value = _cached_dfs_serialize_cache.get(input_object, - _UNCACHED_SENTINEL) - if cached_value is not _UNCACHED_SENTINEL: - return cached_value - obj = _to_serializables(input_object) - - result = obj - - obj_type = type(obj) - if obj_type == dict: - result = dict((_cached_dfs_serialize(key), - _cached_dfs_serialize(value)) - for key, value in obj.iteritems()) - elif obj_type == list or obj_type == tuple: - result = list(_cached_dfs_serialize(x) for x in obj) - - if is_pyrsistent: - _cached_dfs_serialize_cache[input_object] = result - - return result - -# A couple tokens that are used below in the generation hash. -_NULLSET_TOKEN = mmh3_hash_bytes(b'NULLSET') -_MAPPING_TOKEN = mmh3_hash_bytes(b'MAPPING') -_STR_TOKEN = mmh3_hash_bytes(b'STRING') - -_generation_hash_cache = WeakKeyDictionary() - - -def _xor_bytes(aggregating_bytearray, updating_bytes): - """ - Aggregate bytes into a bytearray using XOR. - - This function has a somewhat particular function signature in order for it - to be compatible with a call to `reduce` - - :param bytearray aggregating_bytearray: Resulting bytearray to aggregate - the XOR of both input arguments byte-by-byte. - - :param bytes updating_bytes: Additional bytes to be aggregated into the - other argument. It is assumed that this has the same size as - aggregating_bytearray. - - :returns: aggregating_bytearray, after it has been modified by XORing all - of the bytes in the input bytearray with ``updating_bytes``. - """ - for i in xrange(len(aggregating_bytearray)): - aggregating_bytearray[i] ^= ord(updating_bytes[i]) - return aggregating_bytearray - - -def generation_hash(input_object): - """ - This computes the mmh3 hash for an input object, providing a consistent - hash of deeply persistent objects across python nodes and implementations. - - :returns: An mmh3 hash of input_object. - """ - # Ensure this is a quick function for basic types: - # Note that ``type(x) in frozenset([str, int])`` is faster than - # ``isinstance(x, (str, int))``. - input_type = type(input_object) - if ( - input_object is None or - input_type in _BASIC_JSON_TYPES - ): - if input_type == unicode: - input_type = bytes - input_object = input_object.encode('utf8') - - if input_type == bytes: - # Add a token to identify this as a string. This ensures that - # strings like str('5') are hashed to different values than values - # who have an identical JSON representation like int(5). - object_to_process = b''.join([_STR_TOKEN, bytes(input_object)]) - else: - # For non-string objects, just hash the JSON encoding. - object_to_process = dumps(input_object) - return mmh3_hash_bytes(object_to_process) - - is_pyrsistent = _is_pyrsistent(input_object) - if is_pyrsistent: - cached = _generation_hash_cache.get(input_object, _UNCACHED_SENTINEL) - if cached is not _UNCACHED_SENTINEL: - return cached - - object_to_process = input_object - - if isinstance(object_to_process, PClass): - object_to_process = object_to_process._to_dict() - - if isinstance(object_to_process, Mapping): - # Union a mapping token so that empty maps and empty sets have - # different hashes. - object_to_process = frozenset(object_to_process.iteritems()).union( - [_MAPPING_TOKEN] - ) - - if isinstance(object_to_process, Set): - sub_hashes = (generation_hash(x) for x in object_to_process) - result = bytes( - reduce(_xor_bytes, sub_hashes, bytearray(_NULLSET_TOKEN)) - ) - elif isinstance(object_to_process, Iterable): - result = mmh3_hash_bytes(b''.join( - generation_hash(x) for x in object_to_process - )) - else: - result = mmh3_hash_bytes(wire_encode(object_to_process)) - - if is_pyrsistent: - _generation_hash_cache[input_object] = result - - return result - - -def make_generation_hash(x): - """ - Creates a ``GenerationHash`` for a given argument. - - Simple helper to call ``generation_hash`` and wrap it in the - ``GenerationHash`` ``PClass``. - - :param x: The object to hash. - - :returns: The ``GenerationHash`` for the object. - """ - return GenerationHash( - hash_value=generation_hash(x) - ) - - -def wire_encode(obj): - """ - Encode the given model object into bytes. - - :param obj: An object from the configuration model, e.g. ``Deployment``. - :return bytes: Encoded object. - """ - return dumps(_cached_dfs_serialize(obj)) - - -def wire_decode(data): - """ - Decode the given model object from bytes. - - :param bytes data: Encoded object. - """ - def decode(dictionary): - class_name = dictionary.get(_CLASS_MARKER, None) - if class_name == u"FilePath": - return FilePath(dictionary.get(u"path").encode("utf-8")) - elif class_name == u"PMap": - return pmap(dictionary[u"values"]) - elif class_name == u"UUID": - return UUID(dictionary[u"hex"]) - elif class_name == u"datetime": - return datetime.fromtimestamp(dictionary[u"seconds"], UTC) - elif class_name in _CONFIG_CLASS_MAP: - dictionary = dictionary.copy() - dictionary.pop(_CLASS_MARKER) - return _CONFIG_CLASS_MAP[class_name].create(dictionary) - else: - return dictionary - - return loads(data, object_hook=decode) - - -def to_unserialized_json(obj): - """ - Convert a wire encodeable object into structured Python objects that - are JSON serializable. - - :param obj: An object that can be passed to ``wire_encode``. - :return: Python object that can be JSON serialized. - """ - return _cached_dfs_serialize(obj) - _DEPLOYMENT_FIELD = Field(u"configuration", to_unserialized_json) _LOG_STARTUP = MessageType(u"flocker-control:persistence:startup", [_DEPLOYMENT_FIELD]) diff --git a/flocker/control/_protocol.py b/flocker/control/_protocol.py index af03e98d8e..7a45c8c9ae 100644 --- a/flocker/control/_protocol.py +++ b/flocker/control/_protocol.py @@ -62,7 +62,7 @@ from twisted.application.internet import StreamServerEndpointService from twisted.protocols.tls import TLSMemoryBIOFactory -from ._persistence import wire_encode, wire_decode, make_generation_hash +from ._serialization import wire_encode, wire_decode, make_generation_hash from ._model import ( Deployment, DeploymentState, ChangeSource, UpdateNodeStateEra, BlockDeviceOwnership, DatasetAlreadyOwned, GenerationHash, diff --git a/flocker/control/_serialization.py b/flocker/control/_serialization.py new file mode 100644 index 0000000000..553f41777c --- /dev/null +++ b/flocker/control/_serialization.py @@ -0,0 +1,312 @@ +# Copyright ClusterHQ Inc. See LICENSE file for details. + +""" +Helpers for serialization and de-serialization of cluster model objects. +""" + +from calendar import timegm +from collections import Set, Mapping, Iterable +from datetime import datetime +from json import dumps, loads +from uuid import UUID +from weakref import WeakKeyDictionary + +from mmh3 import hash_bytes as mmh3_hash_bytes +from pyrsistent import PRecord, PVector, PMap, PSet, pmap, PClass +from pytz import UTC +from twisted.python.filepath import FilePath + +from ._model import ( + Deployment, Node, DockerImage, Port, Link, RestartNever, RestartAlways, + RestartOnFailure, Application, Dataset, Manifestation, AttachedVolume, + NodeState, DeploymentState, NonManifestDatasets, Configuration, + Lease, Leases, PersistentState, GenerationHash, +) +from ._diffing import _Set, _Remove, _Add, Diff + + +# Classes that can be serialized to disk or sent over the network: +SERIALIZABLE_CLASSES = [ + Deployment, Node, DockerImage, Port, Link, RestartNever, RestartAlways, + RestartOnFailure, Application, Dataset, Manifestation, AttachedVolume, + NodeState, DeploymentState, NonManifestDatasets, Configuration, + Lease, Leases, PersistentState, GenerationHash, + _Set, _Remove, _Add, Diff, +] + +# Map of serializable class names to classes +_CONFIG_CLASS_MAP = {cls.__name__: cls for cls in SERIALIZABLE_CLASSES} + +_cached_dfs_serialize_cache = WeakKeyDictionary() + +_BASIC_JSON_TYPES = frozenset([str, unicode, int, long, float, bool]) +_BASIC_JSON_LISTS = frozenset([list, tuple]) +_BASIC_JSON_COLLECTIONS = frozenset([dict]).union(_BASIC_JSON_LISTS) + +_UNCACHED_SENTINEL = object() + +# Serialization marker storing the class name: +_CLASS_MARKER = u"$__class__$" + + +# A couple tokens that are used below in the generation hash. +_NULLSET_TOKEN = mmh3_hash_bytes(b'NULLSET') +_MAPPING_TOKEN = mmh3_hash_bytes(b'MAPPING') +_STR_TOKEN = mmh3_hash_bytes(b'STRING') + +_generation_hash_cache = WeakKeyDictionary() + + +def _xor_bytes(aggregating_bytearray, updating_bytes): + """ + Aggregate bytes into a bytearray using XOR. + + This function has a somewhat particular function signature in order for it + to be compatible with a call to `reduce` + + :param bytearray aggregating_bytearray: Resulting bytearray to aggregate + the XOR of both input arguments byte-by-byte. + + :param bytes updating_bytes: Additional bytes to be aggregated into the + other argument. It is assumed that this has the same size as + aggregating_bytearray. + + :returns: aggregating_bytearray, after it has been modified by XORing all + of the bytes in the input bytearray with ``updating_bytes``. + """ + for i in xrange(len(aggregating_bytearray)): + aggregating_bytearray[i] ^= ord(updating_bytes[i]) + return aggregating_bytearray + + +def generation_hash(input_object): + """ + This computes the mmh3 hash for an input object, providing a consistent + hash of deeply persistent objects across python nodes and implementations. + + :returns: An mmh3 hash of input_object. + """ + # Ensure this is a quick function for basic types: + # Note that ``type(x) in frozenset([str, int])`` is faster than + # ``isinstance(x, (str, int))``. + input_type = type(input_object) + if ( + input_object is None or + input_type in _BASIC_JSON_TYPES + ): + if input_type == unicode: + input_type = bytes + input_object = input_object.encode('utf8') + + if input_type == bytes: + # Add a token to identify this as a string. This ensures that + # strings like str('5') are hashed to different values than values + # who have an identical JSON representation like int(5). + object_to_process = b''.join([_STR_TOKEN, bytes(input_object)]) + else: + # For non-string objects, just hash the JSON encoding. + object_to_process = dumps(input_object) + return mmh3_hash_bytes(object_to_process) + + is_pyrsistent = _is_pyrsistent(input_object) + if is_pyrsistent: + cached = _generation_hash_cache.get(input_object, _UNCACHED_SENTINEL) + if cached is not _UNCACHED_SENTINEL: + return cached + + object_to_process = input_object + + if isinstance(object_to_process, PClass): + object_to_process = object_to_process._to_dict() + + if isinstance(object_to_process, Mapping): + # Union a mapping token so that empty maps and empty sets have + # different hashes. + object_to_process = frozenset(object_to_process.iteritems()).union( + [_MAPPING_TOKEN] + ) + + if isinstance(object_to_process, Set): + sub_hashes = (generation_hash(x) for x in object_to_process) + result = bytes( + reduce(_xor_bytes, sub_hashes, bytearray(_NULLSET_TOKEN)) + ) + elif isinstance(object_to_process, Iterable): + result = mmh3_hash_bytes(b''.join( + generation_hash(x) for x in object_to_process + )) + else: + result = mmh3_hash_bytes(wire_encode(object_to_process)) + + if is_pyrsistent: + _generation_hash_cache[input_object] = result + + return result + + +def make_generation_hash(x): + """ + Creates a ``GenerationHash`` for a given argument. + + Simple helper to call ``generation_hash`` and wrap it in the + ``GenerationHash`` ``PClass``. + + :param x: The object to hash. + + :returns: The ``GenerationHash`` for the object. + """ + return GenerationHash( + hash_value=generation_hash(x) + ) + + +def to_unserialized_json(obj): + """ + Convert a wire encodeable object into structured Python objects that + are JSON serializable. + + :param obj: An object that can be passed to ``wire_encode``. + :return: Python object that can be JSON serialized. + """ + return _cached_dfs_serialize(obj) + + +def _to_serializables(obj): + """ + This function turns assorted types into serializable objects (objects that + can be serialized by the default JSON encoder). Note that this is done + shallowly for containers. For example, ``PClass``es will be turned into + dicts, but the values and keys of the dict might still not be serializable. + + It is up to higher layers to traverse containers recursively to achieve + full serialization. + + :param obj: The object to serialize. + + :returns: An object that is shallowly JSON serializable. + """ + if isinstance(obj, PRecord): + result = dict(obj) + result[_CLASS_MARKER] = obj.__class__.__name__ + return result + elif isinstance(obj, PClass): + result = obj._to_dict() + result[_CLASS_MARKER] = obj.__class__.__name__ + return result + elif isinstance(obj, PMap): + return {_CLASS_MARKER: u"PMap", u"values": dict(obj).items()} + elif isinstance(obj, (PSet, PVector, set)): + return list(obj) + elif isinstance(obj, FilePath): + return {_CLASS_MARKER: u"FilePath", + u"path": obj.path.decode("utf-8")} + elif isinstance(obj, UUID): + return {_CLASS_MARKER: u"UUID", + "hex": unicode(obj)} + elif isinstance(obj, datetime): + if obj.tzinfo is None: + raise ValueError( + "Datetime without a timezone: {}".format(obj)) + return {_CLASS_MARKER: u"datetime", + "seconds": timegm(obj.utctimetuple())} + return obj + + +def _is_pyrsistent(obj): + """ + Boolean check if an object is an instance of a pyrsistent object. + """ + return isinstance(obj, (PRecord, PClass, PMap, PSet, PVector)) + + +def _cached_dfs_serialize(input_object): + """ + This serializes an input object into something that can be serialized by + the python json encoder. + + This caches the serialization of pyrsistent objects in a + ``WeakKeyDictionary``, so the cache should be automatically cleared when + the input object that is cached is destroyed. + + :returns: An entirely serializable version of input_object. + """ + # Ensure this is a quick function for basic types: + if input_object is None: + return None + + # Note that ``type(x) in frozenset([str, int])`` is faster than + # ``isinstance(x, (str, int))``. + input_type = type(input_object) + if input_type in _BASIC_JSON_TYPES: + return input_object + + is_pyrsistent = False + if input_type in _BASIC_JSON_COLLECTIONS: + # Don't send basic collections through shallow object serialization, + # isinstance is not a very cheap operation. + obj = input_object + else: + if _is_pyrsistent(input_object): + is_pyrsistent = True + # Using ``dict.get`` and a sentinel rather than the more pythonic + # try/except KeyError for performance. This function is highly + # recursive and the KeyError is guaranteed to happen the first + # time every object is serialized. We do not want to incur the cost + # of a caught exception for every pyrsistent object ever + # serialized. + cached_value = _cached_dfs_serialize_cache.get(input_object, + _UNCACHED_SENTINEL) + if cached_value is not _UNCACHED_SENTINEL: + return cached_value + obj = _to_serializables(input_object) + + result = obj + + obj_type = type(obj) + if obj_type == dict: + result = dict((_cached_dfs_serialize(key), + _cached_dfs_serialize(value)) + for key, value in obj.iteritems()) + elif obj_type == list or obj_type == tuple: + result = list(_cached_dfs_serialize(x) for x in obj) + + if is_pyrsistent: + _cached_dfs_serialize_cache[input_object] = result + + return result + + +def wire_encode(obj): + """ + Encode the given model object into bytes. + + :param obj: An object from the configuration model, e.g. ``Deployment``. + :return bytes: Encoded object. + """ + return dumps(_cached_dfs_serialize(obj)) + + +def wire_decode(data): + """ + Decode the given model object from bytes. + + :param bytes data: Encoded object. + """ + def decode(dictionary): + class_name = dictionary.get(_CLASS_MARKER, None) + if class_name == u"FilePath": + return FilePath(dictionary.get(u"path").encode("utf-8")) + elif class_name == u"PMap": + return pmap(dictionary[u"values"]) + elif class_name == u"UUID": + return UUID(dictionary[u"hex"]) + elif class_name == u"datetime": + return datetime.fromtimestamp(dictionary[u"seconds"], UTC) + elif class_name in _CONFIG_CLASS_MAP: + dictionary = dictionary.copy() + dictionary.pop(_CLASS_MARKER) + return _CONFIG_CLASS_MAP[class_name].create(dictionary) + else: + return dictionary + + return loads(data, object_hook=decode) diff --git a/flocker/control/test/test_generations.py b/flocker/control/test/test_generations.py index ad4f7af930..4de8029aab 100644 --- a/flocker/control/test/test_generations.py +++ b/flocker/control/test/test_generations.py @@ -10,7 +10,7 @@ from ..testtools import related_deployments_strategy from .._generations import GenerationTracker -from .._persistence import make_generation_hash +from .._serialization import make_generation_hash class GenerationTrackerTests(TestCase): diff --git a/flocker/control/test/test_persistence.py b/flocker/control/test/test_persistence.py index b4d0fe9e7a..182ce3caf0 100644 --- a/flocker/control/test/test_persistence.py +++ b/flocker/control/test/test_persistence.py @@ -6,7 +6,7 @@ import json import string -from datetime import datetime, timedelta +from datetime import datetime from uuid import uuid4, UUID from pytz import UTC @@ -22,11 +22,7 @@ from twisted.internet.task import Clock from twisted.python.filepath import FilePath -from pyrsistent import PClass, pset - -from testtools.matchers import Is, Equals, Not - -from ..testtools import deployment_strategy +from pyrsistent import pset from ...testtools import AsyncTestCase, TestCase from .._persistence import ( @@ -34,11 +30,11 @@ _LOG_SAVE, _LOG_STARTUP, migrate_configuration, _CONFIG_VERSION, ConfigurationMigration, ConfigurationMigrationError, _LOG_UPGRADE, MissingMigrationError, update_leases, _LOG_EXPIRE, - _LOG_UNCHANGED_DEPLOYMENT_NOT_SAVED, to_unserialized_json, generation_hash + _LOG_UNCHANGED_DEPLOYMENT_NOT_SAVED, ) from .._model import ( Deployment, Application, DockerImage, Node, Dataset, Manifestation, - AttachedVolume, SERIALIZABLE_CLASSES, NodeState, Configuration, + AttachedVolume, Configuration, Port, Link, Leases, Lease, BlockDeviceOwnership, PersistentState, ) @@ -691,82 +687,6 @@ def _build_node(applications): SUPPORTED_VERSIONS = st.integers(1, _CONFIG_VERSION) -class WireEncodeDecodeTests(TestCase): - """ - Tests for ``to_unserialized_json``, ``wire_encode`` and ``wire_decode``. - """ - def test_encode_to_bytes(self): - """ - ``wire_encode`` converts the given object to ``bytes``. - """ - self.assertIsInstance(wire_encode(LATEST_TEST_DEPLOYMENT), bytes) - - @given(DEPLOYMENTS) - def test_roundtrip(self, deployment): - """ - A range of generated configurations (deployments) can be - roundtripped via the wire encode/decode. - """ - source_json = wire_encode(deployment) - decoded_deployment = wire_decode(source_json) - self.assertEqual(decoded_deployment, deployment) - - @given(DEPLOYMENTS) - def test_to_unserialized_json(self, deployment): - """ - ``to_unserialized_json`` is same output as ``wire_encode`` except - without doing JSON byte encoding. - """ - unserialized = to_unserialized_json(deployment) - self.assertEquals(wire_decode(json.dumps(unserialized)), deployment) - - def test_no_arbitrary_decoding(self): - """ - ``wire_decode`` will not decode classes that are not in - ``SERIALIZABLE_CLASSES``. - """ - class Temp(PClass): - """A class.""" - SERIALIZABLE_CLASSES.append(Temp) - - def cleanup(): - if Temp in SERIALIZABLE_CLASSES: - SERIALIZABLE_CLASSES.remove(Temp) - self.addCleanup(cleanup) - - data = wire_encode(Temp()) - SERIALIZABLE_CLASSES.remove(Temp) - # Possibly future versions might throw exception, the key point is - # that the returned object is not a Temp instance. - self.assertFalse(isinstance(wire_decode(data), Temp)) - - def test_complex_keys(self): - """ - Objects with attributes that are ``PMap``\s with complex keys - (i.e. not strings) can be roundtripped. - """ - node_state = NodeState(hostname=u'127.0.0.1', uuid=uuid4(), - manifestations={}, paths={}, - devices={uuid4(): FilePath(b"/tmp")}) - self.assertEqual(node_state, wire_decode(wire_encode(node_state))) - - def test_datetime(self): - """ - A datetime with a timezone can be roundtripped (with potential loss of - less-than-second resolution). - """ - dt = datetime.now(tz=UTC) - self.assertTrue( - abs(wire_decode(wire_encode(dt)) - dt) < timedelta(seconds=1)) - - def test_naive_datetime(self): - """ - A naive datetime will fail. Don't use those, always use an explicit - timezone. - """ - self.assertRaises(ValueError, wire_encode, datetime.now()) - - class ConfigurationMigrationTests(TestCase): """ Tests for ``ConfigurationMigration`` class that performs individual @@ -834,141 +754,3 @@ def test_can_create_latest_golden(self): "an upgrade test if you are intentionally changing the " "model." % (path.path,) ) - - -class GenerationHashTests(TestCase): - """ - Tests for generation_hash. - """ - - @given(st.data()) - def test_no_hash_collisions(self, data): - """ - Hashes of different deployments do not have hash collisions, hashes of - the same object have the same hash. - """ - # With 128 bits of hash, a collision here indicates a fault in the - # algorithm. - - # Generate the first deployment. - deployment_a = data.draw(deployment_strategy()) - - # Decide if we want to generate a second deployment, or just compare - # the first deployment to a re-serialized version of itself: - simple_comparison = data.draw(st.booleans()) - if simple_comparison: - deployment_b = wire_decode(wire_encode(deployment_a)) - else: - deployment_b = data.draw(deployment_strategy()) - - should_be_equal = (deployment_a == deployment_b) - if simple_comparison: - self.assertThat( - should_be_equal, - Is(True) - ) - - hash_a = generation_hash(deployment_a) - hash_b = generation_hash(deployment_b) - - if should_be_equal: - self.assertThat( - hash_a, - Equals(hash_b) - ) - else: - self.assertThat( - hash_a, - Not(Equals(hash_b)) - ) - - def test_maps_and_sets_differ(self): - """ - Mappings hash to different values than frozensets of their iteritems(). - """ - self.assertThat( - generation_hash(frozenset([('a', 1), ('b', 2)])), - Not(Equals(generation_hash(dict(a=1, b=2)))) - ) - - def test_strings_and_jsonable_types_differ(self): - """ - Strings and integers hash to different values. - """ - self.assertThat( - generation_hash(5), - Not(Equals(generation_hash('5'))) - ) - - def test_sets_and_objects_differ(self): - """ - Sets can be hashed and 1 element sets have a different hash than the - hash of the single element. - """ - self.assertThat( - generation_hash(5), - Not(Equals(generation_hash(frozenset([5])))) - ) - - def test_lists_and_objects_differ(self): - """ - Lists can be hashed, and have a different hash value than scalars with - the same value or sets with the same values. - """ - self.assertThat( - generation_hash(913), - Not(Equals(generation_hash([913]))) - ) - self.assertThat( - generation_hash(frozenset([913])), - Not(Equals(generation_hash([913]))) - ) - - def test_empty_sets_can_be_hashed(self): - """ - Empty sets can be hashed and result in different hashes than empty - strings or the string 'NULLSET'. - """ - self.assertThat( - generation_hash(frozenset()), - Not(Equals(generation_hash(''))) - ) - self.assertThat( - generation_hash(frozenset()), - Not(Equals(generation_hash(b'NULLSET'))) - ) - - def test_unicode_hash(self): - """ - Unicode strings can be hashed, and are hashed to the same value as - their bytes equivalent. - """ - self.assertThat( - generation_hash(unicode(u'abcde')), - Equals(generation_hash(bytes(b'abcde'))) - ) - - def test_consistent_hash(self): - """ - A given deployment hashes to a specific value. - """ - # Unfortunately these are manually created golden values generated by - # running the test with wrong values and copying the output into this - # file. This test mostly adds value in verifying that the hashes - # computed in all of our CI environments are the same. - TEST_DEPLOYMENT_1_HASH = ''.join(chr(x) for x in [ - 0x4e, 0x35, 0x2b, 0xa2, 0x68, 0xde, 0x10, 0x0a, - 0xa5, 0xbc, 0x8a, 0x7e, 0x75, 0xc7, 0xf4, 0xe6 - ]) - TEST_DEPLOYMENT_2_HASH = ''.join(chr(x) for x in [ - 0x96, 0xe6, 0xcb, 0xa9, 0x5f, 0x7c, 0x8e, 0xfa, - 0xf8, 0x76, 0x8a, 0xc6, 0x89, 0x1a, 0xec, 0xc5 - ]) - self.assertThat( - generation_hash(TEST_DEPLOYMENT_1), - Equals(TEST_DEPLOYMENT_1_HASH) - ) - self.assertThat( - generation_hash(TEST_DEPLOYMENT_2), - Equals(TEST_DEPLOYMENT_2_HASH) - ) diff --git a/flocker/control/test/test_protocol.py b/flocker/control/test/test_protocol.py index 194388106d..b4d0743e02 100644 --- a/flocker/control/test/test_protocol.py +++ b/flocker/control/test/test_protocol.py @@ -50,7 +50,7 @@ Deployment, Application, DockerImage, Node, NodeState, Manifestation, Dataset, DeploymentState, NonManifestDatasets, ) -from .._persistence import wire_encode, make_generation_hash +from .._serialization import wire_encode, make_generation_hash from .._diffing import create_diff from .clusterstatetools import advance_some, advance_rest diff --git a/flocker/control/test/test_serialization.py b/flocker/control/test/test_serialization.py new file mode 100644 index 0000000000..0368e924fe --- /dev/null +++ b/flocker/control/test/test_serialization.py @@ -0,0 +1,248 @@ +# Copyright ClusterHQ Inc. See LICENSE file for details. + +""" +Tests for ``flocker.control._serialization``. +""" +from datetime import datetime, timedelta +import json +from uuid import uuid4 + +from hypothesis import given +from hypothesis import strategies as st +from pyrsistent import PClass +from pytz import UTC +from testtools.matchers import Is, Equals, Not +from twisted.python.filepath import FilePath + +from ..testtools import deployment_strategy + +from .._serialization import ( + wire_encode, + wire_decode, + to_unserialized_json, + SERIALIZABLE_CLASSES, + generation_hash, +) + +from .._model import NodeState +from ...testtools import TestCase +from .test_persistence import ( + LATEST_TEST_DEPLOYMENT, + DEPLOYMENTS, + TEST_DEPLOYMENT_1, + TEST_DEPLOYMENT_2, +) + + +class WireEncodeDecodeTests(TestCase): + """ + Tests for ``to_unserialized_json``, ``wire_encode`` and ``wire_decode``. + """ + def test_encode_to_bytes(self): + """ + ``wire_encode`` converts the given object to ``bytes``. + """ + self.assertIsInstance(wire_encode(LATEST_TEST_DEPLOYMENT), bytes) + + @given(DEPLOYMENTS) + def test_roundtrip(self, deployment): + """ + A range of generated configurations (deployments) can be + roundtripped via the wire encode/decode. + """ + source_json = wire_encode(deployment) + decoded_deployment = wire_decode(source_json) + self.assertEqual(decoded_deployment, deployment) + + @given(DEPLOYMENTS) + def test_to_unserialized_json(self, deployment): + """ + ``to_unserialized_json`` is same output as ``wire_encode`` except + without doing JSON byte encoding. + """ + unserialized = to_unserialized_json(deployment) + self.assertEquals(wire_decode(json.dumps(unserialized)), deployment) + + def test_no_arbitrary_decoding(self): + """ + ``wire_decode`` will not decode classes that are not in + ``SERIALIZABLE_CLASSES``. + """ + class Temp(PClass): + """A class.""" + SERIALIZABLE_CLASSES.append(Temp) + + def cleanup(): + if Temp in SERIALIZABLE_CLASSES: + SERIALIZABLE_CLASSES.remove(Temp) + self.addCleanup(cleanup) + + data = wire_encode(Temp()) + SERIALIZABLE_CLASSES.remove(Temp) + # Possibly future versions might throw exception, the key point is + # that the returned object is not a Temp instance. + self.assertFalse(isinstance(wire_decode(data), Temp)) + + def test_complex_keys(self): + """ + Objects with attributes that are ``PMap``\s with complex keys + (i.e. not strings) can be roundtripped. + """ + node_state = NodeState(hostname=u'127.0.0.1', uuid=uuid4(), + manifestations={}, paths={}, + devices={uuid4(): FilePath(b"/tmp")}) + self.assertEqual(node_state, wire_decode(wire_encode(node_state))) + + def test_datetime(self): + """ + A datetime with a timezone can be roundtripped (with potential loss of + less-than-second resolution). + """ + dt = datetime.now(tz=UTC) + self.assertTrue( + abs(wire_decode(wire_encode(dt)) - dt) < timedelta(seconds=1)) + + def test_naive_datetime(self): + """ + A naive datetime will fail. Don't use those, always use an explicit + timezone. + """ + self.assertRaises(ValueError, wire_encode, datetime.now()) + + +class GenerationHashTests(TestCase): + """ + Tests for generation_hash. + """ + + @given(st.data()) + def test_no_hash_collisions(self, data): + """ + Hashes of different deployments do not have hash collisions, hashes of + the same object have the same hash. + """ + # With 128 bits of hash, a collision here indicates a fault in the + # algorithm. + + # Generate the first deployment. + deployment_a = data.draw(deployment_strategy()) + + # Decide if we want to generate a second deployment, or just compare + # the first deployment to a re-serialized version of itself: + simple_comparison = data.draw(st.booleans()) + if simple_comparison: + deployment_b = wire_decode(wire_encode(deployment_a)) + else: + deployment_b = data.draw(deployment_strategy()) + + should_be_equal = (deployment_a == deployment_b) + if simple_comparison: + self.assertThat( + should_be_equal, + Is(True) + ) + + hash_a = generation_hash(deployment_a) + hash_b = generation_hash(deployment_b) + + if should_be_equal: + self.assertThat( + hash_a, + Equals(hash_b) + ) + else: + self.assertThat( + hash_a, + Not(Equals(hash_b)) + ) + + def test_maps_and_sets_differ(self): + """ + Mappings hash to different values than frozensets of their iteritems(). + """ + self.assertThat( + generation_hash(frozenset([('a', 1), ('b', 2)])), + Not(Equals(generation_hash(dict(a=1, b=2)))) + ) + + def test_strings_and_jsonable_types_differ(self): + """ + Strings and integers hash to different values. + """ + self.assertThat( + generation_hash(5), + Not(Equals(generation_hash('5'))) + ) + + def test_sets_and_objects_differ(self): + """ + Sets can be hashed and 1 element sets have a different hash than the + hash of the single element. + """ + self.assertThat( + generation_hash(5), + Not(Equals(generation_hash(frozenset([5])))) + ) + + def test_lists_and_objects_differ(self): + """ + Lists can be hashed, and have a different hash value than scalars with + the same value or sets with the same values. + """ + self.assertThat( + generation_hash(913), + Not(Equals(generation_hash([913]))) + ) + self.assertThat( + generation_hash(frozenset([913])), + Not(Equals(generation_hash([913]))) + ) + + def test_empty_sets_can_be_hashed(self): + """ + Empty sets can be hashed and result in different hashes than empty + strings or the string 'NULLSET'. + """ + self.assertThat( + generation_hash(frozenset()), + Not(Equals(generation_hash(''))) + ) + self.assertThat( + generation_hash(frozenset()), + Not(Equals(generation_hash(b'NULLSET'))) + ) + + def test_unicode_hash(self): + """ + Unicode strings can be hashed, and are hashed to the same value as + their bytes equivalent. + """ + self.assertThat( + generation_hash(unicode(u'abcde')), + Equals(generation_hash(bytes(b'abcde'))) + ) + + def test_consistent_hash(self): + """ + A given deployment hashes to a specific value. + """ + # Unfortunately these are manually created golden values generated by + # running the test with wrong values and copying the output into this + # file. This test mostly adds value in verifying that the hashes + # computed in all of our CI environments are the same. + TEST_DEPLOYMENT_1_HASH = ''.join(chr(x) for x in [ + 0x4e, 0x35, 0x2b, 0xa2, 0x68, 0xde, 0x10, 0x0a, + 0xa5, 0xbc, 0x8a, 0x7e, 0x75, 0xc7, 0xf4, 0xe6 + ]) + TEST_DEPLOYMENT_2_HASH = ''.join(chr(x) for x in [ + 0x96, 0xe6, 0xcb, 0xa9, 0x5f, 0x7c, 0x8e, 0xfa, + 0xf8, 0x76, 0x8a, 0xc6, 0x89, 0x1a, 0xec, 0xc5 + ]) + self.assertThat( + generation_hash(TEST_DEPLOYMENT_1), + Equals(TEST_DEPLOYMENT_1_HASH) + ) + self.assertThat( + generation_hash(TEST_DEPLOYMENT_2), + Equals(TEST_DEPLOYMENT_2_HASH) + )