Added Base64 Serialization for very large arrays (Image-Like)

Author: Luxxii

JsonToObject/reverseEntityAttribute.py

@@ -18,6 +18,14 @@
 __version__ = "0.0.1a"
 __status__ = "Developement"
 
+import base64
+import array
+try: 
+    import urllib.parse as quote
+except ImportError:
+    import urllib as quote
+
+
 ### Error Messages
 TYPE_VALUE_METADATA_NOT_DEFINED_MESSAGE = "One of the following is not defined in json: {type|value}"
 VALUE_EMPTY_MESSAGE = "The Value entered cannot be empty!"
@@ -103,6 +111,29 @@ def __init__(self, _dict, useMetaData=True):
                 rea = ReverseEntityAttribute(value, useMetaData)
                 self.value[key] = rea.getValue()
 
+        elif _dict['type'].lower() == "base64":
+            # Case we have a base64 String:
+            # First Unquote Special Characters
+            tempValue = quote.unquote(_dict['value'])
+
+            # Decode Base64 String into Bytes
+            tempValue = base64.b64decode(tempValue)
+
+            # Retrieve Information about int8 or uint8
+            if "metadata" in _dict and "dataType" in _dict["metadata"]:
+                datatype =  _dict['metadata']['dataType']['value']
+            else: 
+                raise ValueError("Unknown Object-Type: " + _dict['type'] + ". The MetaData does not specify what the actual DataType is.")
+
+            # convert back to integers
+            if datatype == "int8[]":
+                tempValue = array.array("b", tempValue)
+            else:
+                tempValue = array.array("B", tempValue)
+
+            # Change DataType to primitive python list
+            self.value = tempValue.tolist()
+
         else:
             # Maybe a class with key, value or another JSON object, check if you can iterate!
             if (not hasattr(_dict['value'], 'items')):

ObjectToJson/entity.py

@@ -74,7 +74,7 @@ def setObject(self, _object, dataTypeDict, ignorePythonMetaData, showIdValue=Tru
                     # Object contains invalid key-name, ignore!
                     pass
                 else:
-                    self.__dict__[key] = EntityAttribute(value, ignorePythonMetaData, dataTypeDict.get(key)) 
+                    self.__dict__[key] = EntityAttribute(value, ignorePythonMetaData, dataTypeDict.get(key), baseEntity=True) 
         except AttributeError as ex:
             raise ValueError(ERROR_MESSAGE_ATTTRIBUTE, ex)
 

ObjectToJson/entityAttribute.py

@@ -19,7 +19,13 @@
 __status__ = "Developement"
 
 import sys
+import array
+import base64
 
+try: 
+    import urllib.parse as quote
+except ImportError:
+    import urllib as quote
 
 class EntityAttribute():
     """ Here the actual Conversion to the correct JSON-Format happens 
@@ -30,59 +36,72 @@ class EntityAttribute():
     """
     python_version = sys.version_info
 
-    def __init__(self, _object, ipmd, concreteDataType=None):  
+    def __init__(self, _object, ipmd, concreteDataType=None, baseEntity=False):  
         self.value = _object
         self.type = ""
         self.metadata = dict()
+        if baseEntity:
+            self.setConcreteMetaData(concreteDataType)
         objectType = type(_object)
 
-
         # Simply if-then-else to the Json fromat
         if(objectType is type(None)):
             pass
         elif objectType is bool:
             self.type = "boolean"
             self.value = bool(_object)
+            # self.setConcreteMetaData(concreteDataType)
         elif objectType is int:
             self.type = "number"
             self.value = int(_object)
             self.setPythonMetaData(ipmd, "int")
+            # self.setConcreteMetaData(concreteDataType)
         elif objectType is float:
             self.type = "number"
             self.value = float(_object)
             self.setPythonMetaData(ipmd, "float")
+            # self.setConcreteMetaData(concreteDataType)
         elif self.python_version < (3,0) and objectType is long: # Check explicitly if Python 2 is used
             self.type = "number"
             self.value = long(_object)
             self.setPythonMetaData(ipmd, "long")
+            # self.setConcreteMetaData(concreteDataType)
         elif objectType is complex:
             self.type = "array"
             t = complex(_object)
             self.value = [EntityAttribute(t.real, ipmd), EntityAttribute(t.imag, ipmd)]
             self.setPythonMetaData(ipmd, "complex")
+            # self.setConcreteMetaData(concreteDataType)
         elif objectType is str:
             self.type = "string"
             self.value = str(_object)
+            # self.setConcreteMetaData(concreteDataType)
         elif self.python_version < (3,0) and objectType is unicode: # Check explicitly if Python 2 is used
             self.type = "string"
             self.value = unicode(_object)
+            # self.setConcreteMetaData(concreteDataType)
             self.setPythonMetaData(ipmd, "unicode")
         elif objectType is tuple:
             self.type = "array"
             self.value = []
             self.setPythonMetaData(ipmd, "tuple")
+            self.setConcreteMetaData(concreteDataType)
             for item in _object:
                 self.value.append(EntityAttribute(item, ipmd))
         elif objectType is list:
             self.type = "array"
             self.value = []
+            self.setConcreteMetaData(concreteDataType)
             for item in _object:
                 self.value.append(EntityAttribute(item, ipmd))
         elif objectType is dict:
             self.type = "object"
             tempDict = {}
             for key, value in _object.items():
-                tempDict[key] = EntityAttribute(value,ipmd )
+                innerConcreteMetaData = None
+                if concreteDataType is not None and key in concreteDataType:
+                    innerConcreteMetaData = concreteDataType[key]
+                tempDict[key] = EntityAttribute(value, ipmd, innerConcreteMetaData)
             self.value = tempDict
         else:
             # Case it is a Class
@@ -94,22 +113,54 @@ def __init__(self, _object, ipmd, concreteDataType=None):
             else:
                 raise ValueError("Cannot get attrs from {}".format(str(_object)))
 
-            if hasattr(_object, '_type'):   # ROS-Specific Type-Declaration
+            if hasattr(_object, '_type') and hasattr(_object, '_slot_types') and hasattr(_object, '__slots__'):   # ROS-Specific Type-Declaration
+                ### This is a special Class from ROS!
                 self.type = _object._type.replace("/", ".") # Needs to be replaced Fiware does not allow a '/'
-            else:
-                self.type = _object.__class__.__name__
-            self.setPythonMetaData(ipmd, "class")
-            tempDict = {}
-            for key in iterL:
-                if key.startswith('_'):
-                    continue
-                tempDict[key] = EntityAttribute(getattr(_object, key), ipmd)
-            self.value = tempDict
+                self.setPythonMetaData(ipmd, "class")
+                # Special Case 'Image-like'-Data in ROS (very long 'int8[]'- and 'uint8[]' - arrays)
+                # These are converted into Base64 (escaped)
+                tempDict = {}
+                for key, key_type in zip(_object.__slots__, _object._slot_types):
+                    if key.startswith('_'):
+                        continue
+                    if (key_type == 'int8[]' or key_type == 'uint8[]') and len(getattr(_object, key)) >= 256: 
+                        # TODO DL 256 -> Threshold?
+                        # Generate Base64 String of the Array:
+                        tempDict[key] = EntityAttribute(None, ipmd)
+                        tempDict[key].type = "base64"
 
+                        # Either generate unsigned or signed byte-array
+                        if key_type == 'int8[]':
+                            tempDict[key].value = array.array('b', getattr(_object, key)).tostring()
+                        else:
+                            tempDict[key].value = array.array('B', getattr(_object, key)).tostring()
+                        
+                        # Form that Byte-Array: generate Base64 String
+                        tempDict[key].value = base64.b64encode(tempDict[key].value)
 
-        if concreteDataType is not None:
-            self.metadata["dataType"] = dict(type="dataType", value=concreteDataType)
-            pass
+                        # Escape Special Characters:
+                        tempDict[key].value = quote.quote(tempDict[key].value)
+                        tempDict[key].metadata = dict()
+                        self.setConcreteMetaData(concreteDataType[key], tempDict[key])
+                    else:
+                        innerConcreteMetaData = None
+                        if concreteDataType is not None and key in concreteDataType:
+                            innerConcreteMetaData = concreteDataType[key]
+                        tempDict[key] = EntityAttribute(getattr(_object, key), ipmd, innerConcreteMetaData)
+                self.value = tempDict
+            else:
+                # Simple Class. Recursively retrieve the other values
+                self.type = _object.__class__.__name__
+                self.setPythonMetaData(ipmd, "class")
+                tempDict = {}
+                for key in iterL:
+                    if key.startswith('_'):
+                        continue
+                    innerConcreteMetaData = None
+                    if concreteDataType is not None and key in concreteDataType:
+                        innerConcreteMetaData = concreteDataType[key]
+                    tempDict[key] = EntityAttribute(getattr(_object, key), ipmd, innerConcreteMetaData)
+                self.value = tempDict
 
         # Remove metadata-Attribute if it is empty (minimizing the JSON)
         if self.metadata == {} :
@@ -118,4 +169,9 @@ def __init__(self, _object, ipmd, concreteDataType=None):
     def setPythonMetaData(self, ignorePythonMetaData, val):
         if not ignorePythonMetaData:
             self.metadata["python"] = dict(type="dataType", value=val)
-    
+
+    def setConcreteMetaData(self, val, obj= None):
+        if val is not None and obj is None:
+            self.metadata["dataType"] = dict(type="dataType", value=val)
+        elif val is not None:
+            obj.metadata["dataType"] = dict(type="dataType", value=val)

ObjectToJson/testEntityAttribute.py

@@ -140,7 +140,7 @@ def test_EntityAttributeComplex_ignoreMetaData_True(self):
         self.assertEqual(ea.type, "array")
 
     def test_EntityAttributeFloat32List_concreteDataType(self):
-        ea = EA(ClassInt(), True, concreteDataType=dict(int="uint32_t"))
+        ea = EA(ClassInt(), True, concreteDataType=dict(int="uint32_t"), baseEntity=True)
         self.assertTrue(hasattr(ea, 'metadata'))
         self.assertEqual(ea.metadata, dict(
             dataType=dict(type="dataType", value=dict(int='uint32_t'))))
@@ -196,6 +196,7 @@ def __init__(self):
 
 class RosClassWithSlotsInt(object):
     __slots__ = ['val1', '_type']
+    _slot_types = ['uint8', 'string']
     def __init__(self):
         self.val1 = 1
         self._type = "RosClass/Integer"  # Example-Type