Updated vendored dependencies.

7 files changed, 93 insertions, 32 deletions

diff --git a/libs/json_tricks/__init__.py b/libs/json_tricks/__init__.py
index b53a656fc..105bfe1bd 100644
--- a/libs/json_tricks/__init__.py
+++ b/libs/json_tricks/__init__.py
@@ -7,10 +7,11 @@ from .utils import hashodict, NoEnumException, NoNumpyException, NoPandasExcepti
 from .comment import strip_comment_line_with_symbol, strip_comments
 from .encoders import TricksEncoder, json_date_time_encode, class_instance_encode, json_complex_encode, \
 	numeric_types_encode, ClassInstanceEncoder, json_set_encode, pandas_encode, nopandas_encode, \
-	numpy_encode, NumpyEncoder, nonumpy_encode, NoNumpyEncoder, fallback_ignore_unknown, pathlib_encode
+	numpy_encode, NumpyEncoder, nonumpy_encode, NoNumpyEncoder, fallback_ignore_unknown, pathlib_encode, \
+	bytes_encode
 from .decoders import DuplicateJsonKeyException, TricksPairHook, json_date_time_hook, json_complex_hook, \
 	numeric_types_hook, ClassInstanceHook, json_set_hook, pandas_hook, nopandas_hook, json_numpy_obj_hook, \
-	json_nonumpy_obj_hook, pathlib_hook
+	json_nonumpy_obj_hook, pathlib_hook, json_bytes_hook
 from .nonp import dumps, dump, loads, load
 from ._version import VERSION
 
diff --git a/libs/json_tricks/_version.py b/libs/json_tricks/_version.py
index c3eaaf21f..a33d1c6cf 100644
--- a/libs/json_tricks/_version.py
+++ b/libs/json_tricks/_version.py
@@ -1,3 +1,3 @@
 
-VERSION = '3.15.5'
+VERSION = '3.16.1'
 
diff --git a/libs/json_tricks/comment.py b/libs/json_tricks/comment.py
index 102e2592d..97926a00a 100644
--- a/libs/json_tricks/comment.py
+++ b/libs/json_tricks/comment.py
@@ -16,6 +16,8 @@ def strip_comment_line_with_symbol(line, start):
 
 def strip_comments(string, comment_symbols=frozenset(('#', '//'))):
 	"""
+	Stripping comments usually works, but there are a few edge cases that trip it up, like https://github.com/mverleg/pyjson_tricks/issues/57.
+
 	:param string: A string containing json with comments started by comment_symbols.
 	:param comment_symbols: Iterable of symbols that start a line comment (default # or //).
 	:return: The string with the comments removed.
diff --git a/libs/json_tricks/decoders.py b/libs/json_tricks/decoders.py
index b2e65a35b..44ef3beca 100644
--- a/libs/json_tricks/decoders.py
+++ b/libs/json_tricks/decoders.py
@@ -1,4 +1,5 @@
 import warnings
+from base64 import standard_b64decode
 from collections import OrderedDict
 from datetime import datetime, date, time, timedelta
 from decimal import Decimal
@@ -87,7 +88,7 @@ def json_date_time_hook(dct):
 
 def json_complex_hook(dct):
 	"""
-	Return an encoded complex number to it's python representation.
+	Return an encoded complex number to Python complex type.
 
 	:param dct: (dict) json encoded complex number (__complex__)
 	:return: python complex number
@@ -101,6 +102,22 @@ def json_complex_hook(dct):
 	return parts[0] + parts[1] * 1j
 
 
+def json_bytes_hook(dct):
+	"""
+	Return encoded bytes, either base64 or utf8, back to Python bytes.
+
+	:param dct: any object, if it is a dict containing encoded bytes, they will be converted
+	:return: python complex number
+	"""
+	if not isinstance(dct, dict):
+		return dct
+	if '__bytes_b64__' in dct:
+		return standard_b64decode(dct['__bytes_b64__'])
+	if '__bytes_utf8__' in dct:
+		return dct['__bytes_utf8__'].encode('utf-8')
+	return dct
+
+
 def numeric_types_hook(dct):
 	if not isinstance(dct, dict):
 		return dct
diff --git a/libs/json_tricks/encoders.py b/libs/json_tricks/encoders.py
index c42312f5a..bbca9d236 100644
--- a/libs/json_tricks/encoders.py
+++ b/libs/json_tricks/encoders.py
@@ -1,4 +1,5 @@
 import warnings
+from base64 import standard_b64encode
 from datetime import datetime, date, time, timedelta
 from decimal import Decimal
 from fractions import Fraction
@@ -7,7 +8,7 @@ from json import JSONEncoder
 from sys import version, stderr
 
 from .utils import hashodict, get_module_name_from_object, NoEnumException, NoPandasException, \
-	NoNumpyException, str_type, JsonTricksDeprecation, gzip_compress, filtered_wrapper
+	NoNumpyException, str_type, JsonTricksDeprecation, gzip_compress, filtered_wrapper, is_py3
 
 
 def _fallback_wrapper(encoder):
@@ -186,9 +187,12 @@ def class_instance_encode(obj, primitives=False):
 			slots = obj.__slots__
 			if isinstance(slots, str):
 				slots = [slots]
-			slots = list(item for item in slots if item != '__dict__')
 			dct['slots'] = hashodict([])
 			for s in slots:
+				if s == '__dict__':
+					continue
+				if s == '__weakref__':
+					continue
 				dct['slots'][s] = getattr(obj, s)
 		if hasattr(obj, '__dict__'):
 			dct['attributes'] = hashodict(obj.__dict__)
@@ -203,7 +207,7 @@ def class_instance_encode(obj, primitives=False):
 
 def json_complex_encode(obj, primitives=False):
 	"""
-	Encode a complex number as a json dictionary of it's real and imaginary part.
+	Encode a complex number as a json dictionary of its real and imaginary part.
 
 	:param obj: complex number, e.g. `2+1j`
 	:return: (dict) json primitives representation of `obj`
@@ -216,6 +220,29 @@ def json_complex_encode(obj, primitives=False):
 	return obj
 
 
+def bytes_encode(obj, primitives=False):
+	"""
+	Encode bytes as one of these:
+
+	* A utf8-string with special `__bytes_utf8__` marking, if the bytes are valid utf8 and primitives is False.
+	* A base64 encoded string of the bytes with special `__bytes_b64__` marking, if the bytes are not utf8, or if primitives is True.
+
+	:param obj: any object, which will be transformed if it is of type bytes
+	:return: (dict) json primitives representation of `obj`
+	"""
+	if isinstance(obj, bytes):
+		if not is_py3:
+			return obj
+		if primitives:
+			return hashodict(__bytes_b64__=standard_b64encode(obj).decode('ascii'))
+		else:
+			try:
+				return hashodict(__bytes_utf8__=obj.decode('utf-8'))
+			except UnicodeDecodeError:
+				return hashodict(__bytes_b64__=standard_b64encode(obj).decode('ascii'))
+	return obj
+
+
 def numeric_types_encode(obj, primitives=False):
 	"""
 	Encode Decimal and Fraction.
@@ -242,14 +269,14 @@ def numeric_types_encode(obj, primitives=False):
 
 
 def pathlib_encode(obj, primitives=False):
-    from pathlib import Path
-    if not isinstance(obj, Path):
-        return obj
+	from pathlib import Path
+	if not isinstance(obj, Path):
+		return obj
 
-    if primitives:
-        return str(obj)
+	if primitives:
+		return str(obj)
 
-    return {'__pathlib__': str(obj)}
+	return {'__pathlib__': str(obj)}
 
 
 class ClassInstanceEncoder(JSONEncoder):
diff --git a/libs/json_tricks/nonp.py b/libs/json_tricks/nonp.py
index 030bd06ce..f61c10b1a 100644
--- a/libs/json_tricks/nonp.py
+++ b/libs/json_tricks/nonp.py
@@ -10,13 +10,13 @@ from .comment import strip_comments  # keep 'unused' imports
 from .encoders import TricksEncoder, json_date_time_encode, \
 	class_instance_encode, json_complex_encode, json_set_encode, numeric_types_encode, numpy_encode, \
 	nonumpy_encode, nopandas_encode, pandas_encode, noenum_instance_encode, \
-	enum_instance_encode, pathlib_encode # keep 'unused' imports
+	enum_instance_encode, pathlib_encode, bytes_encode  # keep 'unused' imports
 from .decoders import TricksPairHook, \
 	json_date_time_hook, ClassInstanceHook, \
 	json_complex_hook, json_set_hook, numeric_types_hook, json_numpy_obj_hook, \
 	json_nonumpy_obj_hook, \
 	nopandas_hook, pandas_hook, EnumInstanceHook, \
-	noenum_hook, pathlib_hook, nopathlib_hook  # keep 'unused' imports
+	noenum_hook, pathlib_hook, nopathlib_hook, json_bytes_hook  # keep 'unused' imports
 
 
 ENCODING = 'UTF-8'
@@ -25,9 +25,9 @@ ENCODING = 'UTF-8'
 _cih_instance = ClassInstanceHook()
 _eih_instance = EnumInstanceHook()
 DEFAULT_ENCODERS = [json_date_time_encode, json_complex_encode, json_set_encode,
-					numeric_types_encode, class_instance_encode, ]
+					numeric_types_encode, class_instance_encode, bytes_encode,]
 DEFAULT_HOOKS = [json_date_time_hook, json_complex_hook, json_set_hook,
-				numeric_types_hook, _cih_instance, ]
+				numeric_types_hook, _cih_instance, json_bytes_hook,]
 
 
 #TODO @mark: add properties to all built-in encoders (for speed - but it should keep working without)
@@ -103,9 +103,10 @@ def dumps(obj, sort_keys=None, cls=None, obj_encoders=DEFAULT_ENCODERS, extra_ob
 	dict_default(properties, 'allow_nan', allow_nan)
 	if cls is None:
 		cls = TricksEncoder
-	txt = cls(sort_keys=sort_keys, obj_encoders=encoders, allow_nan=allow_nan,
+	combined_encoder = cls(sort_keys=sort_keys, obj_encoders=encoders, allow_nan=allow_nan,
 		primitives=primitives, fallback_encoders=fallback_encoders,
-	  	properties=properties, **jsonkwargs).encode(obj)
+	  	properties=properties, **jsonkwargs)
+	txt = combined_encoder.encode(obj)
 	if not is_py3 and isinstance(txt, str):
 		txt = unicode(txt, ENCODING)
 	if not compression:
@@ -187,7 +188,7 @@ def loads(string, preserve_order=True, ignore_comments=None, decompression=None,
 	:param string: The string containing a json encoded data structure.
 	:param decode_cls_instances: True to attempt to decode class instances (requires the environment to be similar the the encoding one).
 	:param preserve_order: Whether to preserve order by using OrderedDicts or not.
-	:param ignore_comments: Remove comments (starting with # or //).
+	:param ignore_comments: Remove comments (starting with # or //). By default (`None`), try without comments first, and re-try with comments upon failure.
 	:param decompression: True to use gzip decompression, False to use raw data, None to automatically determine (default). Assumes utf-8 encoding!
 	:param obj_pairs_hooks: A list of dictionary hooks to apply.
 	:param extra_obj_pairs_hooks: Like `obj_pairs_hooks` but on top of them: use this to add hooks without replacing defaults. Since v3.5 these happen before default hooks.
@@ -215,16 +216,6 @@ def loads(string, preserve_order=True, ignore_comments=None, decompression=None,
 				'for example bytevar.encode("utf-8") if utf-8 is the encoding. Alternatively you can '
 				'force an attempt by passing conv_str_byte=True, but this may cause decoding issues.')
 					.format(type(string)))
-	if ignore_comments or ignore_comments is None:
-		new_string = strip_comments(string)
-		if ignore_comments is None and not getattr(loads, '_ignore_comments_warned', False) and string != new_string:
-			warnings.warn('`json_tricks.load(s)` stripped some comments, but `ignore_comments` was '
-				'not passed; in the next major release, the behaviour when `ignore_comments` is not '
-				'passed will change; it is recommended to explicitly pass `ignore_comments=True` if '
-				'you want to strip comments; see https://github.com/mverleg/pyjson_tricks/issues/74',
-				JsonTricksDeprecation)
-			loads._ignore_comments_warned = True
-		string = new_string
 	properties = properties or {}
 	dict_default(properties, 'preserve_order', preserve_order)
 	dict_default(properties, 'ignore_comments', ignore_comments)
@@ -233,7 +224,30 @@ def loads(string, preserve_order=True, ignore_comments=None, decompression=None,
 	dict_default(properties, 'allow_duplicates', allow_duplicates)
 	hooks = tuple(extra_obj_pairs_hooks) + tuple(obj_pairs_hooks)
 	hook = TricksPairHook(ordered=preserve_order, obj_pairs_hooks=hooks, allow_duplicates=allow_duplicates, properties=properties)
-	return json_loads(string, object_pairs_hook=hook, **jsonkwargs)
+	if ignore_comments is None:
+		try:
+			# first try to parse without stripping comments
+			return _strip_loads(string, hook, False, **jsonkwargs)
+		except ValueError:
+			# if this fails, re-try parsing after stripping comments
+			result = _strip_loads(string, hook, True, **jsonkwargs)
+			if not getattr(loads, '_ignore_comments_warned', False):
+				warnings.warn('`json_tricks.load(s)` stripped some comments, but `ignore_comments` was '
+					'not passed; in the next major release, the behaviour when `ignore_comments` is not '
+					'passed will change; it is recommended to explicitly pass `ignore_comments=True` if '
+					'you want to strip comments; see https://github.com/mverleg/pyjson_tricks/issues/74',
+					JsonTricksDeprecation)
+				loads._ignore_comments_warned = True
+			return result
+	if ignore_comments:
+		return _strip_loads(string, hook, True, **jsonkwargs)
+	return _strip_loads(string, hook, False, **jsonkwargs)
+
+
+def _strip_loads(string, object_pairs_hook, ignore_comments_bool, **jsonkwargs):
+	if ignore_comments_bool:
+		string = strip_comments(string)
+	return json_loads(string, object_pairs_hook=object_pairs_hook, **jsonkwargs)
 
 
 def load(fp, preserve_order=True, ignore_comments=None, decompression=None, obj_pairs_hooks=DEFAULT_HOOKS,
diff --git a/libs/json_tricks/np.py b/libs/json_tricks/np.py
index 108d6d1a3..5f269c929 100644
--- a/libs/json_tricks/np.py
+++ b/libs/json_tricks/np.py
@@ -10,7 +10,7 @@ from .comment import strip_comment_line_with_symbol, strip_comments  # keep 'unu
 from .encoders import TricksEncoder, json_date_time_encode, class_instance_encode, ClassInstanceEncoder, \
 	numpy_encode, NumpyEncoder # keep 'unused' imports
 from .decoders import DuplicateJsonKeyException, TricksPairHook, json_date_time_hook, ClassInstanceHook, \
-	json_complex_hook, json_set_hook, json_numpy_obj_hook  # keep 'unused' imports
+	json_complex_hook, json_set_hook, json_numpy_obj_hook, json_bytes_hook  # keep 'unused' imports
 
 try:
 	import numpy