Subsync first implementation (only after download/upload).

11 files changed, 1415 insertions, 0 deletions

diff --git a/libs/past/__init__.py b/libs/past/__init__.py
new file mode 100644
index 000000000..3b5d9db17
--- /dev/null
+++ b/libs/past/__init__.py
@@ -0,0 +1,92 @@
+# coding=utf-8
+"""
+past: compatibility with Python 2 from Python 3
+===============================================
+
+``past`` is a package to aid with Python 2/3 compatibility. Whereas ``future``
+contains backports of Python 3 constructs to Python 2, ``past`` provides
+implementations of some Python 2 constructs in Python 3 and tools to import and
+run Python 2 code in Python 3. It is intended to be used sparingly, as a way of
+running old Python 2 code from Python 3 until the code is ported properly.
+
+Potential uses for libraries:
+
+- as a step in porting a Python 2 codebase to Python 3 (e.g. with the ``futurize`` script)
+- to provide Python 3 support for previously Python 2-only libraries with the
+  same APIs as on Python 2 -- particularly with regard to 8-bit strings (the
+  ``past.builtins.str`` type).
+- to aid in providing minimal-effort Python 3 support for applications using
+  libraries that do not yet wish to upgrade their code properly to Python 3, or
+  wish to upgrade it gradually to Python 3 style.
+
+
+Here are some code examples that run identically on Python 3 and 2::
+
+    >>> from past.builtins import str as oldstr
+
+    >>> philosopher = oldstr(u'\u5b54\u5b50'.encode('utf-8'))
+    >>> # This now behaves like a Py2 byte-string on both Py2 and Py3.
+    >>> # For example, indexing returns a Python 2-like string object, not
+    >>> # an integer:
+    >>> philosopher[0]
+    '\xe5'
+    >>> type(philosopher[0])
+    <past.builtins.oldstr>
+
+    >>> # List-producing versions of range, reduce, map, filter
+    >>> from past.builtins import range, reduce
+    >>> range(10)
+    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    >>> reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])
+    15
+
+    >>> # Other functions removed in Python 3 are resurrected ...
+    >>> from past.builtins import execfile
+    >>> execfile('myfile.py')
+
+    >>> from past.builtins import raw_input
+    >>> name = raw_input('What is your name? ')
+    What is your name? [cursor]
+
+    >>> from past.builtins import reload
+    >>> reload(mymodule)   # equivalent to imp.reload(mymodule) in Python 3
+
+    >>> from past.builtins import xrange
+    >>> for i in xrange(10):
+    ...     pass
+
+
+It also provides import hooks so you can import and use Python 2 modules like
+this::
+
+    $ python3
+
+    >>> from past import autotranslate
+    >>> authotranslate('mypy2module')
+    >>> import mypy2module
+
+until the authors of the Python 2 modules have upgraded their code. Then, for
+example::
+
+    >>> mypy2module.func_taking_py2_string(oldstr(b'abcd'))
+
+
+Credits
+-------
+
+:Author:  Ed Schofield
+:Sponsor: Python Charmers Pty Ltd, Australia: http://pythoncharmers.com
+
+
+Licensing
+---------
+Copyright 2013-2018 Python Charmers Pty Ltd, Australia.
+The software is distributed under an MIT licence. See LICENSE.txt.
+"""
+
+
+from past.translation import install_hooks as autotranslate
+from future import __version__, __copyright__, __license__
+
+__title__ = 'past'
+__author__ = 'Ed Schofield'
diff --git a/libs/past/builtins/__init__.py b/libs/past/builtins/__init__.py
new file mode 100644
index 000000000..1b19e373c
--- /dev/null
+++ b/libs/past/builtins/__init__.py
@@ -0,0 +1,72 @@
+"""
+A resurrection of some old functions from Python 2 for use in Python 3. These
+should be used sparingly, to help with porting efforts, since code using them
+is no longer standard Python 3 code.
+
+This module provides the following:
+
+1. Implementations of these builtin functions which have no equivalent on Py3:
+
+- apply
+- chr
+- cmp
+- execfile
+
+2. Aliases:
+
+- intern <- sys.intern
+- raw_input <- input
+- reduce <- functools.reduce
+- reload <- imp.reload
+- unichr <- chr
+- unicode <- str
+- xrange <- range
+
+3. List-producing versions of the corresponding Python 3 iterator-producing functions:
+
+- filter
+- map
+- range
+- zip
+
+4. Forward-ported Py2 types:
+
+- basestring
+- dict
+- str
+- long
+- unicode
+
+"""
+
+from future.utils import PY3
+from past.builtins.noniterators import (filter, map, range, reduce, zip)
+# from past.builtins.misc import (ascii, hex, input, oct, open)
+if PY3:
+    from past.types import (basestring,
+                            olddict as dict,
+                            oldstr as str,
+                            long,
+                            unicode)
+else:
+    from __builtin__ import (basestring, dict, str, long, unicode)
+
+from past.builtins.misc import (apply, chr, cmp, execfile, intern, oct,
+                                raw_input, reload, unichr, unicode, xrange)
+from past import utils
+
+
+if utils.PY3:
+    # We only import names that shadow the builtins on Py3. No other namespace
+    # pollution on Py3.
+
+    # Only shadow builtins on Py3; no new names
+    __all__ = ['filter', 'map', 'range', 'reduce', 'zip',
+               'basestring', 'dict', 'str', 'long', 'unicode',
+               'apply', 'chr', 'cmp', 'execfile', 'intern', 'raw_input',
+               'reload', 'unichr', 'xrange'
+              ]
+
+else:
+    # No namespace pollution on Py2
+    __all__ = []
diff --git a/libs/past/builtins/misc.py b/libs/past/builtins/misc.py
new file mode 100644
index 000000000..06fbb92d2
--- /dev/null
+++ b/libs/past/builtins/misc.py
@@ -0,0 +1,89 @@
+from __future__ import unicode_literals
+import sys
+import inspect
+from collections import Mapping
+
+from future.utils import PY3, exec_
+
+
+if PY3:
+    import builtins
+
+    def apply(f, *args, **kw):
+        return f(*args, **kw)
+
+    from past.builtins import str as oldstr
+
+    def chr(i):
+        """
+        Return a byte-string of one character with ordinal i; 0 <= i <= 256
+        """
+        return oldstr(bytes((i,)))
+
+    def cmp(x, y):
+        """
+        cmp(x, y) -> integer
+
+        Return negative if x<y, zero if x==y, positive if x>y.
+        """
+        return (x > y) - (x < y)
+
+    from sys import intern
+
+    def oct(number):
+        """oct(number) -> string
+
+        Return the octal representation of an integer
+        """
+        return '0' + builtins.oct(number)[2:]
+
+    raw_input = input
+    from imp import reload
+    unicode = str
+    unichr = chr
+    xrange = range
+else:
+    import __builtin__
+    apply = __builtin__.apply
+    chr = __builtin__.chr
+    cmp = __builtin__.cmp
+    execfile = __builtin__.execfile
+    intern = __builtin__.intern
+    oct = __builtin__.oct
+    raw_input = __builtin__.raw_input
+    reload = __builtin__.reload
+    unicode = __builtin__.unicode
+    unichr = __builtin__.unichr
+    xrange = __builtin__.xrange
+
+
+if PY3:
+    def execfile(filename, myglobals=None, mylocals=None):
+        """
+        Read and execute a Python script from a file in the given namespaces.
+        The globals and locals are dictionaries, defaulting to the current
+        globals and locals. If only globals is given, locals defaults to it.
+        """
+        if myglobals is None:
+            # There seems to be no alternative to frame hacking here.
+            caller_frame = inspect.stack()[1]
+            myglobals = caller_frame[0].f_globals
+            mylocals = caller_frame[0].f_locals
+        elif mylocals is None:
+            # Only if myglobals is given do we set mylocals to it.
+            mylocals = myglobals
+        if not isinstance(myglobals, Mapping):
+            raise TypeError('globals must be a mapping')
+        if not isinstance(mylocals, Mapping):
+            raise TypeError('locals must be a mapping')
+        with open(filename, "rbU") as fin:
+             source = fin.read()
+        code = compile(source, filename, "exec")
+        exec_(code, myglobals, mylocals)
+
+
+if PY3:
+    __all__ = ['apply', 'chr', 'cmp', 'execfile', 'intern', 'raw_input',
+               'reload', 'unichr', 'unicode', 'xrange']
+else:
+    __all__ = []
diff --git a/libs/past/builtins/noniterators.py b/libs/past/builtins/noniterators.py
new file mode 100644
index 000000000..5826b97c1
--- /dev/null
+++ b/libs/past/builtins/noniterators.py
@@ -0,0 +1,272 @@
+"""
+This module is designed to be used as follows::
+
+    from past.builtins.noniterators import filter, map, range, reduce, zip
+
+And then, for example::
+
+    assert isinstance(range(5), list)
+
+The list-producing functions this brings in are::
+
+- ``filter``
+- ``map``
+- ``range``
+- ``reduce``
+- ``zip``
+
+"""
+
+from __future__ import division, absolute_import, print_function
+
+from itertools import chain, starmap
+import itertools       # since zip_longest doesn't exist on Py2
+from past.types import basestring
+from past.utils import PY3
+
+
+def flatmap(f, items):
+    return chain.from_iterable(map(f, items))
+
+
+if PY3:
+    import builtins
+
+    # list-producing versions of the major Python iterating functions
+    def oldfilter(*args):
+        """
+        filter(function or None, sequence) -> list, tuple, or string
+
+        Return those items of sequence for which function(item) is true.
+        If function is None, return the items that are true.  If sequence
+        is a tuple or string, return the same type, else return a list.
+        """
+        mytype = type(args[1])
+        if isinstance(args[1], basestring):
+            return mytype().join(builtins.filter(*args))
+        elif isinstance(args[1], (tuple, list)):
+            return mytype(builtins.filter(*args))
+        else:
+            # Fall back to list. Is this the right thing to do?
+            return list(builtins.filter(*args))
+
+    # This is surprisingly difficult to get right. For example, the
+    # solutions here fail with the test cases in the docstring below:
+    # http://stackoverflow.com/questions/8072755/
+    def oldmap(func, *iterables):
+        """
+        map(function, sequence[, sequence, ...]) -> list
+
+        Return a list of the results of applying the function to the
+        items of the argument sequence(s).  If more than one sequence is
+        given, the function is called with an argument list consisting of
+        the corresponding item of each sequence, substituting None for
+        missing values when not all sequences have the same length.  If
+        the function is None, return a list of the items of the sequence
+        (or a list of tuples if more than one sequence).
+
+        Test cases:
+        >>> oldmap(None, 'hello world')
+        ['h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd']
+
+        >>> oldmap(None, range(4))
+        [0, 1, 2, 3]
+
+        More test cases are in past.tests.test_builtins.
+        """
+        zipped = itertools.zip_longest(*iterables)
+        l = list(zipped)
+        if len(l) == 0:
+            return []
+        if func is None:
+            result = l
+        else:
+            result = list(starmap(func, l))
+
+        # Inspect to see whether it's a simple sequence of tuples
+        try:
+            if max([len(item) for item in result]) == 1:
+                return list(chain.from_iterable(result))
+            # return list(flatmap(func, result))
+        except TypeError as e:
+            # Simple objects like ints have no len()
+            pass
+        return result
+
+        ############################
+        ### For reference, the source code for Py2.7 map function:
+        # static PyObject *
+        # builtin_map(PyObject *self, PyObject *args)
+        # {
+        #     typedef struct {
+        #         PyObject *it;           /* the iterator object */
+        #         int saw_StopIteration;  /* bool:  did the iterator end? */
+        #     } sequence;
+        #
+        #     PyObject *func, *result;
+        #     sequence *seqs = NULL, *sqp;
+        #     Py_ssize_t n, len;
+        #     register int i, j;
+        #
+        #     n = PyTuple_Size(args);
+        #     if (n < 2) {
+        #         PyErr_SetString(PyExc_TypeError,
+        #                         "map() requires at least two args");
+        #         return NULL;
+        #     }
+        #
+        #     func = PyTuple_GetItem(args, 0);
+        #     n--;
+        #
+        #     if (func == Py_None) {
+        #         if (PyErr_WarnPy3k("map(None, ...) not supported in 3.x; "
+        #                            "use list(...)", 1) < 0)
+        #             return NULL;
+        #         if (n == 1) {
+        #             /* map(None, S) is the same as list(S). */
+        #             return PySequence_List(PyTuple_GetItem(args, 1));
+        #         }
+        #     }
+        #
+        #     /* Get space for sequence descriptors.  Must NULL out the iterator
+        #      * pointers so that jumping to Fail_2 later doesn't see trash.
+        #      */
+        #     if ((seqs = PyMem_NEW(sequence, n)) == NULL) {
+        #         PyErr_NoMemory();
+        #         return NULL;
+        #     }
+        #     for (i = 0; i < n; ++i) {
+        #         seqs[i].it = (PyObject*)NULL;
+        #         seqs[i].saw_StopIteration = 0;
+        #     }
+        #
+        #     /* Do a first pass to obtain iterators for the arguments, and set len
+        #      * to the largest of their lengths.
+        #      */
+        #     len = 0;
+        #     for (i = 0, sqp = seqs; i < n; ++i, ++sqp) {
+        #         PyObject *curseq;
+        #         Py_ssize_t curlen;
+        #
+        #         /* Get iterator. */
+        #         curseq = PyTuple_GetItem(args, i+1);
+        #         sqp->it = PyObject_GetIter(curseq);
+        #         if (sqp->it == NULL) {
+        #             static char errmsg[] =
+        #                 "argument %d to map() must support iteration";
+        #             char errbuf[sizeof(errmsg) + 25];
+        #             PyOS_snprintf(errbuf, sizeof(errbuf), errmsg, i+2);
+        #             PyErr_SetString(PyExc_TypeError, errbuf);
+        #             goto Fail_2;
+        #         }
+        #
+        #         /* Update len. */
+        #         curlen = _PyObject_LengthHint(curseq, 8);
+        #         if (curlen > len)
+        #             len = curlen;
+        #     }
+        #
+        #     /* Get space for the result list. */
+        #     if ((result = (PyObject *) PyList_New(len)) == NULL)
+        #         goto Fail_2;
+        #
+        #     /* Iterate over the sequences until all have stopped. */
+        #     for (i = 0; ; ++i) {
+        #         PyObject *alist, *item=NULL, *value;
+        #         int numactive = 0;
+        #
+        #         if (func == Py_None && n == 1)
+        #             alist = NULL;
+        #         else if ((alist = PyTuple_New(n)) == NULL)
+        #             goto Fail_1;
+        #
+        #         for (j = 0, sqp = seqs; j < n; ++j, ++sqp) {
+        #             if (sqp->saw_StopIteration) {
+        #                 Py_INCREF(Py_None);
+        #                 item = Py_None;
+        #             }
+        #             else {
+        #                 item = PyIter_Next(sqp->it);
+        #                 if (item)
+        #                     ++numactive;
+        #                 else {
+        #                     if (PyErr_Occurred()) {
+        #                         Py_XDECREF(alist);
+        #                         goto Fail_1;
+        #                     }
+        #                     Py_INCREF(Py_None);
+        #                     item = Py_None;
+        #                     sqp->saw_StopIteration = 1;
+        #                 }
+        #             }
+        #             if (alist)
+        #                 PyTuple_SET_ITEM(alist, j, item);
+        #             else
+        #                 break;
+        #         }
+        #
+        #         if (!alist)
+        #             alist = item;
+        #
+        #         if (numactive == 0) {
+        #             Py_DECREF(alist);
+        #             break;
+        #         }
+        #
+        #         if (func == Py_None)
+        #             value = alist;
+        #         else {
+        #             value = PyEval_CallObject(func, alist);
+        #             Py_DECREF(alist);
+        #             if (value == NULL)
+        #                 goto Fail_1;
+        #         }
+        #         if (i >= len) {
+        #             int status = PyList_Append(result, value);
+        #             Py_DECREF(value);
+        #             if (status < 0)
+        #                 goto Fail_1;
+        #         }
+        #         else if (PyList_SetItem(result, i, value) < 0)
+        #             goto Fail_1;
+        #     }
+        #
+        #     if (i < len && PyList_SetSlice(result, i, len, NULL) < 0)
+        #         goto Fail_1;
+        #
+        #     goto Succeed;
+        #
+        # Fail_1:
+        #     Py_DECREF(result);
+        # Fail_2:
+        #     result = NULL;
+        # Succeed:
+        #     assert(seqs);
+        #     for (i = 0; i < n; ++i)
+        #         Py_XDECREF(seqs[i].it);
+        #     PyMem_DEL(seqs);
+        #     return result;
+        # }
+
+    def oldrange(*args, **kwargs):
+        return list(builtins.range(*args, **kwargs))
+
+    def oldzip(*args, **kwargs):
+        return list(builtins.zip(*args, **kwargs))
+
+    filter = oldfilter
+    map = oldmap
+    range = oldrange
+    from functools import reduce
+    zip = oldzip
+    __all__ = ['filter', 'map', 'range', 'reduce', 'zip']
+
+else:
+    import __builtin__
+    # Python 2-builtin ranges produce lists
+    filter = __builtin__.filter
+    map = __builtin__.map
+    range = __builtin__.range
+    reduce = __builtin__.reduce
+    zip = __builtin__.zip
+    __all__ = []
diff --git a/libs/past/tests/__init__.py b/libs/past/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/libs/past/tests/__init__.py
diff --git a/libs/past/translation/__init__.py b/libs/past/translation/__init__.py
new file mode 100644
index 000000000..c7ae2b7a0
--- /dev/null
+++ b/libs/past/translation/__init__.py
@@ -0,0 +1,497 @@
+# -*- coding: utf-8 -*-
+"""
+past.translation
+==================
+
+The ``past.translation`` package provides an import hook for Python 3 which
+transparently runs ``futurize`` fixers over Python 2 code on import to convert
+print statements into functions, etc.
+
+It is intended to assist users in migrating to Python 3.x even if some
+dependencies still only support Python 2.x.
+
+Usage
+-----
+
+Once your Py2 package is installed in the usual module search path, the import
+hook is invoked as follows:
+
+    >>> from past import autotranslate
+    >>> autotranslate('mypackagename')
+
+Or:
+
+    >>> autotranslate(['mypackage1', 'mypackage2'])
+
+You can unregister the hook using::
+
+    >>> from past.translation import remove_hooks
+    >>> remove_hooks()
+
+Author: Ed Schofield.
+Inspired by and based on ``uprefix`` by Vinay M. Sajip.
+"""
+
+import imp
+import logging
+import marshal
+import os
+import sys
+import copy
+from lib2to3.pgen2.parse import ParseError
+from lib2to3.refactor import RefactoringTool
+
+from libfuturize import fixes
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+myfixes = (list(fixes.libfuturize_fix_names_stage1) +
+           list(fixes.lib2to3_fix_names_stage1) +
+           list(fixes.libfuturize_fix_names_stage2) +
+           list(fixes.lib2to3_fix_names_stage2))
+
+
+# We detect whether the code is Py2 or Py3 by applying certain lib2to3 fixers
+# to it. If the diff is empty, it's Python 3 code.
+
+py2_detect_fixers = [
+# From stage 1:
+    'lib2to3.fixes.fix_apply',
+    # 'lib2to3.fixes.fix_dict',        # TODO: add support for utils.viewitems() etc. and move to stage2
+    'lib2to3.fixes.fix_except',
+    'lib2to3.fixes.fix_execfile',
+    'lib2to3.fixes.fix_exitfunc',
+    'lib2to3.fixes.fix_funcattrs',
+    'lib2to3.fixes.fix_filter',
+    'lib2to3.fixes.fix_has_key',
+    'lib2to3.fixes.fix_idioms',
+    'lib2to3.fixes.fix_import',    # makes any implicit relative imports explicit. (Use with ``from __future__ import absolute_import)
+    'lib2to3.fixes.fix_intern',
+    'lib2to3.fixes.fix_isinstance',
+    'lib2to3.fixes.fix_methodattrs',
+    'lib2to3.fixes.fix_ne',
+    'lib2to3.fixes.fix_numliterals',    # turns 1L into 1, 0755 into 0o755
+    'lib2to3.fixes.fix_paren',
+    'lib2to3.fixes.fix_print',
+    'lib2to3.fixes.fix_raise',   # uses incompatible with_traceback() method on exceptions
+    'lib2to3.fixes.fix_renames',
+    'lib2to3.fixes.fix_reduce',
+    # 'lib2to3.fixes.fix_set_literal',  # this is unnecessary and breaks Py2.6 support
+    'lib2to3.fixes.fix_repr',
+    'lib2to3.fixes.fix_standarderror',
+    'lib2to3.fixes.fix_sys_exc',
+    'lib2to3.fixes.fix_throw',
+    'lib2to3.fixes.fix_tuple_params',
+    'lib2to3.fixes.fix_types',
+    'lib2to3.fixes.fix_ws_comma',
+    'lib2to3.fixes.fix_xreadlines',
+
+# From stage 2:
+    'lib2to3.fixes.fix_basestring',
+    # 'lib2to3.fixes.fix_buffer',    # perhaps not safe. Test this.
+    # 'lib2to3.fixes.fix_callable',  # not needed in Py3.2+
+    # 'lib2to3.fixes.fix_dict',        # TODO: add support for utils.viewitems() etc.
+    'lib2to3.fixes.fix_exec',
+    # 'lib2to3.fixes.fix_future',    # we don't want to remove __future__ imports
+    'lib2to3.fixes.fix_getcwdu',
+    # 'lib2to3.fixes.fix_imports',   # called by libfuturize.fixes.fix_future_standard_library
+    # 'lib2to3.fixes.fix_imports2',  # we don't handle this yet (dbm)
+    # 'lib2to3.fixes.fix_input',
+    # 'lib2to3.fixes.fix_itertools',
+    # 'lib2to3.fixes.fix_itertools_imports',
+    'lib2to3.fixes.fix_long',
+    # 'lib2to3.fixes.fix_map',
+    # 'lib2to3.fixes.fix_metaclass', # causes SyntaxError in Py2! Use the one from ``six`` instead
+    'lib2to3.fixes.fix_next',
+    'lib2to3.fixes.fix_nonzero',     # TODO: add a decorator for mapping __bool__ to __nonzero__
+    # 'lib2to3.fixes.fix_operator',    # we will need support for this by e.g. extending the Py2 operator module to provide those functions in Py3
+    'lib2to3.fixes.fix_raw_input',
+    # 'lib2to3.fixes.fix_unicode',   # strips off the u'' prefix, which removes a potentially helpful source of information for disambiguating unicode/byte strings
+    # 'lib2to3.fixes.fix_urllib',
+    'lib2to3.fixes.fix_xrange',
+    # 'lib2to3.fixes.fix_zip',
+]
+
+
+class RTs:
+    """
+    A namespace for the refactoring tools. This avoids creating these at
+    the module level, which slows down the module import. (See issue #117).
+
+    There are two possible grammars: with or without the print statement.
+    Hence we have two possible refactoring tool implementations.
+    """
+    _rt = None
+    _rtp = None
+    _rt_py2_detect = None
+    _rtp_py2_detect = None
+
+    @staticmethod
+    def setup():
+        """
+        Call this before using the refactoring tools to create them on demand
+        if needed.
+        """
+        if None in [RTs._rt, RTs._rtp]:
+            RTs._rt = RefactoringTool(myfixes)
+            RTs._rtp = RefactoringTool(myfixes, {'print_function': True})
+
+
+    @staticmethod
+    def setup_detect_python2():
+        """
+        Call this before using the refactoring tools to create them on demand
+        if needed.
+        """
+        if None in [RTs._rt_py2_detect, RTs._rtp_py2_detect]:
+            RTs._rt_py2_detect = RefactoringTool(py2_detect_fixers)
+            RTs._rtp_py2_detect = RefactoringTool(py2_detect_fixers,
+                                                  {'print_function': True})
+
+
+# We need to find a prefix for the standard library, as we don't want to
+# process any files there (they will already be Python 3).
+#
+# The following method is used by Sanjay Vinip in uprefix. This fails for
+# ``conda`` environments:
+#     # In a non-pythonv virtualenv, sys.real_prefix points to the installed Python.
+#     # In a pythonv venv, sys.base_prefix points to the installed Python.
+#     # Outside a virtual environment, sys.prefix points to the installed Python.
+
+#     if hasattr(sys, 'real_prefix'):
+#         _syslibprefix = sys.real_prefix
+#     else:
+#         _syslibprefix = getattr(sys, 'base_prefix', sys.prefix)
+
+# Instead, we use the portion of the path common to both the stdlib modules
+# ``math`` and ``urllib``.
+
+def splitall(path):
+    """
+    Split a path into all components. From Python Cookbook.
+    """
+    allparts = []
+    while True:
+        parts = os.path.split(path)
+        if parts[0] == path:  # sentinel for absolute paths
+            allparts.insert(0, parts[0])
+            break
+        elif parts[1] == path: # sentinel for relative paths
+            allparts.insert(0, parts[1])
+            break
+        else:
+            path = parts[0]
+            allparts.insert(0, parts[1])
+    return allparts
+
+
+def common_substring(s1, s2):
+    """
+    Returns the longest common substring to the two strings, starting from the
+    left.
+    """
+    chunks = []
+    path1 = splitall(s1)
+    path2 = splitall(s2)
+    for (dir1, dir2) in zip(path1, path2):
+        if dir1 != dir2:
+            break
+        chunks.append(dir1)
+    return os.path.join(*chunks)
+
+# _stdlibprefix = common_substring(math.__file__, urllib.__file__)
+
+
+def detect_python2(source, pathname):
+    """
+    Returns a bool indicating whether we think the code is Py2
+    """
+    RTs.setup_detect_python2()
+    try:
+        tree = RTs._rt_py2_detect.refactor_string(source, pathname)
+    except ParseError as e:
+        if e.msg != 'bad input' or e.value != '=':
+            raise
+        tree = RTs._rtp.refactor_string(source, pathname)
+
+    if source != str(tree)[:-1]:   # remove added newline
+        # The above fixers made changes, so we conclude it's Python 2 code
+        logger.debug('Detected Python 2 code: {0}'.format(pathname))
+        with open('/tmp/original_code.py', 'w') as f:
+            f.write('### Original code (detected as py2): %s\n%s' %
+                    (pathname, source))
+        with open('/tmp/py2_detection_code.py', 'w') as f:
+            f.write('### Code after running py3 detection (from %s)\n%s' %
+                    (pathname, str(tree)[:-1]))
+        return True
+    else:
+        logger.debug('Detected Python 3 code: {0}'.format(pathname))
+        with open('/tmp/original_code.py', 'w') as f:
+            f.write('### Original code (detected as py3): %s\n%s' %
+                    (pathname, source))
+        try:
+            os.remove('/tmp/futurize_code.py')
+        except OSError:
+            pass
+        return False
+
+
+class Py2Fixer(object):
+    """
+    An import hook class that uses lib2to3 for source-to-source translation of
+    Py2 code to Py3.
+    """
+
+    # See the comments on :class:future.standard_library.RenameImport.
+    # We add this attribute here so remove_hooks() and install_hooks() can
+    # unambiguously detect whether the import hook is installed:
+    PY2FIXER = True
+
+    def __init__(self):
+        self.found = None
+        self.base_exclude_paths = ['future', 'past']
+        self.exclude_paths = copy.copy(self.base_exclude_paths)
+        self.include_paths = []
+
+    def include(self, paths):
+        """
+        Pass in a sequence of module names such as 'plotrique.plotting' that,
+        if present at the leftmost side of the full package name, would
+        specify the module to be transformed from Py2 to Py3.
+        """
+        self.include_paths += paths
+
+    def exclude(self, paths):
+        """
+        Pass in a sequence of strings such as 'mymodule' that, if
+        present at the leftmost side of the full package name, would cause
+        the module not to undergo any source transformation.
+        """
+        self.exclude_paths += paths
+
+    def find_module(self, fullname, path=None):
+        logger.debug('Running find_module: {0}...'.format(fullname))
+        if '.' in fullname:
+            parent, child = fullname.rsplit('.', 1)
+            if path is None:
+                loader = self.find_module(parent, path)
+                mod = loader.load_module(parent)
+                path = mod.__path__
+            fullname = child
+
+        # Perhaps we should try using the new importlib functionality in Python
+        # 3.3: something like this?
+        # thing = importlib.machinery.PathFinder.find_module(fullname, path)
+        try:
+            self.found = imp.find_module(fullname, path)
+        except Exception as e:
+            logger.debug('Py2Fixer could not find {0}')
+            logger.debug('Exception was: {0})'.format(fullname, e))
+            return None
+        self.kind = self.found[-1][-1]
+        if self.kind == imp.PKG_DIRECTORY:
+            self.pathname = os.path.join(self.found[1], '__init__.py')
+        elif self.kind == imp.PY_SOURCE:
+            self.pathname = self.found[1]
+        return self
+
+    def transform(self, source):
+        # This implementation uses lib2to3,
+        # you can override and use something else
+        # if that's better for you
+
+        # lib2to3 likes a newline at the end
+        RTs.setup()
+        source += '\n'
+        try:
+            tree = RTs._rt.refactor_string(source, self.pathname)
+        except ParseError as e:
+            if e.msg != 'bad input' or e.value != '=':
+                raise
+            tree = RTs._rtp.refactor_string(source, self.pathname)
+        # could optimise a bit for only doing str(tree) if
+        # getattr(tree, 'was_changed', False) returns True
+        return str(tree)[:-1] # remove added newline
+
+    def load_module(self, fullname):
+        logger.debug('Running load_module for {0}...'.format(fullname))
+        if fullname in sys.modules:
+            mod = sys.modules[fullname]
+        else:
+            if self.kind in (imp.PY_COMPILED, imp.C_EXTENSION, imp.C_BUILTIN,
+                             imp.PY_FROZEN):
+                convert = False
+            # elif (self.pathname.startswith(_stdlibprefix)
+            #       and 'site-packages' not in self.pathname):
+            #     # We assume it's a stdlib package in this case. Is this too brittle?
+            #     # Please file a bug report at https://github.com/PythonCharmers/python-future
+            #     # if so.
+            #     convert = False
+            # in theory, other paths could be configured to be excluded here too
+            elif any([fullname.startswith(path) for path in self.exclude_paths]):
+                convert = False
+            elif any([fullname.startswith(path) for path in self.include_paths]):
+                convert = True
+            else:
+                convert = False
+            if not convert:
+                logger.debug('Excluded {0} from translation'.format(fullname))
+                mod = imp.load_module(fullname, *self.found)
+            else:
+                logger.debug('Autoconverting {0} ...'.format(fullname))
+                mod = imp.new_module(fullname)
+                sys.modules[fullname] = mod
+
+                # required by PEP 302
+                mod.__file__ = self.pathname
+                mod.__name__ = fullname
+                mod.__loader__ = self
+
+                # This:
+                #     mod.__package__ = '.'.join(fullname.split('.')[:-1])
+                # seems to result in "SystemError: Parent module '' not loaded,
+                # cannot perform relative import" for a package's __init__.py
+                # file. We use the approach below. Another option to try is the
+                # minimal load_module pattern from the PEP 302 text instead.
+
+                # Is the test in the next line more or less robust than the
+                # following one? Presumably less ...
+                # ispkg = self.pathname.endswith('__init__.py')
+
+                if self.kind == imp.PKG_DIRECTORY:
+                    mod.__path__ = [ os.path.dirname(self.pathname) ]
+                    mod.__package__ = fullname
+                else:
+                    #else, regular module
+                    mod.__path__ = []
+                    mod.__package__ = fullname.rpartition('.')[0]
+
+                try:
+                    cachename = imp.cache_from_source(self.pathname)
+                    if not os.path.exists(cachename):
+                        update_cache = True
+                    else:
+                        sourcetime = os.stat(self.pathname).st_mtime
+                        cachetime = os.stat(cachename).st_mtime
+                        update_cache = cachetime < sourcetime
+                    # # Force update_cache to work around a problem with it being treated as Py3 code???
+                    # update_cache = True
+                    if not update_cache:
+                        with open(cachename, 'rb') as f:
+                            data = f.read()
+                            try:
+                                code = marshal.loads(data)
+                            except Exception:
+                                # pyc could be corrupt. Regenerate it
+                                update_cache = True
+                    if update_cache:
+                        if self.found[0]:
+                            source = self.found[0].read()
+                        elif self.kind == imp.PKG_DIRECTORY:
+                            with open(self.pathname) as f:
+                                source = f.read()
+
+                        if detect_python2(source, self.pathname):
+                            source = self.transform(source)
+                            with open('/tmp/futurized_code.py', 'w') as f:
+                                f.write('### Futurized code (from %s)\n%s' %
+                                        (self.pathname, source))
+
+                        code = compile(source, self.pathname, 'exec')
+
+                        dirname = os.path.dirname(cachename)
+                        try:
+                            if not os.path.exists(dirname):
+                                os.makedirs(dirname)
+                            with open(cachename, 'wb') as f:
+                                data = marshal.dumps(code)
+                                f.write(data)
+                        except Exception:   # could be write-protected
+                            pass
+                    exec(code, mod.__dict__)
+                except Exception as e:
+                    # must remove module from sys.modules
+                    del sys.modules[fullname]
+                    raise # keep it simple
+
+        if self.found[0]:
+            self.found[0].close()
+        return mod
+
+_hook = Py2Fixer()
+
+
+def install_hooks(include_paths=(), exclude_paths=()):
+    if isinstance(include_paths, str):
+        include_paths = (include_paths,)
+    if isinstance(exclude_paths, str):
+        exclude_paths = (exclude_paths,)
+    assert len(include_paths) + len(exclude_paths) > 0, 'Pass at least one argument'
+    _hook.include(include_paths)
+    _hook.exclude(exclude_paths)
+    # _hook.debug = debug
+    enable = sys.version_info[0] >= 3   # enabled for all 3.x
+    if enable and _hook not in sys.meta_path:
+        sys.meta_path.insert(0, _hook)  # insert at beginning. This could be made a parameter
+
+    # We could return the hook when there are ways of configuring it
+    #return _hook
+
+
+def remove_hooks():
+    if _hook in sys.meta_path:
+        sys.meta_path.remove(_hook)
+
+
+def detect_hooks():
+    """
+    Returns True if the import hooks are installed, False if not.
+    """
+    return _hook in sys.meta_path
+    # present = any([hasattr(hook, 'PY2FIXER') for hook in sys.meta_path])
+    # return present
+
+
+class hooks(object):
+    """
+    Acts as a context manager. Use like this:
+
+    >>> from past import translation
+    >>> with translation.hooks():
+    ...     import mypy2module
+    >>> import requests        # py2/3 compatible anyway
+    >>> # etc.
+    """
+    def __enter__(self):
+        self.hooks_were_installed = detect_hooks()
+        install_hooks()
+        return self
+
+    def __exit__(self, *args):
+        if not self.hooks_were_installed:
+            remove_hooks()
+
+
+class suspend_hooks(object):
+    """
+    Acts as a context manager. Use like this:
+
+    >>> from past import translation
+    >>> translation.install_hooks()
+    >>> import http.client
+    >>> # ...
+    >>> with translation.suspend_hooks():
+    >>>     import requests     # or others that support Py2/3
+
+    If the hooks were disabled before the context, they are not installed when
+    the context is left.
+    """
+    def __enter__(self):
+        self.hooks_were_installed = detect_hooks()
+        remove_hooks()
+        return self
+    def __exit__(self, *args):
+        if self.hooks_were_installed:
+            install_hooks()
diff --git a/libs/past/types/__init__.py b/libs/past/types/__init__.py
new file mode 100644
index 000000000..91dd270f2
--- /dev/null
+++ b/libs/past/types/__init__.py
@@ -0,0 +1,29 @@
+"""
+Forward-ports of types from Python 2 for use with Python 3:
+
+- ``basestring``: equivalent to ``(str, bytes)`` in ``isinstance`` checks
+- ``dict``: with list-producing .keys() etc. methods
+- ``str``: bytes-like, but iterating over them doesn't product integers
+- ``long``: alias of Py3 int with ``L`` suffix in the ``repr``
+- ``unicode``: alias of Py3 str with ``u`` prefix in the ``repr``
+
+"""
+
+from past import utils
+
+if utils.PY2:
+    import __builtin__
+    basestring = __builtin__.basestring
+    dict = __builtin__.dict
+    str = __builtin__.str
+    long = __builtin__.long
+    unicode = __builtin__.unicode
+    __all__ = []
+else:
+    from .basestring import basestring
+    from .olddict import olddict
+    from .oldstr import oldstr
+    long = int
+    unicode = str
+    # from .unicode import unicode
+    __all__ = ['basestring', 'olddict', 'oldstr', 'long', 'unicode']
diff --git a/libs/past/types/basestring.py b/libs/past/types/basestring.py
new file mode 100644
index 000000000..1cab22f6c
--- /dev/null
+++ b/libs/past/types/basestring.py
@@ -0,0 +1,39 @@
+"""
+An implementation of the basestring type for Python 3
+
+Example use:
+
+>>> s = b'abc'
+>>> assert isinstance(s, basestring)
+>>> from past.types import str as oldstr
+>>> s2 = oldstr(b'abc')
+>>> assert isinstance(s2, basestring)
+
+"""
+
+import sys
+
+from past.utils import with_metaclass, PY2
+
+if PY2:
+    str = unicode
+
+ver = sys.version_info[:2]
+
+
+class BaseBaseString(type):
+    def __instancecheck__(cls, instance):
+        return isinstance(instance, (bytes, str))
+
+    def __subclasshook__(cls, thing):
+        # TODO: What should go here?
+        raise NotImplemented
+
+
+class basestring(with_metaclass(BaseBaseString)):
+    """
+    A minimal backport of the Python 2 basestring type to Py3
+    """
+
+
+__all__ = ['basestring']
diff --git a/libs/past/types/olddict.py b/libs/past/types/olddict.py
new file mode 100644
index 000000000..f4f92a26a
--- /dev/null
+++ b/libs/past/types/olddict.py
@@ -0,0 +1,96 @@
+"""
+A dict subclass for Python 3 that behaves like Python 2's dict
+
+Example use:
+
+>>> from past.builtins import dict
+>>> d1 = dict()    # instead of {} for an empty dict
+>>> d2 = dict(key1='value1', key2='value2')
+
+The keys, values and items methods now return lists on Python 3.x and there are
+methods for iterkeys, itervalues, iteritems, and viewkeys etc.
+
+>>> for d in (d1, d2):
+...     assert isinstance(d.keys(), list)
+...     assert isinstance(d.values(), list)
+...     assert isinstance(d.items(), list)
+"""
+
+import sys
+
+from past.utils import with_metaclass
+
+
+_builtin_dict = dict
+ver = sys.version_info[:2]
+
+
+class BaseOldDict(type):
+    def __instancecheck__(cls, instance):
+        return isinstance(instance, _builtin_dict)
+
+
+class olddict(with_metaclass(BaseOldDict, _builtin_dict)):
+    """
+    A backport of the Python 3 dict object to Py2
+    """
+    iterkeys = _builtin_dict.keys
+    viewkeys = _builtin_dict.keys
+
+    def keys(self):
+        return list(super(olddict, self).keys())
+
+    itervalues = _builtin_dict.values
+    viewvalues = _builtin_dict.values
+
+    def values(self):
+        return list(super(olddict, self).values())
+
+    iteritems = _builtin_dict.items
+    viewitems = _builtin_dict.items
+
+    def items(self):
+        return list(super(olddict, self).items())
+
+    def has_key(self, k):
+        """
+        D.has_key(k) -> True if D has a key k, else False
+        """
+        return k in self
+
+    # def __new__(cls, *args, **kwargs):
+    #     """
+    #     dict() -> new empty dictionary
+    #     dict(mapping) -> new dictionary initialized from a mapping object's
+    #         (key, value) pairs
+    #     dict(iterable) -> new dictionary initialized as if via:
+    #         d = {}
+    #         for k, v in iterable:
+    #             d[k] = v
+    #     dict(**kwargs) -> new dictionary initialized with the name=value pairs
+    #         in the keyword argument list.  For example:  dict(one=1, two=2)
+
+    #     """
+    #
+    #     if len(args) == 0:
+    #         return super(olddict, cls).__new__(cls)
+    #     # Was: elif isinstance(args[0], newbytes):
+    #     # We use type() instead of the above because we're redefining
+    #     # this to be True for all unicode string subclasses. Warning:
+    #     # This may render newstr un-subclassable.
+    #     elif type(args[0]) == olddict:
+    #         return args[0]
+    #     # elif isinstance(args[0], _builtin_dict):
+    #     #     value = args[0]
+    #     else:
+    #         value = args[0]
+    #     return super(olddict, cls).__new__(cls, value)
+
+    def __native__(self):
+        """
+        Hook for the past.utils.native() function
+        """
+        return super(oldbytes, self)
+
+
+__all__ = ['olddict']
diff --git a/libs/past/types/oldstr.py b/libs/past/types/oldstr.py
new file mode 100644
index 000000000..7768d3284
--- /dev/null
+++ b/libs/past/types/oldstr.py
@@ -0,0 +1,132 @@
+"""
+Pure-Python implementation of a Python 2-like str object for Python 3.
+"""
+
+from collections import Iterable
+from numbers import Integral
+
+from past.utils import PY2, with_metaclass
+
+
+_builtin_bytes = bytes
+
+
+class BaseOldStr(type):
+    def __instancecheck__(cls, instance):
+        return isinstance(instance, _builtin_bytes)
+
+
+def unescape(s):
+    """
+    Interprets strings with escape sequences
+
+    Example:
+    >>> s = unescape(r'abc\\def')   # i.e. 'abc\\\\def'
+    >>> print(s)
+    'abc\def'
+    >>> s2 = unescape('abc\\ndef')
+    >>> len(s2)
+    8
+    >>> print(s2)
+    abc
+    def
+    """
+    return s.encode().decode('unicode_escape')
+
+
+class oldstr(with_metaclass(BaseOldStr, _builtin_bytes)):
+    """
+    A forward port of the Python 2 8-bit string object to Py3
+    """
+    # Python 2 strings have no __iter__ method:
+    @property
+    def __iter__(self):
+        raise AttributeError
+
+    def __dir__(self):
+        return [thing for thing in dir(_builtin_bytes) if thing != '__iter__']
+
+    # def __new__(cls, *args, **kwargs):
+    #     """
+    #     From the Py3 bytes docstring:
+
+    #     bytes(iterable_of_ints) -> bytes
+    #     bytes(string, encoding[, errors]) -> bytes
+    #     bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer
+    #     bytes(int) -> bytes object of size given by the parameter initialized with null bytes
+    #     bytes() -> empty bytes object
+    #
+    #     Construct an immutable array of bytes from:
+    #       - an iterable yielding integers in range(256)
+    #       - a text string encoded using the specified encoding
+    #       - any object implementing the buffer API.
+    #       - an integer
+    #     """
+    #
+    #     if len(args) == 0:
+    #         return super(newbytes, cls).__new__(cls)
+    #     # Was: elif isinstance(args[0], newbytes):
+    #     # We use type() instead of the above because we're redefining
+    #     # this to be True for all unicode string subclasses. Warning:
+    #     # This may render newstr un-subclassable.
+    #     elif type(args[0]) == newbytes:
+    #         return args[0]
+    #     elif isinstance(args[0], _builtin_bytes):
+    #         value = args[0]
+    #     elif isinstance(args[0], unicode):
+    #         if 'encoding' not in kwargs:
+    #             raise TypeError('unicode string argument without an encoding')
+    #         ###
+    #         # Was:   value = args[0].encode(**kwargs)
+    #         # Python 2.6 string encode() method doesn't take kwargs:
+    #         # Use this instead:
+    #         newargs = [kwargs['encoding']]
+    #         if 'errors' in kwargs:
+    #             newargs.append(kwargs['errors'])
+    #         value = args[0].encode(*newargs)
+    #         ###
+    #     elif isinstance(args[0], Iterable):
+    #         if len(args[0]) == 0:
+    #             # What is this?
+    #             raise ValueError('unknown argument type')
+    #         elif len(args[0]) > 0 and isinstance(args[0][0], Integral):
+    #             # It's a list of integers
+    #             value = b''.join([chr(x) for x in args[0]])
+    #         else:
+    #             raise ValueError('item cannot be interpreted as an integer')
+    #     elif isinstance(args[0], Integral):
+    #         if args[0] < 0:
+    #             raise ValueError('negative count')
+    #         value = b'\x00' * args[0]
+    #     else:
+    #         value = args[0]
+    #     return super(newbytes, cls).__new__(cls, value)
+
+    def __repr__(self):
+        s = super(oldstr, self).__repr__()   # e.g. b'abc' on Py3, b'abc' on Py3
+        return s[1:]
+
+    def __str__(self):
+        s = super(oldstr, self).__str__()   # e.g. "b'abc'" or "b'abc\\ndef'
+        # TODO: fix this:
+        assert s[:2] == "b'" and s[-1] == "'"
+        return unescape(s[2:-1])            # e.g. 'abc'    or 'abc\ndef'
+
+    def __getitem__(self, y):
+        if isinstance(y, Integral):
+            return super(oldstr, self).__getitem__(slice(y, y+1))
+        else:
+            return super(oldstr, self).__getitem__(y)
+
+    def __getslice__(self, *args):
+        return self.__getitem__(slice(*args))
+
+    def __contains__(self, key):
+        if isinstance(key, int):
+            return False
+
+    def __native__(self):
+        return bytes(self)
+
+
+__all__ = ['oldstr']
diff --git a/libs/past/utils/__init__.py b/libs/past/utils/__init__.py
new file mode 100644
index 000000000..c6606d0b9
--- /dev/null
+++ b/libs/past/utils/__init__.py
@@ -0,0 +1,97 @@
+"""
+Various non-built-in utility functions and definitions for Py2
+compatibility in Py3.
+
+For example:
+
+    >>> # The old_div() function behaves like Python 2's / operator
+    >>> # without "from __future__ import division"
+    >>> from past.utils import old_div
+    >>> old_div(3, 2)    # like 3/2 in Py2
+    0
+    >>> old_div(3, 2.0)  # like 3/2.0 in Py2
+    1.5
+"""
+
+import sys
+import numbers
+
+PY3 = sys.version_info[0] == 3
+PY2 = sys.version_info[0] == 2
+PYPY = hasattr(sys, 'pypy_translation_info')
+
+
+def with_metaclass(meta, *bases):
+    """
+    Function from jinja2/_compat.py. License: BSD.
+
+    Use it like this::
+
+        class BaseForm(object):
+            pass
+
+        class FormType(type):
+            pass
+
+        class Form(with_metaclass(FormType, BaseForm)):
+            pass
+
+    This requires a bit of explanation: the basic idea is to make a
+    dummy metaclass for one level of class instantiation that replaces
+    itself with the actual metaclass.  Because of internal type checks
+    we also need to make sure that we downgrade the custom metaclass
+    for one level to something closer to type (that's why __call__ and
+    __init__ comes back from type etc.).
+
+    This has the advantage over six.with_metaclass of not introducing
+    dummy classes into the final MRO.
+    """
+    class metaclass(meta):
+        __call__ = type.__call__
+        __init__ = type.__init__
+        def __new__(cls, name, this_bases, d):
+            if this_bases is None:
+                return type.__new__(cls, name, (), d)
+            return meta(name, bases, d)
+    return metaclass('temporary_class', None, {})
+
+
+def native(obj):
+    """
+    On Py2, this is a no-op: native(obj) -> obj
+
+    On Py3, returns the corresponding native Py3 types that are
+    superclasses for forward-ported objects from Py2:
+
+    >>> from past.builtins import str, dict
+
+    >>> native(str(b'ABC'))   # Output on Py3 follows. On Py2, output is 'ABC'
+    b'ABC'
+    >>> type(native(str(b'ABC')))
+    bytes
+
+    Existing native types on Py3 will be returned unchanged:
+
+    >>> type(native(b'ABC'))
+    bytes
+    """
+    if hasattr(obj, '__native__'):
+        return obj.__native__()
+    else:
+        return obj
+
+
+# An alias for future.utils.old_div():
+def old_div(a, b):
+    """
+    Equivalent to ``a / b`` on Python 2 without ``from __future__ import
+    division``.
+
+    TODO: generalize this to other objects (like arrays etc.)
+    """
+    if isinstance(a, numbers.Integral) and isinstance(b, numbers.Integral):
+        return a // b
+    else:
+        return a / b
+
+__all__ = ['PY3', 'PY2', 'PYPY', 'with_metaclass', 'native', 'old_div']