jupyter

packages/python/yap_kernel/yap_ipython/lib/__init__.py

@@ -0,0 +1,21 @@
+# encoding: utf-8
+"""
+Extra capabilities for yap_ipython
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The yap_ipython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from yap_ipython.lib.security import passwd
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------

packages/python/yap_kernel/yap_ipython/lib/backgroundjobs.py

@@ -0,0 +1,489 @@
+# -*- coding: utf-8 -*-
+"""Manage background (threaded) jobs conveniently from an interactive shell.
+
+This module provides a BackgroundJobManager class.  This is the main class
+meant for public usage, it implements an object which can create and manage
+new background jobs.
+
+It also provides the actual job classes managed by these BackgroundJobManager
+objects, see their docstrings below.
+
+
+This system was inspired by discussions with B. Granger and the
+BackgroundCommand class described in the book Python Scripting for
+Computational Science, by H. P. Langtangen:
+
+http://folk.uio.no/hpl/scripting
+
+(although ultimately no code from this text was used, as yap_ipython's system is a
+separate implementation).
+
+An example notebook is provided in our documentation illustrating interactive
+use of the system.
+"""
+
+#*****************************************************************************
+#       Copyright (C) 2005-2006 Fernando Perez <fperez@colorado.edu>
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#*****************************************************************************
+
+# Code begins
+import sys
+import threading
+
+from yap_ipython import get_ipython
+from yap_ipython.core.ultratb import AutoFormattedTB
+from logging import error, debug
+
+
+class BackgroundJobManager(object):
+    """Class to manage a pool of backgrounded threaded jobs.
+
+    Below, we assume that 'jobs' is a BackgroundJobManager instance.
+    
+    Usage summary (see the method docstrings for details):
+
+      jobs.new(...) -> start a new job
+      
+      jobs() or jobs.status() -> print status summary of all jobs
+
+      jobs[N] -> returns job number N.
+
+      foo = jobs[N].result -> assign to variable foo the result of job N
+
+      jobs[N].traceback() -> print the traceback of dead job N
+
+      jobs.remove(N) -> remove (finished) job N
+
+      jobs.flush() -> remove all finished jobs
+      
+    As a convenience feature, BackgroundJobManager instances provide the
+    utility result and traceback methods which retrieve the corresponding
+    information from the jobs list:
+
+      jobs.result(N) <--> jobs[N].result
+      jobs.traceback(N) <--> jobs[N].traceback()
+
+    While this appears minor, it allows you to use tab completion
+    interactively on the job manager instance.
+    """
+
+    def __init__(self):
+        # Lists for job management, accessed via a property to ensure they're
+        # up to date.x
+        self._running  = []
+        self._completed = []
+        self._dead = []
+        # A dict of all jobs, so users can easily access any of them
+        self.all = {}
+        # For reporting
+        self._comp_report = []
+        self._dead_report = []
+        # Store status codes locally for fast lookups
+        self._s_created   = BackgroundJobBase.stat_created_c
+        self._s_running   = BackgroundJobBase.stat_running_c
+        self._s_completed = BackgroundJobBase.stat_completed_c
+        self._s_dead      = BackgroundJobBase.stat_dead_c
+
+    @property
+    def running(self):
+        self._update_status()
+        return self._running
+
+    @property
+    def dead(self):
+        self._update_status()
+        return self._dead
+
+    @property
+    def completed(self):
+        self._update_status()
+        return self._completed
+
+    def new(self, func_or_exp, *args, **kwargs):
+        """Add a new background job and start it in a separate thread.
+
+        There are two types of jobs which can be created:
+
+        1. Jobs based on expressions which can be passed to an eval() call.
+        The expression must be given as a string.  For example:
+
+          job_manager.new('myfunc(x,y,z=1)'[,glob[,loc]])
+
+        The given expression is passed to eval(), along with the optional
+        global/local dicts provided.  If no dicts are given, they are
+        extracted automatically from the caller's frame.
+        
+        A Python statement is NOT a valid eval() expression.  Basically, you
+        can only use as an eval() argument something which can go on the right
+        of an '=' sign and be assigned to a variable.
+
+        For example,"print 'hello'" is not valid, but '2+3' is.
+
+        2. Jobs given a function object, optionally passing additional
+        positional arguments:
+
+          job_manager.new(myfunc, x, y)
+
+        The function is called with the given arguments.
+
+        If you need to pass keyword arguments to your function, you must
+        supply them as a dict named kw:
+
+          job_manager.new(myfunc, x, y, kw=dict(z=1))
+
+        The reason for this assymmetry is that the new() method needs to
+        maintain access to its own keywords, and this prevents name collisions
+        between arguments to new() and arguments to your own functions.
+
+        In both cases, the result is stored in the job.result field of the
+        background job object.
+
+        You can set `daemon` attribute of the thread by giving the keyword
+        argument `daemon`.
+
+        Notes and caveats:
+
+        1. All threads running share the same standard output.  Thus, if your
+        background jobs generate output, it will come out on top of whatever
+        you are currently writing.  For this reason, background jobs are best
+        used with silent functions which simply return their output.
+
+        2. Threads also all work within the same global namespace, and this
+        system does not lock interactive variables.  So if you send job to the
+        background which operates on a mutable object for a long time, and
+        start modifying that same mutable object interactively (or in another
+        backgrounded job), all sorts of bizarre behaviour will occur.
+
+        3. If a background job is spending a lot of time inside a C extension
+        module which does not release the Python Global Interpreter Lock
+        (GIL), this will block the yap_ipython prompt.  This is simply because the
+        Python interpreter can only switch between threads at Python
+        bytecodes.  While the execution is inside C code, the interpreter must
+        simply wait unless the extension module releases the GIL.
+
+        4. There is no way, due to limitations in the Python threads library,
+        to kill a thread once it has started."""
+        
+        if callable(func_or_exp):
+            kw  = kwargs.get('kw',{})
+            job = BackgroundJobFunc(func_or_exp,*args,**kw)
+        elif isinstance(func_or_exp, str):
+            if not args:
+                frame = sys._getframe(1)
+                glob, loc = frame.f_globals, frame.f_locals
+            elif len(args)==1:
+                glob = loc = args[0]
+            elif len(args)==2:
+                glob,loc = args
+            else:
+                raise ValueError(
+                      'Expression jobs take at most 2 args (globals,locals)')
+            job = BackgroundJobExpr(func_or_exp, glob, loc)
+        else:
+            raise TypeError('invalid args for new job')
+
+        if kwargs.get('daemon', False):
+            job.daemon = True
+        job.num = len(self.all)+1 if self.all else 0
+        self.running.append(job)
+        self.all[job.num] = job
+        debug('Starting job # %s in a separate thread.' % job.num)
+        job.start()
+        return job
+
+    def __getitem__(self, job_key):
+        num = job_key if isinstance(job_key, int) else job_key.num
+        return self.all[num]
+
+    def __call__(self):
+        """An alias to self.status(),
+
+        This allows you to simply call a job manager instance much like the
+        Unix `jobs` shell command."""
+
+        return self.status()
+
+    def _update_status(self):
+        """Update the status of the job lists.
+
+        This method moves finished jobs to one of two lists:
+          - self.completed: jobs which completed successfully
+          - self.dead: jobs which finished but died.
+
+        It also copies those jobs to corresponding _report lists.  These lists
+        are used to report jobs completed/dead since the last update, and are
+        then cleared by the reporting function after each call."""
+
+        # Status codes
+        srun, scomp, sdead = self._s_running, self._s_completed, self._s_dead
+        # State lists, use the actual lists b/c the public names are properties
+        # that call this very function on access
+        running, completed, dead = self._running, self._completed, self._dead
+
+        # Now, update all state lists
+        for num, job in enumerate(running):
+            stat = job.stat_code
+            if stat == srun:
+                continue
+            elif stat == scomp:
+                completed.append(job)
+                self._comp_report.append(job)
+                running[num] = False
+            elif stat == sdead:
+                dead.append(job)
+                self._dead_report.append(job)
+                running[num] = False
+        # Remove dead/completed jobs from running list
+        running[:] = filter(None, running)
+
+    def _group_report(self,group,name):
+        """Report summary for a given job group.
+
+        Return True if the group had any elements."""
+
+        if group:
+            print('%s jobs:' % name)
+            for job in group:
+                print('%s : %s' % (job.num,job))
+            print()
+            return True
+
+    def _group_flush(self,group,name):
+        """Flush a given job group
+
+        Return True if the group had any elements."""
+
+        njobs = len(group)
+        if njobs:
+            plural = {1:''}.setdefault(njobs,'s')
+            print('Flushing %s %s job%s.' % (njobs,name,plural))
+            group[:] = []
+            return True
+        
+    def _status_new(self):
+        """Print the status of newly finished jobs.
+
+        Return True if any new jobs are reported.
+
+        This call resets its own state every time, so it only reports jobs
+        which have finished since the last time it was called."""
+
+        self._update_status()
+        new_comp = self._group_report(self._comp_report, 'Completed')
+        new_dead = self._group_report(self._dead_report,
+                                      'Dead, call jobs.traceback() for details')
+        self._comp_report[:] = []
+        self._dead_report[:] = []
+        return new_comp or new_dead
+                
+    def status(self,verbose=0):
+        """Print a status of all jobs currently being managed."""
+
+        self._update_status()
+        self._group_report(self.running,'Running')
+        self._group_report(self.completed,'Completed')
+        self._group_report(self.dead,'Dead')
+        # Also flush the report queues
+        self._comp_report[:] = []
+        self._dead_report[:] = []
+
+    def remove(self,num):
+        """Remove a finished (completed or dead) job."""
+
+        try:
+            job = self.all[num]
+        except KeyError:
+            error('Job #%s not found' % num)
+        else:
+            stat_code = job.stat_code
+            if stat_code == self._s_running:
+                error('Job #%s is still running, it can not be removed.' % num)
+                return
+            elif stat_code == self._s_completed:
+                self.completed.remove(job)
+            elif stat_code == self._s_dead:
+                self.dead.remove(job)
+
+    def flush(self):
+        """Flush all finished jobs (completed and dead) from lists.
+
+        Running jobs are never flushed.
+
+        It first calls _status_new(), to update info. If any jobs have
+        completed since the last _status_new() call, the flush operation
+        aborts."""
+
+        # Remove the finished jobs from the master dict
+        alljobs = self.all
+        for job in self.completed+self.dead:
+            del(alljobs[job.num])
+
+        # Now flush these lists completely
+        fl_comp = self._group_flush(self.completed, 'Completed')
+        fl_dead = self._group_flush(self.dead, 'Dead')
+        if not (fl_comp or fl_dead):
+            print('No jobs to flush.')
+
+    def result(self,num):
+        """result(N) -> return the result of job N."""
+        try:
+            return self.all[num].result
+        except KeyError:
+            error('Job #%s not found' % num)
+
+    def _traceback(self, job):
+        num = job if isinstance(job, int) else job.num
+        try:
+            self.all[num].traceback()
+        except KeyError:
+            error('Job #%s not found' % num)
+
+    def traceback(self, job=None):
+        if job is None:
+            self._update_status()
+            for deadjob in self.dead:
+                print("Traceback for: %r" % deadjob)
+                self._traceback(deadjob)
+                print()
+        else:
+            self._traceback(job)
+
+
+class BackgroundJobBase(threading.Thread):
+    """Base class to build BackgroundJob classes.
+
+    The derived classes must implement:
+
+    - Their own __init__, since the one here raises NotImplementedError.  The
+      derived constructor must call self._init() at the end, to provide common
+      initialization.
+
+    - A strform attribute used in calls to __str__.
+
+    - A call() method, which will make the actual execution call and must
+      return a value to be held in the 'result' field of the job object.
+    """
+
+    # Class constants for status, in string and as numerical codes (when
+    # updating jobs lists, we don't want to do string comparisons).  This will
+    # be done at every user prompt, so it has to be as fast as possible
+    stat_created   = 'Created'; stat_created_c = 0
+    stat_running   = 'Running'; stat_running_c = 1
+    stat_completed = 'Completed'; stat_completed_c = 2
+    stat_dead      = 'Dead (Exception), call jobs.traceback() for details'
+    stat_dead_c = -1
+
+    def __init__(self):
+        """Must be implemented in subclasses.
+
+        Subclasses must call :meth:`_init` for standard initialisation.
+        """
+        raise NotImplementedError("This class can not be instantiated directly.")
+
+    def _init(self):
+        """Common initialization for all BackgroundJob objects"""
+        
+        for attr in ['call','strform']:
+            assert hasattr(self,attr), "Missing attribute <%s>" % attr
+        
+        # The num tag can be set by an external job manager
+        self.num = None
+      
+        self.status    = BackgroundJobBase.stat_created
+        self.stat_code = BackgroundJobBase.stat_created_c
+        self.finished  = False
+        self.result    = '<BackgroundJob has not completed>'
+        
+        # reuse the ipython traceback handler if we can get to it, otherwise
+        # make a new one
+        try:
+            make_tb = get_ipython().InteractiveTB.text
+        except:
+            make_tb = AutoFormattedTB(mode = 'Context',
+                                      color_scheme='NoColor',
+                                      tb_offset = 1).text
+        # Note that the actual API for text() requires the three args to be
+        # passed in, so we wrap it in a simple lambda.
+        self._make_tb = lambda : make_tb(None, None, None)
+
+        # Hold a formatted traceback if one is generated.
+        self._tb = None
+        
+        threading.Thread.__init__(self)
+
+    def __str__(self):
+        return self.strform
+
+    def __repr__(self):
+        return '<BackgroundJob #%d: %s>' % (self.num, self.strform)
+
+    def traceback(self):
+        print(self._tb)
+        
+    def run(self):
+        try:
+            self.status    = BackgroundJobBase.stat_running
+            self.stat_code = BackgroundJobBase.stat_running_c
+            self.result    = self.call()
+        except:
+            self.status    = BackgroundJobBase.stat_dead
+            self.stat_code = BackgroundJobBase.stat_dead_c
+            self.finished  = None
+            self.result    = ('<BackgroundJob died, call jobs.traceback() for details>')
+            self._tb       = self._make_tb()
+        else:
+            self.status    = BackgroundJobBase.stat_completed
+            self.stat_code = BackgroundJobBase.stat_completed_c
+            self.finished  = True
+
+
+class BackgroundJobExpr(BackgroundJobBase):
+    """Evaluate an expression as a background job (uses a separate thread)."""
+
+    def __init__(self, expression, glob=None, loc=None):
+        """Create a new job from a string which can be fed to eval().
+
+        global/locals dicts can be provided, which will be passed to the eval
+        call."""
+
+        # fail immediately if the given expression can't be compiled
+        self.code = compile(expression,'<BackgroundJob compilation>','eval')
+                
+        glob = {} if glob is None else glob
+        loc = {} if loc is None else loc
+        self.expression = self.strform = expression
+        self.glob = glob
+        self.loc = loc
+        self._init()
+        
+    def call(self):
+        return eval(self.code,self.glob,self.loc)
+
+
+class BackgroundJobFunc(BackgroundJobBase):
+    """Run a function call as a background job (uses a separate thread)."""
+
+    def __init__(self, func, *args, **kwargs):
+        """Create a new job from a callable object.
+
+        Any positional arguments and keyword args given to this constructor
+        after the initial callable are passed directly to it."""
+
+        if not callable(func):
+            raise TypeError(
+                'first argument to BackgroundJobFunc must be callable')
+        
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+        # The string form will only include the function passed, because
+        # generating string representations of the arguments is a potentially
+        # _very_ expensive operation (e.g. with large arrays).
+        self.strform = str(func)
+        self._init()
+
+    def call(self):
+        return self.func(*self.args, **self.kwargs)

packages/python/yap_kernel/yap_ipython/lib/clipboard.py

@@ -0,0 +1,69 @@
+""" Utilities for accessing the platform's clipboard.
+"""
+
+import subprocess
+
+from yap_ipython.core.error import TryNext
+import yap_ipython.utils.py3compat as py3compat
+
+class ClipboardEmpty(ValueError):
+    pass
+
+def win32_clipboard_get():
+    """ Get the current clipboard's text on Windows.
+
+    Requires Mark Hammond's pywin32 extensions.
+    """
+    try:
+        import win32clipboard
+    except ImportError:
+        raise TryNext("Getting text from the clipboard requires the pywin32 "
+                      "extensions: http://sourceforge.net/projects/pywin32/")
+    win32clipboard.OpenClipboard()
+    try:
+        text = win32clipboard.GetClipboardData(win32clipboard.CF_UNICODETEXT)
+    except (TypeError, win32clipboard.error):
+        try:
+            text = win32clipboard.GetClipboardData(win32clipboard.CF_TEXT)
+            text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
+        except (TypeError, win32clipboard.error):
+            raise ClipboardEmpty
+    finally:
+        win32clipboard.CloseClipboard()
+    return text
+
+def osx_clipboard_get():
+    """ Get the clipboard's text on OS X.
+    """
+    p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'],
+        stdout=subprocess.PIPE)
+    text, stderr = p.communicate()
+    # Text comes in with old Mac \r line endings. Change them to \n.
+    text = text.replace(b'\r', b'\n')
+    text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
+    return text
+
+def tkinter_clipboard_get():
+    """ Get the clipboard's text using Tkinter.
+
+    This is the default on systems that are not Windows or OS X. It may
+    interfere with other UI toolkits and should be replaced with an
+    implementation that uses that toolkit.
+    """
+    try:
+        from tkinter import Tk, TclError 
+    except ImportError:
+        raise TryNext("Getting text from the clipboard on this platform requires tkinter.")
+        
+    root = Tk()
+    root.withdraw()
+    try:
+        text = root.clipboard_get()
+    except TclError:
+        raise ClipboardEmpty
+    finally:
+        root.destroy()
+    text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
+    return text
+
+

packages/python/yap_kernel/yap_ipython/lib/deepreload.py

@@ -0,0 +1,347 @@
+# -*- coding: utf-8 -*-
+"""
+Provides a reload() function that acts recursively.
+
+Python's normal :func:`python:reload` function only reloads the module that it's
+passed. The :func:`reload` function in this module also reloads everything
+imported from that module, which is useful when you're changing files deep
+inside a package.
+
+To use this as your default reload function, type this for Python 2::
+
+    import __builtin__
+    from yap_ipython.lib import deepreload
+    __builtin__.reload = deepreload.reload
+
+Or this for Python 3::
+
+    import builtins
+    from yap_ipython.lib import deepreload
+    builtins.reload = deepreload.reload
+
+A reference to the original :func:`python:reload` is stored in this module as
+:data:`original_reload`, so you can restore it later.
+
+This code is almost entirely based on knee.py, which is a Python
+re-implementation of hierarchical module import.
+"""
+#*****************************************************************************
+#       Copyright (C) 2001 Nathaniel Gray <n8gray@caltech.edu>
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#*****************************************************************************
+
+import builtins as builtin_mod
+from contextlib import contextmanager
+import imp
+import sys
+
+from types import ModuleType
+from warnings import warn
+import types
+
+original_import = builtin_mod.__import__
+
+@contextmanager
+def replace_import_hook(new_import):
+    saved_import = builtin_mod.__import__
+    builtin_mod.__import__ = new_import
+    try:
+        yield
+    finally:
+        builtin_mod.__import__ = saved_import
+
+def get_parent(globals, level):
+    """
+    parent, name = get_parent(globals, level)
+
+    Return the package that an import is being performed in.  If globals comes
+    from the module foo.bar.bat (not itself a package), this returns the
+    sys.modules entry for foo.bar.  If globals is from a package's __init__.py,
+    the package's entry in sys.modules is returned.
+
+    If globals doesn't come from a package or a module in a package, or a
+    corresponding entry is not found in sys.modules, None is returned.
+    """
+    orig_level = level
+
+    if not level or not isinstance(globals, dict):
+        return None, ''
+
+    pkgname = globals.get('__package__', None)
+
+    if pkgname is not None:
+        # __package__ is set, so use it
+        if not hasattr(pkgname, 'rindex'):
+            raise ValueError('__package__ set to non-string')
+        if len(pkgname) == 0:
+            if level > 0:
+                raise ValueError('Attempted relative import in non-package')
+            return None, ''
+        name = pkgname
+    else:
+        # __package__ not set, so figure it out and set it
+        if '__name__' not in globals:
+            return None, ''
+        modname = globals['__name__']
+
+        if '__path__' in globals:
+            # __path__ is set, so modname is already the package name
+            globals['__package__'] = name = modname
+        else:
+            # Normal module, so work out the package name if any
+            lastdot = modname.rfind('.')
+            if lastdot < 0 < level:
+                raise ValueError("Attempted relative import in non-package")
+            if lastdot < 0:
+                globals['__package__'] = None
+                return None, ''
+            globals['__package__'] = name = modname[:lastdot]
+
+    dot = len(name)
+    for x in range(level, 1, -1):
+        try:
+            dot = name.rindex('.', 0, dot)
+        except ValueError:
+            raise ValueError("attempted relative import beyond top-level "
+                             "package")
+    name = name[:dot]
+
+    try:
+        parent = sys.modules[name]
+    except:
+        if orig_level < 1:
+            warn("Parent module '%.200s' not found while handling absolute "
+                 "import" % name)
+            parent = None
+        else:
+            raise SystemError("Parent module '%.200s' not loaded, cannot "
+                              "perform relative import" % name)
+
+    # We expect, but can't guarantee, if parent != None, that:
+    # - parent.__name__ == name
+    # - parent.__dict__ is globals
+    # If this is violated...  Who cares?
+    return parent, name
+
+def load_next(mod, altmod, name, buf):
+    """
+    mod, name, buf = load_next(mod, altmod, name, buf)
+
+    altmod is either None or same as mod
+    """
+
+    if len(name) == 0:
+        # completely empty module name should only happen in
+        # 'from . import' (or '__import__("")')
+        return mod, None, buf
+
+    dot = name.find('.')
+    if dot == 0:
+        raise ValueError('Empty module name')
+
+    if dot < 0:
+        subname = name
+        next = None
+    else:
+        subname = name[:dot]
+        next = name[dot+1:]
+
+    if buf != '':
+        buf += '.'
+    buf += subname
+
+    result = import_submodule(mod, subname, buf)
+    if result is None and mod != altmod:
+        result = import_submodule(altmod, subname, subname)
+        if result is not None:
+            buf = subname
+
+    if result is None:
+        raise ImportError("No module named %.200s" % name)
+
+    return result, next, buf
+
+
+# Need to keep track of what we've already reloaded to prevent cyclic evil
+found_now = {}
+
+def import_submodule(mod, subname, fullname):
+    """m = import_submodule(mod, subname, fullname)"""
+    # Require:
+    # if mod == None: subname == fullname
+    # else: mod.__name__ + "." + subname == fullname
+
+    global found_now
+    if fullname in found_now and fullname in sys.modules:
+        m = sys.modules[fullname]
+    else:
+        print('Reloading', fullname)
+        found_now[fullname] = 1
+        oldm = sys.modules.get(fullname, None)
+
+        if mod is None:
+            path = None
+        elif hasattr(mod, '__path__'):
+            path = mod.__path__
+        else:
+            return None
+
+        try:
+            # This appears to be necessary on Python 3, because imp.find_module()
+            # tries to import standard libraries (like io) itself, and we don't
+            # want them to be processed by our deep_import_hook.
+            with replace_import_hook(original_import):
+                fp, filename, stuff = imp.find_module(subname, path)
+        except ImportError:
+            return None
+
+        try:
+            m = imp.load_module(fullname, fp, filename, stuff)
+        except:
+            # load_module probably removed name from modules because of
+            # the error.  Put back the original module object.
+            if oldm:
+                sys.modules[fullname] = oldm
+            raise
+        finally:
+            if fp: fp.close()
+
+        add_submodule(mod, m, fullname, subname)
+
+    return m
+
+def add_submodule(mod, submod, fullname, subname):
+    """mod.{subname} = submod"""
+    if mod is None:
+        return #Nothing to do here.
+
+    if submod is None:
+        submod = sys.modules[fullname]
+
+    setattr(mod, subname, submod)
+
+    return
+
+def ensure_fromlist(mod, fromlist, buf, recursive):
+    """Handle 'from module import a, b, c' imports."""
+    if not hasattr(mod, '__path__'):
+        return
+    for item in fromlist:
+        if not hasattr(item, 'rindex'):
+            raise TypeError("Item in ``from list'' not a string")
+        if item == '*':
+            if recursive:
+                continue # avoid endless recursion
+            try:
+                all = mod.__all__
+            except AttributeError:
+                pass
+            else:
+                ret = ensure_fromlist(mod, all, buf, 1)
+                if not ret:
+                    return 0
+        elif not hasattr(mod, item):
+            import_submodule(mod, item, buf + '.' + item)
+
+def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
+    """Replacement for __import__()"""
+    parent, buf = get_parent(globals, level)
+
+    head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
+
+    tail = head
+    while name:
+        tail, name, buf = load_next(tail, tail, name, buf)
+
+    # If tail is None, both get_parent and load_next found
+    # an empty module name: someone called __import__("") or
+    # doctored faulty bytecode
+    if tail is None:
+        raise ValueError('Empty module name')
+
+    if not fromlist:
+        return head
+
+    ensure_fromlist(tail, fromlist, buf, 0)
+    return tail
+
+modules_reloading = {}
+
+def deep_reload_hook(m):
+    """Replacement for reload()."""
+    # Hardcode this one  as it would raise a NotImplemeentedError from the
+    # bowels of Python and screw up the import machinery after.
+    # unlike other imports the `exclude` list already in place is not enough.
+
+    if m is types:
+        return m
+    if not isinstance(m, ModuleType):
+        raise TypeError("reload() argument must be module")
+
+    name = m.__name__
+
+    if name not in sys.modules:
+        raise ImportError("reload(): module %.200s not in sys.modules" % name)
+
+    global modules_reloading
+    try:
+        return modules_reloading[name]
+    except:
+        modules_reloading[name] = m
+
+    dot = name.rfind('.')
+    if dot < 0:
+        subname = name
+        path = None
+    else:
+        try:
+            parent = sys.modules[name[:dot]]
+        except KeyError:
+            modules_reloading.clear()
+            raise ImportError("reload(): parent %.200s not in sys.modules" % name[:dot])
+        subname = name[dot+1:]
+        path = getattr(parent, "__path__", None)
+
+    try:
+        # This appears to be necessary on Python 3, because imp.find_module()
+        # tries to import standard libraries (like io) itself, and we don't
+        # want them to be processed by our deep_import_hook.
+        with replace_import_hook(original_import):
+            fp, filename, stuff  = imp.find_module(subname, path)
+    finally:
+        modules_reloading.clear()
+
+    try:
+        newm = imp.load_module(name, fp, filename, stuff)
+    except:
+         # load_module probably removed name from modules because of
+         # the error.  Put back the original module object.
+        sys.modules[name] = m
+        raise
+    finally:
+        if fp: fp.close()
+
+    modules_reloading.clear()
+    return newm
+
+# Save the original hooks
+original_reload = imp.reload
+
+# Replacement for reload()
+def reload(module, exclude=('sys', 'os.path', 'builtins', '__main__',
+                            'numpy', 'numpy._globals')):
+    """Recursively reload all modules used in the given module.  Optionally
+    takes a list of modules to exclude from reloading.  The default exclude
+    list contains sys, __main__, and __builtin__, to prevent, e.g., resetting
+    display, exception, and io hooks.
+    """
+    global found_now
+    for i in exclude:
+        found_now[i] = 1
+    try:
+        with replace_import_hook(deep_import_hook):
+            return deep_reload_hook(module)
+    finally:
+        found_now = {}

packages/python/yap_kernel/yap_ipython/lib/demo.py

@@ -0,0 +1,667 @@
+"""Module for interactive demos using yap_ipython.
+
+This module implements a few classes for running Python scripts interactively
+in yap_ipython for demonstrations.  With very simple markup (a few tags in
+comments), you can control points where the script stops executing and returns
+control to yap_ipython.
+
+
+Provided classes
+----------------
+
+The classes are (see their docstrings for further details):
+
+ - Demo: pure python demos
+
+ - IPythonDemo: demos with input to be processed by yap_ipython as if it had been
+   typed interactively (so magics work, as well as any other special syntax you
+   may have added via input prefilters).
+
+ - LineDemo: single-line version of the Demo class.  These demos are executed
+   one line at a time, and require no markup.
+
+ - IPythonLineDemo: yap_ipython version of the LineDemo class (the demo is
+   executed a line at a time, but processed via yap_ipython).
+
+ - ClearMixin: mixin to make Demo classes with less visual clutter.  It
+   declares an empty marquee and a pre_cmd that clears the screen before each
+   block (see Subclassing below).
+
+ - ClearDemo, ClearIPDemo: mixin-enabled versions of the Demo and IPythonDemo
+   classes.
+
+Inheritance diagram:
+
+.. inheritance-diagram:: yap_ipython.lib.demo
+   :parts: 3
+
+Subclassing
+-----------
+
+The classes here all include a few methods meant to make customization by
+subclassing more convenient.  Their docstrings below have some more details:
+
+  - highlight(): format every block and optionally highlight comments and
+    docstring content.
+
+  - marquee(): generates a marquee to provide visible on-screen markers at each
+    block start and end.
+
+  - pre_cmd(): run right before the execution of each block.
+
+  - post_cmd(): run right after the execution of each block.  If the block
+    raises an exception, this is NOT called.
+
+
+Operation
+---------
+
+The file is run in its own empty namespace (though you can pass it a string of
+arguments as if in a command line environment, and it will see those as
+sys.argv).  But at each stop, the global yap_ipython namespace is updated with the
+current internal demo namespace, so you can work interactively with the data
+accumulated so far.
+
+By default, each block of code is printed (with syntax highlighting) before
+executing it and you have to confirm execution.  This is intended to show the
+code to an audience first so you can discuss it, and only proceed with
+execution once you agree.  There are a few tags which allow you to modify this
+behavior.
+
+The supported tags are:
+
+# <demo> stop
+
+  Defines block boundaries, the points where yap_ipython stops execution of the
+  file and returns to the interactive prompt.
+
+  You can optionally mark the stop tag with extra dashes before and after the
+  word 'stop', to help visually distinguish the blocks in a text editor:
+
+  # <demo> --- stop ---
+
+
+# <demo> silent
+
+  Make a block execute silently (and hence automatically).  Typically used in
+  cases where you have some boilerplate or initialization code which you need
+  executed but do not want to be seen in the demo.
+
+# <demo> auto
+
+  Make a block execute automatically, but still being printed.  Useful for
+  simple code which does not warrant discussion, since it avoids the extra
+  manual confirmation.
+
+# <demo> auto_all
+
+  This tag can _only_ be in the first block, and if given it overrides the
+  individual auto tags to make the whole demo fully automatic (no block asks
+  for confirmation).  It can also be given at creation time (or the attribute
+  set later) to override what's in the file.
+
+While _any_ python file can be run as a Demo instance, if there are no stop
+tags the whole file will run in a single block (no different that calling
+first %pycat and then %run).  The minimal markup to make this useful is to
+place a set of stop tags; the other tags are only there to let you fine-tune
+the execution.
+
+This is probably best explained with the simple example file below.  You can
+copy this into a file named ex_demo.py, and try running it via::
+
+    from yap_ipython.lib.demo import Demo
+    d = Demo('ex_demo.py')
+    d()
+
+Each time you call the demo object, it runs the next block.  The demo object
+has a few useful methods for navigation, like again(), edit(), jump(), seek()
+and back().  It can be reset for a new run via reset() or reloaded from disk
+(in case you've edited the source) via reload().  See their docstrings below.
+
+Note: To make this simpler to explore, a file called "demo-exercizer.py" has
+been added to the "docs/examples/core" directory.  Just cd to this directory in
+an yap_ipython session, and type::
+
+  %run demo-exercizer.py
+
+and then follow the directions.
+
+Example
+-------
+
+The following is a very simple example of a valid demo file.
+
+::
+
+    #################### EXAMPLE DEMO <ex_demo.py> ###############################
+    '''A simple interactive demo to illustrate the use of yap_ipython's Demo class.'''
+
+    print 'Hello, welcome to an interactive yap_ipython demo.'
+
+    # The mark below defines a block boundary, which is a point where yap_ipython will
+    # stop execution and return to the interactive prompt. The dashes are actually
+    # optional and used only as a visual aid to clearly separate blocks while
+    # editing the demo code.
+    # <demo> stop
+
+    x = 1
+    y = 2
+
+    # <demo> stop
+
+    # the mark below makes this block as silent
+    # <demo> silent
+
+    print 'This is a silent block, which gets executed but not printed.'
+
+    # <demo> stop
+    # <demo> auto
+    print 'This is an automatic block.'
+    print 'It is executed without asking for confirmation, but printed.'
+    z = x+y
+
+    print 'z=',x
+
+    # <demo> stop
+    # This is just another normal block.
+    print 'z is now:', z
+
+    print 'bye!'
+    ################### END EXAMPLE DEMO <ex_demo.py> ############################
+"""
+
+
+#*****************************************************************************
+#     Copyright (C) 2005-2006 Fernando Perez. <Fernando.Perez@colorado.edu>
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#
+#*****************************************************************************
+
+import os
+import re
+import shlex
+import sys
+import pygments
+
+from yap_ipython.utils.text import marquee
+from yap_ipython.utils import openpy
+from yap_ipython.utils import py3compat
+__all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
+
+class DemoError(Exception): pass
+
+def re_mark(mark):
+    return re.compile(r'^\s*#\s+<demo>\s+%s\s*$' % mark,re.MULTILINE)
+
+class Demo(object):
+
+    re_stop     = re_mark('-*\s?stop\s?-*')
+    re_silent   = re_mark('silent')
+    re_auto     = re_mark('auto')
+    re_auto_all = re_mark('auto_all')
+
+    def __init__(self,src,title='',arg_str='',auto_all=None, format_rst=False,
+                 formatter='terminal', style='default'):
+        """Make a new demo object.  To run the demo, simply call the object.
+
+        See the module docstring for full details and an example (you can use
+        yap_ipython.Demo? in yap_ipython to see it).
+
+        Inputs:
+
+          - src is either a file, or file-like object, or a
+              string that can be resolved to a filename.
+
+        Optional inputs:
+
+          - title: a string to use as the demo name.  Of most use when the demo
+            you are making comes from an object that has no filename, or if you
+            want an alternate denotation distinct from the filename.
+
+          - arg_str(''): a string of arguments, internally converted to a list
+            just like sys.argv, so the demo script can see a similar
+            environment.
+
+          - auto_all(None): global flag to run all blocks automatically without
+            confirmation.  This attribute overrides the block-level tags and
+            applies to the whole demo.  It is an attribute of the object, and
+            can be changed at runtime simply by reassigning it to a boolean
+            value.
+
+          - format_rst(False): a bool to enable comments and doc strings
+            formatting with pygments rst lexer
+
+          - formatter('terminal'): a string of pygments formatter name to be
+            used. Useful values for terminals: terminal, terminal256,
+            terminal16m
+
+          - style('default'): a string of pygments style name to be used.
+          """
+        if hasattr(src, "read"):
+             # It seems to be a file or a file-like object
+            self.fname = "from a file-like object"
+            if title == '':
+                self.title = "from a file-like object"
+            else:
+                self.title = title
+        else:
+             # Assume it's a string or something that can be converted to one
+            self.fname = src
+            if title == '':
+                (filepath, filename) = os.path.split(src)
+                self.title = filename
+            else:
+                self.title = title
+        self.sys_argv = [src] + shlex.split(arg_str)
+        self.auto_all = auto_all
+        self.src = src
+
+        self.inside_ipython = "get_ipython" in globals()
+        if self.inside_ipython:
+            # get a few things from ipython.  While it's a bit ugly design-wise,
+            # it ensures that things like color scheme and the like are always in
+            # sync with the ipython mode being used.  This class is only meant to
+            # be used inside ipython anyways,  so it's OK.
+            ip = get_ipython()  # this is in builtins whenever yap_ipython is running
+            self.ip_ns       = ip.user_ns
+            self.ip_colorize = ip.pycolorize
+            self.ip_showtb   = ip.showtraceback
+            self.ip_run_cell = ip.run_cell
+            self.shell       = ip
+
+        self.formatter = pygments.formatters.get_formatter_by_name(formatter,
+                                                                   style=style)
+        self.python_lexer = pygments.lexers.get_lexer_by_name("py3")
+        self.format_rst = format_rst
+        if format_rst:
+            self.rst_lexer = pygments.lexers.get_lexer_by_name("rst")
+
+        # load user data and initialize data structures
+        self.reload()
+
+    def fload(self):
+        """Load file object."""
+        # read data and parse into blocks
+        if hasattr(self, 'fobj') and self.fobj is not None:
+           self.fobj.close()
+        if hasattr(self.src, "read"):
+             # It seems to be a file or a file-like object
+            self.fobj = self.src
+        else:
+             # Assume it's a string or something that can be converted to one
+            self.fobj = openpy.open(self.fname)
+
+    def reload(self):
+        """Reload source from disk and initialize state."""
+        self.fload()
+
+        self.src     = "".join(openpy.strip_encoding_cookie(self.fobj))
+        src_b        = [b.strip() for b in self.re_stop.split(self.src) if b]
+        self._silent = [bool(self.re_silent.findall(b)) for b in src_b]
+        self._auto   = [bool(self.re_auto.findall(b)) for b in src_b]
+
+        # if auto_all is not given (def. None), we read it from the file
+        if self.auto_all is None:
+            self.auto_all = bool(self.re_auto_all.findall(src_b[0]))
+        else:
+            self.auto_all = bool(self.auto_all)
+
+        # Clean the sources from all markup so it doesn't get displayed when
+        # running the demo
+        src_blocks = []
+        auto_strip = lambda s: self.re_auto.sub('',s)
+        for i,b in enumerate(src_b):
+            if self._auto[i]:
+                src_blocks.append(auto_strip(b))
+            else:
+                src_blocks.append(b)
+        # remove the auto_all marker
+        src_blocks[0] = self.re_auto_all.sub('',src_blocks[0])
+
+        self.nblocks = len(src_blocks)
+        self.src_blocks = src_blocks
+
+        # also build syntax-highlighted source
+        self.src_blocks_colored = list(map(self.highlight,self.src_blocks))
+
+        # ensure clean namespace and seek offset
+        self.reset()
+
+    def reset(self):
+        """Reset the namespace and seek pointer to restart the demo"""
+        self.user_ns     = {}
+        self.finished    = False
+        self.block_index = 0
+
+    def _validate_index(self,index):
+        if index<0 or index>=self.nblocks:
+            raise ValueError('invalid block index %s' % index)
+
+    def _get_index(self,index):
+        """Get the current block index, validating and checking status.
+
+        Returns None if the demo is finished"""
+
+        if index is None:
+            if self.finished:
+                print('Demo finished.  Use <demo_name>.reset() if you want to rerun it.')
+                return None
+            index = self.block_index
+        else:
+            self._validate_index(index)
+        return index
+
+    def seek(self,index):
+        """Move the current seek pointer to the given block.
+
+        You can use negative indices to seek from the end, with identical
+        semantics to those of Python lists."""
+        if index<0:
+            index = self.nblocks + index
+        self._validate_index(index)
+        self.block_index = index
+        self.finished = False
+
+    def back(self,num=1):
+        """Move the seek pointer back num blocks (default is 1)."""
+        self.seek(self.block_index-num)
+
+    def jump(self,num=1):
+        """Jump a given number of blocks relative to the current one.
+
+        The offset can be positive or negative, defaults to 1."""
+        self.seek(self.block_index+num)
+
+    def again(self):
+        """Move the seek pointer back one block and re-execute."""
+        self.back(1)
+        self()
+
+    def edit(self,index=None):
+        """Edit a block.
+
+        If no number is given, use the last block executed.
+
+        This edits the in-memory copy of the demo, it does NOT modify the
+        original source file.  If you want to do that, simply open the file in
+        an editor and use reload() when you make changes to the file.  This
+        method is meant to let you change a block during a demonstration for
+        explanatory purposes, without damaging your original script."""
+
+        index = self._get_index(index)
+        if index is None:
+            return
+        # decrease the index by one (unless we're at the very beginning), so
+        # that the default demo.edit() call opens up the sblock we've last run
+        if index>0:
+            index -= 1
+
+        filename = self.shell.mktempfile(self.src_blocks[index])
+        self.shell.hooks.editor(filename,1)
+        with open(filename, 'r') as f:
+            new_block = f.read()
+        # update the source and colored block
+        self.src_blocks[index] = new_block
+        self.src_blocks_colored[index] = self.highlight(new_block)
+        self.block_index = index
+        # call to run with the newly edited index
+        self()
+
+    def show(self,index=None):
+        """Show a single block on screen"""
+
+        index = self._get_index(index)
+        if index is None:
+            return
+
+        print(self.marquee('<%s> block # %s (%s remaining)' %
+                           (self.title,index,self.nblocks-index-1)))
+        print(self.src_blocks_colored[index])
+        sys.stdout.flush()
+
+    def show_all(self):
+        """Show entire demo on screen, block by block"""
+
+        fname = self.title
+        title = self.title
+        nblocks = self.nblocks
+        silent = self._silent
+        marquee = self.marquee
+        for index,block in enumerate(self.src_blocks_colored):
+            if silent[index]:
+                print(marquee('<%s> SILENT block # %s (%s remaining)' %
+                              (title,index,nblocks-index-1)))
+            else:
+                print(marquee('<%s> block # %s (%s remaining)' %
+                              (title,index,nblocks-index-1)))
+            print(block, end=' ')
+        sys.stdout.flush()
+
+    def run_cell(self,source):
+        """Execute a string with one or more lines of code"""
+
+        exec(source, self.user_ns)
+
+    def __call__(self,index=None):
+        """run a block of the demo.
+
+        If index is given, it should be an integer >=1 and <= nblocks.  This
+        means that the calling convention is one off from typical Python
+        lists.  The reason for the inconsistency is that the demo always
+        prints 'Block n/N, and N is the total, so it would be very odd to use
+        zero-indexing here."""
+
+        index = self._get_index(index)
+        if index is None:
+            return
+        try:
+            marquee = self.marquee
+            next_block = self.src_blocks[index]
+            self.block_index += 1
+            if self._silent[index]:
+                print(marquee('Executing silent block # %s (%s remaining)' %
+                              (index,self.nblocks-index-1)))
+            else:
+                self.pre_cmd()
+                self.show(index)
+                if self.auto_all or self._auto[index]:
+                    print(marquee('output:'))
+                else:
+                    print(marquee('Press <q> to quit, <Enter> to execute...'), end=' ')
+                    ans = py3compat.input().strip()
+                    if ans:
+                        print(marquee('Block NOT executed'))
+                        return
+            try:
+                save_argv = sys.argv
+                sys.argv = self.sys_argv
+                self.run_cell(next_block)
+                self.post_cmd()
+            finally:
+                sys.argv = save_argv
+
+        except:
+            if self.inside_ipython:
+                self.ip_showtb(filename=self.fname)
+        else:
+            if self.inside_ipython:
+                self.ip_ns.update(self.user_ns)
+
+        if self.block_index == self.nblocks:
+            mq1 = self.marquee('END OF DEMO')
+            if mq1:
+                # avoid spurious print if empty marquees are used
+                print()
+                print(mq1)
+                print(self.marquee('Use <demo_name>.reset() if you want to rerun it.'))
+            self.finished = True
+
+    # These methods are meant to be overridden by subclasses who may wish to
+    # customize the behavior of of their demos.
+    def marquee(self,txt='',width=78,mark='*'):
+        """Return the input string centered in a 'marquee'."""
+        return marquee(txt,width,mark)
+
+    def pre_cmd(self):
+        """Method called before executing each block."""
+        pass
+
+    def post_cmd(self):
+        """Method called after executing each block."""
+        pass
+
+    def highlight(self, block):
+        """Method called on each block to highlight it content"""
+        tokens = pygments.lex(block, self.python_lexer)
+        if self.format_rst:
+            from pygments.token import Token
+            toks = []
+            for token in tokens:
+                if token[0] == Token.String.Doc and len(token[1]) > 6:
+                    toks += pygments.lex(token[1][:3], self.python_lexer)
+                    # parse doc string content by rst lexer
+                    toks += pygments.lex(token[1][3:-3], self.rst_lexer)
+                    toks += pygments.lex(token[1][-3:], self.python_lexer)
+                elif token[0] == Token.Comment.Single:
+                    toks.append((Token.Comment.Single, token[1][0]))
+                    # parse comment content by rst lexer
+                    # remove the extrat newline added by rst lexer
+                    toks += list(pygments.lex(token[1][1:], self.rst_lexer))[:-1]
+                else:
+                    toks.append(token)
+            tokens = toks
+        return pygments.format(tokens, self.formatter)
+
+
+class IPythonDemo(Demo):
+    """Class for interactive demos with yap_ipython's input processing applied.
+
+    This subclasses Demo, but instead of executing each block by the Python
+    interpreter (via exec), it actually calls yap_ipython on it, so that any input
+    filters which may be in place are applied to the input block.
+
+    If you have an interactive environment which exposes special input
+    processing, you can use this class instead to write demo scripts which
+    operate exactly as if you had typed them interactively.  The default Demo
+    class requires the input to be valid, pure Python code.
+    """
+
+    def run_cell(self,source):
+        """Execute a string with one or more lines of code"""
+
+        self.shell.run_cell(source)
+
+class LineDemo(Demo):
+    """Demo where each line is executed as a separate block.
+
+    The input script should be valid Python code.
+
+    This class doesn't require any markup at all, and it's meant for simple
+    scripts (with no nesting or any kind of indentation) which consist of
+    multiple lines of input to be executed, one at a time, as if they had been
+    typed in the interactive prompt.
+
+    Note: the input can not have *any* indentation, which means that only
+    single-lines of input are accepted, not even function definitions are
+    valid."""
+
+    def reload(self):
+        """Reload source from disk and initialize state."""
+        # read data and parse into blocks
+        self.fload()
+        lines           = self.fobj.readlines()
+        src_b           = [l for l in lines if l.strip()]
+        nblocks         = len(src_b)
+        self.src        = ''.join(lines)
+        self._silent    = [False]*nblocks
+        self._auto      = [True]*nblocks
+        self.auto_all   = True
+        self.nblocks    = nblocks
+        self.src_blocks = src_b
+
+        # also build syntax-highlighted source
+        self.src_blocks_colored = list(map(self.highlight,self.src_blocks))
+
+        # ensure clean namespace and seek offset
+        self.reset()
+
+
+class IPythonLineDemo(IPythonDemo,LineDemo):
+    """Variant of the LineDemo class whose input is processed by yap_ipython."""
+    pass
+
+
+class ClearMixin(object):
+    """Use this mixin to make Demo classes with less visual clutter.
+
+    Demos using this mixin will clear the screen before every block and use
+    blank marquees.
+
+    Note that in order for the methods defined here to actually override those
+    of the classes it's mixed with, it must go /first/ in the inheritance
+    tree.  For example:
+
+        class ClearIPDemo(ClearMixin,IPythonDemo): pass
+
+    will provide an IPythonDemo class with the mixin's features.
+    """
+
+    def marquee(self,txt='',width=78,mark='*'):
+        """Blank marquee that returns '' no matter what the input."""
+        return ''
+
+    def pre_cmd(self):
+        """Method called before executing each block.
+
+        This one simply clears the screen."""
+        from yap_ipython.utils.terminal import _term_clear
+        _term_clear()
+
+class ClearDemo(ClearMixin,Demo):
+    pass
+
+
+class ClearIPDemo(ClearMixin,IPythonDemo):
+    pass
+
+
+def slide(file_path, noclear=False, format_rst=True, formatter="terminal",
+          style="native", auto_all=False, delimiter='...'):
+    if noclear:
+        demo_class = Demo
+    else:
+        demo_class = ClearDemo
+    demo = demo_class(file_path, format_rst=format_rst, formatter=formatter,
+                      style=style, auto_all=auto_all)
+    while not demo.finished:
+        demo()
+        try:
+            py3compat.input('\n' + delimiter)
+        except KeyboardInterrupt:
+            exit(1)
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description='Run python demos')
+    parser.add_argument('--noclear', '-C', action='store_true',
+                        help='Do not clear terminal on each slide')
+    parser.add_argument('--rst', '-r', action='store_true',
+                        help='Highlight comments and dostrings as rst')
+    parser.add_argument('--formatter', '-f', default='terminal',
+                        help='pygments formatter name could be: terminal, '
+                        'terminal256, terminal16m')
+    parser.add_argument('--style', '-s', default='default',
+                        help='pygments style name')
+    parser.add_argument('--auto', '-a', action='store_true',
+                        help='Run all blocks automatically without'
+                        'confirmation')
+    parser.add_argument('--delimiter', '-d', default='...',
+                        help='slides delimiter added after each slide run')
+    parser.add_argument('file', nargs=1,
+                        help='python demo file')
+    args = parser.parse_args()
+    slide(args.file[0], noclear=args.noclear, format_rst=args.rst,
+          formatter=args.formatter, style=args.style, auto_all=args.auto,
+          delimiter=args.delimiter)

packages/python/yap_kernel/yap_ipython/lib/display.py

@@ -0,0 +1,557 @@
+"""Various display related classes.
+
+Authors : MinRK, gregcaporaso, dannystaple
+"""
+from os.path import exists, isfile, splitext, abspath, join, isdir
+from os import walk, sep
+
+from yap_ipython.core.display import DisplayObject
+
+__all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument',
+           'FileLink', 'FileLinks']
+
+
+class Audio(DisplayObject):
+    """Create an audio object.
+
+    When this object is returned by an input cell or passed to the
+    display function, it will result in Audio controls being displayed
+    in the frontend (only works in the notebook).
+
+    Parameters
+    ----------
+    data : numpy array, list, unicode, str or bytes
+        Can be one of
+
+          * Numpy 1d array containing the desired waveform (mono)
+          * Numpy 2d array containing waveforms for each channel.
+            Shape=(NCHAN, NSAMPLES). For the standard channel order, see
+            http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
+          * List of float or integer representing the waveform (mono)
+          * String containing the filename
+          * Bytestring containing raw PCM data or
+          * URL pointing to a file on the web.
+
+        If the array option is used the waveform will be normalized.
+
+        If a filename or url is used the format support will be browser
+        dependent.
+    url : unicode
+        A URL to download the data from.
+    filename : unicode
+        Path to a local file to load the data from.
+    embed : boolean
+        Should the audio data be embedded using a data URI (True) or should
+        the original source be referenced. Set this to True if you want the
+        audio to playable later with no internet connection in the notebook.
+
+        Default is `True`, unless the keyword argument `url` is set, then
+        default value is `False`.
+    rate : integer
+        The sampling rate of the raw data.
+        Only required when data parameter is being used as an array
+    autoplay : bool
+        Set to True if the audio should immediately start playing.
+        Default is `False`.
+
+    Examples
+    --------
+    ::
+
+        # Generate a sound
+        import numpy as np
+        framerate = 44100
+        t = np.linspace(0,5,framerate*5)
+        data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t))
+        Audio(data,rate=framerate)
+
+        # Can also do stereo or more channels
+        dataleft = np.sin(2*np.pi*220*t)
+        dataright = np.sin(2*np.pi*224*t)
+        Audio([dataleft, dataright],rate=framerate)
+
+        Audio("http://www.nch.com.au/acm/8k16bitpcm.wav")  # From URL
+        Audio(url="http://www.w3schools.com/html/horse.ogg")
+
+        Audio('/path/to/sound.wav')  # From file
+        Audio(filename='/path/to/sound.ogg')
+
+        Audio(b'RAW_WAV_DATA..)  # From bytes
+        Audio(data=b'RAW_WAV_DATA..)
+
+    """
+    _read_flags = 'rb'
+
+    def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False):
+        if filename is None and url is None and data is None:
+            raise ValueError("No image data found. Expecting filename, url, or data.")
+        if embed is False and url is None:
+            raise ValueError("No url found. Expecting url when embed=False")
+
+        if url is not None and embed is not True:
+            self.embed = False
+        else:
+            self.embed = True
+        self.autoplay = autoplay
+        super(Audio, self).__init__(data=data, url=url, filename=filename)
+
+        if self.data is not None and not isinstance(self.data, bytes):
+            self.data = self._make_wav(data,rate)
+
+    def reload(self):
+        """Reload the raw data from file or URL."""
+        import mimetypes
+        if self.embed:
+            super(Audio, self).reload()
+
+        if self.filename is not None:
+            self.mimetype = mimetypes.guess_type(self.filename)[0]
+        elif self.url is not None:
+            self.mimetype = mimetypes.guess_type(self.url)[0]
+        else:
+            self.mimetype = "audio/wav"
+
+    def _make_wav(self, data, rate):
+        """ Transform a numpy array to a PCM bytestring """
+        import struct
+        from io import BytesIO
+        import wave
+
+        try:
+            import numpy as np
+
+            data = np.array(data, dtype=float)
+            if len(data.shape) == 1:
+                nchan = 1
+            elif len(data.shape) == 2:
+                # In wave files,channels are interleaved. E.g.,
+                # "L1R1L2R2..." for stereo. See
+                # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
+                # for channel ordering
+                nchan = data.shape[0]
+                data = data.T.ravel()
+            else:
+                raise ValueError('Array audio input must be a 1D or 2D array')
+            scaled = np.int16(data/np.max(np.abs(data))*32767).tolist()
+        except ImportError:
+            # check that it is a "1D" list
+            idata = iter(data)  # fails if not an iterable
+            try:
+                iter(idata.next())
+                raise TypeError('Only lists of mono audio are '
+                    'supported if numpy is not installed')
+            except TypeError:
+                # this means it's not a nested list, which is what we want
+                pass
+            maxabsvalue = float(max([abs(x) for x in data]))
+            scaled = [int(x/maxabsvalue*32767) for x in data]
+            nchan = 1
+
+        fp = BytesIO()
+        waveobj = wave.open(fp,mode='wb')
+        waveobj.setnchannels(nchan)
+        waveobj.setframerate(rate)
+        waveobj.setsampwidth(2)
+        waveobj.setcomptype('NONE','NONE')
+        waveobj.writeframes(b''.join([struct.pack('<h',x) for x in scaled]))
+        val = fp.getvalue()
+        waveobj.close()
+
+        return val
+
+    def _data_and_metadata(self):
+        """shortcut for returning metadata with url information, if defined"""
+        md = {}
+        if self.url:
+            md['url'] = self.url
+        if md:
+            return self.data, md
+        else:
+            return self.data
+
+    def _repr_html_(self):
+        src = """
+                <audio controls="controls" {autoplay}>
+                    <source src="{src}" type="{type}" />
+                    Your browser does not support the audio element.
+                </audio>
+              """
+        return src.format(src=self.src_attr(),type=self.mimetype, autoplay=self.autoplay_attr())
+
+    def src_attr(self):
+        import base64
+        if self.embed and (self.data is not None):
+            data = base64=base64.b64encode(self.data).decode('ascii')
+            return """data:{type};base64,{base64}""".format(type=self.mimetype,
+                                                            base64=data)
+        elif self.url is not None:
+            return self.url
+        else:
+            return ""
+
+    def autoplay_attr(self):
+        if(self.autoplay):
+            return 'autoplay="autoplay"'
+        else:
+            return ''
+
+class IFrame(object):
+    """
+    Generic class to embed an iframe in an yap_ipython notebook
+    """
+
+    iframe = """
+        <iframe
+            width="{width}"
+            height="{height}"
+            src="{src}{params}"
+            frameborder="0"
+            allowfullscreen
+        ></iframe>
+        """
+
+    def __init__(self, src, width, height, **kwargs):
+        self.src = src
+        self.width = width
+        self.height = height
+        self.params = kwargs
+
+    def _repr_html_(self):
+        """return the embed iframe"""
+        if self.params:
+            try:
+                from urllib.parse import urlencode # Py 3
+            except ImportError:
+                from urllib import urlencode
+            params = "?" + urlencode(self.params)
+        else:
+            params = ""
+        return self.iframe.format(src=self.src,
+                                  width=self.width,
+                                  height=self.height,
+                                  params=params)
+
+class YouTubeVideo(IFrame):
+    """Class for embedding a YouTube Video in an yap_ipython session, based on its video id.
+
+    e.g. to embed the video from https://www.youtube.com/watch?v=foo , you would
+    do::
+
+        vid = YouTubeVideo("foo")
+        display(vid)
+
+    To start from 30 seconds::
+
+        vid = YouTubeVideo("abc", start=30)
+        display(vid)
+
+    To calculate seconds from time as hours, minutes, seconds use
+    :class:`datetime.timedelta`::
+
+        start=int(timedelta(hours=1, minutes=46, seconds=40).total_seconds())
+
+    Other parameters can be provided as documented at
+    https://developers.google.com/youtube/player_parameters#Parameters
+    
+    When converting the notebook using nbconvert, a jpeg representation of the video
+    will be inserted in the document.
+    """
+
+    def __init__(self, id, width=400, height=300, **kwargs):
+        self.id=id
+        src = "https://www.youtube.com/embed/{0}".format(id)
+        super(YouTubeVideo, self).__init__(src, width, height, **kwargs)
+    
+    def _repr_jpeg_(self):
+        # Deferred import
+        from urllib.request import urlopen
+
+        try:
+            return urlopen("https://img.youtube.com/vi/{id}/hqdefault.jpg".format(id=self.id)).read()
+        except IOError:
+            return None
+
+class VimeoVideo(IFrame):
+    """
+    Class for embedding a Vimeo video in an yap_ipython session, based on its video id.
+    """
+
+    def __init__(self, id, width=400, height=300, **kwargs):
+        src="https://player.vimeo.com/video/{0}".format(id)
+        super(VimeoVideo, self).__init__(src, width, height, **kwargs)
+
+class ScribdDocument(IFrame):
+    """
+    Class for embedding a Scribd document in an yap_ipython session
+
+    Use the start_page params to specify a starting point in the document
+    Use the view_mode params to specify display type one off scroll | slideshow | book
+
+    e.g to Display Wes' foundational paper about PANDAS in book mode from page 3
+
+    ScribdDocument(71048089, width=800, height=400, start_page=3, view_mode="book")
+    """
+
+    def __init__(self, id, width=400, height=300, **kwargs):
+        src="https://www.scribd.com/embeds/{0}/content".format(id)
+        super(ScribdDocument, self).__init__(src, width, height, **kwargs)
+
+class FileLink(object):
+    """Class for embedding a local file link in an yap_ipython session, based on path
+
+    e.g. to embed a link that was generated in the yap_ipython notebook as my/data.txt
+
+    you would do::
+
+        local_file = FileLink("my/data.txt")
+        display(local_file)
+
+    or in the HTML notebook, just::
+
+        FileLink("my/data.txt")
+    """
+
+    html_link_str = "<a href='%s' target='_blank'>%s</a>"
+
+    def __init__(self,
+                 path,
+                 url_prefix='',
+                 result_html_prefix='',
+                 result_html_suffix='<br>'):
+        """
+        Parameters
+        ----------
+        path : str
+            path to the file or directory that should be formatted
+        url_prefix : str
+            prefix to be prepended to all files to form a working link [default:
+            '']
+        result_html_prefix : str
+            text to append to beginning to link [default: '']
+        result_html_suffix : str
+            text to append at the end of link [default: '<br>']
+        """
+        if isdir(path):
+            raise ValueError("Cannot display a directory using FileLink. "
+              "Use FileLinks to display '%s'." % path)
+        self.path = path
+        self.url_prefix = url_prefix
+        self.result_html_prefix = result_html_prefix
+        self.result_html_suffix = result_html_suffix
+
+    def _format_path(self):
+        fp = ''.join([self.url_prefix,self.path])
+        return ''.join([self.result_html_prefix,
+                        self.html_link_str % (fp, self.path),
+                        self.result_html_suffix])
+
+    def _repr_html_(self):
+        """return html link to file
+        """
+        if not exists(self.path):
+            return ("Path (<tt>%s</tt>) doesn't exist. "
+                    "It may still be in the process of "
+                    "being generated, or you may have the "
+                    "incorrect path." % self.path)
+
+        return self._format_path()
+
+    def __repr__(self):
+        """return absolute path to file
+        """
+        return abspath(self.path)
+
+class FileLinks(FileLink):
+    """Class for embedding local file links in an yap_ipython session, based on path
+
+    e.g. to embed links to files that were generated in the yap_ipython notebook
+    under ``my/data``, you would do::
+
+        local_files = FileLinks("my/data")
+        display(local_files)
+
+    or in the HTML notebook, just::
+
+        FileLinks("my/data")
+    """
+    def __init__(self,
+                 path,
+                 url_prefix='',
+                 included_suffixes=None,
+                 result_html_prefix='',
+                 result_html_suffix='<br>',
+                 notebook_display_formatter=None,
+                 terminal_display_formatter=None,
+                 recursive=True):
+        """
+        See :class:`FileLink` for the ``path``, ``url_prefix``,
+        ``result_html_prefix`` and ``result_html_suffix`` parameters.
+
+        included_suffixes : list
+          Filename suffixes to include when formatting output [default: include
+          all files]
+
+        notebook_display_formatter : function
+          Used to format links for display in the notebook. See discussion of
+          formatter functions below.
+
+        terminal_display_formatter : function
+          Used to format links for display in the terminal. See discussion of
+          formatter functions below.
+
+        Formatter functions must be of the form::
+
+            f(dirname, fnames, included_suffixes)
+
+        dirname : str
+          The name of a directory
+        fnames : list
+          The files in that directory
+        included_suffixes : list
+          The file suffixes that should be included in the output (passing None
+          meansto include all suffixes in the output in the built-in formatters)
+        recursive : boolean
+          Whether to recurse into subdirectories. Default is True.
+
+        The function should return a list of lines that will be printed in the
+        notebook (if passing notebook_display_formatter) or the terminal (if
+        passing terminal_display_formatter). This function is iterated over for
+        each directory in self.path. Default formatters are in place, can be
+        passed here to support alternative formatting.
+
+        """
+        if isfile(path):
+            raise ValueError("Cannot display a file using FileLinks. "
+              "Use FileLink to display '%s'." % path)
+        self.included_suffixes = included_suffixes
+        # remove trailing slashs for more consistent output formatting
+        path = path.rstrip('/')
+
+        self.path = path
+        self.url_prefix = url_prefix
+        self.result_html_prefix = result_html_prefix
+        self.result_html_suffix = result_html_suffix
+
+        self.notebook_display_formatter = \
+             notebook_display_formatter or self._get_notebook_display_formatter()
+        self.terminal_display_formatter = \
+             terminal_display_formatter or self._get_terminal_display_formatter()
+
+        self.recursive = recursive
+
+    def _get_display_formatter(self,
+                               dirname_output_format,
+                               fname_output_format,
+                               fp_format,
+                               fp_cleaner=None):
+        """ generate built-in formatter function
+
+           this is used to define both the notebook and terminal built-in
+            formatters as they only differ by some wrapper text for each entry
+
+           dirname_output_format: string to use for formatting directory
+            names, dirname will be substituted for a single "%s" which
+            must appear in this string
+           fname_output_format: string to use for formatting file names,
+            if a single "%s" appears in the string, fname will be substituted
+            if two "%s" appear in the string, the path to fname will be
+             substituted for the first and fname will be substituted for the
+             second
+           fp_format: string to use for formatting filepaths, must contain
+            exactly two "%s" and the dirname will be subsituted for the first
+            and fname will be substituted for the second
+        """
+        def f(dirname, fnames, included_suffixes=None):
+            result = []
+            # begin by figuring out which filenames, if any,
+            # are going to be displayed
+            display_fnames = []
+            for fname in fnames:
+                if (isfile(join(dirname,fname)) and
+                       (included_suffixes is None or
+                        splitext(fname)[1] in included_suffixes)):
+                      display_fnames.append(fname)
+
+            if len(display_fnames) == 0:
+                # if there are no filenames to display, don't print anything
+                # (not even the directory name)
+                pass
+            else:
+                # otherwise print the formatted directory name followed by
+                # the formatted filenames
+                dirname_output_line = dirname_output_format % dirname
+                result.append(dirname_output_line)
+                for fname in display_fnames:
+                    fp = fp_format % (dirname,fname)
+                    if fp_cleaner is not None:
+                        fp = fp_cleaner(fp)
+                    try:
+                        # output can include both a filepath and a filename...
+                        fname_output_line = fname_output_format % (fp, fname)
+                    except TypeError:
+                        # ... or just a single filepath
+                        fname_output_line = fname_output_format % fname
+                    result.append(fname_output_line)
+            return result
+        return f
+
+    def _get_notebook_display_formatter(self,
+                                        spacer="&nbsp;&nbsp;"):
+        """ generate function to use for notebook formatting
+        """
+        dirname_output_format = \
+         self.result_html_prefix + "%s/" + self.result_html_suffix
+        fname_output_format = \
+         self.result_html_prefix + spacer + self.html_link_str + self.result_html_suffix
+        fp_format = self.url_prefix + '%s/%s'
+        if sep == "\\":
+            # Working on a platform where the path separator is "\", so
+            # must convert these to "/" for generating a URI
+            def fp_cleaner(fp):
+                # Replace all occurrences of backslash ("\") with a forward
+                # slash ("/") - this is necessary on windows when a path is
+                # provided as input, but we must link to a URI
+                return fp.replace('\\','/')
+        else:
+            fp_cleaner = None
+
+        return self._get_display_formatter(dirname_output_format,
+                                           fname_output_format,
+                                           fp_format,
+                                           fp_cleaner)
+
+    def _get_terminal_display_formatter(self,
+                                        spacer="  "):
+        """ generate function to use for terminal formatting
+        """
+        dirname_output_format = "%s/"
+        fname_output_format = spacer + "%s"
+        fp_format = '%s/%s'
+
+        return self._get_display_formatter(dirname_output_format,
+                                           fname_output_format,
+                                           fp_format)
+
+    def _format_path(self):
+        result_lines = []
+        if self.recursive:
+            walked_dir = list(walk(self.path))
+        else:
+            walked_dir = [next(walk(self.path))]
+        walked_dir.sort()
+        for dirname, subdirs, fnames in walked_dir:
+            result_lines += self.notebook_display_formatter(dirname, fnames, self.included_suffixes)
+        return '\n'.join(result_lines)
+
+    def __repr__(self):
+        """return newline-separated absolute paths
+        """
+        result_lines = []
+        if self.recursive:
+            walked_dir = list(walk(self.path))
+        else:
+            walked_dir = [next(walk(self.path))]
+        walked_dir.sort()
+        for dirname, subdirs, fnames in walked_dir:
+            result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes)
+        return '\n'.join(result_lines)

packages/python/yap_kernel/yap_ipython/lib/editorhooks.py

@@ -0,0 +1,128 @@
+""" 'editor' hooks for common editors that work well with ipython
+
+They should honor the line number argument, at least.
+
+Contributions are *very* welcome.
+"""
+
+import os
+import pipes
+import shlex
+import subprocess
+import sys
+
+from yap_ipython import get_ipython
+from yap_ipython.core.error import TryNext
+from yap_ipython.utils import py3compat
+
+
+def install_editor(template, wait=False):
+    """Installs the editor that is called by yap_ipython for the %edit magic.
+
+    This overrides the default editor, which is generally set by your EDITOR
+    environment variable or is notepad (windows) or vi (linux). By supplying a
+    template string `run_template`, you can control how the editor is invoked
+    by yap_ipython -- (e.g. the format in which it accepts command line options)
+
+    Parameters
+    ----------
+    template : basestring
+        run_template acts as a template for how your editor is invoked by
+        the shell. It should contain '{filename}', which will be replaced on
+        invokation with the file name, and '{line}', $line by line number
+        (or 0) to invoke the file with.
+    wait : bool
+        If `wait` is true, wait until the user presses enter before returning,
+        to facilitate non-blocking editors that exit immediately after
+        the call.
+    """
+
+    # not all editors support $line, so we'll leave out this check
+    # for substitution in ['$file', '$line']:
+    #    if not substitution in run_template:
+    #        raise ValueError(('run_template should contain %s'
+    #        ' for string substitution. You supplied "%s"' % (substitution,
+    #            run_template)))
+
+    def call_editor(self, filename, line=0):
+        if line is None:
+            line = 0
+        cmd = template.format(filename=pipes.quote(filename), line=line)
+        print(">", cmd)
+        # pipes.quote doesn't work right on Windows, but it does after splitting
+        if sys.platform.startswith('win'):
+            cmd = shlex.split(cmd)
+        proc = subprocess.Popen(cmd, shell=True)
+        if proc.wait() != 0:
+            raise TryNext()
+        if wait:
+            py3compat.input("Press Enter when done editing:")
+
+    get_ipython().set_hook('editor', call_editor)
+    get_ipython().editor = template
+
+
+# in these, exe is always the path/name of the executable. Useful
+# if you don't have the editor directory in your path
+def komodo(exe=u'komodo'):
+    """ Activestate Komodo [Edit] """
+    install_editor(exe + u' -l {line} {filename}', wait=True)
+
+
+def scite(exe=u"scite"):
+    """ SciTE or Sc1 """
+    install_editor(exe + u' {filename} -goto:{line}')
+
+
+def notepadplusplus(exe=u'notepad++'):
+    """ Notepad++ http://notepad-plus.sourceforge.net """
+    install_editor(exe + u' -n{line} {filename}')
+
+
+def jed(exe=u'jed'):
+    """ JED, the lightweight emacsish editor """
+    install_editor(exe + u' +{line} {filename}')
+
+
+def idle(exe=u'idle'):
+    """ Idle, the editor bundled with python
+
+    Parameters
+    ----------
+    exe : str, None
+        If none, should be pretty smart about finding the executable.
+    """
+    if exe is None:
+        import idlelib
+        p = os.path.dirname(idlelib.__filename__)
+        # i'm not sure if this actually works. Is this idle.py script
+        # guaranteed to be executable?
+        exe = os.path.join(p, 'idle.py')
+    install_editor(exe + u' {filename}')
+
+
+def mate(exe=u'mate'):
+    """ TextMate, the missing editor"""
+    # wait=True is not required since we're using the -w flag to mate
+    install_editor(exe + u' -w -l {line} {filename}')
+
+
+# ##########################################
+# these are untested, report any problems
+# ##########################################
+
+
+def emacs(exe=u'emacs'):
+    install_editor(exe + u' +{line} {filename}')
+
+
+def gnuclient(exe=u'gnuclient'):
+    install_editor(exe + u' -nw +{line} {filename}')
+
+
+def crimson_editor(exe=u'cedt.exe'):
+    install_editor(exe + u' /L:{line} {filename}')
+
+
+def kate(exe=u'kate'):
+    install_editor(exe + u' -u -l {line} {filename}')

packages/python/yap_kernel/yap_ipython/lib/guisupport.py

@@ -0,0 +1,155 @@
+# coding: utf-8
+"""
+Support for creating GUI apps and starting event loops.
+
+yap_ipython's GUI integration allows interative plotting and GUI usage in yap_ipython
+session. yap_ipython has two different types of GUI integration:
+
+1. The terminal based yap_ipython supports GUI event loops through Python's
+   PyOS_InputHook. PyOS_InputHook is a hook that Python calls periodically
+   whenever raw_input is waiting for a user to type code. We implement GUI
+   support in the terminal by setting PyOS_InputHook to a function that
+   iterates the event loop for a short while. It is important to note that
+   in this situation, the real GUI event loop is NOT run in the normal
+   manner, so you can't use the normal means to detect that it is running.
+2. In the two process yap_ipython kernel/frontend, the GUI event loop is run in
+   the kernel. In this case, the event loop is run in the normal manner by
+   calling the function or method of the GUI toolkit that starts the event
+   loop.
+
+In addition to starting the GUI event loops in one of these two ways, yap_ipython
+will *always* create an appropriate GUI application object when GUi
+integration is enabled.
+
+If you want your GUI apps to run in yap_ipython you need to do two things:
+
+1. Test to see if there is already an existing main application object. If
+   there is, you should use it. If there is not an existing application object
+   you should create one.
+2. Test to see if the GUI event loop is running. If it is, you should not
+   start it. If the event loop is not running you may start it.
+
+This module contains functions for each toolkit that perform these things
+in a consistent manner. Because of how PyOS_InputHook runs the event loop
+you cannot detect if the event loop is running using the traditional calls
+(such as ``wx.GetApp.IsMainLoopRunning()`` in wxPython). If PyOS_InputHook is
+set These methods will return a false negative. That is, they will say the
+event loop is not running, when is actually is. To work around this limitation
+we proposed the following informal protocol:
+
+* Whenever someone starts the event loop, they *must* set the ``_in_event_loop``
+  attribute of the main application object to ``True``. This should be done
+  regardless of how the event loop is actually run.
+* Whenever someone stops the event loop, they *must* set the ``_in_event_loop``
+  attribute of the main application object to ``False``.
+* If you want to see if the event loop is running, you *must* use ``hasattr``
+  to see if ``_in_event_loop`` attribute has been set. If it is set, you
+  *must* use its value. If it has not been set, you can query the toolkit
+  in the normal manner.
+* If you want GUI support and no one else has created an application or
+  started the event loop you *must* do this. We don't want projects to
+  attempt to defer these things to someone else if they themselves need it.
+
+The functions below implement this logic for each GUI toolkit. If you need
+to create custom application subclasses, you will likely have to modify this
+code for your own purposes. This code can be copied into your own project
+so you don't have to depend on yap_ipython.
+
+"""
+
+# Copyright (c) yap_ipython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from yap_ipython.core.getipython import get_ipython
+
+#-----------------------------------------------------------------------------
+# wx
+#-----------------------------------------------------------------------------
+
+def get_app_wx(*args, **kwargs):
+    """Create a new wx app or return an exiting one."""
+    import wx
+    app = wx.GetApp()
+    if app is None:
+        if 'redirect' not in kwargs:
+            kwargs['redirect'] = False
+        app = wx.PySimpleApp(*args, **kwargs)
+    return app
+
+def is_event_loop_running_wx(app=None):
+    """Is the wx event loop running."""
+    # New way: check attribute on shell instance
+    ip = get_ipython()
+    if ip is not None:
+        if ip.active_eventloop and ip.active_eventloop == 'wx':
+            return True
+        # Fall through to checking the application, because Wx has a native way
+        # to check if the event loop is running, unlike Qt.
+
+    # Old way: check Wx application
+    if app is None:
+        app = get_app_wx()
+    if hasattr(app, '_in_event_loop'):
+        return app._in_event_loop
+    else:
+        return app.IsMainLoopRunning()
+
+def start_event_loop_wx(app=None):
+    """Start the wx event loop in a consistent manner."""
+    if app is None:
+        app = get_app_wx()
+    if not is_event_loop_running_wx(app):
+        app._in_event_loop = True
+        app.MainLoop()
+        app._in_event_loop = False
+    else:
+        app._in_event_loop = True
+
+#-----------------------------------------------------------------------------
+# qt4
+#-----------------------------------------------------------------------------
+
+def get_app_qt4(*args, **kwargs):
+    """Create a new qt4 app or return an existing one."""
+    from yap_ipython.external.qt_for_kernel import QtGui
+    app = QtGui.QApplication.instance()
+    if app is None:
+        if not args:
+            args = ([''],)
+        app = QtGui.QApplication(*args, **kwargs)
+    return app
+
+def is_event_loop_running_qt4(app=None):
+    """Is the qt4 event loop running."""
+    # New way: check attribute on shell instance
+    ip = get_ipython()
+    if ip is not None:
+        return ip.active_eventloop and ip.active_eventloop.startswith('qt')
+
+    # Old way: check attribute on QApplication singleton
+    if app is None:
+        app = get_app_qt4([''])
+    if hasattr(app, '_in_event_loop'):
+        return app._in_event_loop
+    else:
+        # Does qt4 provide a other way to detect this?
+        return False
+
+def start_event_loop_qt4(app=None):
+    """Start the qt4 event loop in a consistent manner."""
+    if app is None:
+        app = get_app_qt4([''])
+    if not is_event_loop_running_qt4(app):
+        app._in_event_loop = True
+        app.exec_()
+        app._in_event_loop = False
+    else:
+        app._in_event_loop = True
+
+#-----------------------------------------------------------------------------
+# Tk
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# gtk
+#-----------------------------------------------------------------------------

packages/python/yap_kernel/yap_ipython/lib/inputhook.py

@@ -0,0 +1,666 @@
+# coding: utf-8
+"""
+Deprecated since yap_ipython 5.0
+
+Inputhook management for GUI event loop integration.
+"""
+
+# Copyright (c) yap_ipython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+try:
+    import ctypes
+except ImportError:
+    ctypes = None
+except SystemError: # IronPython issue, 2/8/2014
+    ctypes = None
+import os
+import platform
+import sys
+from distutils.version import LooseVersion as V
+
+from warnings import warn
+
+
+warn("`yap_ipython.lib.inputhook` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+        DeprecationWarning, stacklevel=2)
+
+
+#-----------------------------------------------------------------------------
+# Constants
+#-----------------------------------------------------------------------------
+
+# Constants for identifying the GUI toolkits.
+GUI_WX = 'wx'
+GUI_QT = 'qt'
+GUI_QT4 = 'qt4'
+GUI_GTK = 'gtk'
+GUI_TK = 'tk'
+GUI_OSX = 'osx'
+GUI_GLUT = 'glut'
+GUI_PYGLET = 'pyglet'
+GUI_GTK3 = 'gtk3'
+GUI_NONE = 'none' # i.e. disable
+
+#-----------------------------------------------------------------------------
+# Utilities
+#-----------------------------------------------------------------------------
+
+def _stdin_ready_posix():
+    """Return True if there's something to read on stdin (posix version)."""
+    infds, outfds, erfds = select.select([sys.stdin],[],[],0)
+    return bool(infds)
+
+def _stdin_ready_nt():
+    """Return True if there's something to read on stdin (nt version)."""
+    return msvcrt.kbhit()
+
+def _stdin_ready_other():
+    """Return True, assuming there's something to read on stdin."""
+    return True
+
+def _use_appnope():
+    """Should we use appnope for dealing with OS X app nap?
+
+    Checks if we are on OS X 10.9 or greater.
+    """
+    return sys.platform == 'darwin' and V(platform.mac_ver()[0]) >= V('10.9')
+
+def _ignore_CTRL_C_posix():
+    """Ignore CTRL+C (SIGINT)."""
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+def _allow_CTRL_C_posix():
+    """Take CTRL+C into account (SIGINT)."""
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+
+def _ignore_CTRL_C_other():
+    """Ignore CTRL+C (not implemented)."""
+    pass
+
+def _allow_CTRL_C_other():
+    """Take CTRL+C into account (not implemented)."""
+    pass
+
+if os.name == 'posix':
+    import select
+    import signal
+    stdin_ready = _stdin_ready_posix
+    ignore_CTRL_C = _ignore_CTRL_C_posix
+    allow_CTRL_C = _allow_CTRL_C_posix
+elif os.name == 'nt':
+    import msvcrt
+    stdin_ready = _stdin_ready_nt
+    ignore_CTRL_C = _ignore_CTRL_C_other
+    allow_CTRL_C = _allow_CTRL_C_other
+else:
+    stdin_ready = _stdin_ready_other
+    ignore_CTRL_C = _ignore_CTRL_C_other
+    allow_CTRL_C = _allow_CTRL_C_other
+
+
+#-----------------------------------------------------------------------------
+# Main InputHookManager class
+#-----------------------------------------------------------------------------
+
+
+class InputHookManager(object):
+    """DEPRECATED since yap_ipython 5.0
+
+    Manage PyOS_InputHook for different GUI toolkits.
+
+    This class installs various hooks under ``PyOSInputHook`` to handle
+    GUI event loop integration.
+    """
+    
+    def __init__(self):
+        if ctypes is None:
+            warn("yap_ipython GUI event loop requires ctypes, %gui will not be available")
+        else:
+            self.PYFUNC = ctypes.PYFUNCTYPE(ctypes.c_int)
+        self.guihooks = {}
+        self.aliases = {}
+        self.apps = {}
+        self._reset()
+
+    def _reset(self):
+        self._callback_pyfunctype = None
+        self._callback = None
+        self._installed = False
+        self._current_gui = None
+
+    def get_pyos_inputhook(self):
+        """DEPRECATED since yap_ipython 5.0
+
+        Return the current PyOS_InputHook as a ctypes.c_void_p."""
+        warn("`get_pyos_inputhook` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        return ctypes.c_void_p.in_dll(ctypes.pythonapi,"PyOS_InputHook")
+
+    def get_pyos_inputhook_as_func(self):
+        """DEPRECATED since yap_ipython 5.0
+
+        Return the current PyOS_InputHook as a ctypes.PYFUNCYPE."""
+        warn("`get_pyos_inputhook_as_func` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        return self.PYFUNC.in_dll(ctypes.pythonapi,"PyOS_InputHook")
+
+    def set_inputhook(self, callback):
+        """DEPRECATED since yap_ipython 5.0
+
+        Set PyOS_InputHook to callback and return the previous one."""
+        # On platforms with 'readline' support, it's all too likely to
+        # have a KeyboardInterrupt signal delivered *even before* an
+        # initial ``try:`` clause in the callback can be executed, so
+        # we need to disable CTRL+C in this situation.
+        ignore_CTRL_C()
+        self._callback = callback
+        self._callback_pyfunctype = self.PYFUNC(callback)
+        pyos_inputhook_ptr = self.get_pyos_inputhook()
+        original = self.get_pyos_inputhook_as_func()
+        pyos_inputhook_ptr.value = \
+            ctypes.cast(self._callback_pyfunctype, ctypes.c_void_p).value
+        self._installed = True
+        return original
+
+    def clear_inputhook(self, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Set PyOS_InputHook to NULL and return the previous one.
+
+        Parameters
+        ----------
+        app : optional, ignored
+          This parameter is allowed only so that clear_inputhook() can be
+          called with a similar interface as all the ``enable_*`` methods.  But
+          the actual value of the parameter is ignored.  This uniform interface
+          makes it easier to have user-level entry points in the main yap_ipython
+          app like :meth:`enable_gui`."""
+        warn("`clear_inputhook` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        pyos_inputhook_ptr = self.get_pyos_inputhook()
+        original = self.get_pyos_inputhook_as_func()
+        pyos_inputhook_ptr.value = ctypes.c_void_p(None).value
+        allow_CTRL_C()
+        self._reset()
+        return original
+
+    def clear_app_refs(self, gui=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Clear yap_ipython's internal reference to an application instance.
+
+        Whenever we create an app for a user on qt4 or wx, we hold a
+        reference to the app.  This is needed because in some cases bad things
+        can happen if a user doesn't hold a reference themselves.  This
+        method is provided to clear the references we are holding.
+
+        Parameters
+        ----------
+        gui : None or str
+            If None, clear all app references.  If ('wx', 'qt4') clear
+            the app for that toolkit.  References are not held for gtk or tk
+            as those toolkits don't have the notion of an app.
+        """
+        warn("`clear_app_refs` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        if gui is None:
+            self.apps = {}
+        elif gui in self.apps:
+            del self.apps[gui]
+
+    def register(self, toolkitname, *aliases):
+        """DEPRECATED since yap_ipython 5.0
+
+        Register a class to provide the event loop for a given GUI.
+        
+        This is intended to be used as a class decorator. It should be passed
+        the names with which to register this GUI integration. The classes
+        themselves should subclass :class:`InputHookBase`.
+        
+        ::
+        
+            @inputhook_manager.register('qt')
+            class QtInputHook(InputHookBase):
+                def enable(self, app=None):
+                    ...
+        """
+        warn("`register` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        def decorator(cls):
+            if ctypes is not None:
+                inst = cls(self)
+                self.guihooks[toolkitname] = inst
+                for a in aliases:
+                    self.aliases[a] = toolkitname
+            return cls
+        return decorator
+
+    def current_gui(self):
+        """DEPRECATED since yap_ipython 5.0
+
+        Return a string indicating the currently active GUI or None."""
+        warn("`current_gui` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        return self._current_gui
+
+    def enable_gui(self, gui=None, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Switch amongst GUI input hooks by name.
+
+        This is a higher level method than :meth:`set_inputhook` - it uses the
+        GUI name to look up a registered object which enables the input hook
+        for that GUI.
+
+        Parameters
+        ----------
+        gui : optional, string or None
+          If None (or 'none'), clears input hook, otherwise it must be one
+          of the recognized GUI names (see ``GUI_*`` constants in module).
+
+        app : optional, existing application object.
+          For toolkits that have the concept of a global app, you can supply an
+          existing one.  If not given, the toolkit will be probed for one, and if
+          none is found, a new one will be created.  Note that GTK does not have
+          this concept, and passing an app if ``gui=="GTK"`` will raise an error.
+
+        Returns
+        -------
+        The output of the underlying gui switch routine, typically the actual
+        PyOS_InputHook wrapper object or the GUI toolkit app created, if there was
+        one.
+        """
+        warn("`enable_gui` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        if gui in (None, GUI_NONE):
+            return self.disable_gui()
+        
+        if gui in self.aliases:
+            return self.enable_gui(self.aliases[gui], app)
+        
+        try:
+            gui_hook = self.guihooks[gui]
+        except KeyError:
+            e = "Invalid GUI request {!r}, valid ones are: {}"
+            raise ValueError(e.format(gui, ', '.join(self.guihooks)))
+        self._current_gui = gui
+
+        app = gui_hook.enable(app)
+        if app is not None:
+            app._in_event_loop = True
+            self.apps[gui] = app
+        return app
+
+    def disable_gui(self):
+        """DEPRECATED since yap_ipython 5.0
+
+        Disable GUI event loop integration.
+        
+        If an application was registered, this sets its ``_in_event_loop``
+        attribute to False. It then calls :meth:`clear_inputhook`.
+        """
+        warn("`disable_gui` is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        gui = self._current_gui
+        if gui in self.apps:
+            self.apps[gui]._in_event_loop = False
+        return self.clear_inputhook()
+
+class InputHookBase(object):
+    """DEPRECATED since yap_ipython 5.0
+
+    Base class for input hooks for specific toolkits.
+    
+    Subclasses should define an :meth:`enable` method with one argument, ``app``,
+    which will either be an instance of the toolkit's application class, or None.
+    They may also define a :meth:`disable` method with no arguments.
+    """
+    def __init__(self, manager):
+        self.manager = manager
+
+    def disable(self):
+        pass
+
+inputhook_manager = InputHookManager()
+
+@inputhook_manager.register('osx')
+class NullInputHook(InputHookBase):
+    """DEPRECATED since yap_ipython 5.0
+
+    A null inputhook that doesn't need to do anything"""
+    def enable(self, app=None):
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+
+@inputhook_manager.register('wx')
+class WxInputHook(InputHookBase):
+    def enable(self, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Enable event loop integration with wxPython.
+
+        Parameters
+        ----------
+        app : WX Application, optional.
+            Running application to use.  If not given, we probe WX for an
+            existing application object, and create a new one if none is found.
+
+        Notes
+        -----
+        This methods sets the ``PyOS_InputHook`` for wxPython, which allows
+        the wxPython to integrate with terminal based applications like
+        yap_ipython.
+
+        If ``app`` is not given we probe for an existing one, and return it if
+        found.  If no existing app is found, we create an :class:`wx.App` as
+        follows::
+
+            import wx
+            app = wx.App(redirect=False, clearSigInt=False)
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        import wx
+        
+        wx_version = V(wx.__version__).version
+        
+        if wx_version < [2, 8]:
+            raise ValueError("requires wxPython >= 2.8, but you have %s" % wx.__version__)
+        
+        from yap_ipython.lib.inputhookwx import inputhook_wx
+        self.manager.set_inputhook(inputhook_wx)
+        if _use_appnope():
+            from appnope import nope
+            nope()
+
+        import wx
+        if app is None:
+            app = wx.GetApp()
+        if app is None:
+            app = wx.App(redirect=False, clearSigInt=False)
+
+        return app
+
+    def disable(self):
+        """DEPRECATED since yap_ipython 5.0
+
+        Disable event loop integration with wxPython.
+
+        This restores appnapp on OS X
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        if _use_appnope():
+            from appnope import nap
+            nap()
+
+@inputhook_manager.register('qt', 'qt4')
+class Qt4InputHook(InputHookBase):
+    def enable(self, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Enable event loop integration with PyQt4.
+        
+        Parameters
+        ----------
+        app : Qt Application, optional.
+            Running application to use.  If not given, we probe Qt for an
+            existing application object, and create a new one if none is found.
+
+        Notes
+        -----
+        This methods sets the PyOS_InputHook for PyQt4, which allows
+        the PyQt4 to integrate with terminal based applications like
+        yap_ipython.
+
+        If ``app`` is not given we probe for an existing one, and return it if
+        found.  If no existing app is found, we create an :class:`QApplication`
+        as follows::
+
+            from PyQt4 import QtCore
+            app = QtGui.QApplication(sys.argv)
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        from yap_ipython.lib.inputhookqt4 import create_inputhook_qt4
+        app, inputhook_qt4 = create_inputhook_qt4(self.manager, app)
+        self.manager.set_inputhook(inputhook_qt4)
+        if _use_appnope():
+            from appnope import nope
+            nope()
+
+        return app
+
+    def disable_qt4(self):
+        """DEPRECATED since yap_ipython 5.0
+
+        Disable event loop integration with PyQt4.
+
+        This restores appnapp on OS X
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        if _use_appnope():
+            from appnope import nap
+            nap()
+
+
+@inputhook_manager.register('qt5')
+class Qt5InputHook(Qt4InputHook):
+    def enable(self, app=None):
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        os.environ['QT_API'] = 'pyqt5'
+        return Qt4InputHook.enable(self, app)
+
+
+@inputhook_manager.register('gtk')
+class GtkInputHook(InputHookBase):
+    def enable(self, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Enable event loop integration with PyGTK.
+
+        Parameters
+        ----------
+        app : ignored
+           Ignored, it's only a placeholder to keep the call signature of all
+           gui activation methods consistent, which simplifies the logic of
+           supporting magics.
+
+        Notes
+        -----
+        This methods sets the PyOS_InputHook for PyGTK, which allows
+        the PyGTK to integrate with terminal based applications like
+        yap_ipython.
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        import gtk
+        try:
+            gtk.set_interactive(True)
+        except AttributeError:
+            # For older versions of gtk, use our own ctypes version
+            from yap_ipython.lib.inputhookgtk import inputhook_gtk
+            self.manager.set_inputhook(inputhook_gtk)
+
+
+@inputhook_manager.register('tk')
+class TkInputHook(InputHookBase):
+    def enable(self, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Enable event loop integration with Tk.
+
+        Parameters
+        ----------
+        app : toplevel :class:`Tkinter.Tk` widget, optional.
+            Running toplevel widget to use.  If not given, we probe Tk for an
+            existing one, and create a new one if none is found.
+
+        Notes
+        -----
+        If you have already created a :class:`Tkinter.Tk` object, the only
+        thing done by this method is to register with the
+        :class:`InputHookManager`, since creating that object automatically
+        sets ``PyOS_InputHook``.
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        if app is None:
+            try:
+                from tkinter import Tk  # Py 3
+            except ImportError:
+                from Tkinter import Tk  # Py 2
+            app = Tk()
+            app.withdraw()
+            self.manager.apps[GUI_TK] = app
+            return app
+
+
+@inputhook_manager.register('glut')
+class GlutInputHook(InputHookBase):
+    def enable(self, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Enable event loop integration with GLUT.
+
+        Parameters
+        ----------
+
+        app : ignored
+            Ignored, it's only a placeholder to keep the call signature of all
+            gui activation methods consistent, which simplifies the logic of
+            supporting magics.
+
+        Notes
+        -----
+
+        This methods sets the PyOS_InputHook for GLUT, which allows the GLUT to
+        integrate with terminal based applications like yap_ipython. Due to GLUT
+        limitations, it is currently not possible to start the event loop
+        without first creating a window. You should thus not create another
+        window but use instead the created one. See 'gui-glut.py' in the
+        docs/examples/lib directory.
+        
+        The default screen mode is set to:
+        glut.GLUT_DOUBLE | glut.GLUT_RGBA | glut.GLUT_DEPTH
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+
+        import OpenGL.GLUT as glut
+        from yap_ipython.lib.inputhookglut import glut_display_mode, \
+                                              glut_close, glut_display, \
+                                              glut_idle, inputhook_glut
+
+        if GUI_GLUT not in self.manager.apps:
+            glut.glutInit( sys.argv )
+            glut.glutInitDisplayMode( glut_display_mode )
+            # This is specific to freeglut
+            if bool(glut.glutSetOption):
+                glut.glutSetOption( glut.GLUT_ACTION_ON_WINDOW_CLOSE,
+                                    glut.GLUT_ACTION_GLUTMAINLOOP_RETURNS )
+            glut.glutCreateWindow( sys.argv[0] )
+            glut.glutReshapeWindow( 1, 1 )
+            glut.glutHideWindow( )
+            glut.glutWMCloseFunc( glut_close )
+            glut.glutDisplayFunc( glut_display )
+            glut.glutIdleFunc( glut_idle )
+        else:
+            glut.glutWMCloseFunc( glut_close )
+            glut.glutDisplayFunc( glut_display )
+            glut.glutIdleFunc( glut_idle)
+        self.manager.set_inputhook( inputhook_glut )
+
+
+    def disable(self):
+        """DEPRECATED since yap_ipython 5.0
+
+        Disable event loop integration with glut.
+        
+        This sets PyOS_InputHook to NULL and set the display function to a
+        dummy one and set the timer to a dummy timer that will be triggered
+        very far in the future.
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        import OpenGL.GLUT as glut
+        from glut_support import glutMainLoopEvent
+
+        glut.glutHideWindow() # This is an event to be processed below
+        glutMainLoopEvent()
+        super(GlutInputHook, self).disable()
+
+@inputhook_manager.register('pyglet')
+class PygletInputHook(InputHookBase):
+    def enable(self, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Enable event loop integration with pyglet.
+
+        Parameters
+        ----------
+        app : ignored
+           Ignored, it's only a placeholder to keep the call signature of all
+           gui activation methods consistent, which simplifies the logic of
+           supporting magics.
+
+        Notes
+        -----
+        This methods sets the ``PyOS_InputHook`` for pyglet, which allows
+        pyglet to integrate with terminal based applications like
+        yap_ipython.
+
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        from yap_ipython.lib.inputhookpyglet import inputhook_pyglet
+        self.manager.set_inputhook(inputhook_pyglet)
+        return app
+
+
+@inputhook_manager.register('gtk3')
+class Gtk3InputHook(InputHookBase):
+    def enable(self, app=None):
+        """DEPRECATED since yap_ipython 5.0
+
+        Enable event loop integration with Gtk3 (gir bindings).
+
+        Parameters
+        ----------
+        app : ignored
+           Ignored, it's only a placeholder to keep the call signature of all
+           gui activation methods consistent, which simplifies the logic of
+           supporting magics.
+
+        Notes
+        -----
+        This methods sets the PyOS_InputHook for Gtk3, which allows
+        the Gtk3 to integrate with terminal based applications like
+        yap_ipython.
+        """
+        warn("This function is deprecated since yap_ipython 5.0 and will be removed in future versions.",
+                DeprecationWarning, stacklevel=2)
+        from yap_ipython.lib.inputhookgtk3 import inputhook_gtk3
+        self.manager.set_inputhook(inputhook_gtk3)
+
+
+clear_inputhook = inputhook_manager.clear_inputhook
+set_inputhook = inputhook_manager.set_inputhook
+current_gui = inputhook_manager.current_gui
+clear_app_refs = inputhook_manager.clear_app_refs
+enable_gui = inputhook_manager.enable_gui
+disable_gui = inputhook_manager.disable_gui
+register = inputhook_manager.register
+guis = inputhook_manager.guihooks
+
+
+def _deprecated_disable():
+    warn("This function is deprecated since yap_ipython 4.0 use disable_gui() instead",
+            DeprecationWarning, stacklevel=2)
+    inputhook_manager.disable_gui()
+    
+disable_wx = disable_qt4 = disable_gtk = disable_gtk3 = disable_glut = \
+        disable_pyglet = disable_osx = _deprecated_disable

packages/python/yap_kernel/yap_ipython/lib/inputhookglut.py

@@ -0,0 +1,172 @@
+# coding: utf-8
+"""
+GLUT Inputhook support functions
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The yap_ipython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+# GLUT is quite an old library and it is difficult to ensure proper
+# integration within yap_ipython since original GLUT does not allow to handle
+# events one by one. Instead, it requires for the mainloop to be entered
+# and never returned (there is not even a function to exit he
+# mainloop). Fortunately, there are alternatives such as freeglut
+# (available for linux and windows) and the OSX implementation gives
+# access to a glutCheckLoop() function that blocks itself until a new
+# event is received. This means we have to setup the idle callback to
+# ensure we got at least one event that will unblock the function.
+#
+# Furthermore, it is not possible to install these handlers without a window
+# being first created. We choose to make this window invisible. This means that
+# display mode options are set at this level and user won't be able to change
+# them later without modifying the code. This should probably be made available
+# via yap_ipython options system.
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import os
+import sys
+import time
+import signal
+import OpenGL.GLUT as glut
+import OpenGL.platform as platform
+from timeit import default_timer as clock
+
+#-----------------------------------------------------------------------------
+# Constants
+#-----------------------------------------------------------------------------
+
+# Frame per second : 60
+# Should probably be an yap_ipython option
+glut_fps = 60
+
+
+# Display mode : double buffeed + rgba + depth
+# Should probably be an yap_ipython option
+glut_display_mode = (glut.GLUT_DOUBLE |
+                     glut.GLUT_RGBA   |
+                     glut.GLUT_DEPTH)
+
+glutMainLoopEvent = None
+if sys.platform == 'darwin':
+    try:
+        glutCheckLoop = platform.createBaseFunction(
+            'glutCheckLoop', dll=platform.GLUT, resultType=None,
+            argTypes=[],
+            doc='glutCheckLoop(  ) -> None',
+            argNames=(),
+            )
+    except AttributeError:
+        raise RuntimeError(
+            '''Your glut implementation does not allow interactive sessions'''
+            '''Consider installing freeglut.''')
+    glutMainLoopEvent = glutCheckLoop
+elif glut.HAVE_FREEGLUT:
+    glutMainLoopEvent = glut.glutMainLoopEvent
+else:
+    raise RuntimeError(
+        '''Your glut implementation does not allow interactive sessions. '''
+        '''Consider installing freeglut.''')
+
+
+#-----------------------------------------------------------------------------
+# Platform-dependent imports and functions
+#-----------------------------------------------------------------------------
+
+if os.name == 'posix':
+    import select
+
+    def stdin_ready():
+        infds, outfds, erfds = select.select([sys.stdin],[],[],0)
+        if infds:
+            return True
+        else:
+            return False
+
+elif sys.platform == 'win32':
+    import msvcrt
+
+    def stdin_ready():
+        return msvcrt.kbhit()
+
+#-----------------------------------------------------------------------------
+# Callback functions
+#-----------------------------------------------------------------------------
+
+def glut_display():
+    # Dummy display function
+    pass
+
+def glut_idle():
+    # Dummy idle function
+    pass
+
+def glut_close():
+    # Close function only hides the current window
+    glut.glutHideWindow()
+    glutMainLoopEvent()
+
+def glut_int_handler(signum, frame):
+    # Catch sigint and print the defautl message
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+    print('\nKeyboardInterrupt')
+    # Need to reprint the prompt at this stage
+
+
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+def inputhook_glut():
+    """Run the pyglet event loop by processing pending events only.
+
+    This keeps processing pending events until stdin is ready.  After
+    processing all pending events, a call to time.sleep is inserted.  This is
+    needed, otherwise, CPU usage is at 100%.  This sleep time should be tuned
+    though for best performance.
+    """
+    # We need to protect against a user pressing Control-C when yap_ipython is
+    # idle and this is running. We trap KeyboardInterrupt and pass.
+
+    signal.signal(signal.SIGINT, glut_int_handler)
+
+    try:
+        t = clock()
+
+        # Make sure the default window is set after a window has been closed
+        if glut.glutGetWindow() == 0:
+            glut.glutSetWindow( 1 )
+            glutMainLoopEvent()
+            return 0
+
+        while not stdin_ready():
+            glutMainLoopEvent()
+            # We need to sleep at this point to keep the idle CPU load
+            # low.  However, if sleep to long, GUI response is poor.  As
+            # a compromise, we watch how often GUI events are being processed
+            # and switch between a short and long sleep time.  Here are some
+            # stats useful in helping to tune this.
+            # time    CPU load
+            # 0.001   13%
+            # 0.005   3%
+            # 0.01    1.5%
+            # 0.05    0.5%
+            used_time = clock() - t
+            if used_time > 10.0:
+                # print 'Sleep for 1 s'  # dbg
+                time.sleep(1.0)
+            elif used_time > 0.1:
+                # Few GUI events coming in, so we can sleep longer
+                # print 'Sleep for 0.05 s'  # dbg
+                time.sleep(0.05)
+            else:
+                # Many GUI events coming in, so sleep only very little
+                time.sleep(0.001)
+    except KeyboardInterrupt:
+        pass
+    return 0

packages/python/yap_kernel/yap_ipython/lib/inputhookgtk.py

@@ -0,0 +1,35 @@
+# encoding: utf-8
+"""
+Enable pygtk to be used interacive by setting PyOS_InputHook.
+
+Authors: Brian Granger
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The yap_ipython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import gtk, gobject
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+def _main_quit(*args, **kwargs):
+    gtk.main_quit()
+    return False
+
+def inputhook_gtk():
+    gobject.io_add_watch(sys.stdin, gobject.IO_IN, _main_quit)
+    gtk.main()
+    return 0
+

packages/python/yap_kernel/yap_ipython/lib/inputhookgtk3.py

@@ -0,0 +1,34 @@
+# encoding: utf-8
+"""
+Enable Gtk3 to be used interacive by yap_ipython.
+
+Authors: Thomi Richards
+"""
+#-----------------------------------------------------------------------------
+# Copyright (c) 2012, the yap_ipython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+from gi.repository import Gtk, GLib
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def _main_quit(*args, **kwargs):
+    Gtk.main_quit()
+    return False
+
+
+def inputhook_gtk3():
+    GLib.io_add_watch(sys.stdin, GLib.PRIORITY_DEFAULT, GLib.IO_IN, _main_quit)
+    Gtk.main()
+    return 0

packages/python/yap_kernel/yap_ipython/lib/inputhookpyglet.py

@@ -0,0 +1,111 @@
+# encoding: utf-8
+"""
+Enable pyglet to be used interacive by setting PyOS_InputHook.
+
+Authors
+-------
+
+* Nicolas P. Rougier
+* Fernando Perez
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The yap_ipython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import sys
+import time
+from timeit import default_timer as clock
+import pyglet
+
+#-----------------------------------------------------------------------------
+# Platform-dependent imports and functions
+#-----------------------------------------------------------------------------
+
+if os.name == 'posix':
+    import select
+
+    def stdin_ready():
+        infds, outfds, erfds = select.select([sys.stdin],[],[],0)
+        if infds:
+            return True
+        else:
+            return False
+
+elif sys.platform == 'win32':
+    import msvcrt
+
+    def stdin_ready():
+        return msvcrt.kbhit()
+
+
+# On linux only, window.flip() has a bug that causes an AttributeError on
+# window close.  For details, see:
+# http://groups.google.com/group/pyglet-users/browse_thread/thread/47c1aab9aa4a3d23/c22f9e819826799e?#c22f9e819826799e
+
+if sys.platform.startswith('linux'):
+    def flip(window):
+        try:
+            window.flip()
+        except AttributeError:
+            pass
+else:
+    def flip(window):
+        window.flip()
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def inputhook_pyglet():
+    """Run the pyglet event loop by processing pending events only.
+
+    This keeps processing pending events until stdin is ready.  After
+    processing all pending events, a call to time.sleep is inserted.  This is
+    needed, otherwise, CPU usage is at 100%.  This sleep time should be tuned
+    though for best performance.
+    """
+    # We need to protect against a user pressing Control-C when yap_ipython is
+    # idle and this is running. We trap KeyboardInterrupt and pass.
+    try:
+        t = clock()
+        while not stdin_ready():
+            pyglet.clock.tick()
+            for window in pyglet.app.windows:
+                window.switch_to()
+                window.dispatch_events()
+                window.dispatch_event('on_draw')
+                flip(window)
+
+            # We need to sleep at this point to keep the idle CPU load
+            # low.  However, if sleep to long, GUI response is poor.  As
+            # a compromise, we watch how often GUI events are being processed
+            # and switch between a short and long sleep time.  Here are some
+            # stats useful in helping to tune this.
+            # time    CPU load
+            # 0.001   13%
+            # 0.005   3%
+            # 0.01    1.5%
+            # 0.05    0.5%
+            used_time = clock() - t
+            if used_time > 10.0:
+                # print 'Sleep for 1 s'  # dbg
+                time.sleep(1.0)
+            elif used_time > 0.1:
+                # Few GUI events coming in, so we can sleep longer
+                # print 'Sleep for 0.05 s'  # dbg
+                time.sleep(0.05)
+            else:
+                # Many GUI events coming in, so sleep only very little
+                time.sleep(0.001)
+    except KeyboardInterrupt:
+        pass
+    return 0

packages/python/yap_kernel/yap_ipython/lib/inputhookqt4.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+"""
+Qt4's inputhook support function
+
+Author: Christian Boos
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2011  The yap_ipython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import signal
+import threading
+
+from yap_ipython.core.interactiveshell import InteractiveShell
+from yap_ipython.external.qt_for_kernel import QtCore, QtGui
+from yap_ipython.lib.inputhook import allow_CTRL_C, ignore_CTRL_C, stdin_ready
+
+#-----------------------------------------------------------------------------
+# Module Globals
+#-----------------------------------------------------------------------------
+
+got_kbdint = False
+sigint_timer = None
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def create_inputhook_qt4(mgr, app=None):
+    """Create an input hook for running the Qt4 application event loop.
+
+    Parameters
+    ----------
+    mgr : an InputHookManager
+
+    app : Qt Application, optional.
+        Running application to use.  If not given, we probe Qt for an
+        existing application object, and create a new one if none is found.
+
+    Returns
+    -------
+    A pair consisting of a Qt Application (either the one given or the
+    one found or created) and a inputhook.
+
+    Notes
+    -----
+    We use a custom input hook instead of PyQt4's default one, as it
+    interacts better with the readline packages (issue #481).
+
+    The inputhook function works in tandem with a 'pre_prompt_hook'
+    which automatically restores the hook as an inputhook in case the
+    latter has been temporarily disabled after having intercepted a
+    KeyboardInterrupt.
+    """
+
+    if app is None:
+        app = QtCore.QCoreApplication.instance()
+        if app is None:
+            app = QtGui.QApplication([" "])
+
+    # Re-use previously created inputhook if any
+    ip = InteractiveShell.instance()
+    if hasattr(ip, '_inputhook_qt4'):
+        return app, ip._inputhook_qt4
+
+    # Otherwise create the inputhook_qt4/preprompthook_qt4 pair of
+    # hooks (they both share the got_kbdint flag)
+
+    def inputhook_qt4():
+        """PyOS_InputHook python hook for Qt4.
+
+        Process pending Qt events and if there's no pending keyboard
+        input, spend a short slice of time (50ms) running the Qt event
+        loop.
+
+        As a Python ctypes callback can't raise an exception, we catch
+        the KeyboardInterrupt and temporarily deactivate the hook,
+        which will let a *second* CTRL+C be processed normally and go
+        back to a clean prompt line.
+        """
+        try:
+            allow_CTRL_C()
+            app = QtCore.QCoreApplication.instance()
+            if not app: # shouldn't happen, but safer if it happens anyway...
+                return 0
+            app.processEvents(QtCore.QEventLoop.AllEvents, 300)
+            if not stdin_ready():
+                # Generally a program would run QCoreApplication::exec()
+                # from main() to enter and process the Qt event loop until
+                # quit() or exit() is called and the program terminates.
+                #
+                # For our input hook integration, we need to repeatedly
+                # enter and process the Qt event loop for only a short
+                # amount of time (say 50ms) to ensure that Python stays
+                # responsive to other user inputs.
+                #
+                # A naive approach would be to repeatedly call
+                # QCoreApplication::exec(), using a timer to quit after a
+                # short amount of time. Unfortunately, QCoreApplication
+                # emits an aboutToQuit signal before stopping, which has
+                # the undesirable effect of closing all modal windows.
+                #
+                # To work around this problem, we instead create a
+                # QEventLoop and call QEventLoop::exec(). Other than
+                # setting some state variables which do not seem to be
+                # used anywhere, the only thing QCoreApplication adds is
+                # the aboutToQuit signal which is precisely what we are
+                # trying to avoid.
+                timer = QtCore.QTimer()
+                event_loop = QtCore.QEventLoop()
+                timer.timeout.connect(event_loop.quit)
+                while not stdin_ready():
+                    timer.start(50)
+                    event_loop.exec_()
+                    timer.stop()
+        except KeyboardInterrupt:
+            global got_kbdint, sigint_timer
+
+            ignore_CTRL_C()
+            got_kbdint = True
+            mgr.clear_inputhook()
+
+            # This generates a second SIGINT so the user doesn't have to
+            # press CTRL+C twice to get a clean prompt.
+            #
+            # Since we can't catch the resulting KeyboardInterrupt here
+            # (because this is a ctypes callback), we use a timer to
+            # generate the SIGINT after we leave this callback.
+            #
+            # Unfortunately this doesn't work on Windows (SIGINT kills
+            # Python and CTRL_C_EVENT doesn't work).
+            if(os.name == 'posix'):
+                pid = os.getpid()
+                if(not sigint_timer):
+                    sigint_timer = threading.Timer(.01, os.kill,
+                                         args=[pid, signal.SIGINT] )
+                    sigint_timer.start()
+            else:
+                print("\nKeyboardInterrupt - Ctrl-C again for new prompt")
+
+
+        except: # NO exceptions are allowed to escape from a ctypes callback
+            ignore_CTRL_C()
+            from traceback import print_exc
+            print_exc()
+            print("Got exception from inputhook_qt4, unregistering.")
+            mgr.clear_inputhook()
+        finally:
+            allow_CTRL_C()
+        return 0
+
+    def preprompthook_qt4(ishell):
+        """'pre_prompt_hook' used to restore the Qt4 input hook
+
+        (in case the latter was temporarily deactivated after a
+        CTRL+C)
+        """
+        global got_kbdint, sigint_timer
+
+        if(sigint_timer):
+            sigint_timer.cancel()
+            sigint_timer = None
+
+        if got_kbdint:
+            mgr.set_inputhook(inputhook_qt4)
+        got_kbdint = False
+
+    ip._inputhook_qt4 = inputhook_qt4
+    ip.set_hook('pre_prompt_hook', preprompthook_qt4)
+
+    return app, inputhook_qt4

packages/python/yap_kernel/yap_ipython/lib/inputhookwx.py

@@ -0,0 +1,167 @@
+# encoding: utf-8
+
+"""
+Enable wxPython to be used interacive by setting PyOS_InputHook.
+
+Authors:  Robin Dunn, Brian Granger, Ondrej Certik
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The yap_ipython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import sys
+import signal
+import time
+from timeit import default_timer as clock
+import wx
+
+from yap_ipython.lib.inputhook import stdin_ready
+
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def inputhook_wx1():
+    """Run the wx event loop by processing pending events only.
+
+    This approach seems to work, but its performance is not great as it
+    relies on having PyOS_InputHook called regularly.
+    """
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+
+            # Make a temporary event loop and process system events until
+            # there are no more waiting, then allow idle events (which
+            # will also deal with pending or posted wx events.)
+            evtloop = wx.EventLoop()
+            ea = wx.EventLoopActivator(evtloop)
+            while evtloop.Pending():
+                evtloop.Dispatch()
+            app.ProcessIdle()
+            del ea
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+class EventLoopTimer(wx.Timer):
+
+    def __init__(self, func):
+        self.func = func
+        wx.Timer.__init__(self)
+
+    def Notify(self):
+        self.func()
+
+class EventLoopRunner(object):
+
+    def Run(self, time):
+        self.evtloop = wx.EventLoop()
+        self.timer = EventLoopTimer(self.check_stdin)
+        self.timer.Start(time)
+        self.evtloop.Run()
+
+    def check_stdin(self):
+        if stdin_ready():
+            self.timer.Stop()
+            self.evtloop.Exit()
+
+def inputhook_wx2():
+    """Run the wx event loop, polling for stdin.
+
+    This version runs the wx eventloop for an undetermined amount of time,
+    during which it periodically checks to see if anything is ready on
+    stdin.  If anything is ready on stdin, the event loop exits.
+
+    The argument to elr.Run controls how often the event loop looks at stdin.
+    This determines the responsiveness at the keyboard.  A setting of 1000
+    enables a user to type at most 1 char per second.  I have found that a
+    setting of 10 gives good keyboard response.  We can shorten it further,
+    but eventually performance would suffer from calling select/kbhit too
+    often.
+    """
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+            elr = EventLoopRunner()
+            # As this time is made shorter, keyboard response improves, but idle
+            # CPU load goes up.  10 ms seems like a good compromise.
+            elr.Run(time=10)  # CHANGE time here to control polling interval
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+def inputhook_wx3():
+    """Run the wx event loop by processing pending events only.
+
+    This is like inputhook_wx1, but it keeps processing pending events
+    until stdin is ready.  After processing all pending events, a call to
+    time.sleep is inserted.  This is needed, otherwise, CPU usage is at 100%.
+    This sleep time should be tuned though for best performance.
+    """
+    # We need to protect against a user pressing Control-C when yap_ipython is
+    # idle and this is running. We trap KeyboardInterrupt and pass.
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+
+            # The import of wx on Linux sets the handler for signal.SIGINT
+            # to 0.  This is a bug in wx or gtk.  We fix by just setting it
+            # back to the Python default.
+            if not callable(signal.getsignal(signal.SIGINT)):
+                signal.signal(signal.SIGINT, signal.default_int_handler)
+
+            evtloop = wx.EventLoop()
+            ea = wx.EventLoopActivator(evtloop)
+            t = clock()
+            while not stdin_ready():
+                while evtloop.Pending():
+                    t = clock()
+                    evtloop.Dispatch()
+                app.ProcessIdle()
+                # We need to sleep at this point to keep the idle CPU load
+                # low.  However, if sleep to long, GUI response is poor.  As
+                # a compromise, we watch how often GUI events are being processed
+                # and switch between a short and long sleep time.  Here are some
+                # stats useful in helping to tune this.
+                # time    CPU load
+                # 0.001   13%
+                # 0.005   3%
+                # 0.01    1.5%
+                # 0.05    0.5%
+                used_time = clock() - t
+                if used_time > 10.0:
+                    # print 'Sleep for 1 s'  # dbg
+                    time.sleep(1.0)
+                elif used_time > 0.1:
+                    # Few GUI events coming in, so we can sleep longer
+                    # print 'Sleep for 0.05 s'  # dbg
+                    time.sleep(0.05)
+                else:
+                    # Many GUI events coming in, so sleep only very little
+                    time.sleep(0.001)
+            del ea
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+if sys.platform == 'darwin':
+    # On OSX, evtloop.Pending() always returns True, regardless of there being
+    # any events pending. As such we can't use implementations 1 or 3 of the
+    # inputhook as those depend on a pending/dispatch loop.
+    inputhook_wx = inputhook_wx2
+else:
+    # This is our default implementation
+    inputhook_wx = inputhook_wx3

packages/python/yap_kernel/yap_ipython/lib/kernel.py

@@ -0,0 +1,13 @@
+"""[DEPRECATED] Utilities for connecting to kernels
+
+Moved to yap_ipython.kernel.connect
+"""
+
+import warnings
+warnings.warn("yap_ipython.lib.kernel moved to yap_ipython.kernel.connect in yap_ipython 1.0," 
+        " and will be removed in yap_ipython 6.0.",
+    DeprecationWarning
+)
+
+from yap_kernel.connect import *
+

packages/python/yap_kernel/yap_ipython/lib/latextools.py

@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+"""Tools for handling LaTeX."""
+
+# Copyright (c) yap_ipython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from io import BytesIO, open
+import os
+import tempfile
+import shutil
+import subprocess
+from base64 import encodebytes
+
+from yap_ipython.utils.process import find_cmd, FindCmdError
+from traitlets.config import get_config
+from traitlets.config.configurable import SingletonConfigurable
+from traitlets import List, Bool, Unicode
+from yap_ipython.utils.py3compat import cast_unicode
+
+
+class LaTeXTool(SingletonConfigurable):
+    """An object to store configuration of the LaTeX tool."""
+    def _config_default(self):
+        return get_config()
+    
+    backends = List(
+        Unicode(), ["matplotlib", "dvipng"],
+        help="Preferred backend to draw LaTeX math equations. "
+        "Backends in the list are checked one by one and the first "
+        "usable one is used.  Note that `matplotlib` backend "
+        "is usable only for inline style equations.  To draw  "
+        "display style equations, `dvipng` backend must be specified. ",
+        # It is a List instead of Enum, to make configuration more
+        # flexible.  For example, to use matplotlib mainly but dvipng
+        # for display style, the default ["matplotlib", "dvipng"] can
+        # be used.  To NOT use dvipng so that other repr such as
+        # unicode pretty printing is used, you can use ["matplotlib"].
+        ).tag(config=True)
+
+    use_breqn = Bool(
+        True,
+        help="Use breqn.sty to automatically break long equations. "
+        "This configuration takes effect only for dvipng backend.",
+        ).tag(config=True)
+
+    packages = List(
+        ['amsmath', 'amsthm', 'amssymb', 'bm'],
+        help="A list of packages to use for dvipng backend. "
+        "'breqn' will be automatically appended when use_breqn=True.",
+        ).tag(config=True)
+
+    preamble = Unicode(
+        help="Additional preamble to use when generating LaTeX source "
+        "for dvipng backend.",
+        ).tag(config=True)
+
+
+def latex_to_png(s, encode=False, backend=None, wrap=False):
+    """Render a LaTeX string to PNG.
+
+    Parameters
+    ----------
+    s : str
+        The raw string containing valid inline LaTeX.
+    encode : bool, optional
+        Should the PNG data base64 encoded to make it JSON'able.
+    backend : {matplotlib, dvipng}
+        Backend for producing PNG data.
+    wrap : bool
+        If true, Automatically wrap `s` as a LaTeX equation.
+
+    None is returned when the backend cannot be used.
+
+    """
+    s = cast_unicode(s)
+    allowed_backends = LaTeXTool.instance().backends
+    if backend is None:
+        backend = allowed_backends[0]
+    if backend not in allowed_backends:
+        return None
+    if backend == 'matplotlib':
+        f = latex_to_png_mpl
+    elif backend == 'dvipng':
+        f = latex_to_png_dvipng
+    else:
+        raise ValueError('No such backend {0}'.format(backend))
+    bin_data = f(s, wrap)
+    if encode and bin_data:
+        bin_data = encodebytes(bin_data)
+    return bin_data
+
+
+def latex_to_png_mpl(s, wrap):
+    try:
+        from matplotlib import mathtext
+        from pyparsing import ParseFatalException
+    except ImportError:
+        return None
+
+    # mpl mathtext doesn't support display math, force inline
+    s = s.replace('$$', '$')
+    if wrap:
+        s = u'${0}$'.format(s)
+
+    try:
+        mt = mathtext.MathTextParser('bitmap')
+        f = BytesIO()
+        mt.to_png(f, s, fontsize=12)
+        return f.getvalue()
+    except (ValueError, RuntimeError, ParseFatalException):
+        return None
+
+
+def latex_to_png_dvipng(s, wrap):
+    try:
+        find_cmd('latex')
+        find_cmd('dvipng')
+    except FindCmdError:
+        return None
+    try:
+        workdir = tempfile.mkdtemp()
+        tmpfile = os.path.join(workdir, "tmp.tex")
+        dvifile = os.path.join(workdir, "tmp.dvi")
+        outfile = os.path.join(workdir, "tmp.png")
+
+        with open(tmpfile, "w", encoding='utf8') as f:
+            f.writelines(genelatex(s, wrap))
+
+        with open(os.devnull, 'wb') as devnull:
+            subprocess.check_call(
+                ["latex", "-halt-on-error", "-interaction", "batchmode", tmpfile],
+                cwd=workdir, stdout=devnull, stderr=devnull)
+
+            subprocess.check_call(
+                ["dvipng", "-T", "tight", "-x", "1500", "-z", "9",
+                 "-bg", "transparent", "-o", outfile, dvifile], cwd=workdir,
+                stdout=devnull, stderr=devnull)
+
+        with open(outfile, "rb") as f:
+            return f.read()
+    except subprocess.CalledProcessError:
+        return None
+    finally:
+        shutil.rmtree(workdir)
+
+
+def kpsewhich(filename):
+    """Invoke kpsewhich command with an argument `filename`."""
+    try:
+        find_cmd("kpsewhich")
+        proc = subprocess.Popen(
+            ["kpsewhich", filename],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        (stdout, stderr) = proc.communicate()
+        return stdout.strip().decode('utf8', 'replace')
+    except FindCmdError:
+        pass
+
+
+def genelatex(body, wrap):
+    """Generate LaTeX document for dvipng backend."""
+    lt = LaTeXTool.instance()
+    breqn = wrap and lt.use_breqn and kpsewhich("breqn.sty")
+    yield r'\documentclass{article}'
+    packages = lt.packages
+    if breqn:
+        packages = packages + ['breqn']
+    for pack in packages:
+        yield r'\usepackage{{{0}}}'.format(pack)
+    yield r'\pagestyle{empty}'
+    if lt.preamble:
+        yield lt.preamble
+    yield r'\begin{document}'
+    if breqn:
+        yield r'\begin{dmath*}'
+        yield body
+        yield r'\end{dmath*}'
+    elif wrap:
+        yield u'$${0}$$'.format(body)
+    else:
+        yield body
+    yield u'\end{document}'
+
+
+_data_uri_template_png = u"""<img src="data:image/png;base64,%s" alt=%s />"""
+
+def latex_to_html(s, alt='image'):
+    """Render LaTeX to HTML with embedded PNG data using data URIs.
+
+    Parameters
+    ----------
+    s : str
+        The raw string containing valid inline LateX.
+    alt : str
+        The alt text to use for the HTML.
+    """
+    base64_data = latex_to_png(s, encode=True).decode('ascii')
+    if base64_data:
+        return _data_uri_template_png  % (base64_data, alt)
+
+

packages/python/yap_kernel/yap_ipython/lib/lexers.py

@@ -0,0 +1,512 @@
+# -*- coding: utf-8 -*-
+"""
+Defines a variety of Pygments lexers for highlighting yap_ipython code.
+
+This includes:
+
+    IPythonLexer, IPython3Lexer
+        Lexers for pure yap_ipython (python + magic/shell commands)
+
+    IPythonPartialTracebackLexer, IPythonTracebackLexer
+        Supports 2.x and 3.x via keyword `python3`.  The partial traceback
+        lexer reads everything but the Python code appearing in a traceback.
+        The full lexer combines the partial lexer with an yap_ipython lexer.
+
+    IPythonConsoleLexer
+        A lexer for yap_ipython console sessions, with support for tracebacks.
+
+    IPyLexer
+        A friendly lexer which examines the first line of text and from it,
+        decides whether to use an yap_ipython lexer or an yap_ipython console lexer.
+        This is probably the only lexer that needs to be explicitly added
+        to Pygments.
+
+"""
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the yap_ipython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+# Standard library
+import re
+
+# Third party
+from pygments.lexers import BashLexer, PythonLexer, Python3Lexer
+from pygments.lexer import (
+    Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
+)
+from pygments.token import (
+    Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
+)
+from pygments.util import get_bool_opt
+
+# Local
+
+line_re = re.compile('.*?\n')
+
+__all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
+           'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
+           'IPythonConsoleLexer', 'IPyLexer']
+
+ipython_tokens = [
+  (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
+  (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
+  (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
+  (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
+  (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
+                                       using(BashLexer), Text)),
+  (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
+  (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+  (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+  (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
+  (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
+]
+
+def build_ipy_lexer(python3):
+    """Builds yap_ipython lexers depending on the value of `python3`.
+
+    The lexer inherits from an appropriate Python lexer and then adds
+    information about yap_ipython specific keywords (i.e. magic commands,
+    shell commands, etc.)
+
+    Parameters
+    ----------
+    python3 : bool
+        If `True`, then build an yap_ipython lexer from a Python 3 lexer.
+
+    """
+    # It would be nice to have a single yap_ipython lexer class which takes
+    # a boolean `python3`.  But since there are two Python lexer classes,
+    # we will also have two yap_ipython lexer classes.
+    if python3:
+        PyLexer = Python3Lexer
+        name = 'IPython3'
+        aliases = ['ipython3']
+        doc = """IPython3 Lexer"""
+    else:
+        PyLexer = PythonLexer
+        name = 'yap_ipython'
+        aliases = ['ipython2', 'ipython']
+        doc = """yap_ipython Lexer"""
+
+    tokens = PyLexer.tokens.copy()
+    tokens['root'] = ipython_tokens + tokens['root']
+
+    attrs = {'name': name, 'aliases': aliases, 'filenames': [],
+             '__doc__': doc, 'tokens': tokens}
+
+    return type(name, (PyLexer,), attrs)
+
+
+IPython3Lexer = build_ipy_lexer(python3=True)
+IPythonLexer = build_ipy_lexer(python3=False)
+
+
+class IPythonPartialTracebackLexer(RegexLexer):
+    """
+    Partial lexer for yap_ipython tracebacks.
+
+    Handles all the non-python output. This works for both Python 2.x and 3.x.
+
+    """
+    name = 'yap_ipython Partial Traceback'
+
+    tokens = {
+        'root': [
+            # Tracebacks for syntax errors have a different style.
+            # For both types of tracebacks, we mark the first line with
+            # Generic.Traceback.  For syntax errors, we mark the filename
+            # as we mark the filenames for non-syntax tracebacks.
+            #
+            # These two regexps define how IPythonConsoleLexer finds a
+            # traceback.
+            #
+            ## Non-syntax traceback
+            (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
+            ## Syntax traceback
+            (r'^(  File)(.*)(, line )(\d+\n)',
+             bygroups(Generic.Traceback, Name.Namespace,
+                      Generic.Traceback, Literal.Number.Integer)),
+
+            # (Exception Identifier)(Whitespace)(Traceback Message)
+            (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
+             bygroups(Name.Exception, Generic.Whitespace, Text)),
+            # (Module/Filename)(Text)(Callee)(Function Signature)
+            # Better options for callee and function signature?
+            (r'(.*)( in )(.*)(\(.*\)\n)',
+             bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
+            # Regular line: (Whitespace)(Line Number)(Python Code)
+            (r'(\s*?)(\d+)(.*?\n)',
+             bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
+            # Emphasized line: (Arrow)(Line Number)(Python Code)
+            # Using Exception token so arrow color matches the Exception.
+            (r'(-*>?\s?)(\d+)(.*?\n)',
+             bygroups(Name.Exception, Literal.Number.Integer, Other)),
+            # (Exception Identifier)(Message)
+            (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
+             bygroups(Name.Exception, Text)),
+            # Tag everything else as Other, will be handled later.
+            (r'.*\n', Other),
+        ],
+    }
+
+
+class IPythonTracebackLexer(DelegatingLexer):
+    """
+    yap_ipython traceback lexer.
+
+    For doctests, the tracebacks can be snipped as much as desired with the
+    exception to the lines that designate a traceback. For non-syntax error
+    tracebacks, this is the line of hyphens. For syntax error tracebacks,
+    this is the line which lists the File and line number.
+
+    """
+    # The lexer inherits from DelegatingLexer.  The "root" lexer is an
+    # appropriate yap_ipython lexer, which depends on the value of the boolean
+    # `python3`.  First, we parse with the partial yap_ipython traceback lexer.
+    # Then, any code marked with the "Other" token is delegated to the root
+    # lexer.
+    #
+    name = 'yap_ipython Traceback'
+    aliases = ['ipythontb']
+
+    def __init__(self, **options):
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipython3tb']
+        else:
+            self.aliases = ['ipython2tb', 'ipythontb']
+
+        if self.python3:
+            IPyLexer = IPython3Lexer
+        else:
+            IPyLexer = IPythonLexer
+
+        DelegatingLexer.__init__(self, IPyLexer,
+                                 IPythonPartialTracebackLexer, **options)
+
+class IPythonConsoleLexer(Lexer):
+    """
+    An yap_ipython console lexer for yap_ipython code-blocks and doctests, such as:
+
+    .. code-block:: rst
+
+        .. code-block:: ipythonconsole
+
+            In [1]: a = 'foo'
+
+            In [2]: a
+            Out[2]: 'foo'
+
+            In [3]: print a
+            foo
+
+            In [4]: 1 / 0
+
+
+    Support is also provided for yap_ipython exceptions:
+
+    .. code-block:: rst
+
+        .. code-block:: ipythonconsole
+
+            In [1]: raise Exception
+
+            ---------------------------------------------------------------------------
+            Exception                                 Traceback (most recent call last)
+            <ipython-input-1-fca2ab0ca76b> in <module>()
+            ----> 1 raise Exception
+
+            Exception:
+
+    """
+    name = 'yap_ipython console session'
+    aliases = ['ipythonconsole']
+    mimetypes = ['text/x-ipython-console']
+
+    # The regexps used to determine what is input and what is output.
+    # The default prompts for yap_ipython are:
+    #
+    #    in           = 'In [#]: '
+    #    continuation = '   .D.: '
+    #    template     = 'Out[#]: '
+    #
+    # Where '#' is the 'prompt number' or 'execution count' and 'D' 
+    # D is a number of dots  matching the width of the execution count 
+    #
+    in1_regex = r'In \[[0-9]+\]: '
+    in2_regex = r'   \.\.+\.: '
+    out_regex = r'Out\[[0-9]+\]: '
+
+    #: The regex to determine when a traceback starts.
+    ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')
+
+    def __init__(self, **options):
+        """Initialize the yap_ipython console lexer.
+
+        Parameters
+        ----------
+        python3 : bool
+            If `True`, then the console inputs are parsed using a Python 3
+            lexer. Otherwise, they are parsed using a Python 2 lexer.
+        in1_regex : RegexObject
+            The compiled regular expression used to detect the start
+            of inputs. Although the yap_ipython configuration setting may have a
+            trailing whitespace, do not include it in the regex. If `None`,
+            then the default input prompt is assumed.
+        in2_regex : RegexObject
+            The compiled regular expression used to detect the continuation
+            of inputs. Although the yap_ipython configuration setting may have a
+            trailing whitespace, do not include it in the regex. If `None`,
+            then the default input prompt is assumed.
+        out_regex : RegexObject
+            The compiled regular expression used to detect outputs. If `None`,
+            then the default output prompt is assumed.
+
+        """
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipython3console']
+        else:
+            self.aliases = ['ipython2console', 'ipythonconsole']
+
+        in1_regex = options.get('in1_regex', self.in1_regex)
+        in2_regex = options.get('in2_regex', self.in2_regex)
+        out_regex = options.get('out_regex', self.out_regex)
+
+        # So that we can work with input and output prompts which have been
+        # rstrip'd (possibly by editors) we also need rstrip'd variants. If
+        # we do not do this, then such prompts will be tagged as 'output'.
+        # The reason can't just use the rstrip'd variants instead is because
+        # we want any whitespace associated with the prompt to be inserted
+        # with the token. This allows formatted code to be modified so as hide
+        # the appearance of prompts, with the whitespace included. One example
+        # use of this is in copybutton.js from the standard lib Python docs.
+        in1_regex_rstrip = in1_regex.rstrip() + '\n'
+        in2_regex_rstrip = in2_regex.rstrip() + '\n'
+        out_regex_rstrip = out_regex.rstrip() + '\n'
+
+        # Compile and save them all.
+        attrs = ['in1_regex', 'in2_regex', 'out_regex',
+                 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
+        for attr in attrs:
+            self.__setattr__(attr, re.compile(locals()[attr]))
+
+        Lexer.__init__(self, **options)
+
+        if self.python3:
+            pylexer = IPython3Lexer
+            tblexer = IPythonTracebackLexer
+        else:
+            pylexer = IPythonLexer
+            tblexer = IPythonTracebackLexer
+
+        self.pylexer = pylexer(**options)
+        self.tblexer = tblexer(**options)
+
+        self.reset()
+
+    def reset(self):
+        self.mode = 'output'
+        self.index = 0
+        self.buffer = u''
+        self.insertions = []
+
+    def buffered_tokens(self):
+        """
+        Generator of unprocessed tokens after doing insertions and before
+        changing to a new state.
+
+        """
+        if self.mode == 'output':
+            tokens = [(0, Generic.Output, self.buffer)]
+        elif self.mode == 'input':
+            tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
+        else: # traceback
+            tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
+
+        for i, t, v in do_insertions(self.insertions, tokens):
+            # All token indexes are relative to the buffer.
+            yield self.index + i, t, v
+
+        # Clear it all
+        self.index += len(self.buffer)
+        self.buffer = u''
+        self.insertions = []
+
+    def get_mci(self, line):
+        """
+        Parses the line and returns a 3-tuple: (mode, code, insertion).
+
+        `mode` is the next mode (or state) of the lexer, and is always equal
+        to 'input', 'output', or 'tb'.
+
+        `code` is a portion of the line that should be added to the buffer
+        corresponding to the next mode and eventually lexed by another lexer.
+        For example, `code` could be Python code if `mode` were 'input'.
+
+        `insertion` is a 3-tuple (index, token, text) representing an
+        unprocessed "token" that will be inserted into the stream of tokens
+        that are created from the buffer once we change modes. This is usually
+        the input or output prompt.
+
+        In general, the next mode depends on current mode and on the contents
+        of `line`.
+
+        """
+        # To reduce the number of regex match checks, we have multiple
+        # 'if' blocks instead of 'if-elif' blocks.
+
+        # Check for possible end of input
+        in2_match = self.in2_regex.match(line)
+        in2_match_rstrip = self.in2_regex_rstrip.match(line)
+        if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
+           in2_match_rstrip:
+            end_input = True
+        else:
+            end_input = False
+        if end_input and self.mode != 'tb':
+            # Only look for an end of input when not in tb mode.
+            # An ellipsis could appear within the traceback.
+            mode = 'output'
+            code = u''
+            insertion = (0, Generic.Prompt, line)
+            return mode, code, insertion
+
+        # Check for output prompt
+        out_match = self.out_regex.match(line)
+        out_match_rstrip = self.out_regex_rstrip.match(line)
+        if out_match or out_match_rstrip:
+            mode = 'output'
+            if out_match:
+                idx = out_match.end()
+            else:
+                idx = out_match_rstrip.end()
+            code = line[idx:]
+            # Use the 'heading' token for output.  We cannot use Generic.Error
+            # since it would conflict with exceptions.
+            insertion = (0, Generic.Heading, line[:idx])
+            return mode, code, insertion
+
+
+        # Check for input or continuation prompt (non stripped version)
+        in1_match = self.in1_regex.match(line)
+        if in1_match or (in2_match and self.mode != 'tb'):
+            # New input or when not in tb, continued input.
+            # We do not check for continued input when in tb since it is
+            # allowable to replace a long stack with an ellipsis.
+            mode = 'input'
+            if in1_match:
+                idx = in1_match.end()
+            else: # in2_match
+                idx = in2_match.end()
+            code = line[idx:]
+            insertion = (0, Generic.Prompt, line[:idx])
+            return mode, code, insertion
+
+        # Check for input or continuation prompt (stripped version)
+        in1_match_rstrip = self.in1_regex_rstrip.match(line)
+        if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
+            # New input or when not in tb, continued input.
+            # We do not check for continued input when in tb since it is
+            # allowable to replace a long stack with an ellipsis.
+            mode = 'input'
+            if in1_match_rstrip:
+                idx = in1_match_rstrip.end()
+            else: # in2_match
+                idx = in2_match_rstrip.end()
+            code = line[idx:]
+            insertion = (0, Generic.Prompt, line[:idx])
+            return mode, code, insertion
+
+        # Check for traceback
+        if self.ipytb_start.match(line):
+            mode = 'tb'
+            code = line
+            insertion = None
+            return mode, code, insertion
+
+        # All other stuff...
+        if self.mode in ('input', 'output'):
+            # We assume all other text is output. Multiline input that
+            # does not use the continuation marker cannot be detected.
+            # For example, the 3 in the following is clearly output:
+            #
+            #    In [1]: print 3
+            #    3
+            #
+            # But the following second line is part of the input:
+            #
+            #    In [2]: while True:
+            #        print True
+            #
+            # In both cases, the 2nd line will be 'output'.
+            #
+            mode = 'output'
+        else:
+            mode = 'tb'
+
+        code = line
+        insertion = None
+
+        return mode, code, insertion
+
+    def get_tokens_unprocessed(self, text):
+        self.reset()
+        for match in line_re.finditer(text):
+            line = match.group()
+            mode, code, insertion = self.get_mci(line)
+
+            if mode != self.mode:
+                # Yield buffered tokens before transitioning to new mode.
+                for token in self.buffered_tokens():
+                    yield token
+                self.mode = mode
+
+            if insertion:
+                self.insertions.append((len(self.buffer), [insertion]))
+            self.buffer += code
+
+        for token in self.buffered_tokens():
+            yield token
+
+class IPyLexer(Lexer):
+    """
+    Primary lexer for all yap_ipython-like code.
+
+    This is a simple helper lexer.  If the first line of the text begins with
+    "In \[[0-9]+\]:", then the entire text is parsed with an yap_ipython console
+    lexer. If not, then the entire text is parsed with an yap_ipython lexer.
+
+    The goal is to reduce the number of lexers that are registered
+    with Pygments.
+
+    """
+    name = 'IPy session'
+    aliases = ['ipy']
+
+    def __init__(self, **options):
+        self.python3 = get_bool_opt(options, 'python3', False)
+        if self.python3:
+            self.aliases = ['ipy3']
+        else:
+            self.aliases = ['ipy2', 'ipy']
+
+        Lexer.__init__(self, **options)
+
+        self.IPythonLexer = IPythonLexer(**options)
+        self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
+
+    def get_tokens_unprocessed(self, text):
+        # Search for the input prompt anywhere...this allows code blocks to
+        # begin with comments as well.
+        if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
+            lex = self.IPythonConsoleLexer
+        else:
+            lex = self.IPythonLexer
+        for token in lex.get_tokens_unprocessed(text):
+            yield token
+

packages/python/yap_kernel/yap_ipython/lib/pretty.py

@@ -0,0 +1,871 @@
+# -*- coding: utf-8 -*-
+"""
+Python advanced pretty printer.  This pretty printer is intended to
+replace the old `pprint` python module which does not allow developers
+to provide their own pretty print callbacks.
+
+This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.
+
+
+Example Usage
+-------------
+
+To directly print the representation of an object use `pprint`::
+
+    from pretty import pprint
+    pprint(complex_object)
+
+To get a string of the output use `pretty`::
+
+    from pretty import pretty
+    string = pretty(complex_object)
+
+
+Extending
+---------
+
+The pretty library allows developers to add pretty printing rules for their
+own objects.  This process is straightforward.  All you have to do is to
+add a `_repr_pretty_` method to your object and call the methods on the
+pretty printer passed::
+
+    class MyObject(object):
+
+        def _repr_pretty_(self, p, cycle):
+            ...
+
+Here is an example implementation of a `_repr_pretty_` method for a list
+subclass::
+
+    class MyList(list):
+
+        def _repr_pretty_(self, p, cycle):
+            if cycle:
+                p.text('MyList(...)')
+            else:
+                with p.group(8, 'MyList([', '])'):
+                    for idx, item in enumerate(self):
+                        if idx:
+                            p.text(',')
+                            p.breakable()
+                        p.pretty(item)
+
+The `cycle` parameter is `True` if pretty detected a cycle.  You *have* to
+react to that or the result is an infinite loop.  `p.text()` just adds
+non breaking text to the output, `p.breakable()` either adds a whitespace
+or breaks here.  If you pass it an argument it's used instead of the
+default space.  `p.pretty` prettyprints another object using the pretty print
+method.
+
+The first parameter to the `group` function specifies the extra indentation
+of the next line.  In this example the next item will either be on the same
+line (if the items are short enough) or aligned with the right edge of the
+opening bracket of `MyList`.
+
+If you just want to indent something you can use the group function
+without open / close parameters.  You can also use this code::
+
+    with p.indent(2):
+        ...
+
+Inheritance diagram:
+
+.. inheritance-diagram:: yap_ipython.lib.pretty
+   :parts: 3
+
+:copyright: 2007 by Armin Ronacher.
+            Portions (c) 2009 by Robert Kern.
+:license: BSD License.
+"""
+from contextlib import contextmanager
+import sys
+import types
+import re
+import datetime
+from collections import deque
+from io import StringIO
+from warnings import warn
+
+from yap_ipython.utils.decorators import undoc
+from yap_ipython.utils.py3compat import PYPY
+from yap_ipython.utils.signatures import signature
+
+__all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',
+    'for_type', 'for_type_by_name']
+
+
+MAX_SEQ_LENGTH = 1000
+_re_pattern_type = type(re.compile(''))
+
+def _safe_getattr(obj, attr, default=None):
+    """Safe version of getattr.
+    
+    Same as getattr, but will return ``default`` on any Exception,
+    rather than raising.
+    """
+    try:
+        return getattr(obj, attr, default)
+    except Exception:
+        return default
+
+@undoc
+class CUnicodeIO(StringIO):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        warn(("CUnicodeIO is deprecated since yap_ipython 6.0. "
+              "Please use io.StringIO instead."),
+             DeprecationWarning, stacklevel=2)
+
+def _sorted_for_pprint(items):
+    """
+    Sort the given items for pretty printing. Since some predictable
+    sorting is better than no sorting at all, we sort on the string
+    representation if normal sorting fails.
+    """
+    items = list(items)
+    try:
+        return sorted(items)
+    except Exception:
+        try:
+            return sorted(items, key=str)
+        except Exception:
+            return items
+
+def pretty(obj, verbose=False, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
+    """
+    Pretty print the object's representation.
+    """
+    stream = StringIO()
+    printer = RepresentationPrinter(stream, verbose, max_width, newline, max_seq_length=max_seq_length)
+    printer.pretty(obj)
+    printer.flush()
+    return stream.getvalue()
+
+
+def pprint(obj, verbose=False, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
+    """
+    Like `pretty` but print to stdout.
+    """
+    printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline, max_seq_length=max_seq_length)
+    printer.pretty(obj)
+    printer.flush()
+    sys.stdout.write(newline)
+    sys.stdout.flush()
+
+class _PrettyPrinterBase(object):
+
+    @contextmanager
+    def indent(self, indent):
+        """with statement support for indenting/dedenting."""
+        self.indentation += indent
+        try:
+            yield
+        finally:
+            self.indentation -= indent
+
+    @contextmanager
+    def group(self, indent=0, open='', close=''):
+        """like begin_group / end_group but for the with statement."""
+        self.begin_group(indent, open)
+        try:
+            yield
+        finally:
+            self.end_group(indent, close)
+
+class PrettyPrinter(_PrettyPrinterBase):
+    """
+    Baseclass for the `RepresentationPrinter` prettyprinter that is used to
+    generate pretty reprs of objects.  Contrary to the `RepresentationPrinter`
+    this printer knows nothing about the default pprinters or the `_repr_pretty_`
+    callback method.
+    """
+
+    def __init__(self, output, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
+        self.output = output
+        self.max_width = max_width
+        self.newline = newline
+        self.max_seq_length = max_seq_length
+        self.output_width = 0
+        self.buffer_width = 0
+        self.buffer = deque()
+
+        root_group = Group(0)
+        self.group_stack = [root_group]
+        self.group_queue = GroupQueue(root_group)
+        self.indentation = 0
+
+    def _break_outer_groups(self):
+        while self.max_width < self.output_width + self.buffer_width:
+            group = self.group_queue.deq()
+            if not group:
+                return
+            while group.breakables:
+                x = self.buffer.popleft()
+                self.output_width = x.output(self.output, self.output_width)
+                self.buffer_width -= x.width
+            while self.buffer and isinstance(self.buffer[0], Text):
+                x = self.buffer.popleft()
+                self.output_width = x.output(self.output, self.output_width)
+                self.buffer_width -= x.width
+
+    def text(self, obj):
+        """Add literal text to the output."""
+        width = len(obj)
+        if self.buffer:
+            text = self.buffer[-1]
+            if not isinstance(text, Text):
+                text = Text()
+                self.buffer.append(text)
+            text.add(obj, width)
+            self.buffer_width += width
+            self._break_outer_groups()
+        else:
+            self.output.write(obj)
+            self.output_width += width
+
+    def breakable(self, sep=' '):
+        """
+        Add a breakable separator to the output.  This does not mean that it
+        will automatically break here.  If no breaking on this position takes
+        place the `sep` is inserted which default to one space.
+        """
+        width = len(sep)
+        group = self.group_stack[-1]
+        if group.want_break:
+            self.flush()
+            self.output.write(self.newline)
+            self.output.write(' ' * self.indentation)
+            self.output_width = self.indentation
+            self.buffer_width = 0
+        else:
+            self.buffer.append(Breakable(sep, width, self))
+            self.buffer_width += width
+            self._break_outer_groups()
+            
+    def break_(self):
+        """
+        Explicitly insert a newline into the output, maintaining correct indentation.
+        """
+        self.flush()
+        self.output.write(self.newline)
+        self.output.write(' ' * self.indentation)
+        self.output_width = self.indentation
+        self.buffer_width = 0
+        
+
+    def begin_group(self, indent=0, open=''):
+        """
+        Begin a group.  If you want support for python < 2.5 which doesn't has
+        the with statement this is the preferred way:
+
+            p.begin_group(1, '{')
+            ...
+            p.end_group(1, '}')
+
+        The python 2.5 expression would be this:
+
+            with p.group(1, '{', '}'):
+                ...
+
+        The first parameter specifies the indentation for the next line (usually
+        the width of the opening text), the second the opening text.  All
+        parameters are optional.
+        """
+        if open:
+            self.text(open)
+        group = Group(self.group_stack[-1].depth + 1)
+        self.group_stack.append(group)
+        self.group_queue.enq(group)
+        self.indentation += indent
+    
+    def _enumerate(self, seq):
+        """like enumerate, but with an upper limit on the number of items"""
+        for idx, x in enumerate(seq):
+            if self.max_seq_length and idx >= self.max_seq_length:
+                self.text(',')
+                self.breakable()
+                self.text('...')
+                return
+            yield idx, x
+    
+    def end_group(self, dedent=0, close=''):
+        """End a group. See `begin_group` for more details."""
+        self.indentation -= dedent
+        group = self.group_stack.pop()
+        if not group.breakables:
+            self.group_queue.remove(group)
+        if close:
+            self.text(close)
+
+    def flush(self):
+        """Flush data that is left in the buffer."""
+        for data in self.buffer:
+            self.output_width += data.output(self.output, self.output_width)
+        self.buffer.clear()
+        self.buffer_width = 0
+
+
+def _get_mro(obj_class):
+    """ Get a reasonable method resolution order of a class and its superclasses
+    for both old-style and new-style classes.
+    """
+    if not hasattr(obj_class, '__mro__'):
+        # Old-style class. Mix in object to make a fake new-style class.
+        try:
+            obj_class = type(obj_class.__name__, (obj_class, object), {})
+        except TypeError:
+            # Old-style extension type that does not descend from object.
+            # FIXME: try to construct a more thorough MRO.
+            mro = [obj_class]
+        else:
+            mro = obj_class.__mro__[1:-1]
+    else:
+        mro = obj_class.__mro__
+    return mro
+
+
+class RepresentationPrinter(PrettyPrinter):
+    """
+    Special pretty printer that has a `pretty` method that calls the pretty
+    printer for a python object.
+
+    This class stores processing data on `self` so you must *never* use
+    this class in a threaded environment.  Always lock it or reinstanciate
+    it.
+
+    Instances also have a verbose flag callbacks can access to control their
+    output.  For example the default instance repr prints all attributes and
+    methods that are not prefixed by an underscore if the printer is in
+    verbose mode.
+    """
+
+    def __init__(self, output, verbose=False, max_width=79, newline='\n',
+        singleton_pprinters=None, type_pprinters=None, deferred_pprinters=None,
+        max_seq_length=MAX_SEQ_LENGTH):
+
+        PrettyPrinter.__init__(self, output, max_width, newline, max_seq_length=max_seq_length)
+        self.verbose = verbose
+        self.stack = []
+        if singleton_pprinters is None:
+            singleton_pprinters = _singleton_pprinters.copy()
+        self.singleton_pprinters = singleton_pprinters
+        if type_pprinters is None:
+            type_pprinters = _type_pprinters.copy()
+        self.type_pprinters = type_pprinters
+        if deferred_pprinters is None:
+            deferred_pprinters = _deferred_type_pprinters.copy()
+        self.deferred_pprinters = deferred_pprinters
+
+    def pretty(self, obj):
+        """Pretty print the given object."""
+        obj_id = id(obj)
+        cycle = obj_id in self.stack
+        self.stack.append(obj_id)
+        self.begin_group()
+        try:
+            obj_class = _safe_getattr(obj, '__class__', None) or type(obj)
+            # First try to find registered singleton printers for the type.
+            try:
+                printer = self.singleton_pprinters[obj_id]
+            except (TypeError, KeyError):
+                pass
+            else:
+                return printer(obj, self, cycle)
+            # Next walk the mro and check for either:
+            #   1) a registered printer
+            #   2) a _repr_pretty_ method
+            for cls in _get_mro(obj_class):
+                if cls in self.type_pprinters:
+                    # printer registered in self.type_pprinters
+                    return self.type_pprinters[cls](obj, self, cycle)
+                else:
+                    # deferred printer
+                    printer = self._in_deferred_types(cls)
+                    if printer is not None:
+                        return printer(obj, self, cycle)
+                    else:
+                        # Finally look for special method names.
+                        # Some objects automatically create any requested
+                        # attribute. Try to ignore most of them by checking for
+                        # callability.
+                        if '_repr_pretty_' in cls.__dict__:
+                            meth = cls._repr_pretty_
+                            if callable(meth):
+                                return meth(obj, self, cycle)
+            return _default_pprint(obj, self, cycle)
+        finally:
+            self.end_group()
+            self.stack.pop()
+
+    def _in_deferred_types(self, cls):
+        """
+        Check if the given class is specified in the deferred type registry.
+
+        Returns the printer from the registry if it exists, and None if the
+        class is not in the registry. Successful matches will be moved to the
+        regular type registry for future use.
+        """
+        mod = _safe_getattr(cls, '__module__', None)
+        name = _safe_getattr(cls, '__name__', None)
+        key = (mod, name)
+        printer = None
+        if key in self.deferred_pprinters:
+            # Move the printer over to the regular registry.
+            printer = self.deferred_pprinters.pop(key)
+            self.type_pprinters[cls] = printer
+        return printer
+
+
+class Printable(object):
+
+    def output(self, stream, output_width):
+        return output_width
+
+
+class Text(Printable):
+
+    def __init__(self):
+        self.objs = []
+        self.width = 0
+
+    def output(self, stream, output_width):
+        for obj in self.objs:
+            stream.write(obj)
+        return output_width + self.width
+
+    def add(self, obj, width):
+        self.objs.append(obj)
+        self.width += width
+
+
+class Breakable(Printable):
+
+    def __init__(self, seq, width, pretty):
+        self.obj = seq
+        self.width = width
+        self.pretty = pretty
+        self.indentation = pretty.indentation
+        self.group = pretty.group_stack[-1]
+        self.group.breakables.append(self)
+
+    def output(self, stream, output_width):
+        self.group.breakables.popleft()
+        if self.group.want_break:
+            stream.write(self.pretty.newline)
+            stream.write(' ' * self.indentation)
+            return self.indentation
+        if not self.group.breakables:
+            self.pretty.group_queue.remove(self.group)
+        stream.write(self.obj)
+        return output_width + self.width
+
+
+class Group(Printable):
+
+    def __init__(self, depth):
+        self.depth = depth
+        self.breakables = deque()
+        self.want_break = False
+
+
+class GroupQueue(object):
+
+    def __init__(self, *groups):
+        self.queue = []
+        for group in groups:
+            self.enq(group)
+
+    def enq(self, group):
+        depth = group.depth
+        while depth > len(self.queue) - 1:
+            self.queue.append([])
+        self.queue[depth].append(group)
+
+    def deq(self):
+        for stack in self.queue:
+            for idx, group in enumerate(reversed(stack)):
+                if group.breakables:
+                    del stack[idx]
+                    group.want_break = True
+                    return group
+            for group in stack:
+                group.want_break = True
+            del stack[:]
+
+    def remove(self, group):
+        try:
+            self.queue[group.depth].remove(group)
+        except ValueError:
+            pass
+
+
+def _default_pprint(obj, p, cycle):
+    """
+    The default print function.  Used if an object does not provide one and
+    it's none of the builtin objects.
+    """
+    klass = _safe_getattr(obj, '__class__', None) or type(obj)
+    if _safe_getattr(klass, '__repr__', None) is not object.__repr__:
+        # A user-provided repr. Find newlines and replace them with p.break_()
+        _repr_pprint(obj, p, cycle)
+        return
+    p.begin_group(1, '<')
+    p.pretty(klass)
+    p.text(' at 0x%x' % id(obj))
+    if cycle:
+        p.text(' ...')
+    elif p.verbose:
+        first = True
+        for key in dir(obj):
+            if not key.startswith('_'):
+                try:
+                    value = getattr(obj, key)
+                except AttributeError:
+                    continue
+                if isinstance(value, types.MethodType):
+                    continue
+                if not first:
+                    p.text(',')
+                p.breakable()
+                p.text(key)
+                p.text('=')
+                step = len(key) + 1
+                p.indentation += step
+                p.pretty(value)
+                p.indentation -= step
+                first = False
+    p.end_group(1, '>')
+
+
+def _seq_pprinter_factory(start, end, basetype):
+    """
+    Factory that returns a pprint function useful for sequences.  Used by
+    the default pprint for tuples, dicts, and lists.
+    """
+    def inner(obj, p, cycle):
+        typ = type(obj)
+        if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
+            # If the subclass provides its own repr, use it instead.
+            return p.text(typ.__repr__(obj))
+
+        if cycle:
+            return p.text(start + '...' + end)
+        step = len(start)
+        p.begin_group(step, start)
+        for idx, x in p._enumerate(obj):
+            if idx:
+                p.text(',')
+                p.breakable()
+            p.pretty(x)
+        if len(obj) == 1 and type(obj) is tuple:
+            # Special case for 1-item tuples.
+            p.text(',')
+        p.end_group(step, end)
+    return inner
+
+
+def _set_pprinter_factory(start, end, basetype):
+    """
+    Factory that returns a pprint function useful for sets and frozensets.
+    """
+    def inner(obj, p, cycle):
+        typ = type(obj)
+        if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
+            # If the subclass provides its own repr, use it instead.
+            return p.text(typ.__repr__(obj))
+
+        if cycle:
+            return p.text(start + '...' + end)
+        if len(obj) == 0:
+            # Special case.
+            p.text(basetype.__name__ + '()')
+        else:
+            step = len(start)
+            p.begin_group(step, start)
+            # Like dictionary keys, we will try to sort the items if there aren't too many
+            if not (p.max_seq_length and len(obj) >= p.max_seq_length):
+                items = _sorted_for_pprint(obj)
+            else:
+                items = obj
+            for idx, x in p._enumerate(items):
+                if idx:
+                    p.text(',')
+                    p.breakable()
+                p.pretty(x)
+            p.end_group(step, end)
+    return inner
+
+
+def _dict_pprinter_factory(start, end, basetype=None):
+    """
+    Factory that returns a pprint function used by the default pprint of
+    dicts and dict proxies.
+    """
+    def inner(obj, p, cycle):
+        typ = type(obj)
+        if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
+            # If the subclass provides its own repr, use it instead.
+            return p.text(typ.__repr__(obj))
+
+        if cycle:
+            return p.text('{...}')
+        step = len(start)
+        p.begin_group(step, start)
+        keys = obj.keys()
+        # if dict isn't large enough to be truncated, sort keys before displaying
+        if not (p.max_seq_length and len(obj) >= p.max_seq_length):
+            keys = _sorted_for_pprint(keys)
+        for idx, key in p._enumerate(keys):
+            if idx:
+                p.text(',')
+                p.breakable()
+            p.pretty(key)
+            p.text(': ')
+            p.pretty(obj[key])
+        p.end_group(step, end)
+    return inner
+
+
+def _super_pprint(obj, p, cycle):
+    """The pprint for the super type."""
+    p.begin_group(8, '<super: ')
+    p.pretty(obj.__thisclass__)
+    p.text(',')
+    p.breakable()
+    if PYPY: # In PyPy, super() objects don't have __self__ attributes
+        dself = obj.__repr__.__self__
+        p.pretty(None if dself is obj else dself)
+    else:
+        p.pretty(obj.__self__)
+    p.end_group(8, '>')
+
+
+def _re_pattern_pprint(obj, p, cycle):
+    """The pprint function for regular expression patterns."""
+    p.text('re.compile(')
+    pattern = repr(obj.pattern)
+    if pattern[:1] in 'uU':
+        pattern = pattern[1:]
+        prefix = 'ur'
+    else:
+        prefix = 'r'
+    pattern = prefix + pattern.replace('\\\\', '\\')
+    p.text(pattern)
+    if obj.flags:
+        p.text(',')
+        p.breakable()
+        done_one = False
+        for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL',
+            'UNICODE', 'VERBOSE', 'DEBUG'):
+            if obj.flags & getattr(re, flag):
+                if done_one:
+                    p.text('|')
+                p.text('re.' + flag)
+                done_one = True
+    p.text(')')
+
+
+def _type_pprint(obj, p, cycle):
+    """The pprint for classes and types."""
+    # Heap allocated types might not have the module attribute,
+    # and others may set it to None.
+
+    # Checks for a __repr__ override in the metaclass. Can't compare the
+    # type(obj).__repr__ directly because in PyPy the representation function
+    # inherited from type isn't the same type.__repr__
+    if [m for m in _get_mro(type(obj)) if "__repr__" in vars(m)][:1] != [type]:
+        _repr_pprint(obj, p, cycle)
+        return
+
+    mod = _safe_getattr(obj, '__module__', None)
+    try:
+        name = obj.__qualname__
+        if not isinstance(name, str):
+            # This can happen if the type implements __qualname__ as a property
+            # or other descriptor in Python 2.
+            raise Exception("Try __name__")
+    except Exception:
+        name = obj.__name__
+        if not isinstance(name, str):
+            name = '<unknown type>'
+
+    if mod in (None, '__builtin__', 'builtins', 'exceptions'):
+        p.text(name)
+    else:
+        p.text(mod + '.' + name)
+
+
+def _repr_pprint(obj, p, cycle):
+    """A pprint that just redirects to the normal repr function."""
+    # Find newlines and replace them with p.break_()
+    output = repr(obj)
+    for idx,output_line in enumerate(output.splitlines()):
+        if idx:
+            p.break_()
+        p.text(output_line)
+
+
+def _function_pprint(obj, p, cycle):
+    """Base pprint for all functions and builtin functions."""
+    name = _safe_getattr(obj, '__qualname__', obj.__name__)
+    mod = obj.__module__
+    if mod and mod not in ('__builtin__', 'builtins', 'exceptions'):
+        name = mod + '.' + name
+    try:
+       func_def = name + str(signature(obj))
+    except ValueError:
+       func_def = name
+    p.text('<function %s>' % func_def)
+
+
+def _exception_pprint(obj, p, cycle):
+    """Base pprint for all exceptions."""
+    name = getattr(obj.__class__, '__qualname__', obj.__class__.__name__)
+    if obj.__class__.__module__ not in ('exceptions', 'builtins'):
+        name = '%s.%s' % (obj.__class__.__module__, name)
+    step = len(name) + 1
+    p.begin_group(step, name + '(')
+    for idx, arg in enumerate(getattr(obj, 'args', ())):
+        if idx:
+            p.text(',')
+            p.breakable()
+        p.pretty(arg)
+    p.end_group(step, ')')
+
+
+#: the exception base
+try:
+    _exception_base = BaseException
+except NameError:
+    _exception_base = Exception
+
+
+#: printers for builtin types
+_type_pprinters = {
+    int:                        _repr_pprint,
+    float:                      _repr_pprint,
+    str:                        _repr_pprint,
+    tuple:                      _seq_pprinter_factory('(', ')', tuple),
+    list:                       _seq_pprinter_factory('[', ']', list),
+    dict:                       _dict_pprinter_factory('{', '}', dict),
+    
+    set:                        _set_pprinter_factory('{', '}', set),
+    frozenset:                  _set_pprinter_factory('frozenset({', '})', frozenset),
+    super:                      _super_pprint,
+    _re_pattern_type:           _re_pattern_pprint,
+    type:                       _type_pprint,
+    types.FunctionType:         _function_pprint,
+    types.BuiltinFunctionType:  _function_pprint,
+    types.MethodType:           _repr_pprint,
+    
+    datetime.datetime:          _repr_pprint,
+    datetime.timedelta:         _repr_pprint,
+    _exception_base:            _exception_pprint
+}
+
+try:
+    # In PyPy, types.DictProxyType is dict, setting the dictproxy printer
+    # using dict.setdefault avoids overwritting the dict printer
+    _type_pprinters.setdefault(types.DictProxyType,
+                               _dict_pprinter_factory('dict_proxy({', '})'))
+    _type_pprinters[types.ClassType] = _type_pprint
+    _type_pprinters[types.SliceType] = _repr_pprint
+except AttributeError: # Python 3
+    _type_pprinters[types.MappingProxyType] = \
+        _dict_pprinter_factory('mappingproxy({', '})')
+    _type_pprinters[slice] = _repr_pprint
+    
+try:
+    _type_pprinters[long] = _repr_pprint
+    _type_pprinters[unicode] = _repr_pprint
+except NameError:
+    _type_pprinters[range] = _repr_pprint
+    _type_pprinters[bytes] = _repr_pprint
+
+#: printers for types specified by name
+_deferred_type_pprinters = {
+}
+
+def for_type(typ, func):
+    """
+    Add a pretty printer for a given type.
+    """
+    oldfunc = _type_pprinters.get(typ, None)
+    if func is not None:
+        # To support easy restoration of old pprinters, we need to ignore Nones.
+        _type_pprinters[typ] = func
+    return oldfunc
+
+def for_type_by_name(type_module, type_name, func):
+    """
+    Add a pretty printer for a type specified by the module and name of a type
+    rather than the type object itself.
+    """
+    key = (type_module, type_name)
+    oldfunc = _deferred_type_pprinters.get(key, None)
+    if func is not None:
+        # To support easy restoration of old pprinters, we need to ignore Nones.
+        _deferred_type_pprinters[key] = func
+    return oldfunc
+
+
+#: printers for the default singletons
+_singleton_pprinters = dict.fromkeys(map(id, [None, True, False, Ellipsis,
+                                      NotImplemented]), _repr_pprint)
+
+
+def _defaultdict_pprint(obj, p, cycle):
+    name = obj.__class__.__name__
+    with p.group(len(name) + 1, name + '(', ')'):
+        if cycle:
+            p.text('...')
+        else:
+            p.pretty(obj.default_factory)
+            p.text(',')
+            p.breakable()
+            p.pretty(dict(obj))
+
+def _ordereddict_pprint(obj, p, cycle):
+    name = obj.__class__.__name__
+    with p.group(len(name) + 1, name + '(', ')'):
+        if cycle:
+            p.text('...')
+        elif len(obj):
+            p.pretty(list(obj.items()))
+
+def _deque_pprint(obj, p, cycle):
+    name = obj.__class__.__name__
+    with p.group(len(name) + 1, name + '(', ')'):
+        if cycle:
+            p.text('...')
+        else:
+            p.pretty(list(obj))
+
+
+def _counter_pprint(obj, p, cycle):
+    name = obj.__class__.__name__
+    with p.group(len(name) + 1, name + '(', ')'):
+        if cycle:
+            p.text('...')
+        elif len(obj):
+            p.pretty(dict(obj))
+
+for_type_by_name('collections', 'defaultdict', _defaultdict_pprint)
+for_type_by_name('collections', 'OrderedDict', _ordereddict_pprint)
+for_type_by_name('collections', 'deque', _deque_pprint)
+for_type_by_name('collections', 'Counter', _counter_pprint)
+
+if __name__ == '__main__':
+    from random import randrange
+    class Foo(object):
+        def __init__(self):
+            self.foo = 1
+            self.bar = re.compile(r'\s+')
+            self.blub = dict.fromkeys(range(30), randrange(1, 40))
+            self.hehe = 23424.234234
+            self.list = ["blub", "blah", self]
+
+        def get_foo(self):
+            print("foo")
+
+    pprint(Foo(), verbose=True)

packages/python/yap_kernel/yap_ipython/lib/security.py

@@ -0,0 +1,114 @@
+"""
+Password generation for the yap_ipython notebook.
+"""
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+# Stdlib
+import getpass
+import hashlib
+import random
+
+# Our own
+from yap_ipython.core.error import UsageError
+from yap_ipython.utils.py3compat import encode
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# Length of the salt in nr of hex chars, which implies salt_len * 4
+# bits of randomness.
+salt_len = 12
+
+#-----------------------------------------------------------------------------
+# Functions
+#-----------------------------------------------------------------------------
+
+def passwd(passphrase=None, algorithm='sha1'):
+    """Generate hashed password and salt for use in notebook configuration.
+
+    In the notebook configuration, set `c.NotebookApp.password` to
+    the generated string.
+
+    Parameters
+    ----------
+    passphrase : str
+        Password to hash.  If unspecified, the user is asked to input
+        and verify a password.
+    algorithm : str
+        Hashing algorithm to use (e.g, 'sha1' or any argument supported
+        by :func:`hashlib.new`).
+
+    Returns
+    -------
+    hashed_passphrase : str
+        Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'.
+
+    Examples
+    --------
+    >>> passwd('mypassword')
+    'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12'
+
+    """
+    if passphrase is None:
+        for i in range(3):
+            p0 = getpass.getpass('Enter password: ')
+            p1 = getpass.getpass('Verify password: ')
+            if p0 == p1:
+                passphrase = p0
+                break
+            else:
+                print('Passwords do not match.')
+        else:
+            raise UsageError('No matching passwords found. Giving up.')
+
+    h = hashlib.new(algorithm)
+    salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len)
+    h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
+
+    return ':'.join((algorithm, salt, h.hexdigest()))
+
+
+def passwd_check(hashed_passphrase, passphrase):
+    """Verify that a given passphrase matches its hashed version.
+
+    Parameters
+    ----------
+    hashed_passphrase : str
+        Hashed password, in the format returned by `passwd`.
+    passphrase : str
+        Passphrase to validate.
+
+    Returns
+    -------
+    valid : bool
+        True if the passphrase matches the hash.
+
+    Examples
+    --------
+    >>> from yap_ipython.lib.security import passwd_check
+    >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
+    ...              'mypassword')
+    True
+
+    >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
+    ...              'anotherpassword')
+    False
+    """
+    try:
+        algorithm, salt, pw_digest = hashed_passphrase.split(':', 2)
+    except (ValueError, TypeError):
+        return False
+
+    try:
+        h = hashlib.new(algorithm)
+    except ValueError:
+        return False
+
+    if len(pw_digest) == 0:
+        return False
+
+    h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
+
+    return h.hexdigest() == pw_digest