Add tqdm v4.21.0

lib/tqdm/__init__.py

@@ -0,0 +1,34 @@
+from ._tqdm import tqdm
+from ._tqdm import trange
+from ._tqdm_gui import tqdm_gui
+from ._tqdm_gui import tgrange
+from ._tqdm_pandas import tqdm_pandas
+from ._main import main
+from ._monitor import TMonitor, TqdmSynchronisationWarning
+from ._version import __version__  # NOQA
+from ._tqdm import TqdmTypeError, TqdmKeyError, TqdmWarning, \
+    TqdmDeprecationWarning, TqdmExperimentalWarning, \
+    TqdmMonitorWarning
+
+__all__ = ['tqdm', 'tqdm_gui', 'trange', 'tgrange', 'tqdm_pandas',
+           'tqdm_notebook', 'tnrange', 'main', 'TMonitor',
+           'TqdmTypeError', 'TqdmKeyError',
+           'TqdmWarning', 'TqdmDeprecationWarning',
+           'TqdmExperimentalWarning',
+           'TqdmMonitorWarning', 'TqdmSynchronisationWarning',
+           '__version__']
+
+
+def tqdm_notebook(*args, **kwargs):  # pragma: no cover
+    """See tqdm._tqdm_notebook.tqdm_notebook for full documentation"""
+    from ._tqdm_notebook import tqdm_notebook as _tqdm_notebook
+    return _tqdm_notebook(*args, **kwargs)
+
+
+def tnrange(*args, **kwargs):  # pragma: no cover
+    """
+    A shortcut for tqdm_notebook(xrange(*args), **kwargs).
+    On Python3+ range is used instead of xrange.
+    """
+    from ._tqdm_notebook import tnrange as _tnrange
+    return _tnrange(*args, **kwargs)

lib/tqdm/__main__.py

@@ -0,0 +1,2 @@
+from ._main import main
+main()

lib/tqdm/_main.py

@@ -0,0 +1,207 @@
+from ._tqdm import tqdm, TqdmTypeError, TqdmKeyError
+from ._version import __version__  # NOQA
+import sys
+import re
+import logging
+__all__ = ["main"]
+
+
+def cast(val, typ):
+    log = logging.getLogger(__name__)
+    log.debug((val, typ))
+    if " or " in typ:
+        for t in typ.split(" or "):
+            try:
+                return cast(val, t)
+            except TqdmTypeError:
+                pass
+        raise TqdmTypeError(val + ' : ' + typ)
+
+    # sys.stderr.write('\ndebug | `val:type`: `' + val + ':' + typ + '`.\n')
+    if typ == 'bool':
+        if (val == 'True') or (val == ''):
+            return True
+        elif val == 'False':
+            return False
+        else:
+            raise TqdmTypeError(val + ' : ' + typ)
+    try:
+        return eval(typ + '("' + val + '")')
+    except:
+        if typ == 'chr':
+            return chr(ord(eval('"' + val + '"')))
+        else:
+            raise TqdmTypeError(val + ' : ' + typ)
+
+
+def posix_pipe(fin, fout, delim='\n', buf_size=256,
+               callback=lambda int: None  # pragma: no cover
+               ):
+    """
+    Params
+    ------
+    fin  : file with `read(buf_size : int)` method
+    fout  : file with `write` (and optionally `flush`) methods.
+    callback  : function(int), e.g.: `tqdm.update`
+    """
+    fp_write = fout.write
+
+    # tmp = ''
+    if not delim:
+        while True:
+            tmp = fin.read(buf_size)
+
+            # flush at EOF
+            if not tmp:
+                getattr(fout, 'flush', lambda: None)()  # pragma: no cover
+                return
+
+            fp_write(tmp)
+            callback(len(tmp))
+        # return
+
+    buf = ''
+    # n = 0
+    while True:
+        tmp = fin.read(buf_size)
+
+        # flush at EOF
+        if not tmp:
+            if buf:
+                fp_write(buf)
+                callback(1 + buf.count(delim))  # n += 1 + buf.count(delim)
+            getattr(fout, 'flush', lambda: None)()  # pragma: no cover
+            return  # n
+
+        while True:
+            try:
+                i = tmp.index(delim)
+            except ValueError:
+                buf += tmp
+                break
+            else:
+                fp_write(buf + tmp[:i + len(delim)])
+                callback(1)  # n += 1
+                buf = ''
+                tmp = tmp[i + len(delim):]
+
+
+# ((opt, type), ... )
+RE_OPTS = re.compile(r'\n {8}(\S+)\s{2,}:\s*([^,]+)')
+# better split method assuming no positional args
+RE_SHLEX = re.compile(r'\s*(?<!\S)--?([^\s=]+)(?:\s*|=|$)')
+
+# TODO: add custom support for some of the following?
+UNSUPPORTED_OPTS = ('iterable', 'gui', 'out', 'file')
+
+# The 8 leading spaces are required for consistency
+CLI_EXTRA_DOC = r"""
+        Extra CLI Options
+        -----------------
+        name  : type, optional
+             TODO: find out why this is needed.
+        delim  : chr, optional
+            Delimiting character [default: '\n']. Use '\0' for null.
+            N.B.: on Windows systems, Python converts '\n' to '\r\n'.
+        buf_size  : int, optional
+            String buffer size in bytes [default: 256]
+            used when `delim` is specified.
+        bytes  : bool, optional
+            If true, will count bytes, ignore `delim`, and default
+            `unit_scale` to True, `unit_divisor` to 1024, and `unit` to 'B'.
+        log  : str, optional
+            CRITICAL|FATAL|ERROR|WARN(ING)|[default: 'INFO']|DEBUG|NOTSET.
+"""
+
+
+def main(fp=sys.stderr):
+    """
+    Paramters (internal use only)
+    ---------
+    fp  : file-like object for tqdm
+    """
+    try:
+        log = sys.argv.index('--log')
+    except ValueError:
+        logLevel = 'INFO'
+    else:
+        # sys.argv.pop(log)
+        # logLevel = sys.argv.pop(log)
+        logLevel = sys.argv[log + 1]
+    logging.basicConfig(level=getattr(logging, logLevel),
+        format="%(levelname)s:%(module)s:%(lineno)d:%(message)s")
+    log = logging.getLogger(__name__)
+
+    d = tqdm.__init__.__doc__ + CLI_EXTRA_DOC
+
+    opt_types = dict(RE_OPTS.findall(d))
+    # opt_types['delim'] = 'chr'
+
+    for o in UNSUPPORTED_OPTS:
+        opt_types.pop(o)
+
+    log.debug(sorted(opt_types.items()))
+
+    # d = RE_OPTS.sub(r'  --\1=<\1>  : \2', d)
+    split = RE_OPTS.split(d)
+    opt_types_desc = zip(split[1::3], split[2::3], split[3::3])
+    d = ''.join('\n  --{0}=<{0}>  : {1}{2}'.format(*otd)
+                for otd in opt_types_desc if otd[0] not in UNSUPPORTED_OPTS)
+
+    d = """Usage:
+  tqdm [--help | options]
+
+Options:
+  -h, --help     Print this help and exit
+  -v, --version  Print version and exit
+
+""" + d.strip('\n') + '\n'
+
+    # opts = docopt(d, version=__version__)
+    if any(v in sys.argv for v in ('-v', '--version')):
+        sys.stdout.write(__version__ + '\n')
+        sys.exit(0)
+    elif any(v in sys.argv for v in ('-h', '--help')):
+        sys.stdout.write(d + '\n')
+        sys.exit(0)
+
+    argv = RE_SHLEX.split(' '.join(["tqdm"] + sys.argv[1:]))
+    opts = dict(zip(argv[1::2], argv[2::2]))
+
+    log.debug(opts)
+    opts.pop('log', True)
+
+    tqdm_args = {'file': fp}
+    try:
+        for (o, v) in opts.items():
+            try:
+                tqdm_args[o] = cast(v, opt_types[o])
+            except KeyError as e:
+                raise TqdmKeyError(str(e))
+        log.debug('args:' + str(tqdm_args))
+    except:
+        fp.write('\nError:\nUsage:\n  tqdm [--help | options]\n')
+        for i in sys.stdin:
+            sys.stdout.write(i)
+        raise
+    else:
+        buf_size = tqdm_args.pop('buf_size', 256)
+        delim = tqdm_args.pop('delim', '\n')
+        delim_per_char = tqdm_args.pop('bytes', False)
+        if delim_per_char:
+            tqdm_args.setdefault('unit', 'B')
+            tqdm_args.setdefault('unit_scale', True)
+            tqdm_args.setdefault('unit_divisor', 1024)
+            log.debug(tqdm_args)
+            with tqdm(**tqdm_args) as t:
+                posix_pipe(sys.stdin, sys.stdout,
+                           '', buf_size, t.update)
+        elif delim == '\n':
+            log.debug(tqdm_args)
+            for i in tqdm(sys.stdin, **tqdm_args):
+                sys.stdout.write(i)
+        else:
+            log.debug(tqdm_args)
+            with tqdm(**tqdm_args) as t:
+                posix_pipe(sys.stdin, sys.stdout,
+                           delim, buf_size, t.update)

lib/tqdm/_monitor.py

@@ -0,0 +1,93 @@
+from threading import Event, Thread
+from time import time
+from warnings import warn
+__all__ = ["TMonitor", "TqdmSynchronisationWarning"]
+
+
+class TqdmSynchronisationWarning(RuntimeWarning):
+    """tqdm multi-thread/-process errors which may cause incorrect nesting
+    but otherwise no adverse effects"""
+    pass
+
+
+class TMonitor(Thread):
+    """
+    Monitoring thread for tqdm bars.
+    Monitors if tqdm bars are taking too much time to display
+    and readjusts miniters automatically if necessary.
+
+    Parameters
+    ----------
+    tqdm_cls  : class
+        tqdm class to use (can be core tqdm or a submodule).
+    sleep_interval  : fload
+        Time to sleep between monitoring checks.
+    """
+
+    # internal vars for unit testing
+    _time = None
+    _event = None
+
+    def __init__(self, tqdm_cls, sleep_interval):
+        Thread.__init__(self)
+        self.daemon = True  # kill thread when main killed (KeyboardInterrupt)
+        self.was_killed = Event()
+        self.woken = 0  # last time woken up, to sync with monitor
+        self.tqdm_cls = tqdm_cls
+        self.sleep_interval = sleep_interval
+        if TMonitor._time is not None:
+            self._time = TMonitor._time
+        else:
+            self._time = time
+        if TMonitor._event is not None:
+            self._event = TMonitor._event
+        else:
+            self._event = Event
+        self.start()
+
+    def exit(self):
+        self.was_killed.set()
+        self.join()
+        return self.report()
+
+    def run(self):
+        cur_t = self._time()
+        while True:
+            # After processing and before sleeping, notify that we woke
+            # Need to be done just before sleeping
+            self.woken = cur_t
+            # Sleep some time...
+            self.was_killed.wait(self.sleep_interval)
+            # Quit if killed
+            if self.was_killed.is_set():
+                return
+            # Then monitor!
+            # Acquire lock (to access _instances)
+            with self.tqdm_cls.get_lock():
+                cur_t = self._time()
+                # Check tqdm instances are waiting too long to print
+                instances = self.tqdm_cls._instances.copy()
+                for instance in instances:
+                    # Check event in loop to reduce blocking time on exit
+                    if self.was_killed.is_set():
+                        return
+                    # Avoid race by checking that the instance started
+                    if not hasattr(instance, 'start_t'):  # pragma: nocover
+                        continue
+                    # Only if mininterval > 1 (else iterations are just slow)
+                    # and last refresh exceeded maxinterval
+                    if instance.miniters > 1 and \
+                            (cur_t - instance.last_print_t) >= \
+                            instance.maxinterval:
+                        # force bypassing miniters on next iteration
+                        # (dynamic_miniters adjusts mininterval automatically)
+                        instance.miniters = 1
+                        # Refresh now! (works only for manual tqdm)
+                        instance.refresh(nolock=True)
+                if instances != self.tqdm_cls._instances:  # pragma: nocover
+                    warn("Set changed size during iteration" +
+                         " (see https://github.com/tqdm/tqdm/issues/481)",
+                         TqdmSynchronisationWarning)
+
+    def report(self):
+        return not self.was_killed.is_set()

lib/tqdm/_tqdm_gui.py

@@ -0,0 +1,351 @@
+"""
+GUI progressbar decorator for iterators.
+Includes a default (x)range iterator printing to stderr.
+
+Usage:
+  >>> from tqdm_gui import tgrange[, tqdm_gui]
+  >>> for i in tgrange(10): #same as: for i in tqdm_gui(xrange(10))
+  ...     ...
+"""
+# future division is important to divide integers and get as
+# a result precise floating numbers (instead of truncated int)
+from __future__ import division, absolute_import
+# import compatibility functions and utilities
+# import sys
+from time import time
+from ._utils import _range
+# to inherit from the tqdm class
+from ._tqdm import tqdm, TqdmExperimentalWarning
+from warnings import warn
+
+
+__author__ = {"github.com/": ["casperdcl", "lrq3000"]}
+__all__ = ['tqdm_gui', 'tgrange']
+
+
+class tqdm_gui(tqdm):  # pragma: no cover
+    """
+    Experimental GUI version of tqdm!
+    """
+
+    # TODO: @classmethod: write() on GUI?
+
+    def __init__(self, *args, **kwargs):
+        import matplotlib as mpl
+        import matplotlib.pyplot as plt
+        from collections import deque
+        kwargs['gui'] = True
+
+        super(tqdm_gui, self).__init__(*args, **kwargs)
+
+        # Initialize the GUI display
+        if self.disable or not kwargs['gui']:
+            return
+
+        warn('GUI is experimental/alpha', TqdmExperimentalWarning)
+        self.mpl = mpl
+        self.plt = plt
+        self.sp = None
+
+        # Remember if external environment uses toolbars
+        self.toolbar = self.mpl.rcParams['toolbar']
+        self.mpl.rcParams['toolbar'] = 'None'
+
+        self.mininterval = max(self.mininterval, 0.5)
+        self.fig, ax = plt.subplots(figsize=(9, 2.2))
+        # self.fig.subplots_adjust(bottom=0.2)
+        if self.total:
+            self.xdata = []
+            self.ydata = []
+            self.zdata = []
+        else:
+            self.xdata = deque([])
+            self.ydata = deque([])
+            self.zdata = deque([])
+        self.line1, = ax.plot(self.xdata, self.ydata, color='b')
+        self.line2, = ax.plot(self.xdata, self.zdata, color='k')
+        ax.set_ylim(0, 0.001)
+        if self.total:
+            ax.set_xlim(0, 100)
+            ax.set_xlabel('percent')
+            self.fig.legend((self.line1, self.line2), ('cur', 'est'),
+                            loc='center right')
+            # progressbar
+            self.hspan = plt.axhspan(0, 0.001,
+                                     xmin=0, xmax=0, color='g')
+        else:
+            # ax.set_xlim(-60, 0)
+            ax.set_xlim(0, 60)
+            ax.invert_xaxis()
+            ax.set_xlabel('seconds')
+            ax.legend(('cur', 'est'), loc='lower left')
+        ax.grid()
+        # ax.set_xlabel('seconds')
+        ax.set_ylabel((self.unit if self.unit else 'it') + '/s')
+        if self.unit_scale:
+            plt.ticklabel_format(style='sci', axis='y',
+                                 scilimits=(0, 0))
+            ax.yaxis.get_offset_text().set_x(-0.15)
+
+        # Remember if external environment is interactive
+        self.wasion = plt.isinteractive()
+        plt.ion()
+        self.ax = ax
+
+    def __iter__(self):
+        # TODO: somehow allow the following:
+        # if not self.gui:
+        #   return super(tqdm_gui, self).__iter__()
+        iterable = self.iterable
+        if self.disable:
+            for obj in iterable:
+                yield obj
+            return
+
+        # ncols = self.ncols
+        mininterval = self.mininterval
+        maxinterval = self.maxinterval
+        miniters = self.miniters
+        dynamic_miniters = self.dynamic_miniters
+        unit = self.unit
+        unit_scale = self.unit_scale
+        ascii = self.ascii
+        start_t = self.start_t
+        last_print_t = self.last_print_t
+        last_print_n = self.last_print_n
+        n = self.n
+        # dynamic_ncols = self.dynamic_ncols
+        smoothing = self.smoothing
+        avg_time = self.avg_time
+        bar_format = self.bar_format
+
+        plt = self.plt
+        ax = self.ax
+        xdata = self.xdata
+        ydata = self.ydata
+        zdata = self.zdata
+        line1 = self.line1
+        line2 = self.line2
+
+        for obj in iterable:
+            yield obj
+            # Update and print the progressbar.
+            # Note: does not call self.update(1) for speed optimisation.
+            n += 1
+            delta_it = n - last_print_n
+            # check the counter first (avoid calls to time())
+            if delta_it >= miniters:
+                cur_t = time()
+                delta_t = cur_t - last_print_t
+                if delta_t >= mininterval:
+                    elapsed = cur_t - start_t
+                    # EMA (not just overall average)
+                    if smoothing and delta_t:
+                        avg_time = delta_t / delta_it \
+                            if avg_time is None \
+                            else smoothing * delta_t / delta_it + \
+                            (1 - smoothing) * avg_time
+
+                    # Inline due to multiple calls
+                    total = self.total
+                    # instantaneous rate
+                    y = delta_it / delta_t
+                    # overall rate
+                    z = n / elapsed
+                    # update line data
+                    xdata.append(n * 100.0 / total if total else cur_t)
+                    ydata.append(y)
+                    zdata.append(z)
+
+                    # Discard old values
+                    # xmin, xmax = ax.get_xlim()
+                    # if (not total) and elapsed > xmin * 1.1:
+                    if (not total) and elapsed > 66:
+                        xdata.popleft()
+                        ydata.popleft()
+                        zdata.popleft()
+
+                    ymin, ymax = ax.get_ylim()
+                    if y > ymax or z > ymax:
+                        ymax = 1.1 * y
+                        ax.set_ylim(ymin, ymax)
+                        ax.figure.canvas.draw()
+
+                    if total:
+                        line1.set_data(xdata, ydata)
+                        line2.set_data(xdata, zdata)
+                        try:
+                            poly_lims = self.hspan.get_xy()
+                        except AttributeError:
+                            self.hspan = plt.axhspan(0, 0.001, xmin=0,
+                                                     xmax=0, color='g')
+                            poly_lims = self.hspan.get_xy()
+                        poly_lims[0, 1] = ymin
+                        poly_lims[1, 1] = ymax
+                        poly_lims[2] = [n / total, ymax]
+                        poly_lims[3] = [poly_lims[2, 0], ymin]
+                        if len(poly_lims) > 4:
+                            poly_lims[4, 1] = ymin
+                        self.hspan.set_xy(poly_lims)
+                    else:
+                        t_ago = [cur_t - i for i in xdata]
+                        line1.set_data(t_ago, ydata)
+                        line2.set_data(t_ago, zdata)
+
+                    ax.set_title(self.format_meter(
+                        n, total, elapsed, 0,
+                        self.desc, ascii, unit, unit_scale,
+                        1 / avg_time if avg_time else None, bar_format),
+                        fontname="DejaVu Sans Mono", fontsize=11)
+                    plt.pause(1e-9)
+
+                    # If no `miniters` was specified, adjust automatically
+                    # to the maximum iteration rate seen so far.
+                    if dynamic_miniters:
+                        if maxinterval and delta_t > maxinterval:
+                            # Set miniters to correspond to maxinterval
+                            miniters = delta_it * maxinterval / delta_t
+                        elif mininterval and delta_t:
+                            # EMA-weight miniters to converge
+                            # towards the timeframe of mininterval
+                            miniters = smoothing * delta_it * mininterval \
+                                / delta_t + (1 - smoothing) * miniters
+                        else:
+                            miniters = smoothing * delta_it + \
+                                (1 - smoothing) * miniters
+
+                    # Store old values for next call
+                    last_print_n = n
+                    last_print_t = cur_t
+
+        # Closing the progress bar.
+        # Update some internal variables for close().
+        self.last_print_n = last_print_n
+        self.n = n
+        self.close()
+
+    def update(self, n=1):
+        # if not self.gui:
+        #   return super(tqdm_gui, self).close()
+        if self.disable:
+            return
+
+        if n < 0:
+            n = 1
+        self.n += n
+
+        delta_it = self.n - self.last_print_n  # should be n?
+        if delta_it >= self.miniters:
+            # We check the counter first, to reduce the overhead of time()
+            cur_t = time()
+            delta_t = cur_t - self.last_print_t
+            if delta_t >= self.mininterval:
+                elapsed = cur_t - self.start_t
+                # EMA (not just overall average)
+                if self.smoothing and delta_t:
+                    self.avg_time = delta_t / delta_it \
+                        if self.avg_time is None \
+                        else self.smoothing * delta_t / delta_it + \
+                        (1 - self.smoothing) * self.avg_time
+
+                # Inline due to multiple calls
+                total = self.total
+                ax = self.ax
+
+                # instantaneous rate
+                y = delta_it / delta_t
+                # smoothed rate
+                z = self.n / elapsed
+                # update line data
+                self.xdata.append(self.n * 100.0 / total
+                                  if total else cur_t)
+                self.ydata.append(y)
+                self.zdata.append(z)
+
+                # Discard old values
+                if (not total) and elapsed > 66:
+                    self.xdata.popleft()
+                    self.ydata.popleft()
+                    self.zdata.popleft()
+
+                ymin, ymax = ax.get_ylim()
+                if y > ymax or z > ymax:
+                    ymax = 1.1 * y
+                    ax.set_ylim(ymin, ymax)
+                    ax.figure.canvas.draw()
+
+                if total:
+                    self.line1.set_data(self.xdata, self.ydata)
+                    self.line2.set_data(self.xdata, self.zdata)
+                    try:
+                        poly_lims = self.hspan.get_xy()
+                    except AttributeError:
+                        self.hspan = self.plt.axhspan(0, 0.001, xmin=0,
+                                                      xmax=0, color='g')
+                        poly_lims = self.hspan.get_xy()
+                    poly_lims[0, 1] = ymin
+                    poly_lims[1, 1] = ymax
+                    poly_lims[2] = [self.n / total, ymax]
+                    poly_lims[3] = [poly_lims[2, 0], ymin]
+                    if len(poly_lims) > 4:
+                        poly_lims[4, 1] = ymin
+                    self.hspan.set_xy(poly_lims)
+                else:
+                    t_ago = [cur_t - i for i in self.xdata]
+                    self.line1.set_data(t_ago, self.ydata)
+                    self.line2.set_data(t_ago, self.zdata)
+
+                ax.set_title(self.format_meter(
+                    self.n, total, elapsed, 0,
+                    self.desc, self.ascii, self.unit, self.unit_scale,
+                    1 / self.avg_time if self.avg_time else None,
+                    self.bar_format),
+                    fontname="DejaVu Sans Mono", fontsize=11)
+                self.plt.pause(1e-9)
+
+                # If no `miniters` was specified, adjust automatically to the
+                # maximum iteration rate seen so far.
+                # e.g.: After running `tqdm.update(5)`, subsequent
+                # calls to `tqdm.update()` will only cause an update after
+                # at least 5 more iterations.
+                if self.dynamic_miniters:
+                    if self.maxinterval and delta_t > self.maxinterval:
+                        self.miniters = self.miniters * self.maxinterval \
+                            / delta_t
+                    elif self.mininterval and delta_t:
+                        self.miniters = self.smoothing * delta_it \
+                            * self.mininterval / delta_t + \
+                            (1 - self.smoothing) * self.miniters
+                    else:
+                        self.miniters = self.smoothing * delta_it + \
+                            (1 - self.smoothing) * self.miniters
+
+                # Store old values for next call
+                self.last_print_n = self.n
+                self.last_print_t = cur_t
+
+    def close(self):
+        # if not self.gui:
+        #   return super(tqdm_gui, self).close()
+        if self.disable:
+            return
+
+        self.disable = True
+
+        self._instances.remove(self)
+
+        # Restore toolbars
+        self.mpl.rcParams['toolbar'] = self.toolbar
+        # Return to non-interactive mode
+        if not self.wasion:
+            self.plt.ioff()
+        if not self.leave:
+            self.plt.close(self.fig)
+
+
+def tgrange(*args, **kwargs):
+    """
+    A shortcut for tqdm_gui(xrange(*args), **kwargs).
+    On Python3+ range is used instead of xrange.
+    """
+    return tqdm_gui(_range(*args), **kwargs)

lib/tqdm/_tqdm_notebook.py

@@ -0,0 +1,236 @@
+"""
+IPython/Jupyter Notebook progressbar decorator for iterators.
+Includes a default (x)range iterator printing to stderr.
+
+Usage:
+  >>> from tqdm_notebook import tnrange[, tqdm_notebook]
+  >>> for i in tnrange(10): #same as: for i in tqdm_notebook(xrange(10))
+  ...     ...
+"""
+# future division is important to divide integers and get as
+# a result precise floating numbers (instead of truncated int)
+from __future__ import division, absolute_import
+# import compatibility functions and utilities
+import sys
+from ._utils import _range
+# to inherit from the tqdm class
+from ._tqdm import tqdm
+
+
+if True:  # pragma: no cover
+    # import IPython/Jupyter base widget and display utilities
+    try:  # IPython 4.x
+        import ipywidgets
+        IPY = 4
+    except ImportError:  # IPython 3.x / 2.x
+        IPY = 32
+        import warnings
+        with warnings.catch_warnings():
+            ipy_deprecation_msg = "The `IPython.html` package" \
+                                  " has been deprecated"
+            warnings.filterwarnings('error',
+                                    message=".*" + ipy_deprecation_msg + ".*")
+            try:
+                import IPython.html.widgets as ipywidgets
+            except Warning as e:
+                if ipy_deprecation_msg not in str(e):
+                    raise
+                warnings.simplefilter('ignore')
+                try:
+                    import IPython.html.widgets as ipywidgets  # NOQA
+                except ImportError:
+                    pass
+            except ImportError:
+                pass
+
+    try:  # IPython 4.x / 3.x
+        if IPY == 32:
+            from IPython.html.widgets import IntProgress, HBox, HTML
+            IPY = 3
+        else:
+            from ipywidgets import IntProgress, HBox, HTML
+    except ImportError:
+        try:  # IPython 2.x
+            from IPython.html.widgets import IntProgressWidget as IntProgress
+            from IPython.html.widgets import ContainerWidget as HBox
+            from IPython.html.widgets import HTML
+            IPY = 2
+        except ImportError:
+            IPY = 0
+
+    try:
+        from IPython.display import display  # , clear_output
+    except ImportError:
+        pass
+
+    # HTML encoding
+    try:  # Py3
+        from html import escape
+    except ImportError:  # Py2
+        from cgi import escape
+
+
+__author__ = {"github.com/": ["lrq3000", "casperdcl", "alexanderkuk"]}
+__all__ = ['tqdm_notebook', 'tnrange']
+
+
+class tqdm_notebook(tqdm):
+    """
+    Experimental IPython/Jupyter Notebook widget using tqdm!
+    """
+
+    @staticmethod
+    def status_printer(_, total=None, desc=None):
+        """
+        Manage the printing of an IPython/Jupyter Notebook progress bar widget.
+        """
+        # Fallback to text bar if there's no total
+        # DEPRECATED: replaced with an 'info' style bar
+        # if not total:
+        #    return super(tqdm_notebook, tqdm_notebook).status_printer(file)
+
+        # fp = file
+
+        # Prepare IPython progress bar
+        if total:
+            pbar = IntProgress(min=0, max=total)
+        else:  # No total? Show info style bar with no progress tqdm status
+            pbar = IntProgress(min=0, max=1)
+            pbar.value = 1
+            pbar.bar_style = 'info'
+        if desc:
+            pbar.description = desc
+        # Prepare status text
+        ptext = HTML()
+        # Only way to place text to the right of the bar is to use a container
+        container = HBox(children=[pbar, ptext])
+        display(container)
+
+        def print_status(s='', close=False, bar_style=None, desc=None):
+            # Note: contrary to native tqdm, s='' does NOT clear bar
+            # goal is to keep all infos if error happens so user knows
+            # at which iteration the loop failed.
+
+            # Clear previous output (really necessary?)
+            # clear_output(wait=1)
+
+            # Get current iteration value from format_meter string
+            if total:
+                # n = None
+                if s:
+                    npos = s.find(r'/|/')  # cause we use bar_format=r'{n}|...'
+                    # Check that n can be found in s (else n > total)
+                    if npos >= 0:
+                        n = int(s[:npos])  # get n from string
+                        s = s[npos + 3:]  # remove from string
+
+                        # Update bar with current n value
+                        if n is not None:
+                            pbar.value = n
+
+            # Print stats
+            if s:  # never clear the bar (signal: s='')
+                s = s.replace('||', '')  # remove inesthetical pipes
+                s = escape(s)  # html escape special characters (like '?')
+                ptext.value = s
+
+            # Change bar style
+            if bar_style:
+                # Hack-ish way to avoid the danger bar_style being overriden by
+                # success because the bar gets closed after the error...
+                if not (pbar.bar_style == 'danger' and bar_style == 'success'):
+                    pbar.bar_style = bar_style
+
+            # Special signal to close the bar
+            if close and pbar.bar_style != 'danger':  # hide only if no error
+                try:
+                    container.close()
+                except AttributeError:
+                    container.visible = False
+
+            # Update description
+            if desc:
+                pbar.description = desc
+
+        return print_status
+
+    def __init__(self, *args, **kwargs):
+        # Setup default output
+        if kwargs.get('file', sys.stderr) is sys.stderr:
+            kwargs['file'] = sys.stdout  # avoid the red block in IPython
+
+        # Remove the bar from the printed string, only print stats
+        if not kwargs.get('bar_format', None):
+            kwargs['bar_format'] = r'{n}/|/{l_bar}{r_bar}'
+
+        # Initialize parent class + avoid printing by using gui=True
+        kwargs['gui'] = True
+        super(tqdm_notebook, self).__init__(*args, **kwargs)
+        if self.disable or not kwargs['gui']:
+            return
+
+        # Delete first pbar generated from super() (wrong total and text)
+        # DEPRECATED by using gui=True
+        # self.sp('', close=True)
+        # Replace with IPython progress bar display (with correct total)
+        self.sp = self.status_printer(self.fp, self.total, self.desc)
+        self.desc = None  # trick to place description before the bar
+
+        # Print initial bar state
+        if not self.disable:
+            self.sp(self.__repr__())  # same as self.refresh without clearing
+
+    def __iter__(self, *args, **kwargs):
+        try:
+            for obj in super(tqdm_notebook, self).__iter__(*args, **kwargs):
+                # return super(tqdm...) will not catch exception
+                yield obj
+        # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt
+        except:
+            self.sp(bar_style='danger')
+            raise
+
+    def update(self, *args, **kwargs):
+        try:
+            super(tqdm_notebook, self).update(*args, **kwargs)
+        except Exception as exc:
+            # cannot catch KeyboardInterrupt when using manual tqdm
+            # as the interrupt will most likely happen on another statement
+            self.sp(bar_style='danger')
+            raise exc
+
+    def close(self, *args, **kwargs):
+        super(tqdm_notebook, self).close(*args, **kwargs)
+        # If it was not run in a notebook, sp is not assigned, check for it
+        if hasattr(self, 'sp'):
+            # Try to detect if there was an error or KeyboardInterrupt
+            # in manual mode: if n < total, things probably got wrong
+            if self.total and self.n < self.total:
+                self.sp(bar_style='danger')
+            else:
+                if self.leave:
+                    self.sp(bar_style='success')
+                else:
+                    self.sp(close=True)
+
+    def moveto(self, *args, **kwargs):
+        # void -> avoid extraneous `\n` in IPython output cell
+        return
+
+    def set_description(self, desc=None, **_):
+        """
+        Set/modify description of the progress bar.
+
+        Parameters
+        ----------
+        desc  : str, optional
+        """
+        self.sp(desc=desc)
+
+
+def tnrange(*args, **kwargs):
+    """
+    A shortcut for tqdm_notebook(xrange(*args), **kwargs).
+    On Python3+ range is used instead of xrange.
+    """
+    return tqdm_notebook(_range(*args), **kwargs)

lib/tqdm/_tqdm_pandas.py

@@ -0,0 +1,46 @@
+import sys
+
+__author__ = "github.com/casperdcl"
+__all__ = ['tqdm_pandas']
+
+
+def tqdm_pandas(tclass, *targs, **tkwargs):
+    """
+    Registers the given `tqdm` instance with
+    `pandas.core.groupby.DataFrameGroupBy.progress_apply`.
+    It will even close() the `tqdm` instance upon completion.
+
+    Parameters
+    ----------
+    tclass  : tqdm class you want to use (eg, tqdm, tqdm_notebook, etc)
+    targs and tkwargs  : arguments for the tqdm instance
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> import numpy as np
+    >>> from tqdm import tqdm, tqdm_pandas
+    >>>
+    >>> df = pd.DataFrame(np.random.randint(0, 100, (100000, 6)))
+    >>> tqdm_pandas(tqdm, leave=True)  # can use tqdm_gui, optional kwargs, etc
+    >>> # Now you can use `progress_apply` instead of `apply`
+    >>> df.groupby(0).progress_apply(lambda x: x**2)
+
+    References
+    ----------
+    https://stackoverflow.com/questions/18603270/
+    progress-indicator-during-pandas-operations-python
+    """
+    from tqdm import TqdmDeprecationWarning
+
+    if isinstance(tclass, type) or (getattr(tclass, '__name__', '').startswith(
+            'tqdm_')):  # delayed adapter case
+        TqdmDeprecationWarning("""\
+Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm, ...)`.
+""", fp_write=getattr(tkwargs.get('file', None), 'write', sys.stderr.write))
+        tclass.pandas(*targs, **tkwargs)
+    else:
+        TqdmDeprecationWarning("""\
+Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm(...))`.
+""", fp_write=getattr(tclass.fp, 'write', sys.stderr.write))
+        type(tclass).pandas(deprecated_t=tclass)

lib/tqdm/_utils.py

@@ -0,0 +1,215 @@
+import os
+import subprocess
+from platform import system as _curos
+CUR_OS = _curos()
+IS_WIN = CUR_OS in ['Windows', 'cli']
+IS_NIX = (not IS_WIN) and any(
+    CUR_OS.startswith(i) for i in
+    ['CYGWIN', 'MSYS', 'Linux', 'Darwin', 'SunOS', 'FreeBSD', 'NetBSD'])
+
+
+# Py2/3 compat. Empty conditional to avoid coverage
+if True:  # pragma: no cover
+    try:
+        _range = xrange
+    except NameError:
+        _range = range
+
+    try:
+        _unich = unichr
+    except NameError:
+        _unich = chr
+
+    try:
+        _unicode = unicode
+    except NameError:
+        _unicode = str
+
+    try:
+        if IS_WIN:
+            import colorama
+            colorama.init()
+        else:
+            colorama = None
+    except ImportError:
+        colorama = None
+
+    try:
+        from weakref import WeakSet
+    except ImportError:
+        WeakSet = set
+
+    try:
+        _basestring = basestring
+    except NameError:
+        _basestring = str
+
+    try:  # py>=2.7,>=3.1
+        from collections import OrderedDict as _OrderedDict
+    except ImportError:
+        try:  # older Python versions with backported ordereddict lib
+            from ordereddict import OrderedDict as _OrderedDict
+        except ImportError:  # older Python versions without ordereddict lib
+            # Py2.6,3.0 compat, from PEP 372
+            from collections import MutableMapping
+
+            class _OrderedDict(dict, MutableMapping):
+                # Methods with direct access to underlying attributes
+                def __init__(self, *args, **kwds):
+                    if len(args) > 1:
+                        raise TypeError('expected at 1 argument, got %d',
+                                        len(args))
+                    if not hasattr(self, '_keys'):
+                        self._keys = []
+                    self.update(*args, **kwds)
+
+                def clear(self):
+                    del self._keys[:]
+                    dict.clear(self)
+
+                def __setitem__(self, key, value):
+                    if key not in self:
+                        self._keys.append(key)
+                    dict.__setitem__(self, key, value)
+
+                def __delitem__(self, key):
+                    dict.__delitem__(self, key)
+                    self._keys.remove(key)
+
+                def __iter__(self):
+                    return iter(self._keys)
+
+                def __reversed__(self):
+                    return reversed(self._keys)
+
+                def popitem(self):
+                    if not self:
+                        raise KeyError
+                    key = self._keys.pop()
+                    value = dict.pop(self, key)
+                    return key, value
+
+                def __reduce__(self):
+                    items = [[k, self[k]] for k in self]
+                    inst_dict = vars(self).copy()
+                    inst_dict.pop('_keys', None)
+                    return self.__class__, (items,), inst_dict
+
+                # Methods with indirect access via the above methods
+                setdefault = MutableMapping.setdefault
+                update = MutableMapping.update
+                pop = MutableMapping.pop
+                keys = MutableMapping.keys
+                values = MutableMapping.values
+                items = MutableMapping.items
+
+                def __repr__(self):
+                    pairs = ', '.join(map('%r: %r'.__mod__, self.items()))
+                    return '%s({%s})' % (self.__class__.__name__, pairs)
+
+                def copy(self):
+                    return self.__class__(self)
+
+                @classmethod
+                def fromkeys(cls, iterable, value=None):
+                    d = cls()
+                    for key in iterable:
+                        d[key] = value
+                    return d
+
+
+def _is_utf(encoding):
+    try:
+        u'\u2588\u2589'.encode(encoding)
+    except UnicodeEncodeError:  # pragma: no cover
+        return False
+    except Exception:  # pragma: no cover
+        try:
+            return encoding.lower().startswith('utf-') or ('U8' == encoding)
+        except:
+            return False
+    else:
+        return True
+
+
+def _supports_unicode(fp):
+    try:
+        return _is_utf(fp.encoding)
+    except AttributeError:
+        return False
+
+
+def _environ_cols_wrapper():  # pragma: no cover
+    """
+    Return a function which gets width and height of console
+    (linux,osx,windows,cygwin).
+    """
+    _environ_cols = None
+    if IS_WIN:
+        _environ_cols = _environ_cols_windows
+        if _environ_cols is None:
+            _environ_cols = _environ_cols_tput
+    if IS_NIX:
+        _environ_cols = _environ_cols_linux
+    return _environ_cols
+
+
+def _environ_cols_windows(fp):  # pragma: no cover
+    try:
+        from ctypes import windll, create_string_buffer
+        import struct
+        from sys import stdin, stdout
+
+        io_handle = -12  # assume stderr
+        if fp == stdin:
+            io_handle = -10
+        elif fp == stdout:
+            io_handle = -11
+
+        h = windll.kernel32.GetStdHandle(io_handle)
+        csbi = create_string_buffer(22)
+        res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
+        if res:
+            (_bufx, _bufy, _curx, _cury, _wattr, left, _top, right, _bottom,
+             _maxx, _maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw)
+            # nlines = bottom - top + 1
+            return right - left  # +1
+    except:
+        pass
+    return None
+
+
+def _environ_cols_tput(*_):  # pragma: no cover
+    """cygwin xterm (windows)"""
+    try:
+        import shlex
+        cols = int(subprocess.check_call(shlex.split('tput cols')))
+        # rows = int(subprocess.check_call(shlex.split('tput lines')))
+        return cols
+    except:
+        pass
+    return None
+
+
+def _environ_cols_linux(fp):  # pragma: no cover
+
+    try:
+        from termios import TIOCGWINSZ
+        from fcntl import ioctl
+        from array import array
+    except ImportError:
+        return None
+    else:
+        try:
+            return array('h', ioctl(fp, TIOCGWINSZ, '\0' * 8))[1]
+        except:
+            try:
+                from os.environ import get
+            except ImportError:
+                return None
+            else:
+                return int(get('COLUMNS', 1)) - 1
+
+
+def _term_move_up():  # pragma: no cover
+    return '' if (os.name == 'nt') and (colorama is None) else '\x1b[A'

lib/tqdm/_version.py

@@ -0,0 +1,59 @@
+# Definition of the version number
+import os
+from io import open as io_open
+
+__all__ = ["__version__"]
+
+# major, minor, patch, -extra
+version_info = 4, 21, 0
+
+# Nice string for the version
+__version__ = '.'.join(map(str, version_info))
+
+
+# auto -extra based on commit hash (if not tagged as release)
+scriptdir = os.path.dirname(__file__)
+gitdir = os.path.abspath(os.path.join(scriptdir, "..", ".git"))
+if os.path.isdir(gitdir):  # pragma: nocover
+    extra = None
+    # Open config file to check if we are in tqdm project
+    with io_open(os.path.join(gitdir, "config"), 'r') as fh_config:
+        if 'tqdm' in fh_config.read():
+            # Open the HEAD file
+            with io_open(os.path.join(gitdir, "HEAD"), 'r') as fh_head:
+                extra = fh_head.readline().strip()
+            # in a branch => HEAD points to file containing last commit
+            if 'ref:' in extra:
+                # reference file path
+                ref_file = extra[5:]
+                branch_name = ref_file.rsplit('/', 1)[-1]
+
+                ref_file_path = os.path.abspath(os.path.join(gitdir, ref_file))
+                # check that we are in git folder
+                # (by stripping the git folder from the ref file path)
+                if os.path.relpath(
+                        ref_file_path, gitdir).replace('\\', '/') != ref_file:
+                    # out of git folder
+                    extra = None
+                else:
+                    # open the ref file
+                    with io_open(ref_file_path, 'r') as fh_branch:
+                        commit_hash = fh_branch.readline().strip()
+                        extra = commit_hash[:8]
+                        if branch_name != "master":
+                            extra += '.' + branch_name
+
+            # detached HEAD mode, already have commit hash
+            else:
+                extra = extra[:8]
+
+    # Append commit hash (and branch) to version string if not tagged
+    if extra is not None:
+        try:
+            with io_open(os.path.join(gitdir, "refs", "tags",
+                                      'v' + __version__)) as fdv:
+                if fdv.readline().strip()[:8] != extra[:8]:
+                    __version__ += '-' + extra
+        except Exception as e:
+            if "No such file" not in str(e):
+                raise

lib/tqdm/tests/tests_main.py

@@ -0,0 +1,94 @@
+import sys
+import subprocess
+from tqdm import main, TqdmKeyError, TqdmTypeError
+
+from tests_tqdm import with_setup, pretest, posttest, _range, closing, \
+    UnicodeIO, StringIO
+
+
+def _sh(*cmd, **kwargs):
+    return subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                            **kwargs).communicate()[0].decode('utf-8')
+
+
+# WARNING: this should be the last test as it messes with sys.stdin, argv
+@with_setup(pretest, posttest)
+def test_main():
+    """Test command line pipes"""
+    ls_out = _sh('ls').replace('\r\n', '\n')
+    ls = subprocess.Popen('ls', stdout=subprocess.PIPE,
+                          stderr=subprocess.STDOUT)
+    res = _sh(sys.executable, '-c', 'from tqdm import main; main()',
+              stdin=ls.stdout, stderr=subprocess.STDOUT)
+    ls.wait()
+
+    # actual test:
+
+    assert (ls_out in res.replace('\r\n', '\n'))
+
+    # semi-fake test which gets coverage:
+    _SYS = sys.stdin, sys.argv
+
+    with closing(StringIO()) as sys.stdin:
+        sys.argv = ['', '--desc', 'Test CLI-delims',
+                    '--ascii', 'True', '--delim', r'\0', '--buf_size', '64']
+        sys.stdin.write('\0'.join(map(str, _range(int(1e3)))))
+        sys.stdin.seek(0)
+        main()
+
+    IN_DATA_LIST = map(str, _range(int(1e3)))
+    sys.stdin = IN_DATA_LIST
+    sys.argv = ['', '--desc', 'Test CLI pipes',
+                '--ascii', 'True', '--unit_scale', 'True']
+    import tqdm.__main__  # NOQA
+
+    IN_DATA = '\0'.join(IN_DATA_LIST)
+    with closing(StringIO()) as sys.stdin:
+        sys.stdin.write(IN_DATA)
+        sys.stdin.seek(0)
+        sys.argv = ['', '--ascii', '--bytes', '--unit_scale', 'False']
+        with closing(UnicodeIO()) as fp:
+            main(fp=fp)
+            assert (str(len(IN_DATA)) in fp.getvalue())
+
+    sys.stdin = IN_DATA_LIST
+    sys.argv = ['', '-ascii', '--unit_scale', 'False',
+                '--desc', 'Test CLI errors']
+    main()
+
+    sys.argv = ['', '-ascii', '-unit_scale', '--bad_arg_u_ment', 'foo']
+    try:
+        main()
+    except TqdmKeyError as e:
+        if 'bad_arg_u_ment' not in str(e):
+            raise
+    else:
+        raise TqdmKeyError('bad_arg_u_ment')
+
+    sys.argv = ['', '-ascii', '-unit_scale', 'invalid_bool_value']
+    try:
+        main()
+    except TqdmTypeError as e:
+        if 'invalid_bool_value' not in str(e):
+            raise
+    else:
+        raise TqdmTypeError('invalid_bool_value')
+
+    sys.argv = ['', '-ascii', '--total', 'invalid_int_value']
+    try:
+        main()
+    except TqdmTypeError as e:
+        if 'invalid_int_value' not in str(e):
+            raise
+    else:
+        raise TqdmTypeError('invalid_int_value')
+
+    for i in ('-h', '--help', '-v', '--version'):
+        sys.argv = ['', i]
+        try:
+            main()
+        except SystemExit:
+            pass
+
+    # clean up
+    sys.stdin, sys.argv = _SYS

lib/tqdm/tests/tests_pandas.py

@@ -0,0 +1,207 @@
+from nose.plugins.skip import SkipTest
+
+from tqdm import tqdm
+from tests_tqdm import with_setup, pretest, posttest, StringIO, closing
+
+
+@with_setup(pretest, posttest)
+def test_pandas_series():
+    """Test pandas.Series.progress_apply and .progress_map"""
+    try:
+        from numpy.random import randint
+        import pandas as pd
+    except ImportError:
+        raise SkipTest
+
+    with closing(StringIO()) as our_file:
+        tqdm.pandas(file=our_file, leave=True, ascii=True)
+
+        series = pd.Series(randint(0, 50, (123,)))
+        res1 = series.progress_apply(lambda x: x + 10)
+        res2 = series.apply(lambda x: x + 10)
+        assert res1.equals(res2)
+
+        res3 = series.progress_map(lambda x: x + 10)
+        res4 = series.map(lambda x: x + 10)
+        assert res3.equals(res4)
+
+        expects = ['100%', '123/123']
+        for exres in expects:
+            our_file.seek(0)
+            if our_file.getvalue().count(exres) < 2:
+                our_file.seek(0)
+                raise AssertionError(
+                    "\nExpected:\n{0}\nIn:\n{1}\n".format(
+                        exres + " at least twice.", our_file.read()))
+
+
+@with_setup(pretest, posttest)
+def test_pandas_data_frame():
+    """Test pandas.DataFrame.progress_apply and .progress_applymap"""
+    try:
+        from numpy.random import randint
+        import pandas as pd
+    except ImportError:
+        raise SkipTest
+
+    with closing(StringIO()) as our_file:
+        tqdm.pandas(file=our_file, leave=True, ascii=True)
+        df = pd.DataFrame(randint(0, 50, (100, 200)))
+
+        def task_func(x):
+            return x + 1
+
+        # applymap
+        res1 = df.progress_applymap(task_func)
+        res2 = df.applymap(task_func)
+        assert res1.equals(res2)
+
+        # apply
+        for axis in [0, 1]:
+            res3 = df.progress_apply(task_func, axis=axis)
+            res4 = df.apply(task_func, axis=axis)
+            assert res3.equals(res4)
+
+        our_file.seek(0)
+        if our_file.read().count('100%') < 3:
+            our_file.seek(0)
+            raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format(
+                '100% at least three times', our_file.read()))
+
+        # apply_map, apply axis=0, apply axis=1
+        expects = ['20000/20000', '200/200', '100/100']
+        for exres in expects:
+            our_file.seek(0)
+            if our_file.getvalue().count(exres) < 1:
+                our_file.seek(0)
+                raise AssertionError(
+                    "\nExpected:\n{0}\nIn:\n {1}\n".format(
+                        exres + " at least once.", our_file.read()))
+
+
+@with_setup(pretest, posttest)
+def test_pandas_groupby_apply():
+    """Test pandas.DataFrame.groupby(...).progress_apply"""
+    try:
+        from numpy.random import randint
+        import pandas as pd
+    except ImportError:
+        raise SkipTest
+
+    with closing(StringIO()) as our_file:
+        tqdm.pandas(file=our_file, leave=False, ascii=True)
+
+        df = pd.DataFrame(randint(0, 50, (500, 3)))
+        df.groupby(0).progress_apply(lambda x: None)
+
+        dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc'))
+        dfs.groupby(['a']).progress_apply(lambda x: None)
+
+        our_file.seek(0)
+
+        # don't expect final output since no `leave` and
+        # high dynamic `miniters`
+        nexres = '100%|##########|'
+        if nexres in our_file.read():
+            our_file.seek(0)
+            raise AssertionError("\nDid not expect:\n{0}\nIn:{1}\n".format(
+                nexres, our_file.read()))
+
+    with closing(StringIO()) as our_file:
+        tqdm.pandas(file=our_file, leave=True, ascii=True)
+
+        dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc'))
+        dfs.loc[0] = [2, 1, 1]
+        dfs['d'] = 100
+
+        expects = ['500/500', '1/1', '4/4', '2/2']
+        dfs.groupby(dfs.index).progress_apply(lambda x: None)
+        dfs.groupby('d').progress_apply(lambda x: None)
+        dfs.groupby(dfs.columns, axis=1).progress_apply(lambda x: None)
+        dfs.groupby([2, 2, 1, 1], axis=1).progress_apply(lambda x: None)
+
+        our_file.seek(0)
+        if our_file.read().count('100%') < 4:
+            our_file.seek(0)
+            raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format(
+                '100% at least four times', our_file.read()))
+
+        for exres in expects:
+            our_file.seek(0)
+            if our_file.getvalue().count(exres) < 1:
+                our_file.seek(0)
+                raise AssertionError(
+                    "\nExpected:\n{0}\nIn:\n {1}\n".format(
+                        exres + " at least once.", our_file.read()))
+
+
+@with_setup(pretest, posttest)
+def test_pandas_leave():
+    """Test pandas with `leave=True`"""
+    try:
+        from numpy.random import randint
+        import pandas as pd
+    except ImportError:
+        raise SkipTest
+
+    with closing(StringIO()) as our_file:
+        df = pd.DataFrame(randint(0, 100, (1000, 6)))
+        tqdm.pandas(file=our_file, leave=True, ascii=True)
+        df.groupby(0).progress_apply(lambda x: None)
+
+        our_file.seek(0)
+
+        exres = '100%|##########| 100/100'
+        if exres not in our_file.read():
+            our_file.seek(0)
+            raise AssertionError(
+                "\nExpected:\n{0}\nIn:{1}\n".format(exres, our_file.read()))
+
+
+@with_setup(pretest, posttest)
+def test_pandas_apply_args_deprecation():
+    """Test warning info in
+    `pandas.Dataframe(Series).progress_apply(func, *args)`"""
+    try:
+        from numpy.random import randint
+        from tqdm import tqdm_pandas
+        import pandas as pd
+    except ImportError:
+        raise SkipTest
+
+    with closing(StringIO()) as our_file:
+        tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20))
+        df = pd.DataFrame(randint(0, 50, (500, 3)))
+        df.progress_apply(lambda x: None, 1)  # 1 shall cause a warning
+        # Check deprecation message
+        res = our_file.getvalue()
+        assert all([i in res for i in (
+            "TqdmDeprecationWarning", "not supported",
+            "keyword arguments instead")])
+
+
+@with_setup(pretest, posttest)
+def test_pandas_deprecation():
+    """Test bar object instance as argument deprecation"""
+    try:
+        from numpy.random import randint
+        from tqdm import tqdm_pandas
+        import pandas as pd
+    except ImportError:
+        raise SkipTest
+
+    with closing(StringIO()) as our_file:
+        tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20))
+        df = pd.DataFrame(randint(0, 50, (500, 3)))
+        df.groupby(0).progress_apply(lambda x: None)
+        # Check deprecation message
+        assert "TqdmDeprecationWarning" in our_file.getvalue()
+        assert "instead of `tqdm_pandas(tqdm(...))`" in our_file.getvalue()
+
+    with closing(StringIO()) as our_file:
+        tqdm_pandas(tqdm, file=our_file, leave=False, ascii=True, ncols=20)
+        df = pd.DataFrame(randint(0, 50, (500, 3)))
+        df.groupby(0).progress_apply(lambda x: None)
+        # Check deprecation message
+        assert "TqdmDeprecationWarning" in our_file.getvalue()
+        assert "instead of `tqdm_pandas(tqdm, ...)`" in our_file.getvalue()

lib/tqdm/tests/tests_perf.py

@@ -0,0 +1,336 @@
+from __future__ import print_function, division
+
+from nose.plugins.skip import SkipTest
+
+from contextlib import contextmanager
+
+import sys
+from time import sleep, time
+
+from tqdm import trange
+from tqdm import tqdm
+
+from tests_tqdm import with_setup, pretest, posttest, StringIO, closing, _range
+
+# Use relative/cpu timer to have reliable timings when there is a sudden load
+try:
+    from time import process_time
+except ImportError:
+    from time import clock
+    process_time = clock
+
+
+def get_relative_time(prevtime=0):
+    return process_time() - prevtime
+
+
+def cpu_sleep(t):
+    """Sleep the given amount of cpu time"""
+    start = process_time()
+    while (process_time() - start) < t:
+        pass
+
+
+def checkCpuTime(sleeptime=0.2):
+    """Check if cpu time works correctly"""
+    if checkCpuTime.passed:
+        return True
+    # First test that sleeping does not consume cputime
+    start1 = process_time()
+    sleep(sleeptime)
+    t1 = process_time() - start1
+
+    # secondly check by comparing to cpusleep (where we actually do something)
+    start2 = process_time()
+    cpu_sleep(sleeptime)
+    t2 = process_time() - start2
+
+    if abs(t1) < 0.0001 and (t1 < t2 / 10):
+        return True
+    raise SkipTest
+
+
+checkCpuTime.passed = False
+
+
+@contextmanager
+def relative_timer():
+    start = process_time()
+
+    def elapser():
+        return process_time() - start
+
+    yield lambda: elapser()
+    spent = process_time() - start
+
+    def elapser():  # NOQA
+        return spent
+
+
+def retry_on_except(n=3):
+    def wrapper(fn):
+        def test_inner():
+            for i in range(1, n + 1):
+                try:
+                    checkCpuTime()
+                    fn()
+                except SkipTest:
+                    if i >= n:
+                        raise
+                else:
+                    return
+
+        test_inner.__doc__ = fn.__doc__
+        return test_inner
+
+    return wrapper
+
+
+class MockIO(StringIO):
+    """Wraps StringIO to mock a file with no I/O"""
+
+    def write(self, data):
+        return
+
+
+def simple_progress(iterable=None, total=None, file=sys.stdout, desc='',
+                    leave=False, miniters=1, mininterval=0.1, width=60):
+    """Simple progress bar reproducing tqdm's major features"""
+    n = [0]  # use a closure
+    start_t = [time()]
+    last_n = [0]
+    last_t = [0]
+    if iterable is not None:
+        total = len(iterable)
+
+    def format_interval(t):
+        mins, s = divmod(int(t), 60)
+        h, m = divmod(mins, 60)
+        if h:
+            return '{0:d}:{1:02d}:{2:02d}'.format(h, m, s)
+        else:
+            return '{0:02d}:{1:02d}'.format(m, s)
+
+    def update_and_print(i=1):
+        n[0] += i
+        if (n[0] - last_n[0]) >= miniters:
+            last_n[0] = n[0]
+
+            if (time() - last_t[0]) >= mininterval:
+                last_t[0] = time()  # last_t[0] == current time
+
+                spent = last_t[0] - start_t[0]
+                spent_fmt = format_interval(spent)
+                rate = n[0] / spent if spent > 0 else 0
+                if 0.0 < rate < 1.0:
+                    rate_fmt = "%.2fs/it" % (1.0 / rate)
+                else:
+                    rate_fmt = "%.2fit/s" % rate
+
+                frac = n[0] / total
+                percentage = int(frac * 100)
+                eta = (total - n[0]) / rate if rate > 0 else 0
+                eta_fmt = format_interval(eta)
+
+                # bar = "#" * int(frac * width)
+                barfill = " " * int((1.0 - frac) * width)
+                bar_length, frac_bar_length = divmod(int(frac * width * 10), 10)
+                bar = '#' * bar_length
+                frac_bar = chr(48 + frac_bar_length) if frac_bar_length \
+                    else ' '
+
+                file.write("\r%s %i%%|%s%s%s| %i/%i [%s<%s, %s]" %
+                           (desc, percentage, bar, frac_bar, barfill, n[0],
+                            total, spent_fmt, eta_fmt, rate_fmt))
+
+                if n[0] == total and leave:
+                    file.write("\n")
+                file.flush()
+
+    def update_and_yield():
+        for elt in iterable:
+            yield elt
+            update_and_print()
+
+    update_and_print(0)
+    if iterable is not None:
+        return update_and_yield()
+    else:
+        return update_and_print
+
+
+@with_setup(pretest, posttest)
+@retry_on_except()
+def test_iter_overhead():
+    """Test overhead of iteration based tqdm"""
+
+    total = int(1e6)
+
+    with closing(MockIO()) as our_file:
+        a = 0
+        with trange(total, file=our_file) as t:
+            with relative_timer() as time_tqdm:
+                for i in t:
+                    a += i
+        assert (a == (total * total - total) / 2.0)
+
+        a = 0
+        with relative_timer() as time_bench:
+            for i in _range(total):
+                a += i
+                our_file.write(a)
+
+    # Compute relative overhead of tqdm against native range()
+    if time_tqdm() > 9 * time_bench():
+        raise AssertionError('trange(%g): %f, range(%g): %f' %
+                             (total, time_tqdm(), total, time_bench()))
+
+
+@with_setup(pretest, posttest)
+@retry_on_except()
+def test_manual_overhead():
+    """Test overhead of manual tqdm"""
+
+    total = int(1e6)
+
+    with closing(MockIO()) as our_file:
+        with tqdm(total=total * 10, file=our_file, leave=True) as t:
+            a = 0
+            with relative_timer() as time_tqdm:
+                for i in _range(total):
+                    a += i
+                    t.update(10)
+
+        a = 0
+        with relative_timer() as time_bench:
+            for i in _range(total):
+                a += i
+                our_file.write(a)
+
+    # Compute relative overhead of tqdm against native range()
+    if time_tqdm() > 10 * time_bench():
+        raise AssertionError('tqdm(%g): %f, range(%g): %f' %
+                             (total, time_tqdm(), total, time_bench()))
+
+
+@with_setup(pretest, posttest)
+@retry_on_except()
+def test_iter_overhead_hard():
+    """Test overhead of iteration based tqdm (hard)"""
+
+    total = int(1e5)
+
+    with closing(MockIO()) as our_file:
+        a = 0
+        with trange(total, file=our_file, leave=True, miniters=1,
+                    mininterval=0, maxinterval=0) as t:
+            with relative_timer() as time_tqdm:
+                for i in t:
+                    a += i
+        assert (a == (total * total - total) / 2.0)
+
+        a = 0
+        with relative_timer() as time_bench:
+            for i in _range(total):
+                a += i
+                our_file.write(("%i" % a) * 40)
+
+    # Compute relative overhead of tqdm against native range()
+    try:
+        assert (time_tqdm() < 60 * time_bench())
+    except AssertionError:
+        raise AssertionError('trange(%g): %f, range(%g): %f' %
+                             (total, time_tqdm(), total, time_bench()))
+
+
+@with_setup(pretest, posttest)
+@retry_on_except()
+def test_manual_overhead_hard():
+    """Test overhead of manual tqdm (hard)"""
+
+    total = int(1e5)
+
+    with closing(MockIO()) as our_file:
+        t = tqdm(total=total * 10, file=our_file, leave=True, miniters=1,
+                 mininterval=0, maxinterval=0)
+        a = 0
+        with relative_timer() as time_tqdm:
+            for i in _range(total):
+                a += i
+                t.update(10)
+
+        a = 0
+        with relative_timer() as time_bench:
+            for i in _range(total):
+                a += i
+                our_file.write(("%i" % a) * 40)
+
+    # Compute relative overhead of tqdm against native range()
+    try:
+        assert (time_tqdm() < 100 * time_bench())
+    except AssertionError:
+        raise AssertionError('tqdm(%g): %f, range(%g): %f' %
+                             (total, time_tqdm(), total, time_bench()))
+
+
+@with_setup(pretest, posttest)
+@retry_on_except()
+def test_iter_overhead_simplebar_hard():
+    """Test overhead of iteration based tqdm vs simple progress bar (hard)"""
+
+    total = int(1e4)
+
+    with closing(MockIO()) as our_file:
+        a = 0
+        with trange(total, file=our_file, leave=True, miniters=1,
+                    mininterval=0, maxinterval=0) as t:
+            with relative_timer() as time_tqdm:
+                for i in t:
+                    a += i
+        assert (a == (total * total - total) / 2.0)
+
+        a = 0
+        s = simple_progress(_range(total), file=our_file, leave=True,
+                            miniters=1, mininterval=0)
+        with relative_timer() as time_bench:
+            for i in s:
+                a += i
+
+    # Compute relative overhead of tqdm against native range()
+    try:
+        assert (time_tqdm() < 2.5 * time_bench())
+    except AssertionError:
+        raise AssertionError('trange(%g): %f, simple_progress(%g): %f' %
+                             (total, time_tqdm(), total, time_bench()))
+
+
+@with_setup(pretest, posttest)
+@retry_on_except()
+def test_manual_overhead_simplebar_hard():
+    """Test overhead of manual tqdm vs simple progress bar (hard)"""
+
+    total = int(1e4)
+
+    with closing(MockIO()) as our_file:
+        t = tqdm(total=total * 10, file=our_file, leave=True, miniters=1,
+                 mininterval=0, maxinterval=0)
+        a = 0
+        with relative_timer() as time_tqdm:
+            for i in _range(total):
+                a += i
+                t.update(10)
+
+        simplebar_update = simple_progress(
+            total=total, file=our_file, leave=True, miniters=1, mininterval=0)
+        a = 0
+        with relative_timer() as time_bench:
+            for i in _range(total):
+                a += i
+                simplebar_update(10)
+
+    # Compute relative overhead of tqdm against native range()
+    try:
+        assert (time_tqdm() < 2.5 * time_bench())
+    except AssertionError:
+        raise AssertionError('tqdm(%g): %f, simple_progress(%g): %f' %
+                             (total, time_tqdm(), total, time_bench()))

lib/tqdm/tests/tests_synchronisation.py

@@ -0,0 +1,164 @@
+from __future__ import division
+from tqdm import tqdm
+from tests_tqdm import with_setup, pretest, posttest, StringIO, closing
+from tests_tqdm import DiscreteTimer, cpu_timify
+
+from time import sleep
+from threading import Event
+from tqdm import TMonitor
+
+
+class FakeSleep(object):
+    """Wait until the discrete timer reached the required time"""
+    def __init__(self, dtimer):
+        self.dtimer = dtimer
+
+    def sleep(self, t):
+        end = t + self.dtimer.t
+        while self.dtimer.t < end:
+            sleep(0.0000001)  # sleep a bit to interrupt (instead of pass)
+
+
+class FakeTqdm(object):
+    _instances = []
+
+
+def make_create_fake_sleep_event(sleep):
+    def wait(self, timeout=None):
+        if timeout is not None:
+            sleep(timeout)
+        return self.is_set()
+
+    def create_fake_sleep_event():
+        event = Event()
+        event.wait = wait
+        return event
+
+    return create_fake_sleep_event
+
+
+@with_setup(pretest, posttest)
+def test_monitor_thread():
+    """Test dummy monitoring thread"""
+    maxinterval = 10
+
+    # Setup a discrete timer
+    timer = DiscreteTimer()
+    TMonitor._time = timer.time
+    # And a fake sleeper
+    sleeper = FakeSleep(timer)
+    TMonitor._event = make_create_fake_sleep_event(sleeper.sleep)
+
+    # Instanciate the monitor
+    monitor = TMonitor(FakeTqdm, maxinterval)
+    # Test if alive, then killed
+    assert monitor.report()
+    monitor.exit()
+    timer.sleep(maxinterval * 2)  # need to go out of the sleep to die
+    assert not monitor.report()
+    # assert not monitor.is_alive()  # not working dunno why, thread not killed
+    del monitor
+
+
+@with_setup(pretest, posttest)
+def test_monitoring_and_cleanup():
+    """Test for stalled tqdm instance and monitor deletion"""
+    # Note: should fix miniters for these tests, else with dynamic_miniters
+    # it's too complicated to handle with monitoring update and maxinterval...
+    maxinterval = 2
+
+    total = 1000
+    # Setup a discrete timer
+    timer = DiscreteTimer()
+    # And a fake sleeper
+    sleeper = FakeSleep(timer)
+    # Setup TMonitor to use the timer
+    TMonitor._time = timer.time
+    TMonitor._event = make_create_fake_sleep_event(sleeper.sleep)
+    # Set monitor interval
+    tqdm.monitor_interval = maxinterval
+    with closing(StringIO()) as our_file:
+        with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1,
+                  maxinterval=maxinterval) as t:
+            cpu_timify(t, timer)
+            # Do a lot of iterations in a small timeframe
+            # (smaller than monitor interval)
+            timer.sleep(maxinterval / 2)  # monitor won't wake up
+            t.update(500)
+            # check that our fixed miniters is still there
+            assert t.miniters == 500
+            # Then do 1 it after monitor interval, so that monitor kicks in
+            timer.sleep(maxinterval * 2)
+            t.update(1)
+            # Wait for the monitor to get out of sleep's loop and update tqdm..
+            timeend = timer.time()
+            while not (t.monitor.woken >= timeend and t.miniters == 1):
+                timer.sleep(1)  # Force monitor to wake up if it woken too soon
+                sleep(0.000001)  # sleep to allow interrupt (instead of pass)
+            assert t.miniters == 1  # check that monitor corrected miniters
+            # Note: at this point, there may be a race condition: monitor saved
+            # current woken time but timer.sleep() happen just before monitor
+            # sleep. To fix that, either sleep here or increase time in a loop
+            # to ensure that monitor wakes up at some point.
+
+            # Try again but already at miniters = 1 so nothing will be done
+            timer.sleep(maxinterval * 2)
+            t.update(2)
+            timeend = timer.time()
+            while not (t.monitor.woken >= timeend):
+                timer.sleep(1)  # Force monitor to wake up if it woken too soon
+                sleep(0.000001)
+            # Wait for the monitor to get out of sleep's loop and update tqdm..
+            assert t.miniters == 1  # check that monitor corrected miniters
+
+    # Check that class var monitor is deleted if no instance left
+    tqdm.monitor_interval = 10
+    assert tqdm.monitor is None
+
+
+@with_setup(pretest, posttest)
+def test_monitoring_multi():
+    """Test on multiple bars, one not needing miniters adjustment"""
+    # Note: should fix miniters for these tests, else with dynamic_miniters
+    # it's too complicated to handle with monitoring update and maxinterval...
+    maxinterval = 2
+
+    total = 1000
+    # Setup a discrete timer
+    timer = DiscreteTimer()
+    # And a fake sleeper
+    sleeper = FakeSleep(timer)
+    # Setup TMonitor to use the timer
+    TMonitor._time = timer.time
+    TMonitor._event = make_create_fake_sleep_event(sleeper.sleep)
+    # Set monitor interval
+    tqdm.monitor_interval = maxinterval
+    with closing(StringIO()) as our_file:
+        with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1,
+                  maxinterval=maxinterval) as t1:
+            # Set high maxinterval for t2 so monitor does not need to adjust it
+            with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1,
+                      maxinterval=1E5) as t2:
+                cpu_timify(t1, timer)
+                cpu_timify(t2, timer)
+                # Do a lot of iterations in a small timeframe
+                timer.sleep(maxinterval / 2)
+                t1.update(500)
+                t2.update(500)
+                assert t1.miniters == 500
+                assert t2.miniters == 500
+                # Then do 1 it after monitor interval, so that monitor kicks in
+                timer.sleep(maxinterval * 2)
+                t1.update(1)
+                t2.update(1)
+                # Wait for the monitor to get out of sleep and update tqdm
+                timeend = timer.time()
+                while not (t1.monitor.woken >= timeend and t1.miniters == 1):
+                    timer.sleep(1)
+                    sleep(0.000001)
+                assert t1.miniters == 1  # check that monitor corrected miniters
+                assert t2.miniters == 500  # check that t2 was not adjusted
+
+    # Check that class var monitor is deleted if no instance left
+    tqdm.monitor_interval = 10
+    assert tqdm.monitor is None

lib/tqdm/tests/tests_version.py

@@ -0,0 +1,12 @@
+import re
+
+
+def test_version():
+    """Test version string"""
+    from tqdm import __version__
+    version_parts = re.split('[.-]', __version__)
+    assert 3 <= len(version_parts)  # must have at least Major.minor.patch
+    try:
+        map(int, version_parts[:3])
+    except ValueError:
+        raise TypeError('Version Major.minor.patch must be 3 integers')