Merge pull request #7962 from CrafterKolyan/patch-1

Speed up `pip list --outdated`

news/7962.bugfix

@@ -0,0 +1 @@
+Significantly speedup ``pip list --outdated`` through parallelizing index interaction.

src/pip/_internal/commands/list.py

@@ -5,8 +5,10 @@ from __future__ import absolute_import
 
 import json
 import logging
+from multiprocessing.dummy import Pool
 
 from pip._vendor import six
+from pip._vendor.requests.adapters import DEFAULT_POOLSIZE
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.req_command import IndexGroupCommand
@@ -183,7 +185,7 @@ class ListCommand(IndexGroupCommand):
         with self._build_session(options) as session:
             finder = self._build_package_finder(options, session)
 
-            for dist in packages:
+            def latest_info(dist):
                 typ = 'unknown'
                 all_candidates = finder.find_all_candidates(dist.key)
                 if not options.pre:
@@ -196,7 +198,7 @@ class ListCommand(IndexGroupCommand):
                 )
                 best_candidate = evaluator.sort_best_candidate(all_candidates)
                 if best_candidate is None:
-                    continue
+                    return None
 
                 remote_version = best_candidate.version
                 if best_candidate.link.is_wheel:
@@ -206,7 +208,19 @@ class ListCommand(IndexGroupCommand):
                 # This is dirty but makes the rest of the code much cleaner
                 dist.latest_version = remote_version
                 dist.latest_filetype = typ
-                yield dist
+                return dist
+
+            # This is done for 2x speed up of requests to pypi.org
+            # so that "real time" of this function
+            # is almost equal to "user time"
+            pool = Pool(DEFAULT_POOLSIZE)
+
+            for dist in pool.imap_unordered(latest_info, packages):
+                if dist is not None:
+                    yield dist
+
+            pool.close()
+            pool.join()
 
     def output_package_listing(self, packages, options):
         packages = sorted(

src/pip/_internal/utils/logging.py

@@ -52,7 +52,6 @@ else:
 
 
 _log_state = threading.local()
-_log_state.indentation = 0
 subprocess_logger = getLogger('pip.subprocessor')
 
 
@@ -104,6 +103,8 @@ def indent_log(num=2):
     A context manager which will cause the log output to be indented for any
     log messages emitted inside it.
     """
+    # For thread-safety
+    _log_state.indentation = get_indentation()
     _log_state.indentation += num
     try:
         yield

tests/unit/test_logging.py

@@ -2,6 +2,7 @@ import errno
 import logging
 import os
 import time
+from threading import Thread
 
 import pytest
 from mock import patch
@@ -11,6 +12,7 @@ from pip._internal.utils.logging import (
     BrokenStdoutLoggingError,
     ColorizedStreamHandler,
     IndentingFormatter,
+    indent_log,
 )
 from pip._internal.utils.misc import captured_stderr, captured_stdout
 
@@ -108,6 +110,39 @@ class TestIndentingFormatter(object):
         f = IndentingFormatter(fmt="%(message)s")
         assert f.format(record) == expected
 
+    def test_thread_safety_base(self):
+        record = self.make_record(
+            'DEPRECATION: hello\nworld', level_name='WARNING',
+        )
+        f = IndentingFormatter(fmt="%(message)s")
+        results = []
+
+        def thread_function():
+            results.append(f.format(record))
+
+        thread_function()
+        thread = Thread(target=thread_function)
+        thread.start()
+        thread.join()
+        assert results[0] == results[1]
+
+    def test_thread_safety_indent_log(self):
+        record = self.make_record(
+            'DEPRECATION: hello\nworld', level_name='WARNING',
+        )
+        f = IndentingFormatter(fmt="%(message)s")
+        results = []
+
+        def thread_function():
+            with indent_log():
+                results.append(f.format(record))
+
+        thread_function()
+        thread = Thread(target=thread_function)
+        thread.start()
+        thread.join()
+        assert results[0] == results[1]
+
 
 class TestColorizedStreamHandler(object):