McSinyx
/
pip
mirror of https://github.com/pypa/pip
1
1
Fork 0
Code Issues Releases Activity

Add parallel_downloads param to PipSesion 

This parameter is used to set the number of parallel downloads in BatchDownloader
as well as to set the pool_connections in the HTTPAdapter to prevent
'Connection pool full' warnings
This commit is contained in:
Neil Botelho 2023-10-29 17:29:34 +05:30
parent ed13cea176
commit 2334399566
2 changed files with 25 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/pip/_internal/cli/req_command.py
View File

@ -118,6 +118,10 @@ class SessionCommandMixin(CommandContextMixIn):
ssl_context = None ssl_context = None
else: else:
ssl_context = None ssl_context = None
if "parallel_downloads" in options.__dict__:
parallel_downloads = options.parallel_downloads
else:
parallel_downloads = None
session = PipSession( session = PipSession(
cache=os.path.join(cache_dir, "http-v2") if cache_dir else None, cache=os.path.join(cache_dir, "http-v2") if cache_dir else None,
@ -125,6 +129,7 @@ class SessionCommandMixin(CommandContextMixIn):
trusted_hosts=options.trusted_hosts, trusted_hosts=options.trusted_hosts,
index_urls=self._get_index_urls(options), index_urls=self._get_index_urls(options),
ssl_context=ssl_context, ssl_context=ssl_context,
parallel_downloads=parallel_downloads,
) )
# Handle custom ca-bundles from the user # Handle custom ca-bundles from the user

22
src/pip/_internal/network/session.py
View File

@ -326,6 +326,7 @@ class PipSession(requests.Session):
trusted_hosts: Sequence[str] = (), trusted_hosts: Sequence[str] = (),
index_urls: Optional[List[str]] = None, index_urls: Optional[List[str]] = None,
ssl_context: Optional["SSLContext"] = None, ssl_context: Optional["SSLContext"] = None,
parallel_downloads: Optional[int] = None,
**kwargs: Any, **kwargs: Any,
) -> None: ) -> None:
""" """
@ -362,12 +363,24 @@ class PipSession(requests.Session):
backoff_factor=0.25, backoff_factor=0.25,
) # type: ignore ) # type: ignore
# Used to set numbers of parallel downloads in
# pip._internal.network.BatchDownloader and to set pool_connection in
# the HTTPAdapter to prevent connection pool from hitting the default(10)
# limit and throwing 'Connection pool is full' warnings
self.parallel_downloads = (
parallel_downloads if (parallel_downloads is not None) else 1
)
pool_maxsize = max(self.parallel_downloads, 10)
# Our Insecure HTTPAdapter disables HTTPS validation. It does not # Our Insecure HTTPAdapter disables HTTPS validation. It does not
# support caching so we'll use it for all http:// URLs. # support caching so we'll use it for all http:// URLs.
# If caching is disabled, we will also use it for # If caching is disabled, we will also use it for
# https:// hosts that we've marked as ignoring # https:// hosts that we've marked as ignoring
# TLS errors for (trusted-hosts). # TLS errors for (trusted-hosts).
insecure_adapter = InsecureHTTPAdapter(max_retries=retries) insecure_adapter = InsecureHTTPAdapter(
max_retries=retries,
pool_connections=pool_maxsize,
pool_maxsize=pool_maxsize,
)
# We want to _only_ cache responses on securely fetched origins or when # We want to _only_ cache responses on securely fetched origins or when
# the host is specified as trusted. We do this because # the host is specified as trusted. We do this because
@ -385,7 +398,12 @@ class PipSession(requests.Session):
max_retries=retries, max_retries=retries,
) )
else: else:
secure_adapter = HTTPAdapter(max_retries=retries, ssl_context=ssl_context) secure_adapter = HTTPAdapter(
max_retries=retries,
ssl_context=ssl_context,
pool_connections=pool_maxsize,
pool_maxsize=pool_maxsize,
)
self._trusted_host_adapter = insecure_adapter self._trusted_host_adapter = insecure_adapter
self.mount("https://", secure_adapter) self.mount("https://", secure_adapter)