Don't need to unpack prematurely

This commit is contained in:
Tzu-ping Chung 2018-10-01 16:49:57 +08:00
parent bb6dae6e71
commit 025dc45ad1
1 changed files with 5 additions and 12 deletions

View File

@ -93,12 +93,8 @@ class _NotHTML(Exception):
pass
def _get_html_data(url, filename, session):
"""Access an HTML page with GET to retrieve data.
Returns a 3-tuple `(content, url, headers)`. `content` should be binary
containing the body; `url` is the response URL, and `headers` a mapping
containing response headers.
def _get_html_response(url, filename, session):
"""Access an HTML page with GET, and return the response.
This consists of five parts:
@ -167,7 +163,7 @@ def _get_html_data(url, filename, session):
if not content_type.lower().startswith("text/html"):
raise _NotHTML(content_type)
return resp.content, resp.url, resp.headers
return resp
def _handle_get_page_fail(link, reason, url, meth=None):
@ -185,10 +181,7 @@ def _get_html_page(link, session=None):
url = link.url.split('#', 1)[0]
try:
content, resp_url, headers = _get_html_data(
url, link.filename, session=session,
)
inst = HTMLPage(content, resp_url, headers)
resp = _get_html_response(url, link.filename, session=session)
except _SchemeIsVCS as exc:
logger.debug('Cannot look at %s URL %s', exc, link)
except _NotHTML as exc:
@ -207,7 +200,7 @@ def _get_html_page(link, session=None):
except requests.Timeout:
_handle_get_page_fail(link, "timed out", url)
else:
return inst
return HTMLPage(resp.content, resp.url, resp.headers)
class PackageFinder(object):