Catch errors to reduce sentry spam

This commit is contained in:
vas3k
2020-02-06 21:05:23 +01:00
parent 75dcf4addd
commit bde93a4b4c

View File

@@ -277,12 +277,16 @@ def parse_text_and_image(entry):
def load_page_safe(url):
response = requests.get(
url=url,
timeout=DEFAULT_REQUEST_TIMEOUT,
headers=DEFAULT_REQUEST_HEADERS,
stream=True # the most important part — stream response to prevent loading everything into memory
)
try:
response = requests.get(
url=url,
timeout=DEFAULT_REQUEST_TIMEOUT,
headers=DEFAULT_REQUEST_HEADERS,
stream=True # the most important part — stream response to prevent loading everything into memory
)
except RequestException as ex:
log.warning(f"Error parsing the page: {url} {ex}")
return ""
html = io.StringIO()
total_bytes = 0