global urllib import urllib.request global html import html global downloadPage def downloadPage(url,headers = False): if not headers: headers = {} response = { "url": url, "body": None, "headers": [] } class rdrh(urllib.request.HTTPRedirectHandler): def redirect_request(self,req,fp,code,msg,hdrs,newurl): response["url"] = newurl try: return urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, hdrs, newurl) except urllib.error.HTTPError as e: return e opener = urllib.request.build_opener(rdrh) outHeaders = [] for header in headers: outHeaders.append((header,headers[header])) opener.addheaders = outHeaders try: requestHandler = opener.open(url) response["headers"] = requestHandler.getheaders() response["body"] = requestHandler.read() except urllib.error.HTTPError as e: response["body"] = e.read() except Exception as e: response["body"] = html.escape(e) return response