BirdyNet/addons/0.pageDownloader.py

global urllib
import urllib.request

global html
import html

global downloadPage
def downloadPage(url,headers = False):
	if not headers: headers = {}
	response = {
		"url": url,
		"body": None,
		"headers": []
	}
	
	class rdrh(urllib.request.HTTPRedirectHandler):
		def redirect_request(self,req,fp,code,msg,hdrs,newurl):
			response["url"] = newurl
			try:
				return urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, hdrs, newurl)
			except urllib.error.HTTPError as e:
				return e
	
	opener = urllib.request.build_opener(rdrh)
	outHeaders = []
	for header in headers:
		outHeaders.append((header,headers[header]))
	opener.addheaders = outHeaders
	
	try:
		requestHandler = opener.open(url)
		response["headers"] = requestHandler.getheaders()
		response["body"] = requestHandler.read()
	except urllib.error.HTTPError as e:
		response["body"] = e.read()
	except Exception as e:
		response["body"] = html.escape(e)
	
	return response