def download_from_url(url,file_name):
	global proxies	
	try:
		if proxies['http']!="":
			proxy_handler = urllib2.ProxyHandler(proxies)
			opener = urllib2.build_opener(proxy_handler)
			urllib2.install_opener(opener)
			# opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299')]
		f = urllib2.urlopen(url,timeout=6)
		data = f.read()
		with open(file_name, "wb") as code:
			code.write(data)
	except urllib2.HTTPError as e:
		print('HTTPError:%s'%url)
		print(e.code)
		if e.code == 404:
			print("访问的url不存在")
		elif e.code == 456:
			print("访问被拦截")
			proxies = get_ip()
			print(proxies)
			download_from_url(url,file_name)
		# print(e.read())
	except urllib2.URLError as e:
		print('URLError:%s'%url)
		print(e)
		download_from_url(url,file_name)
	except Exception as e:
		print('error')
		print(e)

    def __do_login(self):
        cookie_jar2 = cookielib.LWPCookieJar()
        cookie_support2 = request.HTTPCookieProcessor(cookie_jar2)
        opener2 = request.build_opener(cookie_support2, request.HTTPHandler)
        request.install_opener(opener2)

        req = request.Request('https://passport.weibo.cn/sso/login')
        req.add_header('Origin', 'https://passport.weibo.cn')
        req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
        req.add_header('Referer', 'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=http%3A%2F%2Fm.weibo.cn%2F')

        login_data = parse.urlencode([
            ('username', self.name),
            ('password', self.pwd),
            ('entry', 'mweibo'),
            ('client_id', ''),
            ('savestate', '1'),
            ('ec', ''),
            ('pagerefer', 'https://passport.weibo.cn/signin/welcome?entry=mweibo&r=http%3A%2F%2Fm.weibo.cn%2F')
        ])

        with request.urlopen(req,data=login_data.encode('utf-8')) as f:
            cookie_jar2.save(self.cookies_path, ignore_discard=True, ignore_expires=True)
            back_html = f.read()
            back_data = json.loads(back_html)
            if back_data.retcode==20000000:
                self.uid = back_data.data.uid
                return True
            else:
                return False

    def openurl(self,url):
        """
       
        """
        cookie_support= urllib2.HTTPCookieProcessor(cookielib.CookieJar())
        self.opener = urllib2.build_opener(cookie_support,urllib2.HTTPHandler)
        urllib2.install_opener(self.opener)
        user_agents = [
                    'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
                    'Opera/9.25 (Windows NT 5.1; U; en)',
                    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
                    'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
                    'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
                    'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
                    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
                    "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 ",

                    ] 
       
        agent = random.choice(user_agents)
        self.opener.addheaders = [("User-agent",agent),("Accept","*/*"),('Referer','http://www.yahoo.com')]
        try:
            res = self.opener.open(url)
	    #return res
            #print res.read()
        except Exception as e:
            self.speak(str(e))
            raise Exception
        else:
            return res.read()

  def execute(self, method, *args, **kwargs):
    header = {
        'Content-Type' : 'application/json',
        'User-Agent' : 'python-xbmc'
        }
    # Params are given as a dictionnary
    if len(args) == 1:
      args=args[0]
      params = kwargs
      # Use kwargs for param=value style
    else:
      args = kwargs
    params={}
    params['jsonrpc']='2.0'
    params['id']=self.id
    self.id +=1
    params['method']=method
    params['params']=args

    values=json.dumps(params)
    # HTTP Authentication
    password_mgr = HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, self.url, self.username, self.password)
    auth_handler = HTTPBasicAuthHandler(password_mgr)
    opener = build_opener(auth_handler)
    install_opener(opener)

    data = values
    req = Request(self.url, data.encode('utf-8'), header)
    response = urlopen(req)
    the_page = response.read()
    if len(the_page) > 0 :
      return json.load(StringIO(the_page.decode('utf-8')))
    else:
      return None # for readability

def setup_urllib():
    password_mgr = request.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, root_url, USER, PASS)
    handler = request.HTTPBasicAuthHandler(password_mgr)
    opener = request.build_opener(handler)
    opener.open(root_url)
    request.install_opener(opener)

    def download(self, source, dest):
        """
        Download an archive file.

        :param str source: URL pointing to an archive file.
        :param str dest: Local path location to download archive file to.
        """
        # propogate all exceptions
        # URLError, OSError, etc
        proto, netloc, path, params, query, fragment = urlparse(source)
        if proto in ('http', 'https'):
            auth, barehost = splituser(netloc)
            if auth is not None:
                source = urlunparse((proto, barehost, path, params, query, fragment))
                username, password = splitpasswd(auth)
                passman = HTTPPasswordMgrWithDefaultRealm()
                # Realm is set to None in add_password to force the username and password
                # to be used whatever the realm
                passman.add_password(None, source, username, password)
                authhandler = HTTPBasicAuthHandler(passman)
                opener = build_opener(authhandler)
                install_opener(opener)
        response = urlopen(source)
        try:
            with open(dest, 'w') as dest_file:
                dest_file.write(response.read())
        except Exception as e:
            if os.path.isfile(dest):
                os.unlink(dest)
            raise e

 def _build_request(self, url, json_header=False, method='GET', data=None):
     if self.proxies:
         proxy_support = urllib_request.ProxyHandler(self.proxies)
         opener = urllib_request.build_opener(proxy_support)
         urllib_request.install_opener(opener)
     req = urllib_request.Request(url)
     req.get_method = lambda: method
     req.add_header('User-Agent', self.useragent)
     req.add_header('Authorization', 'Bearer ' + self.apikey)
     if json_header:
         req.add_header('Content-Type', 'application/json')
     try:
         if data:
             response = urllib_request.urlopen(req, json.dumps(data))
         else:
             response = urllib_request.urlopen(req, timeout=10)
     except HTTPError as e:
         if 400 <= e.code < 500:
             raise SendGridClientError(e.code, e.read())
         elif 500 <= e.code < 600:
             raise SendGridServerError(e.code, e.read())
         else:
             assert False
     except timeout as e:
         raise SendGridClientError(408, 'Request timeout')
     body = response.read()
     return response.getcode(), body

def atack(ip, users, passwords):
    passwords = passwords.read().split("\n")
    find = False
    realm_router = get_realm(ip)

    for u in users:
        u2 = u.strip()
        for p in passwords:
            p2 = p.strip()
            try:
                auth_handler = ur.HTTPBasicAuthHandler()
                auth_handler.add_password(realm=realm_router,
                                        uri=ip,
                                        user=u2,
                                        passwd=p2)
                opener = ur.build_opener(auth_handler)
                ur.install_opener(opener)
                pag = ur.urlopen("http://" + str(ip))
                if(pag.getcode() == 200):
                    print(chr(27) + "[1;34m[+]"+ chr(27) 
                      + "[0m Login found: " + str(u2) + ":" + str(p2))
                    find = True
            except:
                print(chr(27) + "[1;31m[-] "+ chr(27) 
                      + "[0m" + str(u2) + ":" + str(p2) + " >> failed" )
    if not find:
        print("Login not found.")

def login(n):
	'''
login xiami.
arugments:
	n:
return:
	url opener:when succeeded
	None:when failed or have signed in
'''
	cookie_support= HTTPCookieProcessor(CookieJar())
	opener = build_opener(cookie_support, HTTPHandler)
	install_opener(opener)

	lf=login_form
	lf.update(user_info[n])
	req=Request(url=site_urls["login"],data=urlencode(lf).encode("utf-8"),headers=headers)
	content=urlopen(req).read().decode("utf-8")
	
	if content.find(site_flags["signed-in"])!=-1:
		print("%s：已签过。\r\n"%user_info[n]["email"])
	elif content.find(site_flags["login-failed"])!=-1:
		print("%s：邮箱或密钥错误。\r\n"%user_info[n]["email"])
	elif content.find(site_flags["identify-required"])!=-1:
		print("%s：虾米要求输入验证码，请断网后重新尝试。\r\n"%user_info[n]["email"])
	elif content.find(site_flags["logged-in"])!=-1:
		print("%s：登录成功。"%user_info[n]["email"])
		return opener
	return None

 def getFile(cls, getfile):
     if cls.getProxy():
         proxy = req.ProxyHandler({'http': cls.getProxy(), 'https': cls.getProxy()})
         auth = req.HTTPBasicAuthHandler()
         opener = req.build_opener(proxy, auth, req.HTTPHandler)
         req.install_opener(opener)
     return req.urlopen(getfile)

    def __init__(self, url, userAgent = '', cookieJar = CookieJar(), maxlag = 5):
        if 'http://' not in url and 'https://' not in url:
            url = 'http://' + url # Append http:// if it's not there already.

        if 'api.php' in url:
            self.scriptPath = url[:-7]
        elif 'index.php' in url:
            self.scriptPath = url[:-9]
        elif url[-1:] == '/':
            self.scriptPath = url
        else:
            self.scriptPath = url + '/'

        if userAgent == '':
            out = subprocess.Popen(["git", "log", "-n 1", '--pretty=format:%H'], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
            stdout, stderr = out.communicate()
            if out.returncode == 0:
                userAgent = 'PyMediaWikiClient/git/' + stdout

        self.userAgent = userAgent
        self.cookieJar = cookieJar
        self.maxlagDefault = maxlag
        install_opener(build_opener(HTTPCookieProcessor(self.cookieJar)))
        self.isLoggedIn = False
        self.tokens = {}
        self.getUserInfo()
        try:
            self.getToken(name = 'edit', cached = False)
        except:
            pass

def teardown():
    """Restore the normal HTTP handler"""
    install_opener(build_opener(HTTPHandler))
    # Go back to the original folder
    os.chdir(ORIGINAL_FOLDER)
    # remove the test folder
    shutil.rmtree(TEST_FOLDER)

 def metricCollector3(self):
     try:
         if (self._userName and self._userPass):
                 password_mgr = urlconnection.HTTPPasswordMgr()
                 password_mgr.add_password(self._realm, self._url, self._userName, self._userPass)
                 auth_handler = urlconnection.HTTPBasicAuthHandler(password_mgr)
                 opener = urlconnection.build_opener(auth_handler)
                 urlconnection.install_opener(opener)
         response = urlconnection.urlopen(self._url, timeout=10)
         if response.status == 200:
             byte_responseData = response.read()
             str_responseData = byte_responseData.decode('UTF-8')
             self._parseStats(str_responseData)
         else:
             self.dictApacheData['status'] = 0
             self.dictApacheData['msg'] = 'Error_code' + str(response.status)
     except HTTPError as e:
         self.dictApacheData['status'] = 0
         self.dictApacheData['msg'] = 'Error_code : HTTP Error ' + str(e.code)
     except URLError as e:
         self.dictApacheData['status'] = 0
         self.dictApacheData['msg'] = 'Error_code : URL Error ' + str(e.reason)
     except InvalidURL as e:
         self.dictApacheData['status'] = 0
         self.dictApacheData['msg'] = 'Error_code : Invalid URL'
     except Exception as e:
         self.dictApacheData['status'] = 0
         self.dictApacheData['msg'] = 'Exception occured in collecting data : ' + str(e)

def config_request(request_url, request_data, method, proxy):

    requests = False
    cookie_handler = False
    proxy_handler = False
    headers = False
    request_datas = False
    opener = False

    if request_data and method:
        request_datas = set_request_datas(request_data)
    if proxy and len(proxy) > 0:
        proxy_handler = set_proxy(proxy)

    cookie_handler, cookie = set_cookies()
    headers = set_headers(request_url)
    if cookie_handler and proxy_handler:
        opener = request.build_opener(cookie_handler, proxy_handler)
        request.install_opener(opener)

    if not method:
        requests = request.Request(request_url, headers=headers)  #static
    elif method == 'POST':
        requests = request.Request(
            request_url, request_datas, headers=headers)  #post
    elif method == 'GET':
        get_request_url = request_url + '?' + request_datas
        requests = request.Request(get_request_url, headers=headers)  #get

    return requests, cookie

	def __init__(self,stuid,pwd,io):
		super(Login, self).__init__()
		global INFO
		INFO['id'] = stuid
		INFO['pwd'] = pwd
		INFO['io'] = io
		self._LOGIN_DATA = {
							"Login.Token1":INFO['id'],
							"Login.Token2":INFO['pwd'],
							"goto":"http://portal.uestc.edu.cn/loginSuccess.portal",
							"gotoOnFail":"http://portal.uestc.edu.cn/loginFailure.portal"
							}
		self._HEADERS = {
						"Accept": "Accept:text/html,application/xhtml+xml,image/gif,image/x-xbitmap,image/jpeg,image/pjpeg,application/x-shockwave-flash,application/xml;q=0.9,*/*;q=0.8",
						"Accept-Charset":"GBK,utf-8;q=0.7,*;q=0.3",
						"Accept-Language": "zh-CN,zh;q=0.8",
						"Accept-Encoding": "gzip, deflate",
						"Referer":"http://portal.uestc.edu.cn/index.portal",
						"Host":"portal.uestc.edu.cn",
						"Content-Type": "application/x-www-form-urlencoded",
						'User-agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
						"Connection": "Keep-Alive",
						"Cache-Control": "no-cache",
						}

		self._CJ = cookiejar.CookieJar()
		self._OPENER = request.build_opener(request.HTTPCookieProcessor(self._CJ),SimpleCookieHandler(),MyHTTPHandler())
		request.install_opener(self._OPENER)
		self._LOGIN_TAG = False
		self._URL = 'http://'+INFO['host_portal']+'/userPasswordValidate.portal'
		try:
			self.login()
		except:
			print ("当前网络不稳定，请稍后再试！")
			exit(1)

def POST(url, args={}, cred=None):
    """do http post

    url is the URL you want
    args is a dict of cgi args
    cred is ( host, realm, username, password )
    """

    auth_handler = None

    arg_string = ''

    if cred is not None:
        (host, realm, username, password) = cred
        auth_handler = HTTPBasicAuthHandler()
        auth_handler.add_password(realm, host, username, password)

    if auth_handler:
        opener = build_opener(cookie_processor, auth_handler)
    else:
        opener = build_opener(cookie_processor)

    install_opener(opener)

    print("URL %s" % url)
    data = urlencode(args)
    req = Request(url, data)
    f = urlopen(req)
    return f

 def change_proxy(self):
     self.current_proxy=self.proxy_manager.request_proxy(1,drop=[self.current_proxy])[0]
     print('--- change proxy to ',self.current_proxy,' --- ')
     self.__cj = http.cookiejar.CookieJar()
     proxy_handler=request.ProxyHandler({'http':self.current_proxy})
     self.opener = request.build_opener(request.HTTPCookieProcessor(self.__cj),proxy_handler)
     request.install_opener(self.opener)

    def install_proxy(self):
        """set proxy if one is set in QGIS network settings"""

        # initially support HTTP for now
        if self.settings.value('/proxy/proxyEnabled') == 'true':
            if self.settings.value('/proxy/proxyType') == 'HttpProxy':
                ptype = 'http'
            else:
                return

            user = self.settings.value('/proxy/proxyUser')
            password = self.settings.value('/proxy/proxyPassword')
            host = self.settings.value('/proxy/proxyHost')
            port = self.settings.value('/proxy/proxyPort')

            proxy_up = ''
            proxy_port = ''

            if all([user != '', password != '']):
                proxy_up = '%s:%[email protected]' % (user, password)

            if port != '':
                proxy_port = ':%s' % port

            conn = '%s://%s%s%s' % (ptype, proxy_up, host, proxy_port)
            install_opener(build_opener(ProxyHandler({ptype: conn})))

def exrtract_page(url, title):
    proxies = {'http': '127.0.0.1:8118'}
    proxy = ProxyHandler(proxies)
    opener = build_opener(proxy)
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    install_opener(opener)

    file_path = HTML_dir + title + '.html'
    if os.path.exists(file_path):
        return

    request = Request(url)
    content = urlopen(request, timeout=30).read()
    soup = BeautifulSoup(content)

    page_title = soup.find('h1', {'itemprop': 'name'}).get_text()
    header = \
        '<h1>' \
        + page_title \
        + '</h1>\n'

    main_content = soup.find('div', {'itemprop': 'articleBody'}).prettify(formatter='html')
    main_content = main_content.replace('/images/', 'http://developer.android.com/images/')

    output = open(file_path, 'w')
    output.write(
            header
            + main_content)

    def readUrl(self,host,port,url,user,password):
        error=False
        tomcatUrl = "http://"+host+":"+str(port)+url
        try:
            pwdManager = urlconnection.HTTPPasswordMgrWithDefaultRealm()
            pwdManager.add_password(None,tomcatUrl,user,password)
            authHandler = urlconnection.HTTPBasicAuthHandler(pwdManager)
            opener=urlconnection.build_opener(authHandler)
            urlconnection.install_opener(opener)
            req = urlconnection.Request(tomcatUrl)
            handle = urlconnection.urlopen(req, None)
            data = handle.read()
        except HTTPError as e:
            if(e.code==401):
                data="ERROR: Unauthorized user. Does not have permissions. %s" %(e)
            elif(e.code==403):
                data="ERROR: Forbidden, yours credentials are not correct. %s" %(e)
            else:
                data="ERROR: The server couldn\'t fulfill the request. %s" %(e)
            error=True
        except URLError as e:
            data = 'ERROR: We failed to reach a server. Reason: %s' %(e.reason)
            error = True
        except socket.timeout as e:
            data = 'ERROR: Timeout error'
            error = True
        except socket.error as e:
            data = "ERROR: Unable to connect with host "+self.host+":"+self.port
            error = True
        except:
            data = "ERROR: Unexpected error: %s"%(sys.exc_info()[0])
            error = True

        return data,error