def signon(self, url='localhost:8889', login=None, password=None, startQueue=False, verbose=False):
		"""Sign on to RTS2 JSON server. url - JSON API URL (can include username and login)"""
		# try to get username (and password) from url
		purl = urlparse.urlsplit(url)
		userpass,host = urllib.splituser(purl.netloc)
		(userpass, netloc) = urllib.splituser(purl.netloc)
		if userpass is not None:
			(login, password) = urllib.splitpasswd(userpass)
			url = netloc + purl.path
			if purl.query:
				url += '?' + query
			if purl.fragment:
				url += '#' + fragment

		if login is None:
		  	self.verbose.set_active(verbose)
			self.dialog.show_all()
			if self.run():
				self.dialog.hide()
				sys.exit(-1)
			self.dialog.hide()
		else:
			# or just create server..
			createJsonServer(url, login, password, verbose=verbose)
			getProxy().loadJson('/api/devices')
		if startQueue:
			getProxy().startQueue()
		getProxy().refresh()

    def open_http(self, url, data=None):
        """Use HTTP protocol."""
        import httplib
        user_passwd = None
        if type(url) is type(""):
            host, selector = splithost(url)
            if host:
                user_passwd, host = splituser(host)
                host = unquote(host)
            realhost = host
        else:
            host, selector = url
            urltype, rest = splittype(selector)
            url = rest
            user_passwd = None
            if string.lower(urltype) != 'http':
                realhost = None
            else:
                realhost, rest = splithost(rest)
                if realhost:
                    user_passwd, realhost = splituser(realhost)
                if user_passwd:
                    selector = "%s://%s%s" % (urltype, realhost, rest)
            #print "proxy via http:", host, selector
        if not host: raise IOError, ('http error', 'no host given')
        if user_passwd:
            import base64
            auth = string.strip(base64.encodestring(user_passwd))
        else:
            auth = None
        h = httplib.HTTP(host)
        if data is not None:
            h.putrequest('POST', selector)
            h.putheader('Content-type', 'application/x-www-form-urlencoded')
            h.putheader('Content-length', '%d' % len(data))
        else:
            h.putrequest('GET', selector)
        for cookie in self.cookies.items():
            h.putheader('Cookie', '%s=%s;' % cookie)

        if auth: h.putheader('Authorization', 'Basic %s' % auth)
        if realhost: h.putheader('Host', realhost)
        for args in self.addheaders: apply(h.putheader, args)
        h.endheaders()
        if data is not None:
            h.send(data + '\r\n')
        errcode, errmsg, headers = h.getreply()
        if headers and headers.has_key('set-cookie'):
            cookies = headers.getallmatchingheaders('set-cookie')
            for cookie in cookies: self.cookies.load(cookie)

        fp = h.getfile()
        if errcode == 200:
            return addinfourl(fp, headers, "http:" + url)
        else:
            if data is None:
                return self.http_error(url, fp, errcode, errmsg, headers)
            else:
                return self.http_error(url, fp, errcode, errmsg, headers, data)

def open_http(url, data=None):
    """Use HTTP protocol."""
    import httplib
    user_passwd = None
    proxy_passwd= None
    if isinstance(url, str):
        host, selector = urllib.splithost(url)
        if host:
            user_passwd, host = urllib.splituser(host)
            host = urllib.unquote(host)
        realhost = host
    else:
        host, selector = url
        # check whether the proxy contains authorization information
        proxy_passwd, host = urllib.splituser(host)
        # now we proceed with the url we want to obtain
        urltype, rest = urllib.splittype(selector)
        url = rest
        user_passwd = None
        if urltype.lower() != 'http':
            realhost = None
        else:
            realhost, rest = urllib.splithost(rest)
            if realhost:
                user_passwd, realhost = urllib.splituser(realhost)
            if user_passwd:
                selector = "%s://%s%s" % (urltype, realhost, rest)
            if urllib.proxy_bypass(realhost):
                host = realhost

        #print "proxy via http:", host, selector
    if not host: raise IOError, ('http error', 'no host given')

    if proxy_passwd:
        import base64
        proxy_auth = base64.b64encode(proxy_passwd).strip()
    else:
        proxy_auth = None

    if user_passwd:
        import base64
        auth = base64.b64encode(user_passwd).strip()
    else:
        auth = None
    c = FakeHTTPConnection(host)
    if data is not None:
        c.putrequest('POST', selector)
        c.putheader('Content-Type', 'application/x-www-form-urlencoded')
        c.putheader('Content-Length', '%d' % len(data))
    else:
        c.putrequest('GET', selector)
    if proxy_auth: c.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
    if auth: c.putheader('Authorization', 'Basic %s' % auth)
    if realhost: c.putheader('Host', realhost)
    for args in urllib.URLopener().addheaders: c.putheader(*args)
    c.endheaders()
    return c

def chek_pages(pages):
    try:
        for pages_url in pages:
            urllib.splitpasswd("[email protected]")
            urllib.splituser()
            code = urllib.urlopen(pages_url).getcode()
            print pages_url, code
            if code not in [200, 301]:
                failed_pages.append(pages_url)
    except socket.error, e:
        print "Ping Error", e

def open_https(self, url, data=None, ssl_context=None):
    if ssl_context is not None and isinstance(ssl_context, SSL.Context):
        self.ctx = ssl_context
    else:
        self.ctx = SSL.Context(DEFAULT_PROTOCOL)
    user_passwd = None
    if isinstance(url, basestring):
        host, selector = urllib.splithost(url)
        if host:
            user_passwd, host = urllib.splituser(host)
            host = urllib.unquote(host)
        realhost = host
    else:
        host, selector = url
        urltype, rest = urllib.splittype(selector)
        url = rest
        user_passwd = None
        if urltype.lower() != 'http':
            realhost = None
        else:
            realhost, rest = urllib.splithost(rest)
            if realhost:
                user_passwd, realhost = urllib.splituser(realhost)
            if user_passwd:
                selector = "%s://%s%s" % (urltype, realhost, rest)
        # print("proxy via http:", host, selector)
    if not host:
        raise IOError('http error', 'no host given')
    if user_passwd:
        import base64
        auth = base64.encodestring(user_passwd).strip()
    else:
        auth = None
    # Start here!
    h = httpslib.HTTPSConnection(host=host, ssl_context=self.ctx)
    # h.set_debuglevel(1)
    # Stop here!
    if data is not None:
        h.putrequest('POST', selector)
        h.putheader('Content-type', 'application/x-www-form-urlencoded')
        h.putheader('Content-length', '%d' % len(data))
    else:
        h.putrequest('GET', selector)
    if auth:
        h.putheader('Authorization', 'Basic %s' % auth)
    for args in self.addheaders:
        apply(h.putheader, args)
    h.endheaders()
    if data is not None:
        h.send(data + '\r\n')
    # Here again!
    resp = h.getresponse()
    fp = resp.fp
    return urllib.addinfourl(fp, resp.msg, "https:" + url)

def _urlclean(url):
    """Clean the url of uneccesary parts."""
    # url decode any printable normal characters except reserved characters with special meanings in urls
    for c in _urlencpattern.findall(url):
        r = chr(int(c[1:3],16))
        if r in _okurlchars:
            url = url.replace(c, r)
    # url encode any nonprintable or problematic characters (but not reserved chars)
    url = ''.join(map(lambda x: (x not in _reservedurlchars and x not in _okurlchars) and ('%%%02X' % ord(x)) or x, url))
    # split the url in useful parts (discarding fragment)
    (scheme, netloc, path, query) = urlparse.urlsplit(url)[0:4]
    if ( scheme == "http" or scheme == "https" or scheme == "ftp" ):
        # http(s) urls should have a non-empty path
        if path == "":
            path="/"
        # make hostname lower case
        (userpass, hostport) = urllib.splituser(netloc)
        netloc=hostport.lower()
        # trim trailing :
        if netloc[-1:] == ":":
            netloc = netloc[:-1]
        if userpass is not None:
            netloc = userpass+"@"+netloc
    # put the url back together again
    return urlparse.urlunsplit((scheme, netloc, path, query, ""))

    def do_cmd(self, cmd, **args):
        logging.debug('Repo path: %s' % self.path)

        url = urlparse(self.path)
        username, host = splituser(url.netloc)

        cookie = self.cookiejar[host]
        if cookie is not None:
            return super(formloginhttpsrepo, self).do_cmd(cmd, headers={'cookie' : cookie}, **args)
        else:
            logging.debug('We already have a cookie for host', host)
            try:
                return super(formloginhttpsrepo, self).do_cmd(cmd, **args)
            except error.RepoError, e:
                if 'does not appear to be an hg repository' in str(e):
                    logging.debug('Accessing repo on this host for the first time')
                    # <scheme>://<netloc>/<path>;<params>?<query>#<fragment>
                    # TODO fixed for now, but do we have a cleaner way to get rid of username:passwd from the URL?
                    scheme, netloc, path, params, query, fragment = url
                    path_without_userpasswd = '%s://%s/%s;%s?%s#%s' % (scheme, host, path, params, query, fragment)
                    cookie = get_the_cookie(path_without_userpasswd)
                    logging.info('Got cookie', cookie)
                    self.cookiejar[host] = cookie
                    return super(formloginhttpsrepo, self).do_cmd(cmd, headers={'cookie' : cookie}, **args)
            else:

 def make_connection(self, host):
     self.user_pass, self.realhost = splituser(host)
     proto, proxy, p1,p2,p3,p4 = urlparse.urlparse (self.proxies.get('http', ''))
     if proxy and not self.local:
         return httplib.HTTP(proxy)
     else:
         return httplib.HTTP(self.realhost)

    def __init__(self, url, config=Config):
        proto, uri = urllib.splittype(url)

        # apply some defaults
        if uri[0:2] != "//":
            if proto != None:
                uri = proto + ":" + uri

            uri = "//" + uri
            proto = "http"

        host, path = urllib.splithost(uri)

        try:
            int(host)
            host = "localhost:" + host
        except:
            pass

        if not path:
            path = "/"

        if proto not in ("http", "https", "httpg"):
            raise IOError, "unsupported SOAP protocol"
        if proto == "httpg" and not config.GSIclient:
            raise AttributeError, "GSI client not supported by this Python installation"
        if proto == "https" and not config.SSLclient:
            raise AttributeError, "SSL client not supported by this Python installation"

        self.user, host = urllib.splituser(host)
        self.proto = proto
        self.host = host
        self.path = path

def test_node(request):
    """Runs an SSH call on a node."""
    if authenticated_userid(request) is None:
        raise Forbidden()
    # trying an ssh connection
    connection = paramiko.client.SSHClient()
    connection.load_system_host_keys()
    connection.set_missing_host_key_policy(paramiko.WarningPolicy())
    name = request.matchdict['name']

    host, port = urllib.splitport(name)
    if port is None:
        port = 22

    username, host = urllib.splituser(host)
    credentials = {}

    if username is not None and ':' in username:
        username, password = username.split(':', 1)
        credentials = {"username": username, "password": password}
    elif username is not None:
        password = None
        credentials = {"username": username}

    try:
        connection.connect(host, port=port, timeout=5, **credentials)
        return 'Connection to %r : OK' % name
    except (socket.gaierror, socket.timeout), error:
        return str(error)

    def parse_address_info(cls, server_addr="nats://nats:[email protected]:4222"):
        '''\
        parse the metadata nats server uri;

        Params:
        =====
        addr: nats server address;

        Returns:
        =====
        user: username to login nats server;
        pswd: password to login nats server;
        host: ip address of nats server;
        port: port of nats server
        '''

        if type(server_addr) is not str:
            raise NotImplementException

        protocol, after_split = urllib.splittype(server_addr)

        if not protocol == "nats":
            raise NotImplementException

        auth_len = len(server_addr.split('@'))
        if auth_len > 1:
            auth, after_split = urllib.splituser(after_split)
            user_raw, pswd = urllib.splitpasswd(auth)
            user = user_raw.lstrip("/")
            _, after_split = urllib.splithost(after_split)
            host, port = urllib.splitport(after_split)
        else:
            user = pswd = None
            host, port = urllib.splitport(after_split)
        return user, pswd, host, int(port)

def open_with_auth(url):
    """Open a urllib2 request, handling HTTP authentication"""

    scheme, netloc, path, params, query, frag = urlparse.urlparse(url)

    if scheme in ('http', 'https'):
        auth, host = urllib.splituser(netloc)
    else:
        auth = None

    if auth:
        auth = "Basic " + urllib2.unquote(auth).encode('base64').strip()
        new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
        request = urllib2.Request(new_url)
        request.add_header("Authorization", auth)
    else:
        request = urllib2.Request(url)

    request.add_header('User-Agent', user_agent)
    fp = urllib2.urlopen(request)

    if auth:
        # Put authentication info back into request URL if same host,
        # so that links found on the page will work
        s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)
        if s2==scheme and h2==host:
            fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2))

    return fp

 def open_http(self, url):
     """Use HTTP protocol."""
     if isinstance(url, str):
         host, selector = urllib.splithost(url)
         if host:
             user_passwd, host = urllib.splituser(host)
             host = urllib.unquote(host)
         realhost = host
     else:
         host, selector = url
         urltype, rest = urllib.splittype(selector)
         url = rest
         user_passwd = None
         if urltype.lower() != "http":
             realhost = None
         else:
             realhost, rest = splithost(rest)
             if realhost:
                 user_passwd, realhost = splituser(realhost)
             if user_passwd:
                 selector = "%s://%s%s" % (urltype, realhost, rest)
             if proxy_bypass(realhost):
                 host = realhost
     if not host:
         return -2
     h = httplib.HTTP(host)
     h.putrequest("GET", selector)
     if realhost:
         h.putheader("Host", realhost)
     for args in self.addheaders:
         h.putheader(*args)
     h.endheaders()
     errcode, errmsg, headers = h.getreply()
     return errcode

def getUrlFd(url, headers=None, data=None, timeout=None):
    """getUrlFd(url, headers=None, data=None)

    Opens the given url and returns a file object.  Headers and data are
    a dict and string, respectively, as per urllib2.Request's arguments."""
    if headers is None:
        headers = defaultHeaders
    try:
        if not isinstance(url, urllib2.Request):
            (scheme, loc, path, query, frag) = urlparse.urlsplit(url)
            (user, host) = urllib.splituser(loc)
            url = urlparse.urlunsplit((scheme, host, path, query, ''))
            request = urllib2.Request(url, headers=headers, data=data)
            if user:
                request.add_header('Authorization',
                                   'Basic %s' % base64.b64encode(user))
        else:
            request = url
            request.add_data(data)
        httpProxy = force(proxy)
        if httpProxy:
            request.set_proxy(httpProxy, 'http')
        fd = urllib2.urlopen(request, timeout=timeout)
        return fd
    except socket.timeout, e:
        raise Error, TIMED_OUT

def dnslookup(url):
    """Replaces a hostname by its IP in an url.

    Uses gethostbyname to do a DNS lookup, so the nscd cache is used.

    If gevent has patched the standard library, makes sure it uses the
    original version because gevent uses its own mechanism based on
    the async libevent's evdns_resolve_ipv4, which does not use
    glibc's resolver.
    """
    try:
        from gevent.socket import _socket
        gethostbyname = _socket.gethostbyname
    except ImportError:
        import socket
        gethostbyname = socket.gethostbyname

    # parsing
    parsed_url = urlparse.urlparse(url)
    host, port = urllib.splitport(parsed_url.netloc)
    user, host = urllib.splituser(host)

    # resolving the host
    host = gethostbyname(host)

    # recomposing
    if port is not None:
        host = '%s:%s' % (host, port)

    if user is not None:
        host = '%[email protected]%s' % (user, host)

    parts = [parsed_url[0]] + [host] + list(parsed_url[2:])
    return urlparse.urlunparse(parts)

    def __init__(self, url, config = Config):
        proto, uri = urllib.splittype(url)

        # apply some defaults
        if uri[0:2] != '//':
            if proto != None:
                uri = proto + ':' + uri

            uri = '//' + uri
            proto = 'http'

        host, path = urllib.splithost(uri)

        try:
            int(host)
            host = 'localhost:' + host
        except:
            pass

        if not path:
            path = '/'

        if proto not in ('http', 'https', 'httpg'):
            raise IOError, "unsupported SOAP protocol"
        if proto == 'httpg' and not config.GSIclient:
            raise AttributeError, \
                  "GSI client not supported by this Python installation"
        if proto == 'https' and not config.SSLclient:
            raise AttributeError, \
                "SSL client not supported by this Python installation"

        self.user,host = urllib.splituser(host)
        self.proto = proto
        self.host = host
        self.path = path

    def __init__(self, baseUri, headers=None, maxClients=None,
        maxConnections=None):

        self._headers = headers or HTTPHeaders()

        self._user = None
        self._passwd = None

        baseUri = baseUri.rstrip('/')
        self._scheme, loc, self._path, query, frag = urlparse.urlsplit(baseUri)

        userpass, self._hostport = urllib.splituser(loc)
        if userpass:
            self._user, self._passwd = urllib.splitpasswd(userpass)

        self._baseUri = urlparse.urlunsplit((self._scheme, self._hostport,
            self._path, None, None))

        if self._scheme not in ('http', 'https'):
            raise ValueError(self._scheme)

        self._dispatcher = RequestDispatcher(maxClients=maxClients,
            maxConnections=maxConnections)

        self._queryFragment = urlparse.urlunsplit(('', '', '', query, frag))

    def get_host_info(self, host):
        """Get authorization info from host parameter

        Host may be a string, or a (host, x509-dict) tuple; if a string,
        it is checked for a "user:[email protected]" format, and a "Basic
        Authentication" header is added if appropriate.

        @param host Host descriptor (URL or (URL, x509 info) tuple).
        @return A 3-tuple containing (actual host, extra headers,
        x509 info).  The header and x509 fields may be None.

        """
        x509 = {}
        if isinstance(host, types.TupleType):
            host, x509 = host

        auth, host = urllib.splituser(host)

        if auth:
            auth = base64.encodestring(urllib.unquote(auth))
            auth = string.join(string.split(auth), "") # get rid of whitespace
            extra_headers = [
                ("Authorization", "Basic " + auth)
                ]
        else:
            extra_headers = None

        return host, extra_headers, x509

 def build_url_parts (self):
     """Set userinfo, host, port and anchor from self.urlparts.
     Also checks for obfuscated IP addresses.
     """
     # check [email protected]:port syntax
     self.userinfo, host = urllib.splituser(self.urlparts[1])
     port = urlutil.default_ports.get(self.scheme, 0)
     host, port = urlutil.splitport(host, port=port)
     if port is None:
         raise LinkCheckerError(_("URL host %(host)r has invalid port") %
                 {"host": host})
     self.port = port
     # set host lowercase
     self.host = host.lower()
     if self.scheme in scheme_requires_host:
         if not self.host:
             raise LinkCheckerError(_("URL has empty hostname"))
         self.check_obfuscated_ip()
     if not self.port or self.port == urlutil.default_ports.get(self.scheme):
         host = self.host
     else:
         host = "%s:%d" % (self.host, self.port)
     if self.userinfo:
         self.urlparts[1] = "%[email protected]%s" % (self.userinfo, host)
     else:
         self.urlparts[1] = host
     # safe anchor for later checking
     self.anchor = self.urlparts[4]
     if self.anchor is not None:
         assert isinstance(self.anchor, unicode), repr(self.anchor)

 def url_permutations(url):
     """Try all permutations of hostname and path which can be applied
     to blacklisted URLs"""
     def url_host_permutations(host):
         if re.match(r'\d+\.\d+\.\d+\.\d+', host):
             yield host
             return
         parts = host.split('.')
         l = min(len(parts),5)
         if l > 4:
             yield host
         for i in xrange(l-1):
             yield '.'.join(parts[i-l:])
     def url_path_permutations(path):
         if path != '/':
             yield path
         query = None
         if '?' in path:
             path, query =  path.split('?', 1)
         if query is not None:
             yield path
         path_parts = path.split('/')[0:-1]
         curr_path = ''
         for i in xrange(min(4, len(path_parts))):
             curr_path = curr_path + path_parts[i] + '/'
             yield curr_path
     protocol, address_str = urllib.splittype(url)
     host, path = urllib.splithost(address_str)
     user, host = urllib.splituser(host)
     host, port = urllib.splitport(host)
     host = host.strip('/')
     for h in url_host_permutations(host):
         for p in url_path_permutations(path):
             yield '%s%s' % (h, p)