def get_process_status(hostip, port, username, password, linklocalip, process, hypervisor=None):
    """Double hop and returns a process status"""

    #SSH to the machine
    ssh = SshClient(hostip, port, username, password)
    if (str(hypervisor).lower() == 'vmware'
		or str(hypervisor).lower() == 'hyperv'):
        ssh_command = "ssh -i /var/cloudstack/management/.ssh/id_rsa -ostricthostkeychecking=no "
    else:
        ssh_command = "ssh -i ~/.ssh/id_rsa.cloud -ostricthostkeychecking=no "

    ssh_command = ssh_command +\
                  "-oUserKnownHostsFile=/dev/null -p 3922 %s %s" % (
                      linklocalip,
                      process)

    # Double hop into router
    if str(hypervisor).lower() == 'hyperv':
        timeout = 12
    else:
        timeout = 5
    # Ensure the SSH login is successful
    while True:
        res = ssh.execute(ssh_command)
        if "Connection refused".lower() in res[0].lower():
            pass
        elif res[0] != "Host key verification failed.":
            break
        elif timeout == 0:
            break

        time.sleep(5)
        timeout = timeout - 1
    return res

def download_systemplates_sec_storage(server, services):
    """Download System templates on sec storage"""

    try:
        # Login to management server
        ssh = SshClient(server["ipaddress"], server["port"], server["username"], server["password"])
    except Exception:
        raise Exception("SSH access failed for server with IP address: %s" % server["ipaddess"])
    # Mount Secondary Storage on Management Server
    cmds = [
        "mkdir -p %s" % services["mnt_dir"],
        "mount -t nfs %s:/%s %s" % (services["sec_storage"], services["path"], services["mnt_dir"]),
        "%s -m %s -u %s -h %s -F"
        % (services["command"], services["mnt_dir"], services["download_url"], services["hypervisor"]),
    ]
    for c in cmds:
        result = ssh.execute(c)

    res = str(result)

    # Unmount the Secondary storage
    ssh.execute("umount %s" % (services["mnt_dir"]))

    if res.count("Successfully installed system VM template") == 1:
        return
    else:
        raise Exception("Failed to download System Templates on Sec Storage")
    return

    def setUp(self):
        self.apiclient = self.testClient.getApiClient()
        self.dbclient = self.testClient.getDbConnection()
        self.mgtSvrDetails = self.config.__dict__["mgtSvr"][0].__dict__
        self.cleanup = []
        self.testdata = {
            "account": {
                "email": "[email protected]",
                "firstname": "Marvin",
                "lastname": "TestUser",
                "username": "staticrole_acctest-",
                "password": "password",
            }
        }

        feature_enabled = self.apiclient.listCapabilities(listCapabilities.listCapabilitiesCmd()).dynamicrolesenabled
        if feature_enabled:
            self.skipTest("Dynamic role-based API checker is enabled, skipping tests for static role-base API checker")

        commandsProperties = []
        try:
            sshClient = SshClient(
                self.mgtSvrDetails["mgtSvrIp"],
                22,
                self.mgtSvrDetails["user"],
                self.mgtSvrDetails["passwd"],
                retries=1,
                log_lvl=logging.INFO
            )
            result = sshClient.runCommand("cat /etc/cloudstack/management/commands.properties")
            if 'status' in result and result['status'] == 'SUCCESS' and 'stdout' in result and len(result['stdout']) > 0:
                commandsProperties = result['stdout']
        except Exception:
            self.debug("Failed to ssh into mgmt server host and grab commands.properties file")
            testDir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
            localFileName = os.path.abspath(testDir + "/../../../client/tomcatconf/commands.properties.in")
            if os.path.isfile(localFileName):
                self.info("Detected that we're running in developer mode with maven, using file at:" + localFileName)
                with open(localFileName) as f:
                    commandsProperties = f.readlines()

        if len(commandsProperties) < 1:
            self.skipTest("Unable to find commands.properties, skipping this test")

        apiMap = {}
        for line in commandsProperties:
            if not line or line == '' or line == '\n' or line.startswith('#'):
                continue
            name, value = line.split('=')
            apiMap[name.strip()] = value.strip()

        self.roleApiMap = {} # role to list of apis allowed
        octetKey = {'Admin':1, 'DomainAdmin':4, 'User':8}
        for role in octetKey.keys():
            for api in sorted(apiMap.keys()):
                if (octetKey[role] & int(apiMap[api])) > 0:
                    if role not in self.roleApiMap:
                        self.roleApiMap[role] = []
                    self.roleApiMap[role].append(api)

 def RestartServers(self):
     """ Restart management
     server and usage server """
     sshClient = SshClient(
         self.mgtSvrDetails["mgtSvrIp"], 22, self.mgtSvrDetails["user"], self.mgtSvrDetails["passwd"]
     )
     command = "service cloudstack-management restart"
     sshClient.execute(command)
     return

 def checkHostUp(self, fromHostIp, testHostIp):
     try:
         ssh = SshClient(fromHostIp, 22, "root", "password") 
         res = ssh.execute("ping -c 1 %s" % testHostIp)
         result = str(res)
         if result.count(" 0% packet loss") == 1:
             return True, 1
         else:
             return False, 1
     except Exception as e:
         self.logger.debug("Got exception %s" % e)
         return False, 1

def restartUsageServer(self):
    #Restart usage server

    sshClient = SshClient(
        self.mgtSvrDetails["mgtSvrIp"],
        22,
        self.mgtSvrDetails["user"],
        self.mgtSvrDetails["passwd"]
    )
    command = "service cloudstack-usage restart"
    sshClient.execute(command)
    return

    def RestartServer(cls):
        """Restart management server"""

        sshClient = SshClient(
                cls.mgtSvrDetails["mgtSvrIp"],
                22,
                cls.mgtSvrDetails["user"],
                cls.mgtSvrDetails["passwd"]
                )
        command = "service cloudstack-management restart"
        sshClient.execute(command)

        return

    def try_ssh(self, ip_addr, hostnames):
        try:
            self.debug("SSH into NAT Rule (Public IP: %s)" % ip_addr)

            # If Round Robin Algorithm is chosen,
            # each ssh command should alternate between VMs

            ssh_1 = SshClient(ip_addr, 22, self.services["natrule"]["username"], self.services["natrule"]["password"])
            hostnames.append(ssh_1.execute("hostname")[0])
            self.debug(hostnames)
        except Exception as e:
            self.fail("%s: SSH failed for VM with IP Address: %s" % (e, ip_addr))
        return hostnames

    def setUpClass(self):
        testClient = super(TestDeployvGPUenabledVM, self).getClsTestClient()
        self.apiclient = testClient.getApiClient()
        self.testdata = self.testClient.getParsedTestDataConfig()
        self._cleanup = []
        self.unsupportedHypervisor = False
        self.noSuitableHost = False
        # Need to add check whether zone containing the xen hypervisor or not
        # as well
        hosts = list_hosts(
            self.apiclient,
            hypervisor="XenServer"
        )
        if hosts is None:
             # GPU feature is supported only on XenServer.Check listhosts response
             self.unsupportedHypervisor = True
             return
        else:
            gpuhosts = 0
            for ghost in hosts:
                if ghost.hypervisorversion >= "6.2.0":
                    sshClient = SshClient(
                        host=ghost.ipaddress,
                        port=self.testdata['configurableData']['host']["publicport"],
                        user=self.testdata['configurableData']['host']["username"],
                        passwd=self.testdata['configurableData']['host']["password"])
                    if ghost.hypervisorversion == "6.2.0":
                        res = sshClient.execute(
                            "xe patch-list uuid=0850b186-4d47-11e3-a720-001b2151a503")
                        if len(res) == 0:
                            continue
                    res = sshClient.execute(
                        "xe vgpu-type-list model-name=\"GRID K120Q\"")
                    if len(res) != 0:
                        gpuhosts = gpuhosts + 1
                    else:
                        continue
        if gpuhosts == 0:
            # No XenServer available with GPU Drivers installed
            self.noSuitableHost = True
            return

        self.domain = get_domain(self.apiclient)
        self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
        # Creating Account
        self.account = Account.create(
            self.apiclient,
            self.testdata["account"],
            domainid=self.domain.id
        )
        self._cleanup.append(self.account)

 def test_es_1236_cloudstack_sccs(self):
     """
     @Desc: Test whether cloudstack-sccs is available on management server
     @Steps:
     Step1: run cloudstack-sccs on management server
     Step2: It should return a commit hash
     """
     # Step1: run cloudstack-sccs on management server
     mgmt_ssh = SshClient(
         self.apiClient.connection.mgtSvr, 22, self.apiClient.connection.user, self.apiClient.connection.passwd
     )
     res = mgmt_ssh.execute("cloudstack-sccs")
     # Step2: It should return a commit hash
     return

    def test_multiple_mgmt_srvr_session_timeout(self):
        """
        @Desc: Check whether mgmt server session times out with in 30s
        @Steps:
        Step1: run 'telnet localhot 8250' on the management server
        and see that it times out with in 30seconds
        """
        # Step1: run cloudstack-sccs on management server
        mgmt_ssh = SshClient(
            self.apiClient.connection.mgtSvr, 22, self.apiClient.connection.user, self.apiClient.connection.passwd
        )
        res = mgmt_ssh.execute("time telnet localhost 8250")

        # Step2: It should return a commit hash
        return

def _execute_ssh_command(hostip, port, username, password, ssh_command):
    #SSH to the machine
    ssh = SshClient(hostip, port, username, password)
    # Ensure the SSH login is successful
    while True:
        res = ssh.execute(ssh_command)
        if "Connection refused".lower() in res[0].lower():
            pass
        elif res[0] != "Host key verification failed.":
            break
        elif timeout == 0:
            break

        time.sleep(5)
        timeout = timeout - 1
    return res

def ssh_kvm_host(password, ipaddr, instance_name):
    """Ssh into kvm host and get vm mem details"""
    mem = []
    sshClient = SshClient(
        ipaddr,
        22,
        "root",
        password
    )

    command = "virsh dominfo %s" % instance_name
    vm_detail = sshClient.execute(command)
    max = vm_detail[7].split()
    min = vm_detail[8].split()
    mem.append(int(max[2]))
    mem.append(int(min[2]))
    return mem

def ssh_xen_host(password, ipaddr, instance_name):
    """Ssh into xen host and get vm mem details"""
    mem = []
    sshClient = SshClient(
        ipaddr,
        22,
        "root",
        password
    )
    command = "xe vm-list params=all name-label=%s" % instance_name
    vm_detail = sshClient.execute(command)
    max_str = vm_detail[17].split(":")
    min_str = vm_detail[20].split(":")
    max = int(max_str[1])
    min = int(min_str[1])
    mem.append(max)
    mem.append(min)
    return mem

    def restartServer(cls):
        """Restart management server"""

        sshClient = SshClient(
                    cls.mgtSvrDetails["mgtSvrIp"],
            22,
            cls.mgtSvrDetails["user"],
            cls.mgtSvrDetails["passwd"]
        )
        command = "service cloudstack-management stop"
        sshClient.execute(command)

        command = "service cloudstack-management start"
        sshClient.execute(command)

        #time.sleep(cls.services["sleep"])
        time.sleep(300)

        return

    def test_deploy_vgpu_enabled_vm(self):
        """Test Deploy Virtual Machine

        # Validate the following:
        # 1. Virtual Machine is accessible via SSH
        # 2. Virtual Machine is vGPU enabled (via SSH)
        # 3. listVirtualMachines returns accurate information
        """
        self.virtual_machine = VirtualMachine.create(
            self.apiclient,
            self.testdata["vgpu260q"],
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering.id,
            mode=self.testdata["mode"],
        )

        list_vms = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id)

        self.debug("Verify listVirtualMachines response for virtual machine: %s" % self.virtual_machine.id)

        self.assertEqual(isinstance(list_vms, list), True, "List VM response was not a valid list")
        self.assertNotEqual(len(list_vms), 0, "List VM response was empty")

        vm = list_vms[0]
        self.assertEqual(vm.id, self.virtual_machine.id, "Virtual Machine ids do not match")
        self.assertEqual(vm.name, self.virtual_machine.name, "Virtual Machine names do not match")
        self.assertEqual(vm.state, "Running", msg="VM is not in Running state")
        hosts = list_hosts(self.apiclient, id=vm.hostid)
        hostip = hosts[0].ipaddress
        try:
            sshClient = SshClient(
                host=hostip,
                port=self.testdata["configurableData"]["host"]["publicport"],
                user=self.testdata["configurableData"]["host"]["username"],
                passwd=self.testdata["configurableData"]["host"]["password"],
            )
            res = sshClient.execute("xe vgpu-list vm-name-label=%s params=type-uuid %s" % (vm.instancename))
            self.debug("SSH result: %s" % res)
        except Exception as e:
            self.fail("SSH Access failed for %s: %s" % (hostip, e))
        result = str(res)
        self.assertEqual(result.count("type-uuid"), 1, "VM is vGPU enabled.")

    def restartServer(cls):
        """Restart management server"""

        sshClient = SshClient(
                    cls.mgtSvrDetails["mgtSvrIp"],
            22,
            cls.mgtSvrDetails["user"],
            cls.mgtSvrDetails["passwd"]
        )
        command = "service cloudstack-management stop"
        sshClient.execute(command)

        command = "service cloudstack-management start"
        sshClient.execute(command)

        #Waits for management to come up in 5 mins, when it's up it will continue
        timeout = time.time() + 300
        while time.time() < timeout:
            if cls.isManagementUp() is True: return
            time.sleep(5)
        return cls.fail("Management server did not come up, failing")

    def setUpClass(self):
        testClient = super(TestDeployvGPUenabledVM, self).getClsTestClient()
        self.apiclient = testClient.getApiClient()
        self.testdata = self.testClient.getParsedTestDataConfig()
        #Need to add check whether zone containing the xen hypervisor or not as well
        hosts = list_hosts(
               self.apiclient,
               hypervisor="XenServer"
               )
        if hosts is None:
            raise unittest.SkipTest("There are no XenServers available. GPU feature is supported only on XenServer.Check listhosts response")
        else: 
             gpuhosts=0
             for ghost in hosts :
                    if ghost.hypervisorversion >= "6.2.0":
                       sshClient = SshClient(host=ghost.ipaddress, port=22, user='root',passwd=self.testdata["host_password"]) 
                       if ghost.hypervisorversion == "6.2.0":
                          res = sshClient.execute("xe patch-list uuid=0850b186-4d47-11e3-a720-001b2151a503")
                          if len(res) == 0:
                              continue
                       res = sshClient.execute("xe vgpu-type-list model-name=\"GRID K120Q\"")
                       if len(res) != 0 :
                           gpuhosts=gpuhosts+1 
                       else:        
                           continue
        if gpuhosts == 0:
           raise unittest.SkipTest("No XenServer available with GPU Drivers installed")

        self.domain = get_domain(self.apiclient)
        self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
        #Creating Account 
        self.account = Account.create(
            self.apiclient,
            self.testdata["account"],
            domainid=self.domain.id
            )
        self._cleanup = [
                         self.account
                        ]

 def try_ssh(self, ip_addr, unameCmd, firstAttempt=False):
     try:
         self.debug(
             "SSH into VM (IPaddress: %s) & NAT Rule (Public IP: %s)" %
             (self.vm_1.ipaddress, ip_addr)
         )
         retries = 3
         if firstAttempt:
             retries = 30
         # If Round Robin Algorithm is chosen,
         # each ssh command should alternate between VMs
         ssh_1  = SshClient(
             ip_addr,
             self.services['lbrule']["publicport"],
             self.vm_1.username,
             self.vm_1.password,
             retries=retries
         )
         unameCmd.append(ssh_1.execute("uname")[0])
         self.debug(unameCmd)
     except Exception as e:
         self.fail("%s: SSH failed for VM with IP Address: %s" %
                                 (e, ip_addr))
     time.sleep(5)

#.........这里部分代码省略.........
        cmd.endport = 22
        cmd.cidrlist = "0.0.0.0/0"
        self.apiclient.authorizeSecurityGroupIngress(cmd)

        self.debug("Fetching LB IP for account: %s" % self.account.name)
        ip_addrs = PublicIPAddress.list(
            self.api_client,
            associatednetworkid=self.guest_network.id,
            account=self.account.name,
            domainid=self.account.domainid,
            forloadbalancing=True,
            listall=True,
        )
        self.assertEqual(
            isinstance(ip_addrs, list), True, "List Public IP address should return valid IP address for network"
        )

        lb_ip = ip_addrs[0]
        self.debug("LB IP generated for account: %s is: %s" % (self.account.name, lb_ip.ipaddress))
        # TODO: uncomment this after ssh issue is resolved
        #        self.debug("SSHing into VMs using ELB IP: %s" % lb_ip.ipaddress)
        #        try:
        #            ssh_1 = self.vm_1.get_ssh_client(ipaddress=lb_ip.ipaddress)
        #            self.debug("Command: hostname")
        #            result = ssh_1.execute("hostname")
        #            self.debug("Result: %s" % result)
        #
        #            if isinstance(result, list):
        #                res = result[0]
        #            else:
        #                self.fail("hostname retrieval failed!")
        #
        #            self.assertIn(
        #                          res,
        #                          [self.vm_1.name, self.vm_2.name],
        #                          "SSH should return hostname of one of the VM"
        #                          )
        #
        #            ssh_2 = self.vm_2.get_ssh_client(ipaddress=lb_ip.ipaddress)
        #            self.debug("Command: hostname")
        #            result = ssh_2.execute("hostname")
        #            self.debug("Result: %s" % result)
        #
        #            if isinstance(result, list):
        #                res = result[0]
        #            else:
        #                self.fail("hostname retrieval failed!")
        #            self.assertIn(
        #                          res,
        #                          [self.vm_1.name, self.vm_2.name],
        #                          "SSH should return hostname of one of the VM"
        #                          )
        #        except Exception as e:
        #            self.fail(
        #                "SSH Access failed for %s: %s" % (self.vm_1.ipaddress, e))

        # Fetch details from user_ip_address table in database
        self.debug("select is_system from user_ip_address where public_ip_address='%s';" % lb_ip.ipaddress)

        qresultset = self.dbclient.execute(
            "select is_system from user_ip_address where public_ip_address='%s';" % lb_ip.ipaddress
        )
        self.assertEqual(isinstance(qresultset, list), True, "Check DB query result set for valid data")

        self.assertNotEqual(len(qresultset), 0, "Check DB Query result set")
        qresult = qresultset[0]

        self.assertEqual(qresult[0], 1, "is_system value should be 1 for system generated LB rule")
        self.debug("SSH into netscaler: %s" % self.services["netscaler"]["ipaddress"])
        try:
            ssh_client = SshClient(
                self.services["netscaler"]["ipaddress"],
                22,
                self.services["netscaler"]["username"],
                self.services["netscaler"]["password"],
            )
            self.debug("command: show ip")
            res = ssh_client.execute("show ip")
            result = str(res)
            self.debug("Output: %s" % result)

            self.assertEqual(
                result.count(lb_ip.ipaddress), 1, "One IP from EIP pool should be taken and configured on NS"
            )

            self.debug("Command:show lb vserver")
            res = ssh_client.execute("show lb vserver")

            result = str(res)
            self.debug("Output: %s" % result)

            self.assertEqual(
                result.count("Cloud-VirtualServer-%s-22 (%s:22) - TCP" % (lb_ip.ipaddress, lb_ip.ipaddress)),
                1,
                "User subnet IP should be enabled for LB service",
            )

        except Exception as e:
            self.fail("SSH Access failed for %s: %s" % (self.services["netscaler"]["ipaddress"], e))
        return