def setUpClass(cls):
     super(uao_visimap, cls).setUpClass()
     sqlpath = local_path("sql")
     outpath = local_path("output")
     anspath = local_path("expected")
     if not os.path.exists(outpath):
         os.mkdir(outpath)

 def verify_mpp21545(self):
     """
     After the pg_terminate_backend(), check if new log file has been generated, if so,
     append all new content into a string, plus new content from previous log file(if log
     msg cross two logfiles).
     Check if new content contains 'PANIC', OR 'Unexpected internal error',or 'Stack trace',
     """
     files = [os.path.join(self.PG_LOG_DIR, fname) for fname in os.listdir(self.PG_LOG_DIR)]
     latest_logfile = max(files, key=os.path.getmtime)
     with open (local_path('log.txt')) as fin:
         logfile = fin.read().replace('\n', '')
     with open (local_path('linenum.txt')) as fin:
         linenum = int(fin.read().replace('\n', ''))
     new_logfile_content = ''
     previous_logfile_new_content = ''
     if latest_logfile != logfile:
         with open(latest_logfile) as fin:
             new_logfile_content = fin.read().replace('\n', '')
     with open(logfile) as fin:
         for i, line in enumerate(fin):
             if i>=linenum:
                 previous_logfile_new_content += line
             else:
                 continue
     new_log_content = previous_logfile_new_content+' '+new_logfile_content
     self.assertNotRegexpMatches(new_log_content,
         "PANIC",
         "pg_terminate_backend() should not cause PANIC !")
     self.assertNotRegexpMatches(new_log_content,
         "Unexpected internal error",
         "pg_terminate_backend() should not cause Unexpected error !")
     self.assertNotRegexpMatches(new_log_content,
         "Stack trace",
         "pg_terminate_backend() should not cause Stack trace !")

    def setUpClass(cls):
        super(GppcTestCase, cls).setUpClass()
        """
        compile gppc_test.c and install the gppc_test.so
        """
        gpccutil.pre_process()
        cmd = 'gpssh --version'
        res = {'rc':0, 'stderr':'', 'stdout':''}
        run_shell_command (cmd, 'check product version', res)
        gppkg = Gppkg()
        product_version = res['stdout']
        result = gppkg.gppkg_install(product_version, 'libgppc')
        #makeLog = loal_path('test00MakeLog.out')
        if result:
            cmdMakeInstall = 'cd '+local_path('data')+' && make clean && make CPPFLAGS=-D_GNU_SOURCE && make install'
            res = {'rc':0, 'stderr':'', 'stdout':''}
            run_shell_command (cmdMakeInstall, 'compile gppc_test.c', res)

            # Current make file works for linux, but not for Solaris or OSX.
            # If compilation fails or installation fails, force system quit: os._exit(1)
            if res['rc']:
                os._exit(1) # This will exit the test including the next test suites
            sharedObj = local_path('data/gppc_test.so')
            if not os.path.isfile(sharedObj):
                os._exit(1)

            # For multinode cluster, need to copy shared object tabfunc_gppc_demo.so to all primary segments
            res = {'rc':0, 'stderr':'', 'stdout':''}
            hosts = gpdbconfig.get_hosts(segments=True)
            scp_cmd = 'gpscp  -h ' +' -h '.join(map(str,hosts)) +' '+ sharedObj + ' =:%s' % LIBDIR
            run_shell_command(scp_cmd, 'scp share object to all segment', res)
            tinctest.logger.info('scp shared object result %s' % res['stdout'])
            if res['rc']:
                raise Excpetion('Could not copy shared object to primary segment')

	def test_outof_shmm_exit_slots(self):
		"""
		The issue of MPP-19973 is that a shmem exit callback to reset 
		a temporary namespace is not removed when the temporary namespace is
		reset.

		In situations, where a temporary namespace is multiple times reset
		because of an exception in a subtransaction, the callbacks
		use up all shmem_exit slots.
		"""

		sql_setup_file = local_path('mpp19973_setup.sql')
		PSQL.run_sql_file(sql_file=sql_setup_file)

		# Test case setup verification
		self.assertTrue(table_exists("foo"))
		self.assertTrue(function_exists("testfn"))
	
		sql_file = local_path('mpp19973.sql')
		out_file = local_path('mpp19973.out')
		PSQL.run_sql_file(sql_file=sql_file,
			out_file=out_file, output_to_file=True)

		# There will be different error messages in the output, but
		# we should not run out of shmem exit slots.
		self.assertNotRegexpMatches(open(out_file).read(), 
			"out of on_shmem_exit slots",
			"Database should not run out of shmem_exit slots")

    def pre_process(self, product_version):
        """
        Just process all the sql and ans files to replace the path with new environment
        """
        for sql_file in os.listdir(local_path('sql')):
            if sql_file.endswith('.t'):
                f_input = open(sql_file, 'r')
                f_output = open(sql_file.split('.t')[0], 'w')
                for line in f_input:
                    if line.find('%MYD%') >= 0:
                        f_output.write(line.replace('%MYD%', local_path('')))
                f_input.close()
                f_output.close()

        for ans_file in os.listdir(local_path('expected')):
            if ans_file.endswith('.t'):
                f_input = open(ans_file, 'r')
                f_output = open(ans_file.split('.t')[0], 'w')
                for line in f_input:
                    if line.find('%MYD%') >= 0:
                        f_output.write(line.replace('%MYD%', local_path('')))
                f_input.close()
                f_output.close()
            if ans_file.endswith('4.2') and product_version.startswith('4.2'):
                shutil.move(os.path.join(local_path('expected'), ans_file), os.path.join(local_path('expected'), ans_file.split('.4.2')[0]))

    def setUpClass(cls):
        super(uao_visimap, cls).setUpClass()
	sqlpath=local_path('sql')
	outpath=local_path('output')
	anspath=local_path('expected')
        if not os.path.exists(outpath):
               os.mkdir(outpath)

 def doQuery(self, sqlfile, default=''):
     sql_file = local_path(sqlfile)
     filename_prefix = sqlfile.split('.sql')[0]
     out_file = local_path(filename_prefix + '.out')
     ans_file = local_path(filename_prefix + '.ans')
     PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
     self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))

    def test_with_fault_injection(self):
        """
        add new mirrors run workload to verify if cluster functioning correctly, and 
        inject the mirror to bring cluster into change tracking, then recoverseg
        """
        filerepUtil = Filerepe2e_Util()
        gprecover = GpRecover()
        self._setup_gpaddmirrors()
        self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)

        res = {'rc': 0, 'stdout' : '', 'stderr': ''}
        run_shell_command("gpaddmirrors -a -i %s -d %s --verbose" % (self.mirror_config_file, self.mdd), 'run gpaddmirrros with fault injection', res)
        gprecover.wait_till_insync_transition()
        self.assertEqual(0, res['rc'])
        self.run_simple_ddl_dml()

        # after adding new mirrors, check the intergrity between primary and mirror
        self.check_mirror_seg()
        out_file = local_path('inject_fault_into_ct')
        filerepUtil.inject_fault(f='filerep_consumer', m='async', y='fault', r='mirror', H='ALL', outfile=out_file)
        # trigger the transtion to change tracking
        PSQL.run_sql_command('drop table if exists foo;', dbname = 'template1')
        filerepUtil.wait_till_change_tracking_transition()
        gprecover.incremental()
        gprecover.wait_till_insync_transition()
        out_file=local_path('reset_fault')
        filerepUtil.inject_fault(f='filerep_consumer', m='async', y='reset', r='mirror', H='ALL', outfile=out_file)

    def test_verify_setup_teardown(self):
        test_loader = tinctest.TINCTestLoader()
        test_suite = test_loader.loadTestsFromTestCase(MockSQLTestCase)

        # As explained above, we want MockSQLTestCase to run if and only if
        # it's being invoked by our unit tests. So, it's skipped if discovered
        # directly by unit2. Here, bearing in mind that SQLTestCaseTests is itself
        # triggered by unit2, we override MockSQLTestCase's skip decorator to allow
        # this explicit construction of MockSQLTestCase to proceed.

        for test_case in test_suite._tests:
            test_case.__class__.__unittest_skip__ = False

        if os.path.exists(local_path("output/")):
            shutil.rmtree(local_path("output/"))
        test_result = unittest.TestResult()
        test_suite.run(test_result)

        self.assertEqual(test_result.testsRun, 4)
        self.assertEqual(len(test_result.errors), 0)
        self.assertEqual(len(test_result.skipped), 0)
        self.assertEqual(len(test_result.failures), 1)

        # Verify if setup and teardown sqls were executed
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'setup.out')))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'setup', 'setup1.out')))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'teardown.out')))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'teardown', 'teardown1.out')))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_setup.out')))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_teardown.out')))

    def setUpClass(self):
        super(MapreduceMPPTestCase, self).setUpClass()
        gppkg = Gppkg()
        gppkg.gppkg_install(product_version, 'plperl')
        setup_command = "create language plperl;"
        PSQL.run_sql_command(setup_command, dbname = os.environ.get('PGDATABASE'))

        "compile functions.c and build functions.so"
        makeLog = local_path('testBuildSOLog.out')
        cmdMake = 'cd '+local_path('c_functions') + ' && make clean && make'
        res = {'rc': 0, 'stdout' : '', 'stderr': ''}
        run_shell_command(cmdMake, 'compile functions.c', res)
        file = open(makeLog, 'w')
        file.write(res['stdout'])
        file.close()
        if res['rc']:
            raise Exception('a problem occurred while creating the so files ')
        so_dir = local_path('c_functions')
        sharedObj = local_path('c_functions/functions.so')
        # if not os.path.isfile(sharedObj):
            #raise gptest.GPTestError('so files does not exist')

        # For multinode cluster, need to copy shared object tabfunc_gppc_demo.so to all primary segments
        if gpdbconfig.is_multinode():
            res = {'rc':0, 'stderr':'', 'stdout':''}
            hosts = gpdbconfig.get_hosts(segments=True)
            scp_cmd = 'gpscp  -h ' +' -h '.join(map(str,hosts)) +' '+ sharedObj + ' =:%s' % so_dir
            run_shell_command(scp_cmd)
            if res['rc']:
                raise Exception('Could not copy shared object to primary segment')

 def install_kerberos_conf(self):
     """
     Update the kerberos configuration files according the env
     and copy in appropriate locations
     """
     transforms = {
                     "%DOMAIN%" : self.kdc_domain,
                     "%HOSTNAME%" : self.kdc_host
                 }
     input_file_path = local_path(self.krb_template_conf + "/" + self.KRB_CONF_TEMPLATE)
     output_file_path = local_path(self.krb_template_conf + "/" + self.KRB_CONF_TEMPLATE[:-2])
     with open(input_file_path, 'r') as input:
         with open(output_file_path, 'w') as output:
             for line in input.readlines():
                 for key,value in transforms.iteritems():
                     line = re.sub(key,value,line)
                 output.write(line)
     cmd_str = "sudo cp %s %s" %(output_file_path, self.KRB_CONF)
     if not run_shell_command(cmd_str,"Copying krb5.conf"):
         raise KerberosUtilException("Couldn't copy krb5.conf")
     cmd_str = "sudo cp %s %s" %(local_path(self.krb_template_conf + "/" + self.KDC_CONF_TEMPLATE), self.KDC_CONF)
     if not run_shell_command(cmd_str,"Copying kdc.conf"):
         raise KerberosUtilException("Couldn't copy kdc.conf")
     cmd_str = "sudo cp %s %s" %(local_path(self.krb_template_conf + "/" + self.KADMIN_ACL_CONF_TEMPLATE), self.KADMIN_ACL_CONF)
     if not run_shell_command(cmd_str,"Copying kadm5.acl"):
         raise KerberosUtilException("Couldn't copy kadm5.acl")

 def doTest(self, sql_filename):
     '''Run the file, compare oids in out file '''
     sql_file = local_path(sql_filename)
     out_file = local_path(sql_filename.split('.sql')[0] + '.out')
     PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
     isOk = self.compare_oids(out_file)
     self.assertTrue(isOk)

    def test_PLPERL91010_super_untrust(self):
        """ Language PL/PERL upgrade to 9.1:File system operations are allowed for untrusted PL/PERL """
        if self.checkAPPHOMEandLIB("plperl"):
            print 'installation'
            gp_procedural_languages().installPL('plperlu')
        tmpfilename = local_path('plperl91/plsuperuser.tmp')
        tmpfile = ''
        for i in tmpfilename:
           if i == '/':
               tmpfile = tmpfile + '\/'
           else:
               tmpfile = tmpfile + i
        tmpfilename = tmpfile
        localpath = local_path('')

        if sys.platform == 'sunos5':
            cmd = 'sed \'s/TMPFILENAME/%s/g\' %s/plperl91/test010_super_untrust.sql > %s/plperl91/test010_super_untrust.sql.tmp && mv %s/plperl91/test010_super_untrust.sql.tmp %s/plperl91/test010_super_untrust.sql' % ( tmpfilename, localpath, localpath, localpath, localpath)
        elif sys.platform == 'darwin':
            cmd = 'sed -i \'\' \'s/TMPFILENAME/%s/g\' %s/plperl91/test010_super_untrust.sql' % ( tmpfilename, localpath )
        else:
            cmd = 'sed -i \'s/TMPFILENAME/%s/g\' %s/plperl91/test010_super_untrust.sql' % ( tmpfilename, localpath )
        os.system( cmd )
        self.doPLPERLUbyuser("plperl91/test010_super_untrust", 'plsuperuser')
        checkcmd = 'cat ' + tmpfilename
        res = {'rc': 0, 'stdout' : '', 'stderr': ''}
        run_shell_command(checkcmd, 'run command %s'%checkcmd, res)
        if res['rc']:
            raise Exception("Unable to open created file")

 def test_43_alter_table_with_oid(self):
     '''MPP-13870: Alter table Set Without Oids fails in case of inheritance'''
     sql_file = local_path('alter_table_with_oid.sql')
     out_file = local_path('alter_table_with_oid.out')
     ans_file = local_path('alter_table_with_oid.ans')
     PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
     self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))

    def test_gp_interconnect_fc_ard_142(self):
        if (self.cluster_platform.lower().find('red hat enterprise linux server') < 0):
            self.skipTest('Test only applies to RHEL platform.')
        try:
            out = self.checkGUC(self.gp_interconnect_min_retries_before_timeout)
            self.assertTrue(len(out) > 4)
            out = self.checkGUC(self.gp_interconnect_transmit_timeout)
            self.assertTrue(len(out) > 4)
            out = self.checkGUC(self.gp_interconnect_fc_method)
            self.assertTrue(len(out) > 4)
        except:
            self.skipTest("GUC " + self.gp_interconnect_min_retries_before_timeout + " or " + self.gp_interconnect_transmit_timeout + " or " +  self.gp_interconnect_fc_method + " not defined")


        result = runShellCommand('gpssh ' + self.hoststr +  ' \"export PATH=$PATH:/sbin; \
                                              sudo insmod ickm.ko ict_type=0x101 seq_array=2 drop_times=80\"')
        self.assertTrue(result)
        
        sql_file = local_path(self.common_sql + str(self._testMethodName) + '.sql');
        self.assertTrue(PSQL.run_sql_file(local_path(sql_file)))        
        out_file = sql_file.replace(".sql",".out")
        test_ret = "Failed to send packet (seq 2) to" in open(out_file).read() and "retries in 40 seconds" in open(out_file).read()
        ret_log = runShellCommand(self.log_str + self._testMethodName + '.log' )
        result = runShellCommand('gpssh ' + self.hoststr +  ' \"export PATH=$PATH:/sbin;sudo rmmod ickm.ko \"')

        self.assertTrue(result)  
        self.assertTrue(ret_log)
        self.assertTrue(test_ret)

    def do_test(self, timeout=0, sqlfile=None, host=None, port=None, username=None, password=None, flags='-a', usetemplate=False):
        """
        @summary: Run a PostGIS test case
        
        @param timeout: Number of seconds to run sql file before timing out
        @param sqlfile: The path to sql file (relative to TEST.py directory)
        @param host: The GPDB master host name to use to connect to database
        @param port: The GPDB port used to make connections to the database
        @param username: The database username to use to connect to the database
        @param password: The password for the database user used to connect to database
        """
        if sqlfile is None:
            testcase = inspect.stack()[1][3].split('test_')[1]
            
            #file = mkpath(testcase +".sql")
            file = local_path(testcase +".sql")
        else:
            #file = mkpath(sqlfile)
            file = local_path(sqlfile)
        # run psql on file, and check result
        #psql.runfile(file,timeout=timeout,host=host,port=port,username=username,password=password,flag=flags)
        #self.checkResult(ifile=file, optionalFlags=" -B")

        out_file = local_path(testcase + ".out")
        ans_file = local_path(testcase +".ans")
        PSQL.run_sql_file(sql_file = file, out_file = out_file)
        self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))

    def setUpClass(cls):
        super(EnhancedTableFunctionTest, cls).setUpClass()

        tinctest.logger.info("*** Running the pre-requisite sql files drop.sql and setup.sql")
        PSQL.run_sql_file(local_path('sqls/setup/drop.sql'))
        PSQL.run_sql_file(local_path('sqls/setup/create.sql'))
        tinctest.logger.info("*** Starting the Enhaced table test")

    def replace_new_dirPath(self, orig_filename='recovery.conf', new_filename='recovery_new.conf'):
        """
        @summary: Modifies the template config file with new filespace location
        
        @param orig_filename: name of the template config file
        @param new_filename: name of the new config file
        @return: None
        """

        new_file = open(local_path(new_filename),'w')
        old_file = open(local_path(orig_filename))
        # Finds the gp prefix string from the host entry of the segments
        (host, fileLoc) = self.get_segment_host_fileLoc()
        slashIndex = fileLoc.rfind('/')
        # Extract the gp prefix
        gp_prefix = fileLoc[(slashIndex+1):]
        lineNo = 0
        for line in old_file:
            if lineNo == 1 :
                # If you checkout the template out file from the command gprecoverseg -o, you'll see that,
                # it has failed_host:port:data_dir <SPACE> recovery_host:port:data_dir.
                # we intend to change only the data_dir of the recovery_host and not that of the failed host
                # so we first divide the lines & modify only the second part 
                # and then concatenate them again
                prefix_index = line.find(gp_prefix)
                first_part_of_line = line[:(prefix_index + len(gp_prefix))]
                remaining_part_of_line = line[(prefix_index + len(gp_prefix)):]
                new_gp_prefix = gp_prefix+'new'
                remaining_part_of_line = re.sub(gp_prefix,new_gp_prefix,remaining_part_of_line)
                line = first_part_of_line + remaining_part_of_line
            lineNo = lineNo + 1
            new_file.write(line)
        new_file.close()
        old_file.close()

    def test_run_sql_file(self):
        test_case = MockSQLTestCase('test_query03')
        if os.path.exists(test_case.get_out_dir()):
            shutil.rmtree(test_case.get_out_dir())
        # Default mode
        test_case.run_sql_file(local_path('query03.sql'))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03.sql')))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03.out')))
        self.assertFalse(self._check_str_in_file('SET optimizer',
                                                os.path.join(test_case.get_out_dir(), 'query03.sql')))

        # Optimizer on mode
        test_case.run_sql_file(local_path('query03.sql'), optimizer=True)
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_orca.sql')))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_orca.out')))
        self.assertTrue(self._check_str_in_file('SET optimizer=on;',
                                                os.path.join(test_case.get_out_dir(), 'query03_orca.sql')))
                        

        # Optimizer off mode
        test_case.run_sql_file(local_path('query03.sql'), optimizer=False)
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_planner.sql')))
        self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_planner.out')))
        self.assertTrue(self._check_str_in_file('SET optimizer=off;',
                                                os.path.join(test_case.get_out_dir(), 'query03_planner.sql')))

 def test_with_concurrent_workload(self):
     """
     add new mirrors while concurrent workload in progress, check that mirrors added
     and current workload won't get affected, in the end, run checkmirrorseg.
     Note that: adding mirrors while running workload has checkmirrorseg issue with MPP-24311
     """
     gprecover = GpRecover()
     self._setup_gpaddmirrors()
     self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
     sql_setup_file = local_path('sql/ao_heap_table_setup.sql') 
     sql_file = local_path('sql/ao_heap_table.sql')
     pg_stat_activity = 'SELECT * FROM pg_stat_activity;'
     PSQL.run_sql_file(sql_setup_file)
     subprocess.Popen(["psql", "-f", sql_file])
     time.sleep(15)
     subprocess.Popen(["gpaddmirrors", "-ai", self.mirror_config_file, "-d", self.mdd])
     time.sleep(15)
     result = PSQL.run_sql_command(pg_stat_activity, flags='-q -t', dbname='template1')
     result = result.strip()
     rows = result.split('\n')
     self.assertTrue(len(rows) > 1)
     while len(rows) > 1:
         result = PSQL.run_sql_command(pg_stat_activity, flags='-q -t', dbname='template1')
         result = result.strip()
         rows = result.split('\n')
         time.sleep(3)
     gprecover.wait_till_insync_transition()
     self.verify_config_file_with_gp_config()