def get_max_pageNo(positionName):
    """
    return the max page number of a specific job
    """
    request_url = 'https://m.lagou.com/search.json?city=%E5%85%A8%E5%9B%BD&positionName=' + parse.quote(
        positionName) + '&pageNo=1&pageSize=15'
    headers = {
        'Accept': 'application/json',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Host': 'm.lagou.com',
        'Referer': 'https://m.lagou.com/search.html',
        'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) '
                      'Version/8.0 Mobile/12A4345d Safari/600.1.4',
        'X-Requested-With': 'XMLHttpRequest',
        'Connection': 'keep-alive'
    }
    response = requests.get(request_url, headers=headers, cookies=init_cookies(), timeout=10)
    print("Getting data from %s successfully. URL: " % positionName + request_url)
    if response.status_code == 200:
        max_page_no = int(int(response.json()['content']['data']['page']['totalCount']) / 15 + 1)

        return max_page_no
    elif response.status_code == 403:
        log.error('request is forbidden by the server...')

        return 0
    else:
        log.error(response.status_code)

        return 0

def listener (host):
    interfaces = config["hosts"][host]['interfaces']
    
    for name in sorted(interfaces):
        s = connect(host, 'LISTENER')
        if s == None:
            return
        try:
            interface = interfaces[name]
            addr = interface['bind']
            port = interface['port']
        except:
            log.error('MM: ' + host + 'ERROR: Bad interface spec ' + name)
            continue
            
        try:
            s.send('listener ' + name + ' ' + addr + ' ' + str(port) + '\n')
            while True:
                data = s.recv(1024)
                if len(data) == 0:
                    break
                sys.stdout.write(data)
            s.close()
        except Exception, e:
            log.error('MM:00 ERROR: ' + repr(e))

def connect (host, why):    
    # should be either a listener host or a router host (edge-router)
    try:
        hostdata = hosts[host]
    except:
        if host not in bgprouters:
            log.error('MM:' + host + ' ERROR: ' + why + ': Unknown host: ' + host)
            return None    
    try:
        cmdifc = hostdata['cmdifc']
        cmdport = hostdata['cmdport']
    except:
        if platform.system() == 'Windows':
            cmdifc = '127.0.0.1'
            cmdport = base36(host)
        else:
            cmdifc = '/tmp/' + host
            cmdport = 0
    
    #print 'MM:' + host + ' INFO: ' + why + ': Connecting to ' + host + ' at ' + cmdifc + ':' + str(cmdport)

    try:
        if cmdifc.find('/') >= 0:
            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # @UndefinedVariable
            s.connect(cmdifc)
        else:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.connect((cmdifc, cmdport))
    except Exception, e:
        log.error('MM:' + host + ' ERROR: ' + why + ': ' + repr(e))
        return None

    def _on_connection_success_item(self, connection_item, stream):
        self._off_connection_timeout_handler()
        log.debug(u"Connection Success {}".format(self.client_config.address_str))

        try:
            self.stream = stream
            self.stream.set_close_callback(self._on_connection_close)
            self.stream.set_nodelay(True)

            #: send message
            self._sending_connection_item(connection_item)

            #: fetch message
            read_status = yield self._read_message(connection_item)
            if read_status:
                connection_item.callback(RPCMessage(
                    CONNECTION_TYPE_IN_RESPONSE, self._message.topic, self._message.body))
            else:
                log.error("Malformed Client Request")

        except Exception as e:
            log.error(e)
            traceback.print_exc()
        finally:
            self.close()

def post_check():
    log.info("Check post validation for all posts")
    postid_list = get_postid_list()
    status, obj = check_post_list(postid_list)
    if status is False:
        log.error("post check fail, msg: %s" % obj)
        return False

def get_max_page_no(company_id):
    """
    return the max page number of interviewees' comments based on particular company 
    :param company_id: 
    :return: 
    """
    request_url = 'https://www.lagou.com/gongsi/searchInterviewExperiences.json'
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Accept-Encoding': 'gzip, deflate, br',
        'Host': 'www.lagou.com',
        'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0'
                      ' Mobile/13B143 Safari/601.1',
        'Referer': 'https://www.lagou.com/gongsi/interviewExperiences.html?companyId=%s' % company_id
    }

    params = {
        'companyId': company_id,
        'positionType': '',
        'pageSize': '10',
        'pageNo': '1'
    }

    response = requests.post(request_url, headers=headers, params=params, cookies=init_cookies())
    if response.status_code == 200:
        maxpage = int(response.json()['content']['data']['page']['totalCount'])
    else:
        log.error('Error code is ' + str(response.status_code))
        maxpage = 0

    return int(maxpage / 10) + 1

def crawl_company_stage(company_id):
    req_url = 'https://m.lagou.com/gongsi/%s.html' % str(company_id)
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Host': 'm.lagou.com',
        'Referer': 'https://m.lagou.com',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1'
    }
    response = requests.get(req_url, headers=headers, cookies=m_lagou_spider.init_cookies(), timeout=20)
    print(response.url)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html5lib')
        company_desc = soup.find_all(class_="desc")[0].get_text().strip()
        industryField = company_desc.split('/')[0].strip()
        financeStage = company_desc.split('/')[1].strip()
        staffNum = company_desc.split('/')[2].strip()

    elif response.status_code == 403:
        log.error('403 forbidden...')
    else:
        log.error(response.status_code)
    time.sleep(random.randint(3, 6))

    return [company_id, industryField, financeStage, staffNum]

def generic (host, label, cmd):
    if host in participants:
        log.error('MM:' + host + ' ERROR: ' + label + ': Cannot send to a partipant: ' + host)
        return None   
        
    s = connect(host, label)
    if s == None:
        return None
    
    alldata = ''
    try:
        s.send(cmd)
        
        while True:
            data = s.recv(1024)
            if len(data) == 0:
                break
            alldata += data
            #sys.stdout.write(data)
        s.close()
    except Exception, e:
        log.error('MM:' + host + ' ERROR: ' + label + ': '+ repr(e))
        try:
            s.close()
        except:
            pass
        return None

def listener2(host):
    if host not in hosts:
        log.error('MM:00 ERROR: LISTENER: unknown host: ' + host)
        return
    h = hosts[host]
    for p in h.tcp:
        listener3(host, h.bind, p)

def delay (args):
    if len(args) == 1:
        try:
            log.info('MM:00 DELAY ' + args[0])
            time.sleep(float(args[0]))
        except Exception, e:
            log.error('MM:00 ERROR: DELAY: exception: ' + repr(e))

def connect (host, why):    
    # should be either a listener host or a router host (edge-router)
    if host not in bgprouters and host not in hosts and host not in participants:
            log.error('MM:' + host + ' ERROR: ' + why + ': Unknown host: ' + host)
            return None    
    try:
        hostdata = hosts[host]
    except:
        try:
            hostdata = bgprouters[host]
        except:
            hostdata = participants[host]

    #print 'MM:' + host + ' INFO: ' + why + ': Connecting to ' + host + ' at ' + hostdata.host + ':' + str(hostdata.port)

    try:
        if hostdata.port is None:
            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # @UndefinedVariable
            s.connect(hostdata.host)
        else:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.connect((hostdata.host, int(hostdata.port)))
    except Exception, e:
        log.error('MM:' + host + ' ERROR: ' + why + ': ' + repr(e))
        return None

def test (tn):
    global config
    rand = str(random.randint(1000000000, 9999999999)) # must be 10 characters
    try:
        src = config["tests"][tn]['src']
        baddr = config["tests"][tn]['baddr']
        daddr = config["tests"][tn]['daddr']
        dport = config["tests"][tn]['dport']
        xifc = config["tests"][tn]['xifc']
        xdst = config["tests"][tn]['xdst']
    except:
        log.error('MM:00 ERROR: TEST FAILED unknown or poorly specified test: ' + tn)
        return
            
    s = connect(src, 'TEST')
    if s == None:
        return
    
    try:
        s.send('test ' + rand + ' ' + baddr + ' ' + daddr + ' ' + str(dport) + '\n')
        alldata = ''
        while True:
            data = s.recv(1024)
            if len(data) == 0:
                break
            alldata += data
            #sys.stdout.write(data)
        s.close()
    except Exception, e:
        log.error('MM:' + src + ' ERROR: TEST FAILED ' + repr(e))
        return

def run (args):
    if len(args) < 2:
        print 'MM:00 EXEC: ERROR usage: exec cmd cmd ...'
        print 'Commands are:'
        for c in sorted(commands):
            print '  ' + c + ': ' + commands[c].get('cmd', '<CMD>')
        return
    
    for i in range(1, len(args)):
        cmdname = args[i]
        try:
            c = commands[cmdname]['cmd']
        except:
            log.error('MM:00 ERROR: EXEC FAILED unknown or poorly specified cmd: ' + cmdname)
            continue
        log.info('MM:00 EXEC: ' + cmdname + ' cmd = ' + c)
        ca = c.split()
        try:
            p = subprocess.Popen(ca, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out, err = p.communicate()
        except Exception, e:
            out = ''
            err = 'Command Failed: ' + repr(e)
        r = out + err
        log.debug('MM:00 EXEC: ' + cmdname + ' output = \n' + r.strip())

def connect (host, why):
    global config
    
    try:
        hostdata = config['hosts'][host]
    except:
        log.error('MM:' + host + ' ERROR: ' + why + ': Unknown host: ' + host)
        return None
        
    try:
        cmdifc = hostdata['cmdifc']
        cmdport = hostdata['cmdport']
    except:
        if platform.system() == 'Windows':
            cmdifc = '127.0.0.1'
            cmdport = base36(host)
        else:
            cmdifc = '/tmp/' + host
            cmdport = 0
    
    #print 'MM:' + host + ' INFO: ' + why + ': Connecting to ' + host + ' at ' + cmdifc + ':' + str(cmdport)

    try:
        if cmdifc.find('/') >= 0:
            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # @UndefinedVariable
            s.connect(cmdifc)
        else:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.connect((cmdifc, cmdport))
    except Exception, e:
        log.error('MM:' + host + ' ERROR: ' + why + ': ' + repr(e))
        return None

def parse (line):
    global cmdfuncs
    
    tokens = line.split()
    n = len(tokens)
    if n == 0:
        return
    cmd = tokens[0]
    if cmd[0] == '#':
        return
    
    # search for the command
    # if there is a dictionary in the list, iterate over all the elements or all the arguments
    # else pass the entire set of arguments to the command
    
    foo = cmdfuncs.get(cmd)
    if foo != None:
        func, jset, _ = foo
        if jset is None:
            func(tokens)
            return
        if n == 1:
            for h in sorted(jset):
                func(h)
        else:
            for i in range(1, n):
                func(tokens[i])
        return

    log.error('MM:00 ERROR: unknown command: ' + cmd)

def listener3(host, bind, port):
    if host not in hosts:
        log.error('MM:00 ERROR: LISTENER: unknown host: ' + host)
        return
    #print 'listener ' + host + ' ' + bind + ' ' + port
    r = generic(host, 'LISTENER', 'listener ' + bind + ' ' + port + '\n')
    if r is not None and len(r) > 0:
        log.info('MM:' + host + ' LISTENER: ' + r.strip())

def request_ice(args):
    
    try:
        data=json.loads(args.get('Input'))
        
    except Exception, e:
        print e
        log.error(e)

def parse (line):
    global cmdfuncs
    
    tokens = line.partition('#')[0].split()
    try:
        tokens = shlex.split(line.partition('#')[0])
    except Exception, e:
        log.error('MM:00 PARSE ERROR: ' + repr(e))
        return

def blackholing (args):
    if len(args) < 3:
        log.error('MM:00 EXEC: ERROR usage: blackholing participant_id remove/insert id[,id...]')
        return
    
    part_id = args[0] #participant id
    part_action = args[1] #action insert or remove

    rule_ids = []
    for policy_id in args[2].split(','): #rule ids
        rule_ids.append(int(policy_id)+2**12) #additional 4096 for cookie id
    
    client_path = '/home/vagrant/endeavour/pclnt/participant_client.py'
    config_file = 'participant_' + part_id + '_bh.cfg'

    cmd = ''
    for arg in args:
        cmd += arg + ' '
    log.info('MM:00 BLACKHOLING: ' + cmd + config_file)

    policy_path = os.path.abspath(os.path.join(os.path.realpath(sys.argv[1]), "..", "..", "policies"))
    config_path = os.path.join(policy_path, config_file)

    part_info = config.participants[str(part_id)]

    part_url = 'http://' + str(part_info["EH_SOCKET"][0]) + ':' + str(part_info["EH_SOCKET"][1]) + '/bh/inbound/'
    content_header = {'Content-Type':'application/json'}

    # prepare for insert blackholing policy
    if part_action == 'insert':
        new_policy = []
        # Open File and Parse
        with open(config_path, 'r') as f:
            policies=json.load(f)

            for policy in policies['inbound']: 
                if int(policy['cookie']) in rule_ids:
                    new_policy.append(policy)
        
        # insert only inbound policys
        data = {}
        data['inbound'] = new_policy
        data=json.dumps(data)

        # post to participant api
        r = requests.post(part_url, data=data, headers=content_header)

    # prepare for remove seperate blackholing policy
    elif part_action == 'remove':
    
        for rule_id in rule_ids:
            new_url = part_url + str(rule_id)
            
            # post to participant api
            r = requests.delete(new_url, headers=content_header)
    else:
        log.error('MM:00 EXEC: ERROR usage: error in blackholing - wrong action')

def init(file):
    '''解析excel文件用例'''
    log.info('解析excel文件数据')
    cases_list=[]
    try:
        module_list=open_excel.excel_table_by_index(file)
    except Exception, e:
        print e
        log.error('%s文件打开失败'%file)