def add_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
    status = False
    # identify the user exist
    # identify the attribute exist
    # identify the attribute exist in user_portrait
    # add attribute in user_portrait
    # submit user should has power to change???without
    try:
        user_result = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
    except:
        return 'no user'
    try:
        attribute_result = es.get(index=attribute_index_name, doc_type=user_index_type, id=attribute_name)['_source']
    except:
        return 'no attribute'
    attribute_value_list = json.loads(attribute_result['value'])
    if attribute_value not in attribute_value_list:
        return 'no attribute value'
    if attribute_name in user_result:
        return 'attribute exist'
    add_attribute_dict = {attribute_name: attribute_value}
    
    es.update(index=user_index_name, doc_type=user_index_type, id=uid, body={'doc':body})
    status = True
    return status

def get_group_list(task_name):
    results = []
    try:
        es_results = es.get(index=index_name, doc_type=index_type, id=task_name)['_source']
    except:
        return results
    #print 'es_result:', es_results['uid_list'], type(es_results['uid_list'])
    uid_list = es_results['uid_list']
    user_portrait_attribute = es.mget(index='user_portrait', doc_type='user', body={'ids':uid_list})['docs']
    evaluate_max = get_evaluate_max()
    for item in user_portrait_attribute:
        uid = item['_id']
        try:
            source = item['_source']
            uname = source['uname']
            gender = source['gender']
            location = source['location']
            importance = source['importance']
            normal_importance = math.log(importance / evaluate_max['importance'] * 9 + 1, 10) * 100
            influence = source['influence']
            normal_influence = math.log(influence / evaluate_max['influence'] * 9 + 1, 10) * 100
            results.append([uid, uname, gender, location, normal_importance, normal_influence])
        except:
            results.append([uid])
    return results

def get_text_detail(task_name, ts, text_type, user, order, size=100):
    results = []
    _id = user + '-' + task_name
    task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=_id)["_source"]
    social_sensors = json.loads(task_detail["social_sensors"])

    #print social_sensors
    if int(text_type) == 0: # 热门原创微博
        results = get_origin_weibo_detail(ts, user, task_name, size, order, 1)

    elif int(text_type) == 1: # 热门转发微博
        results = get_origin_weibo_detail(ts, user, task_name, size, order, 2)

    elif int(text_type) == 2: # 普通转发微博
        results = get_retweet_weibo_detail(ts, user, task_name, size, "message_type", 3)

    elif int(text_type) == 3: # 普通评论微博
        results = get_retweet_weibo_detail(ts, user, task_name, size, "message_type", 2)

    elif int(text_type) == 4: # 积极微博
        results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", "1")

    elif int(text_type) == 5: # 中性微博
        results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", "0")

    elif int(text_type) == 6: # 消极微博
        results = get_retweet_weibo_detail(ts, user, task_name, size, "sentiment", ["2", "3", "4", "5", "6"])
    elif int(text_type) == 7: # 敏感微博
        results = get_origin_weibo_detail(ts, user, task_name, size, order, 3)

    else:
        print "error"

    return results

def delete_attribute(attribute_name):
    status = False
    try:
        result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
    except:
        return status
    attribute_value = json.loads(result['value'])
    es.delete(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)
    # delete attribute in user_portrait
    query = []
    for value in attribute_value:
        query.append({'match':{attribute_name: value}})
    try:
        attribute_user_result = es.search(index=user_index_name, doc_type=user_index_type, \
                                         body={'query':{'bool':{'must':query}}})['hits']['hits']
    except:
        attribute_user_result = []
    if attribute_user_result==[]:
        status = True
        return status
    bulk_action = []
    for user_dict in attribute_user_result:
        try:
            user_item = user_dict['_source']
        except:
            next
        user_item.pop(attribute)
        user = user_item['uid']
        action = {'index':{'_id':str(user)}}
        bulk_action.extend([action, user_item])
    es.bulk(bulk_action, index=user_index_name, doc_type=index_type)
    status = True
    return status

def ajax_get_group_detail():
    task_name = request.args.get('task_name','') # task_name
    user = request.args.get('user', '')
    _id = user + '-' + task_name
    portrait_detail = []
    top_activeness = get_top_influence("activeness")
    top_influence = get_top_influence("influence")
    top_importance = get_top_influence("importance")
    search_result = es.get(index=index_group_manage, doc_type=doc_type_group, id=_id).get('_source', {})
    if search_result:
        try:
            uid_list = json.loads(search_result['uid_list'])
        except:
            uid_list = search_result['uid_list']
        if uid_list:
            search_results = es.mget(index=portrait_index_name, doc_type=portrait_index_type, body={"ids":uid_list}, fields=SOCIAL_SENSOR_INFO)['docs']
            for item in search_results:
                temp = []
                if item['found']:
                    for iter_item in SOCIAL_SENSOR_INFO:
                        if iter_item == "topic_string":
                            temp.append(item["fields"][iter_item][0].split('&'))
                            temp.append(item["fields"][iter_item][0].split('&'))
                        elif iter_item == "activeness":
                            temp.append(math.ceil(item["fields"][iter_item][0]/float(top_activeness)*100))
                        elif iter_item == "importance":
                            temp.append(math.ceil(item["fields"][iter_item][0]/float(top_importance)*100))
                        elif iter_item == "influence":
                            temp.append(math.ceil(item["fields"][iter_item][0]/float(top_influence)*100))
                        else:
                            temp.append(item["fields"][iter_item][0])
                    portrait_detail.append(temp)

    return json.dumps(portrait_detail)

def add_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
    status = False
    submit_user_tag = submit_user + "-tag"
    id_attribute = submit_user + "-" + attribute_name
    add_attribute_value = attribute_name + "-" + attribute_value
    # identify the user exist
    # identify the attribute exist
    # identify the attribute exist in user_portrait
    # add attribute in user_portrait
    try:
        user_result = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
    except:
        return 'no user'
    try:
        attribute_result = es_tag.get(index=attribute_index_name, doc_type=attribute_index_type, id=id_attribute)['_source']
    except:
        return 'no attribute'
    attribute_value_list = attribute_result['attribute_value'].split('&')
    if attribute_value not in attribute_value_list:
        return 'no attribute value'
    value_set = set()
    for value in attribute_value_list:
        value_set.add(attribute_name + '-' + value) #
    submit_user_attribute = user_result.get(submit_user_tag, '') # 个人是否存在该管理员打上的个人标签
    tmp_attribute_set = set(submit_user_attribute.split('&'))
    attribute_exist = tmp_attribute_set & value_set
    if attribute_exist:
        return 'attribute exist'
    user_result[submit_user_tag] = "&".join([submit_user_attribute, add_attribute_value])
    es.index(index=user_index_name, doc_type=user_index_type, id=uid, body=user_result)
    status = True
    return status

def change_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
    status = False
    submit_user_tag = submit_user + "-tag"
    id_attribute = submit_user + "-" + attribute_name
    #identify the user exist
    #identify the attribute exist
    #identify the attribute value exist
    #identify the submit_user have been admitted----without 
    try:
        user_result = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
    except:
        return 'no user'
    try:
        attribute_result = es_tag.get(index=attribute_index_name, doc_type=attribute_index_type, id=id_attribute)['_source']
    except:
        return 'no attribute'
    value_list = attribute_result['attribute_value'].split('&')
    if attribute_value not in value_list:
        return 'no attribute value'
    submit_user_attribute = user_result.get(submit_user_tag, '') # 个人是否存在该管理员打上的个人标签
    if attribute_name not in submit_user_attribute:
        return 'personal attribute no exist'
    tmp_attribute_list = submit_user_attribute.split("&")
    attribute_list = []
    for item in tmp_attribute_list:
        if attribute_name in item:
            attribute_list.append(attribute_name + '-' + attribute_value)
        else:
            attribute_list.append(item)
    user_result[submit_user_tag] = "&".join(attribute_list)
    es.index(index=user_index_name, doc_type=user_index_type, id=uid, body=user_result)
    status = True
    return status

def add_tag2group(uid_list, attribute_name, attribute_value, submit_user):
    id_attribute = submit_user + "-" + attribute_name
    add_attribute = attribute_name + "-" + attribute_value
    submit_user_tag = submit_user + "-tag"
    status = False
    #identify the attribute exist
    #for uid in uid_list
    #identify the attribute not in this user
    #add tag to this user
    try:
        attribute_exist = es_tag.get(index=attribute_index_name, doc_type=attribute_index_type, id=id_attribute)['_source']
    except:
        return 'no attribute'
    attribute_exist_value_list = attribute_exist['attribute_value'].split('&')
    if attribute_value not in attribute_exist_value_list:
        return 'no attribute value'
    for uid in uid_list:
        try:
            user_exist = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
        except:
            user_exist = {}
            continue
        submit_user_attribute = user_exist.get(submit_user_tag, '')
        if user_exist and attribute_name not in submit_user_attribute:
            user_exist[submit_user_tag] = "&".join([submit_user_attribute, add_attribute])
            es.index(index=user_index_name, doc_type=user_index_type, id=uid, body=user_exist)
    status = True
    return status

def submit_task(input_data):
    status = 0 # mark it can not submit
    task_name = input_data['task_name']
    try:
        result = es.get(index=index_name, doc_type=index_type, id=task_name)
    except:
        status = 1
    
    if status != 0 and 'uid_file' not in input_data:
        r.lpush('group_task', json.dumps(input_data))
        input_data['status'] = 0 # mark the task not compute
        count = len(input_data['uid_list'])
        input_data['count'] = count
        uid_list_string = json.dumps(input_data['uid_list'])
        es.index(index='group_result', doc_type='group', id=task_name, body=input_data)
    elif status != 0 and 'uid_file' in input_data:
        input_data['status'] = 0 # mark the task not compute
        uid_file = input_data['uid_file']
        uid_list = read_uid_file(uid_file)
        input_data['count'] = len(uid_list)
        input_data['uid_list'] = json.dumps(uid_list)
        r.lpush('group_task', json.dumps(input_data))
        es.index(index='group_result', doc_type='group', id=task_name, body=input_data)
        delete_status = delete_uid_file(uid_file)
        if delete_status == 0:
            print 'fail delete uid file'
        elif delete_status == 1:
            print 'success delete uid file'
    return status

def delete_attribute(attribute_name):
    status = False
    try:
        result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
        print 'result:', result
    except Exception, e:
        raise e
        return status

def get_group_weibo(task_name, date):
    group_weibo = []
    #step1 : get group user list by task_name
    group_index_name = 'group_result'
    group_index_type = 'group'
    try:
        group_task = es.get(index=group_index_name, doc_type=group_index_type, id=task_name)['_source']
    except Exception ,e:
        raise e

def search_identify_uid(uid):
    result = 0
    try:
        user_dict = es_user_portrait.get(index='user_portrait', doc_type='user', id=uid)
        #print 'user_dict:', user_dict
        result = 1
    except:
        result = 0
    return result

def conclusion_on_influence(uid):
    # test
    index_name = copy_portrait_index_name
    index_type = copy_portrait_index_type
    total_number = es.count(index=copy_portrait_index_name, doc_type=copy_portrait_index_type)["count"]

    try:
        influ_result = es.get(index=index_name, doc_type=index_type, id=uid)["_source"]
    except:
        influ_result = {}
        result = [0, 0, 0, 0, 0, 0, total_number]  # aver_activeness, sorted, aver_influence, sorted
        return result

    aver_activeness = influ_result.get("aver_activeness", 0)
    aver_influence = influ_result.get("aver_influence", 0)
    aver_importance = influ_result.get("aver_importance", 0)
    influence_query_body = {"query": {"match_all": {}}, "sort": {"aver_influence": {"order": "desc"}}, "size": 1}
    top_influence = es.search(
        index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=influence_query_body
    )["hits"]["hits"][0]["sort"][0]

    importance_query_body = {"query": {"match_all": {}}, "sort": {"aver_importance": {"order": "desc"}}, "size": 1}
    top_importance = es.search(
        index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=importance_query_body
    )["hits"]["hits"][0]["sort"][0]

    activeness_query_body = {"query": {"match_all": {}}, "sort": {"aver_activeness": {"order": "desc"}}, "size": 1}
    top_activeness = es.search(
        index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=activeness_query_body
    )["hits"]["hits"][0]["sort"][0]

    influence_query_body = {"query": {"filtered": {"filter": {"range": {"aver_influence": {"gt": aver_influence}}}}}}

    activeness_query_body = {"query": {"filtered": {"filter": {"range": {"aver_activeness": {"gt": aver_activeness}}}}}}

    importance_query_body = {"query": {"filtered": {"filter": {"range": {"aver_importance": {"gt": aver_importance}}}}}}

    influence_count = es.count(
        index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=influence_query_body
    )["count"]
    activeness_count = es.count(
        index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=activeness_query_body
    )["count"]
    importance_count = es.count(
        index=copy_portrait_index_name, doc_type=copy_portrait_index_type, body=importance_query_body
    )["count"]

    result = [
        int(aver_activeness * 100.0 / top_activeness),
        activeness_count,
        int(aver_influence * 100.0 / top_influence),
        influence_count,
        int(aver_importance * 100.0 / top_importance),
        importance_count,
        total_number,
    ]
    return result

def ajax_get_task_detail_info():
    task_name = request.args.get('task_name','') # task_name
    task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name)['_source']
    task_detail["social_sensors"] = json.loads(task_detail["social_sensors"])
    task_detail['keywords'] = json.loads(task_detail['keywords'])
    task_detail["sensitive_words"]= json.loads(task_detail["sensitive_words"])
    history_status = json.loads(task_detail['history_status'])
    if history_status:
        temp_list = []
        temp_list.append(history_status[-1])
        for item in history_status[:-1]:
            if int(item[-1]) != 0:
                temp_list.append(item)
        sorted_list = sorted(temp_list, key=lambda x:x[0], reverse=True)
        task_detail['history_status'] = sorted_list
    else:
        task_detail['history_status'] = history_status
    task_detail['social_sensors_portrait'] = []
    portrait_detail = []

    if task_detail["social_sensors"]:
        search_results = es.mget(index=portrait_index_name, doc_type=portrait_index_type, body={"ids": task_detail["social_sensors"]})['docs']
        if search_results:
            for item in search_results:
                temp = []
                if item['found']:
                    for iter_item in SOCIAL_SENSOR_INFO:
                        if iter_item == "topic_string":
                            temp.append(item["_source"][iter_item].split('&'))
                        elif iter_item == "influence":
                            top_influence = get_top_influence("influence")
                            influence = math.log(item["_source"][iter_item]/top_influence*9+1, 10)*100
                            if not influence:
                                influence = 0
                            temp.append(influence)
                        elif iter_item == "importance":
                            top_importance = get_top_influence("importance")
                            importance = math.log(item["_source"][iter_item]/top_importance*9+1, 10)*100
                            if not importance:
                                importance = 0
                            temp.append(importance)
                        elif iter_item == "activeness":
                            top_activeness = get_top_influence("activeness")
                            activeness = math.log(item["_source"][iter_item]/top_activeness*9+1, 10)*100
                            if not activeness:
                                activeness = 0
                            temp.append(activeness)
                        else:
                            temp.append(item["_source"][iter_item])
                    portrait_detail.append(temp)
        if portrait_detail:
            portrait_detail = sorted(portrait_detail, key=lambda x:x[5], reverse=True)
    task_detail['social_sensors_portrait'] = portrait_detail

    #print task_detail
    return json.dumps(task_detail)

def get_attribute_value(attribute_name):
    attribute_value_list = []
    try:
        attribute_result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
    except:
        return 'no attribute'
    print 'attribute_result:', attribute_result
    attribute_value_string = attribute_result['attribute_value']
    attribute_value_list = attribute_value_string.split('&')
    return attribute_value_list

def ajax_stop_task():
    task_name = request.args.get('task_name','') # must
    if task_name:
        task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name)['_source']
        #task_detail["finish"] = finish_signal
        task_detail['processing_status'] = '0'
        es.index(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name, body=task_detail)
        return json.dumps(['1'])
    else:
        return json.dumps([])

def change_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
    status = False
    #identify the user exist
    #identify the attribute exist
    #identify the attribute value exist
    #identify the submit_user have been admitted----without 
    try:
        user_exist = es.get(index=user_index_name, doc_type=user_index_type, id=uid)['_source']
    except:
        return 'no user'
    try:
        attribute_result = es.get(index=attribute_index_name, doc_type=user_index_type, id=uid)['_source']
    except:
        return 'no attribute'
    value_list = json.loads(attribute_result['value'])
    if attribute_value not in value_list:
        return 'no attribute value'
    es.update(index=user_index_name, doc_type=user_index_type, id=uid, body={'doc':body})
    status = True
    return status

def ajax_get_keywords_list():
    task_name = request.args.get('task_name','') # task_name
    keywords = request.args.get('keywords', '') # warning keywords, seperate with ","
    keywords_list = keywords.split(',')
    ts = request.args.get('ts', '') # timestamp: 123456789

    task_detail = es.get(index=index_manage_sensing_task, doc_type=task_doc_type, id=task_name)['_source']
    start_time = task_detail['create_at']

    results = aggregation_hot_keywords(start_time, ts, keywords_list)

    return json.dumps(results)

def get_user_attribute_name(uid):
    result = []
    user_result = es.get(index=user_index_name, doc_type=user_index_type, \
                        id=uid)
    try:
        source = user_result['_source']
    except:
        source = {}
    for key in source:
        if key not in identify_attribute_list:
            result.append(key)
    return result

def get_user_attribute_name(uid, submit_user):
    result = []
    user_result = es.get(index=user_index_name, doc_type=user_index_type, \
                        id=uid)
    try:
        source = user_result['_source']
    except:
        source = {}
    submit_user_tag = submit_user + "-tag"
    submit_user_attribute = source.get(submit_user_tag, '')
    tag_list = submit_user_attribute.split('&')
    for item in tag_list:
        result.append(item.split('-')[0])
    return result