def set_out_paste(decoder_name, message):
    publisher.warning(decoder_name+' decoded')
    #Send to duplicate
    p.populate_set_out(message, 'Duplicate')

    msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
    p.populate_set_out(msg, 'Tags')

def db_import(filename, day):
    with open(filename, 'r') as f:
        entry = ''
        pipeline = routing_db.pipeline()
        i = 0
        for line in f:
            # End of block, extracting the information
            if line == '\n':
                i += 1
                parsed = re.findall('(?:ASPATH|PREFIX): ([^\n{]*)', entry)
                try:
                    block = parsed[0].strip()
                    # RIPE-NCC-RIS BGP IPv6 Anchor Prefix @RRC00
                    # RIPE-NCC-RIS BGP Anchor Prefix @ rrc00 - RIPE NCC
                    if block in ['2001:7fb:ff00::/48', '84.205.80.0/24',
                            '2001:7fb:fe00::/48', '84.205.64.0/24']:
                        asn = 12654
                    else:
                        asn = int(parsed[1].split()[-1].strip())
                    pipeline.hset(block, day, asn)
                except:
                    #FIXME: check the cause of the exception
                    publisher.warning(entry)
                entry = ''
                if i%10000 == 0:
                    pipeline.execute()
                    pipeline = routing_db.pipeline()
            else :
                # append the line to the current block.
                entry += line
        pipeline.execute()
        publisher.info('{f} finished, {nb} entries impported.'.\
                format(f=filename, nb = i))

def main():
    """Main Function"""

    # CONFIG #
    cfg = ConfigParser.ConfigParser()
    cfg.read(configfile)

    # REDIS #
    r_serv = redis.StrictRedis(
        host = cfg.get("Redis_Queues", "host"),
        port = cfg.getint("Redis_Queues", "port"),
        db = cfg.getint("Redis_Queues", "db"))

    # LOGGING #
    publisher.channel = "Queuing"

    # ZMQ #
    channel = cfg.get("PubSub_Words", "channel_0")
    subscriber_name = "curve"
    subscriber_config_section = "PubSub_Words"

    Sub = ZMQ_PubSub.ZMQSub(configfile, subscriber_config_section, channel, subscriber_name)
    # FUNCTIONS #
    publisher.info("""Suscribed to channel {0}""".format(channel))

    while True:
        Sub.get_and_lpush(r_serv)

        if r_serv.sismember("SHUTDOWN_FLAGS", "Curve_Q"):
            r_serv.srem("SHUTDOWN_FLAGS", "Curve_Q")
            print "Shutdown Flag Up: Terminating"
            publisher.warning("Shutdown Flag Up: Terminating.")
            break

def search_key(content, message, paste):
    bitcoin_address = re.findall(regex_bitcoin_public_address, content)
    bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
    validate_address = False
    key = False
    if(len(bitcoin_address) >0):
        #print(message)
        for address in bitcoin_address:
            if(check_bc(address)):
                validate_address = True
                print('Bitcoin address found : {}'.format(address))
                if(len(bitcoin_private_key) > 0):
                    for private_key in bitcoin_private_key:
                        print('Bitcoin private key found : {}'.format(private_key))
                        key = True

        if(validate_address):
            p.populate_set_out(message, 'Duplicate')
            to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
            print(to_print)
            publisher.warning(to_print)
            msg = ('bitcoin;{}'.format(message))
            p.populate_set_out( msg, 'alertHandler')

            msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message)
            p.populate_set_out(msg, 'Tags')

            if(key):
                msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message)
                p.populate_set_out(msg, 'Tags')

                to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date,
                                                    paste.p_name)
                publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
                    to_print, len(bitcoin_private_key),paste.p_path))

def main():
    """Main Function"""

    # CONFIG #
    cfg = ConfigParser.ConfigParser()
    cfg.read(configfile)

    # REDIS #
    r_serv = redis.StrictRedis(
        host = cfg.get("Redis_Queues", "host"),
        port = cfg.getint("Redis_Queues", "port"),
        db = cfg.getint("Redis_Queues", "db"))

    # LOGGING #
    publisher.channel = "Queuing"

    # ZMQ #
    Sub = ZMQ_PubSub.ZMQSub(configfile,"PubSub_Categ", "onion_categ", "tor")

    # FUNCTIONS #
    publisher.info("""Suscribed to channel {0}""".format("onion_categ"))

    while True:
        Sub.get_and_lpush(r_serv)

        if r_serv.sismember("SHUTDOWN_FLAGS", "Onion_Q"):
            r_serv.srem("SHUTDOWN_FLAGS", "Onion_Q")
            print "Shutdown Flag Up: Terminating"
            publisher.warning("Shutdown Flag Up: Terminating.")
            break

def analyse(url, path):
    faup.decode(url)
    url_parsed = faup.get()

    resource_path = url_parsed['resource_path']
    query_string = url_parsed['query_string']

    result_path = 0
    result_query = 0

    if resource_path is not None:
        result_path = is_sql_injection(resource_path)

    if query_string is not None:
        result_query = is_sql_injection(query_string)

    if (result_path > 0) or (result_query > 0):
        paste = Paste.Paste(path)
        if (result_path > 1) or (result_query > 1):
            print "Detected SQL in URL: "
            print urllib2.unquote(url)
            to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL")
            publisher.warning(to_print)
            #Send to duplicate
            p.populate_set_out(path, 'Duplicate')
            #send to Browse_warning_paste
            p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste')
        else:
            print "Potential SQL injection:"
            print urllib2.unquote(url)
            to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection")
            publisher.info(to_print)

def search_phone(message):
    paste = Paste.Paste(message)
    content = paste.get_p_content()
    # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
    reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
    reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})')
    # list of the regex results in the Paste, may be null
    results = reg_phone.findall(content)

    # if the list is greater than 4, we consider the Paste may contain a list of phone numbers
    if len(results) > 4:
        print(results)
        publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))

        msg = 'infoleak:automatic-detection="phone-number";{}'.format(message)
        p.populate_set_out(msg, 'Tags')

        #Send to duplicate
        p.populate_set_out(message, 'Duplicate')
        stats = {}
        for phone_number in results:
            try:
                x = phonenumbers.parse(phone_number, None)
                country_code = x.country_code
                if stats.get(country_code) is None:
                    stats[country_code] = 1
                else:
                    stats[country_code] = stats[country_code] + 1
            except:
                pass
        for country_code in stats:
            if stats[country_code] > 4:
                publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))

def search_gpg(message):
    paste = Paste.Paste(message)
    content = paste.get_p_content()
    if '-----BEGIN PGP MESSAGE-----' in content:
        publisher.warning('{} has a PGP enc message'.format(paste.p_name))
        #Send to duplicate
        p.populate_set_out(message, 'Duplicate')
        #send to Browse_warning_paste
        p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste')

def main():
    """Main Function"""

    # CONFIG #
    cfg = ConfigParser.ConfigParser()
    cfg.read(configfile)

    # REDIS #
    r_serv = redis.StrictRedis(
        host = cfg.get("Redis_Queues", "host"),
        port = cfg.getint("Redis_Queues", "port"),
        db = cfg.getint("Redis_Queues", "db"))

    # LOGGING #
    publisher.channel = "Script"

    # ZMQ #
    channel = cfg.get("PubSub_Longlines", "channel_1")
    subscriber_name = "tokenize"
    subscriber_config_section = "PubSub_Longlines"

    #Publisher
    publisher_config_section = "PubSub_Words"
    publisher_name = "pubtokenize"

    Sub = ZMQ_PubSub.ZMQSub(configfile, subscriber_config_section, channel, subscriber_name)
    Pub = ZMQ_PubSub.ZMQPub(configfile, publisher_config_section, publisher_name)

    channel_0 = cfg.get("PubSub_Words", "channel_0")

    # FUNCTIONS #
    publisher.info("Tokeniser subscribed to channel {0}".format(cfg.get("PubSub_Longlines", "channel_1")))

    while True:
        message = Sub.get_msg_from_queue(r_serv)
        print message
        if message != None:
            PST = P.Paste(message.split(" ",-1)[-1])
        else:
            if r_serv.sismember("SHUTDOWN_FLAGS", "Tokenize"):
                r_serv.srem("SHUTDOWN_FLAGS", "Tokenize")
                print "Shutdown Flag Up: Terminating"
                publisher.warning("Shutdown Flag Up: Terminating.")
                break
            publisher.debug("Tokeniser is idling 10s")
            time.sleep(10)
            print "sleepin"
            continue

        for word, score in PST._get_top_words().items():
            if len(word) >= 4:
                msg = channel_0+' '+PST.p_path+' '+str(word)+' '+str(score)
                Pub.send_message(msg)
                print msg
            else:
                pass

def redis_interbargraph_set(r_serv, year, month, overwrite):
    """Create a Redis sorted set.

    :param r_serv: -- connexion to redis database
    :param year: -- (integer) The year to process
    :param month: -- (integer) The month to process
    :param overwrite: -- (bool) trigger the overwrite mode

    This function create inside redis the intersection of all days in
    a month two by two.
    Example:
    For a month of 31days it will create 30 sorted set between day and
    day+1 until the last day.
    The overwrite mode delete the intersets and re-create them.

    """
    a = date(year, month, 01)
    b = date(year, month, cal.monthrange(year, month)[1])

    if overwrite:
        r_serv.delete("InterSet")

        for dt in rrule(DAILY, dtstart = a, until = b - timedelta(1)):
            dayafter = dt+timedelta(1)

            r_serv.delete(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")))

            r_serv.zinterstore(
                str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")),
                {str(dt.strftime("%Y%m%d")):1,
                str(dayafter.strftime("%Y%m%d")):-1})

            r_serv.zadd(
                "InterSet",
                1,
                str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")))
    else:
        for dt in rrule(DAILY, dtstart = a, until = b - timedelta(1)):
            dayafter = dt+timedelta(1)

            if r_serv.zcard(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))) == 0:

                r_serv.zinterstore(
                    str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")),
                    {str(dt.strftime("%Y%m%d")):1,
                    str(dayafter.strftime("%Y%m%d")):-1})

                r_serv.zadd(
                    "InterSet",
                    1,
                    str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")))

                publisher.info(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))+" Intersection Created")

            else:
                publisher.warning("Data already exist, operation aborted.")

def analyse(url, path):
    faup.decode(url)
    url_parsed = faup.get()

    resource_path = url_parsed['resource_path']
    query_string = url_parsed['query_string']

    result_path = 0
    result_query = 0

    if resource_path is not None:
        ## TODO: # FIXME: remove me
        try:
            resource_path = resource_path.decode()
        except:
            pass
        result_path = is_sql_injection(resource_path)

    if query_string is not None:
        ## TODO: # FIXME: remove me
        try:
            query_string = query_string.decode()
        except:
            pass
        result_query = is_sql_injection(query_string)

    if (result_path > 0) or (result_query > 0):
        paste = Paste.Paste(path)
        if (result_path > 1) or (result_query > 1):
            print("Detected SQL in URL: ")
            print(urllib.request.unquote(url))
            to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path)
            publisher.warning(to_print)
            #Send to duplicate
            p.populate_set_out(path, 'Duplicate')

            msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
            p.populate_set_out(msg, 'Tags')

            #statistics
            tld = url_parsed['tld']
            if tld is not None:
                ## TODO: # FIXME: remove me
                try:
                    tld = tld.decode()
                except:
                    pass
                date = datetime.datetime.now().strftime("%Y%m")
                server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)

        else:
            print("Potential SQL injection:")
            print(urllib.request.unquote(url))
            to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_rel_path)
            publisher.info(to_print)

def __query_logging(ip, user_agent, method, q_ip=None, announce_date=None,
                    days_limit=None, level=None):
    if level == 'warning':
        publisher.warning(__csv2string([ip, user_agent, method, q_ip,
                                        announce_date, days_limit, level]))
    elif level == 'error':
        publisher.error(__csv2string([ip, user_agent, method, q_ip,
                                      announce_date, days_limit, level]))
    else:
        publisher.info(__csv2string([ip, user_agent, method, q_ip,
                                     announce_date, days_limit, level]))

def search_phone(message):
    paste = Paste.Paste(message)
    content = paste.get_p_content()
    # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
    reg_phone = re.compile(r"(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})")
    # list of the regex results in the Paste, may be null
    results = reg_phone.findall(content)

    # if the list is greater than 4, we consider the Paste may contain a list of phone numbers
    if len(results) > 4:
        print results
        publisher.warning("{} contains PID (phone numbers)".format(paste.p_name))

 def test_publisher(self):
     for i in range(0, 21):
         if i % 2 == 0:
             publisher.info('test' + str(i))
         elif i % 3 == 0:
             publisher.warning('test' + str(i))
         elif i % 5 == 0:
             publisher.error('test' + str(i))
         elif i % 7 == 0:
             publisher.critical('test' + str(i))
         else:
             publisher.debug('test' + str(i))
         time.sleep(1)

def analyse(url, path):
    faup.decode(url)
    url_parsed = faup.get()
    pprint.pprint(url_parsed)
    ## TODO: # FIXME: remove me
    try:
        resource_path = url_parsed['resource_path'].encode()
    except:
        resource_path = url_parsed['resource_path']

    ## TODO: # FIXME: remove me
    try:
        query_string = url_parsed['query_string'].encode()
    except:
        query_string = url_parsed['query_string']

    result_path = {'sqli' : False}
    result_query = {'sqli' : False}

    if resource_path is not None:
        result_path = pylibinjection.detect_sqli(resource_path)
        print("path is sqli : {0}".format(result_path))

    if query_string is not None:
        result_query = pylibinjection.detect_sqli(query_string)
        print("query is sqli : {0}".format(result_query))

    if result_path['sqli'] is True or result_query['sqli'] is True:
        paste = Paste.Paste(path)
        print("Detected (libinjection) SQL in URL: ")
        print(urllib.request.unquote(url))
        to_print = 'LibInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path)
        publisher.warning(to_print)
        #Send to duplicate
        p.populate_set_out(path, 'Duplicate')

        msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
        p.populate_set_out(msg, 'Tags')

        #statistics
        ## TODO: # FIXME: remove me
        try:
            tld = url_parsed['tld'].decode()
        except:
            tld = url_parsed['tld']
        if tld is not None:
            date = datetime.datetime.now().strftime("%Y%m")
            server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)

def search_phone(message):
    paste = Paste.Paste(message)
    content = paste.get_p_content()
    # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
    reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
    # list of the regex results in the Paste, may be null
    results = reg_phone.findall(content)

    # if the list is greater than 4, we consider the Paste may contain a list of phone numbers
    if len(results) > 4:
        print results
        publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
        #send to Browse_warning_paste
        p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
        #Send to duplicate
        p.populate_set_out(message, 'Duplicate')

def main():
    publisher.port = 6380
    publisher.channel = "Script"

    config_section = 'DomClassifier'

    p = Process(config_section)
    addr_dns = p.config.get("DomClassifier", "dns")

    publisher.info("""ZMQ DomainClassifier is Running""")

    c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns])

    cc = p.config.get("DomClassifier", "cc")
    cc_tld = p.config.get("DomClassifier", "cc_tld")

    while True:
        try:
            message = p.get_from_set()

            if message is not None:
                PST = Paste.Paste(message)
            else:
                publisher.debug("Script DomClassifier is idling 1s")
                time.sleep(1)
                continue
            paste = PST.get_p_content()
            mimetype = PST._get_p_encoding()

            if mimetype == "text/plain":
                c.text(rawtext=paste)
                c.potentialdomain()
                c.validdomain(rtype=['A'], extended=True)
                localizeddomains = c.include(expression=cc_tld)
                if localizeddomains:
                    print(localizeddomains)
                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path))
                localizeddomains = c.localizedomain(cc=cc)
                if localizeddomains:
                    print(localizeddomains)
                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
        except IOError:
            print("CRC Checksum Failed on :", PST.p_path)
            publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
                PST.p_source, PST.p_date, PST.p_name))

def search_cve(message):
    filepath, count = message.split()
    paste = Paste.Paste(filepath)
    content = paste.get_p_content()
    # regex to find CVE
    reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
    # list of the regex results in the Paste, may be null
    results = set(reg_cve.findall(content))

    # if the list is greater than 2, we consider the Paste may contain a list of cve
    if len(results) > 0:
        print('{} contains CVEs'.format(paste.p_name))
        publisher.warning('{} contains CVEs'.format(paste.p_name))

        #send to Browse_warning_paste
        p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste')
        #Send to duplicate
        p.populate_set_out(filepath, 'Duplicate')

def search_phone(message):
    paste = Paste.Paste(message)
    content = paste.get_p_content()
    # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
    reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
    # list of the regex results in the Paste, may be null
    results = reg_phone.findall(content)

    # if the list is greater than 4, we consider the Paste may contain a list of phone numbers
    if len(results) > 4 :
        print results
        publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))

	if __name__ == '__main__':
    # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
    # Port of the redis instance used by pubsublogger
    publisher.port = 6380
    # Script is the default channel used for the modules.
    publisher.channel = 'Script'

    # Section name in bin/packages/modules.cfg
    config_section = 'Phone'

    # Setup the I/O queues
    p = Process(config_section)

    # Sent to the logging a description of the module
    publisher.info("Run Phone module")

    # Endless loop getting messages from the input queue
    while True:
        # Get one message from the input queue
        message = p.get_from_set()
        if message is None:
            publisher.debug("{} queue is empty, waiting".format(config_section))
            time.sleep(1)
            continue

        # Do something with the message from the queue
        search_phone(message)

def search_api_key(message):
    filename, score = message.split()
    paste = Paste.Paste(filename)
    content = paste.get_p_content()

    aws_access_key = regex_aws_access_key.findall(content)
    aws_secret_key = regex_aws_secret_key.findall(content)
    google_api_key = regex_google_api_key.findall(content)

    if(len(aws_access_key) > 0 or len(aws_secret_key) > 0 or len(google_api_key) > 0):

        to_print = 'ApiKey;{};{};{};'.format(
            paste.p_source, paste.p_date, paste.p_name)
        if(len(google_api_key) > 0):
            print('found google api key')
            print(to_print)
            publisher.warning('{}Checked {} found Google API Key;{}'.format(
                to_print, len(google_api_key), paste.p_path))
            msg = 'infoleak:automatic-detection="google-api-key";{}'.format(filename)
            p.populate_set_out(msg, 'Tags')

        if(len(aws_access_key) > 0 or len(aws_secret_key) > 0):
            print('found AWS key')
            print(to_print)
            total = len(aws_access_key) + len(aws_secret_key)
            publisher.warning('{}Checked {} found AWS Key;{}'.format(
                to_print, total, paste.p_path))
            msg = 'infoleak:automatic-detection="aws-key";{}'.format(filename)
            p.populate_set_out(msg, 'Tags')


        msg = 'infoleak:automatic-detection="api-key";{}'.format(filename)
        p.populate_set_out(msg, 'Tags')

        msg = 'apikey;{}'.format(filename)
        p.populate_set_out(msg, 'alertHandler')
        #Send to duplicate
        p.populate_set_out(filename, 'Duplicate')