def _get_headers(self, resource):
        """
        Get CSV file headers from the provided resource.
        """

        # If the resource is a file we just open it up with the csv
        # reader (after being sure we're reading from the beginning
        # of the file
        if type(resource) == file:
            resource.seek(0)
            reader = csv.reader(resource)
        # If the resource is a basestring it is either a url or a file
        # location, so similarly to the specification mechanism we either
        # access it with an HTTP get request or by opening the file.
        elif isinstance(resource, basestring):
            result = six.moves.urllib.parse.urlparse(resource)
            if result.scheme in ['http', 'https']:
                with closing(requests.get(resource, stream=True)) as response:
                    # Headers are alway the first row of a CSV file
                    # so it's enought to just get the first line and
                    # hopefully save bandwidth
                    header_row = response.iter_lines().next()
            else:
                # It may seem weird to open up a csv file, read its header row
                # and then StringIO that into a new csv reader but this file
                # we want to close and we want the same interface for all
                with open(resource) as resource_file:
                    reader = csv.reader(resource_file)
                    header_row = reader.next()

            reader = csv.reader(cStringIO.StringIO(header_row))
        else:
            raise IOError('Resource type not supported')
        return reader.next()    

def output_lfc_diff_using_csv(path_to_csv_earlier, path_to_csv_later, output_path):
    fleet_time_a = {}
    fleet_time_b = {}
    fleet_diff = {}
    with open(path_to_csv_earlier, 'rUb') as csvfile:
        reader = csv.reader(csvfile, errors='ignore')
        for row in reader:
            if len(row) >= 2:
                fleet_time_a[row[0].strip()] = long(row[1].strip())
    with open(path_to_csv_later, 'rUb') as csvfile:
        reader = csv.reader(csvfile, errors='ignore')
        for row in reader:
            if len(row) >= 2:
                fleet_time_b[row[0].strip()] = long(row[1].strip())
    for account_name in fleet_time_b.keys():
        if account_name in fleet_time_a:
            lfc_diff = fleet_time_b[account_name] - fleet_time_a[account_name]
        else:
            lfc_diff = fleet_time_b[account_name]
        if lfc_diff > 0:
            fleet_diff[account_name] = lfc_diff
    fleet_diff_output = fleet_diff.items()
    fleet_diff_output.sort(key=lambda account_tuple: account_tuple[1], reverse=True)
    with open(output_path, 'wb') as csvfile:
        cwriter = csv.writer(csvfile)
        for account_tuple in fleet_diff_output:
            cwriter.writerow(account_tuple)

    def __init__(self,file=None):

        filesniff = open(file)
        try:
            dialect = unicodecsv.Sniffer().sniff(filesniff.read(1024))
            wb = unicodecsv.reader(open(file),dialect,encoding='utf-8')
        except Exception:
            wb = unicodecsv.reader(open(file),delimiter=',',encoding='utf-8')

        self.wb = wb

        reader = wb
        rows = []
        columns = []
        #
        # for rownum in range(sh1.nrows): # sh1.nrows -> number of rows (ncols -> num columns)
        #     rows.append(sh1.row_values(rownum))

        for row in reader:
            rows.append(row)

        print rows

        columns = self.columnsExtract(rows)


        print columns

        res = Generator().main(rows=rows,columns=columns)

        self.res = res

def open_file(infile, informat='raw', encoding="utf-8", **kwargs):
    logger.debug('Opening file: {}'.format(infile))
    if isinstance(infile, basestring):
        if informat == "vnd.ms-excel" or informat == 'xls':
            import xlrd
            logger.debug('An office file!')
            f = xlrd.open_workbook(infile, on_demand=True)
        elif informat == "xml":
            logger.debug('An XML file!')
            f = etree.parse(infile)
        elif informat == "csv":
            logger.debug('Opening as csv')
            f = csv.reader(open(infile, 'r'),
                           encoding=encoding,
                           **kwargs)
        else:
            f = codecs.open(infile, 'r', encoding)
    else:
        if informat == "vnd.ms-excel" or informat == 'xls':
            import xlrd
            logger.debug('An office file!')
            f = xlrd.open_workbook(file_contents=infile.read(), on_demand=True)
        elif informat == "xml":
            logger.debug('An XML file!')
            f = etree.fromstring(infile)
        elif informat == "csv":
            logger.debug("CSV file")
            f = csv.reader(infile, encoding=encoding, **kwargs)
        else:
            f = codecs.iterdecode(iter(infile.readline, ""), encoding)
    return f

def getBarChartData(): #First function to get Bar Chart Data
    f_artists = open('artists.csv') #Opens artists.csv for editing and makes a variable out of it.
    f_albums = open('albums.csv') #Opens albums.csv for editing and makes a variable out of it.

    artists_rows = csv.reader(f_artists) #Creates rows in artists.csv
    albums_rows = csv.reader(f_albums) #Creates rows in albums.csv

    artists_header = artists_rows.next() #Creates header in artists.csv
    albums_header = albums_rows.next() #Creates header in albums.csv

    artist_names = [] # New list in which to store artist names.
    
    decades = range(1900,2020, 10) #For upcoming decade dictionary, sets limits and intervals for how it will store dates by decade.
    decade_dict = {} #Creates decae dictionary.
    for decade in decades: #Sets conditions on what happens to this dictionary.
        decade_dict[decade] = 0 #Sets initial value to 0
    
    for artist_row in artists_rows: #Sets conditions on rows in artists.csv
        if not artist_row: #condition: skip if it does not correspond to a row in artists.csv
            continue
        artist_id,name,followers, popularity = artist_row # set following values to go in entries to rows section of arists.csv
        artist_names.append(name) #puts all artist names in the list

    for album_row  in albums_rows: #sets conditions on rows in albums.csv
        if not album_row: #condition: skip if it does not correspond to a row in albums.csv
            continue
        artist_id, album_id, album_name, year, popularity = album_row #sets following values to go in entries to rows section of albums.csv
        for decade in decades: #conditions on decades it assigns to album
            if (int(year) >= int(decade)) and (int(year) < (int(decade) + 10)): #takes year in album and counts the decade in which the year belongs
                decade_dict[decade] += 1
                break #ends the condition on the album row

    x_values = decades #for upcoming chart, sets decades as the x value
    y_values = [decade_dict[d] for d in decades] #for upcoming chart, sets number of albums as y value
    return x_values, y_values, artist_names #brings all of the values generated in the function for the upcoming barchart function

    def handle_noargs(self, **options):
        global YEAR, COMMIT
        YEAR = options['year']
        COMMIT = options['commit']

        if not options['candidates'] or not os.path.exists(options['candidates']):
            print >> sys.stderr, "The candidates file doesn't exist"
            sys.exit(1)
        if not YEAR:
            print >> sys.stderr, "You must specify a year"
            sys.exit(1)

        #check all the parties exist
        with open(options['candidates'], 'rb') as csvfile:
            candidiates = unicodecsv.reader(csvfile)
            missingparties = False
            lastmissingparty = ''
            for row in candidiates:
                if not get_party(row[0]):
                    if row[0] != lastmissingparty:
                        print 'Missing party:', row[0]
                        lastmissingparty = row[0]
                    missingparties = True
            if missingparties:
                sys.exit(1)

        #check whether the positions exist, otherwise create them
        check_or_create_positions()

        with open(options['candidates'], 'rb') as csvfile:
            candidiates = unicodecsv.reader(csvfile)
            for row in candidiates:
                if not search(row[3], row[4], row[0], row[2], row[1]):
                    add_new_person(row[0], row[2], row[1], row[3], row[4])

    def _get_py(self, key):
        if isinstance(key, tuple):
            assert len(key) == 2
            result = self._get_py(key[0])

            if isinstance(key[1], list):
                getter = itemgetter(*key[1])
            else:
                getter = itemgetter(key[1])

            if isinstance(key[0], (list, slice)):
                return map(getter, result)
            else:
                return getter(result)

        f = self.open(self.path)
        if self.header:
            next(f)
        if isinstance(key, compatibility._inttypes):
            line = nth(key, f)
            result = next(csv.reader([line], **self.dialect))
        elif isinstance(key, list):
            lines = nth_list(key, f)
            result = csv.reader(lines, **self.dialect)
        elif isinstance(key, slice):
            start, stop, step = key.start, key.stop, key.step
            result = csv.reader(it.islice(f, start, stop, step), **self.dialect)
        else:
            raise IndexError("key '%r' is not valid" % key)
        try:
            if not isinstance(result, Iterator):
                f.close()
        except AttributeError:
            pass
        return result

def validate_and_return_rows(csv_file_form=None, csv_file=None, has_header_row=True, required_fields=[]):
	"""
		Opens a CSV file and optionally checks for required fields in the header. Returns
		the rows of the CSV and a list of the headers as a tuple: (csv_rows, headers)
	"""
	if csv_file_form:
		csv_file = csv_file_form.cleaned_data['file']
		has_header_row = csv_file_form.cleaned_data['has_header_row']

	if not csv_file:
		raise Exception("Pass in CsvFileForm instance or csv_file=request.FILES.get('file')")

	column_headers = []
	if has_header_row:
		# Read and store the header row column names
		r = unicodecsv.reader(csv_file.read().splitlines(), encoding='utf-8')
		row = r.next()
		for column_header in row:
			column_headers.append(column_header)

		# Check for required fields (optional check)
		for field in required_fields:
			if field not in column_headers:
				raise Exception("Invalid CSV file. Must contain %s." % field)

	csv_file.seek(0)
	return (unicodecsv.reader(csv_file.read().splitlines(), encoding='utf-8'), column_headers)

def processData():
    global manualIgnoreRecords
    global yesIgnoreRecords
    global manualProcessedRecords		
    global yesProcessedRecords		
    	
    dirpath = parentdir + "/R3_profiles_YNNM_raw/" 
    with open(dirpath + 'MANUAL_RAW.csv', 'r') as infile, open(processeddir + 'MANUAL_PROCESSED.csv', 'ab') as outfile:
	rows = unicodecsv.reader(infile, delimiter=';', encoding='utf-8')
	writer = unicodecsv.writer(outfile, delimiter=';', encoding='utf-8')
	for row in rows:
	    if(row[6] in manual_ignore_list): #Ignore it
		manualIgnoreRecords += 1
		continue
	    else:
		manualProcessedRecords += 1
		writer.writerow(row)

    with open(dirpath + 'YES_RAW.csv', 'r') as infile, open(processeddir + 'YES_PROCESSED.csv', 'ab') as outfile:
	rows = unicodecsv.reader(infile, delimiter=';', encoding='utf-8')
	writer = unicodecsv.writer(outfile, delimiter=';', encoding='utf-8')	
	for row in rows:
	    if(row[6] in yes_ignore_list): #Ignore it	
		yesIgnoreRecords += 1
		continue
	    else:
		yesProcessedRecords
		writer.writerow(row)

def patchsql(sys_args):

    parser = argparse.ArgumentParser(description='Patch a database.')

    parser.add_argument('url', help='Sqlalchemy-compatible database url')

    parser.add_argument('--patch', nargs=1, required=False, default=None,
                        help="A csv file describing the patch. In the "
                        "format output by daff.")

    parser.add_argument('--follow', nargs=2, required=False, default=None,
                        help="An alternative to --patch option.  Specify"
                        "two csv files to compare, and patch from their diff.")

    parser.add_argument('--table', nargs=1, required=True, default=None,
                        help='Table to which patch should be applied.')

    parser.add_argument('--safe-null', required=False, action='store_true',
                        help='Decode nulls in a reversible way.')

    parser.add_argument('--quiet', required=False, action='store_true',
                       help='Do not show computed diff.')

    args = parser.parse_args(sys_args)

    url = args.url
    tables = args.table

    db = SqlAlchemyDatabase(url)
    st = daff.SqlTable(db, daff.SqlTableName(tables[0]))

    patch = None

    if args.patch:
        with open(args.patch[0], 'rt') as fin:
            patch = list(csv.reader(fin))
            patch = daff.Coopy.tablify(patch)

    if args.follow:
        with open(args.follow[0], 'rt') as fin:
            table0 = list(csv.reader(fin))
            fix_nulls(table0, args.safe_null)
        with open(args.follow[1], 'rt') as fin:
            table1 = list(csv.reader(fin))
            fix_nulls(table1, args.safe_null)
        patch = daff.Coopy.diff(table0, table1)
        ansi_patch = daff.Coopy.diffAsAnsi(table0, table1)
        if not args.quiet:
            print(ansi_patch, file=sys.stderr, end='')

    if not patch:
        raise KeyError('please specify either --patch or --follow')

    daff_patch = daff.HighlightPatch(st, patch)
    daff_patch.apply()
    if db.events['skips'] != 0:
        print(" * {}".format(json.dumps(db.events),
                             file=sys.stderr))

    def compare_csvs(self, csv1, csv2):
        """Compara dos csvs a ver si son iguales."""
        with open(csv1, "rb") as csvfile1, open(csv2, "rb") as csvfile2:
            reader1 = unicodecsv.reader(csvfile1, delimiter=str(","), quotechar=str('"'))
            reader2 = unicodecsv.reader(csvfile2, delimiter=str(","), quotechar=str('"'))
            rows1 = set((",".join(row) for row in reader1))
            rows2 = set((",".join(row) for row in reader2))

            self.assertEqual(len(rows1), len(rows2))
            self.assertEqual(rows1, rows2)

def main(args):

    finished = defaultdict(int)
    input_lines = []
    skipped = defaultdict(int)
    written = defaultdict(int)

    # load input data
    with args.input_file as fh:
        csvread = csv.reader(fh, delimiter=str(args.input_csv_delim),
                             quotechar=b'"', encoding="UTF-8")
        columns = DataLine.get_columns_from_header(csvread.next())
        for row in csvread:
            input_lines.append(DataLine.from_csv_line(row, columns))

    # load all results files provided
    for finished_file in args.finished_files:
        with finished_file as fh:
            csvread = csv.reader(fh, delimiter=str(args.finished_csv_delim),
                                 quotechar=b'"', encoding="UTF-8")
            header = csvread.next();
            columns = DataLine.get_columns_from_header(header)
            try:
                judgment_column = header.index('check_result')
            except ValueError:
                judgment_column = None
            for row in csvread:
                # treat rejected as unfinished
                if judgment_column is not None and row[judgment_column].startswith('N'):
                    continue
                # keep track of how many judgments are finished in the results
                finished_line = DataLine.from_csv_line(row, columns)
                finished[finished_line.signature] += 1

    print >> sys.stderr, "Loaded input: %d, Loaded finished: %d" % (len(input_lines), len(finished))

    with sys.stdout as fh:
        # starting with the header
        csvwrite = csv.writer(fh, delimiter=b"\t", lineterminator="\n", encoding="UTF-8")
        csvwrite.writerow(DataLine.get_headers())
        # write rows requiring different number of judgments, starting from the most judgments
        for judg_req in xrange(args.num_judgments, 0, -1):

            csvwrite.writerow(("# Requiring %d judgments" % judg_req,))
            for line in input_lines:
                if finished[line.signature] != args.num_judgments - judg_req:
                    skipped[judg_req] += 1
                    continue
                csvwrite.writerow(line.as_tuple())
                written[judg_req] += 1

            print >> sys.stderr, ("%d judgments -- written: %d" % (judg_req, written[judg_req]))

    print >> sys.stderr, "Skipped: %d" % (len(input_lines) - sum(written.values()))

def main():

    #Connect to database file (Note: can also pass as a file using sys)
    CONN = sqlite3.connect('melon.db')

    #This cursor object passes commands from python and executes them in the sqlite3 melon.db
    DB = CONN.cursor()

    #Deletes tables if they exist. Good if I made a mistake creating them.
    DB.execute('''DROP TABLE IF EXISTS Customers;''')
    DB.execute('''DROP TABLE IF EXISTS Orders;''')

    #Create 2 tables of Customers & Orders
    DB.execute('''CREATE TABLE Customers (customer_id INTEGER PRIMARY KEY NOT NULL, first varchar(30), last varchar(30), email varchar(60), telephone varchar(30), called DATE);''')

    #Note: Foreign key & Reference needs to be stated at the end of the create table dialog.
    DB.execute('''CREATE TABLE Orders (order_id INTEGER PRIMARY KEY NOT NULL, order_date DATE,status varchar(30), customer_id INTEGER,email varchar(60),address varchar(30),city varchar(30),state varchar(30),postalcode varchar(30),num_watermelons INTEGER,num_othermelons INTEGER ,subtotal INTEGER ,tax INTEGER ,order_total INTEGER, FOREIGN KEY (customer_id) REFERENCES Customers(customer_id)); ''')


    #CSV reader reads each line and strips, splits each line into a list object. :)

    f1reader = unicodecsv.reader(open('customers.csv'), encoding='utf-8')
    f2reader = unicodecsv.reader(open('orders.csv'), encoding='utf-8')

    
    #Next skips collumn names // headers of csv file

    next(f1reader)
    next(f2reader)

# Text formatting in file (nyee '~' & special charecters) cannot be interpreted by sqlite. Need to import unicode csv module to parse code.

#Note: There are some shity looking customer files with strange charecters. What up with dat? Can they be flagged and edited later on? (eg., like editing student records) 


    for row in f1reader:

        DB.executemany('''INSERT INTO customers (customer_id, first, last, email, telephone, called) VALUES(?, ?, ?, ?, ?, ?);''', (row, ))

    for row2 in f2reader:

        DB.executemany('''INSERT INTO orders (order_id, order_date, status, customer_id, email, address, city, state, postalcode, num_watermelons, num_othermelons, subtotal, tax, order_total) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);''', (row2, ))

        #Change empty string row values to Null.
        # DB.executemany('''UPDATE orders SET VALUES 
        #   ''')



    
    CONN.commit()
    CONN.close()

def read_files(qfile_train, qfile_test, catfile):
	"""
	read from .csv files
		qfile_train - .csv file containing the SMS Guru questions for the train set
		qfile_test - .csv file containing the SMS Guru questions for the test set
		catfile - .csv file containing the categories
	"""
	with open(qfile_train, 'rb') as csvfile:
		question_train = list(unicodecsv.reader(csvfile, delimiter=",", quoting=unicodecsv.QUOTE_ALL, escapechar="\\", encoding='utf-8'))
	with open(qfile_test, 'rb') as csvfile:
		question_test = list(unicodecsv.reader(csvfile, delimiter=",", quoting=unicodecsv.QUOTE_ALL, escapechar="\\", encoding='utf-8'))
	with open(catfile, 'rb') as csvfile:
		category = list(unicodecsv.reader(csvfile, delimiter=",", quoting=unicodecsv.QUOTE_ALL, escapechar="\\", encoding='utf-8'))
	return question_train, question_test, category

def read_files(qfile, qcatfile, catfile):
	"""
	read from .csv files
		qfile - .csv file containing the SMS Guru questions
		qcatfile - .csv file containing the relation between questions and category
		catfile - .csv file containing the categories
	"""
	with open(qfile, 'rb') as csvfile:
		question_train = list(unicodecsv.reader(csvfile, delimiter=",", quoting=unicodecsv.QUOTE_ALL, escapechar="\\", encoding='utf-8'))
	with open(qcatfile, 'rb') as csvfile:
		question_category_train = list(unicodecsv.reader(csvfile, delimiter=",", quoting=unicodecsv.QUOTE_ALL, escapechar="\\", encoding='utf-8'))
	with open(catfile, 'rb') as csvfile:
		category = list(unicodecsv.reader(csvfile, delimiter=",", quoting=unicodecsv.QUOTE_ALL, escapechar="\\", encoding='utf-8'))
	return question_train, question_category_train, category

def main(wordlist1, wordlist2, dist_funcs):
    with open(wordlist1, 'rb') as file_a, open(wordlist2, 'rb') as file_b:
        reader_a = csv.reader(file_a, encoding='utf-8')
        reader_b = csv.reader(file_b, encoding='utf-8')
        print('Reading word lists...')
        words = list(zip([(w, g) for (g, w) in reader_a],
                    [(w, g) for (g, w) in reader_b]))
        words_a, words_b = list(zip(*[(a, b) for (a, b) in words if a and b]))
        print('Constructing cost matrix...')
        matrix = construct_cost_matrix(words_a, words_b, dist_funcs)
        m = munkres.Munkres()
        print('Computing matrix using Hungarian Algorithm...')
        indices = m.compute(matrix)
        print(score(indices))
        print('Done.')

def writeUniqueResults(clustered_dupes, input_file, output_file):

    # Write our original data back out to a CSV with a new column called 
    # 'Cluster ID' which indicates which records refer to each other.

    logging.info('saving unique results to: %s' % output_file)

    cluster_membership = {}
    for (cluster_id, cluster) in enumerate(clustered_dupes):
        for record_id in cluster:
            cluster_membership[record_id] = cluster_id

    unique_record_id = cluster_id + 1

    writer = csv.writer(output_file)

    reader = csv.reader(StringIO(input_file))

    heading_row = next(reader)
    heading_row.insert(0, u'Cluster ID')
    writer.writerow(heading_row)

    seen_clusters = set()
    for row_id, row in enumerate(reader):
        if row_id in cluster_membership:
            cluster_id = cluster_membership[row_id]
            if cluster_id not in seen_clusters:
                row.insert(0, cluster_id)
                writer.writerow(row)
                seen_clusters.add(cluster_id)
        else:
            cluster_id = unique_record_id
            unique_record_id += 1
            row.insert(0, cluster_id)
            writer.writerow(row)

def iter_rows(building_data):
    """
    Opens the given file-like object as a CSV file delimited by pipes,
    and yields a list for each row containing the land plot number,
    building id, street address, post code, and latitude, and longitude.
    """
    reader = unicodecsv.reader(building_data, delimiter="|", encoding="latin1")
    reader.next()  # Skip header row.
    for row in reader:
        land_plot_number = int(row[3])
        building_id = int(row[4])
        street_address = u" ".join(row[5].split())  # Normalise whitespace.
        try:
            post_code = int(row[7])
        except ValueError:
            post_code = None
        isn93_x = float(row[22].replace(",", "."))
        isn93_y = float(row[23].replace(",", "."))
        longitude, latitude = isnet93_to_wgs84(isn93_x, isn93_y)
        yield {
            "landnr": land_plot_number,
            "heitinr": building_id,
            "street": street_address,
            "postcode": post_code,
            "ll": (longitude, latitude),
        }

    def handle_label(self, label, **options):
        verbosity = int(options['verbosity'])

        if not os.path.exists(label):
            print >> sys.stderr, "The parties file doesn't exist",
            sys.exit(1)

        #get the party kind object
        partykind = OrganisationKind.objects.get(slug='party')

        #check each party by checking against slug
        with open(label, 'rb') as csvfile:
            parties = unicodecsv.reader(csvfile)
            for slug, name in parties:
                try:
                    party = Organisation.objects.get(slug=slug)
                    if party.name != name:
                        if verbosity >= 1:
                            print 'Updating party %s from %s to %s' % (slug, party.name, name)
                        party.name = name
                        party.save()
                except Organisation.DoesNotExist:
                    #we need to add the party
                    if verbosity >= 1:
                        print 'Adding party %s' % name
                    Organisation.objects.create(
                        name = name,
                        slug = slug,
                        kind = partykind)

    def process(self):
        report = self.receive_message()
        
        if report:
            event = Event()

            columns = ["__IGNORE__", "source_url", "description_url", "source_time", "__IGNORE__", "__IGNORE__", "__IGNORE__", "target"]
            
            for row in unicodecsv.reader(StringIO(report), encoding='utf-8'):

                if "phish_id" in row:
                    continue
                
                for key, value in zip(columns, row):

                    if key == "__IGNORE__":
                        continue
                    
                    event.add(key, value.strip())
                
                event.add('feed', 'phishtank')
                event.add('type', 'phishing')

                event = utils.parse_source_time(event, "source_time")
                event = utils.generate_observation_time(event, "observation_time")
                event = utils.generate_reported_fields(event)
                    
                self.send_message(event)
             
        self.acknowledge_message()