def parse_file(path):
    with open(path, 'rb') as fh:
        ctx = json.load(fh)

    if ctx['source_name'] not in ['MZ']:
        return

    all_name = slugify('%(source_name)s flexicadastre' % ctx, sep='_')
    all_tbl = database[all_name]
    all_tbl.delete()

    layers = ctx.pop('layers')
    for layer in layers:
        lctx = ctx.copy()
        lctx['layer_name'] = layer['name']
        lctx['layer_id'] = layer['id']
        del lctx['rest_url']

        tbl_name = slugify('%(source_name)s %(layer_name)s' % lctx, sep='_')
        tbl = database[tbl_name]
        tbl.delete()

        features = layer['data']['features']
        print ' -> Generating:', tbl_name
        print '    ', layer['name'], layer['id'], len(features)

        for feature in features:
            attrs = convrow(feature.get('attributes'))
            attrs.update(lctx)
            tbl.insert(attrs)
            all_tbl.insert(attrs)

    def scrape_company(self, data):
        if self.COMPANIES_SCRAPED < self.COMPANY_OFFSET:
            self.COMPANIES_SCRAPED += 1
            logging.debug('skipping %s' % data.get('code', 'unknown'))
            return
        if self.COMPANIES_SCRAPED > self.MAX_COMPANIES + self.COMPANY_OFFSET:
            logging.info('finished companies at no. %s' % self.COMPANIES_SCRAPED)
            return
        self.COMPANIES_SCRAPED += 1
        logging.info('scraping %s' % data)
        url = API_URL % data.get('ASX code')
        data.update(requests.get(url).json())
        if 'code' not in data:
            return
        data['Stock Info URL'] = url
        data.pop('ASX code', None)
        data.pop('primary_share', None)
        data.pop('last_dividend', None)
        data.pop('latest_annual_reports', None)
        data.pop('products', None)

        record = {}
        for k, v in data.items():
            record[slugify(k, sep='_')] = v

        category = slugify(record['gics_industry_group'])
        if category not in ['materials', 'energy']:
            logging.info('skipping category %s' % category)
            return

        self.scrape_announcements(data)

def make_filename(source, sep='-'):
    if source is not None:
        source = os.path.basename(source)
        slugs = [slugify(s, sep=sep) for s in source.split('.')]
        source = '.'.join(slugs)
        source = source.strip('.').strip(sep)
    return source

    def crawl(self, directory=None, collection=None, meta={}):
        collection = collection or directory
        collection = Collection.create({
            'foreign_id': 'directory:%s' % slugify(collection),
            'label': collection
        })
        db.session.commit()
        collection_id = collection.id

        if os.path.isfile(directory):
            self.crawl_file(collection_id, directory, meta)

        directory = directory or os.getcwd()
        directory = directory.encode('utf-8')
        for (dirname, dirs, files) in os.walk(directory):
            dirparts = [d for d in dirname.split(os.path.sep)
                        if d in SKIP_DIRECTORIES]
            if len(dirparts):
                continue
            log.info("Descending: %r", dirname)
            for file_name in files:
                dirname = string_value(dirname)
                file_name = string_value(file_name)
                if file_name in SKIP_FILES:
                    continue
                file_path = os.path.join(dirname, file_name)
                self.crawl_file(collection_id, file_path, meta)

def extract_address(ext, prefix, query):
    if query is None:
        return {}
    data = {
        prefix + '_official_name': ext.text(query+'OFFICIALNAME'),
        prefix + '_address': ext.text(query+'ADDRESS'),
        prefix + '_town': ext.text(query+'TOWN'),
        prefix + '_postal_code': ext.text(query+'POSTAL_CODE'),
        prefix + '_country': ext.attr(query+'COUNTRY', 'VALUE'),
        prefix + '_attention': ext.text(query+'ATTENTION'),
        prefix + '_phone': ext.text(query+'PHONE'),
        prefix + '_email': ext.text(query+'EMAIL') or ext.text(query+'E_MAIL'),
        prefix + '_fax': ext.text(query+'FAX'),
        prefix + '_url': ext.text(query+'URL_GENERAL') or ext.text(query+'URL'),
        prefix + '_url_buyer': ext.text(query+'URL_BUYER'),
        prefix + '_url_info': ext.text(query+'URL_INFORMATION'),
        prefix + '_url_participate': ext.text(query+'URL_PARTICIPATE')
    }

    if data[prefix + '_official_name'] is not None:
        data[prefix + '_slug'] = slugify(data[prefix + '_official_name'])

    for k, v in data.items():
        if v is None:
            del data[k]
    return data

def parse_file(path):
    with open(path, 'rb') as fh:
        ctx = json.load(fh)

    if ctx['source_name'] not in ['TZ']:
        return

    for layer in ctx.get('layers'):
        out = {
            "type": "FeatureCollection",
            "features": []
        }

        for fdata in layer.pop('data').get('features'):
            attrs = get_attrs(fdata)
            if not fdata.get('geometry', {}).get('rings'):
                continue

            props = dict(attrs)
            props['layer'] = layer.get('name')
            feature = {
                'type': 'Feature',
                'geometry': {
                    'type': 'Polygon',
                    'coordinates': fdata.get('geometry', {}).get('rings')
                },
                'properties': props
            }
            out['features'].append(feature)

        name = slugify('%s %s' % (ctx['source_name'], layer.get('name')),
                       sep='_')
        name = name + '.json'
        with open(os.path.join(DEST_PATH, name), 'wb') as fh:
            json.dump(out, fh)

    def by_country(self, country):
        country_slug = slugify(country)
        for rig_data in self.rigs_data:
            if slugify(rig_data.get('country')) != country_slug:
                continue
            rig_slug = self.make_entity(rig_data['name'], 'rig', raw=rig_data)
            rig = self.entities[('rig', rig_slug)]

            for role in ['owner', 'operator', 'manager']:
                rig[role] = self.make_entity(rig_data.get(role), 'company')

            rig['flag'] = self.make_entity(rig_data.get('flag'), 'rflag')
            rig['location'] = self.make_entity(rig_data.get('country'),
                                               'location')

        return {'entities': self.entities.values()}

def store_layer_to_db(data, layer, features):
    """Load a layer of features into a database table."""
    # table names are generated from the name of the layer and
    # the name of the country.
    tbl_name = '%s %s' % (data['name'], layer['name'])
    tbl_name = slugify(tbl_name, sep='_')
    log.info('    -> %s: %s rows', tbl_name, len(features))
    tbl = database[tbl_name]
    # clear out all existing data.
    tbl.delete()
    rows = []
    types = {}
    for feature in features:
        row = convrow(feature['attributes'])
        for k, v in row.items():
            if isinstance(v, (int, long)):
                types[k] = BigInteger
        row['layer_name'] = layer['name']
        row['layer_id'] = layer['id']
        row['source_name'] = data['name']
        row['source_title'] = data['title']
        row['source_url'] = data['url']
        if QUERY['returnGeometry'] == 'true':
            # store the geometry as JSON. not sure this is a
            # great idea because it may make the resulting
            # CSV files really hard to parse.
            row['_geometry'] = json.dumps(feature['geometry'])
            row['_attributes'] = json.dumps(feature['attributes'])
        rows.append(row)
    tbl.insert_many(rows, types=types)

    # Dump the table to a CSV file
    csv_file = '%s.csv' % tbl_name
    log.info('    -> %s', csv_file)
    dataset.freeze(tbl, prefix=DATA_PATH, filename=csv_file, format='csv')

def store_layer_to_geojson(data, layer, features):
    """Store the returned data as a GeoJSON file."""
    # skip if we're not loading geometries:
    if QUERY['returnGeometry'] != 'true':
        return

    out = {
        "type": "FeatureCollection",
        "features": []
    }
    for fdata in features:
        attrs = {}
        for k, v in fdata.get('attributes').items():
            k = k.lower().strip()
            attrs[k] = v

        if not fdata.get('geometry', {}).get('rings'):
            continue

        props = dict(attrs)
        props['layer'] = layer.get('name')
        out['features'].append({
            'type': 'Feature',
            'geometry': {
                'type': 'Polygon',
                'coordinates': fdata.get('geometry', {}).get('rings')
            },
            'properties': props
        })

    name = slugify('%s %s' % (data['name'], layer.get('name')), sep='_')
    name = name + '.geojson'
    log.info('    -> %s', name)
    with open(os.path.join(DATA_PATH, name), 'wb') as fh:
        json.dump(out, fh)

def parse_file(path):
    with open(path, 'rb') as fh:
        ctx = json.load(fh)

    #if ctx['source_name'] not in ['MZ']:
    #    return

    layers = ctx.pop('layers')
    for layer in layers:
        lctx = ctx.copy()
        lctx['layer_name'] = layer['name']
        lctx['layer_id'] = layer['id']
        lctx.pop('rest_url', None)

        tbl_name = slugify('%(source_name)s %(layer_name)s' % lctx, sep='_')
        tbl = database[tbl_name]
        tbl.delete()

        features = layer['data']['features']
        print ' -> Generating:', tbl_name
        print '    ', layer['name'], layer['id'], len(features)

        for feature in features:
            attrs = convrow(feature.get('attributes'))
            attrs.update(lctx)
            tbl.insert(attrs)

        dataset.freeze(tbl, prefix=DEST_PATH, filename='%s.csv' % tbl_name, format='csv')

def slugify(mapping, bind, values):
    """ Transform all values into URL-capable slugs. """
    for value in values:
        if isinstance(value, six.string_types):
            value = transliterate(value)
            value = normality.slugify(value)
        yield value

def make_filename(file_name, sep='-'):
    if file_name is not None:
        file_name = os.path.basename(file_name)
        slugs = [slugify(s, sep=sep) for s in file_name.rsplit('.', 1)]
        slugs = [s[:200] for s in slugs if s is not None]
        file_name = '.'.join(slugs)
        file_name = file_name.strip('.').strip(sep)
    return file_name

def load_countries():
    if len(COUNTRIES):
        return COUNTRIES
    with open(os.path.join(DATA_FIXTURES, 'countries.csv'), 'r') as fh:
        for row in unicodecsv.DictReader(fh):
            name = slugify(row['name'], sep=' ').strip()
            code = row['code'].strip().upper()
            REQUESTED.append({'name': row['name'], 'code': code})
            COUNTRIES[name] = code
    return COUNTRIES

def convert_row(row):
    out = {}
    for field, value in row.items():
        field = slugify(field, sep='_')
        value = value.strip()
        # TODO handle excel dates etc.
        if not len(value):
            continue
        out[field] = value
    return out

def create_fixtures():
    if 'engine' not in SHARED:
        conn = dataset.connect('sqlite://')
        for table in ['companies', 'financials']:
            with open(os.path.join(FIXTURE_PATH, table + '.csv'), 'r') as fh:
                for row in unicodecsv.DictReader(fh):
                    data = {slugify(k, sep='_'): v for k, v in row.items()}
                    conn[table].insert(data)
        SHARED['engine'] = conn.engine
    return SHARED['engine']

 def add_codelist(codelist_name, codelist_data):
     codelist = models.Codelist()
     codelist.code = normality.slugify(codelist_name)
     codelist.name = codelist_name
     db.session.add(codelist)
     db.session.commit()
     
     for codelist_code in codelist_data:
         add_codelist_data(codelist_name, codelist_code)
     db.session.commit()

def column_alias(cell, names):
    """ Generate a normalized version of the column name. """
    column = slugify(cell.column or '', sep='_')
    column = column.strip('_')
    column = 'column' if not len(column) else column
    name, i = column, 2
    # de-dupe: column, column_2, column_3, ...
    while name in names:
        name = '%s_%s' % (name, i)
        i += 1
    return name

    def to_proxy(self):
        meta = dict(self.meta)
        headers = meta.pop('headers', {})
        headers = {slugify(k, sep='_'): v for k, v in headers.items()}
        proxy = model.get_proxy({
            'id': str(self.id),
            'schema': self.model,
            'properties': meta
        })
        proxy.set('contentHash', self.content_hash)
        proxy.set('parent', self.parent_id)
        proxy.set('ancestors', self.ancestors)
        proxy.set('processingStatus', self.status)
        proxy.set('processingError', self.error_message)
        proxy.set('fileSize', meta.get('file_size'))
        proxy.set('fileName', meta.get('file_name'))
        if not proxy.has('fileName'):
            disposition = headers.get('content_disposition')
            if disposition is not None:
                _, attrs = cgi.parse_header(disposition)
                proxy.set('fileName', attrs.get('filename'))
        proxy.set('mimeType', meta.get('mime_type'))
        if not proxy.has('mimeType'):
            proxy.set('mimeType', headers.get('content_type'))
        proxy.set('language', meta.get('languages'))
        proxy.set('country', meta.get('countries'))
        proxy.set('authoredAt', meta.get('authored_at'))
        proxy.set('modifiedAt', meta.get('modified_at'))
        proxy.set('publishedAt', meta.get('published_at'))
        proxy.set('retrievedAt', meta.get('retrieved_at'))
        proxy.set('sourceUrl', meta.get('source_url'))
        proxy.set('messageId', meta.get('message_id'), quiet=True)
        proxy.set('inReplyTo', meta.get('in_reply_to'), quiet=True)
        proxy.set('bodyText', self.body_text, quiet=True)
        proxy.set('bodyHtml', self.body_raw, quiet=True)
        columns = meta.get('columns')
        proxy.set('columns', registry.json.pack(columns), quiet=True)
        proxy.set('headers', registry.json.pack(headers), quiet=True)

        pdf = 'application/pdf'
        if meta.get('extension') == 'pdf' or proxy.first('mimeType') == pdf:
            proxy.set('pdfHash', self.content_hash, quiet=True)
        proxy.add('pdfHash', meta.get('pdf_version'), quiet=True)

        q = db.session.query(DocumentTag)
        q = q.filter(DocumentTag.document_id == self.id)
        q = q.filter(DocumentTag.type.in_(DocumentTag.MAPPING.keys()))
        q = q.order_by(DocumentTag.weight.desc())
        q = q.limit(Document.MAX_TAGS)
        for tag in q.all():
            prop = DocumentTag.MAPPING.get(tag.type)
            if prop is not None:
                proxy.add(prop, tag.text)
        return proxy

 def add_column(self, label):
     column = slugify(label or "", sep="_")[:55]
     column = column or "column"
     name, i = column, 2
     # de-dupe: column, column_2, column_3, ...
     while name in [c.name for c in self.columns]:
         name = "%s_%s" % (name, i)
         i += 1
     column = {"label": label, "name": column}
     self.schema["columns"].append(column)
     return TabularColumn(self, column)

 def __init__(self, context, parent, node):
     self.context = context
     self.parent = parent
     self.node = node
     self._results = []
     if node.name is None:
         self.id = 'root'
     else:
         prefix = '_any' if node.name == '*' else node.name
         id = '%s_%s' % (prefix, uuid4().hex[:5])
         self.id = slugify(id, '_')
     self.var = v[self.id]