async function dataExport() {
    await db.connect()

    const tmpFile = "/tmp/owid_chartdata.sql"

    const variablesToExportQuery = `
        SELECT DISTINCT cd.variableId FROM chart_dimensions cd
        WHERE NOT EXISTS (select * from tags t join chart_tags ct on ct.tagId = t.id where ct.chartId=cd.chartId and t.name='Private')
    `

    const variableIds = (await db.query(variablesToExportQuery)).map((row: any) => row.variableId)

    console.log(`Exporting data for ${variableIds.length} variables to ${tmpFile}`)

    await exec(`rm -f ${tmpFile}`)

    let count = 0
    for (const chunk of _.chunk(variableIds, 100)) {
        await exec(`mysqldump --no-create-info ${DB_NAME} data_values --where="variableId IN (${chunk.join(",")})" >> ${tmpFile}`)

        count += chunk.length
        console.log(count)
    }

    await db.end()
}

async function main() {
    await db.connect()
    for (const dataset of await Dataset.find({ namespace: 'owid' })) {
        if (!dataset.isPrivate)
            await syncDatasetToGitRepo(dataset.id, { commitOnly: true })
    }
    await db.end()
}

async function main() {
    await db.connect()
    try {
//    await importCodebook()
        await importData()
    } finally {
        await db.end()
    }
}

async function dataExport() {
    await db.connect()

    const slugs = (await fs.readFile("/Users/mispy/tmp/urls.txt", "utf8")).split("\n").filter(s => s.trim())
    const slugToId = await Chart.mapSlugsToIds()
    const idsToGet = slugs.map(slug => slugToId[slug])

    const variables = await db.query("SELECT v.name, v.id FROM variables v JOIN chart_dimensions cd ON cd.variableId=v.id WHERE cd.chartId IN (?)", [idsToGet])
    const variableIds = variables.map((v: any) => v.id)
    const stream = fs.createWriteStream("/Users/mispy/tmp/sdgs.csv")

    // From dataset CSV export
    const csvHeader = ["Entity", "Year"]
    for (const variable of variables) {
        csvHeader.push(variable.name)
    }

    const columnIndexByVariableId: {[key: number]: number} = {}
    for (const variable of variables) {
        columnIndexByVariableId[variable.id] = csvHeader.indexOf(variable.name)
    }

    stream.write(csvRow(csvHeader))

    const data = await db.query(`
        SELECT e.name AS entity, dv.year, dv.value, dv.variableId FROM data_values dv
        JOIN variables v ON v.id=dv.variableId
        JOIN entities e ON dv.entityId=e.id
        WHERE v.id IN (?)
        ORDER BY e.name ASC, dv.year ASC, dv.variableId ASC`, [variableIds])

    let row: string[] = []
    for (const datum of data) {
        if (datum.entity !== row[0] || datum.year !== row[1]) {
            // New row
            if (row.length) {
                stream.write(csvRow(row))
            }
            row = [datum.entity, datum.year]
            for (const variable of variables) {
                row.push("")
            }
        }

        row[columnIndexByVariableId[datum.variableId]] = datum.value
    }

    // Final row
    stream.write(csvRow(row))

    stream.end()

    await db.end()
}

export async function prepareConsole(): Promise<any> {
    const connection = await db.connect()

    const consoleVars: {[key: string]: any} = { typeorm, db }

    // Expose all typeorm models
    for (const meta of connection.entityMetadatas) {
        consoleVars[meta.targetName] = meta.target
    }

    const r = repl.start({ prompt: 'owid> '})
    Object.assign(r.context, consoleVars)
}

async function dataExport() {
    await db.connect()

    console.log(`Exporting database structure and metadata to /tmp/owid_metadata.sql...`)

    // Dump all tables including schema but exclude the rows of data_values
    await exec(`mysqldump ${DB_NAME} --ignore-table=${DB_NAME}.sessions --ignore-table=${DB_NAME}.user_invitations --ignore-table=${DB_NAME}.data_values -r /tmp/owid_metadata.sql`)
    await exec(`mysqldump --no-data ${DB_NAME} sessions user_invitations data_values >> /tmp/owid_metadata.sql`)

    // Strip passwords
    await exec(`sed -i -e "s/bcrypt[^']*//g" /tmp/owid_metadata.sql`)

    // Add default admin user
    await fs.appendFile("/tmp/owid_metadata.sql", "INSERT INTO users (`password`, `isSuperuser`, `email`, `fullName`, `createdAt`, `updatedAt`, `isActive`) VALUES ('bcrypt$$2b$12$EXfM7cWsjlNchpinv.j6KuOwK92hihg5r3fNssty8tLCUpOubST9u', 1, '[email protected]', 'Admin User', '2016-01-01 00:00:00', '2016-01-01 00:00:00', 1);\n")

    await db.end()
}

async function dataExport() {
    await db.connect()

    const tmpFilename: string = `/tmp/owid_chartdata_${namespaces.join(",")}.sql`

    // This will also retrieve variables that are not in the specified namespace
    // but are used in a chart that has at least one variable from the specified
    // namespace.
    // This is necessary in order to reproduce the charts from the live grapher
    // accurately.
    const rows = await db.query(`
        SELECT DISTINCT chart_dimensions.variableId
        FROM chart_dimensions
        WHERE chart_dimensions.chartId IN (
            SELECT DISTINCT charts.id
            FROM charts
            JOIN chart_dimensions ON chart_dimensions.chartId = charts.id
            JOIN variables ON variables.id = chart_dimensions.variableId
            JOIN datasets ON datasets.id = variables.datasetId
            WHERE datasets.namespace IN (?)
        )
    `, [namespaces])

    const variableIds = rows.map((row: any) => row.variableId)

    console.log(`Exporting data for ${variableIds.length} variables to ${tmpFilename}`)

    await exec(`rm -f ${tmpFilename}`)

    let count = 0
    for (const chunk of _.chunk(variableIds, 100)) {
        await exec(`mysqldump --no-create-info ${DB_NAME} data_values --where="variableId IN (${chunk.join(",")})" >> ${tmpFilename}`)

        count += chunk.length
        console.log(count)
    }

    await db.end()
}