func genNewArticle(title string) {
	fmt.Printf("genNewArticle: %q\n", title)
	store, err := NewStore()
	if err != nil {
		log.Fatalf("NewStore() failed with %s", err)
	}
	newId := findUniqueArticleId(store.articles)
	name := sanitizeForFile(title) + ".md"
	fmt.Printf("new id: %d, name: %s\n", newId, name)
	t := time.Now()
	dir := "blog_posts"
	d := t.Format("2006-01")
	path := filepath.Join(dir, d, name)
	s := fmt.Sprintf(`Id: %d
Title: %s
Date: %s
Format: Markdown
--------------`, newId, title, t.Format(time.RFC3339))
	for i := 1; i < 10; i++ {
		if !u.PathExists(path) {
			break
		}
		name := sanitizeForFile(title) + "-" + strconv.Itoa(i) + ".md"
		path = filepath.Join(dir, d, name)
	}
	u.PanicIf(u.PathExists(path))
	fmt.Printf("path: %s\n", path)
	u.CreateDirForFileMust(path)
	ioutil.WriteFile(path, []byte(s), 0644)
}

func getAppEngineTmplDir() string {
	// when running locally
	d := filepath.Join("..", "tmpl")
	if u.PathExists(d) {
		return d
	}
	// when running on a server
	d = "appengtmpl"
	if u.PathExists(d) {
		return d
	}
	logger.Errorf("getAppEngineTmplDir(): %q dir doesn't exist", d)
	return ""
}

func getWwwDir() string {
	// when running locally
	d := filepath.Join("..", "www")
	if u.PathExists(d) {
		return d
	}
	// when running on a server
	d = "www"
	if u.PathExists(d) {
		return d
	}
	logger.Errorf("getWwwDir(): %q dir doesn't exist", d)
	return ""
}

func serveFileFromDir(w http.ResponseWriter, r *http.Request, dir, fileName string) {
	filePath := filepath.Join(dir, fileName)
	if !u.PathExists(filePath) {
		logger.Noticef("serveFileFromDir() file %q doesn't exist, referer: %q", fileName, getReferer(r))
	}
	http.ServeFile(w, r, filePath)
}

func NewStore(dataDir, forumName string) (*Store, error) {
	dataFilePath := filepath.Join(dataDir, "forum", forumName+".txt")
	store := &Store{
		dataDir:   dataDir,
		forumName: forumName,
		posts:     make([]*Post, 0),
		topics:    make([]Topic, 0),
	}
	var err error
	if u.PathExists(dataFilePath) {
		if err = store.readExistingData(dataFilePath); err != nil {
			fmt.Printf("readExistingData() failed with %s", err)
			return nil, err
		}
	} else {
		f, err := os.Create(dataFilePath)
		if err != nil {
			fmt.Printf("NewStore(): os.Create(%s) failed with %s", dataFilePath, err)
			return nil, err
		}
		f.Close()
	}

	verifyTopics(store.topics)

	store.dataFile, err = os.OpenFile(dataFilePath, os.O_APPEND|os.O_RDWR, 0666)
	if err != nil {
		fmt.Printf("NewStore(): os.OpenFile(%s) failed with %s", dataFilePath, err)
		return nil, err
	}
	return store, nil
}

func serveFileFromDir(w http.ResponseWriter, r *http.Request, dir, fileName string) {
	filePath := filepath.Join(dir, fileName)
	if !u.PathExists(filePath) {
		fmt.Printf("serveFileFromDir() file=%s doesn't exist\n", filePath)
	}
	http.ServeFile(w, r, filePath)
}

func NewStoreCrashes(dataDir string) (*StoreCrashes, error) {
	dataFilePath := filepath.Join(dataDir, "data", "crashesdata.txt")
	store := &StoreCrashes{
		dataDir:       dataDir,
		crashes:       make([]*Crash, 0),
		apps:          make([]*App, 0),
		versions:      make([]*string, 0),
		ips:           make(map[string]*string),
		crashingLines: make(map[string]*string),
	}

	var err error
	if u.PathExists(dataFilePath) {
		err = store.readExistingCrashesData(dataFilePath)
		if err != nil {
			logger.Errorf("NewStoreCrashes(): readExistingCrashesData() failed with %s\n", err)
			return nil, err
		}
	} else {
		f, err := os.Create(dataFilePath)
		if err != nil {
			logger.Errorf("NewStoreCrashes(): os.Create(%s) failed with %s", dataFilePath, err)
			return nil, err
		}
		f.Close()
	}
	store.dataFile, err = os.OpenFile(dataFilePath, os.O_APPEND|os.O_RDWR, 0666)
	if err != nil {
		logger.Errorf("NewStoreCrashes(): os.OpenFile(%s) failed with %s", dataFilePath, err)
		return nil, err
	}
	logger.Noticef("crashes: %d, versions: %d, ips: %d, crashing lines: %d", len(store.crashes), len(store.versions), len(store.ips), len(store.crashingLines))

	return store, nil
}

// data dir is ../../data on the server or ~/data/apptranslator locally
// the important part is that it's outside of directory with the code
func getDataDir() string {
	if dataDir != "" {
		return dataDir
	}
	// on the server, must be done first because ExpandTildeInPath()
	// doesn't work when cross-compiled on mac for linux
	dataDir = filepath.Join("..", "..", "data")
	if u.PathExists(dataDir) {
		return dataDir
	}
	dataDir = u.ExpandTildeInPath("~/data/apptranslator")
	if u.PathExists(dataDir) {
		return dataDir
	}
	log.Fatal("data directory (../../data or ../apptranslatordata) doesn't exist")
	return ""
}

// /static/$rest
func handleStatic(w http.ResponseWriter, r *http.Request) {
	fileName := r.URL.Path[len("/static/"):]
	path := filepath.Join("www", fileName)
	if u.PathExists(path) {
		LogInfof("%s\n", path)
		http.ServeFile(w, r, path)
	} else {
		LogInfof("file %q doesn't exist, referer: %q\n", path, getReferer(r))
		http.NotFound(w, r)
	}
}

func readAppData(app *App) error {
	var path string
	path = app.storeCsvFilePath()
	if u.PathExists(path) {
		if l, err := store.NewStoreCsv(path); err == nil {
			app.store = l
			return nil
		}
	}
	return fmt.Errorf("readAppData: %q data file doesn't exist", path)
}

func serveFileFromDir(w http.ResponseWriter, r *http.Request, dir, fileName string) {
	if redirectIfFoundMatching(w, r, dir, fileName) {
		return
	}
	filePath := filepath.Join(dir, fileName)
	if u.PathExists(filePath) {
		//logger.Noticef("serveFileFromDir(): %q", filePath)
		http.ServeFile(w, r, filePath)
	} else {
		logger.Noticef("serveFileFromDir() file %q doesn't exist, referer: %q", fileName, getReferer(r))
		http.NotFound(w, r)
	}
}

func removeStoreFiles(basePath string) {
	path := idxFilePath(basePath)
	os.Remove(path)
	nSegment := 0
	for {
		path = segmentFilePath(basePath, nSegment)
		if !u.PathExists(path) {
			break
		}
		os.Remove(path)
		nSegment += 1
	}
}

// tests if s3 credentials are valid and aborts if aren't
func ensureValidConfig(config *BackupConfig) {
	if !u.PathExists(config.LocalDir) {
		log.Fatalf("Invalid s3 backup: directory to backup %q doesn't exist\n", config.LocalDir)
	}

	if !strings.HasSuffix(config.S3Dir, bucketDelim) {
		config.S3Dir += bucketDelim
	}
	_, err := listBackupFiles(config, 10)
	if err != nil {
		log.Fatalf("Invalid s3 backup: bucket.List failed %s\n", err)
	}
}

func serveFile(w http.ResponseWriter, r *http.Request, fileName string) {
	//LogVerbosef("serverFile: fileName='%s'\n", fileName)
	path := filepath.Join("www", fileName)
	if hasZipResources() {
		serveResourceFromZip(w, r, path)
		return
	}
	if u.PathExists(path) {
		http.ServeFile(w, r, path)
	} else {
		LogVerbosef("file '%s' doesn't exist\n", path)
		http.NotFound(w, r)
	}
}

func getDataDir() string {
	if dataDir != "" {
		return dataDir
	}

	// on the server, must be done first because ExpandTildeInPath()
	// doesn't work when cross-compiled on mac for linux
	serverDir := filepath.Join("..", "..", "data")
	dataDir = serverDir
	if u.PathExists(dataDir) {
		return dataDir
	}

	// locally
	localDir := u.ExpandTildeInPath("~/data/blog")
	dataDir = localDir
	if u.PathExists(dataDir) {
		return dataDir
	}

	log.Fatalf("data directory (%q or %q) doesn't exist", serverDir, localDir)
	return ""
}

func NewWithLimit(basePath string, maxSegmentSize int) (store *Store, err error) {
	store = &Store{
		basePath:        basePath,
		blobs:           make([]blob, 0),
		sha1HexToBlobNo: make(map[string]int),
		maxSegmentSize:  maxSegmentSize,
		cachedSegmentNo: -1,
	}
	idxPath := idxFilePath(basePath)
	idxDidExist := u.PathExists(idxPath)
	if idxDidExist {
		if err = store.readIndex(); err != nil {
			return nil, err
		}
	}
	if store.idxFile, err = os.OpenFile(idxPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err != nil {
		return nil, err
	}
	store.idxCsvWriter = csv.NewWriter(store.idxFile)
	if !idxDidExist {
		rec := [][]string{{idxHdr}}
		err = store.idxCsvWriter.WriteAll(rec)
		if err != nil {
			return nil, err
		}
	}
	segmentPath := segmentFilePath(store.basePath, store.currSegmentNo)
	stat, err := os.Stat(segmentPath)
	if err != nil {
		// TODO: fail if error is different than "file doesn't exist"
		if store.currSegmentNo != 0 {
			store.Close()
			return nil, errSegmentFileMissing
		}
		store.currSegmentFile, err = os.Create(segmentPath)
		if err != nil {
			store.Close()
			return nil, err
		}
	} else {
		store.currSegmentSize = int(stat.Size())
		store.currSegmentFile, err = os.OpenFile(segmentPath, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0644)
		if err != nil {
			store.Close()
			return nil, err
		}
	}
	return store, nil
}

func NewForum(config *ForumConfig) *Forum {
	forum := &Forum{ForumConfig: *config}
	sidebarTmplPath := filepath.Join("forums", fmt.Sprintf("%s_sidebar.html", forum.ForumUrl))
	if !u.PathExists(sidebarTmplPath) {
		panic(fmt.Sprintf("sidebar template %s for forum %s doesn't exist", sidebarTmplPath, forum.ForumUrl))
	}

	forum.SidebarTmpl = template.Must(template.ParseFiles(sidebarTmplPath))

	store, err := NewStore(getDataDir(), config.DataDir)
	if err != nil {
		panic("failed to create store for a forum")
	}
	logger.Noticef("%d topics, %d posts in forum %q", store.TopicsCount(), store.PostsCount(), config.ForumUrl)
	forum.Store = store
	return forum
}

func (store *Store) readIndex() error {
	// at this point idx file must exist
	file, err := os.Open(idxFilePath(store.basePath))
	if err != nil {
		return err
	}
	defer file.Close()
	// TODO: would be faster (and easier?) to use a bitset since we know
	// segment numbers are consequitive integers
	segments := make([]int, 0)
	csvReader := csv.NewReader(file)
	csvReader.Comma = ','
	csvReader.FieldsPerRecord = -1
	rec, err := csvReader.Read()
	if err != nil || len(rec) != 1 || rec[0] != idxHdr {
		return errInvalidIndexHdr
	}
	var blob blob
	for {
		if rec, err = csvReader.Read(); err != nil {
			break
		}
		if blob, err = decodeIndexLine(rec); err != nil {
			break
		}
		appendIntIfNotExists(&segments, blob.nSegment)
		store.appendBlob(blob)
	}
	if err == io.EOF {
		err = nil
	}
	// verify segment files exist
	// TODO: also verify offset + size is <= size of segment file
	for _, nSegment := range segments {
		path := segmentFilePath(store.basePath, nSegment)
		if !u.PathExists(path) {
			return errSegmentFileMissing
		}
		store.currSegmentNo = nSegment
	}
	return nil
}

func NewStoreCsv(path string) (*StoreCsv, error) {
	//fmt.Printf("NewStoreCsv: %q\n", path)
	var err error
	s := &StoreCsv{
		filePath: path,
		strings:  NewStringInterner(),
		users:    NewStringInterner(),
		edits:    make([]TranslationRec, 0),
	}
	if u.PathExists(path) {
		if err = s.readExistingRecords(path); err != nil {
			return nil, err
		}
	}
	s.setActiveStrings(s.activeStrings)
	if s.file, s.w, err = openCsv(path); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *StoreCrashes) MessageFileExists(sha1 []byte) bool {
	p := s.MessageFilePath(sha1)
	return u.PathExists(p)
}