bool ESDLcompiler::locateIncludedFile(StringBuffer& filepath, const char* prot, const char* srcDir, const char* fname, const char* ext)
{
    StringBuffer alternateExtFilename;
    alternateExtFilename.setf("%s%s%s", (prot && *prot) ? prot : "", srcDir, fname);

    const char* alt_ext;
    if (stricmp(ext, LEGACY_FILE_EXTENSION)==0)
        alt_ext = ESDL_FILE_EXTENSION;
    else
        alt_ext = LEGACY_FILE_EXTENSION;
    alternateExtFilename.append(alt_ext);

    OwnedIFile fileInSrcDir = createIFile(alternateExtFilename.str());
    if (fileInSrcDir->exists())
    {
        filepath.set(alternateExtFilename.str());
        return true;
    }

    ForEachItemIn(x, includeDirs)
    {
        const char* dir = includeDirs.item(x);
        if (dir && *dir)
        {
            StringBuffer pathInInclude(dir);
            pathInInclude.trim();
            if (pathInInclude.charAt(pathInInclude.length() - 1) != PATHSEPCHAR)
                pathInInclude.append(PATHSEPCHAR);
            pathInInclude.append(fname);
            VStringBuffer pathInIncludeFull("%s%s", pathInInclude.str(), ext);
            OwnedIFile fileInInclude = createIFile(pathInIncludeFull.str());
            if (fileInInclude->exists())
            {
                filepath.set(pathInIncludeFull.str());
                return true;
            }
            pathInIncludeFull.setf("%s%s", pathInInclude.str(), alt_ext);
            OwnedIFile altFileInInclude = createIFile(pathInIncludeFull.str());
            if (altFileInInclude->exists())
            {
                filepath.set(pathInIncludeFull.str());
                return true;
            }
        }
    }

    filepath.clear();
    return false;
}

StringBuffer& CLocalDataLogger::writeData(const StringBuffer& dataToCache,const StringBuffer& tokenName,StringBuffer& returnPath)
{
    DBGLOG("CLocalDataLogger::writeData");
    StringBuffer tmpFile;
    getFilePath(tokenName.str(),tmpFile);
    Owned<IFile> file = createIFile(tmpFile.str());
    for (int i = 0; i < RETRIES; ++i)
    {
        try{
            Owned<IFileIO> io = file->open(IFOwrite);
            size32_t filesize = dataToCache.length();
            void* filedata = (void*)dataToCache.str();
            io->write(0,filesize ,filedata );
            if(m_UrlRoot.length()==0)
                returnPath.appendf("/Cache?Name=%s.HTML",tokenName.str());
            else
                returnPath.appendf("/%s/Cache?Name=%s.HTML",m_UrlRoot.str(),tokenName.str());
            break;
        }
        catch(IOSException *ose)
        {
            //The web site could be serving up the page as we try to update it...
            ose->Release();
            Sleep(10);
        }
        catch(...)
        {
            DBGLOG("Unknown exception thrown while reading local data logger file");
        }
    }
    
    return returnPath;
}

bool CTransformerBase::setPartition(RemoteFilename & remoteInputName, offset_t _startOffset, offset_t _length)
{
    inputFile.setown(createIFile(remoteInputName));
    startOffset = _startOffset;
    maxOffset = _startOffset + _length;
    return true;
}

int main(int argc, char* argv[])
{
    InitModuleObjects();
    EnableSEHtoExceptionMapping();

    if (argc<6)
    {
        usage();
        return 0;
    }

    try
    {
        const char *filename = argv[1];
        Owned<IDaliCapabilityCreator> cc = createDaliCapabilityCreator();
        cc->setSystemID(argv[2]);
        cc->setServerPassword(argv[3]);
        for (unsigned i=4;i<argc;i++) {
            const char *cmd = argv[i++];
            if (i==argc)
                break;
            const char *param = argv[i];
            if (stricmp(cmd,"THORWIDTH")==0) {
                cc->setLimit(DCR_ThorSlave,atoi(param));
            }
            else if (stricmp(cmd,"DALINODE")==0) {
                StringBuffer mac;
                if (strchr(param,'.')) { // must be ip
                    IpAddress ip;
                    ip.set(param);
                    if (!getMAC(ip,mac)) {
                        printf("ERROR: could mot get MAC address for %s\n",param);
                        return 1;
                    }
                }
                else
                    mac.append(param);
                cc->addCapability(DCR_DaliServer,mac.str());
            }
            else {
                printf("ERROR: unknown command %s\n",cmd);
                return 1;
            }
        }
        StringBuffer results;
        cc->save(results);
        Owned<IFile> ifile = createIFile(filename);
        Owned<IFileIO> ifileio = ifile->open(IFOcreate);
        ifileio->write(0,results.length(),results.str());
        printf("Dali Capabilities sucessfully exported to %s\n", filename);
    }
    catch (IException *e)
    {
        EXCLOG(e);
        e->Release();
    }

    releaseAtoms();
    return 0;
}

void printKeywordsToXml()
{
     StringBuffer buffer;
     unsigned int nGroups = sizeof(keywordList)/sizeof(keywordList[0]);

     buffer.append("<xml>\n");
     for (unsigned i = 0; i < nGroups; ++i)
     {
         buffer.append("  <cat group=\"").append(keywordList[i].group).append("\">\n");
         unsigned int j = 0;
         while(keywordList[i].keywords[j])
         {
             buffer.append("    <keyword name=\"").append(keywordList[i].keywords[j]).append("\"/>\n");
             ++j;
         }
         buffer.append("  </cat>\n");
     }
     buffer.append("</xml>\n");

     Owned<IFile> treeFile = createIFile("ECLKeywords.xml");
     assertex(treeFile);
     Owned<IFileIO> io = treeFile->open(IFOcreaterw);
     assertex(io);
     Owned<IIOStream> out = createIOStream(io);
     assertex(out);
     out->write(buffer.length(), buffer.str());
}

void SafePluginMap::loadFromList(const char * pluginsList)
{
    const char *pluginDir = pluginsList;
    for (;*pluginDir;)
    {
        StringBuffer thisPlugin;
        while (*pluginDir && *pluginDir != ENVSEPCHAR)
            thisPlugin.append(*pluginDir++);
        if(*pluginDir)
            pluginDir++;

        if(!thisPlugin.length())
            continue;

        Owned<IFile> file = createIFile(thisPlugin.str());
        if (file->isDirectory() == foundYes)
            loadFromDirectory(thisPlugin);
        else
        {
            StringBuffer tail;
            splitFilename(thisPlugin, NULL, NULL, &tail, &tail);
            addPlugin(thisPlugin, tail.str());
        }
    }
}

void CConfigGenEngine::createFakePlugins(StringBuffer& destFilePath) const
{
    String destFilePathStr(destFilePath);
    String* tmpstr = destFilePathStr.toLowerCase();
    if (!tmpstr->endsWith("plugins.xml"))
    {
        int index = tmpstr->indexOf("plugins.xml");
        destFilePath.remove(index + 11, destFilePath.length() - (index + 11));
    }

    delete tmpstr;

    StringBuffer tmpoutbuf("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Plugins/>");

    if (m_instances.ordinality() > 1 && strcmp(m_process.queryName(), XML_TAG_ESPPROCESS))
        destFilePath.replaceString("@temp"PATHSEPSTR, m_cachePath);
    else
    {
        char tempPath[_MAX_PATH];
        getTempPath(tempPath, sizeof(tempPath), m_name);
        ensurePath(tempPath);
        destFilePath.replaceString("@temp"PATHSEPSTR, tempPath);
    }

    Owned<IFile> pTargetFile = createIFile(destFilePath.str());
    if (pTargetFile->exists() && pTargetFile->isReadOnly())
        pTargetFile->setReadOnly(false);
    Owned<IFileIO> pTargetFileIO = pTargetFile->open(IFOcreate);
    pTargetFileIO->write( 0, tmpoutbuf.length(), tmpoutbuf.str());
    m_envDepEngine.addTempFile(destFilePath.str());
}

void CLogSerializer::Open(const char*Directory,const char* NewFileName,const char* Prefix)
{
    m_FilePath.clear();
    m_FilePath.append(Directory);
    if (!EnsureDirectory(m_FilePath))
        throw MakeStringException(-1,"Unable to create directory at %s.",m_FilePath.str());


    m_FilePath.append("/");

    m_FileName.clear();
    m_FileName.append(Prefix);
    m_FileName.append("_");
    m_FileName.append(NewFileName);

    m_FilePath.append(m_FileName);
    
    DBGLOG("Creating tank file %s", m_FilePath.str());
    m_file = createIFile(m_FilePath.str());
    m_fileio  =  m_file->open(IFOcreate);
    if (m_fileio == 0)
        throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());
    else
        DBGLOG("Tank file %s successfully created", m_FilePath.str());

}

//cloned from hthor - a candidate for commoning up.
static IKeyIndex *openKeyFile(IDistributedFilePart *keyFile)
{
    unsigned numCopies = keyFile->numCopies();
    assertex(numCopies);
    for (unsigned copy=0; copy < numCopies; copy++)
    {
        RemoteFilename rfn;
        try
        {
            OwnedIFile ifile = createIFile(keyFile->getFilename(rfn,copy));
            unsigned __int64 thissize = ifile->size();
            if (thissize != -1)
            {
                StringBuffer remotePath;
                rfn.getRemotePath(remotePath);
                unsigned crc = 0;
                keyFile->getCrc(crc);
                return createKeyIndex(remotePath.str(), crc, false, false);
            }
        }
        catch (IException *E)
        {
            EXCLOG(E, "While opening index file");
            E->Release();
        }
    }
    RemoteFilename rfn;
    StringBuffer url;
    keyFile->getFilename(rfn).getRemotePath(url);
    throw MakeStringException(1001, "Could not open key file at %s%s", url.str(), (numCopies > 1) ? " or any alternate location." : ".");
}

            void Do(unsigned i)
            {
                CriticalBlock block(crit);
                if (parent->stopped)
                    return;
                CFileCrcItem &item = parent->list.item(i);
                RemoteFilename &rfn = item.filename;
                Owned<IFile> partfile;
                StringBuffer eps;
                try
                {
                    partfile.setown(createIFile(rfn));
                    // PROGLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
                    if (partfile) {
                        if (parent->stopped)
                            return;
                        CriticalUnblock unblock(crit);
                        item.crc = partfile->getCRC();
                    }
                    else
                        ok = false;

                }
                catch (IException *e)
                {
                    StringBuffer s;
                    s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
                    EXCLOG(e, s.str());
                    e->Release();
                    ok = false;
                }
            }

void CLogSerializer::Remove()
{
    Close();
    Owned<IFile> file = createIFile(m_FilePath.str()); 
    if(file.get() && file->exists() == true)
        file->remove();
}

    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
    {
        isLocal = 0 != (TIWlocal & helper->getFlags());

        mpTag = container.queryJobChannel().deserializeMPTag(data);
        mpTag2 = container.queryJobChannel().deserializeMPTag(data);
        data.read(active);
        if (active)
        {
            data.read(logicalFilename);
            partDesc.setown(deserializePartFileDescriptor(data));
        }

        data.read(singlePartKey);
        data.read(refactor);
        if (singlePartKey)
            buildTlk = false;
        else
        {
            data.read(buildTlk);
            if (firstNode())
            {
                if (buildTlk)
                    tlkDesc.setown(deserializePartFileDescriptor(data));
                else if (!isLocal) // existing tlk then..
                {
                    tlkDesc.setown(deserializePartFileDescriptor(data));
                    unsigned c;
                    data.read(c);
                    while (c--)
                    {
                        RemoteFilename rf;
                        rf.deserialize(data);
                        if (!existingTlkIFile)
                        {
                            Owned<IFile> iFile = createIFile(rf);
                            if (iFile->exists())
                                existingTlkIFile.set(iFile);
                        }
                    }
                    if (!existingTlkIFile)
                        throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key");
                }
            }
        }

        IOutputMetaData * diskSize = helper->queryDiskRecordSize();
        assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk));
        if (diskSize->isVariableSize())
        {
            if (TIWmaxlength & helper->getFlags())
                maxDiskRecordSize = helper->getMaxKeySize();
            else
                maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future
        }
        else
            maxDiskRecordSize = diskSize->getFixedSize();
        reportOverflow = false;
    }

void CDiskWriteSlaveActivityBase::removeFiles()
{
    if (!fName.length())
        return;
    Owned<IFile> primary = createIFile(fName);
    try { primary->remove(); }
    catch (IException *e) { ActPrintLogEx(&queryContainer(), e, thorlog_null, MCwarning, "Failed to remove file: %s", fName.get()); }
    catch (CATCHALL) { ActPrintLogEx(&queryContainer(), thorlog_null, MCwarning, "Failed to remove: %s", fName.get()); }
}

 void removeFiles(IPartDescriptor &partDesc)
 {
     StringBuffer partFname;
     getPartFilename(partDesc, 0, partFname);
     Owned<IFile> primary = createIFile(partFname.str());
     try { primary->remove(); }
     catch (IException *e) { ActPrintLog(e, "Failed to remove file: %s", partFname.str()); e->Release(); }
     catch (CATCHALL) { ActPrintLog("Failed to remove: %s", partFname.str()); }
 }

void CRecieveLogSerializer::LoadDataMap(GuidMap& GUIDmap)
{
    DBGLOG("Loading ACKMap");
    try{
        m_file = createIFile(m_FilePath.str());
        m_fileio = m_file->open(IFOread);
        if (m_fileio == 0)
            throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());
        else
            DBGLOG("File %s successfully opened", m_FilePath.str());

        long finger,bytesRead,dataLen;
        finger = bytesRead = dataLen = 0;

        MemoryBuffer filecontents,dataSize,data;

        bool bOk = true;
        unsigned int total = 0;
        while(bOk)
        {
            bytesRead = m_fileio->read(finger,8,dataSize.reserveTruncate(8));
            if(bytesRead==0)
                break;

            finger+=9;
            dataLen = atoi(dataSize.toByteArray());

            bytesRead = m_fileio->read(finger,dataLen,data.reserveTruncate(dataLen));
            if(bytesRead==0)
                break;
            bool printTrace = false;
            if(total % TRACE_INTERVAL == 0)
            {
                DBGLOG("Loading ack #%u", total);
                printTrace = true;
            }
            LoadMap(data,GUIDmap,printTrace);
            total++;

            finger+=dataLen;
            data.clear();
            dataSize.clear();
        }
        DBGLOG("Total acks loaded %u", total);
    }
    catch(IException* ex)
    {
        StringBuffer errorStr;
        ex->errorMessage(errorStr);
        ERRLOG("Exception caught within CRecieveLogSerializer::LoadDataMap: %s",errorStr.str());
        ex->Release();
    }
    catch(...){
        DBGLOG("Unknown Exception thrown in CRecieveLogSerializer::LoadDataMap");
    }
    Close();
}

    void open()
    {
        char drive       [_MAX_DRIVE];
        char dir         [_MAX_DIR];
        char fname       [_MAX_DIR];
        char ext         [_MAX_EXT];
        _splitpath(fileName.str(), drive, dir, fname, ext);

        StringBuffer directory;
        directory.append(drive).append(dir);

        Owned<IFile> cd = createIFile(directory.str());
        cd->createDirectory();

        IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper();
        void *ekey;
        size32_t ekeylen;
        helper->getEncryptKey(ekeylen,ekey);
        Owned<ICompressor> ecomp;
        if (ekeylen!=0)
        {
            ecomp.setown(createAESCompressor256(ekeylen,ekey));
            memset(ekey,0,ekeylen);
            free(ekey);
            compress = true;
        }
        Owned<IFile> file = createIFile(fileName.str());
        Owned<IFileIO> iFileIO;
        bool fixedRecordSize = queryRowMetaData()->isFixedSize();
        size32_t minrecsize = queryRowMetaData()->getMinRecordSize();

        if (fixedRecordSize)
            ActPrintLog("SPILL: created fixed output %s recsize=%u", (0!=ekeylen)?"[encrypted]":compress?"[compressed]":"",minrecsize);
        else
            ActPrintLog("SPILL: created variable output %s, minrecsize=%u", (0!=ekeylen)?"[encrypted]":compress?"[compressed]":"",minrecsize);
        unsigned rwFlags = (DEFAULT_RWFLAGS & ~rw_autoflush); // flushed by close()
        if (compress)
            rwFlags |= rw_compress;
        else
            rwFlags |= rw_crc; // only if !compress
        if (grouped)
            rwFlags |= rw_grouped;
        out.setown(createRowWriter(file, this, rwFlags));
    }

    offset_t write(IRowStream *input)
    {
        StringBuffer tempname;
        GetTempName(tempname,"srtmrg",false);
        dataFile.setown(createIFile(tempname.str()));
        Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf);

        bool overflowed = false;
        ActPrintLog(&activity, "Local Overflow Merge start");
        unsigned ret=0;
        loop
        {
            const void *_row = input->nextRow();
            if (!_row)
                break;
            ret++;

            OwnedConstThorRow row = _row;
            offset_t start = output->getPosition();
            output->putRow(row.getLink());
            idx++;
            if (idx==interval)
            {
                idx = 0;
                if (!sampleRows.append(row.getClear()))
                {
                    // JCSMORE used to check if 'isFull()' here, but only to warn
                    // I think this is bad news, if has run out of room here...
                    // should at least warn in workunit I suspect
                    overflowsize = output->getPosition();
                    if (!overflowed)
                    {
                        WARNLOG("Sample buffer full");
                        overflowed = true;
                    }
                }
            }
            writeidxofs(start);
        }
        output->flush();
        offset_t end = output->getPosition();
        output.clear();
        writeidxofs(end);
        if (idxFileIO)
        {
            idxFileStream->flush();
            idxFileStream.clear();
            idxFileIO.clear();
        }
        if (overflowed)
            WARNLOG("Overflowed by %"I64F"d", overflowsize);
        ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size());
        return end;
    }

void CLogSerializer::Rollover(const char* ClosedPrefix)
{
    Close();
    Owned<IFile> file = createIFile(m_FilePath.str());
    if(file.get() && file->exists() == true)
    {
        StringBuffer newFileName;
        GetRolloverFileName(m_FileName,newFileName,ClosedPrefix);
        file->rename(newFileName.str());
    }
}

static bool getFileInfo(RemoteFilename &fn, Owned<IFile> &f, offset_t &size,CDateTime &modtime)
{
    f.setown(createIFile(fn));
    bool isdir = false;
    bool ret = f->getInfo(isdir,size,modtime);
    if (ret&&isdir) {
        StringBuffer fs;
        fn.getRemotePath(fs);
        throw MakeStringException(-1,"%s is a directory",fs.str());
    }
    return ret;
}

void CppCompiler::writeLogFile(const char* filepath, StringBuffer& log)
{
    if(!filepath || !*filepath || !log.length())
        return;

    Owned <IFile> f = createIFile(filepath);
    if(f->exists())
        f->remove();

    Owned <IFileIO> fio = f->open(IFOcreaterw);
    if(fio.get())
        fio->write(0, log.length(), log.str());
}