c++ - CreateProcess cmd.exe read/write pipes deadlock

Question

Hello I am trying to make a front end GUI for cmd.exe so I can make it wider but I got stuck.

I try to design an API like this

char* Directory = WriteCommand("dir");
printf("- %s
", Directory);

and the output look exactly like it would in a cmd window, except I have it in a string, so it would be

DATE TIME FILESIZE FILENAME
etc etc etc

and then I can issue

char* Up = WriteCommand ("cd ..");

and it will give me the above directory listing. So I want a terminal control through using pipes to read and write.

I have tried many things based on this MSDN sample code - https://msdn.microsoft.com/en-us/library/ms682499.aspx

But I think this code is only good to issue one command, and read one response, because right after it deadlocks as described here - https://blogs.msdn.microsoft.com/oldnewthing/20110707-00/?p=10223

I see several other questions here, like this one with similar problems - How to read output from cmd.exe using CreateProcess() and CreatePipe() but no solutions posted work for me.

So here is my code.

#include <windows.h> 
#include <tchar.h>
#include <stdio.h> 
#include <strsafe.h>

#define BUFSIZE 4096 

HANDLE g_hChildStd_IN_Rd = NULL;
HANDLE g_hChildStd_IN_Wr = NULL;
HANDLE g_hChildStd_OUT_Rd = NULL;
HANDLE g_hChildStd_OUT_Wr = NULL;

HANDLE g_hInputFile = NULL;

void CreateChildProcess(void);
void WriteToPipe(char* Arg1);
void ReadFromPipe(void);
void ErrorExit(PTSTR);



int _tmain(int argc, TCHAR *argv[])
{
    SECURITY_ATTRIBUTES saAttr;

    printf("
->Start of parent execution.
");

    // Set the bInheritHandle flag so pipe handles are inherited. 

    saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
    saAttr.bInheritHandle = TRUE;
    saAttr.lpSecurityDescriptor = NULL;

    // Create a pipe for the child process's STDOUT. 

    if (!CreatePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr, 0))
        ErrorExit(TEXT("StdoutRd CreatePipe"));

    // Ensure the read handle to the pipe for STDOUT is not inherited.

    if (!SetHandleInformation(g_hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0))
        ErrorExit(TEXT("Stdout SetHandleInformation"));

    // Create a pipe for the child process's STDIN. 

    if (!CreatePipe(&g_hChildStd_IN_Rd, &g_hChildStd_IN_Wr, &saAttr, 0))
        ErrorExit(TEXT("Stdin CreatePipe"));

    // Ensure the write handle to the pipe for STDIN is not inherited. 

    if (!SetHandleInformation(g_hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0))
    ErrorExit(TEXT("Stdin SetHandleInformation"));

    // Create the child process. 

    CreateChildProcess();

    // Get a handle to an input file for the parent. 
    // This example assumes a plain text file and uses string output to verify data flow. 

/*if (argc == 1)
    ErrorExit(TEXT("Please specify an input file.
"));

g_hInputFile = CreateFile(
    argv[1],
    GENERIC_READ,
    0,
    NULL,
    OPEN_EXISTING,
    FILE_ATTRIBUTE_READONLY,
    NULL);

if (g_hInputFile == INVALID_HANDLE_VALUE)
    ErrorExit(TEXT("CreateFile"));*/

    // Write to the pipe that is the standard input for a child process. 
    // Data is written to the pipe's buffers, so it is not necessary to wait
    // until the child process is running before writing data.



// Read from pipe that is the standard output for child process. 


ReadFromPipe();

WriteToPipe("ipconfig");

// THIS IS WHERE DEADLOCK OCCURS, FROM HERE
// PROGRAM BECOMES UNRESPONSIVE - HOW TO FIX THIS?

ReadFromPipe();



printf("
->End of parent execution.
");

// The remaining open handles are cleaned up when this process terminates. 
// To avoid resource leaks in a larger application, close handles explicitly. 

return 0;
}

void CreateChildProcess()
// Create a child process that uses the previously created pipes for     STDIN and STDOUT.
{
   TCHAR szCmdline[] = TEXT("cmd.exe /k");
    PROCESS_INFORMATION piProcInfo;
   STARTUPINFO siStartInfo;
   BOOL bSuccess = FALSE;

    // Set up members of the PROCESS_INFORMATION structure. 

    ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION));

   // Set up members of the STARTUPINFO structure. 
  // This structure specifies the STDIN and STDOUT handles for redirection.

    ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
   siStartInfo.cb = sizeof(STARTUPINFO);
    siStartInfo.hStdError = g_hChildStd_OUT_Wr;
    siStartInfo.hStdOutput = g_hChildStd_OUT_Wr;
    siStartInfo.hStdInput = g_hChildStd_IN_Rd;
    siStartInfo.dwFlags |= STARTF_USESTDHANDLES;

// Create the child process. 

bSuccess = CreateProcess(NULL,
    "cmd.exe",     // command line 
    NULL,          // process security attributes 
    NULL,          // primary thread security attributes 
    TRUE,          // handles are inherited 
    0,             // creation flags 
    NULL,          // use parent's environment 
    NULL,          // use parent's current directory 
    &siStartInfo,  // STARTUPINFO pointer 
    &piProcInfo);  // receives PROCESS_INFORMATION 

                   // If an error occurs, exit the application. 
if (!bSuccess)
    ErrorExit(TEXT("CreateProcess"));
else
{
    // Close handles to the child process and its primary thread.
    // Some applications might keep these handles to monitor the status
    // of the child process, for example. 

    CloseHandle(piProcInfo.hProcess);
    CloseHandle(piProcInfo.hThread);
}
}

void WriteToPipe(char* Command)

// Read from a file and write its contents to the pipe for the    child's STDIN.
// Stop when there is no more data. 
   {
   DWORD dwRead, dwWritten;
    CHAR chBuf[BUFSIZE];
    BOOL bSuccess = FALSE;

    bSuccess = WriteFile(g_hChildStd_IN_Wr, Command, strlen(Command), &dwWritten, NULL);
    if (bSuccess == FALSE)
        printf("write fail
");

    printf("written = %i
", dwWritten);


//for (;;)
//{
    //bSuccess = ReadFile(g_hInputFile, chBuf, BUFSIZE, &dwRead, NULL);
    //if (!bSuccess || dwRead == 0) break;

    //bSuccess = WriteFile(g_hChildStd_IN_Wr, Command, strlen(Command), &dwWritten, NULL);
    //if (bSuccess == FALSE)
        //printf("write fail
");

    //printf("written = %i
", dwWritten);
//}

// Close the pipe handle so the child process stops reading. 

//if (!CloseHandle(g_hChildStd_IN_Wr))
    //ErrorExit(TEXT("StdInWr CloseHandle"));
}

void ReadFromPipe(void)

// Read output from the child process's pipe for STDOUT
// and write to the parent process's pipe for STDOUT. 
// Stop when there is no more data. 
{
DWORD dwRead, dwWritten;
CHAR chBuf[BUFSIZE];
BOOL bSuccess = FALSE;
HANDLE hParentStdOut = GetStdHandle(STD_OUTPUT_HANDLE);

int i;

for (i = 0; i < 4; i++)
{

    /*DWORD dwAvail = 0;
    if (!PeekNamedPipe(g_hChildStd_OUT_Rd, NULL, 0, NULL, &dwAvail, NULL)) {
        // error, the child process might have ended
        break;
    }
    if (!dwAvail) {
        // no data available in the pipe
        break;
    }*/

    bSuccess = ReadFile(g_hChildStd_OUT_Rd, chBuf, BUFSIZE, &dwRead, NULL);
    if (!bSuccess || dwRead == 0) break;

    /*bSuccess = WriteFile(hParentStdOut, chBuf, dwRead, &dwWritten, NULL);
    if (!bSuccess) break;*/

    chBuf[dwRead] = '';

    printf("%i - %s
", i, chBuf);
}

printf("done
");
}

I issue the initial "cmd.exe" command which gives me the start of the command prompt. I now want to issue "ipconfig" (or any other command) to get networking info. The program deadlocks and becomes unresponsive. I can no longer read output of child process. How can I fix this? Thanks for your help.

Answer 1

the most power and effective solution for avoid any deadlocks - use asynchronous io. never wait for IO (read,write,ioctl) complete in place, but handle this in callbacks.

also note about use pipes for redirect output - very common errancy that we need use different handles for STDIN and STDOUT and need create 2 different pipes pair - one for STDIN and another for STDOUT. this is false. we can use single pipe handle for both STDIN and STDOUT (and STDERROR).

1. we need create server pipe handle by using CreateNamedPipeW with PIPE_ACCESS_DUPLEX|FILE_READ_DATA|FILE_WRITE_DATA|FILE_FLAG_OVERLAPPED flags. by using PIPE_ACCESS_DUPLEX we create bi-directional pipe, as result both server and client processes can read from and write to the pipe. and FILE_FLAG_OVERLAPPED give to as asynchronous mode. also we not make this handle inheritable, so not need call SetHandleInformation on it
2. client handle we create by CreateFileW also with FILE_GENERIC_READ|FILE_GENERIC_WRITE access - this give ability assign it both to stdin and stdout. because clients (like cmd.exe) usually assume synchronous io - we not use FILE_FLAG_OVERLAPPED here. also by using lpSecurityAttributes we just make this handle inheritable.
3. we need bind server handle to some IOCP, for callback called when io is ended. here we have 3 variants - use BindIoCompletionCallback - the most simply way or use CreateThreadpoolIo. also we can create IOCP yourself and own thread pool, but for redirect child process output, this way usually not need.
4. after we create child process - we need close client pipe handle (which we duplicate to child) and just call ReadFile on our pipe handle. when this ReadFile complete - we need again call ReadFile from callback and so on - until we not got error from ReadFile in completion (usually ERROR_BROKEN_PIPE). so we need all time have active read request from pipe, until disconnect.
5. and we free call WriteFile at any time and any place - this never cause deadlock, because we use asynchronous io.
6. some time (very very rarely) if we need complex processing on read data(based on previous results and state) and this much more easy handle in plain procedure but not in callbacks, we can create fiber for this task (CreateFiber) and from working thread callback, when read complete - first call ConvertThreadToFiber (if we call this more than once for same working thread - will be error ERROR_ALREADY_FIBER on second and next calls, but this is ok. but all this work begin from vista only. on xp error here). remember current fiber, to where need retirn (GetCurrentFiber()) and call SwitchToFiber (with our dedicated for read fiber)- where we can handle read result and after this return back by call SwitchToFiber (with fiber for worked thread). but all this really can be need in in very rare and specific scenarios. usually handle all is callbacks with state in object related to pipe handle - more than enough.

simply example with cmd

#define _XP_SUPPORT_

struct IO_COUNT 
{
    HANDLE _hFile;
    HANDLE _hEvent;
    LONG _dwIoCount;

    IO_COUNT()
    {
        _dwIoCount = 1;
        _hEvent = 0;
    }

    ~IO_COUNT()
    {
        if (_hEvent)
        {
            CloseHandle(_hEvent);
        }
    }

    ULONG Create(HANDLE hFile);

    void BeginIo()
    {
        InterlockedIncrement(&_dwIoCount);
    }

    void EndIo()
    {
        if (!InterlockedDecrement(&_dwIoCount))
        {
            SetEvent(_hEvent);
        }
    }

    void Wait()
    {
        WaitForSingleObject(_hEvent, INFINITE);
    }
};


struct U_IRP : OVERLAPPED 
{
    enum { read, write };

    IO_COUNT* _pIoObject;
    ULONG _code;
    LONG _dwRef;
    char _buffer[256];

    void AddRef()
    {
        InterlockedIncrement(&_dwRef);
    }

    void Release()
    {
        if (!InterlockedDecrement(&_dwRef)) delete this;
    }

    U_IRP(IO_COUNT* pIoObject) : _pIoObject(pIoObject)
    {
        _dwRef = 1;
        pIoObject->BeginIo();
        RtlZeroMemory(static_cast<OVERLAPPED*>(this), sizeof(OVERLAPPED));
    }

    ~U_IRP()
    {
        _pIoObject->EndIo();
    }

    ULONG CheckIoResult(BOOL fOk)
    {
        if (fOk)
        {
#ifndef _XP_SUPPORT_
            OnIoComplete(NOERROR, InternalHigh);
#endif
            return NOERROR;
        }

        ULONG dwErrorCode = GetLastError();

        if (dwErrorCode != ERROR_IO_PENDING)
        {
            OnIoComplete(dwErrorCode, 0);
        }

        return dwErrorCode;
    }

    ULONG Read()
    {
        _code = read;

        AddRef();

        return CheckIoResult(ReadFile(_pIoObject->_hFile, _buffer, sizeof(_buffer), 0, this));
    }

    ULONG Write(const void* pvBuffer, ULONG cbBuffer)
    {
        _code = write;

        AddRef();

        return CheckIoResult(WriteFile(_pIoObject->_hFile, pvBuffer, cbBuffer, 0, this));
    }

    VOID OnIoComplete(DWORD dwErrorCode, DWORD_PTR dwNumberOfBytesTransfered)
    {
        switch (_code)
        {
        case read:
            if (dwErrorCode == NOERROR)
            {
                if (dwNumberOfBytesTransfered)
                {
                    if (int cchWideChar = MultiByteToWideChar(CP_OEMCP, 0, _buffer, (ULONG)dwNumberOfBytesTransfered, 0, 0))
                    {
                        PWSTR wz = (PWSTR)alloca(cchWideChar * sizeof(WCHAR));

                        if (MultiByteToWideChar(CP_OEMCP, 0, _buffer, (ULONG)dwNumberOfBytesTransfered, wz, cchWideChar))
                        {
                            if (int cbMultiByte = WideCharToMultiByte(CP_ACP, 0, wz, cchWideChar, 0, 0, 0, 0))
                            {
                                PSTR sz = (PSTR)alloca(cbMultiByte);

                                if (WideCharToMultiByte(CP_ACP, 0, wz, cchWideChar, sz, cbMultiByte, 0, 0))
                                {
                                    DbgPrint("%.*s", cbMultiByte, sz);
                                }
                            }
                        }
                    }
                }
                Read();
            }
            break;
        case write:
            break;
        default:
            __debugbreak();
        }

        Release();

        if (dwErrorCode)
        {
            DbgPrint("[%u]: error=%u
", _code, dwErrorCode);
        }
    }

    static VOID WINAPI _OnIoComplete(
        DWORD dwErrorCode,
        DWORD_PTR dwNumberOfBytesTransfered,
        LPOVERLAPPED lpOverlapped
        )
    {
        static_cast<U_IRP*>(lpOverlapped)->OnIoComplete(RtlNtStatusToDosError(dwErrorCode), dwNumberOfBytesTransfered);
    }
};

ULONG IO_COUNT::Create(HANDLE hFile)
{
    _hFile = hFile;
    // error in declaration LPOVERLAPPED_COMPLETION_ROUTINE : 
    // second parameter must be DWORD_PTR but not DWORD
    return BindIoCompletionCallback(hFile, (LPOVERLAPPED_COMPLETION_ROUTINE)U_IRP::_OnIoComplete, 0) && 
#ifndef _XP_SUPPORT_
        SetFileCompletionNotificationModes(hFile, FILE_SKIP_COMPLETION_PORT_ON_SUCCESS) &&
#endif
        (_hEvent = CreateEvent(0, TRUE, FALSE, 0)) ? NOERROR : GetLastError();
}

void ChildTest()
{
    static const WCHAR name[] = L"\\?\pipe\somename";

    HANDLE hFile = CreateNamedPipeW(name, 
        PIPE_ACCESS_DUPLEX|FILE_READ_DATA|FILE_WRITE_DATA|FILE_FLAG_OVERLAPPED, 
        PIPE_TYPE_BYTE|PIPE_READMODE_BYTE, 1, 0, 0, 0, 0);

    if (hFile != INVALID_HANDLE_VALUE)
    {
        IO_COUNT obj;

        if (obj.Create(hFile) == NOERROR)
        {
            BOOL fOk = FALSE;

            SECURITY_ATTRIBUTES sa = { sizeof(sa), 0, TRUE };

            STARTUPINFOW si = { sizeof(si) };
            PROCESS_INFORMATION pi;

            si.dwFlags = STARTF_USESTDHANDLES;

            si.hStdError = CreateFileW(name, FILE_GENERIC_READ|FILE_GENERIC_WRITE, 
                FILE_SHARE_READ|FILE_SHARE_WRITE, &sa, OPEN_EXISTING, 0, 0);

            if (si.hStdError != INVALID_HANDLE_VALUE)
            {
                si.hStdInput = si.hStdOutput = si.hStdError;

                WCHAR ApplicationName[MAX_PATH];
                if (GetEnvironmentVariableW(L"ComSpec", ApplicationName, RTL_NUMBER_OF(ApplicationName)))
                {
                    if (CreateProcessW(ApplicationName, 0, 0, 0, TRUE, 0, 0, 0, &si, &pi))
                    {
                        CloseHandle(pi.hThread);
                        CloseHandle(pi.hProcess);
                        fOk = TRUE;
                    }
                }

                CloseHandle(si.hStdError);
            }

            if (fOk)
            {
                STATIC_ASTRING(help_and_exit, "help
exit
");

                U_IRP* p;

                if (p = new U_IRP(&obj))
                {
                    p->Read();
                    p->Release();
                }

                obj.EndIo();

                //++ simulate user commands
                static PCSTR commands[] = { "help
", "ver
", "dir
", "exit
" };
                ULONG n = RTL_NUMBER_OF(commands);
                PCSTR* psz = commands;
                do 
                {
                    if (MessageBoxW(0,0, L"force close ?", MB_YESNO) == IDYES)
                    {
                        DisconnectNamedPipe(hFile);
                        break;
                    }
                    if (p = new U_IRP(&obj))
                    {
                        PCSTR command = *psz++;
                        p->Write(command, (ULONG)strlen(command) * sizeof(CHAR));
                        p->Release();
                    }    
                } while (--n);
                //--

                obj.Wait();
            }
        }

        CloseHandle(hFile);
    }
}