I am trying to improve the Lua io.popen function to use the Win32 CreateProcess api. By default, it uses the libc popen function, which unfortunately spawns a terminal window on Windows even when running in a Desktop application.

My approach is currently to copy .c, put it in it's own Lua library file and then change some methods. It mostly seems to work, but weirdly reading from the subprocess blocks after 2048 bytes. My assumption is that the pipe buffer is full and is not emptied, but not sure how to configure it.

So, originally the Lua file does this:

#define l_popen(L,c,m)      (_popen(c,m))
#define l_pclose(L,file)    (_pclose(file))

I have replaced this by this code:

static FILE* l_popen(lua_State *L, const char* filename, const char *mode, PROCESS_INFORMATION* pi) {
  ZeroMemory(pi, sizeof(PROCESS_INFORMATION));

  HANDLE g_hChildStd_IN_Rd = NULL;
  HANDLE g_hChildStd_IN_Wr = NULL;
  HANDLE g_hChildStd_OUT_Rd = NULL;
  HANDLE g_hChildStd_OUT_Wr = NULL;

  SECURITY_ATTRIBUTES saAttr;
  ZeroMemory(&saAttr, sizeof(SECURITY_ATTRIBUTES));
  saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
  saAttr.bInheritHandle = TRUE;
  saAttr.lpSecurityDescriptor = NULL;

  // Pipe for child process' STDOUT
  if (!CreatePipe(&g_hChildStd_OUT_Rd, &g_hChildStd_OUT_Wr, &saAttr, 0)) {
    printf("Error creating pipe\n");
    return NULL;
  }

  if ( ! SetHandleInformation(g_hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0) ) {
    printf("SOmething weird\n");
    return NULL;
  }

  // Pipe for child process' STDIN
  if (!CreatePipe(&g_hChildStd_IN_Rd, &g_hChildStd_IN_Wr, &saAttr, 0)) {
    printf("Error creating pipe\n");
    return NULL;
  }

  if ( ! SetHandleInformation(g_hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0) ) {
    printf("SOmething weird\n");
    return NULL;
  }

  STARTUPINFO si;
  ZeroMemory(&si, sizeof(si));
  si.cb = sizeof(si);
  si.hStdError = g_hChildStd_OUT_Wr;
  si.hStdOutput = g_hChildStd_OUT_Wr;
  si.hStdInput = g_hChildStd_IN_Rd;
  si.dwFlags |=  STARTF_USESTDHANDLES;

  if (!CreateProcess( 
    NULL,
    filename, //TODO: check unicode?
    NULL,
    NULL,
    TRUE,
    0,
    NULL,
    NULL,
    &si,
    pi)
  ) {
    printf("Could not create process\n");
    return NULL;
  }

  int fd;
  if (strcmp(mode, "r") == 0) {
    fd = _open_osfhandle((intptr_t) g_hChildStd_OUT_Rd, 0);
  } else if (strcmp(mode, "w") == 0) {
    fd = _open_osfhandle((intptr_t) g_hChildStd_IN_Wr, 0);
  }

  if (fd == -1) {
    printf("Could not open file descriptor");
  }

  printf("Got file descriptor:%d\n", fd);

  FILE* f = _fdopen(fd, mode);
  return f;
}

which is mostly inspired by the MSDN documentation .

As the original file uses the libc functions like getc or fread, I converted the handle to a FILE* so that it will be easier to maintain (That way, i don't have to change everything to ReadFile. Instead in the future, I can just diff the original source file, add my few changes and done).

The problem is now here:

static void read_all (lua_State *L, FILE *f) {
  size_t nr;
  luaL_Buffer b;
  luaL_buffinit(L, &b);
  do {  /* read file in chunks of LUAL_BUFFERSIZE bytes */
    printf("Preparing buffer\n");
    char *p = luaL_prepbuffer(&b);
    nr = fread(p, sizeof(char), LUAL_BUFFERSIZE, f);
    printf("Read %zd bytes\n", nr);
    //printf("Content: %s\n", p);
    luaL_addsize(&b, nr);
  } while (nr == LUAL_BUFFERSIZE);
  printf("Read all\n");
  luaL_pushresult(&b);  /* close buffer */
}

If I load the library to a Lua interpreter, and execute the following file, I get this:

local phandle = mylib.popen("git")
print(phandle:read("a"))

Got file descriptor:3
Preparing buffer
Read 1024 bytes
Preparing buffer
Read 1024 bytes
Preparing buffer

And then it hangs. On the other hand, if I do

local phandle = mylib.popen("git")
for i=1,50 do
    print(phandle:read("l"))
end

it will print the entire output of git, but also block at the last line (So it seems that it does not properly return a EOF).

Not sure where the problem is though...