multithreading windows and embedding python

[freesteel]

I have posted about this problem before. SInce then I found a much
better article to help with embedding python in a multithreaded
application:

http://www.linuxjournal.com/article/3641

I found this article very good and it clarified for me what needs
doing. Now I have an example application that almost works, but it is
far from reliable. If I run this several times I get crashes telling me
that the heap is modified after deallocation. Can anybody else
reproduce this?

At the bottom of this file I left a debug dump, and a stack dump.

Here is my application, compile in windows using a standard windows
application project.


#include <afxmt.h>
#include <afxwin.h>
#include <stdio.h>
#include <Python.h>
#include <Windows.h>
#include <process.h>

static int threadnum = 0;


UINT MyThread(LPVOID lpParam)
{
ASSERT(Py_IsInitialized());
threadnum++;
PyThreadState* mainThreadState = (PyThreadState *)lpParam;

// get the global lock
PyEval_AcquireLock();
// get a reference to the PyInterpreterState
PyInterpreterState * mainInterpreterState = mainThreadState->interp;
PyThreadState_Swap(mainThreadState);

// create a thread state object for this thread
PyThreadState * myThreadState =
PyThreadState_New(mainInterpreterState);
// free the lock
PyEval_ReleaseLock();
// lock - swap in thread state - swap out thread state - unlock
PyEval_AcquireLock();
PyThreadState_Swap(myThreadState);

int num = 0;
int ret = 0;
ret = PyRun_SimpleString("x = []");
ret = PyRun_SimpleString("for i in range(10):\n x.append(i)");
char cmd[100];
sprintf(cmd, "f = open('c:/windows/temp/test%d.txt', 'w')",
threadnum);
ret = PyRun_SimpleString(cmd);
ret = PyRun_SimpleString("f.write('%s\\n' % x.__str__())");
sprintf(cmd, "f.write('0x%d\\n')", &myThreadState);
ret = PyRun_SimpleString(cmd);
ret = PyRun_SimpleString("f.close()");

PyThreadState_Swap(NULL);
PyEval_ReleaseLock();
// clean up
// grab the lock
PyEval_AcquireLock();
// swap my thread state out of the interpreter
PyThreadState_Swap(NULL);
// clear out any cruft from thread state object
PyThreadState_Clear(myThreadState);
// delete my thread state object
PyThreadState_Delete(myThreadState);
// release the lock
PyEval_ReleaseLock();

return 0;
}

class CMyWinApp : public CWinApp
{
public:
CMyWinApp() { }
BOOL InitInstance()
{
Py_Initialize();
PyEval_InitThreads();

// save a pointer to the main PyThreadState object
PyThreadState * mainThreadState = PyThreadState_Get();
// release the lock
PyEval_ReleaseLock();

const int nhandles = 100;
HANDLE hnd[nhandles];
CWinThread* pThread[nhandles];
for (int ih = 0; ih < nhandles; ih++)
{
pThread[ih] = AfxBeginThread(MyThread, mainThreadState,
THREAD_PRIORITY_NORMAL, CREATE_SUSPENDED);
pThread[ih]->m_bAutoDelete = false;
pThread[ih]->ResumeThread();

hnd[ih] = pThread[ih]->m_hThread;
}

int nwaits, nfails;
do
{
nwaits = 0;
nfails = 0;
for (int ih = 0; ih < nhandles; ih++)
{
DWORD ret = WaitForSingleObject(hnd[ih], INFINITE);
switch (ret)
{
case WAIT_OBJECT_0:
printf("WAIT_OBJECT_0\n");
break;

case WAIT_TIMEOUT:
++nwaits;
printf("WAIT_TIMEOUT\n");
break;

case WAIT_FAILED:
++nfails;
printf("WAIT_FAILED\n");
break;
}
}
}
while (nwaits > 0);
ASSERT(nfails == 0);

// delete all windows threads
for (int ih = 0; ih < nhandles; ++ih)
delete pThread[ih];

PyEval_AcquireLock();
PyThreadState_Swap(mainThreadState);
Py_Finalize();

return TRUE;
};
};

CMyWinApp app;


Debug dump:

'pyembed_test.exe': Loaded
'C:\mdunschen\pyembed_test\Debug\pyembed_test.exe' , Symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\ntdll.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\mscoree.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\kernel32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\advapi32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\rpcrt4.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\user32.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\gdi32.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\comctl32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\shlwapi.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\msvcrt.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\oleacc.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\msvcp60.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\ole32.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\oleaut32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\winspool.drv', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\comdlg32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\shell32.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\python24.dll', No
symbols loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\msvcr71.dll', Symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\WinSxS\x86_Microsoft.Windows.Common-Controls_6595b64144ccf1df_6.0.2600.2180_x-ww_a84f1ff9\comctl32.dll',
No symbols loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\msco rwks.dll', No symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\fusi on.dll', No symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\msco rlib.dll', No symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\assembly\NativeImages1_v1.1.4322\mscor lib\1.0.5000.0__b77a5c561934e089_ca105f6f\mscorlib .dll',
No symbols loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\dias ymreader.dll', No
symbols loaded.
'DefaultDomain': Loaded
'c:\windows\microsoft.net\framework\v1.1.4322\msco rlib.dll', No symbols
loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\msco rsn.dll', No symbols
loaded.
'pyembed_test': Loaded
'c:\mdunschen\pyembed_test\Debug\pyembed_test.exe' , Symbols loaded.
'pyembed_test.exe': Loaded
'C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\msco rjit.dll', No symbols
loaded.
'pyembed_test.exe': Loaded 'C:\WINDOWS\system32\uxtheme.dll', No
symbols loaded.
The thread '_threadstartex' (0x500) has exited with code 0 (0x0).
The thread '_threadstartex' (0xadc) has exited with code 0 (0x0).
The thread '<No Name>' (0xadc) has exited with code 0 (0x0).
The thread '<No Name>' (0x500) has exited with code 0 (0x0).
The thread '_threadstartex' (0x928) has exited with code 0 (0x0).
The thread '<No Name>' (0x928) has exited with code 0 (0x0).
The thread '_threadstartex' (0xe48) has exited with code 0 (0x0).
The thread '<No Name>' (0xe48) has exited with code 0 (0x0).
The thread '_threadstartex' (0x984) has exited with code 0 (0x0).
The thread '<No Name>' (0x984) has exited with code 0 (0x0).
The thread '_threadstartex' (0x108) has exited with code 0 (0x0).
HEAP[pyembed_test.exe]: HEAP: Free Heap block eb5bd8 modified at eb5c24
after it was freed
Unhandled exception at 0x7c901230 in pyembed_test.exe: User breakpoint.


Stack dump:

pyembed_test.exe!_heap_alloc_dbg(unsigned int nSize=15400960, int nBlockUse=1073741920, const char * szFileName=0x00000064, int nLine=1242368) Line 359 + 0x1e C

pyembed_test.exe!_heap_alloc_base(unsigned int size=100) Line 212 C
pyembed_test.exe!_heap_alloc_dbg(unsigned int nSize=64, int
nBlockUse=12582916, const char * szFileName=0x004c4dd4, int nLine=311)
Line 397 + 0x9 C
pyembed_test.exe!_nh_malloc_dbg(unsigned int nSize=64, int nhFlag=0,
int nBlockUse=12582916, const char * szFileName=0x004c4dd4, int
nLine=311) Line 260 + 0x15 C
pyembed_test.exe!_malloc_dbg(unsigned int nSize=64, int
nBlockUse=12582916, const char * szFileName=0x004c4dd4, int nLine=311)
Line 176 + 0x1b C
pyembed_test.exe!operator new(unsigned int nSize=64, int
nType=12582916, const char * lpszFileName=0x004c4dd4, int nLine=311)
Line 403 + 0x15 C++
pyembed_test.exe!CObject::operator new(unsigned int nSize=64, const
char * lpszFileName=0x004c4dd4, int nLine=311) Line 93 + 0x16 C++
pyembed_test.exe!AfxBeginThread(unsigned int (void *)*
pfnThreadProc=0x00401b80, void * pParam=0x00df7b10, int nPriority=0,
unsigned int nStackSize=4, unsigned long dwCreateFlags=0,
_SECURITY_ATTRIBUTES * lpSecurityAttrs=0x00000000) Line 311 + 0x11 C++
00aaa43d()
pyembed_test.exe!CMyWinApp::InitInstance() Line 87 + 0x26 bytes C++
pyembed_test.exe!AfxWinMain(HINSTANCE__ * hInstance=0x00400000,
HINSTANCE__ * hPrevInstance=0x00000000, char * lpCmdLine=0x00141f12,
int nCmdShow=5) Line 39 + 0xb C++
pyembed_test.exe!WinMain(HINSTANCE__ * hInstance=0x00400000,
HINSTANCE__ * hPrevInstance=0x00000000, char * lpCmdLine=0x00141f12,
int nCmdShow=5) Line 25 C++
pyembed_test.exe!WinMainCRTStartup() Line 251 + 0x30 C

[freesteel]

freesteel wrote:
....

pThread[ih] = AfxBeginThread(MyThread, mainThreadState,
THREAD_PRIORITY_NORMAL, CREATE_SUSPENDED);

....

Here the call to AfxBeginThread is wrong, there is one argument
missing, it should be:

pThread[ih] = AfxBeginThread(MyThread, mainThreadState,
THREAD_PRIORITY_NORMAL, 0, CREATE_SUSPENDED);

Because there are so many default arguments of similar types the
compiler did not notice that I passed 'CREATE_SUSPENDED' as a stack
size, and use the default 'creation' state of 'start right away' for
the thread.

Don't you love default args and types like 'void*' ?

Martin