SlideShare a Scribd company logo
Threads and Callbacks
for Embedded Python
蔡易⻯⿓龍
Yi-Lung Tsai (Bruce)
Python/C++/C/Java
Coffee/Jazz/Photography
Embedding Python
Embed Python interpreter in C/C++, run a script and
use the result
C/C++
Python
Interpreter Python
Benefit
• Running custom code without recompiling main
program
• Python is more flexible or suitable for solving
specific problems
• Exposing scripting API driving main program
https://www.blender.org/
manual/_images/
Extensions-Python-
Console-Example-bpy-
ops.jpg
C/C++ Layer
Python Interpreter
Python Layer
First Glance
#include <Python.h>
int main(int argc, char* argv[])
{
Py_Initialize();
PyRun_SimpleString("from time import time, ctimen"
"print 'Today is', ctime(time())n");
Py_Finalize();
return 0;
}
http://fannit.com/wp-content/uploads/2014/03/0960a7436008f887aa6bbf0afd34eb722.jpg
Python/C API
• Everything is PyObject*
• Symbols with Py prefix
...
PyObject* sysPath = PySys_GetObject((char*) "path");
PyList_Append(sysPath, PyString_FromString("."));
printf("Input two integers separated by space:n");
int a, b;
scanf("%d %d", &a, &b);
PyObject *pModule = NULL, *pFunc = NULL, *pArgs = NULL, *pValue = NULL;
do
{
pModule = PyImport_ImportModule("func");
if (pModule == NULL) break;
pFunc = PyObject_GetAttrString(pModule, "multiply");
if (pFunc == NULL) break;
pArgs = Py_BuildValue("ii", a, b);
if (pArgs == NULL) break;
pValue = PyObject_Call(pFunc, pArgs, NULL);
if (pValue == NULL) break;
printf("Result of call: %ldn", PyInt_AsLong(pValue));
} while (0);
Py_XDECREF(pValue);
Py_XDECREF(pArgs);
Py_XDECREF(pFunc);
Py_XDECREF(pModule);
...
Python Program Path
...
PyObject* sysPath = PySys_GetObject((char*) "path");
PyList_Append(sysPath, PyString_FromString(“."));
...
PyObject Manipulation
• Reference Python object as PyObject
• Manipulate PyObject
• Dereference PyObject
...
PyObject *pModule = NULL, *pFunc = NULL, *pArgs = NULL, *pValue = NULL;
do
{
pModule = PyImport_ImportModule("func");
if (pModule == NULL) break;
pFunc = PyObject_GetAttrString(pModule, "multiply");
if (pFunc == NULL) break;
pArgs = Py_BuildValue("ii", a, b);
if (pArgs == NULL) break;
pValue = PyObject_Call(pFunc, pArgs, NULL);
if (pValue == NULL) break;
printf("Result of call: %ldn", PyInt_AsLong(pValue));
} while (0);
...
Data Conversion
1. Convert data values from C to Python
2. Perform a function call to a Python interface
routine using the converted values
3. Convert the data values from the call from Python
to C
pArgs = Py_BuildValue("ii", a, b);
Format Python type C type
s string char*
i integer int
b integer char
l integer long int
d float double
o object PyObject*
() tuple
[] list
{} dictionary
Free Resource
PyObject* PyDict_New()
Return value: New reference.
PyObject* PyDict_GetItem(PyObject *p, PyObject *key)
Return value: Borrowed reference.
Need to decrease
reference counting
NO need to decrease
reference counting
Multithreading
• Python interpreter is not fully thread-safe
• Need to deal with global interpreter lock (GIL)
• GIL ensures only one Python instruction runs at one
time
C/C++
Python
class WorkerThread(threading.Thread):
def __init__(self, name):
super(WorkerThread, self).__init__(name=name)
self.stop_event = threading.Event()
def run(self):
while not self.stop_event.is_set():
print self.name, "is working"
time.sleep(1)
def stop(self):
print self.name, "stop"
self.stop_event.set()
class ThreadManager(object):
def __init__(self):
self.worker = WorkerThread("worker")
def start_thread(self):
self.worker.start()
def stop_thread(self):
self.worker.stop()
self.worker.join()
PyEval_InitThreads();
...
PyObject *pModule = NULL, *pClass = NULL, *pInst = NULL;
do
{
PyGILState_STATE state = PyGILState_Ensure();
{
pModule = PyImport_ImportModule("worker");
if (pModule == NULL) break;
pClass = PyObject_GetAttrString(pModule, "ThreadManager");
if (pClass == NULL) break;
pInst = PyObject_CallObject(pClass, NULL);
if (pInst == NULL) break;
PyObject_CallMethod(pInst, "start_thread", NULL);
}
PyGILState_Release(state);
for (int i = 0; i < 5; i++)
{
printf("main thread is runningn");
sleep(1);
}
...
http://www.ralphhoweministries.com/sancfiles/homer.jpg
1. PyEval_InitThreads();
2. state = PyGILState_Ensure();
3. <start thread in Python>
4. PyGILState_Release(state)
5. <main thread in C>
main thread acquires GIL
main thread acquires GIL
main thread acquires GIL
no GIL
Python thread not run
PyEval_InitThreads();
...
PyThreadState* save = PyEval_SaveThread();
PyObject *pModule = NULL, *pClass = NULL, *pInst = NULL;
do
{
PyGILState_STATE state = PyGILState_Ensure();
{
pModule = PyImport_ImportModule("worker");
if (pModule == NULL) break;
pClass = PyObject_GetAttrString(pModule, "ThreadManager");
if (pClass == NULL) break;
pInst = PyObject_CallObject(pClass, NULL);
if (pInst == NULL) break;
PyObject_CallMethod(pInst, "start_thread", NULL);
}
PyGILState_Release(state);
for (int i = 0; i < 5; i++)
{
printf("main thread is runningn");
sleep(1);
}
...
1. PyEval_InitThreads();
2. save = PyEval_SaveThread();
3. state = PyGILState_Ensure();
4. <start thread in Python>
5. PyGILState_Release(state)
6. <main thread in C>
main thread acquires GIL
main thread releases GIL
main thread acquires GIL
main thread releases GIL
no GIL
Python thread run
Multithreading - 2
• Create threads from C
• Each C thread calls Python function
C
P
int main(int argc, char* argv[])
{
PyEval_InitThreads();
Py_Initialize();
PyObject* sysPath = PySys_GetObject((char*) "path");
PyList_Append(sysPath, PyString_FromString("."));
PyThreadState* save = PyEval_SaveThread();
pthread_t tid1, tid2;
char* tname1 = "worker1";
char* tname2 = "worker2";
pthread_create(&tid1, NULL, &run_python_function, &tname1);
pthread_create(&tid2, NULL, &run_python_function, &tname2);
for (int i = 0; i < 5; i++)
{
printf("main thread is runningn");
sleep(1);
}
stop_event = 1;
pthread_join(tid1, NULL);
pthread_join(tid2, NULL);
printf("finishn");
PyEval_RestoreThread(save);
Py_Finalize();
pthread_exit(NULL);
return 0;
}
void* run_python_function(void* arg)
{
PyGILState_STATE state = PyGILState_Ensure();
PyObject *pModule = NULL, *pFunc = NULL;
do
{
pModule = PyImport_ImportModule("work");
if (pModule == NULL) break;
pFunc = PyObject_GetAttrString(pModule, "working");
if (pFunc == NULL) break;
char* name = *((char**) arg);
while (!stop_event)
{
PyObject_CallFunction(pFunc, "s", name);
}
} while (0);
Py_XDECREF(pFunc);
Py_XDECREF(pModule);
PyGILState_Release(state);
pthread_exit(NULL);
return NULL;
}
import time
def working(name):
print name, "is working"
time.sleep(1)
Callback Function
• Asynchronous method invocation
• Push event to C
• Pass heavy computation to C
static PyObject* callback(PyObject* self, PyObject* args)
{
long c;
PyArg_ParseTuple(args, "l", &c);
printf("Result of call: %ldn", c);
Py_RETURN_NONE;
}
int main(int argc, char* argv[])
{
...
PyObject *pModule = NULL, *pFunc = NULL, *pArgs = NULL;
do
{
PyMethodDef CFunctions[] = {
{"callback", callback, METH_VARARGS, ""},
{NULL, NULL, 0, NULL}
};
Py_InitModule("cmodule", CFunctions);
pModule = PyImport_ImportModule("func-module");
if (pModule == NULL) break;
pFunc = PyObject_GetAttrString(pModule, "multiply");
if (pFunc == NULL) break;
pArgs = Py_BuildValue("ii", a, b);
if (pArgs == NULL) break;
PyObject_Call(pFunc, pArgs, NULL);
} while (0);
...
}
Create New Module
static PyObject* callback(PyObject* self, PyObject* args)
{
long c;
PyArg_ParseTuple(args, "l", &c);
printf("Result of call: %ldn", c);
Py_RETURN_NONE;
}
...
PyMethodDef CFunctions[] = {
{"callback", callback, METH_VARARGS, ""},
{NULL, NULL, 0, NULL}
};
Py_InitModule("cmodule", CFunctions);
...
Invoke Python Function
...
pModule = PyImport_ImportModule("func-module");
if (pModule == NULL) break;
pFunc = PyObject_GetAttrString(pModule, "multiply");
if (pFunc == NULL) break;
pArgs = Py_BuildValue("ii", a, b);
if (pArgs == NULL) break;
PyObject_Call(pFunc, pArgs, NULL);
...
Import and Use
import cmodule
def multiply(a, b):
print "Will compute", a, "times", b
c = 0
for i in range(0, a):
c = c + b
cmodule.callback(c)
Callback to C
...
static PyObject* callback(PyObject* self, PyObject* args)
{
long c;
PyArg_ParseTuple(args, "l", &c);
printf("Result of call: %ldn", c);
Py_RETURN_NONE;
}
...
http://pre05.deviantart.net/8b23/th/pre/f/2011/288/e/1/not_bad_by_rober_raik-d4cwbtb.png
Pure Callback
def multiply(a, b, callback):
print "Will compute", a, "times", b
c = 0
for i in range(0, a):
c = c + b
callback(c)
...
PyObject *pCallbackFunc = NULL, *pModule = NULL, *pFunc = NULL, *pArgs = NULL;
do
{
PyMethodDef CFunc = {"callback", callback, METH_VARARGS, ""};
pCallbackFunc = PyCFunction_New(&CFunc, NULL);
if (pCallbackFunc == NULL) break;
pModule = PyImport_ImportModule("func-callback");
if (pModule == NULL) break;
pFunc = PyObject_GetAttrString(pModule, "multiply");
if (pFunc == NULL) break;
pArgs = Py_BuildValue("iiO", a, b, pCallbackFunc);
if (pArgs == NULL) break;
PyObject_Call(pFunc, pArgs, NULL);
} while (0);
...
CFunction Object
Event Loop
C/C++
Python
Blocking Function
C/C++
Python
static PyObject* callback1(PyObject* self, PyObject* args)
{
char* str;
PyArg_ParseTuple(args, "s", &str);
printf("%s push eventn", str);
Py_RETURN_NONE;
}
static PyObject* callback2(PyObject* self, PyObject* args)
{
char* str;
PyArg_ParseTuple(args, "s", &str);
printf("%s start heavy computingn", str);
sleep(5);
printf("%s end heavy computingn", str);
Py_RETURN_NONE;
}
static PyObject* callback1(PyObject* self, PyObject* args)
{
char* str;
PyArg_ParseTuple(args, "s", &str);
printf("%s push eventn", str);
Py_RETURN_NONE;
}
static PyObject* callback2(PyObject* self, PyObject* args)
{
char* str;
PyArg_ParseTuple(args, "s", &str);
printf("%s start heavy computingn", str);
PyThreadState* save = PyEval_SaveThread();
{
sleep(5);
}
PyEval_RestoreThread(save);
printf("%s end heavy computingn", str);
Py_RETURN_NONE;
}
Threads and Callbacks for Embedded Python
Summary
• PyObject manipulation in C
• GIL in multithreading
• CFunction object as callback
• Multithreading with callback

More Related Content

PPTX
Mixing C++ & Python II: Pybind11
PDF
Git Rebase vs Merge
PDF
ドメインロジックに集中せよ 〜ドメイン駆動設計 powered by Spring
PDF
Why rust?
PDF
TIP1 - Overview of C/C++ Debugging/Tracing/Profiling Tools
PDF
クリーンアーキテクチャの考え方にもとづく Laravel との付き合い方 #phpconokinawa
PDF
Linux Namespace
PPTX
Visual Studio を使わず .NET する
Mixing C++ & Python II: Pybind11
Git Rebase vs Merge
ドメインロジックに集中せよ 〜ドメイン駆動設計 powered by Spring
Why rust?
TIP1 - Overview of C/C++ Debugging/Tracing/Profiling Tools
クリーンアーキテクチャの考え方にもとづく Laravel との付き合い方 #phpconokinawa
Linux Namespace
Visual Studio を使わず .NET する

What's hot (20)

PDF
Introduction to Python IDLE | IDLE Tutorial | Edureka
PDF
Redmineとgitの 連携利用事例
PPTX
BuildKitによる高速でセキュアなイメージビルド
PDF
Git pour les (pas si) nuls
PDF
やりなおせる Git 入門
PDF
CMake best practices
PDF
Gitlab Training with GIT and SourceTree
PDF
Pengenalan Git
PDF
Linux network stack
PDF
C# における Redis 徹底活用
PPTX
NginxとLuaを用いた動的なリバースプロキシでデプロイを 100 倍速くした
PPTX
git, 이해부터 활용까지
PDF
Rust system programming language
PDF
Raspberry Pi + Go で IoT した話
PDF
コンテナイメージの脆弱性スキャンについて
PDF
Stargz Snapshotter: イメージのpullを省略しcontainerdでコンテナを高速に起動する
PDF
Spring native について
PDF
Deep drive into rust programming language
PDF
WebAssemblyのWeb以外のことぜんぶ話す
PPTX
Keycloak入門-OpenID ConnectによるAPIセキュリティ
Introduction to Python IDLE | IDLE Tutorial | Edureka
Redmineとgitの 連携利用事例
BuildKitによる高速でセキュアなイメージビルド
Git pour les (pas si) nuls
やりなおせる Git 入門
CMake best practices
Gitlab Training with GIT and SourceTree
Pengenalan Git
Linux network stack
C# における Redis 徹底活用
NginxとLuaを用いた動的なリバースプロキシでデプロイを 100 倍速くした
git, 이해부터 활용까지
Rust system programming language
Raspberry Pi + Go で IoT した話
コンテナイメージの脆弱性スキャンについて
Stargz Snapshotter: イメージのpullを省略しcontainerdでコンテナを高速に起動する
Spring native について
Deep drive into rust programming language
WebAssemblyのWeb以外のことぜんぶ話す
Keycloak入門-OpenID ConnectによるAPIセキュリティ
Ad

Similar to Threads and Callbacks for Embedded Python (20)

PDF
Notes about moving from python to c++ py contw 2020
PDF
PyPy's approach to construct domain-specific language runtime
PDF
Multiprocessing with python
PDF
Interfacing C++ with Python to boost your legacy apps with Python interfaces
PDF
concurrency
PDF
Global Interpreter Lock: Episode I - Break the Seal
PDF
«Python на острие бритвы: PyPy project» Александр Кошкин, Positive Technologies
PDF
25 must know python for Interview by Tutort Academy
PPTX
Chapter 5 - THREADING & REGULAR exp - MAULIK BORSANIYA
PDF
Cluj.py Meetup: Extending Python in C
PDF
Extending Python - EuroPython 2014
PDF
Understanding PyPy - PyConEs 14
KEY
Do more than one thing at the same time, the Python way
PDF
Global Interpreter Lock: Episode III - cat &lt; /dev/zero > GIL;
PPTX
Python UNIT-IV Multi Threading B.Tech CSE
PDF
Byterun, a Python bytecode interpreter - Allison Kaptur at NYCPython
PDF
Multithreaded_Programming_in_Python.pdf
PDF
Take advantage of C++ from Python
PDF
Comparing On-The-Fly Accelerating Packages: Numba, TensorFlow, Dask, etc
PDF
Joblib: Lightweight pipelining for parallel jobs (v2)
Notes about moving from python to c++ py contw 2020
PyPy's approach to construct domain-specific language runtime
Multiprocessing with python
Interfacing C++ with Python to boost your legacy apps with Python interfaces
concurrency
Global Interpreter Lock: Episode I - Break the Seal
«Python на острие бритвы: PyPy project» Александр Кошкин, Positive Technologies
25 must know python for Interview by Tutort Academy
Chapter 5 - THREADING & REGULAR exp - MAULIK BORSANIYA
Cluj.py Meetup: Extending Python in C
Extending Python - EuroPython 2014
Understanding PyPy - PyConEs 14
Do more than one thing at the same time, the Python way
Global Interpreter Lock: Episode III - cat &lt; /dev/zero > GIL;
Python UNIT-IV Multi Threading B.Tech CSE
Byterun, a Python bytecode interpreter - Allison Kaptur at NYCPython
Multithreaded_Programming_in_Python.pdf
Take advantage of C++ from Python
Comparing On-The-Fly Accelerating Packages: Numba, TensorFlow, Dask, etc
Joblib: Lightweight pipelining for parallel jobs (v2)
Ad

More from Yi-Lung Tsai (8)

PDF
Lightning Talk - Raspberry Pi
PDF
Problem Solving with Algorithms and Data Structure - Graphs
PDF
Normal mapping
PDF
Problem Solving with Algorithms and Data Structure - Lists
PDF
Problem Solving with Algorithms and Data Structures
PDF
OpenGL Introduction
PDF
iOS GPUImage introduction
PDF
Android programming introduction
Lightning Talk - Raspberry Pi
Problem Solving with Algorithms and Data Structure - Graphs
Normal mapping
Problem Solving with Algorithms and Data Structure - Lists
Problem Solving with Algorithms and Data Structures
OpenGL Introduction
iOS GPUImage introduction
Android programming introduction

Recently uploaded (20)

PPTX
Agentic AI : A Practical Guide. Undersating, Implementing and Scaling Autono...
PDF
Understanding Forklifts - TECH EHS Solution
PPTX
Transform Your Business with a Software ERP System
PPT
JAVA ppt tutorial basics to learn java programming
PPTX
Lecture 3: Operating Systems Introduction to Computer Hardware Systems
PPTX
Essential Infomation Tech presentation.pptx
PPTX
VVF-Customer-Presentation2025-Ver1.9.pptx
PDF
top salesforce developer skills in 2025.pdf
PDF
Wondershare Filmora 15 Crack With Activation Key [2025
PDF
T3DD25 TYPO3 Content Blocks - Deep Dive by André Kraus
PDF
Addressing The Cult of Project Management Tools-Why Disconnected Work is Hold...
PDF
2025 Textile ERP Trends: SAP, Odoo & Oracle
PPTX
Materi-Enum-and-Record-Data-Type (1).pptx
PDF
System and Network Administration Chapter 2
PDF
Internet Downloader Manager (IDM) Crack 6.42 Build 41
PDF
How to Migrate SBCGlobal Email to Yahoo Easily
PDF
Adobe Premiere Pro 2025 (v24.5.0.057) Crack free
PDF
Upgrade and Innovation Strategies for SAP ERP Customers
PPTX
ISO 45001 Occupational Health and Safety Management System
PPTX
Odoo POS Development Services by CandidRoot Solutions
Agentic AI : A Practical Guide. Undersating, Implementing and Scaling Autono...
Understanding Forklifts - TECH EHS Solution
Transform Your Business with a Software ERP System
JAVA ppt tutorial basics to learn java programming
Lecture 3: Operating Systems Introduction to Computer Hardware Systems
Essential Infomation Tech presentation.pptx
VVF-Customer-Presentation2025-Ver1.9.pptx
top salesforce developer skills in 2025.pdf
Wondershare Filmora 15 Crack With Activation Key [2025
T3DD25 TYPO3 Content Blocks - Deep Dive by André Kraus
Addressing The Cult of Project Management Tools-Why Disconnected Work is Hold...
2025 Textile ERP Trends: SAP, Odoo & Oracle
Materi-Enum-and-Record-Data-Type (1).pptx
System and Network Administration Chapter 2
Internet Downloader Manager (IDM) Crack 6.42 Build 41
How to Migrate SBCGlobal Email to Yahoo Easily
Adobe Premiere Pro 2025 (v24.5.0.057) Crack free
Upgrade and Innovation Strategies for SAP ERP Customers
ISO 45001 Occupational Health and Safety Management System
Odoo POS Development Services by CandidRoot Solutions

Threads and Callbacks for Embedded Python

  • 1. Threads and Callbacks for Embedded Python 蔡易⻯⿓龍 Yi-Lung Tsai (Bruce)
  • 3. Embedding Python Embed Python interpreter in C/C++, run a script and use the result C/C++ Python Interpreter Python
  • 4. Benefit • Running custom code without recompiling main program • Python is more flexible or suitable for solving specific problems • Exposing scripting API driving main program
  • 7. First Glance #include <Python.h> int main(int argc, char* argv[]) { Py_Initialize(); PyRun_SimpleString("from time import time, ctimen" "print 'Today is', ctime(time())n"); Py_Finalize(); return 0; }
  • 9. Python/C API • Everything is PyObject* • Symbols with Py prefix
  • 10. ... PyObject* sysPath = PySys_GetObject((char*) "path"); PyList_Append(sysPath, PyString_FromString(".")); printf("Input two integers separated by space:n"); int a, b; scanf("%d %d", &a, &b); PyObject *pModule = NULL, *pFunc = NULL, *pArgs = NULL, *pValue = NULL; do { pModule = PyImport_ImportModule("func"); if (pModule == NULL) break; pFunc = PyObject_GetAttrString(pModule, "multiply"); if (pFunc == NULL) break; pArgs = Py_BuildValue("ii", a, b); if (pArgs == NULL) break; pValue = PyObject_Call(pFunc, pArgs, NULL); if (pValue == NULL) break; printf("Result of call: %ldn", PyInt_AsLong(pValue)); } while (0); Py_XDECREF(pValue); Py_XDECREF(pArgs); Py_XDECREF(pFunc); Py_XDECREF(pModule); ...
  • 11. Python Program Path ... PyObject* sysPath = PySys_GetObject((char*) "path"); PyList_Append(sysPath, PyString_FromString(“.")); ...
  • 12. PyObject Manipulation • Reference Python object as PyObject • Manipulate PyObject • Dereference PyObject
  • 13. ... PyObject *pModule = NULL, *pFunc = NULL, *pArgs = NULL, *pValue = NULL; do { pModule = PyImport_ImportModule("func"); if (pModule == NULL) break; pFunc = PyObject_GetAttrString(pModule, "multiply"); if (pFunc == NULL) break; pArgs = Py_BuildValue("ii", a, b); if (pArgs == NULL) break; pValue = PyObject_Call(pFunc, pArgs, NULL); if (pValue == NULL) break; printf("Result of call: %ldn", PyInt_AsLong(pValue)); } while (0); ...
  • 14. Data Conversion 1. Convert data values from C to Python 2. Perform a function call to a Python interface routine using the converted values 3. Convert the data values from the call from Python to C
  • 15. pArgs = Py_BuildValue("ii", a, b); Format Python type C type s string char* i integer int b integer char l integer long int d float double o object PyObject* () tuple [] list {} dictionary
  • 16. Free Resource PyObject* PyDict_New() Return value: New reference. PyObject* PyDict_GetItem(PyObject *p, PyObject *key) Return value: Borrowed reference. Need to decrease reference counting NO need to decrease reference counting
  • 17. Multithreading • Python interpreter is not fully thread-safe • Need to deal with global interpreter lock (GIL) • GIL ensures only one Python instruction runs at one time
  • 19. class WorkerThread(threading.Thread): def __init__(self, name): super(WorkerThread, self).__init__(name=name) self.stop_event = threading.Event() def run(self): while not self.stop_event.is_set(): print self.name, "is working" time.sleep(1) def stop(self): print self.name, "stop" self.stop_event.set() class ThreadManager(object): def __init__(self): self.worker = WorkerThread("worker") def start_thread(self): self.worker.start() def stop_thread(self): self.worker.stop() self.worker.join()
  • 20. PyEval_InitThreads(); ... PyObject *pModule = NULL, *pClass = NULL, *pInst = NULL; do { PyGILState_STATE state = PyGILState_Ensure(); { pModule = PyImport_ImportModule("worker"); if (pModule == NULL) break; pClass = PyObject_GetAttrString(pModule, "ThreadManager"); if (pClass == NULL) break; pInst = PyObject_CallObject(pClass, NULL); if (pInst == NULL) break; PyObject_CallMethod(pInst, "start_thread", NULL); } PyGILState_Release(state); for (int i = 0; i < 5; i++) { printf("main thread is runningn"); sleep(1); } ...
  • 22. 1. PyEval_InitThreads(); 2. state = PyGILState_Ensure(); 3. <start thread in Python> 4. PyGILState_Release(state) 5. <main thread in C> main thread acquires GIL main thread acquires GIL main thread acquires GIL no GIL Python thread not run
  • 23. PyEval_InitThreads(); ... PyThreadState* save = PyEval_SaveThread(); PyObject *pModule = NULL, *pClass = NULL, *pInst = NULL; do { PyGILState_STATE state = PyGILState_Ensure(); { pModule = PyImport_ImportModule("worker"); if (pModule == NULL) break; pClass = PyObject_GetAttrString(pModule, "ThreadManager"); if (pClass == NULL) break; pInst = PyObject_CallObject(pClass, NULL); if (pInst == NULL) break; PyObject_CallMethod(pInst, "start_thread", NULL); } PyGILState_Release(state); for (int i = 0; i < 5; i++) { printf("main thread is runningn"); sleep(1); } ...
  • 24. 1. PyEval_InitThreads(); 2. save = PyEval_SaveThread(); 3. state = PyGILState_Ensure(); 4. <start thread in Python> 5. PyGILState_Release(state) 6. <main thread in C> main thread acquires GIL main thread releases GIL main thread acquires GIL main thread releases GIL no GIL Python thread run
  • 25. Multithreading - 2 • Create threads from C • Each C thread calls Python function C P
  • 26. int main(int argc, char* argv[]) { PyEval_InitThreads(); Py_Initialize(); PyObject* sysPath = PySys_GetObject((char*) "path"); PyList_Append(sysPath, PyString_FromString(".")); PyThreadState* save = PyEval_SaveThread(); pthread_t tid1, tid2; char* tname1 = "worker1"; char* tname2 = "worker2"; pthread_create(&tid1, NULL, &run_python_function, &tname1); pthread_create(&tid2, NULL, &run_python_function, &tname2); for (int i = 0; i < 5; i++) { printf("main thread is runningn"); sleep(1); } stop_event = 1; pthread_join(tid1, NULL); pthread_join(tid2, NULL); printf("finishn"); PyEval_RestoreThread(save); Py_Finalize(); pthread_exit(NULL); return 0; }
  • 27. void* run_python_function(void* arg) { PyGILState_STATE state = PyGILState_Ensure(); PyObject *pModule = NULL, *pFunc = NULL; do { pModule = PyImport_ImportModule("work"); if (pModule == NULL) break; pFunc = PyObject_GetAttrString(pModule, "working"); if (pFunc == NULL) break; char* name = *((char**) arg); while (!stop_event) { PyObject_CallFunction(pFunc, "s", name); } } while (0); Py_XDECREF(pFunc); Py_XDECREF(pModule); PyGILState_Release(state); pthread_exit(NULL); return NULL; } import time def working(name): print name, "is working" time.sleep(1)
  • 28. Callback Function • Asynchronous method invocation • Push event to C • Pass heavy computation to C
  • 29. static PyObject* callback(PyObject* self, PyObject* args) { long c; PyArg_ParseTuple(args, "l", &c); printf("Result of call: %ldn", c); Py_RETURN_NONE; } int main(int argc, char* argv[]) { ... PyObject *pModule = NULL, *pFunc = NULL, *pArgs = NULL; do { PyMethodDef CFunctions[] = { {"callback", callback, METH_VARARGS, ""}, {NULL, NULL, 0, NULL} }; Py_InitModule("cmodule", CFunctions); pModule = PyImport_ImportModule("func-module"); if (pModule == NULL) break; pFunc = PyObject_GetAttrString(pModule, "multiply"); if (pFunc == NULL) break; pArgs = Py_BuildValue("ii", a, b); if (pArgs == NULL) break; PyObject_Call(pFunc, pArgs, NULL); } while (0); ... }
  • 30. Create New Module static PyObject* callback(PyObject* self, PyObject* args) { long c; PyArg_ParseTuple(args, "l", &c); printf("Result of call: %ldn", c); Py_RETURN_NONE; } ... PyMethodDef CFunctions[] = { {"callback", callback, METH_VARARGS, ""}, {NULL, NULL, 0, NULL} }; Py_InitModule("cmodule", CFunctions); ...
  • 31. Invoke Python Function ... pModule = PyImport_ImportModule("func-module"); if (pModule == NULL) break; pFunc = PyObject_GetAttrString(pModule, "multiply"); if (pFunc == NULL) break; pArgs = Py_BuildValue("ii", a, b); if (pArgs == NULL) break; PyObject_Call(pFunc, pArgs, NULL); ...
  • 32. Import and Use import cmodule def multiply(a, b): print "Will compute", a, "times", b c = 0 for i in range(0, a): c = c + b cmodule.callback(c)
  • 33. Callback to C ... static PyObject* callback(PyObject* self, PyObject* args) { long c; PyArg_ParseTuple(args, "l", &c); printf("Result of call: %ldn", c); Py_RETURN_NONE; } ...
  • 35. Pure Callback def multiply(a, b, callback): print "Will compute", a, "times", b c = 0 for i in range(0, a): c = c + b callback(c)
  • 36. ... PyObject *pCallbackFunc = NULL, *pModule = NULL, *pFunc = NULL, *pArgs = NULL; do { PyMethodDef CFunc = {"callback", callback, METH_VARARGS, ""}; pCallbackFunc = PyCFunction_New(&CFunc, NULL); if (pCallbackFunc == NULL) break; pModule = PyImport_ImportModule("func-callback"); if (pModule == NULL) break; pFunc = PyObject_GetAttrString(pModule, "multiply"); if (pFunc == NULL) break; pArgs = Py_BuildValue("iiO", a, b, pCallbackFunc); if (pArgs == NULL) break; PyObject_Call(pFunc, pArgs, NULL); } while (0); ... CFunction Object
  • 39. static PyObject* callback1(PyObject* self, PyObject* args) { char* str; PyArg_ParseTuple(args, "s", &str); printf("%s push eventn", str); Py_RETURN_NONE; } static PyObject* callback2(PyObject* self, PyObject* args) { char* str; PyArg_ParseTuple(args, "s", &str); printf("%s start heavy computingn", str); sleep(5); printf("%s end heavy computingn", str); Py_RETURN_NONE; }
  • 40. static PyObject* callback1(PyObject* self, PyObject* args) { char* str; PyArg_ParseTuple(args, "s", &str); printf("%s push eventn", str); Py_RETURN_NONE; } static PyObject* callback2(PyObject* self, PyObject* args) { char* str; PyArg_ParseTuple(args, "s", &str); printf("%s start heavy computingn", str); PyThreadState* save = PyEval_SaveThread(); { sleep(5); } PyEval_RestoreThread(save); printf("%s end heavy computingn", str); Py_RETURN_NONE; }
  • 42. Summary • PyObject manipulation in C • GIL in multithreading • CFunction object as callback • Multithreading with callback