I have issues regarding my own implementation of a parellized database using the TinyDB and multiprocessing libs in python. It always give errors, such as this one:
"c:\Users\1765536\AppData\Local\Programs\Python\Python313\Projects\db.py", line 53, in <module>
print(db.search({'name': 'test'}))
~~~~~~~~~^^^^^^^^^^^^^^^^^^
File "c:\Users\1765536\AppData\Local\Programs\Python\Python313\Projects\db.py", line 34, in search if awnser := DBProcess(table, 'search', query).start():
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
File "c:\Users\1765536\AppData\Local\Programs\Python\Python313\Projects\db.py", line 17, in start
super().start()
~~~~~~~~~~~~~^^
File "C:\Users\1765536\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
~~~~~~~~~~~^^^^^^
File "C:\Users\1765536\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
File "C:\Users\1765536\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\context.py", line 337, in _Popen
return Popen(process_obj)
File "C:\Users\1765536\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\popen_spawn_win32.py", line 97, in __init__
reduction.dump(process_obj, to_child)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\1765536\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\reduction.py",
line 60, in dump
ForkingPickler(file, protocol).dump(obj)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^
TypeError: cannot pickle 'TextIOWrapper' instances
PS C:\Users\1765536\AppData\Local\Programs\Python\Python313> Traceback (most recent call last):
File "<string>", line 1, in <module>
from multiprocessing.spawn import spawn_main; spawn_main(parent_pid=476, pipe_handle=264)
~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\1765536\AppData\Local\Programs\Python\Python313\Lib\multiprocessing\spawn.py", line 108, in spawn_main
source_process = _winapi.OpenProcess(
_winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE,
False, parent_pid)
OSError: [WinError 87] Paramètre incorrect
Here's my code for reference:
from typing import Literal, override, Optional
import tinydb as tdb
import multiprocessing as mp
class DBProcess(mp.Process):
def __init__(self, table: tdb.table.Table, operation: Literal['get', 'search'], data):
super().__init__()
self.table = table
self.operation = operation
self.data = data
def run(self):
if self.operation == 'get':
self.awnser = self.table.get(self.data)
elif self.operation == 'search':
self.awnser = self.table.search(self.data)
@override
def start(self) -> Optional[dict]:
super().start()
return self.awnser
class DB:
def __init__(self, db_path, num_tables: int):
self.db = tdb.TinyDB(db_path)
self.tables = [self.db.table(f'table_{i}') for i in range(num_tables)]
def insert(self, data):
for table in self.tables:
if len(table) < 1000:
table.insert(data)
break
else:
continue
def search(self, query) -> Optional[dict]:
for table in self.tables:
if awnser := DBProcess(table, 'search', query).start():
return awnser
else:
continue
return None
def get(self, id) -> Optional[dict]:
for table in self.tables:
if awnser := DBProcess(table, 'get', id).start():
return awnser
else:
continue
return None
#test
if __name__ == '__main__':
db = DB('test.json', 10)
db.insert({'name': 'test', 'age': 10})
print(db.search({'name': 'test'}))
print(db.get(1))
I was executing some test when the errors happened. It may have to do with something low-level, and I don't want my hand into that.
I've also got this one testing on another computer:
PS C:\Users\jupiter\My code projects> & C:/Users/jupiter/AppData/Local/Microsoft/WindowsApps/python3.13.exe "c:/Users/jupiter/My code projects/test/program.py"
Traceback (most recent call last):
File "c:\Users\jupiter\My code projects\test\program.py", line 53, in <module>
print(db.search({'name': 'test'}))
~~~~~~~~~^^^^^^^^^^^^^^^^^^
File "c:\Users\jupiter\My code projects\test\program.py", line 34, in search
if awnser := DBProcess(table, 'search', query).start():
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
File "c:\Users\jupiter\My code projects\test\program.py", line 17, in start
super().start()
~~~~~~~~~~~~~^^
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.752.0_x64__qbz5n2kfra8p0\Lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
~~~~~~~~~~~^^^^^^
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.752.0_x64__qbz5n2kfra8p0\Lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.752.0_x64__qbz5n2kfra8p0\Lib\multiprocessing\context.py", line 337, in _Popen
return Popen(process_obj)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.752.0_x64__qbz5n2kfra8p0\Lib\multiprocessing\popen_spawn_win32.py", line 97, in __init__
reduction.dump(process_obj, to_child)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.752.0_x64__qbz5n2kfra8p0\Lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^
TypeError: cannot pickle 'TextIOWrapper' instances
PS C:\Users\jupiter\My code projects> Traceback (most recent call last):
File "<string>", line 1, in <module>
from multiprocessing.spawn import spawn_main; spawn_main(parent_pid=19200, pipe_handle=1076)
~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.752.0_x64__qbz5n2kfra8p0\Lib\multiprocessing\spawn.py", line 113, in spawn_main
new_handle = reduction.duplicate(pipe_handle,
source_process=source_process)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.13_3.13.752.0_x64__qbz5n2kfra8p0\Lib\multiprocessing\reduction.py", line 79, in duplicate
return _winapi.DuplicateHandle(
~~~~~~~~~~~~~~~~~~~~~~~^
source_process, handle, target_process,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
0, inheritable, _winapi.DUPLICATE_SAME_ACCESS)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
PermissionError: [WinError 5] Accès refusé
May you give me some directions, please?