CODEWITHSUNDEEP
pythonparallel-processingpicklepartialtqdm
OJT
OJT

Reputation: 899

Python pickling SSLContext TypeError when using tqdm

I have a generic method for handling parallelization using p_tqdm (similar to tqdm) and functools.partial to handle multiple input args, like:

from functools import partial
from p_tqdm import p_umap

def thread_multi(function, non_iterable_args: tuple, iterable_args: tuple):
    results = list()
    if non_iterable_args:
        func = partial(function, *non_iterable_args)
        for result in p_umap(func, *iterable_args):
            if result and isinstance(result, list):
                results.extend([r for r in result if r])
            elif result:
                results.append(result)

When I try to call this method I am generally passing something like a string as a non-iterable arg, and a list of dicts as an iterable arg, like:

def some_function(d, r):
  r["k3"] = d
  return r

d = "2023-03-01"
records = [
  {"k1": "v1", "k2": "v2"}, 
  {"k1": "v1", "k2": "v2"}, 
  ...
]

records = parallelize.thread_multi(
  function=some_function,
  non_iterable_args=(d,),
  iterable_args=(records,)
)

Doing this will often return the following error:

  0%|                                                                                       | 0/21 [00:00<?, ?it/s]
Traceback (most recent call last):
  File "process.py", line 70, in <module>
    processor.run()
  File "process.py", line 32, in run
    records = parallelize.thread_multi(
  File "/dev/utils/parallelize.py", line 28, in thread_multi
    for result in p_umap(func, *iterables):
  File "/usr/local/lib/python3.8/site-packages/p_tqdm/p_tqdm.py", line 84, in p_umap
    result = list(generator)
  File "/usr/local/lib/python3.8/site-packages/p_tqdm/p_tqdm.py", line 54, in _parallel
    for item in tqdm_func(map_func(function, *iterables), total=length, **kwargs):
  File "/usr/local/lib/python3.8/site-packages/tqdm/std.py", line 1195, in __iter__
    for obj in iterable:
  File "/usr/local/lib/python3.8/site-packages/multiprocess/pool.py", line 868, in next
    raise value
  File "/usr/local/lib/python3.8/site-packages/multiprocess/pool.py", line 537, in _handle_tasks
    put(task)
  File "/usr/local/lib/python3.8/site-packages/multiprocess/connection.py", line 209, in send
    self._send_bytes(_ForkingPickler.dumps(obj))
  File "/usr/local/lib/python3.8/site-packages/multiprocess/reduction.py", line 54, in dumps
    cls(buf, protocol, *args, **kwds).dump(obj)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 394, in dump
    StockPickler.dump(self, obj)
  File "/usr/local/lib/python3.8/pickle.py", line 487, in dump
    self.save(obj)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/pickle.py", line 901, in save_tuple
    save(element)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1824, in save_function
    _save_with_postproc(pickler, (_create_function, (
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1089, in _save_with_postproc
    pickler.save_reduce(*reduction)
  File "/usr/local/lib/python3.8/pickle.py", line 692, in save_reduce
    save(args)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 603, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/local/lib/python3.8/pickle.py", line 692, in save_reduce
    save(args)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1427, in save_instancemethod0
    pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
  File "/usr/local/lib/python3.8/pickle.py", line 692, in save_reduce
    save(args)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 603, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/local/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/local/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/local/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 603, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/local/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/local/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/local/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 603, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/local/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/local/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/local/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 603, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/local/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 1186, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/local/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/local/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/local/lib/python3.8/site-packages/dill/_dill.py", line 388, in save
    StockPickler.save(self, obj, save_persistent_id)
  File "/usr/local/lib/python3.8/pickle.py", line 578, in save
    rv = reduce(self.proto)
TypeError: cannot pickle 'SSLContext' object

Is this a datatype issue on the non-iterable arg(s)? If so, how does one resolve?

Upvotes: 1

Views: 220

Answers (0)

Related Questions