Commit c10e82c4 authored by Paul McCarthy's avatar Paul McCarthy 🚵
Browse files

RF: Process results are only saved to disk if using multiprocessing

parent e53f3bd8
......@@ -220,11 +220,14 @@ def runParallelProcess(proc, dtable, vids, workDir):
# New columns are saved to disk,
# rather than being copied back
# to the parent process.
for i, series in enumerate(add):
fname = op.join(workDir, '{}.pkl'.format(i))
add[i] = fname
series.to_pickle(fname)
# to the parent process. We only
# do this if running in a
# multiprocessing context
if not util.inMainProcess():
for i, series in enumerate(add):
fname = op.join(workDir, '{}.pkl'.format(i))
add[i] = fname
series.to_pickle(fname)
return dtable, (remove, add, addvids, addmeta)
......
......@@ -18,7 +18,8 @@ import logging
import warnings
import functools
import contextlib
import subprocess as sp
import subprocess as sp
import multiprocessing as mp
try:
import resource
......@@ -215,6 +216,19 @@ def cat(files, outfile):
break
def inMainProcess():
"""Returns ``True`` if the running process is the main (parent) process.
Returns ``False`` if the running process is a child process (e.g. a
``multiprocessingg`` worker process).
"""
# see docs for mp.Process.name -
# child processes will have a
# name matching this pattern
p = mp.current_process()
return not re.fullmatch(r'Proceess-\d+', p.name)
@contextlib.contextmanager
def timed(op=None, logger=None, lvl=None, fmt=None, minutes=None):
"""Context manager which times a section of code, and prints a log
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment