pool map iterator
if __name__ == "__main__":
with arcpy.da.SearchCursor("c:\temp.gdb\test", fields=["*"]) as s_cursor:
pool = multiprocessing.Pool(processes=4) # lets use 4 workers
cursor_iterator = iterator_slice(s_cursor, 100) # slicer from above, for convinience
queue = [] # a queue for our current worker async results, a deque would be faster
while cursor_iterator or queue: # while we have anything to do...
try:
# add our next slice to the pool:
queue.append(pool.apply_async(insert, [next(cursor_iterator)]))
except (StopIteration, TypeError): # no more data, clear out the slice iterator
cursor_iterator = None
# wait for a free worker or until all remaining finish
while queue and (len(queue) >= pool._processes or not cursor_iterator):
process = queue.pop(0) # grab a process response from the top
process.wait(0.1) # let it breathe a little, 100ms should be enough
if not process.ready(): # a sub-process has not finished execution
queue.append(process) # add it back to the queue
else:
# you can use process.get() to get the result if needed
pass
pool.close()