decreased time of training initialization

core/joblib/SubprocessGenerator.py

@@ -1,7 +1,28 @@
-import queue as Queue
 import multiprocessing
+import queue as Queue
+import threading
+import time
+
 
 class SubprocessGenerator(object):
+    
+    @staticmethod
+    def launch_thread(generator): 
+        generator._start()
+        
+    @staticmethod
+    def start_in_parallel( generator_list ):
+        """
+        Start list of generators in parallel
+        """
+        for generator in generator_list:
+            thread = threading.Thread(target=SubprocessGenerator.launch_thread, args=(generator,) )
+            thread.daemon = True
+            thread.start()
+
+        while not all ([generator._is_started() for generator in generator_list]):
+            time.sleep(0.005)
+    
     def __init__(self, generator_func, user_param=None, prefetch=2, start_now=True):
         super().__init__()
         self.prefetch = prefetch
@@ -17,10 +38,14 @@ class SubprocessGenerator(object):
         if self.p == None:
             user_param = self.user_param
             self.user_param = None
-            self.p = multiprocessing.Process(target=self.process_func, args=(user_param,) )
-            self.p.daemon = True
-            self.p.start()
-
+            p = multiprocessing.Process(target=self.process_func, args=(user_param,) )
+            p.daemon = True
+            p.start()
+            self.p = p
+            
+    def _is_started(self):
+        return self.p is not None
+        
     def process_func(self, user_param):
         self.generator_func = self.generator_func(user_param)
         while True: