initial commit

Go for MVC architecture

mainscripts/Trainer.py

@@ -216,6 +216,7 @@ def main(args, device_args):
         is_waiting_preview = False
         show_last_history_iters_count = 0
         iter = 0
+        autopreview = False
         while True:
             if not c2s.empty():
                 input = c2s.get()
@@ -233,7 +234,13 @@ def main(args, device_args):
                             max_h = max (max_h, h)
                             max_w = max (max_w, w)
 
-                        max_size = 800
+                        #Gan-er comment:
+                        #this is the preview window max possible width.
+                        #I will automate in in the future but for now you can make this as large as you want
+                        #provided that you change the number of previews in the model.py as well. (See comments there)
+                        max_size = 1600 
+                        
+                        
                         if max_h > max_size:
                             max_w = int( max_w / (max_h / max_size) )
                             max_h = max_size
@@ -246,6 +253,14 @@ def main(args, device_args):
                                 previews.remove(preview)
                                 previews.append ( (preview_name, cv2.resize(preview_rgb, (max_w, max_h))) )
                         selected_preview = selected_preview % len(previews)
+                        #Gan-er comment:
+                        #Auto preview implementation for the update preview obsessed users. 
+                        #G---START
+                        if autopreview == True:
+                            is_waiting_preview = True
+                            s2c.put ( {'op': 'preview'} )
+                        #G---END
+                        
                         update_preview = True
                 elif op == 'close':
                     break
@@ -260,9 +275,9 @@ def main(args, device_args):
                 # HEAD
                 head_lines = [
                     '[s]:save [enter]:exit',
-                    '[p]:update [space]:next preview [l]:change history range',
+                    '[p]:update [\]:autoupdate [}]:zoom out [{]:zoom in', #Gan-er: Update the key info presented to the user
-                    'Preview: "%s" [%d/%d]' % (selected_preview_name,selected_preview+1, len(previews) )
+                    'Preview: "%s" ' % (selected_preview_name )
-                    ]
+                    ] 
                 head_line_height = 15
                 head_height = len(head_lines) * head_line_height
                 head = np.ones ( (head_height,w,c) ) * 0.1
@@ -292,30 +307,101 @@ def main(args, device_args):
             key_events = io.get_key_events(wnd_name)
             key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[-1] if len(key_events) > 0 else (0,0,False,False,False)
             
+            #Gan-er comment:
+            #Change the zooming-in to taste. 
+            #Add remove change the values as you see fit. 
+            #The numbers in the show_last_history_iters_count simply tell you how many updates you want to see.
+            #So for example, 50 means the last 50 updates. All the previous ones will be pruned. 
+            #If you want to change the 50 to e.g. 75 go ahead and change the two '50' numbers below to '75', and so on.
+            #I will update this to be more sophisticated once I have the time. 
+            #The truth is that for a correct zooming implementation (where not just the last X updates are visible)
+            #but also the scale of the y-axis in the preview window changes the entire thing has to be refactored/converted
+            #using an MVC architecture. As it is now things that have to do with the presentation are intermingled with
+            #things that have to do with the modeling without any controllers so the whole thing is messy to deal with.
+            #Note that you need to be able to zoom-in/out in order to evaluate the model's effectiveness especially if you are
+            #doing more advanced stuff like warm restarts and what not (which you should be doing). For more details on wth 
+            #I am talking about you should check the wiki. [which at this moment does not exist] 
+            
+            #G---START
             if key == ord('\n') or key == ord('\r'):
-                s2c.put ( {'op': 'close'} )
-            elif key == ord('s'):
                 s2c.put ( {'op': 'save'} )
-            elif key == ord('p'):
+                s2c.put ( {'op': 'close'} )
+            elif key == ord('s') or key ==ord('S'):
+                s2c.put ( {'op': 'save'} )
+            elif key == ord('p') or key == ord('P'):
                 if not is_waiting_preview:
                     is_waiting_preview = True
                     s2c.put ( {'op': 'preview'} )
-            elif key == ord('l'):
+
+            elif key == ord('\\'):
+                if autopreview:
+                    autopreview = False
+                else:
+                    autopreview = True
+                    is_waiting_preview = True
+                    s2c.put ( {'op': 'preview'} )
+            elif key == ord(']'):
                 if show_last_history_iters_count == 0:
+                    show_last_history_iters_count = 50
+                elif show_last_history_iters_count == 50:
+                    show_last_history_iters_count = 150
+                elif show_last_history_iters_count == 150: 
+                    show_last_history_iters_count = 350
+                elif show_last_history_iters_count == 350:
+                    show_last_history_iters_count = 500    
+                elif show_last_history_iters_count == 500:
+                    show_last_history_iters_count = 620
+                elif show_last_history_iters_count == 620:
+                    show_last_history_iters_count = 1240
+                elif show_last_history_iters_count == 1240:
+                    show_last_history_iters_count = 2550
+                elif show_last_history_iters_count == 2550:
+                    show_last_history_iters_count = 3500
+                elif show_last_history_iters_count == 3500:
                     show_last_history_iters_count = 5000
                 elif show_last_history_iters_count == 5000:
+                    show_last_history_iters_count = 10000			                    
+                elif show_last_history_iters_count == 10000:
+                    show_last_history_iters_count = 15000	
+                elif show_last_history_iters_count == 15000:
+                    show_last_history_iters_count = 25000
+                elif show_last_history_iters_count == 25000:
+                    show_last_history_iters_count = 40000
+                elif show_last_history_iters_count == 40000:
+                    show_last_history_iters_count = 0			                    
+                update_preview = True
+            elif key == ord('['):
+                if show_last_history_iters_count == 0:
+                    show_last_history_iters_count = 40000
+                elif show_last_history_iters_count == 40000:
+                    show_last_history_iters_count = 25000
+                elif show_last_history_iters_count == 25000:
+                    show_last_history_iters_count = 15000
+                elif show_last_history_iters_count == 15000:
                     show_last_history_iters_count = 10000
                 elif show_last_history_iters_count == 10000:
-                    show_last_history_iters_count = 50000
+                    show_last_history_iters_count = 5000
-                elif show_last_history_iters_count == 50000:
+                elif show_last_history_iters_count == 5000:
-                    show_last_history_iters_count = 100000
+                    show_last_history_iters_count = 3500
-                elif show_last_history_iters_count == 100000:
+                elif show_last_history_iters_count == 3500:
-                    show_last_history_iters_count = 0
+                    show_last_history_iters_count = 2550
+                elif show_last_history_iters_count == 2550:
+                    show_last_history_iters_count = 1240
+                elif show_last_history_iters_count == 1240:
+                    show_last_history_iters_count = 620    
+                elif show_last_history_iters_count == 620:
+                    show_last_history_iters_count = 500
+                elif show_last_history_iters_count == 500:
+                    show_last_history_iters_count = 350
+                elif show_last_history_iters_count == 350:
+                    show_last_history_iters_count = 150
+                elif show_last_history_iters_count == 150:
+                    show_last_history_iters_count = 50
+                elif show_last_history_iters_count == 50:
+                    show_last_history_iters_count = 0	
+                                    
                 update_preview = True
-            elif key == ord(' '):
+            #G---END
-                selected_preview = (selected_preview + 1) % len(previews)
-                update_preview = True
-
             try:
                 io.process_messages(0.1)
             except KeyboardInterrupt:

models/ModelBase.py

@@ -485,10 +485,17 @@ class ModelBase(object):
                     self.batch_size = d[ keys[-1] ]
 
     @staticmethod
+    #Gan-er comment: This function has no business being here. Should be moved to a 'view' class. 
+    #The values that needed from the ModelBase class, should be passed through the use of a 'controler' class.
+    #i.e. The thing needs to be refactored in an MVC architecture.
     def get_loss_history_preview(loss_history, iter, w, c):
         loss_history = np.array (loss_history.copy())
 
+        #Gan-er comment:
+        #this is the height of the loss history window. You can change it to whatever you want as 
+        #long as it fits in your screen. 
         lh_height = 100
+        
         lh_img = np.ones ( (lh_height,w,c) ) * 0.1
         loss_count = len(loss_history[0])
         lh_len = len(loss_history)
@@ -513,13 +520,40 @@ class ModelBase(object):
                         ]
                         for col in range(w)
                     ]
-
+        
-        plist_abs_max = np.mean(loss_history[ len(loss_history) // 5 : ]) * 2
+        #Gan-er comment:
+        #The following lines affect the resolution/scale of the history window. Ideally they should NOT be in 
+        #this class, but on a utility view-class in a proper MVC architecture manner. 
+        #Refactoring can take care of this... In any case, I am commenting the lines so that you will know
+        #in the meantime what the settings do, so that you can adjust them to taste.
+        #The line below affects the maximum values that can appear in the y-axis of the loss history graph.
+        #Now the way that this is defined is by using the mean, i.e. the average of all the values and then scaling that average 
+        #by some amount.
+        #So as an example... if the average is 4 and I scale it by 2, then the maximum value that will be plotted within the 
+        #visible window, will be an 8. 
+        #Now the mean value, is obviously variable. And what values will be considered in its calculation is something that you can set it up. 
+        #You do so by defining how far back in history you would like to take the values into account. 
+        #The following line does that:
+        #The value should be within 0 and 1. 0 means take the entire history into account (ignore nothing). 1 means consider just the previous value. 
+        #0.25 means IGNORE the first 25% of the loss history.
+        #Why you need that is obvious. When you first start training, your losses will be really high. If you are doing restarts you may want to completely or partially ignore the previous losses. 
+        #So you may not want to consider those extreme values. Remember this affect the max value on the y-axis...i.e. it is a zoom issue.
+        percentage_of_history_to_ignore = 0.25 
+        scale_the_mean_by_this_much = 2 
+        #After refactoring, this can become a GUI option, e.g. mousewheel for the percentage of history and shift+mousewheel for the scale or something like that.
+        #For now just adjust the above two values to taste.
+        plist_abs_max = np.mean(loss_history[ int(len(loss_history)*percentage_of_history_to_ignore) : ]) * scale_the_mean_by_this_much
 
         for col in range(0, w):
             for p in range(0,loss_count):
                 point_color = [1.0]*c
-                point_color[0:3] = colorsys.hsv_to_rgb ( p * (1.0/loss_count), 1.0, 1.0 )
+                
+                #Gan-er comment:
+                #I changed the colors to something easier on the eyes visually at least for me. 
+                #If red/green is too festive/christmacy for you and you prefer iperov's yellow/blue instead 
+                #then delete the next line and uncomment the one after that. If I refactor I will make this a GUI option. 
+                point_color[0:3] = colorsys.hsv_to_rgb ( (p+1) * (1.0/3), 1.0, 1.0 )
+                #point_color[0:3] = colorsys.hsv_to_rgb ( p * (1.0/loss_count), 1.0, 1.0 )
 
                 ph_max = int ( (plist_max[col][p] / plist_abs_max) * (lh_height-1) )
                 ph_max = np.clip( ph_max, 0, lh_height-1 )
@@ -530,7 +564,8 @@ class ModelBase(object):
                 for ph in range(ph_min, ph_max+1):
                     lh_img[ (lh_height-ph-1), col ] = point_color
 
-        lh_lines = 5
+        
+        lh_lines = 5 #Gan-er comment: This setting is affecting the Y axis of the loss history graph. It determines how many 'segments' you want it to have.
         lh_line_height = (lh_height-1)/lh_lines
         for i in range(0,lh_lines+1):
             lh_img[ int(i*lh_line_height), : ] = (0.8,)*c
@@ -538,7 +573,7 @@ class ModelBase(object):
         last_line_t = int((lh_lines-1)*lh_line_height)
         last_line_b = int(lh_lines*lh_line_height)
 
-        lh_text = 'Iter: %d' % (iter) if iter != 0 else ''
+        lh_text = 'Iteration: %d' % (iter) if iter != 0 else ''
 
         lh_img[last_line_t:last_line_b, 0:w] += imagelib.get_text_image (  (last_line_b-last_line_t,w,c), lh_text, color=[0.8]*c )
         return lh_img

models/Model_SAE/Model.py

@@ -271,8 +271,37 @@ class SAEModel(ModelBase):
         psd_target_dst_anti_masked_ar = [ pred_src_dst_sigm_ar[i]*target_dstm_anti_sigm_ar[i]  for i in range(len(pred_src_dst_sigm_ar))]
 
         if self.is_training_mode:
-            self.src_dst_opt      = Adam(lr=5e-5, beta_1=0.5, beta_2=0.999, tf_cpu_mode=self.options['optimizer_mode']-1)
+            #Gan-er comment:
-            self.src_dst_mask_opt = Adam(lr=5e-5, beta_1=0.5, beta_2=0.999, tf_cpu_mode=self.options['optimizer_mode']-1)
+            #Adam related stuff. i.e. the optimizer ..the core/heart and brains of your model... is here.
+            #These values are simply TOO important to be set here, fixed and inaccessible. Ideally should be changeable from GUI.
+            #If I refactor I will implement that. 
+            #For now... you can change the following 3 values depending on what you want to do.
+            #I can not explain in depth what they do here in the comments, but I can give you an idea.
+            
+            #This one is the learning rate, i.e. once you determine the direction that you need to move in descent... how far you are going to go that way.
+            #For an explanation you will have to see the wiki (when filled) or just google info about the learning rate.
+            #Intuitively you want this to be as LARGE as possible especially when you start, and have it getting smaller as you progress in your training
+            #There are countless techniques and methods and strategies about altering this value as you train. If I have the time I will implement manual and automatic alterations of this value from the GUI.
+            #A good starting value is something between 3.5e-4 and 9e-5.
+            myLR = 2.49e-4
+            
+            #Adam the optimizer, already alters the steps as the training goes on. i.e. it has learning adjustment built-in in it. 
+            #The problem is that YOU [having an actual brain and all] are much much MUCH MUUUUUUCH better at judging which values should be used.
+            #Anyway... Adam adjust the learning rate by using the the following two variables. 
+            #The first is pretty much affecting the how far in the past into the training history Adam should be looking for considering the those values.
+            #For example 0 means do not look into the past values at all. Look at just the previous one. 
+            #The second one is similar but for the average value of the history. 
+            #When you start you want the first one to be 0.95 or 0.9... and the second one to be 0.999. In fact these are the default values and you should not be playing with them.
+            #With these values you sort of using Adam '100%' as a matter of speaking.
+            #If you were to change these values to 0, then you would not consider any history at all. The optimizer would simply consider the previous step. 
+            #That is equivalent to using SGD, stochastic gradient descent. 
+            #Values that are in between determine the 'nature' of the optimizer. Higher values make it more 'Adam' lower values make it more SGD.
+            #For more on that either google, or check the wiki.
+            myb1 = 0.95
+            myb2 = 0.999
+            self.src_dst_opt      = Adam(lr=myLR, beta_1=myb1, beta_2=myb2, tf_cpu_mode=self.options['optimizer_mode']-1)
+            self.src_dst_mask_opt = Adam(lr=myLR, beta_1=myb1, beta_2=myb2, tf_cpu_mode=self.options['optimizer_mode']-1)
+
 
             if 'liae' in self.options['archi']:
                 src_dst_loss_train_weights = self.encoder.trainable_weights + self.inter_B.trainable_weights + self.inter_AB.trainable_weights + self.decoder.trainable_weights
@@ -432,6 +461,9 @@ class SAEModel(ModelBase):
 
 
     #override
+    #Gan-er comment: This function has no business being here. Should be moved to a 'view' class. 
+    #The values that needed from the Model class, should be passed through the use of a 'controler' class.
+    #i.e. The thing needs to be refactored in an MVC architecture.
     def onGetPreview(self, sample):
         test_S   = sample[0][1][0:4] #first 4 samples
         test_S_m = sample[0][1+self.ms_count][0:4] #first 4 samples
@@ -447,7 +479,8 @@ class SAEModel(ModelBase):
         result = []
         st = []
         for i in range(0, len(test_S)):
-            ar = S[i], SS[i], D[i], DD[i], SD[i]
+            ar = S[i], SS[i], D[i], DD[i], SD[i] #Gan-er comment: this is where you can add/remove samples to extent the width of your window. Just add more of them to the variable ar, (and make sure that you adjust the for loop appropriately)             
+            
             st.append ( np.concatenate ( ar, axis=1) )
 
         result += [ ('SAE', np.concatenate (st, axis=0 )), ]
@@ -457,8 +490,9 @@ class SAEModel(ModelBase):
             for i in range(0, len(test_S)):
                 ar = S[i]*test_S_m[i], SS[i], D[i]*test_D_m[i], DD[i]*DDM[i], SD[i]*(DDM[i]*SDM[i])
                 st_m.append ( np.concatenate ( ar, axis=1) )
-
+            
-            result += [ ('SAE masked', np.concatenate (st_m, axis=0 )), ]
+            #Gan-er comment: I changed this in order to pass some extra useful information to the GUI
+            result += [ ('SAE ('+self.options['archi']+ ': '+str(self.options['ae_dims'])+','+str(self.options['ed_ch_dims'])+')  minibatch size: '+str(self.options['batch_size'])+' (add stuff here)', np.concatenate (st, axis=0 )), ]
 
         return result