DeepFaceLab/gpufmkmgr/gpufmkmgr.py

import os
import sys
import contextlib

from utils import std_utils
from .pynvml import *


        
dlib_module = None
def import_dlib(device_idx, cpu_only=False):
    global dlib_module
    if dlib_module is not None:
        raise Exception ('Multiple import of dlib is not allowed, reorganize your program.')
        
    import dlib
    dlib_module = dlib
    if not cpu_only:
        dlib_module.cuda.set_device(device_idx)    
    return dlib_module

tf_module = None
tf_session = None
keras_module = None
keras_contrib_module = None
keras_vggface_module = None

def set_prefer_GPUConfig(gpu_config):
    global prefer_GPUConfig
    prefer_GPUConfig = gpu_config
    
def get_tf_session():
    global tf_session
    return tf_session

def import_tf( gpu_config = None ):
    global prefer_GPUConfig
    global tf_module
    global tf_session
    
    if gpu_config is None:
        gpu_config = prefer_GPUConfig
    else:
        prefer_GPUConfig = gpu_config
        
    if tf_module is not None:
        return tf_module

    if 'TF_SUPPRESS_STD' in os.environ.keys() and os.environ['TF_SUPPRESS_STD'] == '1':
        suppressor = std_utils.suppress_stdout_stderr().__enter__()
    else:
        suppressor = None

    if 'CUDA_VISIBLE_DEVICES' in os.environ.keys():
        os.environ.pop('CUDA_VISIBLE_DEVICES')
    
    os.environ['TF_MIN_GPU_MULTIPROCESSOR_COUNT'] = '2'
    
    import tensorflow as tf
    tf_module = tf
    
    if gpu_config.cpu_only:
        config = tf_module.ConfigProto( device_count = {'GPU': 0} )
    else:    
        config = tf_module.ConfigProto()
        visible_device_list = ''
        for idx in gpu_config.gpu_idxs: visible_device_list += str(idx) + ','
        visible_device_list = visible_device_list[:-1]
        config.gpu_options.visible_device_list=visible_device_list
        config.gpu_options.force_gpu_compatible = True
        
    config.gpu_options.allow_growth = gpu_config.allow_growth
    
    tf_session = tf_module.Session(config=config)
        
    if suppressor is not None:  
        suppressor.__exit__()

    return tf_module

def finalize_tf():
    global tf_module
    global tf_session
    
    tf_session.close()
    tf_session = None
    tf_module = None

def get_keras():
    global keras_module
    return keras_module
    
def import_keras():
    global keras_module
    
    if keras_module is not None:
        return keras_module
        
    sess = get_tf_session()
    if sess is None:
        raise Exception ('No TF session found. Import TF first.')
        
    if 'TF_SUPPRESS_STD' in os.environ.keys() and os.environ['TF_SUPPRESS_STD'] == '1':
        suppressor = std_utils.suppress_stdout_stderr().__enter__()
        
    import keras     

    keras.backend.tensorflow_backend.set_session(sess)
    
    if 'TF_SUPPRESS_STD' in os.environ.keys() and os.environ['TF_SUPPRESS_STD'] == '1':        
        suppressor.__exit__()

    keras_module = keras
    return keras_module
    
def finalize_keras():
    global keras_module
    keras_module.backend.clear_session()
    keras_module = None
    
def import_keras_contrib():
    global keras_contrib_module
    
    if keras_contrib_module is not None:
        raise Exception ('Multiple import of keras_contrib is not allowed, reorganize your program.')
    import keras_contrib
    keras_contrib_module = keras_contrib
    return keras_contrib_module
    
def finalize_keras_contrib():
    global keras_contrib_module
    keras_contrib_module = None
    
def import_keras_vggface(optional=False):
    global keras_vggface_module
    
    if keras_vggface_module is not None:
        raise Exception ('Multiple import of keras_vggface_module is not allowed, reorganize your program.')

    try:
        import keras_vggface
    except:
        if optional:
            print ("Unable to import keras_vggface. It will not be used.")
        else:
            raise Exception ("Unable to import keras_vggface.")
        keras_vggface = None
        
    keras_vggface_module = keras_vggface
    return keras_vggface_module
    
def finalize_keras_vggface():
    global keras_vggface_module
    keras_vggface_module = None    

def hasNVML():
    try:
        nvmlInit()
        nvmlShutdown()
    except:
        return False
    return True    
 
#returns [ (device_idx, device_name), ... ]
def getDevicesWithAtLeastFreeMemory(freememsize):
    result = []
    
    nvmlInit()
    for i in range(0, nvmlDeviceGetCount() ):
        handle = nvmlDeviceGetHandleByIndex(i)
        memInfo = nvmlDeviceGetMemoryInfo( handle )
        if (memInfo.total - memInfo.used) >= freememsize:
            result.append (i)
        
    nvmlShutdown()
        
    return result
    
def getDevicesWithAtLeastTotalMemoryGB(totalmemsize_gb):
    result = []
    
    nvmlInit()
    for i in range(0, nvmlDeviceGetCount() ):
        handle = nvmlDeviceGetHandleByIndex(i)
        memInfo = nvmlDeviceGetMemoryInfo( handle )
        if (memInfo.total) >= totalmemsize_gb*1024*1024*1024:
            result.append (i)
        
    nvmlShutdown()
        
    return result
def getAllDevicesIdxsList ():
    nvmlInit()    
    result = [ i for i in range(0, nvmlDeviceGetCount() ) ]    
    nvmlShutdown()        
    return result
    
def getDeviceVRAMFree (idx):
    result = 0
    nvmlInit()
    if idx < nvmlDeviceGetCount():    
        handle = nvmlDeviceGetHandleByIndex(idx)
        memInfo = nvmlDeviceGetMemoryInfo( handle )
        result = (memInfo.total - memInfo.used)        
    nvmlShutdown()
    return result
    
def getDeviceVRAMTotalGb (idx):
    result = 0
    nvmlInit()
    if idx < nvmlDeviceGetCount():    
        handle = nvmlDeviceGetHandleByIndex(idx)
        memInfo = nvmlDeviceGetMemoryInfo( handle )
        result = memInfo.total / (1024*1024*1024)
    nvmlShutdown()
    return round(result)
    
def getBestDeviceIdx():
    nvmlInit()    
    idx = -1
    idx_mem = 0
    for i in range(0, nvmlDeviceGetCount() ):
        handle = nvmlDeviceGetHandleByIndex(i)
        memInfo = nvmlDeviceGetMemoryInfo( handle )
        if memInfo.total > idx_mem:
            idx = i
            idx_mem = memInfo.total

    nvmlShutdown()
    return idx
    
def getWorstDeviceIdx():
    nvmlInit()    
    idx = -1
    idx_mem = sys.maxsize
    for i in range(0, nvmlDeviceGetCount() ):
        handle = nvmlDeviceGetHandleByIndex(i)
        memInfo = nvmlDeviceGetMemoryInfo( handle )
        if memInfo.total < idx_mem:
            idx = i
            idx_mem = memInfo.total

    nvmlShutdown()
    return idx
    
def isValidDeviceIdx(idx):
    nvmlInit()    
    result = (idx < nvmlDeviceGetCount())
    nvmlShutdown()
    return result
    
def getDeviceIdxsEqualModel(idx):
    result = []
    
    nvmlInit()    
    idx_name = nvmlDeviceGetName(nvmlDeviceGetHandleByIndex(idx)).decode()

    for i in range(0, nvmlDeviceGetCount() ):
        if nvmlDeviceGetName(nvmlDeviceGetHandleByIndex(i)).decode() == idx_name:
            result.append (i)
         
    nvmlShutdown()
    return result
    
def getDeviceName (idx):
    result = ''
    nvmlInit()    
    if idx < nvmlDeviceGetCount():    
        result = nvmlDeviceGetName(nvmlDeviceGetHandleByIndex(idx)).decode()
    nvmlShutdown()
    return result
    
    
class GPUConfig():    
    force_best_gpu_idx = -1
    multi_gpu = False
    force_gpu_idxs = None
    choose_worst_gpu = False
    gpu_idxs = []
    gpu_total_vram_gb = 0
    allow_growth = True
    cpu_only = False
    
    def __init__ (self, force_best_gpu_idx = -1, 
                        multi_gpu = False, 
                        force_gpu_idxs = None, 
                        choose_worst_gpu = False,
                        allow_growth = True,
                        cpu_only = False,
                        **in_options):
        if not hasNVML():
            cpu_only = True
            
        if cpu_only:
            self.cpu_only = cpu_only
        else:
            self.force_best_gpu_idx = force_best_gpu_idx
            self.multi_gpu = multi_gpu
            self.force_gpu_idxs = force_gpu_idxs
            self.choose_worst_gpu = choose_worst_gpu        
            self.allow_growth = allow_growth
      
            gpu_idx = force_best_gpu_idx if (force_best_gpu_idx >= 0 and isValidDeviceIdx(force_best_gpu_idx)) else getBestDeviceIdx() if not choose_worst_gpu else getWorstDeviceIdx()

            if force_gpu_idxs is not None:
                self.gpu_idxs = [ int(x) for x in force_gpu_idxs.split(',') ]
            else:
                if self.multi_gpu:
                    self.gpu_idxs = getDeviceIdxsEqualModel( gpu_idx )
                    if len(self.gpu_idxs) <= 1:
                        self.multi_gpu = False
                else:
                    self.gpu_idxs = [gpu_idx]
            
            self.gpu_total_vram_gb = getDeviceVRAMTotalGb ( self.gpu_idxs[0] )
        
prefer_GPUConfig = GPUConfig()