largefiles in WindowsBuilder.py

build/windows/WindowsBuilder.py

@@ -348,6 +348,11 @@ cmd
     def install_pip_package(self, pkg_name):
         subprocess.Popen(args=f'python.exe -m pip install {pkg_name}', cwd=str(self.python_path), shell=True, env=self.env).wait()
 
+    def run_python(self, argsline, cwd=None):
+        if cwd is None:
+            cwd = self.python_path
+        subprocess.Popen(args=f'python.exe {argsline}', cwd=str(cwd), shell=True, env=self.env).wait()
+
     def install_ffmpeg_binaries(self):
         print('Installing ffmpeg binaries.')
         self.ffmpeg_path.mkdir(exist_ok=True, parents=True)
@@ -407,9 +412,11 @@ fr'''{{
         "python.linting.pylamaEnabled": false,
         "python.linting.pydocstyleEnabled": false,
         "python.defaultInterpreterPath": "${{env:PYTHON_EXECUTABLE}}",
+        "telemetry.enableTelemetry": false,
 		"workbench.editor.tabCloseButton": "off",
 		"workbench.editor.tabSizing": "shrink",
 		"workbench.editor.highlightModifiedTabs": true,
+        "workbench.enableExperiments": false,
 		"editor.mouseWheelScrollSensitivity": 3,
 		"editor.folding": false,
 		"editor.glyphMargin": false,
@@ -486,9 +493,11 @@ def build_deepfacelive_windows(release_dir, cache_dir, python_ver='3.7.9'):
         print (f'Moving {target}')
         shutil.move (str(file), str(target) )
 
+    deepfacelive_path = builder.get_internal_path() / 'DeepFaceLive'
+
     print('Copying DeepFaceLive repository.')
-    builder.copyfiletree(Path(__file__).parent.parent.parent, builder.get_internal_path() / 'DeepFaceLive')
-    builder.rmdir_in_all_subdirs(builder.get_internal_path() / 'DeepFaceLive', '.git')
+    builder.copyfiletree(Path(__file__).parent.parent.parent, deepfacelive_path)
+    builder.rmdir_in_all_subdirs(deepfacelive_path, '.git')
 
     builder.install_vscode(project_internal_dir='DeepFaceLive')
 
@@ -505,6 +514,8 @@ def build_deepfacelive_windows(release_dir, cache_dir, python_ver='3.7.9'):
     builder.create_run_python_script('DeepFaceLive.bat', 'DeepFaceLive\\main.py', 'run DeepFaceLive --userdata-dir=%~dp0userdata')
     builder.create_internal_run_python_script('build DeepFaceLive.bat','DeepFaceLive\\build\\windows\\WindowsBuilder.py', '--build-type dfl-windows --release-dir Builds\DeepFaceLive --cache-dir _cache' )
 
+    builder.run_python('main.py dev merge_large_files --delete-parts', cwd=deepfacelive_path)
+
     builder.cleanup()
 
     builder.pack_sfx_release(f'DeepFaceLive_build_{datetime.now().strftime("%m_%d_%Y")}')
@@ -516,7 +527,7 @@ class fixPathAction(argparse.Action):
 
 if __name__ == '__main__':
     p = argparse.ArgumentParser()
-    p.add_argument('--build-type', required=True, choices=['dfl-windows'])#,'dfl-windows-installer'])
+    p.add_argument('--build-type', required=True, choices=['dfl-windows'])
     p.add_argument('--release-dir', action=fixPathAction, default=None)
     p.add_argument('--cache-dir', action=fixPathAction, default=None)
     p.add_argument('--python-ver', default="3.7.9")

main.py

@@ -31,15 +31,23 @@ def main():
     p.set_defaults(func=run_DeepFaceLive)
 
 
-    misc_parser = subparsers.add_parser("misc")
-    misc_subparsers = misc_parser.add_subparsers()
+    dev_parser = subparsers.add_parser("dev")
+    dev_subparsers = dev_parser.add_subparsers()
     
     def run_split_large_files(args):
-        from misc.split_large_files import split_large_files
-        split_large_files()
+        from scripts import dev
+        dev.split_large_files()
     
-    p = misc_subparsers.add_parser('split_large_files')
+    p = dev_subparsers.add_parser('split_large_files')
     p.set_defaults(func=run_split_large_files)
+    
+    def run_merge_large_files(args):
+        from scripts import dev
+        dev.merge_large_files(delete_parts=args.delete_parts)
+    
+    p = dev_subparsers.add_parser('merge_large_files')
+    p.add_argument('--delete-parts', action="store_true", default=False)
+    p.set_defaults(func=run_merge_large_files)
 
     def bad_args(arguments):
         parser.print_help()

misc/split_large_files.py

@@ -1,16 +0,0 @@
-from pathlib import Path
-from xlib.file import SplittedFile
-
-def split_large_files():
-    repo_root = Path(__file__).parent.parent
-    
-    files_list = [ (repo_root / 'modelhub' / 'onnx' / 'S3FD' / 'S3FD.onnx', 48*1024*1024), 
-                   (repo_root / 'modelhub' / 'torch' / 'S3FD' / 'S3FD.pth', 48*1024*1024), 
-                   (repo_root / 'modelhub' / 'cv' / 'FaceMarkerLBF' / 'lbfmodel.yaml', 34*1024*1024), 
-                 ]
-                 
-    for filepath, part_size in files_list:
-        print(f'Splitting {filepath}...')
-        SplittedFile.split(filepath, part_size=part_size, delete_original=False)
-    
-    print('Done')

scripts/dev.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+from xlib.file import SplittedFile
+
+repo_root = Path(__file__).parent.parent
+large_files_list = [ (repo_root / 'modelhub' / 'onnx' / 'S3FD' / 'S3FD.onnx', 48*1024*1024), 
+                     (repo_root / 'modelhub' / 'torch' / 'S3FD' / 'S3FD.pth', 48*1024*1024), 
+                     (repo_root / 'modelhub' / 'cv' / 'FaceMarkerLBF' / 'lbfmodel.yaml', 34*1024*1024), 
+                    ]
+                 
+def merge_large_files(delete_parts=False):
+    for filepath, _ in large_files_list:
+        print(f'Merging {filepath}...')
+        SplittedFile.merge(filepath, delete_parts=delete_parts)
+    print('Done')
+    
+def split_large_files(delete_original=False):
+    for filepath, part_size in large_files_list:
+        print(f'Splitting {filepath}...')
+        SplittedFile.split(filepath, part_size=part_size, delete_original=delete_original)
+    print('Done')

xlib/file/SplittedFile.py

@@ -51,8 +51,6 @@ class SplittedFile:
         ...
         merged to filename.ext
         """
-        if filepath.exists():
-            return
         
         parts : List[Path] = []
         for n in itertools.count(start=0):
@@ -63,13 +61,14 @@ class SplittedFile:
                 break
         
         if len(parts) != 0:
-            bytes_parts = []
-            for part_filepath in parts:
-                bytes_parts.append( part_filepath.read_bytes() )
-            
-            b = b''.join(bytes_parts)
-            
-            filepath.write_bytes(b)
+            if not filepath.exists():
+                bytes_parts = []
+                for part_filepath in parts:
+                    bytes_parts.append( part_filepath.read_bytes() )
+                
+                b = b''.join(bytes_parts)
+                
+                filepath.write_bytes(b)
             
             if delete_parts:
                 for part_filepath in parts: