mirror of
https://github.com/iperov/DeepFaceLab.git
synced 2025-07-05 20:42:11 -07:00
here new whole_face + XSeg workflow: with XSeg model you can train your own mask segmentator for dst(and/or src) faces that will be used by the merger for whole_face. Instead of using a pretrained segmentator model (which does not exist), you control which part of faces should be masked. new scripts: 5.XSeg) data_dst edit masks.bat 5.XSeg) data_src edit masks.bat 5.XSeg) train.bat Usage: unpack dst faceset if packed run 5.XSeg) data_dst edit masks.bat Read tooltips on the buttons (en/ru/zn languages are supported) mask the face using include or exclude polygon mode. repeat for 50/100 faces, !!! you don't need to mask every frame of dst only frames where the face is different significantly, for example: closed eyes changed head direction changed light the more various faces you mask, the more quality you will get Start masking from the upper left area and follow the clockwise direction. Keep the same logic of masking for all frames, for example: the same approximated jaw line of the side faces, where the jaw is not visible the same hair line Mask the obstructions using exclude polygon mode. run XSeg) train.bat train the model Check the faces of 'XSeg dst faces' preview. if some faces have wrong or glitchy mask, then repeat steps: run edit find these glitchy faces and mask them train further or restart training from scratch Restart training of XSeg model is only possible by deleting all 'model\XSeg_*' files. If you want to get the mask of the predicted face (XSeg-prd mode) in merger, you should repeat the same steps for src faceset. New mask modes available in merger for whole_face: XSeg-prd - XSeg mask of predicted face -> faces from src faceset should be labeled XSeg-dst - XSeg mask of dst face -> faces from dst faceset should be labeled XSeg-prd*XSeg-dst - the smallest area of both if workspace\model folder contains trained XSeg model, then merger will use it, otherwise you will get transparent mask by using XSeg-* modes. Some screenshots: XSegEditor: https://i.imgur.com/7Bk4RRV.jpg trainer : https://i.imgur.com/NM1Kn3s.jpg merger : https://i.imgur.com/glUzFQ8.jpg example of the fake using 13 segmented dst faces : https://i.imgur.com/wmvyizU.gifv
152 lines
No EOL
4.2 KiB
Python
152 lines
No EOL
4.2 KiB
Python
import numpy as np
|
|
import cv2
|
|
from enum import IntEnum
|
|
|
|
|
|
class SegIEPolyType(IntEnum):
|
|
EXCLUDE = 0
|
|
INCLUDE = 1
|
|
|
|
|
|
|
|
class SegIEPoly():
|
|
def __init__(self, type=None, pts=None, **kwargs):
|
|
self.type = type
|
|
|
|
if pts is None:
|
|
pts = np.empty( (0,2), dtype=np.float32 )
|
|
else:
|
|
pts = np.float32(pts)
|
|
self.pts = pts
|
|
self.n_max = self.n = len(pts)
|
|
|
|
def dump(self):
|
|
return {'type': int(self.type),
|
|
'pts' : self.get_pts(),
|
|
}
|
|
|
|
def identical(self, b):
|
|
if self.n != b.n:
|
|
return False
|
|
return (self.pts[0:self.n] == b.pts[0:b.n]).all()
|
|
|
|
def get_type(self):
|
|
return self.type
|
|
|
|
def add_pt(self, x, y):
|
|
self.pts = np.append(self.pts[0:self.n], [ ( float(x), float(y) ) ], axis=0).astype(np.float32)
|
|
self.n_max = self.n = self.n + 1
|
|
|
|
def undo(self):
|
|
self.n = max(0, self.n-1)
|
|
return self.n
|
|
|
|
def redo(self):
|
|
self.n = min(len(self.pts), self.n+1)
|
|
return self.n
|
|
|
|
def redo_clip(self):
|
|
self.pts = self.pts[0:self.n]
|
|
self.n_max = self.n
|
|
|
|
def insert_pt(self, n, pt):
|
|
if n < 0 or n > self.n:
|
|
raise ValueError("insert_pt out of range")
|
|
self.pts = np.concatenate( (self.pts[0:n], pt[None,...].astype(np.float32), self.pts[n:]), axis=0)
|
|
self.n_max = self.n = self.n+1
|
|
|
|
def remove_pt(self, n):
|
|
if n < 0 or n >= self.n:
|
|
raise ValueError("remove_pt out of range")
|
|
self.pts = np.concatenate( (self.pts[0:n], self.pts[n+1:]), axis=0)
|
|
self.n_max = self.n = self.n-1
|
|
|
|
def get_last_point(self):
|
|
return self.pts[self.n-1].copy()
|
|
|
|
def get_pts(self):
|
|
return self.pts[0:self.n].copy()
|
|
|
|
def get_pts_count(self):
|
|
return self.n
|
|
|
|
def set_point(self, id, pt):
|
|
self.pts[id] = pt
|
|
|
|
def set_points(self, pts):
|
|
self.pts = np.array(pts)
|
|
self.n_max = self.n = len(pts)
|
|
|
|
|
|
|
|
|
|
class SegIEPolys():
|
|
def __init__(self):
|
|
self.polys = []
|
|
|
|
def identical(self, b):
|
|
polys_len = len(self.polys)
|
|
o_polys_len = len(b.polys)
|
|
if polys_len != o_polys_len:
|
|
return False
|
|
|
|
return all ([ a_poly.identical(b_poly) for a_poly, b_poly in zip(self.polys, b.polys) ])
|
|
|
|
def add_poly(self, ie_poly_type):
|
|
poly = SegIEPoly(ie_poly_type)
|
|
self.polys.append (poly)
|
|
return poly
|
|
|
|
def remove_poly(self, poly):
|
|
if poly in self.polys:
|
|
self.polys.remove(poly)
|
|
|
|
def has_polys(self):
|
|
return len(self.polys) != 0
|
|
|
|
def get_poly(self, id):
|
|
return self.polys[id]
|
|
|
|
def get_polys(self):
|
|
return self.polys
|
|
|
|
def get_pts_count(self):
|
|
return sum([poly.get_pts_count() for poly in self.polys])
|
|
|
|
def sort(self):
|
|
poly_by_type = { SegIEPolyType.EXCLUDE : [], SegIEPolyType.INCLUDE : [] }
|
|
|
|
for poly in self.polys:
|
|
poly_by_type[poly.type].append(poly)
|
|
|
|
self.polys = poly_by_type[SegIEPolyType.INCLUDE] + poly_by_type[SegIEPolyType.EXCLUDE]
|
|
|
|
def __iter__(self):
|
|
for poly in self.polys:
|
|
yield poly
|
|
|
|
def overlay_mask(self, mask):
|
|
h,w,c = mask.shape
|
|
white = (1,)*c
|
|
black = (0,)*c
|
|
for poly in self.polys:
|
|
pts = poly.get_pts().astype(np.int32)
|
|
if len(pts) != 0:
|
|
cv2.fillPoly(mask, [pts], white if poly.type == SegIEPolyType.INCLUDE else black )
|
|
|
|
def dump(self):
|
|
return {'polys' : [ poly.dump() for poly in self.polys ] }
|
|
|
|
@staticmethod
|
|
def load(data=None):
|
|
ie_polys = SegIEPolys()
|
|
if data is not None:
|
|
if isinstance(data, list):
|
|
# Backward comp
|
|
ie_polys.polys = [ SegIEPoly(type=type, pts=pts) for (type, pts) in data ]
|
|
elif isinstance(data, dict):
|
|
ie_polys.polys = [ SegIEPoly(**poly_cfg) for poly_cfg in data['polys'] ]
|
|
|
|
ie_polys.sort()
|
|
|
|
return ie_polys |