# import onnx # import onnx.parser # import onnx.checker # from onnx import helper # from onnx import AttributeProto, TensorProto, GraphProto # import numpy as np # from xlib import onnxruntime as lib_ort # import cv2 # from io import BytesIO # _str_to_type = { # 'float32' : TensorProto.FLOAT, # 'bool' : TensorProto.BOOL, # } # class ONNXGraphBuilder: # """ # onnx graph builder. # Tensors are defined with unknown shape dimensions, # because onnx does not provide shape computation # for example: broadcasting [1,3,5] with [1,5], user should compute it manually and specify output shape of broadcast operation. # So the best way is to define tensors only with rank. # """ # def __init__(self): # self._nodes = [] # self._name_counter = 0 # def _generate_tensor_name(self, ): # result = f'T{self._name_counter}' # self._name_counter += 1 # return result # def _get_shape(self, t): # if isinstance(t, onnx.ValueInfoProto): # return [ x.dim_value for x in t.type.tensor_type.shape.dim ] # raise Exception() # def _get_ndim(self, t): # if isinstance(t, onnx.ValueInfoProto): # return len(t.type.tensor_type.shape.dim) # raise Exception() # def _get_data_type(self, t): # if isinstance(t, onnx.ValueInfoProto): # return t.type.tensor_type.elem_type # raise Exception() # def tensor(self, ndim, data_type='float32', name=None): # if name is None: # name = self._generate_tensor_name() # data_type = _str_to_type[data_type] # return helper.make_tensor_value_info(name, data_type, [-1]*ndim) # def constant(self, np_value : np.ndarray, data_type='float32', name=None): # output_t = self.tensor(ndim=np_value.ndim, name=name) # if isinstance(data_type, str): # data_type = _str_to_type[data_type] # node = helper.make_node('Constant', [], [output_t.name], # value=helper.make_tensor(name=self._generate_tensor_name(), data_type=data_type, dims=np_value.shape, vals=np_value.flatten() ) ) # self._nodes.append(node) # return output_t # def greater_equal(self, input_t, value_t, output_t=None): # input_ndim = self._get_ndim(input_t) # value_ndim = self._get_ndim(value_t) # if output_t is None: # output_t = self.tensor(ndim=max(input_ndim, value_ndim), data_type='bool') # self._nodes.append( helper.make_node('GreaterOrEqual', [input_t.name, value_t.name], [output_t.name]) ) # return output_t # def conv2d(self, input_t, kernel_t, output_t=None, auto_pad='SAME_LOWER', strides=[1,1]): # if output_t is None: # output_t = self.tensor(ndim=self._get_ndim(input_t)) # self._nodes.append( helper.make_node('Conv', [input_t.name, kernel_t.name], [output_t.name], auto_pad=auto_pad, strides=strides) ) # return output_t # def binary_erode(self, input_t, struct_np : np.ndarray): # """ # input_t NCHW tensor # struct_np np.ndarray # HW structuring element # """ # struct_t = self.constant(struct_np, data_type=self._get_data_type(input_t)) # import code # code.interact(local=dict(globals(), **locals())) # def make_model(self, inputs, outputs): # graph_def = helper.make_graph(self._nodes, 'graph', inputs, outputs) # model = helper.make_model(graph_def) # model.ir_version=7 # model.opset_import[0].version=14 # onnx.checker.check_model(model) # return model # gb = ONNXGraphBuilder() # # val1_t = gb.constant( np.ones( (4,4)) ) # # val2_t = gb.constant( np.ones( (1,1,4)) ) # # #val1_t = gb.tensor(ndim=2, name='I') # # #val2_t = gb.tensor(ndim=2, name='W') # # output_t = gb.greater_equal(val1_t, val2_t) # # model = gb.make_model([], [output_t]) # # sess = lib_ort.InferenceSession_with_device(model, lib_ort.get_cpu_device() ) # # #x = sess.run(None, {'I': np.ones( (4,4), np.float32), 'W': np.ones( (1,4), np.float32) })[0] # # x = sess.run(None, {})[0] # # import code # # code.interact(local=dict(globals(), **locals())) # img = np.ones( (5,5), np.uint8 ) # img = np.pad(img, ( (2,2), (1,1) )) # img[4,4] = 0 # img_er = cv2.dilate(img, el, iterations = 1 ) # input_t = gb.tensor(ndim=4, name='I') # #kernel_t = gb.tensor(ndim=4, name='W') # o = gb.binary_erode(input_t, el) # # output_t = gb.conv2d(input_t, kernel_t) # # model = gb.make_model([input_t, kernel_t], [output_t]) # # sess = lib_ort.InferenceSession_with_device(model, lib_ort.get_cpu_device() ) # # x = sess.run(None, {'I': img[None,None,...].astype(np.float32), 'W': el[None,None,...].astype(np.float32) })[0] # import numpy as np # import cv2 # el = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))#.astype(np.float32) # from xlib import tensorcl as tcl # x = tcl.get_available_devices_info() # #dev = tcl.get_device(x[0]) # #tcl.set_argault_device(x[1]) # #x = tcl.Tensor.from_value( np.ones( (2,2))*2) # tcl.test_all() # #x = tcl.Tensor( [4,2], initializer=tcl.initializer.RandomUniform() ) from collections import deque from queue import Queue from typing import Any, Union, Tuple def MTOrderedData(queue_size=0): """ Multithreaded ordered work. Ensures the order of work done by threads. returns (host,client) classes. """ h2c, c2h = Queue(maxsize=queue_size), Queue(maxsize=queue_size) host = _MTOrderedDataHost(h2c, c2h) cli = _MTOrderedDataClient(h2c, c2h) return host, cli class _MTOrderedDataHost: """ """ def __init__(self, h2c : Queue, c2h : Queue): self._h2c = h2c self._c2h = c2h self._counter = 0 self._sent_ids = deque() self._done_datas = {} def send(self, data): """ send the data to the clients """ if data is None: raise ValueError('data cannot be None') c = self._counter self._counter += 1 self._sent_ids.append(c) self._h2c.put( (c, data) ) def recv(self) -> Union[Any, None]: sent_ids = self._sent_ids if len(sent_ids) != 0: done_datas = self._done_datas while not self._c2h.empty(): id, data = self._c2h.get() done_datas[id] = data id = sent_ids[0] if id in done_datas: done_data = done_datas.pop(id) sent_ids.popleft() print('len(sent_ids) ', len(sent_ids)) return done_data return None class _MTOrderedDataClient: def __init__(self, h2c : Queue, c2h : Queue): self._h2c = h2c self._c2h = c2h def send(self, data_id, data): """ """ self._c2h.put( (data_id, data) ) def recv(self, wait=True) -> Tuple[int, Any]: """ returns ( data_id(int), data(Any) ) or None """ h2c = self._h2c if not wait and h2c.empty(): return None id, data = h2c.get() return id, data from xlib import time as lib_time import numpy as np import cv2 from typing import Union, Tuple from xlib.image import ImageProcessor # mat = def_mat = np.array([[ 8.5966533e-01, 8.3356246e-02, 1.9525000e+02 ],# # [-8.3356142e-02, 8.5966533e-01, 8.8052826e+01 ]], np.float32)# # is_cp = False # while True: # print('is_cp : ', is_cp) # img = cv2.imread(r'D:\DevelopPython\test\00000.png') # if is_cp: # img = cp.asarray(img) # is_cp = not is_cp # ip = ImageProcessor(img) # #ip.erode_blur(50,0) # ip.median_blur(5, 100) # #ip.degrade_resize( np.random.rand() ) # #ip.erode_blur(50, 50, fade_to_border=False) # #ip.resize( (500,500) ) # #ip.warpAffine(mat, 1920, 1080) # x = ip.get_image('HWC') # x = cp.asnumpy(x) # cv2.imshow('', x ) # cv2.waitKey(0) # import code # code.interact(local=dict(globals(), **locals())) # from xlib import opencl as lib_cl # import numpy as np # from xlib import time as lib_time # d = lib_cl.get_available_devices_info() # dev = lib_cl.get_device(d[0]) # img1_np = np.ones( (1920,1080,3), dtype=np.float32 ) # img2_np = np.ones( (1920,1080,3), dtype=np.float32 ) # img3_np = np.ones( (1920,1080,3), dtype=np.float32 ) # img1_t = dev.alloc ( size=1920*1080*3*4 ) # img2_t = dev.alloc ( size=1920*1080*3*4 ) # img3_t = dev.alloc ( size=1920*1080*3*4 ) # while True: # with lib_time.timeit(): # img1_t.set(img1_np) # img2_t.set(img2_np) # img3_t.set(img3_np) # dev.wait() # import code # code.interact(local=dict(globals(), **locals())) # def func(): # print('1') # yield 1 # f = func() # #x=np.ndarray((1,), dtype=np.uint8, buffer=bytes(1)) # # q = bytearray([1,2,3]) # class Test(): # class Status: # INITIALIZING, DOWNLOADING, INITIALIZED, ERROR, = range(4) # ... # while True: # with lib_time.timeit(): # for _ in range(10000000): # Test() # from enum import IntEnum # class PreinsertMeta(type): # # def resolvedField(self): # # if isinstance(self.field, basestring): # # tbl, fld = self.field.split(".") # # self.field = (tbl, fld) # # return self.field # # Field = property(resolvedField) # # def __getattr__(self, attrname): # # if attrname == "field": # # if isinstance(self.field, basestring): # # tbl, fld = self.field.split(".") # # self.field = (tbl, fld) # # return self.field # # else: # # return super(PreinsertMeta, self).__getattr__(attrname) # # @classmethod # # def __prepare__(mcs, name, bases, **kwargs): # # print(' Meta.__prepare__(mcs=%s, name=%r, bases=%s, **%s)' % ( # # mcs, name, bases, kwargs # # )) # # import code # # code.interact(local=dict(globals(), **locals())) # # return {} # @classmethod # def __setattr__(self, attrname, value): # print('__setattr__') # super(PreinsertMeta, self).__setattr__(attrname, value) # class CEnum(metaclass=PreinsertMeta): # class ID: # ... # class Status(CEnum): # ST0 = CEnum.ID() # ST1 = CEnum.ID() # # # class __metaclass__(type): # # def __getattr__(self, name): # # print('123') # # return self.values.index(name) # x = Status.ST0 # from xlib import mp as lib_mp # import pickle # data = [0,1,2,3,[1,2]] # rd = lib_mp.MPSPSCMRRingData(table_size=8192, heap_size_mb=1) # rd.write( bytes(1024*1024-24) ) # rd.write( bytes(1024*1024-24) ) # #x = rd.read() # #x=pickle.loads(x) # import code # code.interact(local=dict(globals(), **locals())) # import onnx # m = onnx.load_model(r'D:\DevelopPPP\projects\DeepFaceLive\github_project\xlib\onnxruntime\YoloV5Face\YoloV5Face.onnx') # #from xlib import torch as lib_torch #x = lib_torch.get_available_devices() #s = lib_torch.S3FD(x[0]) #lib_torch.S3FD.save_as_onnx(r'D:\s3fd.onnx') # if __name__ == '__main__': # #from # from xlib import opencl as cl # x = cl.get_available_devices_info() # import xlib.onnxruntime as lib_ort # x = lib_ort.get_available_devices_info() # from xlib.deepface import DFMModelInfo # from xlib import onnxruntime as lib_ort # import numpy as np # c = DFMModelInfo(DFMModelInfo.CelebType.CUSTOM, device=lib_ort.get_cpu_device()) # c.convert( np.zeros( (224,224,3), np.float32 )) # import code # code.interact(local=dict(globals(), **locals())) # yolo = lib_ort.YoloV5Face( lib_ort.CPUDeviceInfo() ) # #img = cv2.imread(r'D:\DevelopPython\test\00009.jpg') # img = cv2.imread(r'D:\DevelopPPP\projects\DeepFaceLive\multiphotos\000004.jpg') # #img = cv2.resize(img, (640,384)) # H,W,C = img.shape # rects = yolo.extract(img)[0] # rects = [ FaceURect.from_ltrb( (l/W, t/H, r/W, b/H) ) for l,t,r,b in rects ] # print(len(rects)) # rects[0].draw(img, (0,255,0)) # cv2.imshow('', img) # cv2.waitKey(0) import io import multiprocessing import pickle import time from xlib.io import FormattedMemoryViewIO from .MPAtomicInt32 import MPAtomicInt32 from .MPSharedMemory import MPSharedMemory class MPDataSlot: """ Multiprocess high performance multireader-multiwriter single data slot. MPDataSlot operates any picklable python object The slot mean only one object can exist in the slot in one time. Has acquire/release methods to init/free resource By default MPDataSlot in non acquired state. """ def __init__(self, size_mb): self._size = size = size_mb*1024*1024 self._shared_mem = None self._atom = None self._avail_size = size-4-4-8 self._last_pop_f = None self._acq_count = 0 def acquire(self): """ acquire the resource. It will be initialized if acquired first time """ self._acq_count += 1 if self._acq_count == 1: shared_mem = self._shared_mem = MPSharedMemory(self._size) self._atom = MPAtomicInt32(ar=shared_mem.get_ar(), index=0) def release(self): """ release the resource. """ if self._acq_count == 0: raise Exception('wrong release call, acq_count == 0') self._acq_count -= 1 if self._acq_count == 0: self._shared_mem = None self._atom = None def push(self, d): """ push obj to the slot arguments d picklable python object returns True if success, otherwise False - the slot is not emptied by receiver side. """ if self._acq_count == 0: return False # Construct the data in local memory d_dumped = pickle.dumps(d, 4) size = len(d_dumped) if size >= self._avail_size: raise Exception('size of MPDataSlot is not enough to push the object') if self._atom.compare_exchange(0, 1) != 0: return False fmv = FormattedMemoryViewIO(self._shared_mem.get_mv()[4:]) ver, = fmv.get_fmt('I') fmv.write_fmt('IQ', ver+1, size) fmv.write (d_dumped) self._atom.set(2, with_lock=False) return True def get_pop(self, your_ver): """ same as pop() but the data will not be popped. Also checks current ver with 'your_ver' returns obj, ver if nothing to get or ver the same, obj is None """ if self._acq_count == 0: return None, your_ver fmv = FormattedMemoryViewIO(self._shared_mem.get_mv()[4:]) ver, = fmv.read_fmt('I') if ver == 0 or ver == your_ver: return None, your_ver f = self._last_pop_f = io.BytesIO() while True: initial_val = self._atom.multi_compare_exchange( (0,2), 1) if initial_val in (0,2): break time.sleep(0.001) fmv.seek(0) ver, size = fmv.read_fmt('IQ') fmv.readinto(f, size ) self._atom.set(initial_val, with_lock=False) f.seek(0) return pickle.load(f), ver def pop(self): """ pop the MPDataSlotData returns MPDataSlotData or None """ if self._acq_count == 0: return None # Acquire the lock and copy the data to the local memory if self._atom.compare_exchange(2, 1) != 2: return None f = self._last_pop_f = io.BytesIO() fmv = FormattedMemoryViewIO(self._shared_mem.get_mv()[4:]) ver, size = fmv.read_fmt('IQ') fmv.readinto(f, size ) self._atom.set(0, with_lock=False) f.seek(0) return pickle.load(f) def get_last_pop(self): """ get last popped data returns MPDataSlotData or None """ if self._acq_count == 0: return None f = self._last_pop_f if f is not None: f.seek(0) return pickle.load(f) return None def __getstate__(self): if self._acq_count == 0: raise Exception('Pickling non-acquired MPDataSlot') d = self.__dict__.copy() d.pop('_last_pop_f') return d def __setstate__(self, d): self.__dict__.update(d) self._last_pop_f = None self._acq_count = 1 from PyQt6.QtCore import * from PyQt6.QtGui import * from PyQt6.QtWidgets import * from .QXFlatPanel import QXFlatPanel from ._part_QXWidget import _part_QXWidget from ..gui.QXPixmap import QXPixmap from ..gui.QXImageSequence import QXImageSequence from ..core.QXTimeLine import QXTimeLine class QXFlatPushButton(QAbstractButton, _part_QXWidget): """ Flat push button for QXFlatPanel by default fills only minimum space if text specified, the minimum_width/height will be set for text the font """ def __init__(self, text=None, pixmap=None, depth_shift=0, checkable=False, toggled=None, released=None, font=None, tooltip_text=None, size_policy=None, minimum_width=None, minimum_height=None, fixed_width=None, fixed_height=None, hided=False, enabled=True): super().__init__() if size_policy is None: size_policy = (QSizePolicy.Policy.Minimum, QSizePolicy.Policy.Minimum) if pixmap is not None: if not isinstance(pixmap, QXPixmap): raise ValueError('pixmap must be an instance of QXPixmap') self._pixmap = pixmap self.setCheckable(checkable) color = 68 self._normal_color = QColor(color, color, color, 255 ) normal_gradient = self._normal_gradient = QLinearGradient(0,0,0,1) normal_gradient.setCoordinateMode(QGradient.CoordinateMode.ObjectMode) normal_gradient.setColorAt(0.0, QColor(255,255,255, 15 ) ) normal_gradient.setColorAt(0.499, QColor(255,255,255, 10 ) ) normal_gradient.setColorAt(0.5, QColor(255,255,255, 3 ) ) normal_gradient.setColorAt(1.0, QColor(255,255,255, 10 ) ) hover_gradient = self._hover_gradient = QLinearGradient(0,0,0,1) hover_gradient.setCoordinateMode(QGradient.CoordinateMode.ObjectMode) hover_gradient.setColorAt(0.0, QColor(255,255,255, 30 ) ) hover_gradient.setColorAt(0.499, QColor(255,255,255, 20 ) ) hover_gradient.setColorAt(0.5, QColor(255,255,255, 6 ) ) hover_gradient.setColorAt(1.0, QColor(255,255,255, 20 ) ) pressed_gradient = self._pressed_gradient = QLinearGradient(0,0,0,1) pressed_gradient.setCoordinateMode(QGradient.CoordinateMode.ObjectMode) pressed_gradient.setColorAt(0.0, QColor(0,0,0, 0 ) ) pressed_gradient.setColorAt(0.33, QColor(0,0,0, 50 ) ) pressed_gradient.setColorAt(0.66, QColor(0,0,0, 50 ) ) pressed_gradient.setColorAt(1.0, QColor(0,0,0, 0 ) ) _part_QXWidget.connect_signal(released, self.released) _part_QXWidget.connect_signal(toggled, self.toggled) _part_QXWidget.__init__(self, font=font, tooltip_text=tooltip_text, size_policy=size_policy, minimum_width=minimum_width, minimum_height=minimum_height, fixed_width=fixed_width, fixed_height=fixed_height, hided=hided, enabled=enabled ) if text is not None: self.setText(text) self.setMinimumWidth(self.fontMetrics().horizontalAdvance(text)+4) self.setMinimumHeight(self.fontMetrics().height()+4) self._qp = QPainter() self._normal_color = QColor(0,0,0,0) self._pixmap_sequence = None self._tl = None def set_pixmap(self, pixmap): """ set static pixmap """ if not isinstance(pixmap, QXPixmap): raise ValueError('pixmap must be an instance of QXPixmap') self.stop_pixmap_sequence() self._pixmap = pixmap self.repaint() def set_pixmap_sequence(self, pixmap_sequence : QXImageSequence, loop_count : int = 1): """ set and play pixmap sequence """ self._pixmap_sequence = pixmap_sequence self._tl = QXTimeLine( duration=pixmap_sequence.get_duration(), frame_range=(0, pixmap_sequence.get_frame_count()-1), loop_count=0, update_interval=int( (1.0/pixmap_sequence.get_fps()) * 1000), frameChanged=self._tl_frameChanged, start=True ) def stop_pixmap_sequence(self): if self._tl is not None: self._tl.stop() self._tl = None self._pixmap_sequence = None def _tl_frameChanged(self, frame_id): self._pixmap = self._pixmap_sequence.get_frame(frame_id) self.repaint() def sizeHint(self) -> QSize: return QSize(0,0) def focusInEvent(self, ev : QFocusEvent): super().focusInEvent(ev) _part_QXWidget.focusInEvent(self, ev) def resizeEvent(self, ev : QResizeEvent): super().resizeEvent(ev) _part_QXWidget.resizeEvent(self, ev) def paintEvent(self, ev : QPaintEvent): rect = self.rect() qp = self._qp qp.begin(self) qp.setRenderHint(QPainter.RenderHint.Antialiasing) qp.setRenderHint(QPainter.RenderHint.SmoothPixmapTransform) qp.fillRect(rect, self._normal_color) pixmap = self._pixmap if pixmap is not None: w, h = rect.width(), rect.height() rect_aspect = w / h size = pixmap.size() pixmap_aspect = size.width() / size.height() if pixmap_aspect != rect_aspect: if pixmap_aspect > rect_aspect: pw, ph = w, int(h * (rect_aspect / pixmap_aspect)) px, py = 0, h/2-ph/2 elif pixmap_aspect < rect_aspect: pw, ph = int( w * (pixmap_aspect / rect_aspect) ), h px, py = w/2-pw/2, 0 pixmap = pixmap.scaled_cached(pw,ph) else: px, py, pw, ph = 0, 0, w, h pixmap = pixmap.scaled_cached(pw,ph) if self.isEnabled(): qp.drawPixmap(px, py, pixmap) else: qp.drawPixmap(px, py, pixmap.grayscaled_cached() ) text = self.text() if text is not None: qp.setFont(self.font()) qp.drawText(rect, Qt.AlignmentFlag.AlignCenter, text) if self.isEnabled(): if self.isDown(): qp.setCompositionMode(QPainter.CompositionMode.CompositionMode_Multiply) qp.fillRect(rect, self._pressed_gradient) elif self.underMouse(): qp.setCompositionMode(QPainter.CompositionMode.CompositionMode_Screen) qp.fillRect(rect, self._hover_gradient) else: qp.setCompositionMode(QPainter.CompositionMode.CompositionMode_Screen) qp.fillRect(rect, self._normal_gradient) else: qp.setCompositionMode(QPainter.CompositionMode.CompositionMode_Screen) qp.fillRect(rect, self._normal_gradient) qp.end() import numpy as np from xlib import opencl as lib_cl from xlib import time as lib_time # # #lib_cl.CLBuffer() # face_mask = np.ones( (1024,1024,1), dtype=np.float32 ) # # # # #d = CL.get_available_devices() dev = lib_cl.get_device(d) # # # # # buf.set(data_np) KW = 5 KH = 5 IH = 64 IW = 64 IC = 1024 OH = IH OW = IW OC = IC input_np = np.ones( (IH,IW,IC), dtype=np.float32 ) kernel_np = np.ones( (KH,KW,OC, IC), dtype=np.float32 ) input_t = dev.alloc ( size=IH*IW*IC*4 ) kernel_t = dev.alloc ( size=KH*KW*OC*IC*4 ) output_t = dev.alloc ( size=OH*OW*OC*4 ) input_t.set(input_np) kernel_t.set(kernel_np) krn1 = lib_cl.OpenCLKernel(kernel_text=f""" #define OH {OH} #define OW {OW} #define OC {OC} #define IC {IC} #define KH {KH} #define KW {KW} #define kh 0 #define kw 0 __kernel void impl(__global float* O, const __global float* I, const __global float* K) {{ size_t gid = get_global_id(0); size_t oc = gid % OC; gid /= OC; size_t ow = gid % OW; gid /= OW; size_t oh = gid % OH; gid /= OH; //size_t lid = get_local_id(0); __local float Iv[IC]; Iv[oc] = I[ oh*OW*OC + ow*OC + oc ]; barrier(CLK_LOCAL_MEM_FENCE); float v = 0.0f; #pragma unroll for (int ic=0; ic numeric symbols will be used example for 'i', AAxes((4,512)) #define I0 4 #define I1 512 #define I_idx(i0,i1) ((size_t)i0)*I1+i1 //access by idx with modulus #define I_idx_mod(i0,i1) MODULO(((size_t)i0), I0)*I1 + MODULO(i1, I1) """ shape = tuple(shape) rank = len(shape) if axes_symbols is None: axes_symbols = "".join([str(i) for i in range(rank)]) axes_symbols = axes_symbols.upper() out = '#define MODULO(x,N) (x % N)\n' for i in range(rank): out += f'#define {axis_letter.upper()}{axes_symbols[i]} {shape[i]}\n' out += f'#define {axis_letter.upper()}_idx({CLKernelHelper.axes_seq_enum(axis_letter, rank)}) ' for i in range(rank): if i == 0: out += f'((size_t)({axis_letter.lower()}{i}))' else: out += f'({axis_letter.lower()}{i})' for j in range(i+1,rank): out += f'*{axis_letter.upper()}{axes_symbols[j]}' if i != rank-1: out += '+' out += '\n' out += f'#define {axis_letter.upper()}_idx_mod(' out += CLKernelHelper.axes_seq_enum(axis_letter, rank) out += ") " for i in range(rank): if i == 0: out += f'MODULO( ((size_t)({axis_letter.lower()}{i})) ,{axis_letter.upper()}{axes_symbols[i]})' else: out += f'MODULO( ({axis_letter.lower()}{i}),{axis_letter.upper()}{axes_symbols[i]})' for j in range(i+1,rank): out += f'*{axis_letter.upper()}{axes_symbols[j]}' if i != rank-1: out += '+' out += "\n" return out @staticmethod def define_axes_sizes(axis_letter, axes_sizes): """ Returns a text of axes sizes, example #define I0 4 #define I1 512 #define I2 512 """ out = "" axes_sizes = tuple(axes_sizes) ndim = len(axes_sizes) for i in range(ndim): out += f'#define {axis_letter.upper()}{i} {axes_sizes[i]}\n' return out @staticmethod def axes_idxs_from_var(axis_letter, rank_or_axes_symbols, var_name): """ decompose a size_t variable to axes indexes. Keeps original variable untouched. Example 'i',3,'gid' size_t gid_original = gid; size_t i2 = gid % I2; gid /= I2; size_t i1 = gid % I1; gid /= I1; size_t i0 = gid % I0; gid = gid_original; 'i','HW','gid' size_t gid_original = gid; size_t iw = gid % IW; gid /= IW; size_t ih = gid % IH; gid = gid_original; """ if isinstance(rank_or_axes_symbols, int): rank = rank_or_axes_symbols axes_symbols = "".join([str(i) for i in range(rank)]) elif isinstance(rank_or_axes_symbols, str): rank = len(rank_or_axes_symbols) axes_symbols = rank_or_axes_symbols else: raise ValueError(f'Unknown type of rank_or_axes_symbols') out = f'size_t {var_name}_original = {var_name};' for i in range(rank-1,-1,-1): if i == 0: if rank > 1: out += f'size_t {axis_letter.lower()}{axes_symbols[i].lower()} = {var_name} / {axis_letter.upper()}{axes_symbols[i+1].upper()};' else: out += f'size_t {axis_letter.lower()}{axes_symbols[i].lower()} = {var_name};' else: out += f'size_t {axis_letter.lower()}{axes_symbols[i].lower()} = {var_name} % {axis_letter.upper()}{axes_symbols[i].upper()};' if i > 1: out += f' {var_name} /= {axis_letter.upper()}{axes_symbols[i].upper()};\n' out += f'{var_name} = {var_name}_original;\n' return out @staticmethod def axes_order_enum(axis_letter, axes_order): """ returns axis enumeration with given order Example ('i', (1,2,0)) returns 'i1,i2,i0' ('i', 'HW') return 'ih,iw' """ if isinstance(axes_order, str): axes_order = axes_order.lower() else: axes_order = tuple(axes_order) return ','.join( [ f'{axis_letter.lower()}{axes_order[axis]}' for axis in range(len(axes_order)) ]) @staticmethod def axes_seq_enum(axis_letter, rank, new_axis=None, zero_axes=None): """ returns axis sequental enumeration with given rank Example ('i', 4) returns 'i0,i1,i2,i3' ('i', 4, new_axis=('name',1) ) returns 'i0,name,i1,i2,i3' ('i', 3, zero_axes=(1,) ) returns 'i0,0,i2' """ if zero_axes is not None: axes = [ '0' if axis in zero_axes else f'{axis_letter.lower()}{axis}' for axis in range(rank) ] else: axes = [ f'{axis_letter.lower()}{axis}' for axis in range(rank) ] if new_axis is not None: name, axis = new_axis return','.join(axes[:axis] + [name] + axes [axis:]) else: return ','.join(axes) @staticmethod def include_constants_pi(): """ defines PI constants PI_F PI_2_F PI_4_F """ return f""" #define PI_F 3.14159274101257f #define PI_2_F 1.57079637050629f #define PI_4_F 0.78539818525314f """ @staticmethod def include_hash(): """ returns hash functions: uint hash_uint_uint(uint v) uint2 hash_uint2_uint2(uint2 v) uint3 hash_uint3_uint3(uint3 v) float hash_float_uint(uint q) float2 hash_float2_uint(uint q) float3 hash_float3_uint (uint v) float hash_float_float(float p) float hash_float_float2(float2 p) """ return f""" //---------- PCG hashes from https://www.shadertoy.com/view/XlGcRh #define UIF (1.0 / (float)(0xffffffffU)) uint hash_uint_uint(uint v) {{ uint state = v * 747796405u + 2891336453u; uint word = ((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u; return (word >> 22u) ^ word; }} uint2 hash_uint2_uint2 (uint2 v) {{ v = v * 1664525u + 1013904223u; v.x += v.y * 1664525u; v.y += v.x * 1664525u; v ^= v>>16u; v.x += v.y * 1664525u; v.y += v.x * 1664525u; v ^= v>>16u; return v; }} uint3 hash_uint3_uint3(uint3 v) {{ v = v * 1664525u + 1013904223u; v.x += v.y*v.z; v.y += v.z*v.x; v.z += v.x*v.y; v ^= v >> 16u; v.x += v.y*v.z; v.y += v.z*v.x; v.z += v.x*v.y; return v; }} float hash_float_uint(uint v) {{ return (float)( hash_uint_uint(v) ) * UIF; }} float2 hash_float2_uint (uint v) {{ uint2 q = hash_uint2_uint2( (uint2)(v, 1) ); return (float2)(q.x, q.y) * UIF; }} float3 hash_float3_uint (uint v) {{ uint3 q = hash_uint3_uint3( (uint3)(v, 1, 1) ); return (float3)(q.x, q.y, q.z) * UIF; }} //---------- Classic hashes used in shaders float hash_float_float(float p) {{ float x = sin(p*12.9898)*43758.5453; return x - floor(x); }} float hash_float_float2(float2 p) {{ float x = sin( dot(p, (float2)(12.9898, 78.233)) )*43758.5453; return x - floor(x); }} """ ph = CLKernelHelper from xlib import opencl as lib_cl # # #lib_cl.CLBuffer() # # face_mask = np.ones( (1024,1024,1), dtype=np.float32 ) # # # # #d = CL.get_available_devices() dev = lib_cl.get_device(d) # # # # # buf.set(data_np) inp_np = np.ones( (1024,18,18), dtype=np.float32 ) sh = (102760448//2,) #kernel_np = np.ones( (102760448,), dtype=np.float32 ) input_t = dev.alloc ( size=1*200000*4 ) dense_kernel = dev.alloc ( size=256*200000*4 ) working_t = dev.alloc ( size=256*200000*2*4 ) krn1 = lib_cl.OpenCLKernel(kernel_text=f""" __kernel void impl(__global float* O, const __global float* I) {{ size_t gid = get_global_id(0); size_t i0 = gid % 256; gid /= 256; size_t i1 = gid % 200000; gid /= 200000; O[i0*200000 + i1 + 0] = I[i1]; }} """) krn2 = lib_cl.OpenCLKernel(kernel_text=f""" __kernel void impl(__global float* O, const __global float* I) {{ size_t gid = get_global_id(0); size_t i0 = gid % 256; gid /= 256; size_t i1 = gid % 200000; gid /= 200000; O[i0*200000 + i1 + 1] = I[i0*200000 + i1]; }} """) krn3 = lib_cl.OpenCLKernel(kernel_text=f""" __kernel void impl(__global float* O) {{ size_t gid = get_global_id(0); O[gid*2+0] = O[gid*2+0]*O[gid*2+1]; }} """) # 1, 200000 # 256, 200000 # # #buf.set(kernel_np) # while True: # with lib_time.timeit(): # #for _ in range(1000): # dev.run(krn1, working_t, input_t, global_shape=(256*200000,) ) # dev.run(krn2, working_t, dense_kernel, global_shape=(256*200000,) ) # dev.run(krn3, working_t, global_shape=(256*200000,) ) # dev.wait() # import onnxruntime as rt # import onnx # model = onnx.load_model(r'D:\DevelopPPP\projects\DeepFaceLive\github_project\xlib\onnxruntime\FaceMesh\FaceMesh.onnx') # g=model.graph # chs = {} # for node in g.node: # if 'channel_padding' in node.name: # chs[node.name] = node #from xlib.directml import # def GetPlatforms() -> List[platform]: # num_platforms = cl_uint() # lib_GetPlatformIDs(0, None, byref(num_platforms)) # n = num_platforms.value # if n > 0: # platform_array = (platform * n)() # lib_GetPlatformIDs(num_platforms, platform_array, None) # return tuple(x for x in platform_array) # else: # return () #HRESULT WINAPI CreateDXGIFactory(REFIID riid, _COM_Outptr_ void **ppFactory); # for i,node in enumerate(g.node): # print(i,node.name) # onnx.save_model(model, r'D:\DevelopPPP\projects\DeepFaceLive\github_project\xlib\onnxruntime\FaceMesh\FaceMesh2.onnx') # x = rt.get_available_providers() # model = onnx.load_model(r'D:\DevelopPPP\projects\DeepFaceLive\github_project\xlib\deepface\CELEB_MODEL\TOM_CRUISE.onnx') # g=model.graph # y=[x for x in g.node if x.name == 'Conv2D_62'] # mat = np.eye(2,3) # with lib_time.timeit(): # x = cv2.warpAffine( face_mask, mat, frame_image.shape[1::-1], np.zeros(frame_image.shape[0:2], dtype=np.float32), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LINEAR ) # x = cv2.warpAffine( face_mask, mat, frame_image.shape[1::-1], np.zeros(frame_image.shape[0:2], dtype=np.float32), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LINEAR ) # x = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(15,15)) # x = np.float32( [ (0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) # x = lib_math.Affine2DMat( [ [1,1,1],[1,1,1] ]) # p = Path(r'D:\qwe\asd.txt') # import itertools # from xlib.api.win32.dxgi import * # from xlib.api.win32.d3d12 import * # from xlib.api.win32.directml import * # factory = create_DXGIFactory1() # if factory is None: # raise Exception('Unable to CreateDXGIFactory') # adapters = [] # for i in itertools.count(): # adapter = factory.enum_adapters1(i) # if adapter is not None: # adapters.append(adapter) # else: # break # device = d3d12_create_device(adapters[0], D3D_FEATURE_LEVEL.D3D_FEATURE_LEVEL_11_0) # desc = D3D12_COMMAND_QUEUE_DESC() # desc.Type = D3D12_COMMAND_LIST_TYPE.D3D12_COMMAND_LIST_TYPE_DIRECT # desc.Flags = D3D12_COMMAND_QUEUE_FLAGS.D3D12_COMMAND_QUEUE_FLAG_NONE # comm_q = device.create_command_queue(desc) # comm_alloc = device.create_command_allocator(D3D12_COMMAND_LIST_TYPE.D3D12_COMMAND_LIST_TYPE_DIRECT) # comm_list = device.create_command_list(0, D3D12_COMMAND_LIST_TYPE.D3D12_COMMAND_LIST_TYPE_DIRECT, comm_alloc) # #D3D12_COMMAND_QUEUE_DESC # # D3D12_COMMAND_QUEUE_DESC commandQueueDesc{}; # # commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; # # commandQueueDesc.Flags = D3D12_COMMAND_QUEUE_FLAG_NONE; # comm_list.GetType() # import code # code.interact(local=dict(globals(), **locals())) # dxgi_factory = IDXGIFactory1() # hr = dxgi_CreateDXGIFactory1( byref(IDXGIFactory1.IID), byref(dxgi_factory) ) # adapters = dxgi_factory.EnumAdapters1() # for adapter in adapters: # x = adapter.GetDesc1() # from xlib import radeonml # from xlib.torch import S3FD # S3FD.save_as_onnx( Path(__file__).parent / 'S3FD.onnx') # # CenterFace_to_onnx( Path(__file__).parent / 'CenterFace.onnx' ) # import code # code.interact(local=dict(globals(), **locals())) [0.499976992607117, 0.652534008026123], [0.500025987625122, 0.547487020492554], [0.499974012374878, 0.602371990680695], [0.482113003730774, 0.471979022026062], [0.500150978565216, 0.527155995368958], [0.499909996986389, 0.498252987861633], [0.499523013830185, 0.40106201171875], [0.289712011814117, 0.380764007568359], [0.499954998493195, 0.312398016452789], [0.499987006187439, 0.269918978214264], [0.500023007392883, 0.107050001621246], [0.500023007392883, 0.666234016418457], [0.5000159740448, 0.679224014282227], [0.500023007392883, 0.692348003387451], [0.499976992607117, 0.695277988910675], [0.499976992607117, 0.70593398809433], [0.499976992607117, 0.719385027885437], [0.499976992607117, 0.737019002437592], [0.499967992305756, 0.781370997428894], [0.499816000461578, 0.562981009483337], [0.473773002624512, 0.573909997940063], [0.104906998574734, 0.254140973091125], [0.365929991006851, 0.409575998783112], [0.338757991790771, 0.41302502155304], [0.311120003461838, 0.409460008144379], [0.274657994508743, 0.389131009578705], [0.393361985683441, 0.403706014156342], [0.345234006643295, 0.344011008739471], [0.370094001293182, 0.346076011657715], [0.319321990013123, 0.347265005111694], [0.297903001308441, 0.353591024875641], [0.24779200553894, 0.410809993743896], [0.396889001131058, 0.842755019664764], [0.280097991228104, 0.375599980354309], [0.106310002505779, 0.399955987930298], [0.2099249958992, 0.391353011131287], [0.355807989835739, 0.534406006336212], [0.471751004457474, 0.65040397644043], [0.474155008792877, 0.680191993713379], [0.439785003662109, 0.657229006290436], [0.414617002010345, 0.66654098033905], [0.450374007225037, 0.680860996246338], [0.428770989179611, 0.682690978050232], [0.374971002340317, 0.727805018424988], [0.486716985702515, 0.547628998756409], [0.485300987958908, 0.527395009994507], [0.257764995098114, 0.314490020275116], [0.401223003864288, 0.455172002315521], [0.429818987846375, 0.548614978790283], [0.421351999044418, 0.533740997314453], [0.276895999908447, 0.532056987285614], [0.483370006084442, 0.499586999416351], [0.33721199631691, 0.282882988452911], [0.296391993761063, 0.293242990970612], [0.169294998049736, 0.193813979625702], [0.447580009698868, 0.302609980106354], [0.392390012741089, 0.353887975215912], [0.354490011930466, 0.696784019470215], [0.067304998636246, 0.730105042457581], [0.442739009857178, 0.572826027870178], [0.457098007202148, 0.584792017936707], [0.381974011659622, 0.694710969924927], [0.392388999462128, 0.694203019142151], [0.277076005935669, 0.271932005882263], [0.422551989555359, 0.563233017921448], [0.385919004678726, 0.281364023685455], [0.383103013038635, 0.255840003490448], [0.331431001424789, 0.119714021682739], [0.229923993349075, 0.232002973556519], [0.364500999450684, 0.189113974571228], [0.229622006416321, 0.299540996551514], [0.173287004232407, 0.278747975826263], [0.472878992557526, 0.666198015213013], [0.446828007698059, 0.668527007102966], [0.422762006521225, 0.673889994621277], [0.445307999849319, 0.580065965652466], [0.388103008270264, 0.693961024284363], [0.403039008378983, 0.706539988517761], [0.403629004955292, 0.693953037261963], [0.460041999816895, 0.557139039039612], [0.431158006191254, 0.692366003990173], [0.452181994915009, 0.692366003990173], [0.475387006998062, 0.692366003990173], [0.465828001499176, 0.779190003871918], [0.472328990697861, 0.736225962638855], [0.473087012767792, 0.717857003211975], [0.473122000694275, 0.704625964164734], [0.473033010959625, 0.695277988910675], [0.427942007780075, 0.695277988910675], [0.426479011774063, 0.703539967536926], [0.423162013292313, 0.711845993995667], [0.4183090031147, 0.720062971115112], [0.390094995498657, 0.639572978019714], [0.013953999616206, 0.560034036636353], [0.499913990497589, 0.58014702796936], [0.413199990987778, 0.69539999961853], [0.409626007080078, 0.701822996139526], [0.468080013990402, 0.601534962654114], [0.422728985548019, 0.585985004901886], [0.463079988956451, 0.593783974647522], [0.37211999297142, 0.47341400384903], [0.334562003612518, 0.496073007583618], [0.411671012639999, 0.546965003013611], [0.242175996303558, 0.14767599105835], [0.290776997804642, 0.201445996761322], [0.327338010072708, 0.256527006626129], [0.399509996175766, 0.748921036720276], [0.441727995872498, 0.261676013469696], [0.429764986038208, 0.187834024429321], [0.412198007106781, 0.108901023864746], [0.288955003023148, 0.398952007293701], [0.218936994671822, 0.435410976409912], [0.41278201341629, 0.398970007896423], [0.257135003805161, 0.355440020561218], [0.427684992551804, 0.437960982322693], [0.448339998722076, 0.536936044692993], [0.178560003638268, 0.45755398273468], [0.247308000922203, 0.457193970680237], [0.286267012357712, 0.467674970626831], [0.332827985286713, 0.460712015628815], [0.368755996227264, 0.447206974029541], [0.398963987827301, 0.432654976844788], [0.476410001516342, 0.405806005001068], [0.189241006970406, 0.523923993110657], [0.228962004184723, 0.348950982093811], [0.490725994110107, 0.562400996685028], [0.404670000076294, 0.485132992267609], [0.019469000399113, 0.401564002037048], [0.426243007183075, 0.420431017875671], [0.396993011236191, 0.548797011375427], [0.266469985246658, 0.376977026462555], [0.439121007919312, 0.51895797252655], [0.032313998788595, 0.644356966018677], [0.419054001569748, 0.387154996395111], [0.462783008813858, 0.505746960639954], [0.238978996872902, 0.779744982719421], [0.198220998048782, 0.831938028335571], [0.107550002634525, 0.540755033493042], [0.183610007166862, 0.740257024765015], [0.134409993886948, 0.333683013916016], [0.385764002799988, 0.883153975009918], [0.490967005491257, 0.579378008842468], [0.382384985685349, 0.508572995662689], [0.174399003386497, 0.397670984268188], [0.318785011768341, 0.39623498916626], [0.343364000320435, 0.400596976280212], [0.396100014448166, 0.710216999053955], [0.187885001301765, 0.588537991046906], [0.430987000465393, 0.944064974784851], [0.318993002176285, 0.898285031318665], [0.266247987747192, 0.869701027870178], [0.500023007392883, 0.190576016902924], [0.499976992607117, 0.954452991485596], [0.366169989109039, 0.398822009563446], [0.393207013607025, 0.39553701877594], [0.410373002290726, 0.391080021858215], [0.194993004202843, 0.342101991176605], [0.388664990663528, 0.362284004688263], [0.365961998701096, 0.355970978736877], [0.343364000320435, 0.355356991291046], [0.318785011768341, 0.35834002494812], [0.301414996385574, 0.363156020641327], [0.058132998645306, 0.319076001644135], [0.301414996385574, 0.387449026107788], [0.499987989664078, 0.618434011936188], [0.415838003158569, 0.624195992946625], [0.445681989192963, 0.566076993942261], [0.465844005346298, 0.620640993118286], [0.49992299079895, 0.351523995399475], [0.288718998432159, 0.819945991039276], [0.335278987884521, 0.852819979190826], [0.440512001514435, 0.902418971061707], [0.128294005990028, 0.791940987110138], [0.408771991729736, 0.373893976211548], [0.455606997013092, 0.451801002025604], [0.499877005815506, 0.908990025520325], [0.375436991453171, 0.924192011356354], [0.11421000212431, 0.615022003650665], [0.448662012815475, 0.695277988910675], [0.4480200111866, 0.704632043838501], [0.447111994028091, 0.715808033943176], [0.444831997156143, 0.730794012546539], [0.430011987686157, 0.766808986663818], [0.406787008047104, 0.685672998428345], [0.400738000869751, 0.681069016456604], [0.392399996519089, 0.677703022956848], [0.367855995893478, 0.663918972015381], [0.247923001646996, 0.601333022117615], [0.452769994735718, 0.420849978923798], [0.43639200925827, 0.359887003898621], [0.416164010763168, 0.368713974952698], [0.413385987281799, 0.692366003990173], [0.228018000721931, 0.683571994304657], [0.468268007040024, 0.352671027183533], [0.411361992359161, 0.804327011108398], [0.499989002943039, 0.469825029373169], [0.479153990745544, 0.442654013633728], [0.499974012374878, 0.439637005329132], [0.432112008333206, 0.493588984012604], [0.499886006116867, 0.866917014122009], [0.49991300702095, 0.821729004383087], [0.456548988819122, 0.819200992584229], [0.344549000263214, 0.745438992977142], [0.37890899181366, 0.574010014533997], [0.374292999505997, 0.780184984207153], [0.319687992334366, 0.570737957954407], [0.357154995203018, 0.604269981384277], [0.295284003019333, 0.621580958366394], [0.447750002145767, 0.862477004528046], [0.410986006259918, 0.508723020553589], [0.31395098567009, 0.775308012962341], [0.354128003120422, 0.812552988529205], [0.324548006057739, 0.703992962837219], [0.189096003770828, 0.646299958229065], [0.279776990413666, 0.71465802192688], [0.1338230073452, 0.682700991630554], [0.336768001317978, 0.644733011722565], [0.429883986711502, 0.466521978378296], [0.455527991056442, 0.548622965812683], [0.437114000320435, 0.558896005153656], [0.467287987470627, 0.529924988746643], [0.414712011814117, 0.335219979286194], [0.37704598903656, 0.322777986526489], [0.344107985496521, 0.320150971412659], [0.312875986099243, 0.32233202457428], [0.283526003360748, 0.333190023899078], [0.241245999932289, 0.382785975933075], [0.102986000478268, 0.468762993812561], [0.267612010240555, 0.424560010433197], [0.297879010438919, 0.433175981044769], [0.333433985710144, 0.433878004550934], [0.366427004337311, 0.426115989685059], [0.396012008190155, 0.416696012020111], [0.420121014118195, 0.41022801399231], [0.007561000064015, 0.480777025222778], [0.432949006557465, 0.569517970085144], [0.458638995885849, 0.479089021682739], [0.473466008901596, 0.545744001865387], [0.476087987422943, 0.563830018043518], [0.468472003936768, 0.555056989192963], [0.433990985155106, 0.582361996173859], [0.483518004417419, 0.562983989715576], [0.482482999563217, 0.57784903049469], [0.42645001411438, 0.389798998832703], [0.438998997211456, 0.39649498462677], [0.450067013502121, 0.400434017181396], [0.289712011814117, 0.368252992630005], [0.276670008897781, 0.363372981548309], [0.517862021923065, 0.471948027610779], [0.710287988185883, 0.380764007568359], [0.526226997375488, 0.573909997940063], [0.895093023777008, 0.254140973091125], [0.634069979190826, 0.409575998783112], [0.661242008209229, 0.41302502155304], [0.688880026340485, 0.409460008144379], [0.725341975688934, 0.389131009578705], [0.606630027294159, 0.40370500087738], [0.654766023159027, 0.344011008739471], [0.629905998706818, 0.346076011657715], [0.680678009986877, 0.347265005111694], [0.702096998691559, 0.353591024875641], [0.75221198797226, 0.410804986953735], [0.602918028831482, 0.842862963676453], [0.719901978969574, 0.375599980354309], [0.893692970275879, 0.399959981441498], [0.790081977844238, 0.391354024410248], [0.643998026847839, 0.534487962722778], [0.528249025344849, 0.65040397644043], [0.525849997997284, 0.680191040039062], [0.560214996337891, 0.657229006290436], [0.585384011268616, 0.66654098033905], [0.549625992774963, 0.680860996246338], [0.57122802734375, 0.682691991329193], [0.624852001667023, 0.72809898853302], [0.513050019741058, 0.547281980514526], [0.51509702205658, 0.527251958847046], [0.742246985435486, 0.314507007598877], [0.598631024360657, 0.454979002475739], [0.570338010787964, 0.548575043678284], [0.578631997108459, 0.533622980117798], [0.723087012767792, 0.532054007053375], [0.516445994377136, 0.499638974666595], [0.662801027297974, 0.282917976379395], [0.70362401008606, 0.293271005153656], [0.830704987049103, 0.193813979625702], [0.552385985851288, 0.302568018436432], [0.607609987258911, 0.353887975215912], [0.645429015159607, 0.696707010269165], [0.932694971561432, 0.730105042457581], [0.557260990142822, 0.572826027870178], [0.542901992797852, 0.584792017936707], [0.6180260181427, 0.694710969924927], [0.607590973377228, 0.694203019142151], [0.722943007946014, 0.271963000297546], [0.577413976192474, 0.563166975975037], [0.614082992076874, 0.281386971473694], [0.616907000541687, 0.255886018276215], [0.668509006500244, 0.119913995265961], [0.770092010498047, 0.232020974159241], [0.635536015033722, 0.189248979091644], [0.77039098739624, 0.299556016921997], [0.826722025871277, 0.278755009174347], [0.527121007442474, 0.666198015213013], [0.553171992301941, 0.668527007102966], [0.577238023281097, 0.673889994621277], [0.554691970348358, 0.580065965652466], [0.611896991729736, 0.693961024284363], [0.59696102142334, 0.706539988517761], [0.596370995044708, 0.693953037261963], [0.539958000183105, 0.557139039039612], [0.568841993808746, 0.692366003990173], [0.547818005084991, 0.692366003990173], [0.52461302280426, 0.692366003990173], [0.534089982509613, 0.779141008853912], [0.527670979499817, 0.736225962638855], [0.526912987232208, 0.717857003211975], [0.526877999305725, 0.704625964164734], [0.526966989040375, 0.695277988910675], [0.572058022022247, 0.695277988910675], [0.573521018028259, 0.703539967536926], [0.57683801651001, 0.711845993995667], [0.581691026687622, 0.720062971115112], [0.609944999217987, 0.639909982681274], [0.986046016216278, 0.560034036636353], [0.5867999792099, 0.69539999961853], [0.590372025966644, 0.701822996139526], [0.531915009021759, 0.601536989212036], [0.577268004417419, 0.585934996604919], [0.536915004253387, 0.593786001205444], [0.627542972564697, 0.473352015018463], [0.665585994720459, 0.495950996875763], [0.588353991508484, 0.546862006187439], [0.757824003696442, 0.14767599105835], [0.709249973297119, 0.201507985591888], [0.672684013843536, 0.256581008434296], [0.600408971309662, 0.74900496006012], [0.55826598405838, 0.261672019958496], [0.570303976535797, 0.187870979309082], [0.588165998458862, 0.109044015407562], [0.711045026779175, 0.398952007293701], [0.781069993972778, 0.435405015945435], [0.587247014045715, 0.398931980133057], [0.742869973182678, 0.355445981025696], [0.572156012058258, 0.437651991844177], [0.55186802148819, 0.536570012569427], [0.821442008018494, 0.457556009292603], [0.752701997756958, 0.457181990146637], [0.71375697851181, 0.467626988887787], [0.66711300611496, 0.460672974586487], [0.631101012229919, 0.447153985500336], [0.6008620262146, 0.432473003864288], [0.523481011390686, 0.405627012252808], [0.810747981071472, 0.523926019668579], [0.771045982837677, 0.348959028720856], [0.509127020835876, 0.562718033790588], [0.595292985439301, 0.485023975372314], [0.980530977249146, 0.401564002037048], [0.573499977588654, 0.420000016689301], [0.602994978427887, 0.548687994480133], [0.733529984951019, 0.376977026462555], [0.560611009597778, 0.519016981124878], [0.967685997486115, 0.644356966018677], [0.580985009670258, 0.387160003185272], [0.537728011608124, 0.505385041236877], [0.760966002941132, 0.779752969741821], [0.801778972148895, 0.831938028335571], [0.892440974712372, 0.54076099395752], [0.816350996494293, 0.740260004997253], [0.865594983100891, 0.333687007427216], [0.614073991775513, 0.883246004581451], [0.508952975273132, 0.579437971115112], [0.617941975593567, 0.508316040039062], [0.825608015060425, 0.397674977779388], [0.681214988231659, 0.39623498916626], [0.656635999679565, 0.400596976280212], [0.603900015354156, 0.710216999053955], [0.81208598613739, 0.588539004325867], [0.56801301240921, 0.944564998149872], [0.681007981300354, 0.898285031318665], [0.733752012252808, 0.869701027870178], [0.633830010890961, 0.398822009563446], [0.606792986392975, 0.39553701877594], [0.589659988880157, 0.391062021255493], [0.805015981197357, 0.342108011245728], [0.611334979534149, 0.362284004688263], [0.634037971496582, 0.355970978736877], [0.656635999679565, 0.355356991291046], [0.681214988231659, 0.35834002494812], [0.698584973812103, 0.363156020641327], [0.941866993904114, 0.319076001644135], [0.698584973812103, 0.387449026107788], [0.584177017211914, 0.624107003211975], [0.554318010807037, 0.566076993942261], [0.534153997898102, 0.62064003944397], [0.711217999458313, 0.819975018501282], [0.664629995822906, 0.852871000766754], [0.559099972248077, 0.902631998062134], [0.871706008911133, 0.791940987110138], [0.591234028339386, 0.373893976211548], [0.544341027736664, 0.451583981513977], [0.624562978744507, 0.924192011356354], [0.88577002286911, 0.615028977394104], [0.551338016986847, 0.695277988910675], [0.551980018615723, 0.704632043838501], [0.552887976169586, 0.715808033943176], [0.555167973041534, 0.730794012546539], [0.569944024085999, 0.767035007476807], [0.593203008174896, 0.685675978660583], [0.599261999130249, 0.681069016456604], [0.607599973678589, 0.677703022956848], [0.631937980651855, 0.663500010967255], [0.752032995223999, 0.601315021514893], [0.547226011753082, 0.420395016670227], [0.563543975353241, 0.359827995300293], [0.583841025829315, 0.368713974952698], [0.586614012718201, 0.692366003990173], [0.771915018558502, 0.683578014373779], [0.531597018241882, 0.352482974529266], [0.588370978832245, 0.804440975189209], [0.52079701423645, 0.442565023899078], [0.567984998226166, 0.493479013442993], [0.543282985687256, 0.819254994392395], [0.655317008495331, 0.745514988899231], [0.621008992195129, 0.574018001556396], [0.625559985637665, 0.78031200170517], [0.680198013782501, 0.570719003677368], [0.64276397228241, 0.604337990283966], [0.704662978649139, 0.621529996395111], [0.552012026309967, 0.862591981887817], [0.589071989059448, 0.508637011051178], [0.685944974422455, 0.775357007980347], [0.645735025405884, 0.812640011310577], [0.675342977046967, 0.703978002071381], [0.810858011245728, 0.646304965019226], [0.72012197971344, 0.714666962623596], [0.866151988506317, 0.682704985141754], [0.663187026977539, 0.644596993923187], [0.570082008838654, 0.466325998306274], [0.544561982154846, 0.548375964164734], [0.562758982181549, 0.558784961700439], [0.531987011432648, 0.530140042304993], [0.585271000862122, 0.335177004337311], [0.622952997684479, 0.32277899980545], [0.655896008014679, 0.320163011550903], [0.687132000923157, 0.322345972061157], [0.716481983661652, 0.333200991153717], [0.758756995201111, 0.382786989212036], [0.897013008594513, 0.468769013881683], [0.732392013072968, 0.424547016620636], [0.70211398601532, 0.433162987232208], [0.66652500629425, 0.433866024017334], [0.633504986763, 0.426087975502014], [0.603875994682312, 0.416586995124817], [0.579657971858978, 0.409945011138916], [0.992439985275269, 0.480777025222778], [0.567192018032074, 0.569419980049133], [0.54136598110199, 0.478899002075195], [0.526564002037048, 0.546118021011353], [0.523913025856018, 0.563830018043518], [0.531529009342194, 0.555056989192963], [0.566035985946655, 0.582329034805298], [0.51631098985672, 0.563053965568542], [0.5174720287323, 0.577877044677734], [0.573594987392426, 0.389806985855103], [0.560697972774506, 0.395331978797913], [0.549755990505219, 0.399751007556915], [0.710287988185883, 0.368252992630005], [0.723330020904541, 0.363372981548309] ]) # import torch #from xlib import onnxruntime as lib_ort # img = lib_cv.imread(r'D:\DevelopPython\test\00008.jpg') # img = cv2.resize(img, (224,224) ) # img = img[None,...].transpose( (0,3,1,2) ) # img = img.astype(np.float32) / 255.0 # sess = lib_ort.InferenceSession_with_device(r"D:\DevelopPPP\projects\DeepFaceLive\github_project\xlib\deepface\celeb\TOM_CRUISE\model.onnx", lib_ort.get_available_devices().get_best_device()) # sess = lib_ort.InferenceSession_with_device(r"D:\DevelopPPP\projects\DeepFaceLive\github_project\xlib\torch\CenterFace\CenterFace.onnx", lib_ort.get_available_devices().get_worst_device()) # # # input = sess.get_inputs()[0] # # # # input_name = sess.get_inputs()[0].name # while True: # img = np.random.random ( (1,3,672,672 )).astype(np.float32) # with lib_time.timeit(): # #try: # preds = sess.run(None, {sess.get_inputs()[0].name: img}) # #except: # # print('errro') # import onnx # g = onnx.load(r"D:\DevelopPPP\projects\DeepFaceLive\github_project\xlib\torch\CenterFace\CenterFace.onnx") # g.graph.input[0].type.tensor_type.shape.dim[0].dim_param = '?' # g.graph.input[0].type.tensor_type.shape.dim[2].dim_param = '?' # g.graph.input[0].type.tensor_type.shape.dim[3].dim_param = '?' # onnx.save(g, r"D:\DevelopPPP\projects\DeepFaceLive\github_project\xlib\torch\CenterFace\CenterFace.onnx") # img = np.ones( (1100,600), np.float32 ) # x = fit_in(img, 800,800) # cv2.imshow('',img) # cv2.waitKey(0) # cv2.imshow('',x) # cv2.waitKey(0) # poly = np.array([ [0,0],[2,0],[2,2],[0,2] ]) # print(polygon_area(poly)) # def gen_not_None(func): # def wrapper(*args, gen_not_None=False, **kwargs): # result = func(*args, **kwargs) # if gen_not_None: # for v in [ result ]: # if v is not None: # yield v # else: # return result # return wrapper # @gen_not_None # def test(v = 0): # return 1 # for v in test( gen_not_None=True ): # print(v) #from xlib.torch import S3FD #S3FD.save_as_onnx(r'D:\S3FD.onnx') # #CenterFace_to_onnx( Path(__file__).parent / 'CenterFace.onnx' ) # class A: # def __new__(cls, *args, **kwargs): # print('__new__ A') # obj = super().__new__(cls) # obj.asd : int = 0 # def __init__(self): # print('__init__ A') # super().__init__() # class B(A): # def __new__(cls, *args, **kwargs): # print('__new__ B') # return super().__new__(cls, *args, **kwargs) # def __init__(self, x=0): # print('__init__ B') # print(x) # super().__init__() # x = pickle.loads(pickle.dumps(B())) # #x = B(3) # # for pred in preds: # # pred = pred.transpose( (0,2,3,1)) # # cv2.imshow('', pred[0]) # # cv2.waitKey(0) # class A(lib_serialization.Serializable): # _a = None # _b = None # _c = None # _d = None # _e = None # _f = None # _g = None # def __init__(self): # super().__init__(#_on_serialize={'_d': self._on_serialize_a}, # _on_deserialize={'_d': self._on_deserialize_a} # ) # def _on_serialize_a(self, obj): # return 1 # def _on_deserialize_a(self, obj): # return obj # def set_test_vars(self): # self._a = None # self._b = 'asd' # self._c = 'asd'.encode('utf-8') # nested_a = A() # nested_a._a = 'qwe' # nested_a._b = np.ones( (256,4) ) # self._d = [ [nested_a], nested_a ] # self._e = ( nested_a, nested_a ) # self._f = { 0: [nested_a] } # self._g = 123 # class A(lib_serialization.Serializable): # _a = np.ones( (256,4) ) # a = A() # #a.set_test_vars() # # b = bytearray(16*1024*1024) # # mv = memoryview(b) # # bf = lib_serialization.BinaryMemoryViewSerializer(mv) # # #bf = lib_serialization.BinaryBytesIOSerializer() # # with lib_time.timeit(): # # for i in range(1000): # # bf.set_cursor(0) # # lib_serialization.Serializable.serialize(a, bf) # x = np.ones( (256,4) ) from xlib import python as lib_python import zlib class AdditiveFileStorage(lib_python.Disposable): """ Tiny file storage designed for fastest addition files not larger than 4Gb. Removing is unsupported. File with the same name will just overwrite old file. """ """ File format name table block (4096) aligned in 4096: prev table cluster offset(4) offset to prev table from this offset * 4096 0 = is last table records_count(4) 511 fixed records: name_crc_32_hash(4) 511 fixed records: file sizes(4) file block (not higher 4*1024*1024*1024-1): pickled name pickled data always at the end of file: [backward offset to latest name table = eof_offset - offset - 8] VERSION (4) MAGIC NUMBER (4) File format file block ( up to 15Tb ): file size(8) data padding last to cluster size 4096 Current table at the end of file: name table block (4096) aligned in 4096: prev table offset(8) 0 = is last table records_count(4) reserved(4) 255 fixed records: name(12 bytes max) cluster file offset from table (4) * 4096 always at the end of file after table: VERSION (4) MAGIC NUMBER (4) """ class _Table: def __init__(self): ... @staticmethod def load(f : lib_io.FormattedFileIO): ... class FileIterator: def __init__(self): ... def __enter__(self): ... def __exit__(self, *_): ... def __iter__(self): return self def __next__(self): ... #raise StopIteration() def __init__(self, filepath, create_new=False): self.filepath = filepath = Path(filepath) self._f = f = lib_io.FormattedFileIO (filepath, 'w+' if create_new else 'a+') self._table_files_count_max = 511 size = f.seek(0,2) if size == 0: self._table_g_offset = 0 self._add_table() else: f.seek(0,2) magic_number, = f.read_backward_fmt('I') if magic_number != 0xFED1D1AD: raise Exception(f'File {filepath} is not an AdditiveFileStorage.') last_table_offset, version, = f.read_backward_fmt('QI') f.seek(-last_table_offset,1) self._load_table() def _on_dispose(self): f = getattr(self, '_f', None) if f is not None: f.close() def close(self): self.dispose() def _add_table(self): """ add and write new table at current file offset """ f = self._f # Pad to 4096 cluster_overlap = f.tell() % 4096 if cluster_overlap != 0: f.fill(0, 4096 - cluster_overlap) self._table_prev_offset = (f.tell() - self._table_g_offset) // 4096 self._table_files_count = 0 self._table_name_hashes = [0]*self._table_files_count_max self._table_file_sizes = [0]*self._table_files_count_max self._table_g_offset = f.tell() f.write_fmt('I', self._table_prev_offset) self._table_files_count_offset = f.tell() f.write_fmt('I', 0) self._table_name_hashes_offset = f.tell() f.fill(0, 4*511) self._table_file_sizes_offset = f.tell() f.fill(0, 4*511) self._write_header() f.truncate() def _table_get_next_free_space_offset(self): files_count = self._table_files_count offset = self._table_g_offset + 4096 if files_count != 0: offset += sum(self._table_file_sizes[:files_count]) return offset def _add_new_table(self): # Seek to free space self._f.seek( self._table_get_next_free_space_offset() ) self._add_table() def _load_table(self): """load table from current offset""" f = self._f self._table_g_offset = f.tell() self._table_prev_offset = self._table_g_offset - f.read_fmt('I')[0]*4096 self._table_files_count_offset = f.tell() self._table_files_count = f.read_fmt('I')[0] self._table_name_hashes_offset = f.tell() self._table_name_hashes = list(f.read_fmt('I'*self._table_files_count_max)) self._table_file_sizes_offset = f.tell() self._table_file_sizes = list(f.read_fmt('I'*self._table_files_count_max)) def _write_header(self): f = self._f f.write_fmt('QII', f.tell()-self._table_g_offset, 1, 0xFED1D1AD) def add_file(self, filename, data): f = self._f if self._table_files_count == self._table_files_count_max: # Current table is full, add new self._add_new_table() file_offset = self._table_get_next_free_space_offset() f.seek(file_offset) # Seek to free space f.write_utf8(filename) # Write filename f.write(data) # Write the data # Calc total file size file_size = f.tell() - file_offset if file_size > 4*1024*1024*1024-1: raise Exception(f'File cannot be more than {4*1024*1024*1024}. Written: {file_size}') # Inc and write table files count file_idx = self._table_files_count self._table_files_count += 1 f.write_fmt_at(self._table_files_count_offset, 'I', self._table_files_count) # Set and write name CRC 32 hash self._table_name_hashes[file_idx] = namehash = zlib.crc32( filename.encode('utf-8') ) f.write_fmt_at(self._table_name_hashes_offset + file_idx*4, 'I', namehash) # Set and write file size self._table_file_sizes[file_idx] = file_size f.write_fmt_at(self._table_file_sizes_offset + file_idx*4, 'I', file_size) self._write_header() self._f.truncate() self._f.flush() # b = bytearray(8*1000) # bf = lib_serialization.BinaryMemoryViewSerializer( memoryview(b) ) class PickleFS: """ Mini file storage inside a file. Designed for several files combined together, for example metadata + images. Unpickled class has a function to load other data from the file where it was stored. """ # from xlib import path as lib_path # filenames = lib_path.get_files_paths(r'E:\FakeFaceVideoSources\Datasets\CelebA\data_src') # with lib_time.timeit(): # for filename in filenames: # with open(filename, 'rb+') as f: # f.read(4096) # x = 56 # import code # code.interact(local=dict(globals(), **locals())) # fs = AdditiveFileStorage(r'E:\1.faceset', create_new=True) # b = bytearray(4*1024*1024*1024 -8) # fs.add_file('asd', b ) # fs.close() # fs = AdditiveFileStorage(r'E:\1.faceset') # b = bytearray(8) # fs.add_file('asd', b ) # fs.close() # #fs = AdditiveFileStorage(r'E:\1.faceset') # #fs.add_file('qwe', bytes(range(8)) ) # import code # code.interact(local=dict(globals(), **locals())) # fs = AdditiveFileStorage(r'E:\1.faceset') # fs._add_table() # fs.add_file('asd', bytes(range(8)) ) # import code # code.interact(local=dict(globals(), **locals())) # with lib_io.FormattedFileIO(r'E:\test.txt', 'w+') as f1: # f1.truncate(4*1024*1024*1024) #with lib_io.FormattedFileIO(r'E:\test.txt', 'a+') as f1: # f1.read(4*1024*1024*1024) # import uuid # x=uuid.uuid4() # y=uuid.uuid4() # import code # code.interact(local=dict(globals(), **locals())) # ar= [] # with lib_time.timeit(): # with lib_io.FormattedFileIO(r'E:\test.txt', 'a+') as f1: # for i in range(1024*1024): # f1.seek( i*4096 ) # ar.append( f1.read(256) ) # # a = { f'0123456789_{i}': 0xFFFFFFFF for i in range(1024*1024) } # # with lib_time.timeit(): # # d = pickle.dumps(a,4) # # with lib_time.timeit(): # # pickle.loads(d) # x = np.array( [ [-2,3], [9,12]]) # x = None # import code # code.interact(local=dict(globals(), **locals())) # a = 1 # b = 2 # c = 3 # d = 4 # with lib_time.timeit(): # for _ in range(100000): # if a is not None and b is not None and c is not None and d is not None: # ... # with lib_time.timeit(): # for _ in range(100000): # if all(x for x in [a,b,c,d] if x is not None): # ... # with lib_io.FormattedFileIO(r'E:\test.txt', 'a+') as f1: # f1.seek(4*1024*1024*1024) # #fs.save_file('asd', bytes(range(8)) ) # #x = fs.load_file('asd') # bf.write_object( (1,) ) # with lib_time.timeit(): # bf.fill_raw(0x01, 4) # f1.close() # d = { x:'asd' for x in range(1000) } # b = d.copy() # class B(): # def __init__(self): # print('__init__ B') # def __getstate__(self): # print('__getstate__ B') # return self.__dict__.copy() # def __setstate__(self, d): # print('__setstate__ B') # self.__init__() # self.__dict__.update(d) # class A(): # def __init__(self): # print('__init__ A') # self.b = B() # def __getstate__(self): # print('__getstate__ A') # return self.__dict__.copy() # def __setstate__(self, d): # self.__init__() # self.__dict__.update(d) # a = A() # pickle.loads(pickle.dumps(a)) # #pickle.load(myf) # class A(lib_serialization.Serializable): # _a = None # _b = None # _c = None # _d = None # _e = None # _f = None # _g = None # def __init__(self, nested=True): # self._a = None # self._b = 'asd' # self._c = 'asd'.encode('utf-8') # if nested: # nested_a = A(nested=False) # nested_a._a = 'qwe' # nested_a._b = np.ones( (1080,1920,3), np.uint8 ) # self._d = [ [nested_a], nested_a ] # self._e = ( nested_a, nested_a ) # self._f = { 0: [nested_a] } # self._g = 123 # f = io.BytesIO() # f2 = io.BytesIO() # myf = MyIO() # a = A() # with lib_time.timeit(): # pickle.dump(a, f) # f.getbuffer() # with lib_time.timeit(): # pickle.dumps(a) # # #f.seek(0) # # #x = pickle.load(f) # # # ar = bytearray(1080*1920*3) # # # ar[0] = 1 # # # mar = memoryview(ar) # # # with lib_time.timeit(): # # # f.write(mar) # bf = lib_serialization.BinaryBytesIOSerializer(f) # with lib_time.timeit(): # lib_serialization.Serializable.serialize(a, bf) # import code # code.interact(local=dict(globals(), **locals())) # with lib_time.timeit(): # pickle.dumps(a) # b = bytearray(8) # with lib_io.FormattedFileIO(r'C:\test.txt', 'a+') as f1: # f1.seek(0) # # f1.write_fmt('II',1, 2) # # f1.seek(0) # # f1.readinto(b, 4) # import code # code.interact(local=dict(globals(), **locals())) # with lib_time.timeit(): # bf.write_fmt('I'*1000000, *([1]*1000000)) # bf.set_cursor(0) # x = bf.read_fmt('I'*1000000) # f2 = open(r'C:\test.txt', 'ab+') # f2.seek(0,0) # bf1 = lib_serialization.BinaryFileSerializer(f1) # bf2 = lib_serialization.BinaryFileSerializer(f2) # #bf.write_fmt('I', 4) # #f.close() # a = np.ones( (1080,1920,3), dtype=np.uint8 ) # av = memoryview(a.reshape(-1)) # import hashlib # with lib_time.timeit(): # sha = hashlib.sha1() # sha.update(av) # sha.digest() # lib_python.Serializable.serialize(a, bf) # bf.set_cursor(0) # b = lib_python.Serializable.deserialize(bf) # dsd = backend.BackendConnectionData(frame_image=np.zeros((1920,1080,3), np.uint8)) # with lib_time.timeit(): # for i in range(1000): # bf.set_cursor(0) # lib_python.Serializable.serialize(dsd, bf) # f = open(r'D:\asd', 'rb+') # #bf = lib_serialization.BinaryFileSerializer(f) # bf.write_ndarray( np.zeros((5,), dtype=np.int32)) # bf.set_cursor(0) # x=bf.read_ndarray() # class ASD(lib_serialization.Serializable): # _a = None # from backend import BackendConnectionData # d = BackendConnectionData() # mat = np.array([ [2,0,10], [0,2,10]]) # pts = np.array([ [10,10],[30,30] ]) # pts2 = Affine2DMat.transform_points(pts, mat, invert=True) # result = [] # for x in MyEnum: # x_splits = [] # for s in x.name.split('_'): # x_splits.append( s.capitalize() ) # result.append ( ' '.join(x_splits) ) # # >>> result # # ['Foo Bar', 'John Doe'] # # Now let's simplify # # First step: # result = [] # for x in MyEnum: # x_splits = [ s.capitalize() for s in x.name.split('_') ] # result.append ( ' '.join(x_splits) ) # # Second step: # result = [] # for x in MyEnum: # result.append ( ' '.join( s.capitalize() for s in x.name.split('_') ) ) # # Third step: # result = [ ' '.join( s.capitalize() for s in x.name.split('_') ) for x in MyEnum ] # # >>> result # # ['Foo Bar', 'John Doe'] # from xlib import torch as lib_torch # from collections import Iterable # a = lib_torch.Devices.get_all() # import zipfile # with lib_time.timeit(): # zip = zipfile.ZipFile(r'C:\test.zip', mode='a', compression=zipfile.ZIP_STORED) # import random # try: # x = zip.read('asd') # except KeyError as e: # print(e) # for i in range(1000000): # zip.writestr( str(i), bytes(0) ) # import code # code.interact(local=dict(globals(), **locals())) # with lib_time.timeit(): # for i in range(1000000): # #x = zip.read( str(i) ) # x = zip.read( str( random.randint(0, 1000000-1) ) ) # #print(x.__class__) # #zip.writestr( str(i), bytes(64) ) # zip.close() # import h5py # f = h5py.File(r'C:\1.faceset', 'a') # with lib_time.timeit(): # for i in range(1000000): # f[str(i)] = np.zeros( (64,), np.uint8 ) # f.flush() # b = bytearray(8) # n = np.frombuffer(b, np.uint8, 8) # b[0] = 1 # if 'a' in f: # del f['a'] # f['a'] = n # b2 = bytearray(8) # n2 = np.frombuffer(b2, np.uint8, 8) # f['a'].read_direct(n2) # from enum import IntEnum, Enum # def is_IntEnum(obj): # try: # return isinstance(obj, Iterable) and isinstance (next(iter(obj)), IntEnum) # except: # return False # Handle StopIteration, if obj has no elements # class MyEnum(IntEnum): # FOO_BAR = 0 # JOHN_DOE = 1 # x = [] # #print( issubclass(x, IntEnum) ) # print( issubclass(MyEnum, IntEnum) ) # False # import code # code.interact(local=dict(globals(), **locals())) # class B(lib_serialization.Serializable): # _a = 2 # class A(lib_serialization.Serializable): # _a = 1 # a = A() # a._b = B() # f = open(r'D:\qwe', 'wb+') # bf = lib_serialization.BinaryFileSerializer(f) # a.serialize(bf) # import numpy as np # x = np.int32(12) # import pickle # class A(): # ... # class B(): # ... # a = A() # a.b = a.a = B() # x = pickle.loads(pickle.dumps(a)) # from timeit import timeit # print( timeit("[None]*1000",number=10000) ) # print( timeit("[None for _ in range(1000)]",number=10000) ) # print( timeit("list(range(1000))",number=10000) ) # import io # x = io.BytesIO() # import code # code.interact(local=dict(globals(), **locals())) # def subproc( ds : lib_mp.MPDataSlot): # while True: # d = ds.pop() # if d is not None: # b = d._a # if b is None: # print('b is None') # else: # print('b is not None') # # val = b[0] # # for i in range(len(b)): # # if b[i] != val: # # print(b[i], val) # #del d # if __name__ == '__main__': # ds = lib_mp.MPDataSlot(size=512*1024*1024) # p = multiprocessing.Process(target=subproc, args=(ds,),) # p.daemon = True # p.start() # #import code # #code.interact(local=dict(globals(), **locals())) # #b = bytes(1024) # #c = 0 # #for i in range(1024): # # b[i] = c # #c = (c + 1) % 256 # #time.sleep(0.001) # a = ASD() # a._a = bytes(312*1024*1024) # while True: # ds.push(a) """ UUIDDB FaceDB(UUIDDB) _FaceBase | | uuid | image_name _FacePersonBase(_FaceBase) | | person_name FaceMark (_FacePersonBase) | | rect : FaceRect | lmrks_list FaceLandmarks | type (5, 68, ...) | np.ndarray FaceAlign (_FacePersonBase) | | uuid_face_mark | lmrks_type : FaceLandmarks.Type | source_to_align_mat : np.ndarray | coverage FaceSwap (_FacePersonBase) | | uuid_face_align | FaceSeg (_FaceBase) | | uuid_face_base """ # class UUIDBase(PicklableExpandable): # def __init__(self): # super().__init__() # self._uuid : uuid.UUID = uuid.uuid4() # def get_uuid(self) -> uuid.UUID: return self._uuid # class _FaceBase(PicklableExpandable): # def __init__(self): # super().__init__() # self._uuid : uuid.UUID = uuid.uuid4() # self._image_name : Union[str, None] = None # def get_uuid(self) -> uuid.UUID: return self._uuid # def get_image_name(self) -> Union[str, None]: return self._image_name # def set_image_name(self, image_name : Union[str, None]): # if image_name is not None and not isinstance(image_name, str): # raise ValueError(f'image_name must be an instance of str or None') # self._image_name = image_name # class _FacePersonBase(_FaceBase): # def __init__(self): # super().__init__() # self._person_name : Union[str, None] = None # def get_person_name(self) -> Union[str, None]: return self._person_name # def set_person_name(self, person_name : Union[str, None]): # if person_name is not None and not isinstance(person_name, str): # raise ValueError(f'person_name must be an instance of str or None') # self._person_name = person_name import pickle import struct import numpy as np class BinarySerializer: """ base binary serializer class """ def __init__(self): self.idx_stack = [] def get_cursor(self): raise NotImplementedError() def set_cursor(self, cursor): raise NotImplementedError() def write_raw_bytes(self, b_bytes): raise NotImplementedError() def read_raw_bytes(self, len): raise NotImplementedError() def read_raw_bytes_as_mv(self, len): """ read raw bytes as memoryview """ return memoryview(self.read_raw_bytes(len)) def get_raw_bytes(self, len, offset=None): self.push_idx(offset) result = self.read_raw_bytes(len) self.pop_idx() return result def push_idx(self, new_idx=None): self.idx_stack.append(self.get_cursor()) if new_idx is not None: self.set_cursor(new_idx) def pop_idx(self): self.set_cursor(self.idx_stack.pop()) def write_bytes(self, b_bytes, noneable=False): """ writes bytes() object """ if noneable: self.write_fmt('?', b_bytes is not None) if b_bytes is None: return self.get_cursor() self.write_fmt('Q', len(b_bytes)) return self.write_raw_bytes(b_bytes) def read_bytes(self, noneable=False): if noneable: has, = self.read_fmt('?') if not has: return None return self.read_raw_bytes( self.read_fmt('Q')[0] ) def write_fmt(self, fmt, *args): if noneables: self.write_fmt('?'*len(fmt), *(arg is not None for arg in args) ) self.write_fmt(''.join( fmt[i] if arg is not None else 'B' for i,arg in enumerate(args) ), *(arg if arg is not None else 0 for arg in args) ) else: self.write_raw_bytes( struct.pack(fmt, *args) ) def get_fmt(self, fmt): return struct.unpack (fmt, self.get_raw_bytes(struct.calcsize(fmt)) ) def read_fmt(self, fmt, noneables=False): if noneables: bools = self.read_fmt( '?'*len(fmt) ) y = self.read_fmt( ''.join(fmt[i] if has else 'B' for i,has in enumerate(bools)) ) return ( y[i] if has else None for i,has in enumerate(bools) ) else: fmt_size = struct.calcsize(fmt) return struct.unpack (fmt, self.read_raw_bytes(fmt_size)) def write_utf8(self, s): self.write_bytes( s.encode('utf-8') ) def read_utf8(self): return self.read_bytes().decode('utf-8') def write_ndarray(self, npar : np.ndarray, noneable=False): if noneable: self.write_fmt('?', npar is not None) if npar is None: return self.c shape = npar.shape dtype = npar.dtype np_view = memoryview(npar.reshape(-1)) format = np_view.format nbytes = np_view.nbytes ar_info = pickle.dumps( (shape, dtype, nbytes, format) ) self.write_bytes(ar_info) self.write_raw_bytes(np_view.cast('B')) return self.get_cursor() def read_ndarray(self, noneable=False): if noneable: has, = self.read_fmt('?') if not has: return None shape, dtype, nbytes, format = pickle.loads(self.read_bytes()) np_ar = np.empty(shape, dtype) memoryview(np_ar.reshape(-1))[:] = self.read_raw_bytes_as_mv(nbytes).cast(format) return np_ar class BinaryFileSerializer(BinarySerializer): def __init__(self, f, offset=0): super().__init__() self.f = f self.base_cursor = offset if offset != 0: self.set_cursor(0) def get_cursor(self): return self.f.tell()-self.base_cursor def set_cursor(self, cursor): self.f.seek(self.base_cursor+cursor, 0) def write_raw_bytes(self, b_bytes): self.f.write(b_bytes) return self.f.tell() def read_raw_bytes(self, len): return self.f.read(len) class BinaryMemorySerializer(BinarySerializer): def __init__(self, mv : memoryview, offset=0): super().__init__() if offset != 0: mv = mv[offset:] self.mv = mv self.c = 0 def get_cursor(self): return self.c def set_cursor(self, cursor): self.c = cursor def write_raw_bytes(self, b_bytes): b_bytes_len = len(b_bytes) self.mv[self.c:self.c+b_bytes_len] = b_bytes self.c += b_bytes_len return self.c def read_raw_bytes(self, size): result = self.mv[self.c:self.c+size].tobytes() self.c += size return result def read_raw_bytes_as_mv(self, size): result = self.mv[self.c:self.c+size] self.c += size return result def write_raw_bytes_from_mv(self, mv : memoryview, size): self.mv[self.c:self.c+size] = mv[:size] self.c += size return self.c def read_raw_bytes_to_mv (self, mv : memoryview, size): mv[:size] = self.mv[self.c:self.c+size] self.c += size # class MemoryViewFormatter: # def __init__(self, mv, offset=0): # if offset != 0: # mv = mv[offset:] # self.mv = mv # self.c = 0 # self.idx_stack = [] # def push_idx(self, new_idx): # self.idx_stack.append(self.c) # self.c = new_idx # def pop_idx(self): # self.c = self.idx_stack.pop() # def get_cursor(self): return self.c # def set_cursor(self, idx): self.c = idx # def write_utf8(self, s): # self.write_bytes( s.encode('utf-8') ) # def read_utf8(self): # return self.read_bytes().decode('utf-8') # def write_object(self, obj, noneable=False): # if noneable: # self.write_fmt('?', obj is not None) # if obj is None: # return self.c # if isinstance(obj, bytes): # self.write_fmt('I', 0) # elif isinstance(obj, np.ndarray): # self.write_fmt('I', 1) # # self.write_bytes( pickle.dumps(obj) if obj is not None else None, noneable=noneable ) # # def read_object(self, noneable=False): # # b = self.read_bytes(noneable=noneable) # # return pickle.loads(b) if b is not None else None # def write_bytes(self, b_bytes, noneable=False): # if noneable: # self.write_fmt('?', b_bytes is not None) # if b_bytes is None: # return self.c # b_bytes_len = len(b_bytes) # self.write_fmt('Q', b_bytes_len) # self.mv[self.c:self.c+b_bytes_len] = b_bytes # self.c += b_bytes_len # return self.c # def write_ndarray(self, npar : np.ndarray, noneable=False): # if noneable: # self.write_fmt('?', npar is not None) # if npar is None: # return self.c # shape = npar.shape # dtype = npar.dtype # np_view = memoryview(npar.reshape(-1)) # format = np_view.format # nbytes = np_view.nbytes # ar_info = pickle.dumps( (shape, dtype, nbytes, format) ) # self.write_bytes(ar_info) # self.mv[self.c:self.c+nbytes] = np_view.cast('B') # self.c += nbytes # return self.c # def read_ndarray(self, noneable=False): # if noneable: # has, = self.read_fmt('?') # if not has: # return None # shape, dtype, nbytes, format = pickle.loads(self.read_bytes()) # np_ar = np.empty(shape, dtype) # memoryview(np_ar.reshape(-1))[:] = self.mv[self.c:self.c+nbytes].cast(format) # self.c += nbytes # return np_ar # def read_bytes(self, noneable=False): # if noneable: # has, = self.read_fmt('?') # if not has: # return None # b_bytes_len, = self.read_fmt('Q') # b_bytes = self.mv[self.c:self.c+b_bytes_len].tobytes() # self.c += b_bytes_len # return b_bytes # def write_mvf(self, mvf : 'MemoryViewFormatter', offset, size): # self.write_fmt('Q', size) # self.mv[self.c:self.c+size] = mvf.mv[offset:offset+size] # self.c += size # def read_mvf(self, to_mv : memoryview = None): # size, = self.read_fmt('Q') # if to_mv is not None: # to_mv[0:size] = self.mv[self.c:self.c+size] # mvf = MemoryViewFormatter(to_mv, offset=0) # else: # mvf = MemoryViewFormatter(self.mv, offset=self.c) # self.c += size # return mvf # def write_fmt(self, fmt, *args, noneables=False): # if noneables: # self.write_fmt('?'*len(fmt), *(arg is not None for arg in args) ) # self.write_fmt(''.join( fmt[i] if arg is not None else 'B' for i,arg in enumerate(args) ), # *(arg if arg is not None else 0 for arg in args) ) # else: # fmt_size = struct.calcsize(fmt) # self.mv[self.c:self.c+fmt_size] = struct.pack(fmt, *args) # self.c += fmt_size # def read_fmt(self, fmt, noneables=False): # if noneables: # bools = self.read_fmt( '?'*len(fmt) ) # y = self.read_fmt( ''.join(fmt[i] if has else 'B' for i,has in enumerate(bools)) ) # return ( y[i] if has else None for i,has in enumerate(bools) ) # else: # fmt_size = struct.calcsize(fmt) # result = struct.unpack (fmt, self.mv[self.c:self.c+fmt_size]) # self.c += fmt_size # return result # def get(self, fmt): # fmt_size = struct.calcsize(fmt) # return struct.unpack (fmt, self.mv[self.c:self.c+fmt_size]) from localization import LQTFonts, LStrings from xlib import torch as lib_torch from xlib import qt as lib_qt from PyQt6.QtCore import * from PyQt6.QtGui import * from PyQt6.QtWidgets import * class QDeviceChooser(QFrame): """ on_choosed func(lib_torch.Devices) """ def __init__(self, devices : lib_torch.Devices, choose_multi=True, on_choosed=None): super().__init__() self.devices = devices self.choose_multi = choose_multi self.on_choosed = on_choosed if choose_multi: main_l = lib_qt.QXVBoxLayout() device_chboxes = self.device_chboxes = [] for i, device in enumerate(devices): c = lib_qt.QXCheckBox(text=f'{device.name}', font=LQTFonts.get_fixedwidth_font(size=8), toggled=lambda checked, i=i: self.on_toggled(i, checked) ) device_chboxes.append(c) main_l.addWidget(c) self.setLayout( lib_qt.QXVBoxLayout([ lib_qt.QXCollapsibleSection('Device', main_l, is_opened=True) ], contents_margins=(0,0,0,0) ) ) else: label_device = lib_qt.QXLabel('Device') cbox_choose = self.cbox_choose = lib_qt.QXComboBox(font=LQTFonts.get_fixedwidth_font(size=8), maximum_width=200 ) cbox_choose.addItem( QIcon(), '') for i, device in enumerate(devices): cbox_choose.addItem( QIcon(), f'{device.name}{device.name}{device.name}') cbox_choose.setCurrentIndex(0) cbox_choose.currentIndexChanged.connect(lambda idx: self.on_toggled(idx, True)) #self.setLayout( lib_qt.QXVBoxLayout([ lib_qt.QXCollapsibleSection('Device', lib_qt.QXHBoxLayout([cbox_choose], contents_margins=(0,0,0,0)) , is_opened=True, allow_open_close=False, show_content_frame=False) ], contents_margins=(0,0,0,0) ) ) self.setLayout( lib_qt.QXHBoxLayout([label_device, cbox_choose], contents_margins=(0,0,0,0)) ) def get_devices(self) -> lib_torch.Devices : return self.devices def unselect_all(self, block_signals : bool): if self.choose_multi: with lib_qt.BlockSignals(self.device_chboxes, block_signals=block_signals): for device_chbox in self.device_chboxes: device_chbox.setChecked(False) else: with lib_qt.BlockSignals(self.cbox_choose, block_signals=block_signals): self.cbox_choose.setCurrentIndex(0) def set_selected_index(self, index, is_selected, block_signals : bool): if self.choose_multi: with lib_qt.BlockSignals(self.device_chboxes, block_signals=block_signals): self.device_chboxes[index].setChecked(is_selected) elif is_selected: with lib_qt.BlockSignals(self.cbox_choose, block_signals=block_signals): self.cbox_choose.setCurrentIndex(index+1) def set_selected(self, devices : lib_torch.Devices, block_signals : bool): if self.choose_multi: with lib_qt.BlockSignals(self.device_chboxes, block_signals=block_signals): for i, device in enumerate(self.devices): self.device_chboxes[i].setChecked( True if device in devices else False ) else: with lib_qt.BlockSignals(self.cbox_choose, block_signals=block_signals): if len(devices) == 0: self.cbox_choose.setCurrentIndex(0) else: for i, device in enumerate(self.devices): if device == devices[0]: self.cbox_choose.setCurrentIndex(i+1) break def get_selected(self) -> lib_torch.Devices: if self.choose_multi: devices_list = [] for i, device_chbox in enumerate(self.device_chboxes): if device_chbox.isChecked(): devices_list.append(self.devices[i]) return lib_torch.Devices(devices_list) else: idx = self.cbox_choose.currentIndex() if idx == 0: return lib_torch.Devices() return lib_torch.Devices( [ self.devices[idx-1] ] ) def on_toggled(self, idx, checked): if self.choose_multi: with lib_qt.BlockSignals(self.device_chboxes): for device_chbox in self.device_chboxes: device_chbox.setChecked(False) self.device_chboxes[idx].setChecked(checked) if self.on_choosed is not None: self.on_choosed( self.get_selected() ) import multiprocessing import pickle import numpy as np from xlib import python as lib_python from .RWLock import RWLock from .MPAtomicInt32 import MPAtomicInt32 class MPDataSlotMirror: """ mirror for MPDataSlot for single advisor. """ def __init__(self, buffer_size=128*1024*1024): self._ar = ar = multiprocessing.RawArray('B', buffer_size) self._ar_view = memoryview(ar).cast('B') self._lock = RWLock() self._cached_dict = {} self._cached_dict_ver = -1 def commit(self, mvf_src, offset, size): mvf = lib_python.MemoryViewFormatter(self._ar_view) self._lock.write_lock() ver, = mvf.get('I') mvf.write('I', ver+1) mvf.write_mvf(mvf_src, offset, size) self._lock.write_unlock() def get(self): mvf = lib_python.MemoryViewFormatter(self._ar_view) self._lock.read_lock() ver, = mvf.read('I') mvf_bytes = mvf.read_bytes() self._lock.read_unlock() mvf = lib_python.MemoryViewFormatter( memoryview(mvf_bytes) ) import code code.interact(local=dict(globals(), **locals())) class MPDataSlot: """ Multiprocess high performance multireader-multiwriter DataSlot. The slot mean only one dict can exist in slot in one time. """ def __init__(self, buffer_size=128*1024*1024, use_mirror=False): self._ar = ar = multiprocessing.RawArray('B', buffer_size) self._ar_view = memoryview(ar).cast('B') self._atom = MPAtomicInt32(ar=ar, index=0) self.use_mirror = use_mirror self._dsm = MPDataSlotMirror(buffer_size) if use_mirror else None #self.push( {} ) #self.pop() def get_dsm(self): return self._dsm def push(self, d, wait=True): """ arguments d dict wait True: wait while other side pop the data and push the dict returns True False: returns True if success push right now, otherwise the slot is busy and returns False """ if not isinstance(d, dict): raise ValueError('only dict can be pushed') bytes_buffers = [] ndarray_buffers = [] dict_key_vals = [] for key in d: value = d[key] if isinstance(value, bytes): bytes_buffers.append( (pickle.dumps(key), value) ) elif isinstance(value, np.ndarray): ndarray_buffers.append( (pickle.dumps(key), value) ) else: dict_key_vals.append( (key, value) ) dict_key_vals_bytes = pickle.dumps(dict_key_vals, 4) mvf = lib_python.MemoryViewFormatter(self._ar_view, offset=4) if wait: while True: if self._atom.compare_exchange(0, 1) == 1: break elif self._atom.compare_exchange(0, 1) != 1: return False mvf.write_bytes(dict_key_vals_bytes) mvf.write('II', len(bytes_buffers), len(ndarray_buffers)) for key_bytes, bytes_buffer in bytes_buffers: mvf.write_bytes(key_bytes) mvf.write_bytes(bytes_buffer) for key_bytes, nd_ar in ndarray_buffers: mvf.write_bytes(key_bytes) mvf.write_ndarray(nd_ar) if self.use_mirror: self._dsm.commit( mvf, 0, mvf.get_idx() ) self._atom.set(2) return True def pop(self, wait=True): """ """ mvf = lib_python.MemoryViewFormatter(self._ar_view, offset=4) ndarray_buffers = [] bytes_buffers = [] if wait: while True: if self._atom.compare_exchange(2, 1) == 1: break elif self._atom.compare_exchange(2, 1) != 1: return None dict_key_vals = pickle.loads(mvf.read_bytes()) bytes_buffers_len, ndarray_buffers_len = mvf.read('II') for i in range(bytes_buffers_len): bytes_buffers.append( (mvf.read_bytes(), mvf.read_bytes()) ) for i in range(ndarray_buffers_len): ndarray_buffers.append( (mvf.read_bytes(), mvf.read_ndarray()) ) self._atom.set(0) d = {} for key,value in dict_key_vals: d[key] = value for key_bytes, bytes_buffer in bytes_buffers: d[pickle.loads(key_bytes)] = bytes_buffer for key_bytes, nd_ar in ndarray_buffers: d[pickle.loads(key_bytes)] = nd_ar return d def __getstate__(self): d = self.__dict__.copy() # pop unpicklable memoryview object d.pop('_ar_view') return d def __setstate__(self, d): # restore memoryview of RawArray d['_ar_view'] = memoryview(d['_ar']).cast('B') self.__dict__.update(d) import multiprocessing import pickle import numpy as np from xlib import python as lib_python from .RWLock import RWLock class MPDataPipe: """ DataPipe between two processes. """ def __init__(self, buffer_size=4096*4096*3*4): self._ar = ar = multiprocessing.RawArray('B', buffer_size) self._ar_view = memoryview(ar).cast('B') self._ev = multiprocessing.Event() self._lock = RWLock() self.set( {} ) def set(self, d): """ set dict without pushing an event increments current version arguments d dict """ if not isinstance(d, dict): raise ValueError('only dict can be pushed') bytes_buffers = [] ndarray_buffers = [] for key in d: value = d[key] if isinstance(value, bytes): # Remove bytes() value to separated buffer bytes_buffers.append( (pickle.dumps(key), value) ) d[key] = None elif isinstance(value, np.ndarray): # Remove np.ndarray value to separated buffer ndarray_buffers.append( (pickle.dumps(key), value) ) d[key] = None dict_bytes = pickle.dumps(d, 4) bf = lib_python.MemoryViewFormatter(self._ar_view) self._lock.write_lock() idx = bf.get_idx() ver, = bf.read('Q') bf.set_idx(idx) bf.write('Q', ver+1) bf.write_bytes(dict_bytes) bf.write('II', len(bytes_buffers), len(ndarray_buffers)) for key_bytes, bytes_buffer in bytes_buffers: bf.write_bytes(key_bytes) bf.write_bytes(bytes_buffer) for key_bytes, nd_ar in ndarray_buffers: bf.write_bytes(key_bytes) bf.write_ndarray(nd_ar) self._lock.write_unlock() def push(self, d): """ set dict, increments current version, and set an event arguments d dict """ self.set(d) self._ev.set() def get_ver(self) -> int: """ Get current version of dict. """ bf = lib_python.MemoryViewFormatter(self._ar_view) ver, = bf.read('Q') return ver def get(self, with_ver=False): """ Get current or last dict without checking and clear event. """ bf = lib_python.MemoryViewFormatter(self._ar_view) bytes_buffers = [] ndarray_buffers = [] self._lock.read_lock() ver, = bf.read('Q') dict_bytes = bf.read_bytes() bytes_buffers_len, ndarray_buffers_len = bf.read('II') for i in range(bytes_buffers_len): bytes_buffers.append( (bf.read_bytes(), bf.read_bytes()) ) for i in range(ndarray_buffers_len): ndarray_buffers.append( (bf.read_bytes(), bf.read_ndarray()) ) self._lock.read_unlock() d = pickle.loads(dict_bytes) for key_bytes, bytes_buffer in bytes_buffers: d[pickle.loads(key_bytes)] = bytes_buffer for key_bytes, nd_ar in ndarray_buffers: d[pickle.loads(key_bytes)] = nd_ar if with_ver: return ver, d else: return d def pop(self, timeout_sec=None): """ Get dict only if event is set returns None if event is not set """ ev = self._ev if (timeout_sec is None and ev.is_set()) or \ (timeout_sec is not None and ev.wait(timeout_sec)): d = self.get() ev.clear() return d return None def is_pushed(self): """ return True if data is already pushed, and still did not retrieved by other side """ return self._ev.is_set() def __getstate__(self): d = self.__dict__.copy() # pop unpicklable memoryview object d.pop('_ar_view') return d def __setstate__(self, d): # restore memoryview of RawArray d['_ar_view'] = memoryview(d['_ar']).cast('B') self.__dict__.update(d) class HCDataSheet1: """ Base class for data-sheet between host and client process. arguments var_names list of string of variables in datasheet Host "commits" suggested data. Client validate the data and can send changes back to the host with error messages for every param. Suitable when speed of data transfer does not matter. """ def __init__(self, var_names): self.var_names = var_names self.pickled = False self.conn, self.other_conn = multiprocessing.Pipe() self.error_msgs = {} for v in var_names: setattr(self, v, None) def commit(self): """ Commit the data """ self.conn.send( (self._dump_vars(), self.error_msgs) ) self.error_msgs = {} def pull(self, timeout=0): """ Pulls new version of sheet from other process. Returns None if nothing received. otherwise returns dotdict (dict with dot access) of variables, and dotdict of error msgs of vars You should merge these variables manually timeout 0sec : only check, don't wait None : wait infinite """ if self.conn.poll(timeout): vars, errs = self.conn.recv() return dotdict(vars), dotdict(errs) return None, None def set_error(self, var_name, msg): """ set error message for var_name for new commit. All error messages will be cleaned after commit. """ if var_name not in self.var_names: raise ValueError(f'{var_name} is not registered var_name in data sheet.') self.error_msgs[var_name] = msg def has_errors(self): return len(self.error_msgs.keys()) != 0 def _dump_vars(self): return { v : getattr(self, v) for v in self.var_names } def __getstate__(self): if self.pickled: raise Exception(f'{self.__class__.__name__} can be pickled only once') self.pickled = True d = dict() d['conn'] = self.other_conn d['vars'] = self._dump_vars() return d def __setstate__(self, d): self.conn = d['conn'] vars = d['vars'] self.var_names = list(vars.keys()) for v in vars: setattr(self, v, vars[v]) self.error_msgs = {} class StreamNode: def __init__(self, fssh): self.fssh = fssh # def fssh_initialize(self): # fssh = self.fssh # fssh.image_sequence_dir = None # fssh.fps = 30 # fssh.commit() # def fssh_merge(self, fssh, fssh_new): # fps_new = fssh_new.fps # if fssh.fps != fps_new: # if fps_new < 1: # fssh.set_error('fps', 'fps cannot be < 1') # else: # fssh.fps = fps_new # if fssh.has_errors(): # fssh.commit() @staticmethod def proc(*args): StreamNode(*args).run() def run(self): #self.fssh_initialize() #fssh = self.fssh while True: """ check new sheet merge and check vars if some vars are invalid, flag to commit new version """ #fssh_new, _ = fssh.pull() #if fssh_new is not None: # self.fssh_merge(fssh, fssh_new) time.sleep(0.001) class HCDataSheet: """ DataSheet between Host and Client """ class Variable(): def __init__(self): self.value = None self.err_msg = None self.on_merge_func = None def err(self, msg): self.err_msg = msg def call_on_merge(self, func): if self.on_merge_func is not None: raise Exception('on_merge_func is already set') self.on_merge_func = func def __call__(self, *args, err_msg=None, by_pull=False, **kwargs): args_len = len(args) if args_len == 0: return self.value elif args_len == 1: old_value = self.value new_value = args[0] if by_pull and self.on_merge_func is not None: self.on_merge_func(self, old_value, new_value, err_msg) else: self.value = new_value def __init__(self): self.pickled = False self._on_merge_funcs = [] self.conn, self.other_conn = multiprocessing.Pipe() def call_on_merge(self, func): """Call the func when the datasheet is updated by other side""" self._on_merge_funcs.append(func) def commit(self, err_msg=None): """ Commit the data """ self.conn.send( (self._dump_vars(), err_msg) ) def pull(self, timeout=0): """ Pulls new version of sheet from other process. timeout 0sec : only check, don't wait None : wait infinite """ if self.conn.poll(timeout): vars, errs = self.conn.recv() for v in vars: v_var = getattr(self, v, None) v_var(vars[v], err_msg=errs[v], by_pull=True) def has_errors(self): for v in vars(self): v_val = getattr(self, v) if isinstance(v_val, HCDataSheet.Variable): if v_val.err_msg is not None: return True return False def _get_var_names(self): d = [] for v in vars(self): v_val = getattr(self, v) if isinstance(v_val, HCDataSheet.Variable): d.append(v) return d def _dump_vars(self): d = {} for v in vars(self): v_val = getattr(self, v) if isinstance(v_val, HCDataSheet.Variable): d[v] = v_val.value return d def _dump_errs(self, clear=False): d = {} for v in vars(self): v_val = getattr(self, v) if isinstance(v_val, HCDataSheet.Variable): d[v] = v_val.err_msg if clear: v_val.err_msg = None return d def __getstate__(self): #if self.pickled: # raise Exception(f'{self.__class__.__name__} can be pickled only once') #self.pickled = True d = dict() d['conn'] = self.other_conn d['vars'] = self._get_var_names() return d def __setstate__(self, d): self.conn = d['conn'] vars = d['vars'] for v in vars: setattr(self, v, HCDataSheet.Variable())