code release

This commit is contained in:
iperov 2021-07-23 17:34:49 +04:00
commit a902f11f74
354 changed files with 826570 additions and 1 deletions

View file

@ -0,0 +1,261 @@
import multiprocessing
import sys
import time
import traceback
from PyQt6.QtCore import *
from PyQt6.QtGui import *
from PyQt6.QtWidgets import *
from .qtex import *
class QSubprocessor(object):
"""
"""
class Cli(object):
def __init__ ( self, client_dict ):
s2c = multiprocessing.Queue()
c2s = multiprocessing.Queue()
self.p = multiprocessing.Process(target=self._subprocess_run, args=(client_dict,s2c,c2s) )
self.s2c = s2c
self.c2s = c2s
self.p.daemon = True
self.p.start()
self.state = None
self.sent_time = None
self.sent_data = None
self.name = None
self.host_dict = None
def kill(self):
self.p.terminate()
self.p.join()
#overridable optional
def on_initialize(self, client_dict):
#initialize your subprocess here using client_dict
pass
#overridable optional
def on_finalize(self):
#finalize your subprocess here
pass
#overridable
def process_data(self, data):
#process 'data' given from host and return result
raise NotImplementedError
#overridable optional
def get_data_name (self, data):
#return string identificator of your 'data'
return "undefined"
def log_info(self, msg): self.c2s.put ( {'op': 'log_info', 'msg':msg } )
def log_err(self, msg): self.c2s.put ( {'op': 'log_err' , 'msg':msg } )
def progress_bar_inc(self, c): self.c2s.put ( {'op': 'progress_bar_inc' , 'c':c } )
def _subprocess_run(self, client_dict, s2c, c2s):
self.c2s = c2s
data = None
try:
self.on_initialize(client_dict)
c2s.put ( {'op': 'init_ok'} )
while True:
msg = s2c.get()
op = msg.get('op','')
if op == 'data':
data = msg['data']
result = self.process_data (data)
c2s.put ( {'op': 'success', 'data' : data, 'result' : result} )
data = None
elif op == 'close':
break
time.sleep(0.001)
self.on_finalize()
c2s.put ( {'op': 'finalized'} )
except Exception as e:
c2s.put ( {'op': 'error', 'data' : data} )
if data is not None:
print ('Exception while process data [%s]: %s' % (self.get_data_name(data), traceback.format_exc()) )
else:
print ('Exception: %s' % (traceback.format_exc()) )
c2s.close()
s2c.close()
self.c2s = None
# disable pickling
def __getstate__(self):
return dict()
def __setstate__(self, d):
self.__dict__.update(d)
#overridable
def __init__(self, name, SubprocessorCli_class, no_response_time_sec = 0, io_loop_sleep_time=0.005):
if not issubclass(SubprocessorCli_class, QSubprocessor.Cli):
raise ValueError("SubprocessorCli_class must be subclass of QSubprocessor.Cli")
self.name = name
self.SubprocessorCli_class = SubprocessorCli_class
self.no_response_time_sec = no_response_time_sec
self.io_loop_sleep_time = io_loop_sleep_time
self.clis = []
#getting info about name of subprocesses, host and client dicts, and spawning them
for name, host_dict, client_dict in self.process_info_generator():
try:
cli = self.SubprocessorCli_class(client_dict)
cli.state = 1
cli.sent_time = 0
cli.sent_data = None
cli.name = name
cli.host_dict = host_dict
self.clis.append (cli)
except:
raise Exception (f"Unable to start subprocess {name}. Error: {traceback.format_exc()}")
if len(self.clis) == 0:
raise Exception ("Unable to start QSubprocessor '%s' " % (self.name))
#waiting subprocesses their success(or not) initialization
while True:
for cli in self.clis[:]:
while not cli.c2s.empty():
obj = cli.c2s.get()
op = obj.get('op','')
if op == 'init_ok':
cli.state = 0
elif op == 'log_info':
print(obj['msg'])
elif op == 'log_err':
print(obj['msg'])
elif op == 'error':
cli.kill()
self.clis.remove(cli)
break
if all ([cli.state == 0 for cli in self.clis]):
break
time.sleep(0.005)
if len(self.clis) == 0:
raise Exception ( "Unable to start subprocesses." )
#ok some processes survived, initialize host logic
self.on_clients_initialized()
self.q_timer = QTimer()
self.q_timer.timeout.connect(self.tick)
self.q_timer.start(5)
#overridable
def process_info_generator(self):
#yield per process (name, host_dict, client_dict)
for i in range(min(multiprocessing.cpu_count(), 8) ):
yield 'CPU%d' % (i), {}, {}
#overridable optional
def on_clients_initialized(self):
#logic when all subprocesses initialized and ready
pass
#overridable optional
def on_clients_finalized(self):
#logic when all subprocess finalized
pass
#overridable
def get_data(self, host_dict):
#return data for processing here
raise NotImplementedError
#overridable
def on_data_return (self, host_dict, data):
#you have to place returned 'data' back to your queue
raise NotImplementedError
#overridable
def on_result (self, host_dict, data, result):
#your logic what to do with 'result' of 'data'
raise NotImplementedError
def tick(self):
for cli in self.clis[:]:
while not cli.c2s.empty():
obj = cli.c2s.get()
op = obj.get('op','')
if op == 'success':
#success processed data, return data and result to on_result
self.on_result (cli.host_dict, obj['data'], obj['result'])
self.sent_data = None
cli.state = 0
elif op == 'error':
#some error occured while process data, returning chunk to on_data_return
if 'data' in obj.keys():
self.on_data_return (cli.host_dict, obj['data'] )
#and killing process
cli.kill()
self.clis.remove(cli)
elif op == 'log_info':
print(obj['msg'])
elif op == 'log_err':
print(obj['msg'])
elif op == 'progress_bar_inc':
...
#io.progress_bar_inc(obj['c'])
for cli in self.clis[:]:
if cli.state == 1:
if cli.sent_time != 0 and self.no_response_time_sec != 0 and (time.time() - cli.sent_time) > self.no_response_time_sec:
#subprocess busy too long
print ( '%s doesnt response, terminating it.' % (cli.name) )
self.on_data_return (cli.host_dict, cli.sent_data )
cli.kill()
self.clis.remove(cli)
for cli in self.clis[:]:
if cli.state == 0:
#free state of subprocess, get some data from get_data
data = self.get_data(cli.host_dict)
if data is not None:
#and send it to subprocess
cli.s2c.put ( {'op': 'data', 'data' : data} )
cli.sent_time = time.time()
cli.sent_data = data
cli.state = 1
if all ([cli.state == 0 for cli in self.clis]):
#gracefully terminating subprocesses
for cli in self.clis[:]:
cli.s2c.put ( {'op': 'close'} )
cli.sent_time = time.time()
while True:
for cli in self.clis[:]:
terminate_it = False
while not cli.c2s.empty():
obj = cli.c2s.get()
obj_op = obj['op']
if obj_op == 'finalized':
terminate_it = True
break
if (time.time() - cli.sent_time) > 30:
terminate_it = True
if terminate_it:
cli.state = 2
cli.kill()
if all ([cli.state == 2 for cli in self.clis]):
break
#finalizing host logic
self.q_timer.stop()
self.q_timer = None
self.on_clients_finalized()

235
xlib/qt/_unused/_unused.py Normal file
View file

@ -0,0 +1,235 @@
# from PyQt6.QtCore import *
# from PyQt6.QtGui import *
# from PyQt6.QtWidgets import *
# #from localization import StringsDB
# from .QXMainWindow import *
# class QXIconButton(QPushButton):
# """
# Custom Icon button that works through keyEvent system, without shortcut of QAction
# works only with QXMainWindow as global window class
# currently works only with one-key shortcut
# """
# def __init__(self, icon,
# tooltip=None,
# shortcut=None,
# click_func=None,
# first_repeat_delay=300,
# repeat_delay=20,
# ):
# super().__init__(icon, "")
# self.setIcon(icon)
# if shortcut is not None:
# tooltip = f"{tooltip} ( S_HOT_KEY: {shortcut} )"
# self.setToolTip(tooltip)
# self.seq = QKeySequence(shortcut) if shortcut is not None else None
# QXMainWindow.inst.add_keyPressEvent_listener ( self.on_keyPressEvent )
# QXMainWindow.inst.add_keyReleaseEvent_listener ( self.on_keyReleaseEvent )
# self.click_func = click_func
# self.first_repeat_delay = first_repeat_delay
# self.repeat_delay = repeat_delay
# self.repeat_timer = None
# self.op_device = None
# self.pressed.connect( lambda : self.action(is_pressed=True) )
# self.released.connect( lambda : self.action(is_pressed=False) )
# def action(self, is_pressed=None, op_device=None):
# if self.click_func is None:
# return
# if is_pressed is not None:
# if is_pressed:
# if self.repeat_timer is None:
# self.click_func()
# self.repeat_timer = QTimer()
# self.repeat_timer.timeout.connect(self.action)
# self.repeat_timer.start(self.first_repeat_delay)
# else:
# if self.repeat_timer is not None:
# self.repeat_timer.stop()
# self.repeat_timer = None
# else:
# self.click_func()
# if self.repeat_timer is not None:
# self.repeat_timer.setInterval(self.repeat_delay)
# def on_keyPressEvent(self, ev):
# key = ev.nativeVirtualKey()
# if ev.isAutoRepeat():
# return
# if self.seq is not None:
# if key == self.seq[0]:
# self.action(is_pressed=True)
# def on_keyReleaseEvent(self, ev):
# key = ev.nativeVirtualKey()
# if ev.isAutoRepeat():
# return
# if self.seq is not None:
# if key == self.seq[0]:
# self.action(is_pressed=False)
############################
############################
############################
############################
############################
# class QXTabWidget(QTabWidget):
# def __init__(self, tabs=None, tab_shape=None, size_policy=None, maximum_width=None, hided=False, enabled=True):
# super().__init__()
# if tabs is not None:
# for tab,icon,name in tabs:
# self.addTab(tab, icon, name)
# if tab_shape is not None:
# self.setTabShape(tab_shape)
# if maximum_width is not None:
# self.setMaximumWidth(maximum_width)
# if size_policy is not None:
# self.setSizePolicy(*size_policy)
# if hided:
# self.hide()
# self.setEnabled(enabled)
# class QXComboObjectBox(QXComboBox):
# """
# as QComboBox but str'able Iterable of objects
# and more functionality
# """
# def __init__(self, choices : Iterable, none_choice=None, font=None, size_policy=None, maximum_width=None, hided=False, enabled=True, on_choosed=None):
# super().__init__(font=font, size_policy=size_policy, maximum_width=maximum_width, hided=hided, enabled=enabled)
# self.choices = tuple(choices)
# if len(self.choices) == 0:
# raise ValueError('Number of choices are 0')
# self.none_choice = none_choice
# self.on_choosed = on_choosed
# if none_choice is not None:
# self.addItem( QIcon(), str(none_choice) )
# for i, choice in enumerate(choices):
# self.addItem( QIcon(), str(choice) )
# self.setCurrentIndex(0)
# self.currentIndexChanged.connect(self.on_toggled)
# def get_choices(self):
# return self.choices
# def get_selected_choice(self):
# idx = self.currentIndex()
# if self.none_choice is not None:
# idx -= 1
# if idx == -1:
# return None
# return self.choices[idx]
# def unselect(self, block_signals : bool = False):
# if self.none_choice is not None:
# with BlockSignals(self, block_signals=block_signals):
# self.setCurrentIndex(0)
# def set_selected_index(self, index, block_signals : bool = False):
# if index >= 0 and index < len(self.choices):
# if self.none_choice is not None:
# index += 1
# with BlockSignals(self, block_signals=block_signals):
# self.setCurrentIndex(index)
# def set_selected_choice(self, choice, block_signals : bool = False):
# with BlockSignals(self, block_signals=block_signals):
# if choice is None:
# if self.none_choice is not None:
# self.setCurrentIndex(0)
# else:
# raise ValueError('unable to change to None with none_choice=False')
# else:
# for i, schoice in enumerate(self.choices):
# if schoice == choice:
# self.setCurrentIndex(i+1)
# break
# def on_toggled(self, idx):
# if self.on_choosed is not None:
# self.on_choosed( self.get_selected_choice() )
# class QXCollapsibleSection(QWidget):
# def __init__(self, title, content_layout, is_opened=False, allow_open_close=True, show_content_frame=True):
# super().__init__()
# btn = self.btn = QToolButton()
# btn.setToolButtonStyle(Qt.ToolButtonStyle.ToolButtonTextBesideIcon)
# btn.setStyleSheet('border: none;')
# btn.setArrowType(Qt.ArrowType.RightArrow)
# btn.setText(title)
# #btn.setIconSize( QSize(8,8))
# btn.setCheckable(True)
# btn.setChecked(False)
# if allow_open_close:
# btn.toggled.connect(self.on_btn_toggled)
# line = QXFrame( size_policy=(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Maximum) )
# line.setFrameShape(QFrame.Shape.HLine)
# frame = self.frame = QXFrame( size_policy=(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed), layout=content_layout, hided=True)
# if show_content_frame:
# frame.setFrameShape(QFrame.Shape.StyledPanel)
# main_l = QXGridLayout( contents_margins=0 )
# main_l.addWidget(btn, 0, 0, alignment=Qt.AlignmentFlag.AlignLeft)
# main_l.addWidget(QXHorizontalLine() , 0, 1)
# main_l.addWidget(frame, 1, 0, 1, 2)
# self.setLayout(main_l)
# if is_opened:
# self.open()
# else:
# self.close()
# def is_opened(self):
# return self.frame.isVisible()
# def open(self):
# self.btn.setArrowType(Qt.ArrowType.DownArrow)
# self.frame.show()
# def close(self):
# self.btn.setArrowType(Qt.ArrowType.RightArrow)
# self.frame.hide()
# def on_btn_toggled(self):
# if self.btn.isChecked():
# self.open()
# else:
# self.close()

View file

@ -0,0 +1,62 @@
import numpy as np
from PyQt6.QtCore import *
from PyQt6.QtGui import *
from PyQt6.QtWidgets import *
# class QActionEx(QAction):
# def __init__(self, icon, text, shortcut=None, trigger_func=None, shortcut_in_tooltip=False, is_checkable=False, is_auto_repeat=False ):
# super().__init__(icon, text)
# if shortcut is not None:
# self.setShortcut(shortcut)
# if shortcut_in_tooltip:
# self.setToolTip( f"{text} ( S_HOT_KEY : {shortcut} )")
# if trigger_func is not None:
# self.triggered.connect(trigger_func)
# if is_checkable:
# self.setCheckable(True)
# self.setAutoRepeat(is_auto_repeat)
# def QImage_from_np(img):
# if img.dtype != np.uint8:
# raise ValueError("img should be in np.uint8 format")
# h,w,c = img.shape
# if c == 1:
# fmt = QImage.Format_Grayscale8
# elif c == 3:
# fmt = QImage.Format_BGR888
# elif c == 4:
# fmt = QImage.Format_ARGB32
# else:
# raise ValueError("unsupported channel count")
# return QImage(img.data, w, h, c*w, fmt )
# def QImage_to_np(q_img, fmt=QImage.Format_BGR888):
# q_img = q_img.convertToFormat(fmt)
# width = q_img.width()
# height = q_img.height()
# b = q_img.constBits()
# b.setsize(height * width * 3)
# arr = np.frombuffer(b, np.uint8).reshape((height, width, 3))
# return arr#[::-1]
# def QPixmap_from_np(img):
# return QPixmap.fromImage(QImage_from_np(img))
# def QPoint_from_np(n):
# return QPoint(*n.astype(np.int))
# def QPoint_to_np(q):
# return np.int32( [q.x(), q.y()] )
# def QSize_to_np(q):
# return np.int32( [q.width(), q.height()] )