StreamOutput: added mpegts udp output

apps/DeepFaceLive/backend/StreamOutput.py

@@ -10,6 +10,7 @@ from xlib import os as lib_os
 from xlib import time as lib_time
 from xlib.image import ImageProcessor
 from xlib.mp import csw as lib_csw
+from xlib.streamer import FFMPEGStreamer
 
 from .BackendBase import (BackendConnection, BackendDB, BackendHost,
                           BackendSignal, BackendWeakHeap, BackendWorker,
@@ -43,16 +44,17 @@ class SourceType(IntEnum):
     SOURCE_N_MERGED_FRAME = 5
     SOURCE_N_MERGED_FRAME_OR_SOURCE_FRAME = 6
 
-ViewModeNames = ['@StreamOutput.SourceType.SOURCE_FRAME', 
+ViewModeNames = ['@StreamOutput.SourceType.SOURCE_FRAME',
-                 '@StreamOutput.SourceType.ALIGNED_FACE', 
+                 '@StreamOutput.SourceType.ALIGNED_FACE',
                  '@StreamOutput.SourceType.SWAPPED_FACE',
-                 '@StreamOutput.SourceType.MERGED_FRAME', 
+                 '@StreamOutput.SourceType.MERGED_FRAME',
-                 '@StreamOutput.SourceType.MERGED_FRAME_OR_SOURCE_FRAME', 
+                 '@StreamOutput.SourceType.MERGED_FRAME_OR_SOURCE_FRAME',
                  '@StreamOutput.SourceType.SOURCE_N_MERGED_FRAME',
                  '@StreamOutput.SourceType.SOURCE_N_MERGED_FRAME_OR_SOURCE_FRAME',
                  ]
 
 
+
 class StreamOutputWorker(BackendWorker):
     def get_state(self) -> 'WorkerState': return super().get_state()
     def get_control_sheet(self) -> 'Sheet.Worker': return super().get_control_sheet()
@@ -73,7 +75,9 @@ class StreamOutputWorker(BackendWorker):
 
         self._wnd_name = 'DeepFaceLive output'
         self._wnd_showing = False
-
+        
+        self._streamer = FFMPEGStreamer()
+        
         lib_os.set_timer_resolution(1)
 
         state, cs = self.get_state(), self.get_control_sheet()
@@ -84,7 +88,10 @@ class StreamOutputWorker(BackendWorker):
         cs.target_delay.call_on_number(self.on_cs_target_delay)
         cs.save_sequence_path.call_on_paths(self.on_cs_save_sequence_path)
         cs.save_fill_frame_gap.call_on_flag(self.on_cs_save_fill_frame_gap)
-
+        cs.is_streaming.call_on_flag(self.on_cs_is_streaming)
+        cs.stream_addr.call_on_text(self.on_cs_stream_addr)
+        cs.stream_port.call_on_number(self.on_cs_stream_port)
+        
         cs.source_type.enable()
         cs.source_type.set_choices(SourceType, ViewModeNames, none_choice_name='@misc.menu_select')
         cs.source_type.select(state.source_type)
@@ -107,16 +114,26 @@ class StreamOutputWorker(BackendWorker):
             state.is_showing_window = not state.is_showing_window
             cs.show_hide_window.signal()
 
-
         cs.save_sequence_path.enable()
         cs.save_sequence_path.set_config( lib_csw.Paths.Config.Directory('Choose output sequence directory', directory_path=save_default_path) )
         cs.save_sequence_path.set_paths(state.sequence_path)
 
         cs.save_fill_frame_gap.enable()
         cs.save_fill_frame_gap.set_flag(state.save_fill_frame_gap if state.save_fill_frame_gap is not None else True )
+            
+        cs.is_streaming.enable()
+        cs.is_streaming.set_flag(state.is_streaming if state.is_streaming is not None else False )
+        
+        cs.stream_addr.enable()
+        cs.stream_addr.set_text(state.stream_addr if state.stream_addr is not None else '127.0.0.1')
+        
+        cs.stream_port.enable()
+        cs.stream_port.set_config(lib_csw.Number.Config(min=1, max=9999, decimals=0, allow_instant_update=True))
+        cs.stream_port.set_number(state.stream_port if state.stream_port is not None else 1234)
 
-
+    def on_stop(self):
-
+        self._streamer.stop()
+        
     def on_cs_source_type(self, idx, source_type):
         state, cs = self.get_state(), self.get_control_sheet()
         if source_type == SourceType.ALIGNED_FACE:
@@ -126,10 +143,10 @@ class StreamOutputWorker(BackendWorker):
         else:
             cs.aligned_face_id.disable()
         state.source_type = source_type
-        
+
         self.save_state()
         self.reemit_frame_signal.send()
-        
+
     def show_window(self):
         state, cs = self.get_state(), self.get_control_sheet()
         cv2.namedWindow(self._wnd_name)
@@ -189,6 +206,23 @@ class StreamOutputWorker(BackendWorker):
         state.save_fill_frame_gap = save_fill_frame_gap
         self.save_state()
 
+    def on_cs_is_streaming(self, is_streaming):
+        state, cs = self.get_state(), self.get_control_sheet()
+        state.is_streaming = is_streaming
+        self.save_state()
+    
+    def on_cs_stream_addr(self, stream_addr):
+        state, cs = self.get_state(), self.get_control_sheet()
+        state.stream_addr = stream_addr
+        self.save_state()
+        self._streamer.set_addr_port(state.stream_addr, state.stream_port)
+        
+    def on_cs_stream_port(self, stream_port):
+        state, cs = self.get_state(), self.get_control_sheet()
+        state.stream_port = stream_port
+        self.save_state()
+        self._streamer.set_addr_port(state.stream_addr, state.stream_port)
+    
     def on_tick(self):
         cs, state = self.get_control_sheet(), self.get_state()
 
@@ -204,7 +238,9 @@ class StreamOutputWorker(BackendWorker):
 
             source_type = state.source_type
             if source_type is not None and \
-                (state.is_showing_window or state.sequence_path is not None):
+                (state.is_showing_window or \
+                 state.sequence_path is not None or \
+                 state.is_streaming):
                 buffered_frames = self.buffered_frames
 
                 view_image = None
@@ -213,7 +249,7 @@ class StreamOutputWorker(BackendWorker):
                     view_image = bcd.get_image(bcd.get_frame_image_name())
                 elif source_type in [SourceType.MERGED_FRAME, SourceType.MERGED_FRAME_OR_SOURCE_FRAME]:
                     view_image = bcd.get_image(bcd.get_merged_image_name())
-                    if view_image is None and source_type == SourceType.MERGED_FRAME_OR_SOURCE_FRAME:                       
+                    if view_image is None and source_type == SourceType.MERGED_FRAME_OR_SOURCE_FRAME:
                         view_image = bcd.get_image(bcd.get_frame_image_name())
 
                 elif source_type == SourceType.ALIGNED_FACE:
@@ -228,17 +264,17 @@ class StreamOutputWorker(BackendWorker):
                         view_image = bcd.get_image(fsi.face_swap_image_name)
                         if view_image is not None:
                             break
-                    
+
                 elif source_type in [SourceType.SOURCE_N_MERGED_FRAME, SourceType.SOURCE_N_MERGED_FRAME_OR_SOURCE_FRAME]:
                     source_frame = bcd.get_image(bcd.get_frame_image_name())
                     if source_frame is not None:
                         source_frame = ImageProcessor(source_frame).to_ufloat32().get_image('HWC')
-                    
+
                     merged_frame = bcd.get_image(bcd.get_merged_image_name())
-                    
+
-                    if merged_frame is None and source_type == SourceType.SOURCE_N_MERGED_FRAME_OR_SOURCE_FRAME:                       
+                    if merged_frame is None and source_type == SourceType.SOURCE_N_MERGED_FRAME_OR_SOURCE_FRAME:
                         merged_frame = source_frame
-                    
+
                     if source_frame is not None and merged_frame is not None:
                         view_image = np.concatenate( (source_frame, merged_frame), 1 )
 
@@ -259,8 +295,13 @@ class StreamOutputWorker(BackendWorker):
                 pr = buffered_frames.process()
 
                 img = pr.new_data
-                if state.is_showing_window and img is not None:
+                if img is not None:
-                    cv2.imshow(self._wnd_name, img)
+                    if state.is_streaming:
+                        img = ImageProcessor(view_image).to_uint8().get_image('HWC')
+                        self._streamer.push_frame(img)
+
+                    if state.is_showing_window:
+                        cv2.imshow(self._wnd_name, img)
 
         if state.is_showing_window:
             cv2.waitKey(1)
@@ -277,7 +318,10 @@ class Sheet:
             self.save_sequence_path = lib_csw.Paths.Client()
             self.save_sequence_path_error = lib_csw.Error.Client()
             self.save_fill_frame_gap = lib_csw.Flag.Client()
-
+            self.is_streaming = lib_csw.Flag.Client()
+            self.stream_addr = lib_csw.Text.Client()
+            self.stream_port = lib_csw.Number.Client()
+            
     class Worker(lib_csw.Sheet.Worker):
         def __init__(self):
             super().__init__()
@@ -289,7 +333,10 @@ class Sheet:
             self.save_sequence_path = lib_csw.Paths.Host()
             self.save_sequence_path_error = lib_csw.Error.Host()
             self.save_fill_frame_gap = lib_csw.Flag.Host()
-
+            self.is_streaming = lib_csw.Flag.Host()
+            self.stream_addr = lib_csw.Text.Host()
+            self.stream_port = lib_csw.Number.Host()
+            
 class WorkerState(BackendWorkerState):
     source_type : SourceType = None
     is_showing_window : bool = None
@@ -297,3 +344,6 @@ class WorkerState(BackendWorkerState):
     target_delay : int = None
     sequence_path : Path = None
     save_fill_frame_gap : bool = None
+    is_streaming : bool = None
+    stream_addr : str = None
+    stream_port : int = None

apps/DeepFaceLive/ui/QStreamOutput.py

@@ -1,5 +1,7 @@
 from localization import L
+from resources.fonts import QXFontDB
 from xlib import qt as qtx
+from xlib.qt.widgets.QXLabel import QXLabel
 
 from ..backend import StreamOutput
 from .widgets.QBackendPanel import QBackendPanel
@@ -9,6 +11,7 @@ from .widgets.QComboBoxCSWDynamicSingleSwitch import \
 from .widgets.QErrorCSWError import QErrorCSWError
 from .widgets.QLabelCSWNumber import QLabelCSWNumber
 from .widgets.QLabelPopupInfo import QLabelPopupInfo
+from .widgets.QLineEditCSWText import QLineEditCSWText
 from .widgets.QPathEditCSWPaths import QPathEditCSWPaths
 from .widgets.QSpinBoxCSWNumber import QSpinBoxCSWNumber
 from .widgets.QXPushButtonCSWSignal import QXPushButtonCSWSignal
@@ -39,6 +42,12 @@ class QStreamOutput(QBackendPanel):
         q_save_fill_frame_gap_label = QLabelPopupInfo(label=L('@QStreamOutput.save_fill_frame_gap'), popup_info_text=L('@QStreamOutput.help.save_fill_frame_gap'))
         q_save_fill_frame_gap       = QCheckBoxCSWFlag(cs.save_fill_frame_gap, reflect_state_widgets=[q_save_fill_frame_gap_label])
 
+        q_is_streaming_label = QLabelPopupInfo(label='mpegts udp://')
+        q_is_streaming       = QCheckBoxCSWFlag(cs.is_streaming, reflect_state_widgets=[q_is_streaming_label])
+
+        q_stream_addr = QLineEditCSWText(cs.stream_addr, font=QXFontDB.get_fixedwidth_font())
+        q_stream_port = QSpinBoxCSWNumber(cs.stream_port)
+
         grid_l = qtx.QXGridLayout(spacing=5)
         row = 0
         grid_l.addWidget(q_average_fps_label, row, 0, 1, 1, alignment=qtx.AlignRight | qtx.AlignVCenter )
@@ -61,9 +70,11 @@ class QStreamOutput(QBackendPanel):
         row += 1
         grid_l.addLayout( qtx.QXHBoxLayout([q_save_fill_frame_gap, 4, q_save_fill_frame_gap_label]), row, 1, 1, 2, alignment=qtx.AlignLeft | qtx.AlignVCenter )
         row += 1
-
         grid_l.addWidget(q_save_sequence_path_error, row, 0, 1, 3)
         row += 1
+        grid_l.addLayout( qtx.QXHBoxLayout([q_is_streaming, 4, q_is_streaming_label]), row, 0, 1, 1, alignment=qtx.AlignRight | qtx.AlignVCenter )
+        grid_l.addLayout( qtx.QXHBoxLayout([q_stream_addr, qtx.QXLabel(text=':'), q_stream_port]), row, 1, 1, 2, alignment=qtx.AlignLeft | qtx.AlignVCenter )
+        row += 1
 
         super().__init__(backend, L('@QStreamOutput.module_title'),
                          layout=grid_l)

apps/DeepFaceLive/ui/widgets/QLineEditCSWText.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+from resources.fonts import QXFontDB
+from resources.gfx import QXImageDB
+from xlib import qt as qtx
+from xlib.mp import csw as lib_csw
+
+from .QCSWControl import QCSWControl
+
+
+class QLineEditCSWText(QCSWControl):
+    def __init__(self, csw_text : lib_csw.Text.Client, 
+                       font = None,
+                       reflect_state_widgets=None):
+        """
+        Implements lib_csw.Text control as LineEdit
+        """
+        if not isinstance(csw_text, lib_csw.Text.Client):
+            raise ValueError('csw_path must be an instance of Text.Client')
+
+        self._csw_text = csw_text
+        self._dlg = None
+
+        csw_text.call_on_text(self._on_csw_text)
+        
+        if font is None:
+            font = QXFontDB.get_default_font()
+        lineedit = self._lineedit = qtx.QXLineEdit(font=font,
+                                                   placeholder_text='...',
+                                                   size_policy=('expanding', 'fixed'),
+                                                   editingFinished=self.on_lineedit_editingFinished)
+
+        super().__init__(csw_control=csw_text, reflect_state_widgets=reflect_state_widgets,
+                         layout=qtx.QXHBoxLayout([lineedit]) )
+
+    def _on_csw_text(self, text):
+
+        with qtx.BlockSignals(self._lineedit):
+            self._lineedit.setText(text)
+
+    def on_lineedit_editingFinished(self):
+        text = self._lineedit.text()
+        if len(text) == 0:
+            text = None
+        self._csw_text.set_text(text)

doc/setup_tutorial_windows/setup_for_streaming.md

@@ -9,7 +9,7 @@ DeepFaceLive only provides a video window of the replaced face. Since the face m
 
 So, what do we need for streaming?
 
-Capture window and sound with some delay, transmit to streaming service (e.g. twitch, youtube, ...)
+Capture window (or receive mpegts udp stream) and sound with some delay, transmit to streaming service (e.g. twitch, youtube, ...)
 
 Below, one of the solutions.
 
@@ -63,8 +63,27 @@ Below, one of the solutions.
 </td></tr>
 <tr><td colspan=2 align="center">
 
-### DONE !
+### **DONE** !
 ### Now you can stream yourself to a stream service.
 
 </td></tr>
+
+<tr><td colspan=2 align="center">
+
+### instead of **Window capture** 
+
+you can use **Media Source**
+
+<img src="media_source.png"></img>
+
+with configuration:
+
+<img src="media_source_config.png"></img>
+
+enable mpegts in **_Stream Output_**
+
+<img src="Media_source_stream_output.png"></img>
+
+</td></tr>
+
 </table>

xlib/streamer/FFMPEGStreamer.py

@@ -0,0 +1,52 @@
+import numpy as np
+
+from .. import ffmpeg as lib_ffmpeg
+
+
+class FFMPEGStreamer:
+    def __init__(self):
+        self._ffmpeg_proc = None
+        self._addr = '127.0.0.1'
+        self._port = 1234
+        self._width = 320
+        self._height = 240
+
+    def set_addr_port(self, addr : str, port : int):
+        if self._addr != addr or self._port != port:
+            self._addr = addr
+            self._port = port
+            self.stop()
+    
+    def stop(self):
+        if self._ffmpeg_proc is not None:
+            self._ffmpeg_proc.kill()
+            self._ffmpeg_proc = None
+        
+    def _restart(self):
+        self.stop()
+        args = ['-y', '-re',
+                '-f', 'rawvideo',
+                '-vcodec','rawvideo',
+                '-pix_fmt', 'bgr24',
+                '-s', f'{self._width}:{self._height}',
+                '-i', '-',
+                '-f', 'mpegts',
+                '-q:v', '2',
+                f'udp://{self._addr}:{self._port}'
+        ]
+        self._ffmpeg_proc = lib_ffmpeg.run (args, pipe_stdin=True, quiet_stderr=True)#, pipe_stderr=True)
+
+    def push_frame(self, img : np.ndarray):
+        H,W,C = img.shape
+        if self._width != W or self._height != H:
+            self._width = W
+            self._height = H
+            self.stop()
+            
+        if self._ffmpeg_proc is None:
+            self._restart()
+        
+        try:
+            self._ffmpeg_proc.stdin.write(img)
+        except:
+            self.stop()

xlib/streamer/__init__.py

@@ -0,0 +1 @@
+from .FFMPEGStreamer import FFMPEGStreamer