From 3c559bd6892ba19f7b6c6fa961c5314f61da93f6 Mon Sep 17 00:00:00 2001 From: flandre Date: Mon, 14 Apr 2025 01:34:05 +0800 Subject: [PATCH] add midi function --- flandre/launcher.py | 2 ++ flandre/nodes/Beamformer.py | 2 +- flandre/nodes/MainUI.py | 39 +++++++++++++++++++++ flandre/nodes/Midi.py | 69 +++++++++++++++++++++++++++++++++---- flandre/pyqt/Main.py | 46 +++++++++++++------------ flandre/pyqt/Main.ui | 23 ++++++++----- flandre/utils/Msg.py | 8 +++-- flandre/utils/RfMat.py | 5 +++ 8 files changed, 154 insertions(+), 40 deletions(-) diff --git a/flandre/launcher.py b/flandre/launcher.py index 5870cd3..42928b3 100644 --- a/flandre/launcher.py +++ b/flandre/launcher.py @@ -17,6 +17,7 @@ from flandre.nodes.Muxer import Muxer from flandre.nodes.Robot import Robot from flandre.nodes.ImageFFMPEG import ImageFFMPEG from flandre.nodes.ImageQt import ImageQt +from flandre.nodes.Midi import Midi from flandre.utils.Msg import KillMsg from flandre.config import CONFIG_FOLDER @@ -31,6 +32,7 @@ class LaunchComponent(Enum): Beamformer = Beamformer ImageFFMPEG = ImageFFMPEG ImageQt = ImageQt + Midi = Midi def launch(arg: dict[LaunchComponent, dict]): diff --git a/flandre/nodes/Beamformer.py b/flandre/nodes/Beamformer.py index 73404fd..e2783ba 100644 --- a/flandre/nodes/Beamformer.py +++ b/flandre/nodes/Beamformer.py @@ -40,7 +40,7 @@ class Beamformer(Node): .call(cp.asarray, order='C') .argrelextrema() .conv_guass(b=arg.beta * 0.01) - .crop(arg.t_start, arg.t_end) + .crop_center(arg.t_start, arg.t_end) .rotate90() .cpu() ) diff --git a/flandre/nodes/MainUI.py b/flandre/nodes/MainUI.py index 3ae1282..83a3158 100644 --- a/flandre/nodes/MainUI.py +++ b/flandre/nodes/MainUI.py @@ -345,12 +345,25 @@ class Adv(QMainWindow, Ui_MainWindow): elif isinstance(msg, ImageArgMsg): self.arg = msg self.arg.sender = 'ui' + self.s_t_start.setValue(msg.t_start) self.s_t_end.setValue(msg.t_end) self.s_f_rows.setValue(msg.f_rows) self.s_v2.setValue(msg.v2) self.s_dct_center.setValue(msg.dct_center) self.s_dct_bandwidth.setValue(msg.dct_bandwidth) + self.s_beta.setValue(msg.beta) + + self.sp_crop_center.setValue(msg.t_start) + self.sp_crop_width.setValue(msg.t_end) + self.sp_f_rows.setValue(msg.f_rows) + self.sp_v2.setValue(msg.v2) + self.sp_dct_center.setValue(msg.dct_center) + self.sp_dct_bandwidth.setValue(msg.dct_bandwidth) + self.sp_beta.setValue(msg.beta) + + + elif isinstance(msg, MoveAxisMsg): match msg.axis: case 'S': @@ -376,11 +389,19 @@ class Adv(QMainWindow, Ui_MainWindow): elif isinstance(msg, SetSeqMetaMsg): self.seq_meta = RfSequenceMeta.from_name(msg.name) mmax_shape0 = max(self.seq_meta.shape) + self.s_t_start.setMaximum(mmax_shape0) self.s_t_end.setMaximum(mmax_shape0) self.s_dct_center.setMaximum(mmax_shape0) self.s_dct_bandwidth.setMaximum(mmax_shape0) self.s_f_rows.setMaximum(mmax_shape0) + + self.sp_crop_center.setMaximum(mmax_shape0) + self.sp_crop_width.setMaximum(mmax_shape0) + self.sp_dct_center.setMaximum(mmax_shape0) + self.sp_dct_bandwidth.setMaximum(mmax_shape0) + self.sp_f_rows.setMaximum(mmax_shape0) + elif isinstance(msg, DeviceConnectedMsg): if msg.value: self.set_device_connection(LinkStatus.GREEN) @@ -456,6 +477,24 @@ class Adv(QMainWindow, Ui_MainWindow): if self.cb_bscan.sender() is None: self.p.send(SetWindowVisibleMsg('ui', 'bscan', v == 2)) + @pyqtSlot(int) + def on_sp_crop_center_valueChanged(self, v): + if self.sp_crop_center.sender() is None: + self.arg.t_start = v + self.p.send(self.arg) + + @pyqtSlot(int) + def on_sp_crop_width_valueChanged(self, v): + if self.sp_crop_width.sender() is None: + self.arg.t_end = v + self.p.send(self.arg) + + @pyqtSlot(int) + def on_sp_v2_valueChanged(self, v): + if self.sp_v2.sender() is None: + self.arg.v2 = v + self.p.send(self.arg) + @pyqtSlot(int) def on_s_beta_valueChanged(self, v): if self.s_beta.sender() is None: diff --git a/flandre/nodes/Midi.py b/flandre/nodes/Midi.py index 8b2fd77..2514246 100644 --- a/flandre/nodes/Midi.py +++ b/flandre/nodes/Midi.py @@ -1,5 +1,6 @@ import logging from threading import Thread +from unittest import case import mido import zmq @@ -7,12 +8,15 @@ from mido import Message from mido.backends.rtmidi import Input, Output from flandre.nodes.Node import Node -from flandre.utils.Msg import KillMsg, MidiMsg, Msg, ImageArgMsg +from flandre.utils.Msg import KillMsg, MidiMsg, Msg, ImageArgMsg, SetSeqMetaMsg, SetSidMsg +from flandre.utils.RfMeta import RfSequenceMeta logger = logging.getLogger(__name__) class Midi(Node): + topics = [ImageArgMsg, SetSeqMetaMsg] + def __init__(self, level=logging.INFO): super(Midi, self).__init__(level=level) self.m_input: Input = None @@ -20,6 +24,15 @@ class Midi(Node): self.do_loop = True self.t_midi_event_loop: Thread = None self.isa: zmq.Socket = None + self.arg = ImageArgMsg('midi', 0) + + self.m_t_start = 100 + self.m_t_end = 100 + self.m_dct_center = 100 + self.m_dct_bandwidth = 100 + self.m_f_rows = 100 + self.last_pitch: dict[int, int] = dict() + self.sid = 0 def custom_setup(self): self.isa = self.c.ctx.socket(zmq.PUSH) @@ -33,16 +46,32 @@ class Midi(Node): def midi_event_loop(self): while self.do_loop: midi_msg: Message = self.m_input.receive() - # print(midi_msg) + print(midi_msg) d = midi_msg.dict() match d['type']: case 'pitchwheel': channel = d['channel'] # 0-127 pitch = int(d['pitch'] / 128) + 64 - # pitch_p = int(pitch / 127) - self.isa.send(MidiMsg(type='pitchwheel', channel=channel, pitch=pitch).encode_msg()) - + if channel not in self.last_pitch: + self.last_pitch[channel] = pitch + elif abs(pitch - self.last_pitch[channel]) > 5: + pass + else: + self.last_pitch[channel] = pitch + # pitch_p = int(pitch / 127) + self.isa.send(MidiMsg(type='pitchwheel', channel=channel, pitch=pitch).encode_msg()) + case 'control_change': + for i in range(16, 24): + if i in [d['control'], d['value']]: + if 1 in [d['control'], d['value']]: + self.isa.send(MidiMsg(type='control_change', value=1, control=i - 16).encode_msg()) + else: + self.isa.send(MidiMsg(type='control_change', value=-1, control=i - 16).encode_msg()) + case 'note_on': + self.isa.send(MidiMsg(type='note_on', note=d['note'], velocity=d['velocity']).encode_msg()) + case 'note_off': + self.isa.send(MidiMsg(type='note_off', note=d['note'], velocity=d['velocity']).encode_msg()) def loop(self): isb = self.c.ctx.socket(zmq.PULL) isb.connect("inproc://midi") @@ -50,16 +79,42 @@ class Midi(Node): while True: p = dict(self.c.poller.poll()) if isb in p: - msg = Msg.decode_msg(isb.recv()) + msg: MidiMsg = Msg.decode_msg(isb.recv()) match msg.type: case 'pitchwheel': match msg.channel: case 0: + self.arg.v2 = int(100 + 6000 * (msg.pitch / 127)) # print(msg.pitch) - self.send(ImageArgMsg('midi', 0, msg.pitch * 3)) + self.send(self.arg) + case 'control_change': + match msg.control: + case 0: + self.arg.t_start = sorted((1, self.arg.t_start + msg.value * 10, self.m_t_start))[1] + case 1: + self.arg.t_end = sorted((1, self.arg.t_end + msg.value * 10, self.m_t_end))[1] + case 2: + self.arg.v2 = sorted((500, self.arg.v2 + msg.value * 10, 7000))[1] + case 'note_on': + self.sid += 1 + self.send(SetSidMsg(self.sid)) + + self.send(self.arg) if self.c.sub in p: msg = self.recv() if isinstance(msg, KillMsg): if msg.name == '': self.do_loop = False return + elif isinstance(msg, ImageArgMsg): + if msg.sender != 'midi': + self.arg = msg + self.arg.sender = 'midi' + elif isinstance(msg, SetSeqMetaMsg): + seq_meta = RfSequenceMeta.from_name(msg.name) + mmax_shape0 = max(seq_meta.shape) + self.m_t_start = mmax_shape0 + self.m_t_end = mmax_shape0 + self.m_dct_center = mmax_shape0 + self.m_dct_bandwidth = mmax_shape0 + self.m_f_rows = mmax_shape0 diff --git a/flandre/pyqt/Main.py b/flandre/pyqt/Main.py index 2970f4e..4c28324 100644 --- a/flandre/pyqt/Main.py +++ b/flandre/pyqt/Main.py @@ -485,9 +485,9 @@ class Ui_MainWindow(object): self.label_7 = QtWidgets.QLabel(parent=self.centralwidget) self.label_7.setObjectName("label_7") self.gridLayout_5.addWidget(self.label_7, 6, 0, 1, 1) - self.spinBox = QtWidgets.QSpinBox(parent=self.centralwidget) - self.spinBox.setObjectName("spinBox") - self.gridLayout_5.addWidget(self.spinBox, 4, 2, 1, 1) + self.sp_crop_width = QtWidgets.QSpinBox(parent=self.centralwidget) + self.sp_crop_width.setObjectName("sp_crop_width") + self.gridLayout_5.addWidget(self.sp_crop_width, 4, 2, 1, 1) spacerItem2 = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Expanding) self.gridLayout_5.addItem(spacerItem2, 10, 1, 1, 1) self.s_t_end = QJumpSlider(parent=self.centralwidget) @@ -496,9 +496,9 @@ class Ui_MainWindow(object): self.s_t_end.setOrientation(QtCore.Qt.Orientation.Horizontal) self.s_t_end.setObjectName("s_t_end") self.gridLayout_5.addWidget(self.s_t_end, 4, 1, 1, 1) - self.spinBox_7 = QtWidgets.QSpinBox(parent=self.centralwidget) - self.spinBox_7.setObjectName("spinBox_7") - self.gridLayout_5.addWidget(self.spinBox_7, 7, 2, 1, 1) + self.sp_dct_bandwidth = QtWidgets.QSpinBox(parent=self.centralwidget) + self.sp_dct_bandwidth.setObjectName("sp_dct_bandwidth") + self.gridLayout_5.addWidget(self.sp_dct_bandwidth, 7, 2, 1, 1) self.label_5 = QtWidgets.QLabel(parent=self.centralwidget) self.label_5.setObjectName("label_5") self.gridLayout_5.addWidget(self.label_5, 4, 0, 1, 1) @@ -518,15 +518,17 @@ class Ui_MainWindow(object): self.label_6 = QtWidgets.QLabel(parent=self.centralwidget) self.label_6.setObjectName("label_6") self.gridLayout_5.addWidget(self.label_6, 5, 0, 1, 1) - self.spinBox_2 = QtWidgets.QSpinBox(parent=self.centralwidget) - self.spinBox_2.setObjectName("spinBox_2") - self.gridLayout_5.addWidget(self.spinBox_2, 5, 2, 1, 1) - self.spinBox_3 = QtWidgets.QSpinBox(parent=self.centralwidget) - self.spinBox_3.setObjectName("spinBox_3") - self.gridLayout_5.addWidget(self.spinBox_3, 6, 2, 1, 1) - self.spinBox_13 = QtWidgets.QSpinBox(parent=self.centralwidget) - self.spinBox_13.setObjectName("spinBox_13") - self.gridLayout_5.addWidget(self.spinBox_13, 8, 2, 1, 1) + self.sp_v2 = QtWidgets.QSpinBox(parent=self.centralwidget) + self.sp_v2.setMinimum(500) + self.sp_v2.setMaximum(7000) + self.sp_v2.setObjectName("sp_v2") + self.gridLayout_5.addWidget(self.sp_v2, 5, 2, 1, 1) + self.sp_dct_center = QtWidgets.QSpinBox(parent=self.centralwidget) + self.sp_dct_center.setObjectName("sp_dct_center") + self.gridLayout_5.addWidget(self.sp_dct_center, 6, 2, 1, 1) + self.sp_f_rows = QtWidgets.QSpinBox(parent=self.centralwidget) + self.sp_f_rows.setObjectName("sp_f_rows") + self.gridLayout_5.addWidget(self.sp_f_rows, 8, 2, 1, 1) self.s_f_rows = QtWidgets.QSlider(parent=self.centralwidget) self.s_f_rows.setMinimum(500) self.s_f_rows.setMaximum(7000) @@ -538,7 +540,7 @@ class Ui_MainWindow(object): self.s_dct_bandwidth.setObjectName("s_dct_bandwidth") self.gridLayout_5.addWidget(self.s_dct_bandwidth, 7, 1, 1, 1) self.s_v2 = QtWidgets.QSlider(parent=self.centralwidget) - self.s_v2.setMinimum(1) + self.s_v2.setMinimum(500) self.s_v2.setMaximum(7000) self.s_v2.setProperty("value", 5900) self.s_v2.setOrientation(QtCore.Qt.Orientation.Horizontal) @@ -556,9 +558,9 @@ class Ui_MainWindow(object): self.label_2.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter) self.label_2.setObjectName("label_2") self.gridLayout_5.addWidget(self.label_2, 0, 0, 1, 3) - self.spinBox_12 = QtWidgets.QSpinBox(parent=self.centralwidget) - self.spinBox_12.setObjectName("spinBox_12") - self.gridLayout_5.addWidget(self.spinBox_12, 3, 2, 1, 1) + self.sp_crop_center = QtWidgets.QSpinBox(parent=self.centralwidget) + self.sp_crop_center.setObjectName("sp_crop_center") + self.gridLayout_5.addWidget(self.sp_crop_center, 3, 2, 1, 1) self.s_beta = QtWidgets.QSlider(parent=self.centralwidget) self.s_beta.setMinimum(1) self.s_beta.setMaximum(30) @@ -569,9 +571,9 @@ class Ui_MainWindow(object): self.label_35 = QtWidgets.QLabel(parent=self.centralwidget) self.label_35.setObjectName("label_35") self.gridLayout_5.addWidget(self.label_35, 9, 0, 1, 1) - self.spinBox_14 = QtWidgets.QSpinBox(parent=self.centralwidget) - self.spinBox_14.setObjectName("spinBox_14") - self.gridLayout_5.addWidget(self.spinBox_14, 9, 2, 1, 1) + self.sp_beta = QtWidgets.QSpinBox(parent=self.centralwidget) + self.sp_beta.setObjectName("sp_beta") + self.gridLayout_5.addWidget(self.sp_beta, 9, 2, 1, 1) self.gridLayout.addLayout(self.gridLayout_5, 3, 0, 1, 1) self.gridLayout.setRowStretch(0, 1) MainWindow.setCentralWidget(self.centralwidget) diff --git a/flandre/pyqt/Main.ui b/flandre/pyqt/Main.ui index c2ea02d..a9aca95 100644 --- a/flandre/pyqt/Main.ui +++ b/flandre/pyqt/Main.ui @@ -910,7 +910,7 @@ border-radius: 7px; - + @@ -939,7 +939,7 @@ border-radius: 7px; - + @@ -977,13 +977,20 @@ border-radius: 7px; - + + + 500 + + + 7000 + + - + - + @@ -1008,7 +1015,7 @@ border-radius: 7px; - 1 + 500 7000 @@ -1045,7 +1052,7 @@ border-radius: 7px; - + @@ -1071,7 +1078,7 @@ border-radius: 7px; - + diff --git a/flandre/utils/Msg.py b/flandre/utils/Msg.py index cecd9fa..5f76240 100644 --- a/flandre/utils/Msg.py +++ b/flandre/utils/Msg.py @@ -403,8 +403,12 @@ class DeviceZero(Msg): @dataclasses.dataclass class MidiMsg(Msg): type: str - channel: int - pitch: int + channel: int = None + pitch: int = None + control: int = None + value: int = None + velocity: int = None + note: int = None @dataclasses.dataclass diff --git a/flandre/utils/RfMat.py b/flandre/utils/RfMat.py index b5a2e9c..5b7ea4c 100644 --- a/flandre/utils/RfMat.py +++ b/flandre/utils/RfMat.py @@ -210,6 +210,11 @@ class RfMat: def crop(self, t_start: int, t_end: int): return self.copy(self.m[:, t_start:t_end]) + def crop_center(self, center: float, width: float): + mmin = max(0, int(center - width / 2)) + mmax = min(self.duration, int(center + width / 2)) + return self.crop(mmin, mmax) + def watermark(self, watermark=None): assert self.m.dtype == np.uint8 canvas = np.zeros(self.m.shape, dtype=np.uint8)