Add piano roll track editing

This commit is contained in:
Skyler Lehmkuhl 2025-10-23 23:10:56 -04:00
parent 3de1b05fb3
commit 976b41cb83
9 changed files with 942 additions and 46 deletions

View File

@ -504,6 +504,31 @@ impl Engine {
// Add a pre-loaded MIDI clip to the track // Add a pre-loaded MIDI clip to the track
let _ = self.project.add_midi_clip(track_id, clip); let _ = self.project.add_midi_clip(track_id, clip);
} }
Command::UpdateMidiClipNotes(track_id, clip_id, notes) => {
// Update all notes in a MIDI clip
if let Some(crate::audio::track::TrackNode::Midi(track)) = self.project.get_track_mut(track_id) {
if let Some(clip) = track.clips.iter_mut().find(|c| c.id == clip_id) {
// Clear existing events
clip.events.clear();
// Add new events from the notes array
for (start_time, note, velocity, duration) in notes {
// Convert time to sample timestamp
let timestamp = (start_time * self.sample_rate as f64) as u64;
let note_on = MidiEvent::note_on(timestamp, 0, note, velocity);
clip.events.push(note_on);
// Add note off event
let note_off_timestamp = ((start_time + duration) * self.sample_rate as f64) as u64;
let note_off = MidiEvent::note_off(note_off_timestamp, 0, note, 64);
clip.events.push(note_off);
}
// Sort events by timestamp
clip.events.sort_by_key(|e| e.timestamp);
}
}
}
Command::RequestBufferPoolStats => { Command::RequestBufferPoolStats => {
// Send buffer pool statistics back to UI // Send buffer pool statistics back to UI
let stats = self.buffer_pool.stats(); let stats = self.buffer_pool.stats();
@ -1010,6 +1035,11 @@ impl EngineController {
let _ = self.command_tx.push(Command::AddLoadedMidiClip(track_id, clip)); let _ = self.command_tx.push(Command::AddLoadedMidiClip(track_id, clip));
} }
/// Update all notes in a MIDI clip
pub fn update_midi_clip_notes(&mut self, track_id: TrackId, clip_id: MidiClipId, notes: Vec<(f64, u8, u8, f64)>) {
let _ = self.command_tx.push(Command::UpdateMidiClipNotes(track_id, clip_id, notes));
}
/// Request buffer pool statistics /// Request buffer pool statistics
/// The statistics will be sent via an AudioEvent::BufferPoolStats event /// The statistics will be sent via an AudioEvent::BufferPoolStats event
pub fn request_buffer_pool_stats(&mut self) { pub fn request_buffer_pool_stats(&mut self) {

View File

@ -76,6 +76,9 @@ pub enum Command {
AddMidiNote(TrackId, MidiClipId, f64, u8, u8, f64), AddMidiNote(TrackId, MidiClipId, f64, u8, u8, f64),
/// Add a pre-loaded MIDI clip to a track /// Add a pre-loaded MIDI clip to a track
AddLoadedMidiClip(TrackId, MidiClip), AddLoadedMidiClip(TrackId, MidiClip),
/// Update MIDI clip notes (track_id, clip_id, notes: Vec<(start_time, note, velocity, duration)>)
/// NOTE: May need to switch to individual note operations if this becomes slow on clips with many notes
UpdateMidiClipNotes(TrackId, MidiClipId, Vec<(f64, u8, u8, f64)>),
// Diagnostics commands // Diagnostics commands
/// Request buffer pool statistics /// Request buffer pool statistics

Binary file not shown.

View File

@ -507,6 +507,23 @@ pub async fn audio_load_midi_file(
} }
} }
#[tauri::command]
pub async fn audio_update_midi_clip_notes(
state: tauri::State<'_, Arc<Mutex<AudioState>>>,
track_id: u32,
clip_id: u32,
notes: Vec<(f64, u8, u8, f64)>,
) -> Result<(), String> {
let mut audio_state = state.lock().unwrap();
if let Some(controller) = &mut audio_state.controller {
controller.update_midi_clip_notes(track_id, clip_id, notes);
Ok(())
} else {
Err("Audio not initialized".to_string())
}
}
#[derive(serde::Serialize, Clone)] #[derive(serde::Serialize, Clone)]
#[serde(tag = "type")] #[serde(tag = "type")]
pub enum SerializedAudioEvent { pub enum SerializedAudioEvent {

View File

@ -211,6 +211,7 @@ pub fn run() {
audio::audio_create_midi_clip, audio::audio_create_midi_clip,
audio::audio_add_midi_note, audio::audio_add_midi_note,
audio::audio_load_midi_file, audio::audio_load_midi_file,
audio::audio_update_midi_clip_notes,
audio::audio_send_midi_note_on, audio::audio_send_midi_note_on,
audio::audio_send_midi_note_off, audio::audio_send_midi_note_off,
]) ])

105
src/assets/piano-roll.svg Normal file
View File

@ -0,0 +1,105 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
width="48"
height="48"
viewBox="0 0 12.7 12.7"
version="1.1"
id="svg1"
inkscape:version="1.4 (e7c3feb100, 2024-10-09)"
sodipodi:docname="piano.svg"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg">
<sodipodi:namedview
id="namedview1"
pagecolor="#505050"
bordercolor="#ffffff"
borderopacity="1"
inkscape:showpageshadow="0"
inkscape:pageopacity="0"
inkscape:pagecheckerboard="1"
inkscape:deskcolor="#505050"
inkscape:document-units="mm"
inkscape:zoom="16.035511"
inkscape:cx="22.231908"
inkscape:cy="24.102756"
inkscape:window-width="2256"
inkscape:window-height="1432"
inkscape:window-x="0"
inkscape:window-y="0"
inkscape:window-maximized="1"
inkscape:current-layer="layer1" />
<defs
id="defs1">
<inkscape:path-effect
effect="fillet_chamfer"
id="path-effect1"
is_visible="true"
lpeversion="1"
nodesatellites_param="F,0,0,1,0,0.79375001,0,1 @ F,0,0,1,0,0.79375001,0,1 @ F,0,0,1,0,0.79375001,0,1 @ F,0,0,1,0,0.79375001,0,1"
radius="3"
unit="px"
method="auto"
mode="F"
chamfer_steps="1"
flexible="false"
use_knot_distance="true"
apply_no_radius="true"
apply_with_radius="true"
only_selected="false"
hide_knots="false" />
</defs>
<g
inkscape:label="Layer 1"
inkscape:groupmode="layer"
id="layer1">
<path
style="fill:none;stroke:#ffffff;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round"
id="rect1"
width="10.427897"
height="10.427897"
x="1.1219889"
y="1.1714885"
inkscape:path-effect="#path-effect1"
sodipodi:type="rect"
d="m 1.9157389,1.1714885 h 8.8403971 a 0.79375001,0.79375001 45 0 1 0.79375,0.79375 v 8.8403975 a 0.79375001,0.79375001 135 0 1 -0.79375,0.79375 H 1.9157389 a 0.79375001,0.79375001 45 0 1 -0.79375,-0.79375 l 0,-8.8403975 a 0.79375001,0.79375001 135 0 1 0.79375,-0.79375 z" />
<path
style="fill:none;stroke:#ffffff;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round"
d="M 6.3946773,6.6255579 V 11.361012"
id="path1" />
<path
style="fill:none;stroke:#ffffff;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round"
d="M 3.6389113,6.6255579 V 11.361012"
id="path1-5" />
<path
style="fill:none;stroke:#ffffff;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round"
d="M 9.1504432,6.6255589 V 11.361012"
id="path1-2" />
<rect
style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round"
id="rect2"
width="1.6664836"
height="5.4449468"
x="5.537518"
y="1.2044882" />
<rect
style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round"
id="rect2-7"
width="1.6664836"
height="5.4449468"
x="8.4957237"
y="1.245263"
inkscape:transform-center-x="2.557475"
inkscape:transform-center-y="1.4189861" />
<rect
style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round"
id="rect2-6"
width="1.6664836"
height="5.4449468"
x="2.5793126"
y="1.2770908" />
</g>
</svg>

After

Width:  |  Height:  |  Size: 3.6 KiB

View File

@ -61,7 +61,7 @@ import {
shadow, shadow,
} from "./styles.js"; } from "./styles.js";
import { Icon } from "./icon.js"; import { Icon } from "./icon.js";
import { AlphaSelectionBar, ColorSelectorWidget, ColorWidget, HueSelectionBar, SaturationValueSelectionGradient, TimelineWindow, TimelineWindowV2, VirtualPiano, Widget } from "./widgets.js"; import { AlphaSelectionBar, ColorSelectorWidget, ColorWidget, HueSelectionBar, SaturationValueSelectionGradient, TimelineWindow, TimelineWindowV2, VirtualPiano, PianoRollEditor, Widget } from "./widgets.js";
// State management // State management
import { import {
@ -746,42 +746,6 @@ window.addEventListener("DOMContentLoaded", () => {
createPane(panes.stage), createPane(panes.stage),
); );
// Add audio test button (temporary for Phase 0)
const testBtn = document.createElement('button');
testBtn.textContent = 'Test Audio';
testBtn.style.position = 'fixed';
testBtn.style.top = '10px';
testBtn.style.right = '10px';
testBtn.style.zIndex = '10000';
testBtn.style.padding = '10px';
testBtn.style.backgroundColor = '#4CAF50';
testBtn.style.color = 'white';
testBtn.style.border = 'none';
testBtn.style.borderRadius = '4px';
testBtn.style.cursor = 'pointer';
testBtn.onclick = async () => {
try {
console.log('Initializing audio...');
const result = await invoke('audio_init');
console.log(result);
console.log('Creating MIDI beep...');
await invoke('audio_test_beep');
console.log('Playing...');
await invoke('audio_play');
setTimeout(async () => {
await invoke('audio_stop');
console.log('Stopped');
}, 3000);
} catch (error) {
console.error('Audio test failed:', error);
alert('Audio test failed: ' + error);
}
};
document.body.appendChild(testBtn);
// Initialize audio system on startup // Initialize audio system on startup
(async () => { (async () => {
try { try {
@ -960,6 +924,11 @@ async function playPause() {
console.error('Failed to start audio playback:', error); console.error('Failed to start audio playback:', error);
} }
// Re-enable auto-scroll when playback starts
if (context.pianoRollEditor) {
context.pianoRollEditor.autoScrollEnabled = true;
}
playbackLoop(); playbackLoop();
} else { } else {
// Stop recording if active // Stop recording if active
@ -1134,16 +1103,11 @@ async function handleAudioEvent(event) {
// Sync frontend time with DAW time // Sync frontend time with DAW time
if (playing) { if (playing) {
// Quantize time to framerate for animation playback // Quantize time to framerate for animation playback
console.log('[PlaybackPosition] context.activeObject:', context.activeObject, 'root:', root, 'same?', context.activeObject === root);
console.log('[PlaybackPosition] root.frameRate:', root.frameRate, 'activeObject.frameRate:', context.activeObject.frameRate);
const framerate = context.activeObject.frameRate; const framerate = context.activeObject.frameRate;
console.log('[PlaybackPosition] framerate:', framerate, 'event.time:', event.time, 'currentTime before:', context.activeObject.currentTime);
const frameDuration = 1 / framerate; const frameDuration = 1 / framerate;
const quantizedTime = Math.floor(event.time / frameDuration) * frameDuration; const quantizedTime = Math.floor(event.time / frameDuration) * frameDuration;
console.log('[PlaybackPosition] frameDuration:', frameDuration, 'quantizedTime:', quantizedTime);
context.activeObject.currentTime = quantizedTime; context.activeObject.currentTime = quantizedTime;
console.log('[PlaybackPosition] currentTime after:', context.activeObject.currentTime);
if (context.timelineWidget?.timelineState) { if (context.timelineWidget?.timelineState) {
context.timelineWidget.timelineState.currentTime = quantizedTime; context.timelineWidget.timelineState.currentTime = quantizedTime;
} }
@ -1158,6 +1122,11 @@ async function handleAudioEvent(event) {
context.pianoWidget.setPlayingNotes(playingNotes); context.pianoWidget.setPlayingNotes(playingNotes);
context.pianoRedraw(); context.pianoRedraw();
} }
// Update piano roll editor to show playhead
if (context.pianoRollRedraw) {
context.pianoRollRedraw();
}
} }
break; break;
@ -4474,10 +4443,34 @@ function createPane(paneType = undefined, div = undefined) {
// Create and append the new menu to the DOM // Create and append the new menu to the DOM
popupMenu = createPaneMenu(div); popupMenu = createPaneMenu(div);
// Position the menu below the button // Position the menu intelligently to stay onscreen
const buttonRect = event.target.getBoundingClientRect(); const buttonRect = event.target.getBoundingClientRect();
popupMenu.style.left = `${buttonRect.left}px`; const menuRect = popupMenu.getBoundingClientRect();
popupMenu.style.top = `${buttonRect.bottom + window.scrollY}px`;
// Default: position below and to the right of the button
let left = buttonRect.left;
let top = buttonRect.bottom + window.scrollY;
// Check if menu goes off the right edge
if (left + menuRect.width > window.innerWidth) {
// Align right edge of menu with right edge of button
left = buttonRect.right - menuRect.width;
}
// Check if menu goes off the bottom edge
if (buttonRect.bottom + menuRect.height > window.innerHeight) {
// Position above the button instead
top = buttonRect.top + window.scrollY - menuRect.height;
}
// Ensure menu doesn't go off the left edge
left = Math.max(0, left);
// Ensure menu doesn't go off the top edge
top = Math.max(window.scrollY, top);
popupMenu.style.left = `${left}px`;
popupMenu.style.top = `${top}px`;
} }
// Prevent the click event from propagating to the window click listener // Prevent the click event from propagating to the window click listener
@ -6024,6 +6017,87 @@ function piano() {
return piano_cvs; return piano_cvs;
} }
function pianoRoll() {
let canvas = document.createElement("canvas");
canvas.className = "piano-roll";
// Create the piano roll editor widget
canvas.pianoRollEditor = new PianoRollEditor(0, 0, 0, 0);
function updateCanvasSize() {
const canvasStyles = window.getComputedStyle(canvas);
const width = parseInt(canvasStyles.width);
const height = parseInt(canvasStyles.height);
// Update widget dimensions
canvas.pianoRollEditor.width = width;
canvas.pianoRollEditor.height = height;
// Set actual size in memory (scaled for retina displays)
canvas.width = width * window.devicePixelRatio;
canvas.height = height * window.devicePixelRatio;
// Normalize coordinate system to use CSS pixels
const ctx = canvas.getContext("2d");
ctx.scale(window.devicePixelRatio, window.devicePixelRatio);
// Render the piano roll
canvas.pianoRollEditor.draw(ctx);
}
// Store references in context for global access and playback updates
context.pianoRollEditor = canvas.pianoRollEditor;
context.pianoRollCanvas = canvas;
context.pianoRollRedraw = updateCanvasSize;
const resizeObserver = new ResizeObserver(() => {
updateCanvasSize();
});
resizeObserver.observe(canvas);
// Pointer event handlers (works with mouse and touch)
canvas.addEventListener("pointerdown", (e) => {
const rect = canvas.getBoundingClientRect();
const x = e.clientX - rect.left;
const y = e.clientY - rect.top;
canvas.pianoRollEditor.handleMouseEvent("mousedown", x, y);
updateCanvasSize();
});
canvas.addEventListener("pointermove", (e) => {
const rect = canvas.getBoundingClientRect();
const x = e.clientX - rect.left;
const y = e.clientY - rect.top;
canvas.pianoRollEditor.handleMouseEvent("mousemove", x, y);
// Update cursor based on widget state
if (canvas.pianoRollEditor.cursor) {
canvas.style.cursor = canvas.pianoRollEditor.cursor;
}
updateCanvasSize();
});
canvas.addEventListener("pointerup", (e) => {
const rect = canvas.getBoundingClientRect();
const x = e.clientX - rect.left;
const y = e.clientY - rect.top;
canvas.pianoRollEditor.handleMouseEvent("mouseup", x, y);
updateCanvasSize();
});
canvas.addEventListener("wheel", (e) => {
e.preventDefault();
canvas.pianoRollEditor.wheel(e);
updateCanvasSize();
});
// Prevent text selection
canvas.addEventListener("selectstart", (e) => e.preventDefault());
return canvas;
}
const panes = { const panes = {
stage: { stage: {
name: "stage", name: "stage",
@ -6053,6 +6127,10 @@ const panes = {
name: "piano", name: "piano",
func: piano, func: piano,
}, },
pianoRoll: {
name: "piano-roll",
func: pianoRoll,
},
}; };
/** /**

View File

@ -9,6 +9,12 @@ body {
user-select: none; user-select: none;
} }
/* Allow text selection in input fields and textareas */
input,
textarea {
user-select: text;
}
.logo.vanilla:hover { .logo.vanilla:hover {
filter: drop-shadow(0 0 2em #ffe21c); filter: drop-shadow(0 0 2em #ffe21c);
} }

View File

@ -4460,6 +4460,661 @@ class VirtualPiano extends Widget {
} }
} }
/**
* Piano Roll Editor
* MIDI note editor with piano keyboard on left and grid on right
*/
class PianoRollEditor extends Widget {
constructor(width, height, x, y) {
super(x, y)
this.width = width
this.height = height
// Display settings
this.keyboardWidth = 60 // Width of piano keyboard on left
this.noteHeight = 16 // Height of each note row
this.pixelsPerSecond = 100 // Horizontal zoom
this.minNote = 21 // A0
this.maxNote = 108 // C8
this.totalNotes = this.maxNote - this.minNote + 1
// Scroll state
this.scrollX = 0
this.scrollY = 0
this.initialScrollSet = false // Track if we've set initial scroll position
// Interaction state
this.selectedNotes = new Set() // Set of note indices
this.dragMode = null // null, 'move', 'resize-left', 'resize-right', 'create'
this.dragStartX = 0
this.dragStartY = 0
this.creatingNote = null // Temporary note being created
this.isDragging = false
// Note preview playback state
this.playingNote = null // Currently playing note number
this.playingNoteMaxDuration = null // Max duration in seconds
this.playingNoteStartTime = null // Timestamp when note started playing
// Auto-scroll state
this.autoScrollEnabled = true // Auto-scroll to follow playhead during playback
this.lastPlayheadTime = 0 // Track last playhead position
// Start timer to check for note duration expiry
this.checkNoteDurationTimer = setInterval(() => this.checkNoteDuration(), 50)
}
checkNoteDuration() {
if (this.playingNote !== null && this.playingNoteMaxDuration !== null && this.playingNoteStartTime !== null) {
const elapsed = (Date.now() - this.playingNoteStartTime) / 1000
if (elapsed >= this.playingNoteMaxDuration) {
// Stop the note
const clipData = this.getSelectedClip()
if (clipData) {
invoke('audio_send_midi_note_off', {
trackId: clipData.trackId,
note: this.playingNote
})
this.playingNote = null
this.playingNoteMaxDuration = null
this.playingNoteStartTime = null
}
}
}
}
// Get the currently selected MIDI clip from context
getSelectedClip() {
if (typeof context === 'undefined' || !context.activeObject || !context.activeObject.audioTracks) {
return null
}
// Find the first MIDI track with a selected clip
for (const track of context.activeObject.audioTracks) {
if (track.type === 'midi' && track.clips && track.clips.length > 0) {
// For now, just return the first clip on the first MIDI track
// TODO: Add proper clip selection mechanism
return { clip: track.clips[0], trackId: track.audioTrackId }
}
}
return null
}
hitTest(x, y) {
return x >= 0 && x <= this.width && y >= 0 && y <= this.height
}
// Convert screen coordinates to note/time
screenToNote(y) {
const gridY = y + this.scrollY
const noteIndex = Math.floor(gridY / this.noteHeight)
return this.maxNote - noteIndex // Invert (higher notes at top)
}
screenToTime(x) {
const gridX = x - this.keyboardWidth + this.scrollX
return gridX / this.pixelsPerSecond
}
// Convert note/time to screen coordinates
noteToScreenY(note) {
const noteIndex = this.maxNote - note
return noteIndex * this.noteHeight - this.scrollY
}
timeToScreenX(time) {
return time * this.pixelsPerSecond - this.scrollX + this.keyboardWidth
}
// Find note at screen position
findNoteAtPosition(x, y) {
const clipData = this.getSelectedClip()
if (!clipData || !clipData.clip.notes) {
return -1
}
const note = this.screenToNote(y)
const time = this.screenToTime(x)
// Search in reverse order so we find top-most notes first
for (let i = clipData.clip.notes.length - 1; i >= 0; i--) {
const n = clipData.clip.notes[i]
const noteMatches = Math.round(n.note) === Math.round(note)
const timeInRange = time >= n.start_time && time <= (n.start_time + n.duration)
if (noteMatches && timeInRange) {
return i
}
}
return -1
}
// Check if clicking on the right edge resize handle
isOnResizeHandle(x, noteIndex) {
const clipData = this.getSelectedClip()
if (!clipData || noteIndex < 0 || noteIndex >= clipData.clip.notes.length) {
return false
}
const note = clipData.clip.notes[noteIndex]
const noteEndX = this.timeToScreenX(note.start_time + note.duration)
// Consider clicking within 8 pixels of the right edge as resize
return Math.abs(x - noteEndX) < 8
}
mousedown(x, y) {
this._globalEvents.add("mousemove")
this._globalEvents.add("mouseup")
this.isDragging = true
this.dragStartX = x
this.dragStartY = y
// Check if clicking on keyboard or grid
if (x < this.keyboardWidth) {
// Clicking on keyboard - could preview note
return
}
const note = this.screenToNote(y)
const time = this.screenToTime(x)
// Check if clicking on an existing note
const noteIndex = this.findNoteAtPosition(x, y)
if (noteIndex >= 0) {
// Clicking on an existing note
const clipData = this.getSelectedClip()
if (this.isOnResizeHandle(x, noteIndex)) {
// Start resizing
this.dragMode = 'resize'
this.resizingNoteIndex = noteIndex
this.selectedNotes.clear()
this.selectedNotes.add(noteIndex)
} else {
// Start moving
this.dragMode = 'move'
this.movingStartTime = time
this.movingStartNote = note
// Select this note (or add to selection with Ctrl/Cmd)
if (!this.selectedNotes.has(noteIndex)) {
this.selectedNotes.clear()
this.selectedNotes.add(noteIndex)
}
// Play preview of the note
if (clipData && clipData.clip.notes[noteIndex]) {
const clickedNote = clipData.clip.notes[noteIndex]
this.playingNote = clickedNote.note
this.playingNoteMaxDuration = clickedNote.duration
this.playingNoteStartTime = Date.now()
invoke('audio_send_midi_note_on', {
trackId: clipData.trackId,
note: clickedNote.note,
velocity: clickedNote.velocity
})
}
}
} else {
// Clicking on empty space - start creating a new note
this.dragMode = 'create'
this.selectedNotes.clear()
// Create a temporary note for preview
const newNoteValue = Math.round(note)
this.creatingNote = {
note: newNoteValue,
start_time: time,
duration: 0.1, // Minimum duration
velocity: 100
}
// Play preview of the new note
const clipData = this.getSelectedClip()
if (clipData) {
this.playingNote = newNoteValue
this.playingNoteMaxDuration = null // No max duration for creating notes
this.playingNoteStartTime = Date.now()
invoke('audio_send_midi_note_on', {
trackId: clipData.trackId,
note: newNoteValue,
velocity: 100
})
}
}
}
mousemove(x, y) {
// Update cursor based on hover position even when not dragging
if (!this.isDragging && x >= this.keyboardWidth) {
const noteIndex = this.findNoteAtPosition(x, y)
if (noteIndex >= 0 && this.isOnResizeHandle(x, noteIndex)) {
this.cursor = 'ew-resize'
} else {
this.cursor = 'default'
}
}
if (!this.isDragging) return
const clipData = this.getSelectedClip()
if (!clipData) return
if (this.dragMode === 'create') {
// Extend the note being created
if (this.creatingNote) {
const currentTime = this.screenToTime(x)
const duration = Math.max(0.1, currentTime - this.creatingNote.start_time)
this.creatingNote.duration = duration
}
} else if (this.dragMode === 'move') {
// Move selected notes
const currentTime = this.screenToTime(x)
const currentNote = this.screenToNote(y)
const deltaTime = currentTime - this.movingStartTime
const deltaNote = Math.round(currentNote - this.movingStartNote)
// Check if pitch changed
if (deltaNote !== 0) {
const firstSelectedIndex = Array.from(this.selectedNotes)[0]
if (firstSelectedIndex >= 0 && firstSelectedIndex < clipData.clip.notes.length) {
const movedNote = clipData.clip.notes[firstSelectedIndex]
const newPitch = Math.max(0, Math.min(127, movedNote.note + deltaNote))
// Stop old note if one is playing
if (this.playingNote !== null) {
invoke('audio_send_midi_note_off', {
trackId: clipData.trackId,
note: this.playingNote
})
}
// Update playing note to new pitch
this.playingNote = newPitch
this.playingNoteMaxDuration = movedNote.duration
this.playingNoteStartTime = Date.now()
// Play new note at new pitch
invoke('audio_send_midi_note_on', {
trackId: clipData.trackId,
note: newPitch,
velocity: movedNote.velocity
})
}
}
// Update positions of all selected notes
for (const noteIndex of this.selectedNotes) {
if (noteIndex >= 0 && noteIndex < clipData.clip.notes.length) {
const note = clipData.clip.notes[noteIndex]
note.start_time = Math.max(0, note.start_time + deltaTime)
note.note = Math.max(0, Math.min(127, note.note + deltaNote))
}
}
// Update drag start positions for next move
this.movingStartTime = currentTime
this.movingStartNote = currentNote
// Trigger timeline redraw to show updated notes
if (context.timelineWidget) {
context.timelineWidget.requestRedraw()
}
} else if (this.dragMode === 'resize') {
// Resize the selected note
if (this.resizingNoteIndex >= 0 && this.resizingNoteIndex < clipData.clip.notes.length) {
const note = clipData.clip.notes[this.resizingNoteIndex]
const currentTime = this.screenToTime(x)
const newDuration = Math.max(0.1, currentTime - note.start_time)
note.duration = newDuration
// Trigger timeline redraw to show updated notes
if (context.timelineWidget) {
context.timelineWidget.requestRedraw()
}
}
}
}
mouseup(x, y) {
this._globalEvents.delete("mousemove")
this._globalEvents.delete("mouseup")
const clipData = this.getSelectedClip()
// Stop playing note
if (this.playingNote !== null && clipData) {
invoke('audio_send_midi_note_off', {
trackId: clipData.trackId,
note: this.playingNote
})
this.playingNote = null
this.playingNoteMaxDuration = null
this.playingNoteStartTime = null
}
// If we were creating a note, add it to the clip
if (this.dragMode === 'create' && this.creatingNote && clipData) {
if (!clipData.clip.notes) {
clipData.clip.notes = []
}
// Binary search to find insertion position to maintain sorted order
const newNote = { ...this.creatingNote }
let left = 0
let right = clipData.clip.notes.length
while (left < right) {
const mid = Math.floor((left + right) / 2)
if (clipData.clip.notes[mid].start_time < newNote.start_time) {
left = mid + 1
} else {
right = mid
}
}
clipData.clip.notes.splice(left, 0, newNote)
// Trigger timeline redraw to show new note
if (context.timelineWidget) {
context.timelineWidget.requestRedraw()
}
// Sync to backend
this.syncNotesToBackend(clipData)
}
// If we moved or resized notes, sync to backend
if ((this.dragMode === 'move' || this.dragMode === 'resize') && clipData) {
if (context.timelineWidget) {
context.timelineWidget.requestRedraw()
}
// Sync to backend
this.syncNotesToBackend(clipData)
}
this.isDragging = false
this.dragMode = null
this.creatingNote = null
this.resizingNoteIndex = -1
}
wheel(e) {
// Support horizontal scrolling from trackpad (deltaX) or Shift+scroll (deltaY)
if (e.deltaX !== 0) {
// Trackpad horizontal scroll
this.scrollX += e.deltaX
} else if (e.shiftKey) {
// Shift+wheel for horizontal scroll
this.scrollX += e.deltaY
} else {
// Normal vertical scroll
this.scrollY += e.deltaY
}
this.scrollX = Math.max(0, this.scrollX)
this.scrollY = Math.max(0, this.scrollY)
// Disable auto-scroll when user manually scrolls
this.autoScrollEnabled = false
}
syncNotesToBackend(clipData) {
// Convert notes to backend format: (start_time, note, velocity, duration)
const notes = clipData.clip.notes.map(n => [
n.start_time,
n.note,
n.velocity,
n.duration
])
// Send to backend
invoke('audio_update_midi_clip_notes', {
trackId: clipData.trackId,
clipId: clipData.clip.clipId,
notes: notes
}).catch(err => {
console.error('Failed to update MIDI notes:', err)
})
}
draw(ctx) {
// Update dimensions
// (width/height will be set by parent container)
// Set initial scroll position to center on G4 (MIDI note 67) on first draw
if (!this.initialScrollSet && this.height > 0) {
const g4Index = this.maxNote - 67 // G4 is MIDI note 67
const g4Y = g4Index * this.noteHeight
// Center G4 in the viewport
this.scrollY = g4Y - (this.height / 2)
this.initialScrollSet = true
}
// Auto-scroll to follow playhead during playback
if (this.autoScrollEnabled && context.activeObject && this.width > 0) {
const playheadTime = context.activeObject.currentTime || 0
// Check if playhead is moving forward (playing)
if (playheadTime > this.lastPlayheadTime) {
// Center playhead in viewport
const gridWidth = this.width - this.keyboardWidth
const playheadScreenX = playheadTime * this.pixelsPerSecond
const targetScrollX = playheadScreenX - (gridWidth / 2)
this.scrollX = Math.max(0, targetScrollX)
}
this.lastPlayheadTime = playheadTime
}
// Clear
ctx.fillStyle = backgroundColor
ctx.fillRect(0, 0, this.width, this.height)
// Draw piano keyboard
this.drawKeyboard(ctx, this.width, this.height)
// Draw grid
this.drawGrid(ctx, this.width, this.height)
// Draw notes if we have a selected clip
const selected = this.getSelectedClip()
if (selected && selected.clip && selected.clip.notes) {
this.drawNotes(ctx, this.width, this.height, selected.clip)
}
// Draw playhead
this.drawPlayhead(ctx, this.width, this.height)
}
drawKeyboard(ctx, width, height) {
const keyboardWidth = this.keyboardWidth
// Draw keyboard background
ctx.fillStyle = shade
ctx.fillRect(0, 0, keyboardWidth, height)
// Draw keys
for (let note = this.minNote; note <= this.maxNote; note++) {
const y = this.noteToScreenY(note)
if (y < 0 || y > height) continue
const isBlackKey = [1, 3, 6, 8, 10].includes(note % 12)
ctx.fillStyle = isBlackKey ? '#333' : '#fff'
ctx.fillRect(5, y, keyboardWidth - 10, this.noteHeight - 1)
// Draw note label for C notes
if (note % 12 === 0) {
ctx.fillStyle = '#999'
ctx.font = '10px sans-serif'
ctx.textAlign = 'right'
ctx.textBaseline = 'middle'
ctx.fillText(`C${Math.floor(note / 12) - 1}`, keyboardWidth - 15, y + this.noteHeight / 2)
}
}
}
drawGrid(ctx, width, height) {
const gridLeft = this.keyboardWidth
const gridWidth = width - gridLeft
ctx.save()
ctx.beginPath()
ctx.rect(gridLeft, 0, gridWidth, height)
ctx.clip()
// Draw background
ctx.fillStyle = backgroundColor
ctx.fillRect(gridLeft, 0, gridWidth, height)
// Draw horizontal lines (note separators)
ctx.strokeStyle = 'rgba(255, 255, 255, 0.1)'
ctx.lineWidth = 1
for (let note = this.minNote; note <= this.maxNote; note++) {
const y = this.noteToScreenY(note)
if (y < 0 || y > height) continue
// Highlight C notes
if (note % 12 === 0) {
ctx.strokeStyle = 'rgba(255, 255, 255, 0.3)'
} else {
ctx.strokeStyle = 'rgba(255, 255, 255, 0.1)'
}
ctx.beginPath()
ctx.moveTo(gridLeft, y)
ctx.lineTo(width, y)
ctx.stroke()
}
// Draw vertical lines (time grid)
const beatInterval = 0.5 // Half second intervals
const startTime = Math.floor(this.scrollX / this.pixelsPerSecond / beatInterval) * beatInterval
const endTime = (this.scrollX + gridWidth) / this.pixelsPerSecond
for (let time = startTime; time <= endTime; time += beatInterval) {
const x = this.timeToScreenX(time)
if (x < gridLeft || x > width) continue
// Every second is brighter
if (Math.abs(time % 1.0) < 0.01) {
ctx.strokeStyle = 'rgba(255, 255, 255, 0.3)'
} else {
ctx.strokeStyle = 'rgba(255, 255, 255, 0.1)'
}
ctx.beginPath()
ctx.moveTo(x, 0)
ctx.lineTo(x, height)
ctx.stroke()
}
ctx.restore()
}
drawNotes(ctx, width, height, clip) {
const gridLeft = this.keyboardWidth
ctx.save()
ctx.beginPath()
ctx.rect(gridLeft, 0, width - gridLeft, height)
ctx.clip()
// Draw existing notes
ctx.fillStyle = '#6fdc6f'
for (let i = 0; i < clip.notes.length; i++) {
const note = clip.notes[i]
const x = this.timeToScreenX(note.start_time)
const y = this.noteToScreenY(note.note)
const noteWidth = note.duration * this.pixelsPerSecond
const noteHeight = this.noteHeight - 2
// Skip if off-screen
if (x + noteWidth < gridLeft || x > width || y + noteHeight < 0 || y > height) {
continue
}
// Highlight selected notes
if (this.selectedNotes.has(i)) {
ctx.fillStyle = '#8ffc8f'
} else {
ctx.fillStyle = '#6fdc6f'
}
ctx.fillRect(x, y, noteWidth, noteHeight)
// Draw border
ctx.strokeStyle = 'rgba(0, 0, 0, 0.3)'
ctx.strokeRect(x, y, noteWidth, noteHeight)
}
// Draw note being created
if (this.creatingNote) {
const x = this.timeToScreenX(this.creatingNote.start_time)
const y = this.noteToScreenY(this.creatingNote.note)
const noteWidth = this.creatingNote.duration * this.pixelsPerSecond
const noteHeight = this.noteHeight - 2
// Draw with a slightly transparent color to indicate it's being created
ctx.fillStyle = 'rgba(111, 220, 111, 0.7)'
ctx.fillRect(x, y, noteWidth, noteHeight)
ctx.strokeStyle = 'rgba(0, 0, 0, 0.5)'
ctx.setLineDash([4, 4])
ctx.strokeRect(x, y, noteWidth, noteHeight)
ctx.setLineDash([])
}
ctx.restore()
}
drawPlayhead(ctx, width, height) {
// Get current playhead time from context
if (typeof context === 'undefined' || !context.activeObject) {
return
}
const playheadTime = context.activeObject.currentTime || 0
const gridLeft = this.keyboardWidth
// Convert time to screen X position
const playheadX = this.timeToScreenX(playheadTime)
// Only draw if playhead is visible
if (playheadX < gridLeft || playheadX > width) {
return
}
ctx.save()
ctx.beginPath()
ctx.rect(gridLeft, 0, width - gridLeft, height)
ctx.clip()
// Draw playhead line
ctx.strokeStyle = 'rgba(255, 100, 100, 0.8)'
ctx.lineWidth = 2
ctx.beginPath()
ctx.moveTo(playheadX, 0)
ctx.lineTo(playheadX, height)
ctx.stroke()
ctx.restore()
}
}
export { export {
SCROLL, SCROLL,
Widget, Widget,
@ -4473,5 +5128,6 @@ export {
ScrollableWindowHeaders, ScrollableWindowHeaders,
TimelineWindow, TimelineWindow,
TimelineWindowV2, TimelineWindowV2,
VirtualPiano VirtualPiano,
PianoRollEditor
}; };