24 KiB
24 KiB
phase, plan, type, wave, depends_on, files_modified, autonomous, requirements, must_haves
| phase | plan | type | wave | depends_on | files_modified | autonomous | requirements | must_haves | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 06-ai-sync | 02 | execute | 2 |
|
|
true |
|
|
Purpose: Provides the audio analysis foundation (onsets, chroma, RMS, segments) that the AI sync engine (Plan 06-03) consumes, plus the visible segment overlay on the waveform canvas that enables per-section AI fill (D-11).
Output: features.py and segments.py backend modules, /api/audio/segments endpoint, and interactive segment band overlay on the timeline canvas.
<execution_context> @$HOME/.claude/get-shit-done/workflows/execute-plan.md @$HOME/.claude/get-shit-done/templates/summary.md </execution_context>
@.planning/PROJECT.md @.planning/ROADMAP.md @.planning/STATE.md @.planning/phases/06-ai-sync/06-CONTEXT.md @.planning/phases/06-ai-sync/06-RESEARCH.md @.planning/phases/06-ai-sync/06-01-SUMMARY.mdFrom lightsync/audio/beats.py:
def analyze(path: str) -> dict[str, Any]:
"""Returns {"tempo": float, "beats": [float, ...]}"""
import librosa
y, sr = librosa.load(path, sr=None, mono=True)
...
async def analyze_async(path: str) -> dict[str, Any]:
return await asyncio.to_thread(analyze, path)
From lightsync/main.py (lifespan):
app.state.beats = {}
# Add app.state.segments = {} here
From lightsync/api/audio.py (after Plan 06-01):
router = APIRouter()
SHOWS_DIR = Path("/app/shows")
# Existing endpoints: /files, /upload, /beats, /waveform, /fetch
From lightsync/frontend/timeline/timeline.js:
class TimelineCanvas {
constructor(canvasEl, audioEl) { ... }
render() {
// 1. Background
// 2. Track row banding (lines 117-121)
// >>> INSERT segment bands HERE (between step 2 and step 3) <<<
// 3. Waveform background (lines 124-136)
// 4. Beat marks (lines 139-177)
// 5. Block rendering (lines 180+)
}
timeToX(t) { ... }
xToTime(x) { ... }
async loadWaveform(audioPath) { ... }
async loadBeats(audioPath) { ... }
}
export { TimelineCanvas };
Constants from timeline.js:
const TRACK_HEIGHT = 48;
const HEADER_WIDTH = 140;
const TIME_AXIS_HEIGHT = 24;
From lightsync/frontend/style.css (CSS variables):
--accent: #00ffff;
--border-dim: #1a4a4a;
--text-dim: #555555;
"""Audio feature extraction: onsets, chroma, RMS energy (Phase 6).
extract_features(path) -> {
"onset_times": [float, ...],
"rms_1s": [float, ...],
"chroma_1s": [int, ...],
"duration": float,
"sr": int,
}
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
def extract_features(path: str) -> dict[str, Any]:
"""Extract onsets, chroma, RMS from audio file.
Args:
path: Absolute path to audio file.
Returns:
Dictionary with onset_times (seconds), rms_1s (per-second mean RMS),
chroma_1s (per-second dominant pitch class 0-11), duration, sr.
"""
import librosa
import numpy as np
p = Path(path)
if not p.exists():
raise FileNotFoundError(f"Audio file not found: {path}")
y, sr = librosa.load(path, sr=None, mono=True)
# Onsets — note attacks / transients
onset_frames = librosa.onset.onset_detect(y=y, sr=sr, units='frames')
onset_times = librosa.frames_to_time(onset_frames, sr=sr).tolist()
# Chroma — tonal/harmonic content (CQT-based)
chroma = librosa.feature.chroma_cqt(y=y, sr=sr) # shape (12, T)
# RMS energy — loudness envelope
rms = librosa.feature.rms(y=y)[0] # shape (T,)
# Reduce to 1s resolution for downstream consumers (heuristic + LLM)
hop_length = 512
frame_rate = sr / hop_length
duration = len(y) / sr
n_seconds = int(duration)
rms_1s = []
chroma_1s = []
for i in range(n_seconds):
start_frame = int(i * frame_rate)
end_frame = int((i + 1) * frame_rate)
# RMS mean for this second
rms_slice = rms[start_frame:min(end_frame, len(rms))]
rms_1s.append(float(np.mean(rms_slice)) if len(rms_slice) > 0 else 0.0)
# Dominant pitch class for this second
chroma_slice = chroma[:, start_frame:min(end_frame, chroma.shape[1])]
if chroma_slice.shape[1] > 0:
chroma_1s.append(int(np.argmax(np.mean(chroma_slice, axis=1))))
else:
chroma_1s.append(0)
return {
"onset_times": onset_times,
"rms_1s": rms_1s,
"chroma_1s": chroma_1s,
"duration": duration,
"sr": sr,
}
async def extract_features_async(path: str) -> dict[str, Any]:
"""Async wrapper — runs in thread pool."""
return await asyncio.to_thread(extract_features, path)
Create lightsync/audio/segments.py — Structural segmentation module.
"""Structural segmentation via librosa agglomerative clustering (Phase 6).
detect_segments(path, n_segments=None) -> [{"start": float, "end": float, "label": str}, ...]
Uses chroma features + recurrence matrix + agglomerative clustering.
Target 4-12 sections for a typical 3-5 minute song.
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
def detect_segments(path: str, n_segments: int | None = None) -> list[dict[str, Any]]:
"""Detect structural sections using agglomerative clustering on chroma features.
Args:
path: Absolute path to audio file.
n_segments: Number of segments (default: auto based on duration, ~1 per 30s).
Returns:
List of dicts: [{"start": float, "end": float, "label": "SEC 1"}, ...]
"""
import librosa
import numpy as np
p = Path(path)
if not p.exists():
raise FileNotFoundError(f"Audio file not found: {path}")
y, sr = librosa.load(path, sr=None, mono=True)
duration = len(y) / sr
# Auto-select segment count if not specified: ~1 per 30s, clamped to 4-12
if n_segments is None:
n_segments = max(4, min(12, int(duration / 30)))
# Build chroma feature matrix
chroma = librosa.feature.chroma_cqt(y=y, sr=sr, bins_per_octave=36)
# Recurrence matrix — similarity structure
R = librosa.segment.recurrence_matrix(chroma, width=3, mode='affinity', sym=True)
# Filter recurrence matrix to enhance diagonal consistency
R_filtered = librosa.segment.path_enhance(R, 15)
# Agglomerative clustering — returns boundary frame indices
bounds = librosa.segment.agglomerative(R_filtered, k=n_segments)
bound_times = librosa.frames_to_time(bounds, sr=sr).tolist()
# Build section list: [{start, end, label}, ...]
# Pitfall 2 from RESEARCH: agglomerative returns k boundaries, yielding k+1 sections
all_times = [0.0] + bound_times + [duration]
sections = []
for i in range(len(all_times) - 1):
sections.append({
"start": all_times[i],
"end": all_times[i + 1],
"label": f"SEC {i + 1}",
})
return sections
async def detect_segments_async(path: str, n_segments: int | None = None) -> list[dict[str, Any]]:
"""Async wrapper — runs in thread pool."""
return await asyncio.to_thread(detect_segments, path, n_segments)
Edit lightsync/main.py — Add app.state.segments = {} in the lifespan function, right after app.state.beats = {} (line 27):
app.state.segments = {}
Edit lightsync/api/audio.py — Add segments endpoint. Add import at top:
from lightsync.audio.segments import detect_segments_async
from lightsync.audio.features import extract_features_async
Add endpoints after the existing get_waveform endpoint:
@router.get("/segments")
async def get_segments(request: Request, path: str, n_segments: int | None = None):
"""Return structural segments for an audio file. Cached in app.state.segments."""
segments_cache: dict = getattr(request.app.state, "segments", None) or {}
cache_key = f"{path}:{n_segments or 'auto'}"
if cache_key not in segments_cache:
p = Path(path)
if not p.exists():
raise HTTPException(status_code=404, detail=f"File not found: {path}")
try:
segments = await detect_segments_async(path, n_segments)
if not hasattr(request.app.state, "segments") or request.app.state.segments is None:
request.app.state.segments = {}
request.app.state.segments[cache_key] = segments
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Segmentation failed: {exc}") from exc
else:
segments = segments_cache[cache_key]
return {"segments": segments, "count": len(segments), "file": path}
@router.get("/features")
async def get_features(request: Request, path: str):
"""Return extracted audio features (onsets, chroma, RMS) for a file."""
p = Path(path)
if not p.exists():
raise HTTPException(status_code=404, detail=f"File not found: {path}")
try:
features = await extract_features_async(path)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Feature extraction failed: {exc}") from exc
return {**features, "file": path}
// SegmentOverlay — draws structural section bands on the timeline canvas (Phase 6)
// Bands are translucent, drawn below waveform. Clicking selects a section.
const BAND_COLORS = [
'rgba(0,255,255,0.15)', // --accent cyan variant
'rgba(180,0,255,0.15)', // purple complement
'rgba(0,200,200,0.12)', // darker cyan variant
];
class SegmentOverlay {
constructor() {
this.segments = []; // [{start, end, label}, ...]
this.selectedIndex = -1; // -1 = no selection
this.loading = false;
this.error = null;
}
/**
* Load segments from the API.
* @param {string} audioPath — absolute path to audio file (e.g. /app/shows/file.mp3)
*/
async load(audioPath) {
this.loading = true;
this.error = null;
this.segments = [];
this.selectedIndex = -1;
try {
const res = await fetch(`/api/audio/segments?path=${encodeURIComponent(audioPath)}`);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const data = await res.json();
this.segments = data.segments || [];
} catch (e) {
this.error = `SEGMENTATION FAILED: ${e.message}`;
console.warn('[segments]', e);
} finally {
this.loading = false;
}
}
/**
* Get the currently selected segment, or null.
* @returns {{start: number, end: number, label: string} | null}
*/
getSelected() {
if (this.selectedIndex >= 0 && this.selectedIndex < this.segments.length) {
return this.segments[this.selectedIndex];
}
return null;
}
/**
* Handle a click at a given time position. Selects the segment containing that time,
* or deselects if clicking outside any segment.
* @param {number} time — time in seconds
* @returns {boolean} — true if selection changed
*/
handleClick(time) {
const prev = this.selectedIndex;
this.selectedIndex = -1;
for (let i = 0; i < this.segments.length; i++) {
if (time >= this.segments[i].start && time < this.segments[i].end) {
this.selectedIndex = i;
break;
}
}
return this.selectedIndex !== prev;
}
/**
* Draw segment bands on the canvas context.
* Must be called BETWEEN track row banding and waveform in the render pipeline.
*
* @param {CanvasRenderingContext2D} ctx
* @param {number} W — canvas logical width
* @param {number} H — canvas logical height
* @param {number} trackCount — number of tracks
* @param {Function} timeToX — converts time (seconds) to canvas X coordinate
*/
draw(ctx, W, H, trackCount, timeToX) {
if (this.segments.length === 0) return;
const totalH = trackCount * 48; // TRACK_HEIGHT
const topY = 24; // TIME_AXIS_HEIGHT
const headerW = 140; // HEADER_WIDTH
for (let i = 0; i < this.segments.length; i++) {
const seg = this.segments[i];
const x1 = timeToX(seg.start);
const x2 = timeToX(seg.end);
// Skip segments entirely off-screen
if (x2 < headerW || x1 > W) continue;
// Clamp to visible area
const drawX1 = Math.max(x1, headerW);
const drawX2 = Math.min(x2, W);
const drawW = drawX2 - drawX1;
const isSelected = this.selectedIndex === i;
// D-09: Translucent colored band, full waveform height, alternating colors
ctx.fillStyle = BAND_COLORS[i % BAND_COLORS.length];
ctx.fillRect(drawX1, topY, drawW, totalH);
// D-11: Selected section gets accent-color border highlight
if (isSelected) {
ctx.strokeStyle = 'rgba(0,255,255,1.0)';
ctx.lineWidth = 1;
ctx.strokeRect(drawX1, topY, drawW, totalH);
}
// D-10: Label — small monospace uppercase at top-left of band
if (drawW > 30) {
ctx.fillStyle = 'rgba(0,255,255,0.6)';
ctx.font = '9px monospace';
ctx.fillText(seg.label, Math.max(x1 + 3, headerW + 2), topY + 10);
}
}
}
}
export { SegmentOverlay };
Edit lightsync/frontend/timeline/timeline.js — Integrate SegmentOverlay into the timeline.
- Add import at the top (after existing imports, around line 7):
import { SegmentOverlay } from './segments.js';
- In the
constructor()method, add afterthis.beatGrid = new BeatGrid();(around line 30):
this.segmentOverlay = new SegmentOverlay();
- In the
render()method, insert segment band drawing BETWEEN step 2 (track row banding, after theforEachat line ~121) and step 3 (waveform background, starting at line ~124). Add this code:
// 2.5 Segment overlay bands (drawn below waveform)
if (this.segmentOverlay.segments.length > 0) {
this.segmentOverlay.draw(ctx, W, H, this.tracks.length, (t) => this.timeToX(t));
}
- Add a
loadSegmentsmethod afterloadBeats(around line 624):
async loadSegments(audioPath) {
await this.segmentOverlay.load(audioPath);
}
- In the existing
_setupInteractions()method, add segment click handling in the mousedown handler. In the existing mousedown logic, BEFORE the block selection/move/resize logic, add:
// Check for segment click (only if click is in track area, not on a block)
// This should be checked and the overlay updated; block interactions take priority
Actually, segment selection should happen on clicks that do NOT land on a block. The existing mousedown handler already checks for block hits. Add segment overlay click handling in the existing mousedown flow — after the block hit detection fails (no block found), call:
const clickTime = this.xToTime(mx);
this.segmentOverlay.handleClick(clickTime);
This ensures: click on block = select block (existing behavior), click on empty track area = select segment.
Edit lightsync/frontend/app.js — Load segments when audio loads.
In the initTimeline() function, inside the audio.addEventListener('loadedmetadata', ...) callback (around line 404-417), add after timeline.loadBeats(path).then(...):
// Load structural segments for overlay
timeline.loadSegments(path);
This triggers segment detection (or cache hit) when audio loads, displaying the overlay as soon as analysis completes.
docker exec lightsync python -c "
import pathlib
seg_js = pathlib.Path('/app/lightsync/frontend/timeline/segments.js').read_text()
assert 'class SegmentOverlay' in seg_js, 'missing SegmentOverlay class'
assert 'export { SegmentOverlay }' in seg_js, 'missing export'
assert 'BAND_COLORS' in seg_js, 'missing BAND_COLORS'
assert 'handleClick' in seg_js, 'missing handleClick'
tl_js = pathlib.Path('/app/lightsync/frontend/timeline/timeline.js').read_text()
assert 'SegmentOverlay' in tl_js, 'missing SegmentOverlay import'
assert 'segmentOverlay' in tl_js, 'missing segmentOverlay instance'
assert 'loadSegments' in tl_js, 'missing loadSegments method'
app_js = pathlib.Path('/app/lightsync/frontend/app.js').read_text()
assert 'loadSegments' in app_js, 'missing loadSegments call'
print('All segment overlay checks passed')
"
<acceptance_criteria>
- lightsync/frontend/timeline/segments.js exists and contains class SegmentOverlay
- lightsync/frontend/timeline/segments.js contains export { SegmentOverlay }
- lightsync/frontend/timeline/segments.js contains BAND_COLORS with at least 2 colors
- lightsync/frontend/timeline/segments.js contains rgba(0,255,255,0.15) (cyan band)
- lightsync/frontend/timeline/segments.js contains rgba(180,0,255,0.15) (purple band)
- lightsync/frontend/timeline/segments.js contains handleClick(time) method
- lightsync/frontend/timeline/segments.js contains getSelected() method
- lightsync/frontend/timeline/segments.js contains seg.label (renders D-10 labels)
- lightsync/frontend/timeline/segments.js contains 9px monospace (small monospace font for labels)
- lightsync/frontend/timeline/timeline.js contains import { SegmentOverlay } from ./segments.js
- lightsync/frontend/timeline/timeline.js contains this.segmentOverlay = new SegmentOverlay()
- lightsync/frontend/timeline/timeline.js contains segmentOverlay.draw( call in render()
- lightsync/frontend/timeline/timeline.js contains async loadSegments(audioPath)
- lightsync/frontend/timeline/timeline.js render() calls segmentOverlay.draw BEFORE waveform drawing
- lightsync/frontend/app.js contains loadSegments(path) in the loadedmetadata handler
</acceptance_criteria>
Segment bands appear as translucent colored overlays on the waveform canvas when audio is loaded. Clicking a band selects it (highlighted border). Clicking outside deselects. Labels show SEC 1, SEC 2, etc.
<success_criteria>
- Structural segmentation detects 4-12 sections from audio
- Segment bands render below waveform in correct Z-order
- Section selection works via click interaction
- Features API returns onsets, chroma, RMS at 1s resolution
- All analysis runs in asyncio.to_thread, never blocks the event loop </success_criteria>