feat(06-02): add feature extraction and segmentation backend modules + API

- Create lightsync/audio/features.py with extract_features and extract_features_async
- Create lightsync/audio/segments.py with detect_segments via agglomerative clustering
- Add /api/audio/segments and /api/audio/features endpoints to audio.py
- Initialize app.state.segments cache in main.py lifespan
This commit is contained in:
Claude
2026-04-07 11:40:29 +00:00
parent 8c1f0a0fbc
commit d339ba588d
4 changed files with 185 additions and 0 deletions

View File

@@ -0,0 +1,78 @@
"""Audio feature extraction: onsets, chroma, RMS energy (Phase 6).
extract_features(path) -> {
"onset_times": [float, ...],
"rms_1s": [float, ...],
"chroma_1s": [int, ...],
"duration": float,
"sr": int,
}
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
def extract_features(path: str) -> dict[str, Any]:
"""Extract onsets, chroma, RMS from audio file.
Args:
path: Absolute path to audio file.
Returns:
Dictionary with onset_times (seconds), rms_1s (per-second mean RMS),
chroma_1s (per-second dominant pitch class 0-11), duration, sr.
"""
import librosa
import numpy as np
p = Path(path)
if not p.exists():
raise FileNotFoundError(f"Audio file not found: {path}")
y, sr = librosa.load(path, sr=None, mono=True)
# Onsets — note attacks / transients
onset_frames = librosa.onset.onset_detect(y=y, sr=sr, units='frames')
onset_times = librosa.frames_to_time(onset_frames, sr=sr).tolist()
# Chroma — tonal/harmonic content (CQT-based)
chroma = librosa.feature.chroma_cqt(y=y, sr=sr) # shape (12, T)
# RMS energy — loudness envelope
rms = librosa.feature.rms(y=y)[0] # shape (T,)
# Reduce to 1s resolution for downstream consumers (heuristic + LLM)
hop_length = 512
frame_rate = sr / hop_length
duration = len(y) / sr
n_seconds = int(duration)
rms_1s = []
chroma_1s = []
for i in range(n_seconds):
start_frame = int(i * frame_rate)
end_frame = int((i + 1) * frame_rate)
# RMS mean for this second
rms_slice = rms[start_frame:min(end_frame, len(rms))]
rms_1s.append(float(np.mean(rms_slice)) if len(rms_slice) > 0 else 0.0)
# Dominant pitch class for this second
chroma_slice = chroma[:, start_frame:min(end_frame, chroma.shape[1])]
if chroma_slice.shape[1] > 0:
chroma_1s.append(int(np.argmax(np.mean(chroma_slice, axis=1))))
else:
chroma_1s.append(0)
return {
"onset_times": onset_times,
"rms_1s": rms_1s,
"chroma_1s": chroma_1s,
"duration": duration,
"sr": sr,
}
async def extract_features_async(path: str) -> dict[str, Any]:
"""Async wrapper — runs in thread pool."""
return await asyncio.to_thread(extract_features, path)