feat(06-02): add feature extraction and segmentation backend modules + API
- Create lightsync/audio/features.py with extract_features and extract_features_async - Create lightsync/audio/segments.py with detect_segments via agglomerative clustering - Add /api/audio/segments and /api/audio/features endpoints to audio.py - Initialize app.state.segments cache in main.py lifespan
This commit is contained in:
78
lightsync/audio/features.py
Normal file
78
lightsync/audio/features.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""Audio feature extraction: onsets, chroma, RMS energy (Phase 6).
|
||||
|
||||
extract_features(path) -> {
|
||||
"onset_times": [float, ...],
|
||||
"rms_1s": [float, ...],
|
||||
"chroma_1s": [int, ...],
|
||||
"duration": float,
|
||||
"sr": int,
|
||||
}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def extract_features(path: str) -> dict[str, Any]:
|
||||
"""Extract onsets, chroma, RMS from audio file.
|
||||
|
||||
Args:
|
||||
path: Absolute path to audio file.
|
||||
|
||||
Returns:
|
||||
Dictionary with onset_times (seconds), rms_1s (per-second mean RMS),
|
||||
chroma_1s (per-second dominant pitch class 0-11), duration, sr.
|
||||
"""
|
||||
import librosa
|
||||
import numpy as np
|
||||
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(f"Audio file not found: {path}")
|
||||
|
||||
y, sr = librosa.load(path, sr=None, mono=True)
|
||||
|
||||
# Onsets — note attacks / transients
|
||||
onset_frames = librosa.onset.onset_detect(y=y, sr=sr, units='frames')
|
||||
onset_times = librosa.frames_to_time(onset_frames, sr=sr).tolist()
|
||||
|
||||
# Chroma — tonal/harmonic content (CQT-based)
|
||||
chroma = librosa.feature.chroma_cqt(y=y, sr=sr) # shape (12, T)
|
||||
|
||||
# RMS energy — loudness envelope
|
||||
rms = librosa.feature.rms(y=y)[0] # shape (T,)
|
||||
|
||||
# Reduce to 1s resolution for downstream consumers (heuristic + LLM)
|
||||
hop_length = 512
|
||||
frame_rate = sr / hop_length
|
||||
duration = len(y) / sr
|
||||
|
||||
n_seconds = int(duration)
|
||||
rms_1s = []
|
||||
chroma_1s = []
|
||||
for i in range(n_seconds):
|
||||
start_frame = int(i * frame_rate)
|
||||
end_frame = int((i + 1) * frame_rate)
|
||||
# RMS mean for this second
|
||||
rms_slice = rms[start_frame:min(end_frame, len(rms))]
|
||||
rms_1s.append(float(np.mean(rms_slice)) if len(rms_slice) > 0 else 0.0)
|
||||
# Dominant pitch class for this second
|
||||
chroma_slice = chroma[:, start_frame:min(end_frame, chroma.shape[1])]
|
||||
if chroma_slice.shape[1] > 0:
|
||||
chroma_1s.append(int(np.argmax(np.mean(chroma_slice, axis=1))))
|
||||
else:
|
||||
chroma_1s.append(0)
|
||||
|
||||
return {
|
||||
"onset_times": onset_times,
|
||||
"rms_1s": rms_1s,
|
||||
"chroma_1s": chroma_1s,
|
||||
"duration": duration,
|
||||
"sr": sr,
|
||||
}
|
||||
|
||||
|
||||
async def extract_features_async(path: str) -> dict[str, Any]:
|
||||
"""Async wrapper — runs in thread pool."""
|
||||
return await asyncio.to_thread(extract_features, path)
|
||||
Reference in New Issue
Block a user