d339ba588d
- Create lightsync/audio/features.py with extract_features and extract_features_async - Create lightsync/audio/segments.py with detect_segments via agglomerative clustering - Add /api/audio/segments and /api/audio/features endpoints to audio.py - Initialize app.state.segments cache in main.py lifespan
79 lines
2.3 KiB
Python
79 lines
2.3 KiB
Python
"""Audio feature extraction: onsets, chroma, RMS energy (Phase 6).
|
|
|
|
extract_features(path) -> {
|
|
"onset_times": [float, ...],
|
|
"rms_1s": [float, ...],
|
|
"chroma_1s": [int, ...],
|
|
"duration": float,
|
|
"sr": int,
|
|
}
|
|
"""
|
|
from __future__ import annotations
|
|
import asyncio
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def extract_features(path: str) -> dict[str, Any]:
|
|
"""Extract onsets, chroma, RMS from audio file.
|
|
|
|
Args:
|
|
path: Absolute path to audio file.
|
|
|
|
Returns:
|
|
Dictionary with onset_times (seconds), rms_1s (per-second mean RMS),
|
|
chroma_1s (per-second dominant pitch class 0-11), duration, sr.
|
|
"""
|
|
import librosa
|
|
import numpy as np
|
|
|
|
p = Path(path)
|
|
if not p.exists():
|
|
raise FileNotFoundError(f"Audio file not found: {path}")
|
|
|
|
y, sr = librosa.load(path, sr=None, mono=True)
|
|
|
|
# Onsets — note attacks / transients
|
|
onset_frames = librosa.onset.onset_detect(y=y, sr=sr, units='frames')
|
|
onset_times = librosa.frames_to_time(onset_frames, sr=sr).tolist()
|
|
|
|
# Chroma — tonal/harmonic content (CQT-based)
|
|
chroma = librosa.feature.chroma_cqt(y=y, sr=sr) # shape (12, T)
|
|
|
|
# RMS energy — loudness envelope
|
|
rms = librosa.feature.rms(y=y)[0] # shape (T,)
|
|
|
|
# Reduce to 1s resolution for downstream consumers (heuristic + LLM)
|
|
hop_length = 512
|
|
frame_rate = sr / hop_length
|
|
duration = len(y) / sr
|
|
|
|
n_seconds = int(duration)
|
|
rms_1s = []
|
|
chroma_1s = []
|
|
for i in range(n_seconds):
|
|
start_frame = int(i * frame_rate)
|
|
end_frame = int((i + 1) * frame_rate)
|
|
# RMS mean for this second
|
|
rms_slice = rms[start_frame:min(end_frame, len(rms))]
|
|
rms_1s.append(float(np.mean(rms_slice)) if len(rms_slice) > 0 else 0.0)
|
|
# Dominant pitch class for this second
|
|
chroma_slice = chroma[:, start_frame:min(end_frame, chroma.shape[1])]
|
|
if chroma_slice.shape[1] > 0:
|
|
chroma_1s.append(int(np.argmax(np.mean(chroma_slice, axis=1))))
|
|
else:
|
|
chroma_1s.append(0)
|
|
|
|
return {
|
|
"onset_times": onset_times,
|
|
"rms_1s": rms_1s,
|
|
"chroma_1s": chroma_1s,
|
|
"duration": duration,
|
|
"sr": sr,
|
|
}
|
|
|
|
|
|
async def extract_features_async(path: str) -> dict[str, Any]:
|
|
"""Async wrapper — runs in thread pool."""
|
|
return await asyncio.to_thread(extract_features, path)
|