Partial tracking with Kalman filters
In [1]:
import random
from IPython.display import display
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import pandas as pd
import librosa as lr
import mir_eval as me
from librosa.display import specshow
# Compressed audio embedding (Ogg Vorbis) keeps the notebook file itself
# small; IPython's own Audio(data, rate=...) embeds uncompressed 16-bit PCM
# WAV, which for anything more than a few seconds bloats the .ipynb a lot.
from ipytoolbox import compressed_audio
%matplotlib inline
In [2]:
sample_rate = 16000
hop_length = 512
top_db = 30
# Seed everything so the Kalman tracker below (which draws process/observation
# noise from np.random) gives a reproducible partial count and plot run to run.
random.seed(0)
np.random.seed(0)
# NOTE: the original local dataset paths
# (/media/carl/Data/solo-instrument-recordings/...) no longer exist, so we
# fall back to librosa's bundled "trumpet" example recording -- a monophonic
# solo instrument clip, close in spirit to the original guitar/violin/piano
# recordings and well suited to partial tracking. There is no ground-truth
# annotation for it, so `reference_annotation` is left empty (columns only)
# -- the "reference" scatter in the note-event plot below will simply show
# nothing.
audio_path = lr.ex('trumpet')
reference_annotation = pd.DataFrame(columns=['Onset', 'Note'])
audio, _ = lr.load(audio_path, sr=sample_rate)
compressed_audio(audio, rate=sample_rate)
Out[2]:
In [3]:
def spectral_transform(audio_samples, sample_rate, hop_length, top_db):
audio_samples = lr.util.normalize(audio_samples)
n_fft = hop_length * 2
z = lr.stft(y=audio_samples, n_fft=n_fft, hop_length=hop_length)
bands = lr.fft_frequencies(sr=sample_rate, n_fft=n_fft)
spectrogram = lr.power_to_db(np.abs(z) ** 2, top_db=top_db)
return spectrogram, bands
spectrogram, bands = spectral_transform(audio, sample_rate, hop_length, top_db)
specshow(spectrogram, sr=sample_rate, hop_length=hop_length);
Peak Detection¶
In [4]:
def partials(spectrogram, top_db):
"""Determine spectral peaks."""
# Go through audio frame-by-frame.
frames = []
for spectrum in spectrogram.T:
# Don't try to pick peaks if overall audio volume is too low.
if np.all(spectrum < top_db * 0.1):
frames.append(([], []))
continue
# Pick spectral peaks.
powers = []
peaks = []
dy = 0.01 * top_db
ys = spectrum
xs = np.arange(len(spectrum))
ymin = np.inf
ymax = -np.inf
uphill = True
for x, y in zip(xs, ys):
if y > ymax:
ymax = y
xmax = x
if y < ymin:
ymin = y
xmin = x
if uphill:
if y < ymax - dy:
powers.append(ymax)
peaks.append(xmax)
ymin = y
xmin = x
uphill = False
else:
if y > ymin + dy:
ymax = y
xmax = x
uphill = True
frames.append((peaks, powers))
peaks = [x[0] for x in frames]
powers = [x[1] for x in frames]
return peaks, powers
peaks, powers = partials(spectrogram, top_db=top_db)
print("Number of peaks: ", int(np.sum([len(x) for x in peaks])))
frequencies = [list(bands[list(x)]) for x in peaks]
# Plot a frame.
for frame in zip(spectrogram.T, peaks, powers):
if len(frame[1]) > 1:
plt.plot(frame[0])
plt.scatter(frame[1], frame[2], color='r')
plt.show()
plt.xlim([0, 100])
break
# Plot all frames over time.
X, Y = [], []
for x, ys in enumerate(peaks):
xs = [x] * len(ys)
X.extend(xs)
Y.extend(ys)
specshow(spectrogram, sr=sample_rate, hop_length=hop_length)
plt.gca().scatter(X, Y, marker='.', color='g', alpha=0.1);
Kalman Tracker¶
In [5]:
frames = list(zip(powers, frequencies))
tracks = []
# Transition matrix
power_variation = 0.99
frequency_variation = 0.9
A = np.array([[1, 1, 0, 0, 0],
[0, 1, 1, 0, 0],
[0, 0, power_variation, 0, 0],
[0, 0, 0, 1, 1],
[0, 0, 0, 0, frequency_variation]])
# State matrix
noise_gain = 1
power_gain = 10
frequency_gain = 0.1
B = np.array([[0, 0, 0],
[0, 0, noise_gain],
[power_gain, 0, 0],
[0, 0, 0],
[0, frequency_gain, 0]])
# Observation matrix
C = np.array([[1, 0, 0, 0, 0],
[0, 0, 0, 1, 0]])
# Process noise covariance
Q = np.eye(3)
# Measurement noise covariance
R = np.eye(2)
# The process noise and observation noise could be thought of as spectral fluctuations caused by inharmonicity, timbral variations, beats, etc.
n_states = A.shape[0]
I_n = np.eye(n_states)
# Initial state-estimate uncertainty. Large relative to Q/R so a brand new
# track's very first correction step trusts the measurement, not the guess.
initial_covariance_scale = 10.0
P_init = np.eye(n_states) * initial_covariance_scale
initial_v = 1.0
initial_w = 1.0
initial_z = 1.0
alive_tracks = [[(0, np.array([p, initial_v, initial_w, f, initial_z]))]
for p, f in list(zip(*frames[0]))]
alive_covariances = [P_init.copy() for _ in alive_tracks]
# Gate on Mahalanobis distance (in units of standard deviations of the
# innovation), not raw Euclidean distance -- power and frequency live on very
# different scales, so a fixed Euclidean threshold either gates on almost
# nothing but frequency or almost nothing but power depending on units.
gate = 6.0
for frame_idx, frame in enumerate(frames[1:], start=1):
# --- Predict step: propagate each track's state estimate and its
# uncertainty (covariance) forward one frame. This is deterministic --
# unlike the earlier version, no random noise sample is injected into the
# predicted state itself. Randomness only ever entered Q/R as their
# *covariance*, describing how uncertain the prediction is, not as an
# actual perturbation of the estimate.
predictions = []
for track, P in zip(alive_tracks, alive_covariances):
x = track[-1][1]
x_pred = A.dot(x)
P_pred = A.dot(P).dot(A.T) + B.dot(Q).dot(B.T)
y_pred = C.dot(x_pred)
S = C.dot(P_pred).dot(C.T) + R # innovation covariance
K = P_pred.dot(C.T).dot(np.linalg.inv(S)) # Kalman gain
predictions.append((x_pred, P_pred, y_pred, S, K))
# Go through all peaks and assign them to ongoing tracks.
new_tracks = []
new_covariances = []
updated = [False] * len(alive_tracks)
pending_updates = {} # track index -> (frame_idx, x_updated, P_updated, distance)
for peak in list(zip(*frame)):
observation = np.array(peak).T
# Go through all ongoing tracks and pick the closest to the new
# observation, using the Mahalanobis distance so power and frequency
# are compared in units of their own predicted uncertainty.
candidate_track_i = None
candidate_distance = np.inf
candidate_update = None
for i, (x_pred, P_pred, y_pred, S, K) in enumerate(predictions):
innovation = observation - y_pred
distance = float(np.sqrt(innovation.dot(np.linalg.inv(S)).dot(innovation)))
if distance < candidate_distance:
candidate_distance = distance
candidate_track_i = i
# --- Correct/update step: fold the real observation back
# into the state estimate via the Kalman gain. This is the
# step that was missing entirely before -- the observation
# was previously used only to pick a track and then thrown
# away, so tracks were really just an unconstrained random
# walk instead of being pulled toward the measured peak.
x_updated = x_pred + K.dot(innovation)
P_updated = (I_n - K.dot(C)).dot(P_pred)
candidate_update = (frame_idx, x_updated, P_updated)
if candidate_track_i is not None and candidate_distance < gate:
if (candidate_track_i not in pending_updates or
candidate_distance < pending_updates[candidate_track_i][3]):
# Pick best fitting candidate peak for this track this frame.
pending_updates[candidate_track_i] = (
candidate_update[0], candidate_update[1], candidate_update[2], candidate_distance)
updated[candidate_track_i] = True
else:
# No candidate track was found: add current observation to a new track.
p, f = observation
new_track = [(frame_idx, np.array([p, initial_v, initial_w, f, initial_z]))]
new_tracks.append(new_track)
new_covariances.append(P_init.copy())
# Apply the winning update (if any) to each track that got matched.
for i, (idx, x_updated, P_updated, _distance) in pending_updates.items():
alive_tracks[i].append((idx, x_updated))
alive_covariances[i] = P_updated
# Go through non-updated tracks and kill them.
# (Built as a fresh list rather than popping from `alive_tracks` while
# enumerating it in place, which used to skip every other stale track.)
still_alive = []
still_alive_covariances = []
for i, track in enumerate(alive_tracks):
if updated[i]:
still_alive.append(track)
still_alive_covariances.append(alive_covariances[i])
else:
tracks.append(track)
alive_tracks = still_alive
alive_covariances = still_alive_covariances
print('Frame: ' + str(frame_idx) + '/' + str(len(frames)) + ', Partial tracks: ' + str(len(tracks)) + '\r', end='')
# Make sure new tracks are alive.
alive_tracks += new_tracks
alive_covariances += new_covariances
# At the final frame, add all remaining ongoing tracks.
tracks += alive_tracks
_tracks = []
for t in tracks:
xs = np.array([x[0] for x in t])
ps = np.array([x[1][0] for x in t])
fs = np.array([x[1][3] for x in t])
# Skip partials outside a reasonable frequency range.
if np.any(fs < 20):
continue
if np.any(fs > 20000):
continue
if np.all(fs < lr.note_to_hz('A0')):
continue
if np.all(fs > lr.note_to_hz('C8')):
continue
# Skip short partials.
if len(xs) <= 1:
continue
_tracks.append(t)
plt.plot(xs, fs)
plt.title('Partial Tracking')
plt.xlabel('Time [frame]')
plt.ylabel('Frequency [Hz]')
tracks = _tracks
len(tracks)
Out[5]:
In [6]:
# Convert partials to note events.
events = []
for t in tracks:
fs = np.array([x[1][3] for x in t])
note = float(lr.hz_to_midi(np.median(fs)))
onset, offset = lr.frames_to_time([t[0][0], t[-1][0]],
sr=sample_rate,
hop_length=hop_length)
events.append((note, onset, offset))
estimated_annotation = pd.DataFrame(events, columns=['Note', 'Onset', 'Offset'])
plt.scatter(reference_annotation.Onset, reference_annotation.Note, marker='o', color='r', alpha=0.5)
plt.scatter(estimated_annotation.Onset, estimated_annotation.Note, marker='x', color='g', alpha=0.5);
In [7]:
# Listen to partials.
synthesis = None
for t in tracks:
xs = np.array([x[0] for x in t])
ps = np.array([x[1][0] for x in t])
fs = np.array([x[1][3] for x in t])
times = np.array(lr.frames_to_time(xs, sr=sample_rate, hop_length=hop_length))
a = me.sonify.pitch_contour(times=times,
frequencies=np.array(fs),
fs=sample_rate)
if a.size > 0:
if synthesis is None:
synthesis = a
else:
synthesis = lr.util.fix_length(synthesis, size=int(np.maximum(synthesis.size, a.size)))
synthesis[:a.size] += a
synthesis = lr.util.normalize(synthesis)
compressed_audio(synthesis, rate=sample_rate)
Out[7]:
Comments
Comments powered by Disqus