Skip to main content

Partial tracking with Kalman filters

In [1]:
import random

from IPython.display import display
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import pandas as pd
import librosa as lr
import mir_eval as me
from librosa.display import specshow
# Compressed audio embedding (Ogg Vorbis) keeps the notebook file itself
# small; IPython's own Audio(data, rate=...) embeds uncompressed 16-bit PCM
# WAV, which for anything more than a few seconds bloats the .ipynb a lot.
from ipytoolbox import compressed_audio

%matplotlib inline
In [2]:
sample_rate = 16000
hop_length = 512
top_db = 30

# Seed everything so the Kalman tracker below (which draws process/observation
# noise from np.random) gives a reproducible partial count and plot run to run.
random.seed(0)
np.random.seed(0)

# NOTE: the original local dataset paths
# (/media/carl/Data/solo-instrument-recordings/...) no longer exist, so we
# fall back to librosa's bundled "trumpet" example recording -- a monophonic
# solo instrument clip, close in spirit to the original guitar/violin/piano
# recordings and well suited to partial tracking. There is no ground-truth
# annotation for it, so `reference_annotation` is left empty (columns only)
# -- the "reference" scatter in the note-event plot below will simply show
# nothing.
audio_path = lr.ex('trumpet')
reference_annotation = pd.DataFrame(columns=['Onset', 'Note'])

audio, _ = lr.load(audio_path, sr=sample_rate)

compressed_audio(audio, rate=sample_rate)
Out[2]:
In [3]:
def spectral_transform(audio_samples, sample_rate, hop_length, top_db):
    audio_samples = lr.util.normalize(audio_samples)
    n_fft = hop_length * 2
    z = lr.stft(y=audio_samples, n_fft=n_fft, hop_length=hop_length)
    bands = lr.fft_frequencies(sr=sample_rate, n_fft=n_fft)
    spectrogram = lr.power_to_db(np.abs(z) ** 2, top_db=top_db)
    return spectrogram, bands


spectrogram, bands = spectral_transform(audio, sample_rate, hop_length, top_db)
specshow(spectrogram, sr=sample_rate, hop_length=hop_length);
No description has been provided for this image

Peak Detection

In [4]:
def partials(spectrogram, top_db):
    """Determine spectral peaks."""

    # Go through audio frame-by-frame.
    frames = []
    for spectrum in spectrogram.T:

        # Don't try to pick peaks if overall audio volume is too low.
        if np.all(spectrum < top_db * 0.1):
            frames.append(([], []))
            continue

        # Pick spectral peaks.
        powers = []
        peaks = []
        dy = 0.01 * top_db
        ys = spectrum
        xs = np.arange(len(spectrum))
        ymin = np.inf
        ymax = -np.inf
        uphill = True
        for x, y in zip(xs, ys):

            if y > ymax:
                ymax = y
                xmax = x

            if y < ymin:
                ymin = y
                xmin = x

            if uphill:
                if y < ymax - dy:
                    powers.append(ymax)
                    peaks.append(xmax)
                    ymin = y
                    xmin = x
                    uphill = False
            else:
                if y > ymin + dy:
                    ymax = y
                    xmax = x
                    uphill = True

        frames.append((peaks, powers))

    peaks = [x[0] for x in frames]
    powers = [x[1] for x in frames]
    return peaks, powers


peaks, powers = partials(spectrogram, top_db=top_db)

print("Number of peaks: ", int(np.sum([len(x) for x in peaks])))

frequencies = [list(bands[list(x)]) for x in peaks]

# Plot a frame.
for frame in zip(spectrogram.T, peaks, powers):
    if len(frame[1]) > 1:
        plt.plot(frame[0])
        plt.scatter(frame[1], frame[2], color='r')
        plt.show()
        plt.xlim([0, 100])
        break

# Plot all frames over time.
X, Y = [], []
for x, ys in enumerate(peaks):
    xs = [x] * len(ys)
    X.extend(xs)
    Y.extend(ys)
specshow(spectrogram, sr=sample_rate, hop_length=hop_length)
plt.gca().scatter(X, Y, marker='.', color='g', alpha=0.1);
Number of peaks:  818
No description has been provided for this image
No description has been provided for this image

Kalman Tracker

In [5]:
frames = list(zip(powers, frequencies))

tracks = []

# Transition matrix
power_variation = 0.99
frequency_variation = 0.9
A = np.array([[1, 1, 0, 0, 0],
              [0, 1, 1, 0, 0],
              [0, 0, power_variation, 0, 0],
              [0, 0, 0, 1, 1],
              [0, 0, 0, 0, frequency_variation]])

# State matrix
noise_gain = 1
power_gain = 10
frequency_gain = 0.1
B = np.array([[0, 0, 0],
              [0, 0, noise_gain],
              [power_gain, 0, 0],
              [0, 0, 0],
              [0, frequency_gain, 0]])

# Observation matrix
C = np.array([[1, 0, 0, 0, 0],
              [0, 0, 0, 1, 0]])

# Process noise covariance
Q = np.eye(3)

# Measurement noise covariance
R = np.eye(2)

# The process noise and observation noise could be thought of as spectral fluctuations caused by inharmonicity, timbral variations, beats, etc.
n_states = A.shape[0]
I_n = np.eye(n_states)

# Initial state-estimate uncertainty. Large relative to Q/R so a brand new
# track's very first correction step trusts the measurement, not the guess.
initial_covariance_scale = 10.0
P_init = np.eye(n_states) * initial_covariance_scale

initial_v = 1.0
initial_w = 1.0
initial_z = 1.0
alive_tracks = [[(0, np.array([p, initial_v, initial_w, f, initial_z]))]
                for p, f in list(zip(*frames[0]))]
alive_covariances = [P_init.copy() for _ in alive_tracks]

# Gate on Mahalanobis distance (in units of standard deviations of the
# innovation), not raw Euclidean distance -- power and frequency live on very
# different scales, so a fixed Euclidean threshold either gates on almost
# nothing but frequency or almost nothing but power depending on units.
gate = 6.0

for frame_idx, frame in enumerate(frames[1:], start=1):

    # --- Predict step: propagate each track's state estimate and its
    # uncertainty (covariance) forward one frame. This is deterministic --
    # unlike the earlier version, no random noise sample is injected into the
    # predicted state itself. Randomness only ever entered Q/R as their
    # *covariance*, describing how uncertain the prediction is, not as an
    # actual perturbation of the estimate.
    predictions = []
    for track, P in zip(alive_tracks, alive_covariances):
        x = track[-1][1]

        x_pred = A.dot(x)
        P_pred = A.dot(P).dot(A.T) + B.dot(Q).dot(B.T)

        y_pred = C.dot(x_pred)
        S = C.dot(P_pred).dot(C.T) + R  # innovation covariance
        K = P_pred.dot(C.T).dot(np.linalg.inv(S))  # Kalman gain

        predictions.append((x_pred, P_pred, y_pred, S, K))

    # Go through all peaks and assign them to ongoing tracks.
    new_tracks = []
    new_covariances = []
    updated = [False] * len(alive_tracks)
    pending_updates = {}  # track index -> (frame_idx, x_updated, P_updated, distance)
    for peak in list(zip(*frame)):
        observation = np.array(peak).T

        # Go through all ongoing tracks and pick the closest to the new
        # observation, using the Mahalanobis distance so power and frequency
        # are compared in units of their own predicted uncertainty.
        candidate_track_i = None
        candidate_distance = np.inf
        candidate_update = None
        for i, (x_pred, P_pred, y_pred, S, K) in enumerate(predictions):
            innovation = observation - y_pred
            distance = float(np.sqrt(innovation.dot(np.linalg.inv(S)).dot(innovation)))
            if distance < candidate_distance:
                candidate_distance = distance
                candidate_track_i = i
                # --- Correct/update step: fold the real observation back
                # into the state estimate via the Kalman gain. This is the
                # step that was missing entirely before -- the observation
                # was previously used only to pick a track and then thrown
                # away, so tracks were really just an unconstrained random
                # walk instead of being pulled toward the measured peak.
                x_updated = x_pred + K.dot(innovation)
                P_updated = (I_n - K.dot(C)).dot(P_pred)
                candidate_update = (frame_idx, x_updated, P_updated)

        if candidate_track_i is not None and candidate_distance < gate:
            if (candidate_track_i not in pending_updates or
                    candidate_distance < pending_updates[candidate_track_i][3]):
                # Pick best fitting candidate peak for this track this frame.
                pending_updates[candidate_track_i] = (
                    candidate_update[0], candidate_update[1], candidate_update[2], candidate_distance)
                updated[candidate_track_i] = True
        else:
            # No candidate track was found: add current observation to a new track.
            p, f = observation
            new_track = [(frame_idx, np.array([p, initial_v, initial_w, f, initial_z]))]
            new_tracks.append(new_track)
            new_covariances.append(P_init.copy())

    # Apply the winning update (if any) to each track that got matched.
    for i, (idx, x_updated, P_updated, _distance) in pending_updates.items():
        alive_tracks[i].append((idx, x_updated))
        alive_covariances[i] = P_updated

    # Go through non-updated tracks and kill them.
    # (Built as a fresh list rather than popping from `alive_tracks` while
    # enumerating it in place, which used to skip every other stale track.)
    still_alive = []
    still_alive_covariances = []
    for i, track in enumerate(alive_tracks):
        if updated[i]:
            still_alive.append(track)
            still_alive_covariances.append(alive_covariances[i])
        else:
            tracks.append(track)
    alive_tracks = still_alive
    alive_covariances = still_alive_covariances

    print('Frame: ' + str(frame_idx) + '/' + str(len(frames)) + ', Partial tracks: ' + str(len(tracks)) + '\r', end='')

    # Make sure new tracks are alive.
    alive_tracks += new_tracks
    alive_covariances += new_covariances

# At the final frame, add all remaining ongoing tracks.
tracks += alive_tracks

_tracks = []
for t in tracks:
    xs = np.array([x[0] for x in t])
    ps = np.array([x[1][0] for x in t])
    fs = np.array([x[1][3] for x in t])

    # Skip partials outside a reasonable frequency range.
    if np.any(fs < 20):
        continue
    if np.any(fs > 20000):
        continue
    if np.all(fs < lr.note_to_hz('A0')):
        continue
    if np.all(fs > lr.note_to_hz('C8')):
        continue

    # Skip short partials.
    if len(xs) <= 1:
        continue

    _tracks.append(t)

    plt.plot(xs, fs)

plt.title('Partial Tracking')
plt.xlabel('Time [frame]')
plt.ylabel('Frequency [Hz]')
tracks = _tracks
len(tracks)
Frame: 1/167, Partial tracks: 2
Frame: 2/167, Partial tracks: 6
Frame: 3/167, Partial tracks: 11
Frame: 4/167, Partial tracks: 14
Frame: 5/167, Partial tracks: 18
Frame: 6/167, Partial tracks: 21
Frame: 7/167, Partial tracks: 25
Frame: 8/167, Partial tracks: 29
Frame: 9/167, Partial tracks: 40
Frame: 10/167, Partial tracks: 43
Frame: 11/167, Partial tracks: 46
Frame: 12/167, Partial tracks: 49
Frame: 13/167, Partial tracks: 57
Frame: 14/167, Partial tracks: 59
Frame: 15/167, Partial tracks: 60
Frame: 16/167, Partial tracks: 64
Frame: 17/167, Partial tracks: 68
Frame: 18/167, Partial tracks: 71
Frame: 19/167, Partial tracks: 75
Frame: 20/167, Partial tracks: 78
Frame: 21/167, Partial tracks: 84
Frame: 22/167, Partial tracks: 88
Frame: 23/167, Partial tracks: 95
Frame: 24/167, Partial tracks: 99
Frame: 25/167, Partial tracks: 99
Frame: 26/167, Partial tracks: 100
Frame: 27/167, Partial tracks: 106
Frame: 28/167, Partial tracks: 110
Frame: 29/167, Partial tracks: 116
Frame: 30/167, Partial tracks: 130
Frame: 31/167, Partial tracks: 134
Frame: 32/167, Partial tracks: 147
Frame: 33/167, Partial tracks: 157
Frame: 34/167, Partial tracks: 162
Frame: 35/167, Partial tracks: 177
Frame: 36/167, Partial tracks: 185
Frame: 37/167, Partial tracks: 190
Frame: 38/167, Partial tracks: 192
Frame: 39/167, Partial tracks: 193
Frame: 40/167, Partial tracks: 193
Frame: 41/167, Partial tracks: 200
Frame: 42/167, Partial tracks: 201
Frame: 43/167, Partial tracks: 202
Frame: 44/167, Partial tracks: 202
Frame: 45/167, Partial tracks: 215
Frame: 46/167, Partial tracks: 225
Frame: 47/167, Partial tracks: 227
Frame: 48/167, Partial tracks: 233
Frame: 49/167, Partial tracks: 240
Frame: 50/167, Partial tracks: 243
Frame: 51/167, Partial tracks: 245
Frame: 52/167, Partial tracks: 250
Frame: 53/167, Partial tracks: 255
Frame: 54/167, Partial tracks: 255
Frame: 55/167, Partial tracks: 256
Frame: 56/167, Partial tracks: 256
Frame: 57/167, Partial tracks: 258
Frame: 58/167, Partial tracks: 261
Frame: 59/167, Partial tracks: 266
Frame: 60/167, Partial tracks: 266
Frame: 61/167, Partial tracks: 266
Frame: 62/167, Partial tracks: 266
Frame: 63/167, Partial tracks: 266
Frame: 64/167, Partial tracks: 269
Frame: 65/167, Partial tracks: 284
Frame: 66/167, Partial tracks: 284
Frame: 67/167, Partial tracks: 287
Frame: 68/167, Partial tracks: 290
Frame: 69/167, Partial tracks: 297
Frame: 70/167, Partial tracks: 300
Frame: 71/167, Partial tracks: 300
Frame: 72/167, Partial tracks: 300
Frame: 73/167, Partial tracks: 300
Frame: 74/167, Partial tracks: 301
Frame: 75/167, Partial tracks: 308
Frame: 76/167, Partial tracks: 308
Frame: 77/167, Partial tracks: 310
Frame: 78/167, Partial tracks: 314
Frame: 79/167, Partial tracks: 321
Frame: 80/167, Partial tracks: 331
Frame: 81/167, Partial tracks: 350
Frame: 82/167, Partial tracks: 353
Frame: 83/167, Partial tracks: 354
Frame: 84/167, Partial tracks: 358
Frame: 85/167, Partial tracks: 360
Frame: 86/167, Partial tracks: 360
Frame: 87/167, Partial tracks: 362
Frame: 88/167, Partial tracks: 362
Frame: 89/167, Partial tracks: 362
Frame: 90/167, Partial tracks: 362
Frame: 91/167, Partial tracks: 362
Frame: 92/167, Partial tracks: 364
Frame: 93/167, Partial tracks: 365
Frame: 94/167, Partial tracks: 369
Frame: 95/167, Partial tracks: 373
Frame: 96/167, Partial tracks: 373
Frame: 97/167, Partial tracks: 373
Frame: 98/167, Partial tracks: 373
Frame: 99/167, Partial tracks: 373
Frame: 100/167, Partial tracks: 373
Frame: 101/167, Partial tracks: 373
Frame: 102/167, Partial tracks: 373
Frame: 103/167, Partial tracks: 373
Frame: 104/167, Partial tracks: 373
Frame: 105/167, Partial tracks: 373
Frame: 106/167, Partial tracks: 373
Frame: 107/167, Partial tracks: 373
Frame: 108/167, Partial tracks: 373
Frame: 109/167, Partial tracks: 373
Frame: 110/167, Partial tracks: 373
Frame: 111/167, Partial tracks: 373
Frame: 112/167, Partial tracks: 373
Frame: 113/167, Partial tracks: 373
Frame: 114/167, Partial tracks: 373
Frame: 115/167, Partial tracks: 373
Frame: 116/167, Partial tracks: 373
Frame: 117/167, Partial tracks: 373
Frame: 118/167, Partial tracks: 373
Frame: 119/167, Partial tracks: 373
Frame: 120/167, Partial tracks: 373
Frame: 121/167, Partial tracks: 373
Frame: 122/167, Partial tracks: 373
Frame: 123/167, Partial tracks: 373
Frame: 124/167, Partial tracks: 373
Frame: 125/167, Partial tracks: 373
Frame: 126/167, Partial tracks: 373
Frame: 127/167, Partial tracks: 373
Frame: 128/167, Partial tracks: 373
Frame: 129/167, Partial tracks: 373
Frame: 130/167, Partial tracks: 373
Frame: 131/167, Partial tracks: 373
Frame: 132/167, Partial tracks: 373
Frame: 133/167, Partial tracks: 373
Frame: 134/167, Partial tracks: 373
Frame: 135/167, Partial tracks: 373
Frame: 136/167, Partial tracks: 373
Frame: 137/167, Partial tracks: 373
Frame: 138/167, Partial tracks: 373
Frame: 139/167, Partial tracks: 373
Frame: 140/167, Partial tracks: 373
Frame: 141/167, Partial tracks: 373
Frame: 142/167, Partial tracks: 373
Frame: 143/167, Partial tracks: 373
Frame: 144/167, Partial tracks: 373
Frame: 145/167, Partial tracks: 373
Frame: 146/167, Partial tracks: 373
Frame: 147/167, Partial tracks: 373
Frame: 148/167, Partial tracks: 373
Frame: 149/167, Partial tracks: 373
Frame: 150/167, Partial tracks: 373
Frame: 151/167, Partial tracks: 373
Frame: 152/167, Partial tracks: 373
Frame: 153/167, Partial tracks: 373
Frame: 154/167, Partial tracks: 373
Frame: 155/167, Partial tracks: 373
Frame: 156/167, Partial tracks: 373
Frame: 157/167, Partial tracks: 373
Frame: 158/167, Partial tracks: 373
Frame: 159/167, Partial tracks: 373
Frame: 160/167, Partial tracks: 373
Frame: 161/167, Partial tracks: 373
Frame: 162/167, Partial tracks: 373
Frame: 163/167, Partial tracks: 373
Frame: 164/167, Partial tracks: 373
Frame: 165/167, Partial tracks: 373
Frame: 166/167, Partial tracks: 373
Out[5]:
207
No description has been provided for this image
In [6]:
# Convert partials to note events.
events = []
for t in tracks:
    fs = np.array([x[1][3] for x in t])
    note = float(lr.hz_to_midi(np.median(fs)))
    onset, offset = lr.frames_to_time([t[0][0], t[-1][0]],
                                      sr=sample_rate,
                                      hop_length=hop_length)
    events.append((note, onset, offset))
estimated_annotation = pd.DataFrame(events, columns=['Note', 'Onset', 'Offset'])

plt.scatter(reference_annotation.Onset, reference_annotation.Note, marker='o', color='r', alpha=0.5)
plt.scatter(estimated_annotation.Onset, estimated_annotation.Note, marker='x', color='g', alpha=0.5);
No description has been provided for this image
In [7]:
# Listen to partials.
synthesis = None
for t in tracks:
    xs = np.array([x[0] for x in t])
    ps = np.array([x[1][0] for x in t])
    fs = np.array([x[1][3] for x in t])

    times = np.array(lr.frames_to_time(xs, sr=sample_rate, hop_length=hop_length))

    a = me.sonify.pitch_contour(times=times,
                                frequencies=np.array(fs),
                                fs=sample_rate)
    if a.size > 0:
        if synthesis is None:
            synthesis = a
        else:
            synthesis = lr.util.fix_length(synthesis, size=int(np.maximum(synthesis.size, a.size)))
            synthesis[:a.size] += a

synthesis = lr.util.normalize(synthesis)
compressed_audio(synthesis, rate=sample_rate)
Out[7]:

Comments

Comments powered by Disqus