first continous analysis
This commit is contained in:
@@ -0,0 +1,350 @@
|
||||
import librosa
|
||||
import librosa.display
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
import soundfile as sf
|
||||
from pydub import AudioSegment
|
||||
from pydub.silence import split_on_silence, detect_nonsilent
|
||||
import math
|
||||
import wave
|
||||
import contextlib
|
||||
import random
|
||||
from mpl_toolkits.axes_grid1.axes_divider import VBoxDivider
|
||||
import mpl_toolkits.axes_grid1.axes_size as Size
|
||||
import cv2
|
||||
|
||||
import webrtcvad
|
||||
|
||||
min_silence_len = 400
|
||||
|
||||
|
||||
def calc_dtw_sim(y1, y2, sr1, sr2, plot_result=False):
|
||||
hop_length = 64
|
||||
assert sr1 == sr2
|
||||
|
||||
l = min(len(y1), len(y2))
|
||||
|
||||
to_consider = min(l, max(round(0.2*l), 2048))
|
||||
min_len = millisecond_to_samples(100, sr1)
|
||||
bound = round(0.5 * l)
|
||||
if bound < min_len:
|
||||
bound = min_len
|
||||
#bound = max(round(0.2 * l), millisecond_to_samples(200, sr1))
|
||||
|
||||
y1 = y1[0:bound]
|
||||
y2 = y2[0:bound]
|
||||
|
||||
if bound < 2048:
|
||||
n_fft = bound
|
||||
n_mels = 64
|
||||
else:
|
||||
n_fft = 2048
|
||||
n_mels = 128
|
||||
|
||||
mfcc1 = librosa.feature.mfcc(y=y1, sr=sr1, hop_length=hop_length, n_mfcc=42, n_fft=n_fft, n_mels=n_mels)[1:,:]
|
||||
mfcc2 = librosa.feature.mfcc(y=y2, sr=sr2, hop_length=hop_length, n_mfcc=42, n_fft=n_fft, n_mels=n_mels)[1:,:]
|
||||
|
||||
D, wp = librosa.sequence.dtw(mfcc1, mfcc2)
|
||||
|
||||
if plot_result:
|
||||
fig, ax = plt.subplots(nrows=4)
|
||||
|
||||
img = librosa.display.specshow(D, x_axis='frames', y_axis='frames',
|
||||
ax=ax[0])
|
||||
|
||||
ax[0].set(title='DTW cost', xlabel='Noisy sequence', ylabel='Target')
|
||||
|
||||
ax[0].plot(wp[:, 1], wp[:, 0], label='Optimal path', color='y')
|
||||
|
||||
ax[0].legend()
|
||||
|
||||
fig.colorbar(img, ax=ax[0])
|
||||
|
||||
ax[1].plot(D[-1, :] / wp.shape[0])
|
||||
|
||||
ax[1].set(xlim=[0, mfcc1.shape[1]],
|
||||
title='Matching cost function')
|
||||
|
||||
ax[2].imshow(mfcc1)
|
||||
ax[3].imshow(mfcc2)
|
||||
|
||||
plt.show()
|
||||
total_alignment_cost = D[-1, -1] / wp.shape[0]
|
||||
|
||||
return total_alignment_cost
|
||||
|
||||
|
||||
def calc_xcorr_sim(y1, y2, sr1, sr2):
|
||||
hop_length = 256
|
||||
|
||||
y1 = y1[0:round(len(y1)*0.2)]
|
||||
y2 = y2[0:round(len(y2)*0.2)]
|
||||
|
||||
mfcc1 = librosa.feature.mfcc(y=y1, sr=sr1, hop_length=hop_length, n_mfcc=13)[1:,:]
|
||||
mfcc2 = librosa.feature.mfcc(y=y2, sr=sr2, hop_length=hop_length, n_mfcc=13)[1:,:]
|
||||
|
||||
xsim = librosa.segment.cross_similarity(mfcc1, mfcc2, mode='distance')
|
||||
return xsim
|
||||
|
||||
|
||||
def match_target_amplitude(aChunk, target_dBFS):
|
||||
''' Normalize given audio chunk '''
|
||||
change_in_dBFS = target_dBFS - aChunk.dBFS
|
||||
return aChunk.apply_gain(change_in_dBFS)
|
||||
|
||||
|
||||
def spl_on_silence():
|
||||
# Import the AudioSegment class for processing audio and the
|
||||
|
||||
# Load your audio.
|
||||
song = AudioSegment.from_wav("recording.wav")
|
||||
|
||||
# Split track where the silence is 2 seconds or more and get chunks using
|
||||
# the imported function.
|
||||
chunks = split_on_silence (
|
||||
# Use the loaded audio.
|
||||
song,
|
||||
# Specify that a silent chunk must be at least 2 seconds or 2000 ms long.
|
||||
min_silence_len = 1000,
|
||||
# Consider a chunk silent if it's quieter than -16 dBFS.
|
||||
# (You may want to adjust this parameter.)
|
||||
silence_thresh = -50,
|
||||
timestamps=True
|
||||
)
|
||||
|
||||
## Process each chunk with your parameters
|
||||
#for i, chunk in enumerate(chunks):
|
||||
# # Create a silence chunk that's 0.5 seconds (or 500 ms) long for padding.
|
||||
# silence_chunk = AudioSegment.silent(duration=500)
|
||||
|
||||
# # Add the padding chunk to beginning and end of the entire chunk.
|
||||
# audio_chunk = silence_chunk + chunk + silence_chunk
|
||||
|
||||
# # Normalize the entire chunk.
|
||||
# normalized_chunk = match_target_amplitude(audio_chunk, -20.0)
|
||||
|
||||
# # Export the audio chunk with new bitrate.
|
||||
# print("Exporting chunk{0}.mp3.".format(i))
|
||||
# normalized_chunk.export(
|
||||
# ".//chunk{0}.wav".format(i),
|
||||
# bitrate = "192k",
|
||||
# format = "wav"
|
||||
# )
|
||||
return ([ audiosegment_to_librosawav(c) for c in chunks ], song.frame_rate)
|
||||
|
||||
|
||||
def non_silent_chunks(song):
|
||||
#song = AudioSegment.from_wav("recording.wav")
|
||||
|
||||
return detect_nonsilent(song, min_silence_len=min_silence_len, silence_thresh=-50)
|
||||
|
||||
|
||||
def audiosegment_to_librosawav(audiosegment):
|
||||
channel_sounds = audiosegment.split_to_mono()
|
||||
samples = [s.get_array_of_samples() for s in channel_sounds]
|
||||
|
||||
fp_arr = np.array(samples).T.astype(np.float32)
|
||||
fp_arr /= np.iinfo(samples[0].typecode).max
|
||||
fp_arr = fp_arr.reshape(-1)
|
||||
|
||||
return fp_arr
|
||||
|
||||
|
||||
# sr = samples / second
|
||||
def millisecond_to_samples(ms, sr):
|
||||
return round((ms / 1000) * sr)
|
||||
|
||||
|
||||
def samples_to_millisecond(samples, sr):
|
||||
return (samples / sr) * 1000
|
||||
|
||||
|
||||
def ms_to_time(ms):
|
||||
secs = ms / 1000
|
||||
return "{0}:{1}".format(math.floor(secs / 60), secs % 60)
|
||||
|
||||
|
||||
def seg_is_speech(seg):
|
||||
f = lambda x: int(32768 * x)
|
||||
x = np.vectorize(f)(seg)
|
||||
pcm_data = x.tobytes()
|
||||
|
||||
speeches = 0
|
||||
total = 0
|
||||
offset = 0
|
||||
n = int(sr * (frame_duration_ms / 1000.0) * 2)
|
||||
duration = (float(n) / sr) / 2.0
|
||||
|
||||
while offset + n < len(pcm_data):
|
||||
frame = pcm_data[offset:(offset+n)]
|
||||
if vad.is_speech(frame, sr):
|
||||
speeches += 1
|
||||
offset = offset + n
|
||||
total += 1
|
||||
|
||||
#return speeches / total
|
||||
return 1.0
|
||||
|
||||
|
||||
def calculate_best_offset(mfcc_ref, mfcc_seg, sr):
|
||||
return librosa.segment.cross_similarity(mfcc_seg, mfcc_ref, mode='affinity', metric='cosine')
|
||||
|
||||
|
||||
def detect_lines(img, duration_x, duration_y):
|
||||
#print(img.shape)
|
||||
#print(np.min(img), np.max(img))
|
||||
img = cv2.imread('affine_similarity.png')
|
||||
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
|
||||
kernel_size = 5
|
||||
blur_gray = cv2.GaussianBlur(gray, (kernel_size, kernel_size), 0)
|
||||
|
||||
low_threshold = 50
|
||||
high_threshold = 150
|
||||
edges = cv2.Canny(blur_gray, low_threshold, high_threshold)
|
||||
|
||||
rho = 1 # distance resolution in pixels of the Hough grid
|
||||
theta = np.pi / 180 # angular resolution in radians of the Hough grid
|
||||
threshold = 15 # minimum number of votes (intersections in Hough grid cell)
|
||||
min_line_length = 50 # minimum number of pixels making up a line
|
||||
max_line_gap = 20 # maximum gap in pixels between connectable line segments
|
||||
line_image = np.copy(img) * 0 # creating a blank to draw lines on
|
||||
|
||||
# Run Hough on edge detected image
|
||||
# Output "lines" is an array containing endpoints of detected line segments
|
||||
lines = cv2.HoughLinesP(edges, rho, theta, threshold, np.array([]),
|
||||
min_line_length, max_line_gap)
|
||||
|
||||
width, height = img.shape[1], img.shape[0]
|
||||
|
||||
scale_x = duration_x / width
|
||||
scale_y = duration_y / height
|
||||
print(img.shape, scale_x, scale_y, duration_x, duration_y)
|
||||
|
||||
#slope = duration_y / duration_x
|
||||
slope = 1
|
||||
|
||||
expected_slope = scale_x / scale_y
|
||||
#expected_slope = 0.101694915
|
||||
|
||||
print(expected_slope)
|
||||
offsets = []
|
||||
for line in lines:
|
||||
for x1,y1,x2,y2 in line:
|
||||
# swapped y1 and y2 since y is measured from the top
|
||||
slope = (y1-y2)/(x2-x1)
|
||||
if abs(slope - expected_slope) < 0.03:
|
||||
cv2.line(line_image,(x1,y1),(x2,y2),(255,0,0),5)
|
||||
cv2.putText(img, "{:.2f}".format(slope), (x1, y1), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5,color=(0, 0, 255))
|
||||
if (x1 / width) < 0.15:
|
||||
print(height-y1)
|
||||
y = height - y1
|
||||
y0 = y - x1 * slope
|
||||
offsets.append(y0 * scale_y)
|
||||
#actual_lines.append((x1 * scale_x, (height - y1) * scale_y, x2 * scale_x, (height - y2) * scale_y))
|
||||
#print(max(slopes))
|
||||
|
||||
lines_edges = cv2.addWeighted(img, 0.8, line_image, 1, 0)
|
||||
#cv2.imshow("lines", lines_edges)
|
||||
#cv2.waitKey(0)
|
||||
return offsets
|
||||
|
||||
|
||||
def map2d(x, y, f):
|
||||
n_x = len(x)
|
||||
n_y = len(y)
|
||||
res = np.zeros((n_x, n_y))
|
||||
for i in range(n_x):
|
||||
for j in range(n_y):
|
||||
res[i,j] = f(x[i], y[j])
|
||||
return res
|
||||
|
||||
|
||||
def make_widths_equal(fig, rect, ax1, ax2, ax3, pad):
|
||||
# pad in inches
|
||||
divider = VBoxDivider(
|
||||
fig, rect,
|
||||
horizontal=[Size.AxesX(ax1), Size.Scaled(1), Size.AxesX(ax2), Size.Scaled(1), Size.AxesX(ax3)],
|
||||
vertical=[Size.AxesY(ax1), Size.Fixed(pad), Size.AxesY(ax2), Size.Fixed(pad), Size.AxesY(ax3)])
|
||||
ax1.set_axes_locator(divider.new_locator(0))
|
||||
ax2.set_axes_locator(divider.new_locator(2))
|
||||
ax3.set_axes_locator(divider.new_locator(4))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
#vad = webrtcvad.Vad()
|
||||
#hop_length = 128
|
||||
#n_mfcc = 13
|
||||
|
||||
#frame_duration_ms = 10
|
||||
fp = "hard_piece_7.wav"
|
||||
y, sr = librosa.load(fp, mono=True)
|
||||
song = AudioSegment.from_wav(fp)
|
||||
|
||||
ts_non_sil_ms = non_silent_chunks(song)
|
||||
|
||||
#print(y.shape)
|
||||
#mfcc = librosa.feature.mfcc(y=y, sr=sr, hop_length=hop_length, n_mfcc=n_mfcc)[1:,:]
|
||||
#print(mfcc.shape)
|
||||
|
||||
#seg_duration_ms = 100
|
||||
#seg_duration_samples = millisecond_to_samples(seg_duration_ms, sr)
|
||||
|
||||
## split complete audio in 10ms segments, only keep those that have voice in it
|
||||
##segs = []
|
||||
##offset = 0
|
||||
###i = 0
|
||||
##while offset + seg_duration_samples < len(y):
|
||||
## seg = y[ offset : offset + seg_duration_samples ]
|
||||
## if seg_is_speech(seg):
|
||||
## segs.append((seg, offset))
|
||||
## offset += seg_duration_samples
|
||||
|
||||
##segs = segs[1:]
|
||||
##n_segs = len(segs)
|
||||
|
||||
##(seg, offset) = segs[0]
|
||||
|
||||
fp_segment = "segment.wav"
|
||||
seg, sr_seg = librosa.load(fp_segment, mono=True)
|
||||
|
||||
assert sr==sr_seg
|
||||
|
||||
##for seg in segs:
|
||||
#mfcc_seg = librosa.feature.mfcc(y=seg, sr=sr_seg, hop_length=hop_length, n_mfcc=n_mfcc)[1:,:]
|
||||
#xsim = calculate_best_offset(mfcc, mfcc_seg, sr)
|
||||
|
||||
#fig, ax = plt.subplots(nrows=1, sharex=True)
|
||||
#img = librosa.display.specshow(xsim, x_axis='s', y_axis='s', hop_length=hop_length, ax=ax, cmap='magma_r')
|
||||
#print(detect_lines(xsim))
|
||||
#ax.imshow(np.transpose(xsim), aspect='auto')
|
||||
#ax[1].imshow(diffs_penalised)
|
||||
#ax[1].imshow(np.reshape(vad_coeffs, (1, n_segs)))
|
||||
#ax[2].imshow(np.reshape(lengths, (1, n_segs)))
|
||||
|
||||
#make_widths_equal(fig, 111, ax[0], ax[1], ax[2], pad=0.5)
|
||||
#plt.show()
|
||||
found_starts = sorted([ samples_to_millisecond(y0, sr) for y0 in detect_lines(None, len(seg), len(y))])
|
||||
|
||||
def f(ts, start):
|
||||
return abs(ts[0] - start)
|
||||
|
||||
closest = map2d(ts_non_sil_ms, found_starts, f)
|
||||
plt.imshow(closest)
|
||||
plt.show()
|
||||
latest = -1
|
||||
for i, row in enumerate(closest):
|
||||
# min silence len = 400
|
||||
if min(row) < min_silence_len / 2:
|
||||
latest = ts_non_sil_ms[i]
|
||||
print("delete until:", ms_to_time(latest[0]))
|
||||
|
||||
#print("possible starts:", [ ms_to_time(t) for t in found_starts])
|
||||
#for n, seg in enumerate(segs):
|
||||
# sf.write('part' + str(n) + '.wav', seg, sr)
|
||||
#print(segs)
|
||||
|
||||
#y1, sr1 = librosa.load("out000.wav")
|
||||
#y2, sr2 = librosa.load("out004.wav")
|
||||
|
||||
#print("total alignment cost:", calc_dtw_sim(y1, y2, sr1, sr2, plot_result=True))
|
||||
#print("xcorr:", np.trace(calc_xcorr_sim(y1, y2, sr1, sr2)))
|
||||
Reference in New Issue
Block a user