Create results_dejavu.py

scripts/results_dejavu.py

@@ -0,0 +1,355 @@
+# result generator for dejavu
+
+# TODO: Don't work very well with musics with special chars.
+# use test file on the format below, with no special chars and only one "-" to separate artist from song
+
+import os, subprocess, json, re, sys
+import logging, time
+from os import listdir
+from os.path import isfile, join
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.animation as animation
+from optparse import OptionParser
+
+#####
+### 	Test files are in specific format:
+###		'artist_name'-'song_name'_'start_time'_'duration'sec.wav
+#####
+
+DEFAULT_FS = 44100
+DEFAULT_WINDOW_SIZE = 4096
+DEFAULT_OVERLAP_RATIO = 0.5
+
+#FIELD_SONG_ID 				= 'song_id'
+FIELD_SONG_NAME				= 'song_name'
+FIELD_CONFIDENCE			= 'confidence'
+FIELD_QUERY_TIME			= 'match_time'
+FIELD_OFFSET				= 'offset'
+
+# Parse options 
+
+usage = "usage: %prog [options] DEJAVU_PATH TEST_FOLDER"
+parser = OptionParser(usage=usage, version="%prog 1.1")
+
+parser.add_option("--no-log",
+				  action="store_false",
+                  dest="log",
+                  default=True,
+                  help='Disables logging'
+                  )
+parser.add_option("--log-file",
+                  dest="log_file",
+                  default="results-compare.log",
+                  metavar="LOG_FILE",
+                  help='Set the path and filename of the log file'
+                  )
+parser.add_option("--test-seconds",
+				  action="append",
+                  dest="test_seconds",
+                  default=[],
+                  metavar="Xsec",
+                  help='Appends seconds to test suit'
+                  )
+parser.add_option("--results-folder",
+				  action="store",
+                  dest="results_folder",
+                  metavar="FOLDER",
+                  help='Sets the path where the results are saved'
+                  )
+
+(options, args) = parser.parse_args()
+
+if len(args) != 2:
+	parser.error("wrong number of arguments")
+
+if len(options.test_seconds) == 0:
+	options.test_seconds = ['1sec','2sec','3sec','4sec','5sec','6sec','7sec','8sec','9sec','10sec']
+
+if options.log == True:
+	logging.basicConfig( filename=options.log_file, level=logging.DEBUG )
+
+if options.results_folder != "" and options.results_folder[len(options.results_folder)-1] != '/':
+	options.results_folder += "/"
+
+def log_msg(msg):
+	if options.log == True:
+		logging.debug(msg)
+
+class DejavuTest (object):
+	def __init__(self, folder, seconds):
+		super(DejavuTest, self).__init__()
+
+		self.test_folder = folder
+		self.test_seconds = seconds
+		self.test_songs = []
+		self.test_files = [ f for f in listdir(self.test_folder) if isfile(join(self.test_folder,f)) 
+			and re.findall("[0-9]*sec",f)[0] in self.test_seconds ]
+		self.n_columns = len(self.test_seconds)
+		self.n_lines = len(self.test_files) / self.n_columns 
+
+		# variable match results (yes, no, invalid)
+		self.result_match = [[0 for x in xrange(self.n_columns)] for x in xrange(self.n_lines)] 
+
+		# variable match precision (if matched in the corrected time)
+		self.result_matching_times = [[0 for x in xrange(self.n_columns)] for x in xrange(self.n_lines)] 
+
+		# variable mahing time (query time) 
+		self.result_query_duration = [[0 for x in xrange(self.n_columns)] for x in xrange(self.n_lines)] 
+
+		# variable confidence
+		self.result_match_confidence = [[0 for x in xrange(self.n_columns)] for x in xrange(self.n_lines)] 
+
+		self.begin()
+
+	def get_column_id ( self,secs ):
+		for i, sec in enumerate(self.test_seconds):
+			if secs == sec:
+				return i
+
+	def get_line_id ( self,artist, song ):
+		elem = artist + " - " + song
+
+		for i, s in enumerate(self.test_songs):
+			if elem == s:
+				return i
+
+		self.test_songs.append(elem)
+		return len(self.test_songs)-1
+
+	def begin(self):
+		for f in self.test_files:
+			log_msg('--------------------------------------------------')
+			log_msg('file: %s' % f)
+			# get column 
+			col = self.get_column_id(re.findall("[0-9]*sec",f)[0])
+
+			# get artist and song
+			artist = re.findall("^[^\-]+",f)
+			artist = artist[0].rstrip()
+
+			song = re.findall("\-[^\_]+",f)
+			song = song[0].lstrip("- ")
+
+			line = self.get_line_id ( artist, song)
+
+			result = subprocess.check_output([args[0], 'recognize', 'file', self.test_folder+"/"+f])
+			log_msg('RESULT: %s' % result.strip() )
+
+			if result.strip() == "None":
+				log_msg('No match')
+				self.result_match[line][col] = 'no'
+				self.result_matching_times[line][col] = 0
+				self.result_query_duration[line][col] = 0
+				self.result_match_confidence[line][col] = 0
+			
+			else:
+				result = result.strip()
+				result = result.replace(" \'", ' "')
+				result = result.replace("{\'", '{"')
+				result = result.replace("\':", '":')
+				result = result.replace("\',", '",')
+
+				result = json.loads(result)
+
+				artist_result = re.findall("^[^\-]+",result[FIELD_SONG_NAME])
+				artist_result = artist_result[0].rstrip()
+
+				song_result = re.findall("\-[^\_]+",result[FIELD_SONG_NAME])
+				song_result = song_result[0].lstrip("- ")
+
+				log_msg('artist: %s' % artist)
+				log_msg('artist_result: %s' % artist_result)
+				log_msg('song: %s' % song)
+				log_msg('song_result: %s' % song_result)
+
+				if artist_result != artist or song_result != song:
+					log_msg('invalid match')
+					self.result_match[line][col] = 'invalid'
+					self.result_matching_times[line][col] = 0
+					self.result_query_duration[line][col] = 0
+					self.result_match_confidence[line][col] = 0
+				else:
+					log_msg('correct match')
+					self.result_match[line][col] = 'yes'
+					self.result_query_duration[line][col] = round(result[FIELD_QUERY_TIME],3)
+					self.result_match_confidence[line][col] = result[FIELD_CONFIDENCE]
+
+					song_start_time = re.findall("\_[^\_]+",f)
+					song_start_time = song_start_time[0].lstrip("_ ")
+
+					#result_start_time = round((result[FIELD_SONG_DURATION] * result[FIELD_OFFSET]) / float(result[FIELD_SONG_SPEC_DURATION]), 0)
+					result_start_time = round((result[FIELD_OFFSET] * DEFAULT_WINDOW_SIZE * DEFAULT_OVERLAP_RATIO) / (DEFAULT_FS),0)
+
+					self.result_matching_times[line][col] = int(result_start_time) - int(song_start_time)
+					if (abs(self.result_matching_times[line][col]) == 1):
+						self.result_matching_times[line][col] = 0
+
+					log_msg('query duration: %s' % round(result[FIELD_QUERY_TIME],3))
+					log_msg('confidence: %s' % result[FIELD_CONFIDENCE])
+					log_msg('song start_time: %s' % song_start_time)
+					log_msg('result start time: %s' % result_start_time)
+					if (self.result_matching_times[line][col] == 0):
+						log_msg('accurate match')
+					else:
+						log_msg('inaccurate match')
+			log_msg('--------------------------------------------------\n')
+
+print "obtaining results from dejavu"
+log_msg('obtaining results from dejavu')
+tm = time.time()
+djv = DejavuTest(args[1], options.test_seconds)
+print "finished obtaining results from dejavu in %s" % (time.time() - tm)
+log_msg("finished obtaining results from dejavu in %s" % (time.time() - tm))
+
+tests_n_lines = djv.n_lines
+tests_n_columns = djv.n_columns # len(options.test_seconds)
+tests = 1 # djv
+n_secs = len(options.test_seconds) # = tests.n_columns
+
+# set result variables -> 4d variables
+all_match_counter = [[[0 for x in xrange(tests)] for x in xrange(3)] for x in xrange(n_secs)]
+all_matching_times_counter = [[[0 for x in xrange(tests)] for x in xrange(2)] for x in xrange(n_secs)]
+all_query_duration = [[[0 for x in xrange(tests)] for x in xrange(tests_n_lines)] for x in xrange(n_secs)]
+all_match_confidence = [[[0 for x in xrange(tests)] for x in xrange(tests_n_lines)] for x in xrange(n_secs)]
+
+# agroup results by seconds
+for line in range(0, tests_n_lines):
+	for col in range(0, tests_n_columns):
+		# for dejavu
+		all_query_duration[col][line][0] = djv.result_query_duration[line][col]
+		all_match_confidence[col][line][0] = djv.result_match_confidence[line][col]
+
+		djv_match_result = djv.result_match[line][col]
+
+		if djv_match_result == 'yes':
+			all_match_counter[col][0][0] += 1
+		elif djv_match_result == 'no':
+			all_match_counter[col][1][0] += 1
+		else:
+			all_match_counter[col][2][0] += 1
+
+		djv_match_acc = djv.result_matching_times[line][col]
+
+		if djv_match_acc == 0 and djv_match_result == 'yes':
+			all_matching_times_counter[col][0][0] += 1
+		elif djv_match_acc != 0:
+			all_matching_times_counter[col][1][0] += 1
+
+def autolabel(rects,ax):
+    # attach some text labels
+    for rect in rects:
+        height = rect.get_height()
+        ax.text(rect.get_x()+rect.get_width()/2., 1.05*height, '%d'%int(height),
+                ha='center', va='bottom')
+
+def autolabeldoubles(rects,ax):
+    # attach some text labels
+    for rect in rects:
+        height = rect.get_height()
+        ax.text(rect.get_x()+rect.get_width()/2., 1.05*height, '%s'%round(float(height),3),
+                ha='center', va='bottom')
+
+def create_plots(name,results):
+	for sec in range(0,n_secs):
+		ind = np.arange(tests_n_lines) #
+		width = 0.25       # the width of the bars
+
+		fig = plt.figure()
+		ax = fig.add_subplot(111)
+		ax.set_xlim([-1*width, 2*width])
+
+		means_dvj = [x[0] for x in results[sec]]
+		rects1 = ax.bar(ind, means_dvj, width, color='r')
+
+		# add some
+		ax.set_ylabel(name)
+		ax.set_title("%s %s Results" % (options.test_seconds[sec],name)) 
+		ax.set_xticks(ind+width)
+
+		labels = [0 for x in range(0,tests_n_lines)]
+		for x in range(0,tests_n_lines):
+			labels[x] = "song %s" % (x+1)
+		ax.set_xticklabels( labels )
+
+		box = ax.get_position()
+		ax.set_position([box.x0, box.y0, box.width * 0.75, box.height])
+
+		#ax.legend( (rects1[0]), ('Dejavu'), loc='center left', bbox_to_anchor=(1, 0.5))
+
+		if name == 'Confidence':
+			autolabel(rects1,ax)
+		else:
+			autolabeldoubles(rects1,ax)
+
+		plt.grid()
+
+		fig_name = "%s%s_%s.png" % (options.results_folder,name,options.test_seconds[sec])
+		fig.savefig(fig_name)
+
+create_plots('Confidence',all_match_confidence)
+create_plots('Query duration',all_query_duration)
+
+for sec in range(0,n_secs):
+	ind = np.arange(3) #
+	width = 0.25       # the width of the bars
+
+	fig = plt.figure()
+	ax = fig.add_subplot(111)
+	ax.set_xlim([-1*width, 2.75])
+
+	means_dvj = [round(x[0]*100/tests_n_lines,1) for x in all_match_counter[sec]]
+	rects1 = ax.bar(ind, means_dvj, width, color='r')
+
+	# add some
+	ax.set_ylabel('Matching Percentage')
+	ax.set_title('%s Matching Percentage' % options.test_seconds[sec])
+	ax.set_xticks(ind+width)
+
+	labels = ['yes','no','invalid']
+	ax.set_xticklabels( labels )
+
+	box = ax.get_position()
+	ax.set_position([box.x0, box.y0, box.width * 0.75, box.height])
+
+	#ax.legend( (rects1[0]), ('Dejavu'), loc='center left', bbox_to_anchor=(1, 0.5))
+	autolabeldoubles(rects1,ax)
+
+	plt.grid()
+
+ 	fig_name = "%smatching_perc_%s.png" % (options.results_folder,options.test_seconds[sec])
+ 	fig.savefig(fig_name)
+
+for sec in range(0,n_secs):
+	ind = np.arange(2) #
+	width = 0.25       # the width of the bars
+
+	fig = plt.figure()
+	ax = fig.add_subplot(111)
+	ax.set_xlim([-1*width, 1.75])
+
+	div = all_match_counter[sec][0][0]
+	if div == 0 :
+		div = 1000000
+
+	means_dvj = [round(x[0]*100/div,1) for x in all_matching_times_counter[sec]]
+	rects1 = ax.bar(ind, means_dvj, width, color='r')
+
+	# add some
+	ax.set_ylabel('Matching Accuracy')
+	ax.set_title('%s Matching Times Accuracy' % options.test_seconds[sec])
+	ax.set_xticks(ind+width)
+
+	labels = ['yes','no']
+	ax.set_xticklabels( labels )
+
+	box = ax.get_position()
+	ax.set_position([box.x0, box.y0, box.width * 0.75, box.height])
+
+	#ax.legend( (rects1[0]), ('Dejavu'), loc='center left', bbox_to_anchor=(1, 0.5))
+	autolabeldoubles(rects1,ax)
+
+	plt.grid()
+
+ 	fig_name = "%smatching_acc_%s.png" % (options.results_folder,options.test_seconds[sec])
+ 	fig.savefig(fig_name)