ytsearch/ytsearch

#!/usr/bin/python3

from tg_youtube_search import YoutubeSearch
import os
import configparser
import sys, getopt
import requests, json
import random, string
import logging

##Import list of songs, artists from list.txt
##parse this file into a list of dictionaries
##for each dictionary pair, search youtube and output formatted links in file
##
## - Download over tor
##   - Check if Tor installed
##   - Warn user if NOT using tor
##
## - Multi-thread conversion to audio format
##
## - Logging
##
## - Check for dependencies
##    - Tor?
##    - Write Permissions in log files?
##    - YoutubeSearch
##
## - Differentiate between releases and masters when calling api
##
## - check for dependencies on start
##   - youtube-dl
##   - tor (if used)
##
## - Download the first X results from YoutubeSearch and pick best result
##
## - Allow interrupt to stop script (CTRL + C)
##

##Vars
if os.path.exists('config.ini'):
	config = configparser.ConfigParser()
	config.read('config.ini')
else:
	print("Config.ini file not found in", os.getcwd())
	print("Exiting...")
	exit(1)

VERSION="0.0.4"
DOWNLOAD=config['DEFAULT'].getboolean('Download')	#Download True/False
MUSICFILE=config['DEFAULT']['Musicfile']		#location of text file containing songs
RETRIES=config['DEFAULT'].getint('Retries')		#Number of retries to search for songs
LOGPATH=config['DEFAULT']['LogLocation']
ITERATOR=0						#Number or current tries
STORAGEPATH=config['DEFAULT']['DefaultStoragePath']
VERBOSITY=config['DEFAULT'].getint('Verbosity')
KEY=config['DEFAULT']['Key']
DISCOG=""
DESTFOLDER=""
ALBUM=""
ARTIST=""
MASTER=True
TOR=False
TESTFOLDER=config['DEFAULT'].getboolean('TestFolder')
HELP=	"Takes list.txt or Discogs.com Master/Release number \n" \
	"And downloads albums by stripping the audio from Yuotube videos. \n" \
	"USAGE: ytsearch [-flag] [Discog Num] \n" \
	"	-h 		This help file \n" \
	"	-d --discog	set Discog.com Release or Master number \n" \
	"	-r --release	search for Discog.com RELEASE instead of MASTER [default MASTER] \n" \
	"	-D --download	override config.ini and set Download=True \n" \
	"	-f --file	Allows quick download of a single Youtube link \n" \
	"	-t --tor	Will perform actions over tor using torsocks\n" \
#	"	-D --download	override config.ini and set Download=True"
JSONDATA=[]

music=[]	# list to hold dictionaries of songnum, Title, Artist
logresults=[]	# list to hold results of link creation attempts, and download attempts
linkresults=[]	#
completed=[]	# list to hold song numbers of completed downloads so they can be removed from MUSICFILE


def msg(message, level):
	##Takes message and log level as arguments
	##Based  on verbosity from config.ini, return messages to user
	## 1-ERROR, 2-WARN, 3-INFO, -1 [No flag, always show]
	tlevel = {-1: '', 1: "ERROR", 2: "WARN", 3: "INFO"}
	if level <= VERBOSITY:
		print(tlevel.get(level), message)
	
	## Add logging ##T

def arguments(argv):
	msg("Starting arguments", 3)
	try:
		opts, args = getopt.getopt(argv, "hvrtDf:d:", ["discog", "release", "help", "download", "version", "file", "tor"])
		for opt, arg in opts:
			if opt in ('-h', '--help'):
				print(HELP)
				sys.exit()
			elif opt in ("-d", "--discog"):
				global DISCOG
				DISCOG = arg
				msg("Discog number:" + DISCOG, 3)
			elif opt in ("-D", "--download"):
				global DOWNLOAD
				DOWNLOAD = arg
				msg("Override DOWNLOAD from agrs", 2)
			elif opt in ("-v", "--version"):
				msg("Version: " + VERSION, -1)
				sys.exit()
			elif opt in ("-r", "--release"):
				global MASTER
				MASTER=False
				msg("searching for Release, not Master at Discogs.com", 1)
			elif opt in ("-f", "--file"):
				msg("call singlesong with: " + arg, 3)
				singlesong(arg)
			elif opt in ("-t", "--tor"):
				msg("perform operations over tor", 1)
				if not checktor():
					msg("Torsocks not found!  Check that torsocks is in /usr/bin/torsocks",1)
				else:
					TOR=True
					pass

	except getopt.GetoptError as err:
		msg("cannot get arguments, {0}".format(err), 1)

def checktor():
	msg("Starting checktor", 3)
	if not os.system('which torsocks'):
		msg("Torsocks installed!", 3)
		return 1
	else:
		return 0

def fetchjson(discogno, master=True, show=True):
	msg("Starting fetchjson", 3)
	if master:
		url = 'https://api.discogs.com/masters/'
	else:
		url = 'https://api.discogs.com/releases/'
	url = url + discogno
	msg("Downloading " + url, 1)
	r = requests.get(url)
	global JSONDATA
#	JSONDATA = r.json()
	JSONDATA = json.loads(r.text)
	msg("fetchjson complete!", 3)
	return JSONDATA

def buildlist(jsondata, write=False):
	## takes raw jsons data from Discogs.com and extracts Album, Artist, and tracklist list
	## passes tracklist to gettracks to create list of track names
	## appends "," + artist to end of track names
	## calls buildfolders to create a home for the new file
	## writes list to list.txt in appropriate folder
	msg("Staring Buildlist", 3)
	try:
		Artist = jsondata['artists'][0]['name']
		Album = jsondata['title']
	except:
		msg("Could not read Artist or Album Name from Jsonfile", 1)
		print(sys.exc_info()[0])
		sys.exit()
	if Artist.find( '(' ) != -1:	## Discovered a Artist 'Tool (2)' (Discogs 1181).  This removes ()
		Artist = Artist[:Artist.find( '(' )-1]
		msg("Correcting Artist name to " + Artist, 2)
	global ALBUM
	try:
		ALBUM = Album
		msg("Set ALBUM var to: " + ALBUM, 3)
	except Exception as e:
		msg("Could not set ALBUM var." + e, 2)
	else:
		print(Artist)

	tracks = gettracks(jsondata['tracklist'])

	for i in range(len(tracks)):
		tracks[i] = tracks[i] + ", " + Artist

	if write:
		with open('list.txt', 'w') as f:
			for j in range (len(tracks)):
				f.write(tracks[j] + "\n")
		f.close()

def gettracks(tracks):
	## takes raw json data from Discogs.com and creates a tracklist for the album
	## This will return a list
	msg("Starting gettracks", 3)
	goodtracks = []

	for track in tracks:
		goodtracks.append(track['title'].replace(',',''))

	return goodtracks

def randomizer(length=8):
	#Creates a Random directory name
	randoms = string.ascii_letters + string.digits
	return ''.join((random.choice(randoms) for i in range(length)))

def buildfolders(artist, album=""):
	## Takes raw json data and creates foldes in parent_directory for Artist/Album
	msg("buildfolders started", 3)
	if len(ALBUM) == 0:
		album="UNKNOWN ALBUM"
	else:
		album = ALBUM.replace("'","")
#	msg("buildfolders local album is set to: " + album, 3)
	global DESTFOLDER
	DESTFOLDER = artist + "/" + album + "/"
	home = os.path.expanduser('~')
	if TESTFOLDER:
		DESTFOLDER = os.path.join(home, randomizer(), DESTFOLDER)
	DESTFOLDER = os.path.join(home, STORAGEPATH, DESTFOLDER)
	try:
		os.makedirs(DESTFOLDER)
		msg("Folder " + DESTFOLDER + " created", 2)
	except Exception as e:
		msg("Could not create destination folder!", 1)
		msg(e, 1)

def readlist(file):
	msg("Starting readlist", 3)
	##Open list.txt, read into music[]
	if not os.path.exists(MUSICFILE):
		msg("List.txt file not found.  Exiting", 1)
		sys.exit()
	songnum = 0
	with open(file) as f:
	  for line in f:
	    song={}
	    (key, val) = line.split(", ")
	    songnum += 1
	    song['songnum'] = songnum
	    song['Title'] = key
	    song['Artist'] = val.rstrip()
	    music.append(song)
	f.close()
	return music

def searchlinks(links, artist):
	## Takes a list of dictionaries and parses the results
	## Discards bad choices
	## Returns a dictionary of one entry (best result)
	## Good results include published by artist,
	## bad results include words live "live" or "Video"
	msg("Starting searchlinks", 3)
	list_badterms = ["live", "video", "sexy"]

	### FIX RANKINGS! ##

	for link in links:
		rating = 0
		for term in list_badterms:
			if term.lower() in link['title'].lower():
#				print("Contains Term!")
				rating -= 1
		if artist != "":
			if artist.lower() == link['publisher'].lower():
#				print("Published by Artist!")
				rating += 10
		link["rating"] = rating

	links.sort(reverse=True, key = lambda i: i['rating'])	## Sort links based on rating
	msg("Ending searchlinks", 3)
	return links[0]

def generatelink(searchterm, max_results=10, tries=7):
	## This will retry the link generation routine up to *tries* times and return results
	msg("Starting generatelink for " + searchterm, 3)
	counter = 0
	while counter <= tries:
	  try:
	    ytresult = YoutubeSearch(searchterm, max_results).to_dict()
	    if len(ytresult) > 0:
	      msg("Link Generated!", 3)
	      break
	    else:
	      raise IndexError("Index Empty")
	  except:
	    msg("Unable to generate link on try " + str(counter), 3)
	    counter += 1
	    if counter >= tries:
	      msg("Could Not Generate link for " + searchterm, 2)
	      raise IndexError("Could not Generate Link")
#	  finally:
#	    msg("Ending generatelink on try " + str(counter), 3)

#	searchlinks(ytresult)
	return ytresult

def parselist(musiclist):
	msg("Starting parselist", 3)
	global ITERATOR
	if ITERATOR == 0 and DOWNLOAD: ## <- Original Line
#	if ITERATOR == 0: ## <- Used only for testing buildfolders
		buildfolders(musiclist[0]['Artist'])
	ITERATOR+=1
	for song in musiclist:
#	  searchterm = song['Title'] + " " + song['Artist'] + ' lyrics HD'
	  searchterm = song['Title'] + " " + song['Artist']
	  dictlink={}

	  try:
	    ytresult = generatelink(searchterm)
	    bestlink = searchlinks(ytresult, song['Artist'])
#	    ytresult = generatelink(searchterm)
#	    link = 'https://youtube.com' + ytresult[0]['link']
	    link = 'https://youtube.com' + bestlink['link']
	    logresults.append(song['Title'] + ", " + song['Artist'] + " Link Created")
	    if DOWNLOAD:
              msg("Attempting to download " + song['Title'], 2)
              downloadsong(link, song)
	    else:
	      print("Not downloading " + song['Title'] + ".  Change this in config.ini")
	  except Exception as ex:
	    print(song['Title'], ex)

#	  searchlinks(ytresult, song['Artist'])

	if DOWNLOAD:
		cleanup(MUSICFILE)

def downloadsong(link, song):
       msg("Starling Downloadsong for " + song['Title'], 3)
       msg("Downloadsong DESTFOLDER: " + DESTFOLDER, 3)
       try:
         if TOR:
           os.system("torsocks youtube-dl --extract-audio --audio-format best --audio-quality 0 --output ''" + DESTFOLDER + "%(title)s.%(ext)s' --ignore-errors " + link)
         else:
           os.system("youtube-dl --extract-audio --audio-format best --audio-quality 0 --output '''" + DESTFOLDER + "%(title)s.%(ext)s' --ignore-errors " + link)
         completed.append(song['songnum'])
         logresults.append(song['Title'] + ", " + song['Artist'] + " Audio downloaded")
         msg(song['Title'] + " Download Complete!", 2)
       except e as youtubedlexception:
         logresults.append(song['Title'] + ", " + song['Artist'] + " FAILED TO DOWNLOAD SONG (youtube-dl)")
         print(youtubedlexception)

def singlesong(link):
	try:
		os.system("youtube-dl --extract-audio --audio-format best --audio-quality 0 --output '%(title)s.%(ext)s' --ignore-errors " + link)
	except Exception as e:
		msg("Could not download file. " + e, 1)
	sys.exit()

def cleanup(file):
	print("Cleaning completed files from list")
	print("Completed Downloads:", completed)

	linenum=0
	count=0
	with open(file, "r") as f:
	  lines = f.readlines()
	with open(file, "w") as f:
	  for line in lines:
	    linenum += 1
	    if linenum not in completed:
	     f.write(line)
	     count += 1

	f.close()

	if count >=1:
		print(count, "TRACKS REMAIN")
		print(RETRIES - ITERATOR, "tries remaining")
		if ITERATOR <= RETRIES:
			print("Retrying")
			global music
			music = []
			parselist(readlist(MUSICFILE))
	else:
		msg("All downloads complete!", -1)

if __name__ == "__main__":

	arguments(sys.argv[1:])

	if DISCOG != "":
		msg("DISCOG found, fetch json", 3)
		buildlist(fetchjson(DISCOG), MASTER)
	readlist(MUSICFILE)
	parselist(music)

	print("ytsearch complete, exiting")