QYTTS.sh 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395
  1. #!/bin/bash
  2. #
  3. # #####################################################
  4. # QsYouTubeTrailerScraper - QYTTS - v0.25
  5. # #####################################################
  6. #
  7. # This script will scrape your Kodi library for movie titles, then search
  8. # YouTube for it's trailer. It'll download the trailers to a specified
  9. # directory, which is used by the Cinema Experience script for Kodi to
  10. # load trailers. This way you can see the trailers for films you own,
  11. # withought having to dea with all the buffering that comes from streaming.
  12. #
  13. # I know there are other tools out there to get trailers for your library,
  14. # but most were windows based. The ones that weren't would just grab trailers
  15. # for any film, whether you have it or not.
  16. #
  17. # This was designed for my original Raspberry Pi Model B, running OSMC.
  18. # As such, it'll probably run on any Linux Kodi setup.
  19. #
  20. #
  21. # Please see the included README.md file for more information
  22. #
  23. #
  24. #
  25. # Copyright © 2015 Category <categoryNOSPAM@quintendo.uk>
  26. # This work is free. You can redistribute it and/or modify it under the
  27. # terms of the Do What The Fuck You Want To Public License, Version 2,
  28. # as published by Sam Hocevar. See the COPYING file for more details.
  29. #################
  30. # #
  31. # CONFIGURATION #
  32. # #
  33. #################
  34. ###
  35. # REQUIRED
  36. ###
  37. # Update the below before using, if this default doesn't match your storage plans
  38. ts_trailerdir="$HOME/Trailers/"
  39. # Location to save trailers, with trailing slash
  40. ###
  41. # OPTIONAL
  42. ###
  43. ts_databasedir="$HOME/.kodi/userdata/Database/"
  44. # Location of your Kodi database directory, normally "$HOME/.kodi/userdata/Database/"
  45. ts_tempdir="/tmp/"
  46. # Location to store temporary woking files, with trailing slash
  47. ts_maxsize=52428800
  48. # Maximum filesize of Trailers in bytes (to prevent full films being saved as trailers)
  49. # Default 50MB is 52428800
  50. ts_fix_dependencies=false
  51. # Option to fix dependencies if they are missing - script may ask for sudo password,
  52. # and install files to /usr/bin, or via apt-get
  53. # Default is false, for safety
  54. #############
  55. ### ###
  56. # # FUNCTIONS # #
  57. ### ###
  58. #############
  59. function trailer_dl {
  60. # Clear temp files/variable
  61. cat /dev/null > $ts_tempdir"YTSearchList"
  62. cat /dev/null > $ts_tempdir"TopResult"
  63. cat /dev/null > $ts_tempdir"YTDLErrors"
  64. YTCode=""
  65. # Prepare search target
  66. # Convert passed film name to the format "Film+Name+trailer", to pipe into the YouTube search URL
  67. YTTarget=${1// /+}+"trailer"
  68. # Prepare new filename
  69. YTNewName=$1" [Trailer].mp4"
  70. # Check if local trailer already exists
  71. # -e returns positive if file of the new name already exists
  72. if [ -e $ts_trailerdir"$YTNewName" ]
  73. then
  74. # File exists
  75. echo Trailer for $1 already exists, skipping download
  76. return
  77. else
  78. # No file found
  79. echo Searching YouTube for $1 trailer
  80. # Scrape top result from YouTube search
  81. # The process is as follows:-
  82. # 1. curl downloads html of the results page, after searching for the "Film+Name+trailer"
  83. # 2. grep returns any section containing 'videoId":' and the following code. -o to return just the search term, and not the entire line as the current YouTube results page is quite obfuscated and very few lines, all exceedingly long
  84. # 3. sed strips away the cruft, leaving just the YouTube videoId string
  85. # 4. List of all codes is piped to the "YTSearchList" temp file
  86. # 5. head saves the first result to "TopResult" temp file - the first videoId in a YouTube results page is always the top search result
  87. curl -s https://www.youtube.com/results\?search_query\=$YTTarget | grep -o videoId\"\:............ | sed 's/videoId\"\:\"//' > $ts_tempdir"YTSearchList"
  88. head -n 1 $ts_tempdir"YTSearchList" > $ts_tempdir"TopResult"
  89. # Extract YouTube ID from TopResult
  90. YTCode=$(cat $ts_tempdir"TopResult")
  91. echo YouTube ID: $YTCode
  92. # Check if first result is a trailer
  93. # youtube-dl with -e returns the title of the video store as YTTitle
  94. YTTitle=$(youtube-dl -e "https://www.youtube.com/watch?v=$YTCode" 2> $ts_tempdir"YTDLErrors")
  95. # Skip film if error encountered extracting title
  96. if [ $(cat $ts_tempdir"YTDLErrors" | grep ERROR | wc -l) -gt 0 ]
  97. then
  98. echo youtube-dl encountered an error, skipping film
  99. LogFailure "[youtube-dl error] |"
  100. return
  101. fi
  102. # If the title doesn't include the word "trailer" it is disregarded
  103. if [ $(echo $YTTitle | grep -i trailer | wc -l) = 1 ]
  104. then
  105. echo First result is a trailer, proceeding
  106. echo Downloading trailer for $1
  107. # Breakdown of the download request:-
  108. # -q - Quiet to suppress
  109. # --id - Use the videoId as filename
  110. # -f best - Save highest quality as .mp4 file
  111. # Uses the full url in quotes, to hopefully avoid issues where videoId includes punctuation
  112. # 2> to pipe all error output into "YTDLErrors" temp file
  113. youtube-dl -q --id -f best "https://www.youtube.com/watch?v=$YTCode" 2> $ts_tempdir"YTDLErrors"
  114. # Check to see if any of the error log contains the "ERROR" keyword - if so consider it a failure and move on
  115. if [ $(cat $ts_tempdir"YTDLErrors" | grep ERROR | wc -l) -gt 0 ]
  116. then
  117. echo youtube-dl encountered an error, skipping film
  118. LogFailure "[youtube-dl error] |"
  119. return
  120. else
  121. # Store downloaded filename based on videoId
  122. YTFileName=$YTCode.mp4
  123. # Store filesize of trailer
  124. YTFileSize=$(wc -c <"$YTFileName")
  125. # Check filesize against user-set max size
  126. # If too large, delete
  127. # If not, move to the trailer directory in the form "Film Name [Trailer].mp4"
  128. if [ $YTFileSize -gt $ts_maxsize ]
  129. then
  130. echo File too large, removing
  131. rm "./$YTFileName"
  132. LogFailure "[filesize too large] |"
  133. return
  134. else
  135. mv "./$YTFileName" $ts_trailerdir"$YTNewName"
  136. echo Saved as $YTNewName
  137. fi
  138. fi
  139. else
  140. echo Top YouTube result is not a trailer, skipping download
  141. LogFailure "[No trailer found] |"
  142. return
  143. fi
  144. fi
  145. }
  146. function LogFailure {
  147. # Count how many downloads failed
  148. let FailedDL=$FailedDL+1
  149. # Store failure reason & film name to temp file
  150. echo "$1 $ScanFilm" >> $ts_tempdir"FailedFilms"
  151. }
  152. # ################ #
  153. # ### ### #
  154. ###### # SCRIPT START # ######
  155. # ### ### #
  156. # ################ #
  157. echo QsYouTubeTrailerScraper - QYTTS
  158. echo v0.25- 2020-11-15
  159. echo
  160. #########################
  161. # #
  162. # Test for dependencies #
  163. # #
  164. #########################
  165. if [ $(which youtube-dl) ]
  166. then
  167. echo youtube-dl found
  168. if [ $(which sqlite3) ]
  169. then
  170. echo SQLite3 found
  171. echo Dependencies met, continuing...
  172. else
  173. if [ "$ts_fix_dependencies" = true ]
  174. then
  175. echo Attempting to install SQLite3
  176. echo sudo password may be requested
  177. sudo apt-get update -y
  178. sudo apt-get install sqlite3 -y
  179. if [ $? = 0 ]
  180. then
  181. echo Successfully installed SQLite3
  182. else
  183. echo Something has gone wrong, please install SQLite3 manually \(see README\)
  184. exit
  185. fi
  186. else
  187. echo SQLite3 not installed, please see README
  188. exit
  189. fi
  190. fi
  191. else
  192. if [ "$ts_fix_dependencies" = true ]
  193. then
  194. echo Attempting to install youtube-dl
  195. echo sudo password may be requested
  196. sudo curl https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl ; sudo chmod a+rx /usr/local/bin/youtube-dl
  197. if [ $? = 0 ]
  198. then
  199. echo Successfully installed youtube-dl
  200. else
  201. echo Something has gone wrong, please install youtube-dl manually \(see README\)
  202. exit
  203. fi
  204. else
  205. echo youtube-dl not installed, please see README
  206. exit
  207. fi
  208. fi
  209. ##############
  210. # #
  211. # Initialize #
  212. # #
  213. ##############
  214. echo Initializing...
  215. # Create clean temp files
  216. cat /dev/null > $ts_tempdir"CurrentMovie"
  217. cat /dev/null > $ts_tempdir"MovieList"
  218. cat /dev/null > $ts_tempdir"YTSearchList"
  219. cat /dev/null > $ts_tempdir"TopResult"
  220. cat /dev/null > $ts_tempdir"FailedFilms"
  221. cat /dev/null > $ts_tempdir"YTDLErrors"
  222. FailedDL=0
  223. # Log start directory, then move to temp folder ready to download
  224. StartDir=$(pwd)
  225. cd $ts_tempdir
  226. # Create trailer directory if it doesn't exist
  227. if [ ! -e $ts_trailerdir ]
  228. then
  229. echo "Trailer directory doesn't exist, creating it now"
  230. mkdir $ts_trailerdir
  231. fi
  232. ########################
  233. # #
  234. # Select Database File #
  235. # #
  236. ########################
  237. echo Finding curent Kodi database
  238. # All Kodi video databases come in the format "MyVideosXXX.db", where the XXX is any seemingly
  239. # random number that increases as Kodi is updated. To find the latest, this line performs:-
  240. # 1. List all files in databse directory
  241. # 2. Limit to "MyVideos" files
  242. # 3. Show only the trailing numbers from these files
  243. # 4. Sort in decreasing numerical value
  244. # 5. Store the highest number as DBNumber, to be the base for the variable DBFile
  245. DBNumber=$(ls $ts_databasedir | grep MyVideos | grep -o '[0-9]\{1,\}' | sort -gr | head -n 1)
  246. DBFile=MyVideos"$DBNumber".db
  247. echo Using database file $DBFile
  248. #####################
  249. # #
  250. # Extract MovieList #
  251. # #
  252. #####################
  253. echo Extracting MovieList from Kodi database
  254. sqlite3 $ts_databasedir$DBFile "select c00 from movie order by c00;" >> $ts_tempdir"MovieList"
  255. MovieCount=$(wc -l < $ts_tempdir"MovieList")
  256. echo $MovieCount movies found
  257. #############
  258. ### ###
  259. # # MAIN LOOP # #
  260. ### ###
  261. #############
  262. # Loop through the Movie list
  263. COUNTER=0
  264. while [ $COUNTER -lt $MovieCount ]; do
  265. # Cut a single film out of the list based on the Counter
  266. let COUNTER=$COUNTER+1
  267. head -n$COUNTER $ts_tempdir"MovieList" > $ts_tempdir"CurrentMovie"
  268. ScanFilm=$(tail -n1 $ts_tempdir"CurrentMovie")
  269. # Pass movie name to the download handler
  270. trailer_dl "$ScanFilm"
  271. done
  272. echo All Movies checked.
  273. ###################
  274. # #
  275. # Report failures #
  276. # #
  277. ###################
  278. # Print list of failed films, so user can find get them manually
  279. if [ $FailedDL -gt 0 ]
  280. then
  281. echo Failed to find trailers for the following $FailedDL movies...
  282. echo -------------
  283. cat $ts_tempdir"FailedFilms"
  284. echo -------------
  285. # Save a copy of all failed films in a log file
  286. CurrDate=$(date)
  287. echo "QYTTS failure log for "$CurrDate > $StartDir"/FailedTrailers.log"
  288. cat $ts_tempdir"FailedFilms" >> $StartDir"/FailedTrailers.log"
  289. echo Log of failures saved as "FailedTrailers.log"
  290. else
  291. echo All missing trailers downloaded successfully
  292. fi
  293. ############
  294. # #
  295. # CLEANING #
  296. # #
  297. ############
  298. echo Cleaning up...
  299. # Remove temp files
  300. rm $ts_tempdir"CurrentMovie"
  301. rm $ts_tempdir"MovieList"
  302. rm $ts_tempdir"YTSearchList"
  303. rm $ts_tempdir"TopResult"
  304. rm $ts_tempdir"FailedFilms"
  305. rm $ts_tempdir"YTDLErrors"
  306. # Check for mp4 files in temp directory and remove if found
  307. # (sometimes left behind from youtube-dl issues)
  308. if [ $(ls $ts_tempdir | grep .mp4 | wc -l) -gt 0 ]
  309. then
  310. rm $ts_tempdir*.mp4
  311. fi
  312. # Return to initial directory
  313. cd $StartDir
  314. ########
  315. # Exit #
  316. ########
  317. echo Thanks for using QYTTS
  318. exit