diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7fd074756..4b9b8a4e4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 11.8.1 +current_version = 12.0.0 commit = True tag = False diff --git a/.gitignore b/.gitignore index c0ea25fc8..a8fa8e37f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ /userscripts/ /logs/ /.idea/ +*.dist-info +*.egg-info diff --git a/README.md b/README.md index 89642047e..418bea206 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -nzbToMedia v11.8.1 +nzbToMedia v12.0.0 ================== Provides an [efficient](https://github.com/clinton-hall/nzbToMedia/wiki/Efficient-on-demand-post-processing) way to handle postprocessing for [CouchPotatoServer](https://couchpota.to/ "CouchPotatoServer") and [SickBeard](http://sickbeard.com/ "SickBeard") (and its [forks](https://github.com/clinton-hall/nzbToMedia/wiki/Failed-Download-Handling-%28FDH%29#sick-beard-and-its-forks)) @@ -50,9 +50,11 @@ Sorry for any inconvenience caused here. ### General -1. Install python 2.7. +1. Install Python -2. Clone or copy all files into a directory wherever you want to keep them (eg. /scripts/ in the home directory of your download client) +1. Install `pywin32` + +1. Clone or copy all files into a directory wherever you want to keep them (eg. /scripts/ in the home directory of your download client) and change the permission accordingly so the download client can access these files. `git clone git://github.com/clinton-hall/nzbToMedia.git` diff --git a/TorrentToMedia.py b/TorrentToMedia.py index e6dfbcad9..821e60ab6 100755 --- a/TorrentToMedia.py +++ b/TorrentToMedia.py @@ -4,178 +4,180 @@ import cleanup cleanup.clean('core', 'libs') - import datetime import os import sys -import core -from libs.six import text_type -from core import logger, nzbToMediaDB -from core.nzbToMediaUtil import convert_to_ascii, CharReplace, plex_update, replace_links -from core.nzbToMediaUserScript import external_script +import core +from core import logger, main_db +from core.auto_process import comics, games, movies, music, tv +from core.auto_process.common import ProcessResult +from core.user_scripts import external_script +from core.utils import char_replace, convert_to_ascii, plex_update, replace_links +from six import text_type -def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, clientAgent): +def process_torrent(input_directory, input_name, input_category, input_hash, input_id, client_agent): status = 1 # 1 = failed | 0 = success root = 0 - foundFile = 0 + found_file = 0 - if clientAgent != 'manual' and not core.DOWNLOADINFO: - logger.debug('Adding TORRENT download info for directory {0} to database'.format(inputDirectory)) + if client_agent != 'manual' and not core.DOWNLOADINFO: + logger.debug('Adding TORRENT download info for directory {0} to database'.format(input_directory)) - myDB = nzbToMediaDB.DBConnection() + my_db = main_db.DBConnection() - inputDirectory1 = inputDirectory - inputName1 = inputName + input_directory1 = input_directory + input_name1 = input_name try: - encoded, inputDirectory1 = CharReplace(inputDirectory) - encoded, inputName1 = CharReplace(inputName) - except: + encoded, input_directory1 = char_replace(input_directory) + encoded, input_name1 = char_replace(input_name) + except Exception: pass - controlValueDict = {"input_directory": text_type(inputDirectory1)} - newValueDict = {"input_name": text_type(inputName1), - "input_hash": text_type(inputHash), - "input_id": text_type(inputID), - "client_agent": text_type(clientAgent), - "status": 0, - "last_update": datetime.date.today().toordinal() - } - myDB.upsert("downloads", newValueDict, controlValueDict) + control_value_dict = {'input_directory': text_type(input_directory1)} + new_value_dict = { + 'input_name': text_type(input_name1), + 'input_hash': text_type(input_hash), + 'input_id': text_type(input_id), + 'client_agent': text_type(client_agent), + 'status': 0, + 'last_update': datetime.date.today().toordinal(), + } + my_db.upsert('downloads', new_value_dict, control_value_dict) - logger.debug("Received Directory: {0} | Name: {1} | Category: {2}".format(inputDirectory, inputName, inputCategory)) + logger.debug('Received Directory: {0} | Name: {1} | Category: {2}'.format(input_directory, input_name, input_category)) # Confirm the category by parsing directory structure - inputDirectory, inputName, inputCategory, root = core.category_search(inputDirectory, inputName, inputCategory, - root, core.CATEGORIES) - if inputCategory == "": - inputCategory = "UNCAT" + input_directory, input_name, input_category, root = core.category_search(input_directory, input_name, input_category, + root, core.CATEGORIES) + if input_category == '': + input_category = 'UNCAT' - usercat = inputCategory + usercat = input_category try: - inputName = inputName.encode(core.SYS_ENCODING) + input_name = input_name.encode(core.SYS_ENCODING) except UnicodeError: pass try: - inputDirectory = inputDirectory.encode(core.SYS_ENCODING) + input_directory = input_directory.encode(core.SYS_ENCODING) except UnicodeError: pass - logger.debug("Determined Directory: {0} | Name: {1} | Category: {2}".format - (inputDirectory, inputName, inputCategory)) + logger.debug('Determined Directory: {0} | Name: {1} | Category: {2}'.format + (input_directory, input_name, input_category)) # auto-detect section - section = core.CFG.findsection(inputCategory).isenabled() + section = core.CFG.findsection(input_category).isenabled() if section is None: - section = core.CFG.findsection("ALL").isenabled() + section = core.CFG.findsection('ALL').isenabled() if section is None: logger.error('Category:[{0}] is not defined or is not enabled. ' 'Please rename it or ensure it is enabled for the appropriate section ' 'in your autoProcessMedia.cfg and try again.'.format - (inputCategory)) - return [-1, ""] + (input_category)) + return [-1, ''] else: - usercat = "ALL" + usercat = 'ALL' if len(section) > 1: logger.error('Category:[{0}] is not unique, {1} are using it. ' 'Please rename it or disable all other sections using the same category name ' 'in your autoProcessMedia.cfg and try again.'.format (usercat, section.keys())) - return [-1, ""] + return [-1, ''] if section: - sectionName = section.keys()[0] - logger.info('Auto-detected SECTION:{0}'.format(sectionName)) + section_name = section.keys()[0] + logger.info('Auto-detected SECTION:{0}'.format(section_name)) else: - logger.error("Unable to locate a section with subsection:{0} " - "enabled in your autoProcessMedia.cfg, exiting!".format - (inputCategory)) - return [-1, ""] + logger.error('Unable to locate a section with subsection:{0} ' + 'enabled in your autoProcessMedia.cfg, exiting!'.format + (input_category)) + return [-1, ''] - section = dict(section[sectionName][usercat]) # Type cast to dict() to allow effective usage of .get() + section = dict(section[section_name][usercat]) # Type cast to dict() to allow effective usage of .get() - Torrent_NoLink = int(section.get("Torrent_NoLink", 0)) - keep_archive = int(section.get("keep_archive", 0)) + torrent_no_link = int(section.get('Torrent_NoLink', 0)) + keep_archive = int(section.get('keep_archive', 0)) extract = int(section.get('extract', 0)) - extensions = section.get('user_script_mediaExtensions', "").lower().split(',') - uniquePath = int(section.get("unique_path", 1)) + extensions = section.get('user_script_mediaExtensions', '').lower().split(',') + unique_path = int(section.get('unique_path', 1)) - if clientAgent != 'manual': - core.pause_torrent(clientAgent, inputHash, inputID, inputName) + if client_agent != 'manual': + core.pause_torrent(client_agent, input_hash, input_id, input_name) # In case input is not directory, make sure to create one. # This way Processing is isolated. - if not os.path.isdir(os.path.join(inputDirectory, inputName)): - basename = os.path.basename(inputDirectory) - basename = core.sanitizeName(inputName) \ - if inputName == basename else os.path.splitext(core.sanitizeName(inputName))[0] - outputDestination = os.path.join(core.OUTPUTDIRECTORY, inputCategory, basename) - elif uniquePath: - outputDestination = os.path.normpath( - core.os.path.join(core.OUTPUTDIRECTORY, inputCategory, core.sanitizeName(inputName).replace(" ","."))) + if not os.path.isdir(os.path.join(input_directory, input_name)): + basename = os.path.basename(input_directory) + basename = core.sanitize_name(input_name) \ + if input_name == basename else os.path.splitext(core.sanitize_name(input_name))[0] + output_destination = os.path.join(core.OUTPUTDIRECTORY, input_category, basename) + elif unique_path: + output_destination = os.path.normpath( + core.os.path.join(core.OUTPUTDIRECTORY, input_category, core.sanitize_name(input_name).replace(' ', '.'))) else: - outputDestination = os.path.normpath( - core.os.path.join(core.OUTPUTDIRECTORY, inputCategory)) + output_destination = os.path.normpath( + core.os.path.join(core.OUTPUTDIRECTORY, input_category)) try: - outputDestination = outputDestination.encode(core.SYS_ENCODING) + output_destination = output_destination.encode(core.SYS_ENCODING) except UnicodeError: pass - if outputDestination in inputDirectory: - outputDestination = inputDirectory + if output_destination in input_directory: + output_destination = input_directory - logger.info("Output directory set to: {0}".format(outputDestination)) + logger.info('Output directory set to: {0}'.format(output_destination)) - if core.SAFE_MODE and outputDestination == core.TORRENT_DEFAULTDIR: + if core.SAFE_MODE and output_destination == core.TORRENT_DEFAULTDIR: logger.error('The output directory:[{0}] is the Download Directory. ' 'Edit outputDirectory in autoProcessMedia.cfg. Exiting'.format - (inputDirectory)) - return [-1, ""] + (input_directory)) + return [-1, ''] - logger.debug("Scanning files in directory: {0}".format(inputDirectory)) + logger.debug('Scanning files in directory: {0}'.format(input_directory)) - if sectionName in ['HeadPhones', 'Lidarr']: + if section_name in ['HeadPhones', 'Lidarr']: core.NOFLATTEN.extend( - inputCategory) # Make sure we preserve folder structure for HeadPhones. + input_category) # Make sure we preserve folder structure for HeadPhones. now = datetime.datetime.now() if extract == 1: - inputFiles = core.listMediaFiles(inputDirectory, archives=False, other=True, otherext=extensions) + input_files = core.list_media_files(input_directory, archives=False, other=True, otherext=extensions) else: - inputFiles = core.listMediaFiles(inputDirectory, other=True, otherext=extensions) - if len(inputFiles) == 0 and os.path.isfile(inputDirectory): - inputFiles = [inputDirectory] - logger.debug("Found 1 file to process: {0}".format(inputDirectory)) + input_files = core.list_media_files(input_directory, other=True, otherext=extensions) + if len(input_files) == 0 and os.path.isfile(input_directory): + input_files = [input_directory] + logger.debug('Found 1 file to process: {0}'.format(input_directory)) else: - logger.debug("Found {0} files in {1}".format(len(inputFiles), inputDirectory)) - for inputFile in inputFiles: - filePath = os.path.dirname(inputFile) - fileName, fileExt = os.path.splitext(os.path.basename(inputFile)) - fullFileName = os.path.basename(inputFile) - - targetFile = core.os.path.join(outputDestination, fullFileName) - if inputCategory in core.NOFLATTEN: - if not os.path.basename(filePath) in outputDestination: - targetFile = core.os.path.join( - core.os.path.join(outputDestination, os.path.basename(filePath)), fullFileName) - logger.debug("Setting outputDestination to {0} to preserve folder structure".format - (os.path.dirname(targetFile))) + logger.debug('Found {0} files in {1}'.format(len(input_files), input_directory)) + for inputFile in input_files: + file_path = os.path.dirname(inputFile) + file_name, file_ext = os.path.splitext(os.path.basename(inputFile)) + full_file_name = os.path.basename(inputFile) + + target_file = core.os.path.join(output_destination, full_file_name) + if input_category in core.NOFLATTEN: + if not os.path.basename(file_path) in output_destination: + target_file = core.os.path.join( + core.os.path.join(output_destination, os.path.basename(file_path)), full_file_name) + logger.debug('Setting outputDestination to {0} to preserve folder structure'.format + (os.path.dirname(target_file))) try: - targetFile = targetFile.encode(core.SYS_ENCODING) + target_file = target_file.encode(core.SYS_ENCODING) except UnicodeError: pass if root == 1: - if not foundFile: - logger.debug("Looking for {0} in: {1}".format(inputName, inputFile)) - if any([core.sanitizeName(inputName) in core.sanitizeName(inputFile), - core.sanitizeName(fileName) in core.sanitizeName(inputName)]): - foundFile = True - logger.debug("Found file {0} that matches Torrent Name {1}".format - (fullFileName, inputName)) + if not found_file: + logger.debug('Looking for {0} in: {1}'.format(input_name, inputFile)) + if any([core.sanitize_name(input_name) in core.sanitize_name(inputFile), + core.sanitize_name(file_name) in core.sanitize_name(input_name)]): + found_file = True + logger.debug('Found file {0} that matches Torrent Name {1}'.format + (full_file_name, input_name)) else: continue @@ -183,106 +185,103 @@ def processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, mtime_lapse = now - datetime.datetime.fromtimestamp(os.path.getmtime(inputFile)) ctime_lapse = now - datetime.datetime.fromtimestamp(os.path.getctime(inputFile)) - if not foundFile: - logger.debug("Looking for files with modified/created dates less than 5 minutes old.") + if not found_file: + logger.debug('Looking for files with modified/created dates less than 5 minutes old.') if (mtime_lapse < datetime.timedelta(minutes=5)) or (ctime_lapse < datetime.timedelta(minutes=5)): - foundFile = True - logger.debug("Found file {0} with date modified/created less than 5 minutes ago.".format - (fullFileName)) + found_file = True + logger.debug('Found file {0} with date modified/created less than 5 minutes ago.'.format + (full_file_name)) else: continue # This file has not been recently moved or created, skip it - if Torrent_NoLink == 0: + if torrent_no_link == 0: try: - core.copy_link(inputFile, targetFile, core.USELINK) - core.rmReadOnly(targetFile) - except: - logger.error("Failed to link: {0} to {1}".format(inputFile, targetFile)) + core.copy_link(inputFile, target_file, core.USELINK) + core.remove_read_only(target_file) + except Exception: + logger.error('Failed to link: {0} to {1}'.format(inputFile, target_file)) - inputName, outputDestination = convert_to_ascii(inputName, outputDestination) + input_name, output_destination = convert_to_ascii(input_name, output_destination) if extract == 1: - logger.debug('Checking for archives to extract in directory: {0}'.format(inputDirectory)) - core.extractFiles(inputDirectory, outputDestination, keep_archive) + logger.debug('Checking for archives to extract in directory: {0}'.format(input_directory)) + core.extract_files(input_directory, output_destination, keep_archive) - if inputCategory not in core.NOFLATTEN: + if input_category not in core.NOFLATTEN: # don't flatten hp in case multi cd albums, and we need to copy this back later. - core.flatten(outputDestination) + core.flatten(output_destination) # Now check if video files exist in destination: - if sectionName in ["SickBeard", "NzbDrone", "Sonarr", "CouchPotato", "Radarr"]: - numVideos = len( - core.listMediaFiles(outputDestination, media=True, audio=False, meta=False, archives=False)) - if numVideos > 0: - logger.info("Found {0} media files in {1}".format(numVideos, outputDestination)) + if section_name in ['SickBeard', 'NzbDrone', 'Sonarr', 'CouchPotato', 'Radarr']: + num_videos = len( + core.list_media_files(output_destination, media=True, audio=False, meta=False, archives=False)) + if num_videos > 0: + logger.info('Found {0} media files in {1}'.format(num_videos, output_destination)) status = 0 elif extract != 1: - logger.info("Found no media files in {0}. Sending to {1} to process".format(outputDestination, sectionName)) + logger.info('Found no media files in {0}. Sending to {1} to process'.format(output_destination, section_name)) status = 0 else: - logger.warning("Found no media files in {0}".format(outputDestination)) + logger.warning('Found no media files in {0}'.format(output_destination)) # Only these sections can handling failed downloads # so make sure everything else gets through without the check for failed - if sectionName not in ['CouchPotato', 'Radarr', 'SickBeard', 'NzbDrone', 'Sonarr']: + if section_name not in ['CouchPotato', 'Radarr', 'SickBeard', 'NzbDrone', 'Sonarr']: status = 0 - logger.info("Calling {0}:{1} to post-process:{2}".format(sectionName, usercat, inputName)) + logger.info('Calling {0}:{1} to post-process:{2}'.format(section_name, usercat, input_name)) if core.TORRENT_CHMOD_DIRECTORY: - core.rchmod(outputDestination, core.TORRENT_CHMOD_DIRECTORY) - - result = [0, ""] - if sectionName == 'UserScript': - result = external_script(outputDestination, inputName, inputCategory, section) - - elif sectionName in ['CouchPotato', 'Radarr']: - result = core.autoProcessMovie().process(sectionName, outputDestination, inputName, - status, clientAgent, inputHash, inputCategory) - elif sectionName in ['SickBeard', 'NzbDrone', 'Sonarr']: - if inputHash: - inputHash = inputHash.upper() - result = core.autoProcessTV().processEpisode(sectionName, outputDestination, inputName, - status, clientAgent, inputHash, inputCategory) - elif sectionName in ['HeadPhones', 'Lidarr']: - result = core.autoProcessMusic().process(sectionName, outputDestination, inputName, - status, clientAgent, inputCategory) - elif sectionName == 'Mylar': - result = core.autoProcessComics().processEpisode(sectionName, outputDestination, inputName, - status, clientAgent, inputCategory) - elif sectionName == 'Gamez': - result = core.autoProcessGames().process(sectionName, outputDestination, inputName, - status, clientAgent, inputCategory) - - plex_update(inputCategory) - - if result[0] != 0: + core.rchmod(output_destination, core.TORRENT_CHMOD_DIRECTORY) + + result = ProcessResult( + message='', + status_code=0, + ) + if section_name == 'UserScript': + result = external_script(output_destination, input_name, input_category, section) + elif section_name in ['CouchPotato', 'Radarr']: + result = movies.process(section_name, output_destination, input_name, status, client_agent, input_hash, input_category) + elif section_name in ['SickBeard', 'NzbDrone', 'Sonarr']: + if input_hash: + input_hash = input_hash.upper() + result = tv.process(section_name, output_destination, input_name, status, client_agent, input_hash, input_category) + elif section_name in ['HeadPhones', 'Lidarr']: + result = music.process(section_name, output_destination, input_name, status, client_agent, input_category) + elif section_name == 'Mylar': + result = comics.process(section_name, output_destination, input_name, status, client_agent, input_category) + elif section_name == 'Gamez': + result = games.process(section_name, output_destination, input_name, status, client_agent, input_category) + + plex_update(input_category) + + if result.status_code != 0: if not core.TORRENT_RESUME_ON_FAILURE: - logger.error("A problem was reported in the autoProcess* script. " - "Torrent won't resume seeding (settings)") - elif clientAgent != 'manual': - logger.error("A problem was reported in the autoProcess* script. " - "If torrent was paused we will resume seeding") - core.resume_torrent(clientAgent, inputHash, inputID, inputName) + logger.error('A problem was reported in the autoProcess* script. ' + 'Torrent won\'t resume seeding (settings)') + elif client_agent != 'manual': + logger.error('A problem was reported in the autoProcess* script. ' + 'If torrent was paused we will resume seeding') + core.resume_torrent(client_agent, input_hash, input_id, input_name) else: - if clientAgent != 'manual': + if client_agent != 'manual': # update download status in our DB - core.update_downloadInfoStatus(inputName, 1) + core.update_download_info_status(input_name, 1) # remove torrent if core.USELINK == 'move-sym' and not core.DELETE_ORIGINAL == 1: - logger.debug('Checking for sym-links to re-direct in: {0}'.format(inputDirectory)) - for dirpath, dirs, files in os.walk(inputDirectory): + logger.debug('Checking for sym-links to re-direct in: {0}'.format(input_directory)) + for dirpath, dirs, files in os.walk(input_directory): for file in files: logger.debug('Checking symlink: {0}'.format(os.path.join(dirpath, file))) replace_links(os.path.join(dirpath, file)) - core.remove_torrent(clientAgent, inputHash, inputID, inputName) + core.remove_torrent(client_agent, input_hash, input_id, input_name) - if not sectionName == 'UserScript': + if not section_name == 'UserScript': # for user script, we assume this is cleaned by the script or option USER_SCRIPT_CLEAN # cleanup our processing folders of any misc unwanted files and empty directories - core.cleanDir(outputDestination, sectionName, inputCategory) + core.clean_dir(output_destination, section_name, input_category) return result @@ -292,82 +291,85 @@ def main(args): core.initialize() # clientAgent for Torrents - clientAgent = core.TORRENT_CLIENTAGENT + client_agent = core.TORRENT_CLIENTAGENT - logger.info("#########################################################") - logger.info("## ..::[{0}]::.. ##".format(os.path.basename(__file__))) - logger.info("#########################################################") + logger.info('#########################################################') + logger.info('## ..::[{0}]::.. ##'.format(os.path.basename(__file__))) + logger.info('#########################################################') # debug command line options - logger.debug("Options passed into TorrentToMedia: {0}".format(args)) + logger.debug('Options passed into TorrentToMedia: {0}'.format(args)) # Post-Processing Result - result = [0, ""] + result = ProcessResult( + message='', + status_code=0, + ) try: - inputDirectory, inputName, inputCategory, inputHash, inputID = core.parse_args(clientAgent, args) - except: - logger.error("There was a problem loading variables") + input_directory, input_name, input_category, input_hash, input_id = core.parse_args(client_agent, args) + except Exception: + logger.error('There was a problem loading variables') return -1 - if inputDirectory and inputName and inputHash and inputID: - result = processTorrent(inputDirectory, inputName, inputCategory, inputHash, inputID, clientAgent) + if input_directory and input_name and input_hash and input_id: + result = process_torrent(input_directory, input_name, input_category, input_hash, input_id, client_agent) else: # Perform Manual Post-Processing - logger.warning("Invalid number of arguments received from client, Switching to manual run mode ...") + logger.warning('Invalid number of arguments received from client, Switching to manual run mode ...') for section, subsections in core.SECTIONS.items(): for subsection in subsections: if not core.CFG[section][subsection].isenabled(): continue - for dirName in core.getDirs(section, subsection, link='hard'): - logger.info("Starting manual run for {0}:{1} - Folder:{2}".format - (section, subsection, dirName)) + for dir_name in core.get_dirs(section, subsection, link='hard'): + logger.info('Starting manual run for {0}:{1} - Folder:{2}'.format + (section, subsection, dir_name)) - logger.info("Checking database for download info for {0} ...".format - (os.path.basename(dirName))) - core.DOWNLOADINFO = core.get_downloadInfo(os.path.basename(dirName), 0) + logger.info('Checking database for download info for {0} ...'.format + (os.path.basename(dir_name))) + core.DOWNLOADINFO = core.get_download_info(os.path.basename(dir_name), 0) if core.DOWNLOADINFO: - clientAgent = text_type(core.DOWNLOADINFO[0].get('client_agent', 'manual')) - inputHash = text_type(core.DOWNLOADINFO[0].get('input_hash', '')) - inputID = text_type(core.DOWNLOADINFO[0].get('input_id', '')) - logger.info("Found download info for {0}, " - "setting variables now ...".format(os.path.basename(dirName))) + client_agent = text_type(core.DOWNLOADINFO[0].get('client_agent', 'manual')) + input_hash = text_type(core.DOWNLOADINFO[0].get('input_hash', '')) + input_id = text_type(core.DOWNLOADINFO[0].get('input_id', '')) + logger.info('Found download info for {0}, ' + 'setting variables now ...'.format(os.path.basename(dir_name))) else: logger.info('Unable to locate download info for {0}, ' 'continuing to try and process this release ...'.format - (os.path.basename(dirName))) - clientAgent = 'manual' - inputHash = '' - inputID = '' + (os.path.basename(dir_name))) + client_agent = 'manual' + input_hash = '' + input_id = '' - if clientAgent.lower() not in core.TORRENT_CLIENTS: + if client_agent.lower() not in core.TORRENT_CLIENTS: continue try: - dirName = dirName.encode(core.SYS_ENCODING) + dir_name = dir_name.encode(core.SYS_ENCODING) except UnicodeError: pass - inputName = os.path.basename(dirName) + input_name = os.path.basename(dir_name) try: - inputName = inputName.encode(core.SYS_ENCODING) + input_name = input_name.encode(core.SYS_ENCODING) except UnicodeError: pass - results = processTorrent(dirName, inputName, subsection, inputHash or None, inputID or None, - clientAgent) + results = process_torrent(dir_name, input_name, subsection, input_hash or None, input_id or None, + client_agent) if results[0] != 0: - logger.error("A problem was reported when trying to perform a manual run for {0}:{1}.".format + logger.error('A problem was reported when trying to perform a manual run for {0}:{1}.'.format (section, subsection)) result = results - if result[0] == 0: - logger.info("The {0} script completed successfully.".format(args[0])) + if result.status_code == 0: + logger.info('The {0} script completed successfully.'.format(args[0])) else: - logger.error("A problem was reported in the {0} script.".format(args[0])) + logger.error('A problem was reported in the {0} script.'.format(args[0])) del core.MYAPP - return result[0] + return result.status_code -if __name__ == "__main__": +if __name__ == '__main__': exit(main(sys.argv)) diff --git a/autoProcessMedia.cfg.spec b/autoProcessMedia.cfg.spec index e04f2dc24..ef3c362b3 100644 --- a/autoProcessMedia.cfg.spec +++ b/autoProcessMedia.cfg.spec @@ -40,6 +40,11 @@ # Set the ionice scheduling class data. This defines the class data, if the class accepts an argument. For real time and best-effort, 0-7 is valid data. ionice_classdata = 0 +[Windows] + ### Set specific settings for Windows systems + # Set this to 1 to allow extraction (7zip) windows to be lunched visble (for debugging) otherwise 0 to have this run in background. + show_extraction = 0 + [CouchPotato] #### autoProcessing for Movies #### movie - category that gets called for post-processing with CPS diff --git a/changelog.txt b/changelog.txt index 7f611bac4..9ad64594f 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,39 @@ Change_LOG / History +V12.0.0 + +NOTE: +- This release contains major backwards-incompatible changes to the internal API +- Windows users will need to manually install pywin32 + +Add Python 3 support +Add cleanup script for post-update cleanup +Update all dependencies +Move vendored packages in `core` to `libs` +Move common libs to `libs/common` +Move custom libs to `libs/custom` +Move Python 2 libs to `libs/py2` +Move Windows libs to `libs/windows` +Fix PEP8 +Add feature to make libs importable +Add feature to auto-update libs +Add path parent option to module path and default to using local path +Update invisible.cmd to return errorlevel +Update invisible.vbs to return exit code of 7zip +Update extractor.py for correct return code +Added debugging to extractor +Add option for windows extraction debugging +Remove surplus debug +Fix handling of None Password file +Fix invisible windows extraction +Fix execution of extraction +Start vbs directly from extractor +Delete invisible.cmd +Use args instead of Wscript.Arguments +Fix postprocessing of failed / bad downloads (#1091) +Fix release is None +Fix UnRAR failing + V11.8.1 12/29/2018 Fix cleanup for nzbToMedia installed as a git submodule diff --git a/core/__init__.py b/core/__init__.py index c4cfdf4b0..9f24b22fa 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -5,51 +5,54 @@ import itertools import locale import os +import platform import re import subprocess import sys -import platform import time +import libs.autoload +import libs.util + +if not libs.autoload.completed: + sys.exit('Could not load vendored libraries.') -# init libs -PROGRAM_DIR = os.path.dirname(os.path.normpath(os.path.abspath(os.path.join(__file__, os.pardir)))) -LIBS_DIR = os.path.join(PROGRAM_DIR, 'libs') -sys.path.insert(0, LIBS_DIR) +try: + import win32event +except ImportError: + if sys.platform == 'win32': + sys.exit('Please install pywin32') + +APP_ROOT = libs.util.module_path(parent=True) +SOURCE_ROOT = libs.util.module_path() # init preliminaries SYS_ARGV = sys.argv[1:] APP_FILENAME = sys.argv[0] APP_NAME = os.path.basename(APP_FILENAME) -LOG_DIR = os.path.join(PROGRAM_DIR, 'logs') +LOG_DIR = os.path.join(APP_ROOT, 'logs') LOG_FILE = os.path.join(LOG_DIR, 'nzbtomedia.log') PID_FILE = os.path.join(LOG_DIR, 'nzbtomedia.pid') -CONFIG_FILE = os.path.join(PROGRAM_DIR, 'autoProcessMedia.cfg') -CONFIG_SPEC_FILE = os.path.join(PROGRAM_DIR, 'autoProcessMedia.cfg.spec') -CONFIG_MOVIE_FILE = os.path.join(PROGRAM_DIR, 'autoProcessMovie.cfg') -CONFIG_TV_FILE = os.path.join(PROGRAM_DIR, 'autoProcessTv.cfg') -TEST_FILE = os.path.join(os.path.join(PROGRAM_DIR, 'tests'), 'test.mp4') +CONFIG_FILE = os.path.join(APP_ROOT, 'autoProcessMedia.cfg') +CONFIG_SPEC_FILE = os.path.join(APP_ROOT, 'autoProcessMedia.cfg.spec') +CONFIG_MOVIE_FILE = os.path.join(APP_ROOT, 'autoProcessMovie.cfg') +CONFIG_TV_FILE = os.path.join(APP_ROOT, 'autoProcessTv.cfg') +TEST_FILE = os.path.join(APP_ROOT, 'tests', 'test.mp4') MYAPP = None +import six from six.moves import reload_module -from core.autoProcess.autoProcessComics import autoProcessComics -from core.autoProcess.autoProcessGames import autoProcessGames -from core.autoProcess.autoProcessMovie import autoProcessMovie -from core.autoProcess.autoProcessMusic import autoProcessMusic -from core.autoProcess.autoProcessTV import autoProcessTV -from core import logger, versionCheck, nzbToMediaDB -from core.nzbToMediaConfig import config -from core.nzbToMediaUtil import ( - category_search, sanitizeName, copy_link, parse_args, flatten, getDirs, - rmReadOnly, rmDir, pause_torrent, resume_torrent, remove_torrent, listMediaFiles, - extractFiles, cleanDir, update_downloadInfoStatus, get_downloadInfo, WakeUp, makeDir, cleanDir, - create_torrent_class, listMediaFiles, RunningProcess, - ) -from core.transcoder import transcoder -from core.databases import mainDB - -__version__ = '11.8.1' +from core import logger, main_db, version_check, databases, transcoder +from core.configuration import config +from core.utils import ( + RunningProcess, wake_up, category_search, clean_dir, clean_dir, copy_link, + create_torrent_class, extract_files, flatten, get_dirs, get_download_info, + list_media_files, make_dir, parse_args, pause_torrent, remove_torrent, + resume_torrent, remove_dir, remove_read_only, sanitize_name, update_download_info_status, +) + +__version__ = '12.0.0' # Client Agents NZB_CLIENTS = ['sabnzbd', 'nzbget', 'manual'] @@ -61,23 +64,25 @@ # sickbeard fork/branch constants FORKS = {} -FORK_DEFAULT = "default" -FORK_FAILED = "failed" -FORK_FAILED_TORRENT = "failed-torrent" -FORK_SICKRAGE = "SickRage" -FORK_SICKCHILL = "SickChill" -FORK_SICKBEARD_API = "SickBeard-api" -FORK_MEDUSA = "Medusa" -FORK_SICKGEAR = "SickGear" -FORKS[FORK_DEFAULT] = {"dir": None} -FORKS[FORK_FAILED] = {"dirName": None, "failed": None} -FORKS[FORK_FAILED_TORRENT] = {"dir": None, "failed": None, "process_method": None} -FORKS[FORK_SICKRAGE] = {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None} -FORKS[FORK_SICKCHILL] = {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "force_next": None} -FORKS[FORK_SICKBEARD_API] = {"path": None, "failed": None, "process_method": None, "force_replace": None, "return_data": None, "type": None, "delete": None, "force_next": None} -FORKS[FORK_MEDUSA] = {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs":None} -FORKS[FORK_SICKGEAR] = {"dir": None, "failed": None, "process_method": None, "force": None} -ALL_FORKS = {k:None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} +FORK_DEFAULT = 'default' +FORK_FAILED = 'failed' +FORK_FAILED_TORRENT = 'failed-torrent' +FORK_SICKRAGE = 'SickRage' +FORK_SICKCHILL = 'SickChill' +FORK_SICKBEARD_API = 'SickBeard-api' +FORK_MEDUSA = 'Medusa' +FORK_SICKGEAR = 'SickGear' +FORK_STHENO = 'Stheno' +FORKS[FORK_DEFAULT] = {'dir': None} +FORKS[FORK_FAILED] = {'dirName': None, 'failed': None} +FORKS[FORK_FAILED_TORRENT] = {'dir': None, 'failed': None, 'process_method': None} +FORKS[FORK_SICKRAGE] = {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None} +FORKS[FORK_SICKCHILL] = {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'force_next': None} +FORKS[FORK_SICKBEARD_API] = {'path': None, 'failed': None, 'process_method': None, 'force_replace': None, 'return_data': None, 'type': None, 'delete': None, 'force_next': None} +FORKS[FORK_MEDUSA] = {'proc_dir': None, 'failed': None, 'process_method': None, 'force': None, 'delete_on': None, 'ignore_subs': None} +FORKS[FORK_SICKGEAR] = {'dir': None, 'failed': None, 'process_method': None, 'force': None} +FORKS[FORK_STHENO] = {"proc_dir": None, "failed": None, "process_method": None, "force": None, "delete_on": None, "ignore_subs": None} +ALL_FORKS = {k: None for k in set(list(itertools.chain.from_iterable([FORKS[x].keys() for x in FORKS.keys()])))} # NZBGet Exit Codes NZBGET_POSTPROCESS_PARCHECK = 92 @@ -204,6 +209,7 @@ OUTPUTQUALITYPERCENT = None FFMPEG = None SEVENZIP = None +SHOWEXTRACT = 0 PAR2CMD = None FFPROBE = None CHECK_MEDIA = None @@ -227,7 +233,7 @@ def initialize(section=None): global NZBGET_POSTPROCESS_ERROR, NZBGET_POSTPROCESS_NONE, NZBGET_POSTPROCESS_PARCHECK, NZBGET_POSTPROCESS_SUCCESS, \ - NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, \ + NZBTOMEDIA_TIMEOUT, FORKS, FORK_DEFAULT, FORK_FAILED_TORRENT, FORK_FAILED, NOEXTRACTFAILED, SHOWEXTRACT, \ NZBTOMEDIA_BRANCH, NZBTOMEDIA_VERSION, NEWEST_VERSION, NEWEST_VERSION_STRING, VERSION_NOTIFY, SYS_ARGV, CFG, \ SABNZB_NO_OF_ARGUMENTS, SABNZB_0717_NO_OF_ARGUMENTS, CATEGORIES, TORRENT_CLIENTAGENT, USELINK, OUTPUTDIRECTORY, \ NOFLATTEN, UTORRENTPWD, UTORRENTUSR, UTORRENTWEBUI, DELUGEHOST, DELUGEPORT, DELUGEUSR, DELUGEPWD, VLEVEL, \ @@ -252,16 +258,16 @@ def initialize(section=None): LOG_FILE = os.environ['NTM_LOGFILE'] LOG_DIR = os.path.split(LOG_FILE)[0] - if not makeDir(LOG_DIR): - print("No log folder, logging to screen only") + if not make_dir(LOG_DIR): + print('No log folder, logging to screen only') MYAPP = RunningProcess() while MYAPP.alreadyrunning(): - print("Waiting for existing session to end") + print('Waiting for existing session to end') time.sleep(30) try: - locale.setlocale(locale.LC_ALL, "") + locale.setlocale(locale.LC_ALL, '') SYS_ENCODING = locale.getpreferredencoding() except (locale.Error, IOError): pass @@ -270,28 +276,29 @@ def initialize(section=None): if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): SYS_ENCODING = 'UTF-8' - if not hasattr(sys, "setdefaultencoding"): - reload_module(sys) + if six.PY2: + if not hasattr(sys, 'setdefaultencoding'): + reload_module(sys) - try: - # pylint: disable=E1101 - # On non-unicode builds this will raise an AttributeError, if encoding type is not valid it throws a LookupError - sys.setdefaultencoding(SYS_ENCODING) - except: - print('Sorry, you MUST add the nzbToMedia folder to the PYTHONPATH environment variable' - '\nor find another way to force Python to use {codec} for string encoding.'.format - (codec=SYS_ENCODING)) - if 'NZBOP_SCRIPTDIR' in os.environ: - sys.exit(NZBGET_POSTPROCESS_ERROR) - else: - sys.exit(1) + try: + # pylint: disable=E1101 + # On non-unicode builds this will raise an AttributeError, if encoding type is not valid it throws a LookupError + sys.setdefaultencoding(SYS_ENCODING) + except Exception: + print('Sorry, you MUST add the nzbToMedia folder to the PYTHONPATH environment variable' + '\nor find another way to force Python to use {codec} for string encoding.'.format + (codec=SYS_ENCODING)) + if 'NZBOP_SCRIPTDIR' in os.environ: + sys.exit(NZBGET_POSTPROCESS_ERROR) + else: + sys.exit(1) # init logging - logger.ntm_log_instance.initLogging() + logger.ntm_log_instance.init_logging() # run migrate to convert old cfg to new style cfg plus fix any cfg missing values/options. if not config.migrate(): - logger.error("Unable to migrate config file {0}, exiting ...".format(CONFIG_FILE)) + logger.error('Unable to migrate config file {0}, exiting ...'.format(CONFIG_FILE)) if 'NZBOP_SCRIPTDIR' in os.environ: pass # We will try and read config from Environment. else: @@ -302,7 +309,7 @@ def initialize(section=None): CFG = config.addnzbget() else: # load newly migrated config - logger.info("Loading config from [{0}]".format(CONFIG_FILE)) + logger.info('Loading config from [{0}]'.format(CONFIG_FILE)) CFG = config() # Enable/Disable DEBUG Logging @@ -313,10 +320,10 @@ def initialize(section=None): if LOG_ENV: for item in os.environ: - logger.info("{0}: {1}".format(item, os.environ[item]), "ENVIRONMENT") + logger.info('{0}: {1}'.format(item, os.environ[item]), 'ENVIRONMENT') # initialize the main SB database - nzbToMediaDB.upgradeDatabase(nzbToMediaDB.DBConnection(), mainDB.InitialSchema) + main_db.upgrade_database(main_db.DBConnection(), databases.InitialSchema) # Set Version and GIT variables NZBTOMEDIA_VERSION = '11.06' @@ -326,80 +333,80 @@ def initialize(section=None): GIT_PATH = CFG['General']['git_path'] GIT_USER = CFG['General']['git_user'] or 'clinton-hall' GIT_BRANCH = CFG['General']['git_branch'] or 'master' - FORCE_CLEAN = int(CFG["General"]["force_clean"]) - FFMPEG_PATH = CFG["General"]["ffmpeg_path"] - CHECK_MEDIA = int(CFG["General"]["check_media"]) - SAFE_MODE = int(CFG["General"]["safe_mode"]) - NOEXTRACTFAILED = int(CFG["General"]["no_extract_failed"]) + FORCE_CLEAN = int(CFG['General']['force_clean']) + FFMPEG_PATH = CFG['General']['ffmpeg_path'] + CHECK_MEDIA = int(CFG['General']['check_media']) + SAFE_MODE = int(CFG['General']['safe_mode']) + NOEXTRACTFAILED = int(CFG['General']['no_extract_failed']) # Check for updates via GitHUB - if versionCheck.CheckVersion().check_for_new_version(): + if version_check.CheckVersion().check_for_new_version(): if AUTO_UPDATE == 1: - logger.info("Auto-Updating nzbToMedia, Please wait ...") - updated = versionCheck.CheckVersion().update() + logger.info('Auto-Updating nzbToMedia, Please wait ...') + updated = version_check.CheckVersion().update() if updated: # restart nzbToMedia try: del MYAPP - except: + except Exception: pass restart() else: - logger.error("Update wasn't successful, not restarting. Check your log for more information.") + logger.error('Update wasn\'t successful, not restarting. Check your log for more information.') # Set Current Version logger.info('nzbToMedia Version:{version} Branch:{branch} ({system} {release})'.format (version=NZBTOMEDIA_VERSION, branch=GIT_BRANCH, system=platform.system(), release=platform.release())) - if int(CFG["WakeOnLan"]["wake"]) == 1: - WakeUp() + if int(CFG['WakeOnLan']['wake']) == 1: + wake_up() - NZB_CLIENTAGENT = CFG["Nzb"]["clientAgent"] # sabnzbd - SABNZBDHOST = CFG["Nzb"]["sabnzbd_host"] - SABNZBDPORT = int(CFG["Nzb"]["sabnzbd_port"] or 8080) # defaults to accomodate NzbGet - SABNZBDAPIKEY = CFG["Nzb"]["sabnzbd_apikey"] - NZB_DEFAULTDIR = CFG["Nzb"]["default_downloadDirectory"] - GROUPS = CFG["Custom"]["remove_group"] + NZB_CLIENTAGENT = CFG['Nzb']['clientAgent'] # sabnzbd + SABNZBDHOST = CFG['Nzb']['sabnzbd_host'] + SABNZBDPORT = int(CFG['Nzb']['sabnzbd_port'] or 8080) # defaults to accomodate NzbGet + SABNZBDAPIKEY = CFG['Nzb']['sabnzbd_apikey'] + NZB_DEFAULTDIR = CFG['Nzb']['default_downloadDirectory'] + GROUPS = CFG['Custom']['remove_group'] if isinstance(GROUPS, str): GROUPS = GROUPS.split(',') if GROUPS == ['']: GROUPS = None - TORRENT_CLIENTAGENT = CFG["Torrent"]["clientAgent"] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other - USELINK = CFG["Torrent"]["useLink"] # no | hard | sym - OUTPUTDIRECTORY = CFG["Torrent"]["outputDirectory"] # /abs/path/to/complete/ - TORRENT_DEFAULTDIR = CFG["Torrent"]["default_downloadDirectory"] - CATEGORIES = (CFG["Torrent"]["categories"]) # music,music_videos,pictures,software - NOFLATTEN = (CFG["Torrent"]["noFlatten"]) + TORRENT_CLIENTAGENT = CFG['Torrent']['clientAgent'] # utorrent | deluge | transmission | rtorrent | vuze | qbittorrent |other + USELINK = CFG['Torrent']['useLink'] # no | hard | sym + OUTPUTDIRECTORY = CFG['Torrent']['outputDirectory'] # /abs/path/to/complete/ + TORRENT_DEFAULTDIR = CFG['Torrent']['default_downloadDirectory'] + CATEGORIES = (CFG['Torrent']['categories']) # music,music_videos,pictures,software + NOFLATTEN = (CFG['Torrent']['noFlatten']) if isinstance(NOFLATTEN, str): NOFLATTEN = NOFLATTEN.split(',') if isinstance(CATEGORIES, str): CATEGORIES = CATEGORIES.split(',') - DELETE_ORIGINAL = int(CFG["Torrent"]["deleteOriginal"]) - TORRENT_CHMOD_DIRECTORY = int(str(CFG["Torrent"]["chmodDirectory"]), 8) - TORRENT_RESUME_ON_FAILURE = int(CFG["Torrent"]["resumeOnFailure"]) - TORRENT_RESUME = int(CFG["Torrent"]["resume"]) - UTORRENTWEBUI = CFG["Torrent"]["uTorrentWEBui"] # http://localhost:8090/gui/ - UTORRENTUSR = CFG["Torrent"]["uTorrentUSR"] # mysecretusr - UTORRENTPWD = CFG["Torrent"]["uTorrentPWD"] # mysecretpwr - - TRANSMISSIONHOST = CFG["Torrent"]["TransmissionHost"] # localhost - TRANSMISSIONPORT = int(CFG["Torrent"]["TransmissionPort"]) - TRANSMISSIONUSR = CFG["Torrent"]["TransmissionUSR"] # mysecretusr - TRANSMISSIONPWD = CFG["Torrent"]["TransmissionPWD"] # mysecretpwr - - DELUGEHOST = CFG["Torrent"]["DelugeHost"] # localhost - DELUGEPORT = int(CFG["Torrent"]["DelugePort"]) # 8084 - DELUGEUSR = CFG["Torrent"]["DelugeUSR"] # mysecretusr - DELUGEPWD = CFG["Torrent"]["DelugePWD"] # mysecretpwr - - QBITTORRENTHOST = CFG["Torrent"]["qBittorrenHost"] # localhost - QBITTORRENTPORT = int(CFG["Torrent"]["qBittorrentPort"]) # 8080 - QBITTORRENTUSR = CFG["Torrent"]["qBittorrentUSR"] # mysecretusr - QBITTORRENTPWD = CFG["Torrent"]["qBittorrentPWD"] # mysecretpwr - - REMOTEPATHS = CFG["Network"]["mount_points"] or [] + DELETE_ORIGINAL = int(CFG['Torrent']['deleteOriginal']) + TORRENT_CHMOD_DIRECTORY = int(str(CFG['Torrent']['chmodDirectory']), 8) + TORRENT_RESUME_ON_FAILURE = int(CFG['Torrent']['resumeOnFailure']) + TORRENT_RESUME = int(CFG['Torrent']['resume']) + UTORRENTWEBUI = CFG['Torrent']['uTorrentWEBui'] # http://localhost:8090/gui/ + UTORRENTUSR = CFG['Torrent']['uTorrentUSR'] # mysecretusr + UTORRENTPWD = CFG['Torrent']['uTorrentPWD'] # mysecretpwr + + TRANSMISSIONHOST = CFG['Torrent']['TransmissionHost'] # localhost + TRANSMISSIONPORT = int(CFG['Torrent']['TransmissionPort']) + TRANSMISSIONUSR = CFG['Torrent']['TransmissionUSR'] # mysecretusr + TRANSMISSIONPWD = CFG['Torrent']['TransmissionPWD'] # mysecretpwr + + DELUGEHOST = CFG['Torrent']['DelugeHost'] # localhost + DELUGEPORT = int(CFG['Torrent']['DelugePort']) # 8084 + DELUGEUSR = CFG['Torrent']['DelugeUSR'] # mysecretusr + DELUGEPWD = CFG['Torrent']['DelugePWD'] # mysecretpwr + + QBITTORRENTHOST = CFG['Torrent']['qBittorrenHost'] # localhost + QBITTORRENTPORT = int(CFG['Torrent']['qBittorrentPort']) # 8080 + QBITTORRENTUSR = CFG['Torrent']['qBittorrentUSR'] # mysecretusr + QBITTORRENTPWD = CFG['Torrent']['qBittorrentPWD'] # mysecretpwr + + REMOTEPATHS = CFG['Network']['mount_points'] or [] if REMOTEPATHS: if isinstance(REMOTEPATHS, list): REMOTEPATHS = ','.join(REMOTEPATHS) # fix in case this imported as list. @@ -408,11 +415,11 @@ def initialize(section=None): REMOTEPATHS = [(local.strip(), remote.strip()) for local, remote in REMOTEPATHS] # strip trailing and leading whitespaces - PLEXSSL = int(CFG["Plex"]["plex_ssl"]) - PLEXHOST = CFG["Plex"]["plex_host"] - PLEXPORT = CFG["Plex"]["plex_port"] - PLEXTOKEN = CFG["Plex"]["plex_token"] - PLEXSEC = CFG["Plex"]["plex_sections"] or [] + PLEXSSL = int(CFG['Plex']['plex_ssl']) + PLEXHOST = CFG['Plex']['plex_host'] + PLEXPORT = CFG['Plex']['plex_port'] + PLEXTOKEN = CFG['Plex']['plex_token'] + PLEXSEC = CFG['Plex']['plex_sections'] or [] if PLEXSEC: if isinstance(PLEXSEC, list): PLEXSEC = ','.join(PLEXSEC) # fix in case this imported as list. @@ -420,34 +427,34 @@ def initialize(section=None): devnull = open(os.devnull, 'w') try: - subprocess.Popen(["nice"], stdout=devnull, stderr=devnull).communicate() - NICENESS.extend(['nice', '-n{0}'.format(int(CFG["Posix"]["niceness"]))]) - except: + subprocess.Popen(['nice'], stdout=devnull, stderr=devnull).communicate() + NICENESS.extend(['nice', '-n{0}'.format(int(CFG['Posix']['niceness']))]) + except Exception: pass try: - subprocess.Popen(["ionice"], stdout=devnull, stderr=devnull).communicate() + subprocess.Popen(['ionice'], stdout=devnull, stderr=devnull).communicate() try: - NICENESS.extend(['ionice', '-c{0}'.format(int(CFG["Posix"]["ionice_class"]))]) - except: + NICENESS.extend(['ionice', '-c{0}'.format(int(CFG['Posix']['ionice_class']))]) + except Exception: pass try: if 'ionice' in NICENESS: - NICENESS.extend(['-n{0}'.format(int(CFG["Posix"]["ionice_classdata"]))]) + NICENESS.extend(['-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) else: - NICENESS.extend(['ionice', '-n{0}'.format(int(CFG["Posix"]["ionice_classdata"]))]) - except: + NICENESS.extend(['ionice', '-n{0}'.format(int(CFG['Posix']['ionice_classdata']))]) + except Exception: pass - except: + except Exception: pass devnull.close() - COMPRESSEDCONTAINER = [re.compile('.r\d{2}$', re.I), - re.compile('.part\d+.rar$', re.I), + COMPRESSEDCONTAINER = [re.compile(r'.r\d{2}$', re.I), + re.compile(r'.part\d+.rar$', re.I), re.compile('.rar$', re.I)] - COMPRESSEDCONTAINER += [re.compile('{0}$'.format(ext), re.I) for ext in CFG["Extensions"]["compressedExtensions"]] - MEDIACONTAINER = CFG["Extensions"]["mediaExtensions"] - AUDIOCONTAINER = CFG["Extensions"]["audioExtensions"] - METACONTAINER = CFG["Extensions"]["metaExtensions"] # .nfo,.sub,.srt + COMPRESSEDCONTAINER += [re.compile('{0}$'.format(ext), re.I) for ext in CFG['Extensions']['compressedExtensions']] + MEDIACONTAINER = CFG['Extensions']['mediaExtensions'] + AUDIOCONTAINER = CFG['Extensions']['audioExtensions'] + METACONTAINER = CFG['Extensions']['metaExtensions'] # .nfo,.sub,.srt if isinstance(COMPRESSEDCONTAINER, str): COMPRESSEDCONTAINER = COMPRESSEDCONTAINER.split(',') if isinstance(MEDIACONTAINER, str): @@ -457,15 +464,15 @@ def initialize(section=None): if isinstance(METACONTAINER, str): METACONTAINER = METACONTAINER.split(',') - GETSUBS = int(CFG["Transcoder"]["getSubs"]) - TRANSCODE = int(CFG["Transcoder"]["transcode"]) - DUPLICATE = int(CFG["Transcoder"]["duplicate"]) - CONCAT = int(CFG["Transcoder"]["concat"]) - IGNOREEXTENSIONS = (CFG["Transcoder"]["ignoreExtensions"]) + GETSUBS = int(CFG['Transcoder']['getSubs']) + TRANSCODE = int(CFG['Transcoder']['transcode']) + DUPLICATE = int(CFG['Transcoder']['duplicate']) + CONCAT = int(CFG['Transcoder']['concat']) + IGNOREEXTENSIONS = (CFG['Transcoder']['ignoreExtensions']) if isinstance(IGNOREEXTENSIONS, str): IGNOREEXTENSIONS = IGNOREEXTENSIONS.split(',') - OUTPUTFASTSTART = int(CFG["Transcoder"]["outputFastStart"]) - GENERALOPTS = (CFG["Transcoder"]["generalOptions"]) + OUTPUTFASTSTART = int(CFG['Transcoder']['outputFastStart']) + GENERALOPTS = (CFG['Transcoder']['generalOptions']) if isinstance(GENERALOPTS, str): GENERALOPTS = GENERALOPTS.split(',') if GENERALOPTS == ['']: @@ -475,93 +482,93 @@ def initialize(section=None): if '+genpts' not in GENERALOPTS: GENERALOPTS.append('+genpts') try: - OUTPUTQUALITYPERCENT = int(CFG["Transcoder"]["outputQualityPercent"]) - except: + OUTPUTQUALITYPERCENT = int(CFG['Transcoder']['outputQualityPercent']) + except Exception: pass - OUTPUTVIDEOPATH = CFG["Transcoder"]["outputVideoPath"] - PROCESSOUTPUT = int(CFG["Transcoder"]["processOutput"]) - ALANGUAGE = CFG["Transcoder"]["audioLanguage"] - AINCLUDE = int(CFG["Transcoder"]["allAudioLanguages"]) - SLANGUAGES = CFG["Transcoder"]["subLanguages"] + OUTPUTVIDEOPATH = CFG['Transcoder']['outputVideoPath'] + PROCESSOUTPUT = int(CFG['Transcoder']['processOutput']) + ALANGUAGE = CFG['Transcoder']['audioLanguage'] + AINCLUDE = int(CFG['Transcoder']['allAudioLanguages']) + SLANGUAGES = CFG['Transcoder']['subLanguages'] if isinstance(SLANGUAGES, str): SLANGUAGES = SLANGUAGES.split(',') if SLANGUAGES == ['']: SLANGUAGES = [] - SINCLUDE = int(CFG["Transcoder"]["allSubLanguages"]) - SEXTRACT = int(CFG["Transcoder"]["extractSubs"]) - SEMBED = int(CFG["Transcoder"]["embedSubs"]) - SUBSDIR = CFG["Transcoder"]["externalSubDir"] - VEXTENSION = CFG["Transcoder"]["outputVideoExtension"].strip() - VCODEC = CFG["Transcoder"]["outputVideoCodec"].strip() - VCODEC_ALLOW = CFG["Transcoder"]["VideoCodecAllow"].strip() + SINCLUDE = int(CFG['Transcoder']['allSubLanguages']) + SEXTRACT = int(CFG['Transcoder']['extractSubs']) + SEMBED = int(CFG['Transcoder']['embedSubs']) + SUBSDIR = CFG['Transcoder']['externalSubDir'] + VEXTENSION = CFG['Transcoder']['outputVideoExtension'].strip() + VCODEC = CFG['Transcoder']['outputVideoCodec'].strip() + VCODEC_ALLOW = CFG['Transcoder']['VideoCodecAllow'].strip() if isinstance(VCODEC_ALLOW, str): VCODEC_ALLOW = VCODEC_ALLOW.split(',') if VCODEC_ALLOW == ['']: VCODEC_ALLOW = [] - VPRESET = CFG["Transcoder"]["outputVideoPreset"].strip() + VPRESET = CFG['Transcoder']['outputVideoPreset'].strip() try: - VFRAMERATE = float(CFG["Transcoder"]["outputVideoFramerate"].strip()) - except: + VFRAMERATE = float(CFG['Transcoder']['outputVideoFramerate'].strip()) + except Exception: pass try: - VCRF = int(CFG["Transcoder"]["outputVideoCRF"].strip()) - except: + VCRF = int(CFG['Transcoder']['outputVideoCRF'].strip()) + except Exception: pass try: - VLEVEL = CFG["Transcoder"]["outputVideoLevel"].strip() - except: + VLEVEL = CFG['Transcoder']['outputVideoLevel'].strip() + except Exception: pass try: - VBITRATE = int((CFG["Transcoder"]["outputVideoBitrate"].strip()).replace('k', '000')) - except: + VBITRATE = int((CFG['Transcoder']['outputVideoBitrate'].strip()).replace('k', '000')) + except Exception: pass - VRESOLUTION = CFG["Transcoder"]["outputVideoResolution"] - ACODEC = CFG["Transcoder"]["outputAudioCodec"].strip() - ACODEC_ALLOW = CFG["Transcoder"]["AudioCodecAllow"].strip() + VRESOLUTION = CFG['Transcoder']['outputVideoResolution'] + ACODEC = CFG['Transcoder']['outputAudioCodec'].strip() + ACODEC_ALLOW = CFG['Transcoder']['AudioCodecAllow'].strip() if isinstance(ACODEC_ALLOW, str): ACODEC_ALLOW = ACODEC_ALLOW.split(',') if ACODEC_ALLOW == ['']: ACODEC_ALLOW = [] try: - ACHANNELS = int(CFG["Transcoder"]["outputAudioChannels"].strip()) - except: + ACHANNELS = int(CFG['Transcoder']['outputAudioChannels'].strip()) + except Exception: pass try: - ABITRATE = int((CFG["Transcoder"]["outputAudioBitrate"].strip()).replace('k', '000')) - except: + ABITRATE = int((CFG['Transcoder']['outputAudioBitrate'].strip()).replace('k', '000')) + except Exception: pass - ACODEC2 = CFG["Transcoder"]["outputAudioTrack2Codec"].strip() - ACODEC2_ALLOW = CFG["Transcoder"]["AudioCodec2Allow"].strip() + ACODEC2 = CFG['Transcoder']['outputAudioTrack2Codec'].strip() + ACODEC2_ALLOW = CFG['Transcoder']['AudioCodec2Allow'].strip() if isinstance(ACODEC2_ALLOW, str): ACODEC2_ALLOW = ACODEC2_ALLOW.split(',') if ACODEC2_ALLOW == ['']: ACODEC2_ALLOW = [] try: - ACHANNELS2 = int(CFG["Transcoder"]["outputAudioTrack2Channels"].strip()) - except: + ACHANNELS2 = int(CFG['Transcoder']['outputAudioTrack2Channels'].strip()) + except Exception: pass try: - ABITRATE2 = int((CFG["Transcoder"]["outputAudioTrack2Bitrate"].strip()).replace('k', '000')) - except: + ABITRATE2 = int((CFG['Transcoder']['outputAudioTrack2Bitrate'].strip()).replace('k', '000')) + except Exception: pass - ACODEC3 = CFG["Transcoder"]["outputAudioOtherCodec"].strip() - ACODEC3_ALLOW = CFG["Transcoder"]["AudioOtherCodecAllow"].strip() + ACODEC3 = CFG['Transcoder']['outputAudioOtherCodec'].strip() + ACODEC3_ALLOW = CFG['Transcoder']['AudioOtherCodecAllow'].strip() if isinstance(ACODEC3_ALLOW, str): ACODEC3_ALLOW = ACODEC3_ALLOW.split(',') if ACODEC3_ALLOW == ['']: ACODEC3_ALLOW = [] try: - ACHANNELS3 = int(CFG["Transcoder"]["outputAudioOtherChannels"].strip()) - except: + ACHANNELS3 = int(CFG['Transcoder']['outputAudioOtherChannels'].strip()) + except Exception: pass try: - ABITRATE3 = int((CFG["Transcoder"]["outputAudioOtherBitrate"].strip()).replace('k', '000')) - except: + ABITRATE3 = int((CFG['Transcoder']['outputAudioOtherBitrate'].strip()).replace('k', '000')) + except Exception: pass - SCODEC = CFG["Transcoder"]["outputSubtitleCodec"].strip() - BURN = int(CFG["Transcoder"]["burnInSubtitle"].strip()) - DEFAULTS = CFG["Transcoder"]["outputDefault"].strip() - HWACCEL = int(CFG["Transcoder"]["hwAccel"]) + SCODEC = CFG['Transcoder']['outputSubtitleCodec'].strip() + BURN = int(CFG['Transcoder']['burnInSubtitle'].strip()) + DEFAULTS = CFG['Transcoder']['outputDefault'].strip() + HWACCEL = int(CFG['Transcoder']['hwAccel']) allow_subs = ['.mkv', '.mp4', '.m4v', 'asf', 'wma', 'wmv'] codec_alias = { @@ -570,118 +577,118 @@ def initialize(section=None): 'libfaac': ['libfaac', 'aac', 'faac'] } transcode_defaults = { - 'iPad':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':None,'VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':None, 'ACHANNELS':2, - 'ACODEC2':'ac3','ACODEC2_ALLOW':['ac3'],'ABITRATE2':None, 'ACHANNELS2':6, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'iPad-1080p':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':'1920:1080','VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':None, 'ACHANNELS':2, - 'ACODEC2':'ac3','ACODEC2_ALLOW':['ac3'],'ABITRATE2':None, 'ACHANNELS2':6, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'iPad-720p':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':'1280:720','VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':None, 'ACHANNELS':2, - 'ACODEC2':'ac3','ACODEC2_ALLOW':['ac3'],'ABITRATE2':None, 'ACHANNELS2':6, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'Apple-TV':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':'1280:720','VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'ac3','ACODEC_ALLOW':['ac3'],'ABITRATE':None, 'ACHANNELS':6, - 'ACODEC2':'aac','ACODEC2_ALLOW':['libfaac'],'ABITRATE2':None, 'ACHANNELS2':2, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'iPod':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':'1280:720','VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':128000, 'ACHANNELS':2, - 'ACODEC2':None,'ACODEC2_ALLOW':[],'ABITRATE2':None, 'ACHANNELS2':None, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'iPhone':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':'460:320','VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':128000, 'ACHANNELS':2, - 'ACODEC2':None,'ACODEC2_ALLOW':[],'ABITRATE2':None, 'ACHANNELS2':None, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'PS3':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':None,'VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'ac3','ACODEC_ALLOW':['ac3'],'ABITRATE':None, 'ACHANNELS':6, - 'ACODEC2':'aac','ACODEC2_ALLOW':['libfaac'],'ABITRATE2':None, 'ACHANNELS2':2, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'xbox':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':None,'VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'ac3','ACODEC_ALLOW':['ac3'],'ABITRATE':None, 'ACHANNELS':6, - 'ACODEC2':None,'ACODEC2_ALLOW':[],'ABITRATE2':None, 'ACHANNELS2':None, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'Roku-480p':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':None,'VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':128000, 'ACHANNELS':2, - 'ACODEC2':'ac3','ACODEC2_ALLOW':['ac3'],'ABITRATE2':None, 'ACHANNELS2':6, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'Roku-720p':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':None,'VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':128000, 'ACHANNELS':2, - 'ACODEC2':'ac3','ACODEC2_ALLOW':['ac3'],'ABITRATE2':None, 'ACHANNELS2':6, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'Roku-1080p':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':None,'VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':160000, 'ACHANNELS':2, - 'ACODEC2':'ac3','ACODEC2_ALLOW':['ac3'],'ABITRATE2':None, 'ACHANNELS2':6, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - }, - 'mkv':{ - 'VEXTENSION':'.mkv','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':None,'VLEVEL':None, - 'VRESOLUTION':None,'VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], - 'ACODEC':'dts','ACODEC_ALLOW':['libfaac', 'dts', 'ac3', 'mp2', 'mp3'],'ABITRATE':None, 'ACHANNELS':8, - 'ACODEC2':None,'ACODEC2_ALLOW':[],'ABITRATE2':None, 'ACHANNELS2':None, - 'ACODEC3':'ac3','ACODEC3_ALLOW':['libfaac', 'dts', 'ac3', 'mp2', 'mp3'],'ABITRATE3':None, 'ACHANNELS3':8, - 'SCODEC':'mov_text' - }, - 'mp4-scene-release':{ - 'VEXTENSION':'.mp4','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':None,'VCRF':19,'VLEVEL':'3.1', - 'VRESOLUTION':None,'VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], - 'ACODEC':'dts','ACODEC_ALLOW':['libfaac', 'dts', 'ac3', 'mp2', 'mp3'],'ABITRATE':None, 'ACHANNELS':8, - 'ACODEC2':None,'ACODEC2_ALLOW':[],'ABITRATE2':None, 'ACHANNELS2':None, - 'ACODEC3':'ac3','ACODEC3_ALLOW':['libfaac', 'dts', 'ac3', 'mp2', 'mp3'],'ABITRATE3':None, 'ACHANNELS3':8, - 'SCODEC':'mov_text' - }, - 'MKV-SD':{ - 'VEXTENSION':'.mkv','VCODEC':'libx264','VPRESET':None,'VFRAMERATE':None,'VBITRATE':'1200k','VCRF':None,'VLEVEL':None, - 'VRESOLUTION':'720:-1','VCODEC_ALLOW':['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], - 'ACODEC':'aac','ACODEC_ALLOW':['libfaac'],'ABITRATE':128000, 'ACHANNELS':2, - 'ACODEC2':'ac3','ACODEC2_ALLOW':['ac3'],'ABITRATE2':None, 'ACHANNELS2':6, - 'ACODEC3':None,'ACODEC3_ALLOW':[],'ABITRATE3':None, 'ACHANNELS3':None, - 'SCODEC':'mov_text' - } + 'iPad': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, + 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'iPad-1080p': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': '1920:1080', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, + 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'iPad-720p': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': '1280:720', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': None, 'ACHANNELS': 2, + 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'Apple-TV': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': '1280:720', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, + 'ACODEC2': 'aac', 'ACODEC2_ALLOW': ['libfaac'], 'ABITRATE2': None, 'ACHANNELS2': 2, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'iPod': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': '1280:720', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, + 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'iPhone': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': '460:320', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, + 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'PS3': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, + 'ACODEC2': 'aac', 'ACODEC2_ALLOW': ['libfaac'], 'ABITRATE2': None, 'ACHANNELS2': 2, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'xbox': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'ac3', 'ACODEC_ALLOW': ['ac3'], 'ABITRATE': None, 'ACHANNELS': 6, + 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'Roku-480p': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, + 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'Roku-720p': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, + 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'Roku-1080p': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 160000, 'ACHANNELS': 2, + 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + }, + 'mkv': { + 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, + 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, + 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, + 'SCODEC': 'mov_text' + }, + 'mp4-scene-release': { + 'VEXTENSION': '.mp4', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': None, 'VCRF': 19, 'VLEVEL': '3.1', + 'VRESOLUTION': None, 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4', 'mpeg2video'], + 'ACODEC': 'dts', 'ACODEC_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE': None, 'ACHANNELS': 8, + 'ACODEC2': None, 'ACODEC2_ALLOW': [], 'ABITRATE2': None, 'ACHANNELS2': None, + 'ACODEC3': 'ac3', 'ACODEC3_ALLOW': ['libfaac', 'dts', 'ac3', 'mp2', 'mp3'], 'ABITRATE3': None, 'ACHANNELS3': 8, + 'SCODEC': 'mov_text' + }, + 'MKV-SD': { + 'VEXTENSION': '.mkv', 'VCODEC': 'libx264', 'VPRESET': None, 'VFRAMERATE': None, 'VBITRATE': '1200k', 'VCRF': None, 'VLEVEL': None, + 'VRESOLUTION': '720: -1', 'VCODEC_ALLOW': ['libx264', 'h264', 'h.264', 'AVC', 'avc', 'mpeg4', 'msmpeg4', 'MPEG-4'], + 'ACODEC': 'aac', 'ACODEC_ALLOW': ['libfaac'], 'ABITRATE': 128000, 'ACHANNELS': 2, + 'ACODEC2': 'ac3', 'ACODEC2_ALLOW': ['ac3'], 'ABITRATE2': None, 'ACHANNELS2': 6, + 'ACODEC3': None, 'ACODEC3_ALLOW': [], 'ABITRATE3': None, 'ACHANNELS3': None, + 'SCODEC': 'mov_text' + } } if DEFAULTS and DEFAULTS in transcode_defaults: VEXTENSION = transcode_defaults[DEFAULTS]['VEXTENSION'] @@ -738,52 +745,53 @@ def initialize(section=None): ACODEC3_ALLOW.extend(extra) codec_alias = {} # clear memory - PASSWORDSFILE = CFG["passwords"]["PassWordFile"] + PASSWORDSFILE = CFG['passwords']['PassWordFile'] # Setup FFMPEG, FFPROBE and SEVENZIP locations if platform.system() == 'Windows': FFMPEG = os.path.join(FFMPEG_PATH, 'ffmpeg.exe') FFPROBE = os.path.join(FFMPEG_PATH, 'ffprobe.exe') - SEVENZIP = os.path.join(PROGRAM_DIR, 'core', 'extractor', 'bin', platform.machine(), '7z.exe') + SEVENZIP = os.path.join(APP_ROOT, 'core', 'extractor', 'bin', platform.machine(), '7z.exe') + SHOWEXTRACT = int(str(CFG['Windows']['show_extraction']), 0) if not (os.path.isfile(FFMPEG)): # problem FFMPEG = None - logger.warning("Failed to locate ffmpeg.exe. Transcoding disabled!") - logger.warning("Install ffmpeg with x264 support to enable this feature ...") + logger.warning('Failed to locate ffmpeg.exe. Transcoding disabled!') + logger.warning('Install ffmpeg with x264 support to enable this feature ...') if not (os.path.isfile(FFPROBE)): FFPROBE = None if CHECK_MEDIA: - logger.warning("Failed to locate ffprobe.exe. Video corruption detection disabled!") - logger.warning("Install ffmpeg with x264 support to enable this feature ...") + logger.warning('Failed to locate ffprobe.exe. Video corruption detection disabled!') + logger.warning('Install ffmpeg with x264 support to enable this feature ...') else: try: SEVENZIP = subprocess.Popen(['which', '7z'], stdout=subprocess.PIPE).communicate()[0].strip() - except: + except Exception: pass if not SEVENZIP: try: SEVENZIP = subprocess.Popen(['which', '7zr'], stdout=subprocess.PIPE).communicate()[0].strip() - except: + except Exception: pass if not SEVENZIP: try: SEVENZIP = subprocess.Popen(['which', '7za'], stdout=subprocess.PIPE).communicate()[0].strip() - except: + except Exception: pass if not SEVENZIP: SEVENZIP = None logger.warning( - "Failed to locate 7zip. Transcoding of disk images and extraction of .7z files will not be possible!") + 'Failed to locate 7zip. Transcoding of disk images and extraction of .7z files will not be possible!') try: PAR2CMD = subprocess.Popen(['which', 'par2'], stdout=subprocess.PIPE).communicate()[0].strip() - except: + except Exception: pass if not PAR2CMD: PAR2CMD = None logger.warning( - "Failed to locate par2. Repair and rename using par files will not be possible!") + 'Failed to locate par2. Repair and rename using par files will not be possible!') if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffmpeg')) or os.access(os.path.join(FFMPEG_PATH, 'ffmpeg'), os.X_OK): FFMPEG = os.path.join(FFMPEG_PATH, 'ffmpeg') @@ -793,17 +801,17 @@ def initialize(section=None): else: try: FFMPEG = subprocess.Popen(['which', 'ffmpeg'], stdout=subprocess.PIPE).communicate()[0].strip() - except: + except Exception: pass if not FFMPEG: try: FFMPEG = subprocess.Popen(['which', 'avconv'], stdout=subprocess.PIPE).communicate()[0].strip() - except: + except Exception: pass if not FFMPEG: FFMPEG = None - logger.warning("Failed to locate ffmpeg. Transcoding disabled!") - logger.warning("Install ffmpeg with x264 support to enable this feature ...") + logger.warning('Failed to locate ffmpeg. Transcoding disabled!') + logger.warning('Install ffmpeg with x264 support to enable this feature ...') if os.path.isfile(os.path.join(FFMPEG_PATH, 'ffprobe')) or os.access(os.path.join(FFMPEG_PATH, 'ffprobe'), os.X_OK): @@ -814,18 +822,18 @@ def initialize(section=None): else: try: FFPROBE = subprocess.Popen(['which', 'ffprobe'], stdout=subprocess.PIPE).communicate()[0].strip() - except: + except Exception: pass if not FFPROBE: try: FFPROBE = subprocess.Popen(['which', 'avprobe'], stdout=subprocess.PIPE).communicate()[0].strip() - except: + except Exception: pass if not FFPROBE: FFPROBE = None if CHECK_MEDIA: - logger.warning("Failed to locate ffprobe. Video corruption detection disabled!") - logger.warning("Install ffmpeg with x264 support to enable this feature ...") + logger.warning('Failed to locate ffprobe. Video corruption detection disabled!') + logger.warning('Install ffmpeg with x264 support to enable this feature ...') # check for script-defied section and if None set to allow sections SECTIONS = CFG[tuple(x for x in CFG if CFG[x].sections and CFG[x].isenabled()) if not section else (section,)] @@ -841,7 +849,7 @@ def initialize(section=None): def restart(): - install_type = versionCheck.CheckVersion().install_type + install_type = version_check.CheckVersion().install_type status = 0 popen_list = [] @@ -851,7 +859,7 @@ def restart(): if popen_list: popen_list += SYS_ARGV - logger.log(u"Restarting nzbToMedia with {args}".format(args=popen_list)) + logger.log(u'Restarting nzbToMedia with {args}'.format(args=popen_list)) logger.close() p = subprocess.Popen(popen_list, cwd=os.getcwd()) p.wait() @@ -861,7 +869,7 @@ def restart(): def rchmod(path, mod): - logger.log("Changing file mode of {0} to {1}".format(path, oct(mod))) + logger.log('Changing file mode of {0} to {1}'.format(path, oct(mod))) os.chmod(path, mod) if not os.path.isdir(path): return # Skip files diff --git a/core/autoProcess/autoProcessComics.py b/core/autoProcess/autoProcessComics.py deleted file mode 100644 index 0e4facf73..000000000 --- a/core/autoProcess/autoProcessComics.py +++ /dev/null @@ -1,77 +0,0 @@ -# coding=utf-8 - -import os -import core -import requests - -from core.nzbToMediaUtil import convert_to_ascii, remoteDir, server_responding -from core import logger - -requests.packages.urllib3.disable_warnings() - - -class autoProcessComics(object): - def processEpisode(self, section, dirName, inputName=None, status=0, clientAgent='manual', inputCategory=None): - - apc_version = "2.04" - comicrn_version = "1.01" - - cfg = dict(core.CFG[section][inputCategory]) - - host = cfg["host"] - port = cfg["port"] - apikey = cfg["apikey"] - ssl = int(cfg.get("ssl", 0)) - web_root = cfg.get("web_root", "") - remote_path = int(cfg.get("remote_path"), 0) - protocol = "https://" if ssl else "http://" - - url = "{0}{1}:{2}{3}/api".format(protocol, host, port, web_root) - if not server_responding(url): - logger.error("Server did not respond. Exiting", section) - return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)] - - inputName, dirName = convert_to_ascii(inputName, dirName) - clean_name, ext = os.path.splitext(inputName) - if len(ext) == 4: # we assume this was a standard extension. - inputName = clean_name - - params = { - 'cmd': 'forceProcess', - 'apikey': apikey, - 'nzb_folder': remoteDir(dirName) if remote_path else dirName, - } - - if inputName is not None: - params['nzb_name'] = inputName - params['failed'] = int(status) - params['apc_version'] = apc_version - params['comicrn_version'] = comicrn_version - - success = False - - logger.debug("Opening URL: {0}".format(url), section) - try: - r = requests.post(url, params=params, stream=True, verify=False, timeout=(30, 300)) - except requests.ConnectionError: - logger.error("Unable to open URL", section) - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] - - result = r.content - if not type(result) == list: - result = result.split('\n') - for line in result: - if line: - logger.postprocess("{0}".format(line), section) - if "Post Processing SUCCESSFUL" in line: - success = True - - if success: - logger.postprocess("SUCCESS: This issue has been processed successfully", section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - else: - logger.warning("The issue does not appear to have successfully processed. Please check your Logs", section) - return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] diff --git a/core/autoProcess/autoProcessGames.py b/core/autoProcess/autoProcessGames.py deleted file mode 100644 index ef7e7cb9d..000000000 --- a/core/autoProcess/autoProcessGames.py +++ /dev/null @@ -1,77 +0,0 @@ -# coding=utf-8 - -import os -import core -import requests -import shutil - -from core.nzbToMediaUtil import convert_to_ascii, server_responding -from core import logger - -requests.packages.urllib3.disable_warnings() - - -class autoProcessGames(object): - def process(self, section, dirName, inputName=None, status=0, clientAgent='manual', inputCategory=None): - status = int(status) - - cfg = dict(core.CFG[section][inputCategory]) - - host = cfg["host"] - port = cfg["port"] - apikey = cfg["apikey"] - library = cfg.get("library") - ssl = int(cfg.get("ssl", 0)) - web_root = cfg.get("web_root", "") - protocol = "https://" if ssl else "http://" - - url = "{0}{1}:{2}{3}/api".format(protocol, host, port, web_root) - if not server_responding(url): - logger.error("Server did not respond. Exiting", section) - return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)] - - inputName, dirName = convert_to_ascii(inputName, dirName) - - fields = inputName.split("-") - - gamezID = fields[0].replace("[", "").replace("]", "").replace(" ", "") - - downloadStatus = 'Downloaded' if status == 0 else 'Wanted' - - params = { - 'api_key': apikey, - 'mode': 'UPDATEREQUESTEDSTATUS', - 'db_id': gamezID, - 'status': downloadStatus - } - - logger.debug("Opening URL: {0}".format(url), section) - - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 300)) - except requests.ConnectionError: - logger.error("Unable to open URL") - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - - result = r.json() - logger.postprocess("{0}".format(result), section) - if library: - logger.postprocess("moving files to library: {0}".format(library), section) - try: - shutil.move(dirName, os.path.join(library, inputName)) - except: - logger.error("Unable to move {0} to {1}".format(dirName, os.path.join(library, inputName)), section) - return [1, "{0}: Failed to post-process - Unable to move files".format(section)] - else: - logger.error("No library specified to move files to. Please edit your configuration.", section) - return [1, "{0}: Failed to post-process - No library defined in {1}".format(section, section)] - - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] - elif result['success']: - logger.postprocess("SUCCESS: Status for {0} has been set to {1} in Gamez".format(gamezID, downloadStatus), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - else: - logger.error("FAILED: Status for {0} has NOT been updated in Gamez".format(gamezID), section) - return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] diff --git a/core/autoProcess/autoProcessMovie.py b/core/autoProcess/autoProcessMovie.py deleted file mode 100644 index 233531995..000000000 --- a/core/autoProcess/autoProcessMovie.py +++ /dev/null @@ -1,464 +0,0 @@ -# coding=utf-8 - -import os -import time -import requests -import json -import core - -from core.nzbToMediaSceneExceptions import process_all_exceptions -from core.nzbToMediaUtil import convert_to_ascii, rmDir, find_imdbid, find_download, listMediaFiles, remoteDir, import_subs, server_responding, reportNzb -from core import logger -from core.transcoder import transcoder - -requests.packages.urllib3.disable_warnings() - - -class autoProcessMovie(object): - def get_release(self, baseURL, imdbid=None, download_id=None, release_id=None): - results = {} - params = {} - - # determine cmd and params to send to CouchPotato to get our results - section = 'movies' - cmd = "media.list" - if release_id or imdbid: - section = 'media' - cmd = "media.get" - params['id'] = release_id or imdbid - - if not (release_id or imdbid or download_id): - logger.debug("No information available to filter CP results") - return results - - url = "{0}{1}".format(baseURL, cmd) - logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params)) - - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL {0}".format(url)) - return results - - try: - result = r.json() - except ValueError: - # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error("CouchPotato returned the following non-json data") - for line in r.iter_lines(): - logger.error("{0}".format(line)) - return results - - if not result['success']: - if 'error' in result: - logger.error('{0}'.format(result['error'])) - else: - logger.error("no media found for id {0}".format(params['id'])) - return results - - # Gather release info and return it back, no need to narrow results - if release_id: - try: - id = result[section]['_id'] - results[id] = result[section] - return results - except: - pass - - # Gather release info and proceed with trying to narrow results to one release choice - - movies = result[section] - if not isinstance(movies, list): - movies = [movies] - for movie in movies: - if movie['status'] not in ['active', 'done']: - continue - releases = movie['releases'] - for release in releases: - try: - if release['status'] not in ['snatched', 'downloaded', 'done']: - continue - if download_id: - if download_id.lower() != release['download_info']['id'].lower(): - continue - - id = release['_id'] - results[id] = release - results[id]['title'] = movie['title'] - except: - continue - - # Narrow results by removing old releases by comparing their last_edit field - if len(results) > 1: - for id1, x1 in results.items(): - for id2, x2 in results.items(): - try: - if x2["last_edit"] > x1["last_edit"]: - results.pop(id1) - except: - continue - - # Search downloads on clients for a match to try and narrow our results down to 1 - if len(results) > 1: - for id, x in results.items(): - try: - if not find_download(str(x['download_info']['downloader']).lower(), x['download_info']['id']): - results.pop(id) - except: - continue - - return results - - def command_complete(self, url, params, headers, section): - try: - r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url), section) - return None - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return None - else: - try: - return r.json()['state'] - except (ValueError, KeyError): - # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error("{0} did not return expected json data.".format(section), section) - return None - - def CDH(self, url2, headers, section="MAIN"): - try: - r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url2), section) - return False - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return False - else: - try: - return r.json().get("enableCompletedDownloadHandling", False) - except ValueError: - # ValueError catches simplejson's JSONDecodeError and json's ValueError - return False - - def process(self, section, dirName, inputName=None, status=0, clientAgent="manual", download_id="", inputCategory=None, failureLink=None): - - cfg = dict(core.CFG[section][inputCategory]) - - host = cfg["host"] - port = cfg["port"] - apikey = cfg["apikey"] - if section == "CouchPotato": - method = cfg["method"] - else: - method = None - #added importMode for Radarr config - if section == "Radarr": - importMode = cfg.get("importMode","Move") - else: - importMode = None - delete_failed = int(cfg["delete_failed"]) - wait_for = int(cfg["wait_for"]) - ssl = int(cfg.get("ssl", 0)) - web_root = cfg.get("web_root", "") - remote_path = int(cfg.get("remote_path", 0)) - protocol = "https://" if ssl else "http://" - omdbapikey = cfg.get("omdbapikey", "") - status = int(status) - if status > 0 and core.NOEXTRACTFAILED: - extract = 0 - else: - extract = int(cfg.get("extract", 0)) - - imdbid = find_imdbid(dirName, inputName, omdbapikey) - if section == "CouchPotato": - baseURL = "{0}{1}:{2}{3}/api/{4}/".format(protocol, host, port, web_root, apikey) - if section == "Radarr": - baseURL = "{0}{1}:{2}{3}/api/command".format(protocol, host, port, web_root) - url2 = "{0}{1}:{2}{3}/api/config/downloadClient".format(protocol, host, port, web_root) - headers = {'X-Api-Key': apikey} - if not apikey: - logger.info('No CouchPotato or Radarr apikey entered. Performing transcoder functions only') - release = None - elif server_responding(baseURL): - if section == "CouchPotato": - release = self.get_release(baseURL, imdbid, download_id) - else: - release = None - else: - logger.error("Server did not respond. Exiting", section) - return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)] - - # pull info from release found if available - release_id = None - media_id = None - downloader = None - release_status_old = None - if release: - try: - release_id = release.keys()[0] - media_id = release[release_id]['media_id'] - download_id = release[release_id]['download_info']['id'] - downloader = release[release_id]['download_info']['downloader'] - release_status_old = release[release_id]['status'] - except: - pass - - if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name. - dirName = os.path.split(os.path.normpath(dirName))[0] - - SpecificPath = os.path.join(dirName, str(inputName)) - cleanName = os.path.splitext(SpecificPath) - if cleanName[1] == ".nzb": - SpecificPath = cleanName[0] - if os.path.isdir(SpecificPath): - dirName = SpecificPath - - process_all_exceptions(inputName, dirName) - inputName, dirName = convert_to_ascii(inputName, dirName) - - if not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True) and extract: - logger.debug('Checking for archives to extract in directory: {0}'.format(dirName)) - core.extractFiles(dirName) - inputName, dirName = convert_to_ascii(inputName, dirName) - - good_files = 0 - num_files = 0 - # Check video files for corruption - for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): - num_files += 1 - if transcoder.isVideoGood(video, status): - import_subs(video) - good_files += 1 - if num_files and good_files == num_files: - if status: - logger.info("Status shown as failed from Downloader, but {0} valid video files found. Setting as success.".format(good_files), section) - status = 0 - elif num_files and good_files < num_files: - logger.info("Status shown as success from Downloader, but corrupt video files found. Setting as failed.", section) - if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': - print('[NZB] MARK=BAD') - if failureLink: - failureLink += '&corrupt=true' - status = 1 - elif clientAgent == "manual": - logger.warning("No media files found in directory {0} to manually process.".format(dirName), section) - return [0, ""] # Success (as far as this script is concerned) - else: - logger.warning("No media files found in directory {0}. Processing this as a failed download".format(dirName), section) - status = 1 - if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': - print('[NZB] MARK=BAD') - - if status == 0: - if core.TRANSCODE == 1: - result, newDirName = transcoder.Transcode_directory(dirName) - if result == 0: - logger.debug("Transcoding succeeded for files in {0}".format(dirName), section) - dirName = newDirName - - chmod_directory = int(str(cfg.get("chmodDirectory", "0")), 8) - logger.debug("Config setting 'chmodDirectory' currently set to {0}".format(oct(chmod_directory)), section) - if chmod_directory: - logger.info("Attempting to set the octal permission of '{0}' on directory '{1}'".format(oct(chmod_directory), dirName), section) - core.rchmod(dirName, chmod_directory) - else: - logger.error("Transcoding failed for files in {0}".format(dirName), section) - return [1, "{0}: Failed to post-process - Transcoding failed".format(section)] - for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): - if not release and ".cp(tt" not in video and imdbid: - videoName, videoExt = os.path.splitext(video) - video2 = "{0}.cp({1}){2}".format(videoName, imdbid, videoExt) - if not (clientAgent in [core.TORRENT_CLIENTAGENT, 'manual'] and core.USELINK == 'move-sym'): - logger.debug('Renaming: {0} to: {1}'.format(video, video2)) - os.rename(video, video2) - - if not apikey: #If only using Transcoder functions, exit here. - logger.info('No CouchPotato or Radarr apikey entered. Processing completed.') - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - - params = {} - if download_id and release_id: - params['downloader'] = downloader or clientAgent - params['download_id'] = download_id - - params['media_folder'] = remoteDir(dirName) if remote_path else dirName - - if section == "CouchPotato": - if method == "manage": - command = "manage.update" - params = {} - else: - command = "renamer.scan" - - url = "{0}{1}".format(baseURL, command) - logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section) - logger.postprocess("Starting {0} scan for {1}".format(method, inputName), section) - - if section == "Radarr": - payload = {'name': 'DownloadedMoviesScan', 'path': params['media_folder'], 'downloadClientId': download_id,'importMode' : importMode} - if not download_id: - payload.pop("downloadClientId") - logger.debug("Opening URL: {0} with PARAMS: {1}".format(baseURL, payload), section) - logger.postprocess("Starting DownloadedMoviesScan scan for {0}".format(inputName), section) - - try: - if section == "CouchPotato": - r = requests.get(url, params=params, verify=False, timeout=(30, 1800)) - else: - r = requests.post(baseURL, data=json.dumps(payload), headers=headers, stream=True, verify=False, timeout=(30, 1800)) - except requests.ConnectionError: - logger.error("Unable to open URL", section) - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - - result = r.json() - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] - elif section == "CouchPotato" and result['success']: - logger.postprocess("SUCCESS: Finished {0} scan for folder {1}".format(method, dirName), section) - if method == "manage": - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - elif section == "Radarr": - logger.postprocess("Radarr response: {0}".format(result['state'])) - try: - res = json.loads(r.content) - scan_id = int(res['id']) - logger.debug("Scan started with id: {0}".format(scan_id), section) - Started = True - except Exception as e: - logger.warning("No scan id was returned due to: {0}".format(e), section) - scan_id = None - else: - logger.error("FAILED: {0} scan was unable to finish for folder {1}. exiting!".format(method, dirName), - section) - return [1, "{0}: Failed to post-process - Server did not return success".format(section)] - - else: - core.FAILED = True - logger.postprocess("FAILED DOWNLOAD DETECTED FOR {0}".format(inputName), section) - if failureLink: - reportNzb(failureLink, clientAgent) - - if section == "Radarr": - logger.postprocess("FAILED: The download failed. Sending failed download to {0} for CDH processing".format(section), section) - return [1, "{0}: Download Failed. Sending back to {1}".format(section, section)] # Return as failed to flag this in the downloader. - - if delete_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName: - logger.postprocess("Deleting failed files and folder {0}".format(dirName), section) - rmDir(dirName) - - if not release_id and not media_id: - logger.error("Could not find a downloaded movie in the database matching {0}, exiting!".format(inputName), - section) - return [1, "{0}: Failed to post-process - Failed download not found in {1}".format(section, section)] - - if release_id: - logger.postprocess("Setting failed release {0} to ignored ...".format(inputName), section) - - url = "{url}release.ignore".format(url=baseURL) - params = {'id': release_id} - - logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section) - - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error("Unable to open URL {0}".format(url), section) - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - - result = r.json() - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] - elif result['success']: - logger.postprocess("SUCCESS: {0} has been set to ignored ...".format(inputName), section) - else: - logger.warning("FAILED: Unable to set {0} to ignored!".format(inputName), section) - return [1, "{0}: Failed to post-process - Unable to set {1} to ignored".format(section, inputName)] - - logger.postprocess("Trying to snatch the next highest ranked release.", section) - - url = "{0}movie.searcher.try_next".format(baseURL) - logger.debug("Opening URL: {0}".format(url), section) - - try: - r = requests.get(url, params={'media_id': media_id}, verify=False, timeout=(30, 600)) - except requests.ConnectionError: - logger.error("Unable to open URL {0}".format(url), section) - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - - result = r.json() - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] - elif result['success']: - logger.postprocess("SUCCESS: Snatched the next highest release ...", section) - return [0, "{0}: Successfully snatched next highest release".format(section)] - else: - logger.postprocess("SUCCESS: Unable to find a new release to snatch now. CP will keep searching!", section) - return [0, "{0}: No new release found now. {1} will keep searching".format(section, section)] - - # Added a release that was not in the wanted list so confirm rename successful by finding this movie media.list. - if not release: - download_id = None # we don't want to filter new releases based on this. - - # we will now check to see if CPS has finished renaming before returning to TorrentToMedia and unpausing. - timeout = time.time() + 60 * wait_for - while time.time() < timeout: # only wait 2 (default) minutes, then return. - logger.postprocess("Checking for status change, please stand by ...", section) - if section == "CouchPotato": - release = self.get_release(baseURL, imdbid, download_id, release_id) - scan_id = None - else: - release = None - if release: - try: - release_id = release.keys()[0] - title = release[release_id]['title'] - release_status_new = release[release_id]['status'] - if release_status_old is None: # we didn't have a release before, but now we do. - logger.postprocess("SUCCESS: Movie {0} has now been added to CouchPotato with release status of [{1}]".format( - title, str(release_status_new).upper()), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - - if release_status_new != release_status_old: - logger.postprocess("SUCCESS: Release for {0} has now been marked with a status of [{1}]".format( - title, str(release_status_new).upper()), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - except: - pass - elif scan_id: - url = "{0}/{1}".format(baseURL, scan_id) - command_status = self.command_complete(url, params, headers, section) - if command_status: - logger.debug("The Scan command return status: {0}".format(command_status), section) - if command_status in ['completed']: - logger.debug("The Scan command has completed successfully. Renaming was successful.", section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - elif command_status in ['failed']: - logger.debug("The Scan command has failed. Renaming was not successful.", section) - # return [1, "%s: Failed to post-process %s" % (section, inputName) ] - - if not os.path.isdir(dirName): - logger.postprocess("SUCCESS: Input Directory [{0}] has been processed and removed".format( - dirName), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - - elif not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=True): - logger.postprocess("SUCCESS: Input Directory [{0}] has no remaining media files. This has been fully processed.".format( - dirName), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - - # pause and let CouchPotatoServer/Radarr catch its breath - time.sleep(10 * wait_for) - - # The status hasn't changed. we have waited wait_for minutes which is more than enough. uTorrent can resume seeding now. - if section == "Radarr" and self.CDH(url2, headers, section=section): - logger.debug("The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {0}.".format(section), section) - return [status, "{0}: Complete DownLoad Handling is enabled. Passing back to {1}".format(section, section)] - logger.warning( - "{0} does not appear to have changed status after {1} minutes, Please check your logs.".format(inputName, wait_for), - section) - return [1, "{0}: Failed to post-process - No change in status".format(section)] diff --git a/core/autoProcess/autoProcessMusic.py b/core/autoProcess/autoProcessMusic.py deleted file mode 100644 index be64d73c9..000000000 --- a/core/autoProcess/autoProcessMusic.py +++ /dev/null @@ -1,238 +0,0 @@ -# coding=utf-8 - -import os -import time -import requests -import core -import json - -from core.nzbToMediaUtil import convert_to_ascii, rmDir, remoteDir, listMediaFiles, server_responding -from core.nzbToMediaSceneExceptions import process_all_exceptions -from core import logger - -requests.packages.urllib3.disable_warnings() - - -class autoProcessMusic(object): - def command_complete(self, url, params, headers, section): - try: - r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url), section) - return None - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return None - else: - try: - return r.json()['state'] - except (ValueError, KeyError): - # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error("{0} did not return expected json data.".format(section), section) - return None - - def get_status(self, url, apikey, dirName): - logger.debug("Attempting to get current status for release:{0}".format(os.path.basename(dirName))) - - params = { - 'apikey': apikey, - 'cmd': "getHistory" - } - - logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params)) - - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.RequestException: - logger.error("Unable to open URL") - return None - - try: - result = r.json() - except ValueError: - # ValueError catches simplejson's JSONDecodeError and json's ValueError - return None - - for album in result: - if os.path.basename(dirName) == album['FolderName']: - return album["Status"].lower() - - def forceProcess(self, params, url, apikey, inputName, dirName, section, wait_for): - release_status = self.get_status(url, apikey, dirName) - if not release_status: - logger.error("Could not find a status for {0}, is it in the wanted list ?".format(inputName), section) - - logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section) - - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 300)) - except requests.ConnectionError: - logger.error("Unable to open URL {0}".format(url), section) - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - - logger.debug("Result: {0}".format(r.text), section) - - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] - elif r.text == "OK": - logger.postprocess("SUCCESS: Post-Processing started for {0} in folder {1} ...".format(inputName, dirName), section) - else: - logger.error("FAILED: Post-Processing has NOT started for {0} in folder {1}. exiting!".format(inputName, dirName), section) - return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] - - # we will now wait for this album to be processed before returning to TorrentToMedia and unpausing. - timeout = time.time() + 60 * wait_for - while time.time() < timeout: - current_status = self.get_status(url, apikey, dirName) - if current_status is not None and current_status != release_status: # Something has changed. CPS must have processed this movie. - logger.postprocess("SUCCESS: This release is now marked as status [{0}]".format(current_status), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - if not os.path.isdir(dirName): - logger.postprocess("SUCCESS: The input directory {0} has been removed Processing must have finished.".format(dirName), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - time.sleep(10 * wait_for) - # The status hasn't changed. - return [2, "no change"] - - def process(self, section, dirName, inputName=None, status=0, clientAgent="manual", inputCategory=None): - status = int(status) - - cfg = dict(core.CFG[section][inputCategory]) - - host = cfg["host"] - port = cfg["port"] - apikey = cfg["apikey"] - wait_for = int(cfg["wait_for"]) - ssl = int(cfg.get("ssl", 0)) - delete_failed = int(cfg["delete_failed"]) - web_root = cfg.get("web_root", "") - remote_path = int(cfg.get("remote_path", 0)) - protocol = "https://" if ssl else "http://" - status = int(status) - if status > 0 and core.NOEXTRACTFAILED: - extract = 0 - else: - extract = int(cfg.get("extract", 0)) - - if section == "Lidarr": - url = "{0}{1}:{2}{3}/api/v1".format(protocol, host, port, web_root) - else: - url = "{0}{1}:{2}{3}/api".format(protocol, host, port, web_root) - if not server_responding(url): - logger.error("Server did not respond. Exiting", section) - return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)] - - if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name. - dirName = os.path.split(os.path.normpath(dirName))[0] - - SpecificPath = os.path.join(dirName, str(inputName)) - cleanName = os.path.splitext(SpecificPath) - if cleanName[1] == ".nzb": - SpecificPath = cleanName[0] - if os.path.isdir(SpecificPath): - dirName = SpecificPath - - process_all_exceptions(inputName, dirName) - inputName, dirName = convert_to_ascii(inputName, dirName) - - if not listMediaFiles(dirName, media=False, audio=True, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True) and extract: - logger.debug('Checking for archives to extract in directory: {0}'.format(dirName)) - core.extractFiles(dirName) - inputName, dirName = convert_to_ascii(inputName, dirName) - - #if listMediaFiles(dirName, media=False, audio=True, meta=False, archives=False) and status: - # logger.info("Status shown as failed from Downloader, but valid video files found. Setting as successful.", section) - # status = 0 - - if status == 0 and section == "HeadPhones": - - params = { - 'apikey': apikey, - 'cmd': "forceProcess", - 'dir': remoteDir(dirName) if remote_path else dirName - } - - res = self.forceProcess(params, url, apikey, inputName, dirName, section, wait_for) - if res[0] in [0, 1]: - return res - - params = { - 'apikey': apikey, - 'cmd': "forceProcess", - 'dir': os.path.split(remoteDir(dirName))[0] if remote_path else os.path.split(dirName)[0] - } - - res = self.forceProcess(params, url, apikey, inputName, dirName, section, wait_for) - if res[0] in [0, 1]: - return res - - # The status hasn't changed. uTorrent can resume seeding now. - logger.warning("The music album does not appear to have changed status after {0} minutes. Please check your Logs".format(wait_for), section) - return [1, "{0}: Failed to post-process - No change in wanted status".format(section)] - - elif status == 0 and section == "Lidarr": - url = "{0}{1}:{2}{3}/api/v1/command".format(protocol, host, port, web_root) - headers = {"X-Api-Key": apikey} - if remote_path: - logger.debug("remote_path: {0}".format(remoteDir(dirName)), section) - data = {"name": "Rename", "path": remoteDir(dirName)} - else: - logger.debug("path: {0}".format(dirName), section) - data = {"name": "Rename", "path": dirName} - data = json.dumps(data) - try: - logger.debug("Opening URL: {0} with data: {1}".format(url, data), section) - r = requests.post(url, data=data, headers=headers, stream=True, verify=False, timeout=(30, 1800)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url), section) - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - - Success = False - Queued = False - Started = False - try: - res = json.loads(r.content) - scan_id = int(res['id']) - logger.debug("Scan started with id: {0}".format(scan_id), section) - Started = True - except Exception as e: - logger.warning("No scan id was returned due to: {0}".format(e), section) - scan_id = None - Started = False - return [1, "{0}: Failed to post-process - Unable to start scan".format(section)] - - n = 0 - params = {} - url = "{0}/{1}".format(url, scan_id) - while n < 6: # set up wait_for minutes to see if command completes.. - time.sleep(10 * wait_for) - command_status = self.command_complete(url, params, headers, section) - if command_status and command_status in ['completed', 'failed']: - break - n += 1 - if command_status: - logger.debug("The Scan command return status: {0}".format(command_status), section) - if not os.path.exists(dirName): - logger.debug("The directory {0} has been removed. Renaming was successful.".format(dirName), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - elif command_status and command_status in ['completed']: - logger.debug("The Scan command has completed successfully. Renaming was successful.", section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - elif command_status and command_status in ['failed']: - logger.debug("The Scan command has failed. Renaming was not successful.", section) - # return [1, "%s: Failed to post-process %s" % (section, inputName) ] - else: - logger.debug("The Scan command did not return status completed. Passing back to {0} to attempt complete download handling.".format(section), section) - return [status, "{0}: Passing back to {1} to attempt Complete Download Handling".format(section, section)] - - else: - if section == "Lidarr": - logger.postprocess("FAILED: The download failed. Sending failed download to {0} for CDH processing".format(section), section) - return [1, "{0}: Download Failed. Sending back to {1}".format(section, section)] # Return as failed to flag this in the downloader. - else: - logger.warning("FAILED DOWNLOAD DETECTED", section) - if delete_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName: - logger.postprocess("Deleting failed files and folder {0}".format(dirName), section) - rmDir(dirName) - return [1, "{0}: Failed to post-process. {1} does not support failed downloads".format(section, section)] # Return as failed to flag this in the downloader. \ No newline at end of file diff --git a/core/autoProcess/autoProcessTV.py b/core/autoProcess/autoProcessTV.py deleted file mode 100644 index b9d92fcd2..000000000 --- a/core/autoProcess/autoProcessTV.py +++ /dev/null @@ -1,373 +0,0 @@ -# coding=utf-8 - -import copy -import os -import time -import errno -import requests -import json -import core - -from core.nzbToMediaAutoFork import autoFork -from core.nzbToMediaSceneExceptions import process_all_exceptions -from core.nzbToMediaUtil import convert_to_ascii, flatten, rmDir, listMediaFiles, remoteDir, import_subs, server_responding, reportNzb -from core import logger -from core.transcoder import transcoder - -requests.packages.urllib3.disable_warnings() - - -class autoProcessTV(object): - def command_complete(self, url, params, headers, section): - try: - r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url), section) - return None - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return None - else: - try: - return r.json()['state'] - except (ValueError, KeyError): - # ValueError catches simplejson's JSONDecodeError and json's ValueError - logger.error("{0} did not return expected json data.".format(section), section) - return None - - def CDH(self, url2, headers, section="MAIN"): - try: - r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url2), section) - return False - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return False - else: - try: - return r.json().get("enableCompletedDownloadHandling", False) - except ValueError: - # ValueError catches simplejson's JSONDecodeError and json's ValueError - return False - - def processEpisode(self, section, dirName, inputName=None, failed=False, clientAgent="manual", download_id=None, inputCategory=None, failureLink=None): - - cfg = dict(core.CFG[section][inputCategory]) - - host = cfg["host"] - port = cfg["port"] - ssl = int(cfg.get("ssl", 0)) - web_root = cfg.get("web_root", "") - protocol = "https://" if ssl else "http://" - username = cfg.get("username", "") - password = cfg.get("password", "") - apikey = cfg.get("apikey", "") - - if server_responding("{0}{1}:{2}{3}".format(protocol, host, port, web_root)): - # auto-detect correct fork - fork, fork_params = autoFork(section, inputCategory) - elif not username and not apikey: - logger.info('No SickBeard username or Sonarr apikey entered. Performing transcoder functions only') - fork, fork_params = "None", {} - else: - logger.error("Server did not respond. Exiting", section) - return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)] - - delete_failed = int(cfg.get("delete_failed", 0)) - nzbExtractionBy = cfg.get("nzbExtractionBy", "Downloader") - process_method = cfg.get("process_method") - if clientAgent == core.TORRENT_CLIENTAGENT and core.USELINK == "move-sym": - process_method = "symlink" - remote_path = int(cfg.get("remote_path", 0)) - wait_for = int(cfg.get("wait_for", 2)) - force = int(cfg.get("force", 0)) - delete_on = int(cfg.get("delete_on", 0)) - ignore_subs = int(cfg.get("ignore_subs", 0)) - status = int(failed) - if status > 0 and core.NOEXTRACTFAILED: - extract = 0 - else: - extract = int(cfg.get("extract", 0)) - #get importmode, default to "Move" for consistency with legacy - importMode = cfg.get("importMode","Move") - - if not os.path.isdir(dirName) and os.path.isfile(dirName): # If the input directory is a file, assume single file download and split dir/name. - dirName = os.path.split(os.path.normpath(dirName))[0] - - SpecificPath = os.path.join(dirName, str(inputName)) - cleanName = os.path.splitext(SpecificPath) - if cleanName[1] == ".nzb": - SpecificPath = cleanName[0] - if os.path.isdir(SpecificPath): - dirName = SpecificPath - - # Attempt to create the directory if it doesn't exist and ignore any - # error stating that it already exists. This fixes a bug where SickRage - # won't process the directory because it doesn't exist. - try: - os.makedirs(dirName) # Attempt to create the directory - except OSError as e: - # Re-raise the error if it wasn't about the directory not existing - if e.errno != errno.EEXIST: - raise - - if 'process_method' not in fork_params or (clientAgent in ['nzbget', 'sabnzbd'] and nzbExtractionBy != "Destination"): - if inputName: - process_all_exceptions(inputName, dirName) - inputName, dirName = convert_to_ascii(inputName, dirName) - - # Now check if tv files exist in destination. - if not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): - if listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True) and extract: - logger.debug('Checking for archives to extract in directory: {0}'.format(dirName)) - core.extractFiles(dirName) - inputName, dirName = convert_to_ascii(inputName, dirName) - - if listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): # Check that a video exists. if not, assume failed. - flatten(dirName) - - # Check video files for corruption - good_files = 0 - num_files = 0 - for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False): - num_files += 1 - if transcoder.isVideoGood(video, status): - good_files += 1 - import_subs(video) - if num_files > 0: - if good_files == num_files and not status == 0: - logger.info('Found Valid Videos. Setting status Success') - status = 0 - failed = 0 - if good_files < num_files and status == 0: - logger.info('Found corrupt videos. Setting status Failed') - status = 1 - failed = 1 - if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': - print('[NZB] MARK=BAD') - if failureLink: - failureLink += '&corrupt=true' - elif clientAgent == "manual": - logger.warning("No media files found in directory {0} to manually process.".format(dirName), section) - return [0, ""] # Success (as far as this script is concerned) - elif nzbExtractionBy == "Destination": - logger.info("Check for media files ignored because nzbExtractionBy is set to Destination.") - if int(failed) == 0: - logger.info("Setting Status Success.") - status = 0 - failed = 0 - else: - logger.info("Downloader reported an error during download or verification. Processing this as a failed download.") - status = 1 - failed = 1 - else: - logger.warning("No media files found in directory {0}. Processing this as a failed download".format(dirName), section) - status = 1 - failed = 1 - if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': - print('[NZB] MARK=BAD') - - if status == 0 and core.TRANSCODE == 1: # only transcode successful downloads - result, newDirName = transcoder.Transcode_directory(dirName) - if result == 0: - logger.debug("SUCCESS: Transcoding succeeded for files in {0}".format(dirName), section) - dirName = newDirName - - chmod_directory = int(str(cfg.get("chmodDirectory", "0")), 8) - logger.debug("Config setting 'chmodDirectory' currently set to {0}".format(oct(chmod_directory)), section) - if chmod_directory: - logger.info("Attempting to set the octal permission of '{0}' on directory '{1}'".format(oct(chmod_directory), dirName), section) - core.rchmod(dirName, chmod_directory) - else: - logger.error("FAILED: Transcoding failed for files in {0}".format(dirName), section) - return [1, "{0}: Failed to post-process - Transcoding failed".format(section)] - - # configure SB params to pass - fork_params['quiet'] = 1 - fork_params['proc_type'] = 'manual' - if inputName is not None: - fork_params['nzbName'] = inputName - - for param in copy.copy(fork_params): - if param == "failed": - fork_params[param] = failed - del fork_params['proc_type'] - if "type" in fork_params: - del fork_params['type'] - - if param == "return_data": - fork_params[param] = 0 - del fork_params['quiet'] - - if param == "type": - fork_params[param] = 'manual' - if "proc_type" in fork_params: - del fork_params['proc_type'] - - if param in ["dirName", "dir", "proc_dir", "process_directory", "path"]: - fork_params[param] = dirName - if remote_path: - fork_params[param] = remoteDir(dirName) - - if param == "process_method": - if process_method: - fork_params[param] = process_method - else: - del fork_params[param] - - if param in ["force", "force_replace"]: - if force: - fork_params[param] = force - else: - del fork_params[param] - - if param in ["delete_on", "delete"]: - if delete_on: - fork_params[param] = delete_on - else: - del fork_params[param] - - if param == "ignore_subs": - if ignore_subs: - fork_params[param] = ignore_subs - else: - del fork_params[param] - - if param == "force_next": - fork_params[param] = 1 - - # delete any unused params so we don't pass them to SB by mistake - [fork_params.pop(k) for k, v in fork_params.items() if v is None] - - if status == 0: - if section == "NzbDrone" and not apikey: - logger.info('No Sonarr apikey entered. Processing completed.') - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - logger.postprocess("SUCCESS: The download succeeded, sending a post-process request", section) - else: - core.FAILED = True - if failureLink: - reportNzb(failureLink, clientAgent) - if 'failed' in fork_params: - logger.postprocess("FAILED: The download failed. Sending 'failed' process request to {0} branch".format(fork), section) - elif section == "NzbDrone": - logger.postprocess("FAILED: The download failed. Sending failed download to {0} for CDH processing".format(fork), section) - return [1, "{0}: Download Failed. Sending back to {1}".format(section, section)] # Return as failed to flag this in the downloader. - else: - logger.postprocess("FAILED: The download failed. {0} branch does not handle failed downloads. Nothing to process".format(fork), section) - if delete_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName: - logger.postprocess("Deleting failed files and folder {0}".format(dirName), section) - rmDir(dirName) - return [1, "{0}: Failed to post-process. {1} does not support failed downloads".format(section, section)] # Return as failed to flag this in the downloader. - - url = None - if section == "SickBeard": - if apikey: - url = "{0}{1}:{2}{3}/api/{4}/?cmd=postprocess".format(protocol, host, port, web_root, apikey) - else: - url = "{0}{1}:{2}{3}/home/postprocess/processEpisode".format(protocol, host, port, web_root) - elif section == "NzbDrone": - url = "{0}{1}:{2}{3}/api/command".format(protocol, host, port, web_root) - url2 = "{0}{1}:{2}{3}/api/config/downloadClient".format(protocol, host, port, web_root) - headers = {"X-Api-Key": apikey} - # params = {'sortKey': 'series.title', 'page': 1, 'pageSize': 1, 'sortDir': 'asc'} - if remote_path: - logger.debug("remote_path: {0}".format(remoteDir(dirName)), section) - data = {"name": "DownloadedEpisodesScan", "path": remoteDir(dirName), "downloadClientId": download_id, "importMode": importMode} - else: - logger.debug("path: {0}".format(dirName), section) - data = {"name": "DownloadedEpisodesScan", "path": dirName, "downloadClientId": download_id, "importMode": importMode} - if not download_id: - data.pop("downloadClientId") - data = json.dumps(data) - - try: - if section == "SickBeard": - logger.debug("Opening URL: {0} with params: {1}".format(url, fork_params), section) - s = requests.Session() - if not apikey and username and password: - login = "{0}{1}:{2}{3}/login".format(protocol, host, port, web_root) - login_params = {'username': username, 'password': password} - r = s.get(login, verify=False, timeout=(30,60)) - if r.status_code == 401 and r.cookies.get('_xsrf'): - login_params['_xsrf'] = r.cookies.get('_xsrf') - s.post(login, data=login_params, stream=True, verify=False, timeout=(30, 60)) - r = s.get(url, auth=(username, password), params=fork_params, stream=True, verify=False, timeout=(30, 1800)) - elif section == "NzbDrone": - logger.debug("Opening URL: {0} with data: {1}".format(url, data), section) - r = requests.post(url, data=data, headers=headers, stream=True, verify=False, timeout=(30, 1800)) - except requests.ConnectionError: - logger.error("Unable to open URL: {0}".format(url), section) - return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)] - - if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: - logger.error("Server returned status {0}".format(r.status_code), section) - return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)] - - Success = False - Queued = False - Started = False - if section == "SickBeard": - if apikey: - if r.json()['result'] == 'success': - Success = True - else: - for line in r.iter_lines(): - if line: - logger.postprocess("{0}".format(line), section) - if "Moving file from" in line: - inputName = os.path.split(line)[1] - if "added to the queue" in line: - Queued = True - if "Processing succeeded" in line or "Successfully processed" in line: - Success = True - - if Queued: - time.sleep(60) - elif section == "NzbDrone": - try: - res = json.loads(r.content) - scan_id = int(res['id']) - logger.debug("Scan started with id: {0}".format(scan_id), section) - Started = True - except Exception as e: - logger.warning("No scan id was returned due to: {0}".format(e), section) - scan_id = None - Started = False - - if status != 0 and delete_failed and not os.path.dirname(dirName) == dirName: - logger.postprocess("Deleting failed files and folder {0}".format(dirName), section) - rmDir(dirName) - - if Success: - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - elif section == "NzbDrone" and Started: - n = 0 - params = {} - url = "{0}/{1}".format(url, scan_id) - while n < 6: # set up wait_for minutes to see if command completes.. - time.sleep(10 * wait_for) - command_status = self.command_complete(url, params, headers, section) - if command_status and command_status in ['completed', 'failed']: - break - n += 1 - if command_status: - logger.debug("The Scan command return status: {0}".format(command_status), section) - if not os.path.exists(dirName): - logger.debug("The directory {0} has been removed. Renaming was successful.".format(dirName), section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - elif command_status and command_status in ['completed']: - logger.debug("The Scan command has completed successfully. Renaming was successful.", section) - return [0, "{0}: Successfully post-processed {1}".format(section, inputName)] - elif command_status and command_status in ['failed']: - logger.debug("The Scan command has failed. Renaming was not successful.", section) - # return [1, "%s: Failed to post-process %s" % (section, inputName) ] - if self.CDH(url2, headers, section=section): - logger.debug("The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {0}.".format(section), section) - return [status, "{0}: Complete DownLoad Handling is enabled. Passing back to {1}".format(section, section)] - else: - logger.warning("The Scan command did not return a valid status. Renaming was not successful.", section) - return [1, "{0}: Failed to post-process {1}".format(section, inputName)] - else: - return [1, "{0}: Failed to post-process - Returned log from {1} was not as expected.".format(section, section)] # We did not receive Success confirmation. diff --git a/libs/dogpile/cache/plugins/__init__.py b/core/auto_process/__init__.py similarity index 100% rename from libs/dogpile/cache/plugins/__init__.py rename to core/auto_process/__init__.py diff --git a/core/auto_process/comics.py b/core/auto_process/comics.py new file mode 100644 index 000000000..7049704bb --- /dev/null +++ b/core/auto_process/comics.py @@ -0,0 +1,92 @@ +# coding=utf-8 + +import os + +import requests + +import core +from core import logger +from core.auto_process.common import ProcessResult +from core.utils import convert_to_ascii, remote_dir, server_responding + +requests.packages.urllib3.disable_warnings() + + +def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None): + apc_version = '2.04' + comicrn_version = '1.01' + + cfg = dict(core.CFG[section][input_category]) + + host = cfg['host'] + port = cfg['port'] + apikey = cfg['apikey'] + ssl = int(cfg.get('ssl', 0)) + web_root = cfg.get('web_root', '') + remote_path = int(cfg.get('remote_path'), 0) + protocol = 'https://' if ssl else 'http://' + + url = '{0}{1}:{2}{3}/api'.format(protocol, host, port, web_root) + if not server_responding(url): + logger.error('Server did not respond. Exiting', section) + return ProcessResult( + message='{0}: Failed to post-process - {0} did not respond.'.format(section), + status_code=1, + ) + + input_name, dir_name = convert_to_ascii(input_name, dir_name) + clean_name, ext = os.path.splitext(input_name) + if len(ext) == 4: # we assume this was a standard extension. + input_name = clean_name + + params = { + 'cmd': 'forceProcess', + 'apikey': apikey, + 'nzb_folder': remote_dir(dir_name) if remote_path else dir_name, + } + + if input_name is not None: + params['nzb_name'] = input_name + params['failed'] = int(status) + params['apc_version'] = apc_version + params['comicrn_version'] = comicrn_version + + success = False + + logger.debug('Opening URL: {0}'.format(url), section) + try: + r = requests.post(url, params=params, stream=True, verify=False, timeout=(30, 300)) + except requests.ConnectionError: + logger.error('Unable to open URL', section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + status_code=1 + ) + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + status_code=1, + ) + + result = r.content + if not type(result) == list: + result = result.split('\n') + for line in result: + if line: + logger.postprocess('{0}'.format(line), section) + if 'Post Processing SUCCESSFUL' in line: + success = True + + if success: + logger.postprocess('SUCCESS: This issue has been processed successfully', section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + else: + logger.warning('The issue does not appear to have successfully processed. Please check your Logs', section) + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(section), + status_code=1, + ) diff --git a/core/auto_process/common.py b/core/auto_process/common.py new file mode 100644 index 000000000..8da834858 --- /dev/null +++ b/core/auto_process/common.py @@ -0,0 +1,62 @@ +import requests + +from core import logger + + +class ProcessResult(object): + def __init__(self, message, status_code): + self.message = message + self.status_code = status_code + + def __iter__(self): + return self.status_code, self.message + + def __bool__(self): + return not bool(self.status_code) + + def __str__(self): + return 'Processing {0}: {1}'.format( + 'succeeded' if bool(self) else 'failed', + self.message + ) + + def __repr__(self): + return ''.format( + self.status_code, + self.message, + ) + + +def command_complete(url, params, headers, section): + try: + r = requests.get(url, params=params, headers=headers, stream=True, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error('Unable to open URL: {0}'.format(url), section) + return None + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return None + else: + try: + return r.json()['state'] + except (ValueError, KeyError): + # ValueError catches simplejson's JSONDecodeError and json's ValueError + logger.error('{0} did not return expected json data.'.format(section), section) + return None + + +def completed_download_handling(url2, headers, section='MAIN'): + try: + r = requests.get(url2, params={}, headers=headers, stream=True, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error('Unable to open URL: {0}'.format(url2), section) + return False + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return False + else: + try: + return r.json().get('enableCompletedDownloadHandling', False) + except ValueError: + # ValueError catches simplejson's JSONDecodeError and json's ValueError + return False diff --git a/core/auto_process/games.py b/core/auto_process/games.py new file mode 100644 index 000000000..c412a690f --- /dev/null +++ b/core/auto_process/games.py @@ -0,0 +1,99 @@ +# coding=utf-8 + +import os +import shutil + +import requests + +import core +from core import logger +from core.auto_process.common import ProcessResult +from core.utils import convert_to_ascii, server_responding + +requests.packages.urllib3.disable_warnings() + + +def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None): + status = int(status) + + cfg = dict(core.CFG[section][input_category]) + + host = cfg['host'] + port = cfg['port'] + apikey = cfg['apikey'] + library = cfg.get('library') + ssl = int(cfg.get('ssl', 0)) + web_root = cfg.get('web_root', '') + protocol = 'https://' if ssl else 'http://' + + url = '{0}{1}:{2}{3}/api'.format(protocol, host, port, web_root) + if not server_responding(url): + logger.error('Server did not respond. Exiting', section) + return ProcessResult( + message='{0}: Failed to post-process - {0} did not respond.'.format(section), + status_code=1, + ) + + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + fields = input_name.split('-') + + gamez_id = fields[0].replace('[', '').replace(']', '').replace(' ', '') + + download_status = 'Downloaded' if status == 0 else 'Wanted' + + params = { + 'api_key': apikey, + 'mode': 'UPDATEREQUESTEDSTATUS', + 'db_id': gamez_id, + 'status': download_status + } + + logger.debug('Opening URL: {0}'.format(url), section) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 300)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {1}'.format(section, section), + status_code=1, + ) + + result = r.json() + logger.postprocess('{0}'.format(result), section) + if library: + logger.postprocess('moving files to library: {0}'.format(library), section) + try: + shutil.move(dir_name, os.path.join(library, input_name)) + except Exception: + logger.error('Unable to move {0} to {1}'.format(dir_name, os.path.join(library, input_name)), section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to move files'.format(section), + status_code=1, + ) + else: + logger.error('No library specified to move files to. Please edit your configuration.', section) + return ProcessResult( + message='{0}: Failed to post-process - No library defined in {0}'.format(section), + status_code=1, + ) + + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + status_code=1, + ) + elif result['success']: + logger.postprocess('SUCCESS: Status for {0} has been set to {1} in Gamez'.format(gamez_id, download_status), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + else: + logger.error('FAILED: Status for {0} has NOT been updated in Gamez'.format(gamez_id), section) + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(section), + status_code=1, + ) diff --git a/core/auto_process/movies.py b/core/auto_process/movies.py new file mode 100644 index 000000000..cde56f27a --- /dev/null +++ b/core/auto_process/movies.py @@ -0,0 +1,506 @@ +# coding=utf-8 + +import json +import os +import time + +import requests + +import core +from core import logger, transcoder +from core.auto_process.common import command_complete, completed_download_handling, ProcessResult +from core.scene_exceptions import process_all_exceptions +from core.utils import convert_to_ascii, find_download, find_imdbid, import_subs, list_media_files, remote_dir, remove_dir, report_nzb, server_responding + +requests.packages.urllib3.disable_warnings() + + +def process(section, dir_name, input_name=None, status=0, client_agent='manual', download_id='', input_category=None, failure_link=None): + + cfg = dict(core.CFG[section][input_category]) + + host = cfg['host'] + port = cfg['port'] + apikey = cfg['apikey'] + if section == 'CouchPotato': + method = cfg['method'] + else: + method = None + # added importMode for Radarr config + if section == 'Radarr': + import_mode = cfg.get('importMode', 'Move') + else: + import_mode = None + delete_failed = int(cfg['delete_failed']) + wait_for = int(cfg['wait_for']) + ssl = int(cfg.get('ssl', 0)) + web_root = cfg.get('web_root', '') + remote_path = int(cfg.get('remote_path', 0)) + protocol = 'https://' if ssl else 'http://' + omdbapikey = cfg.get('omdbapikey', '') + status = int(status) + if status > 0 and core.NOEXTRACTFAILED: + extract = 0 + else: + extract = int(cfg.get('extract', 0)) + + imdbid = find_imdbid(dir_name, input_name, omdbapikey) + if section == 'CouchPotato': + base_url = '{0}{1}:{2}{3}/api/{4}/'.format(protocol, host, port, web_root, apikey) + if section == 'Radarr': + base_url = '{0}{1}:{2}{3}/api/command'.format(protocol, host, port, web_root) + url2 = '{0}{1}:{2}{3}/api/config/downloadClient'.format(protocol, host, port, web_root) + headers = {'X-Api-Key': apikey} + if not apikey: + logger.info('No CouchPotato or Radarr apikey entered. Performing transcoder functions only') + release = None + elif server_responding(base_url): + if section == 'CouchPotato': + release = get_release(base_url, imdbid, download_id) + else: + release = None + else: + logger.error('Server did not respond. Exiting', section) + return ProcessResult( + message='{0}: Failed to post-process - {0} did not respond.'.format(section), + status_code=1, + ) + + # pull info from release found if available + release_id = None + media_id = None + downloader = None + release_status_old = None + if release: + try: + release_id = list(release.keys())[0] + media_id = release[release_id]['media_id'] + download_id = release[release_id]['download_info']['id'] + downloader = release[release_id]['download_info']['downloader'] + release_status_old = release[release_id]['status'] + except Exception: + pass + + if not os.path.isdir(dir_name) and os.path.isfile(dir_name): # If the input directory is a file, assume single file download and split dir/name. + dir_name = os.path.split(os.path.normpath(dir_name))[0] + + specific_path = os.path.join(dir_name, str(input_name)) + clean_name = os.path.splitext(specific_path) + if clean_name[1] == '.nzb': + specific_path = clean_name[0] + if os.path.isdir(specific_path): + dir_name = specific_path + + process_all_exceptions(input_name, dir_name) + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + if not list_media_files(dir_name, media=True, audio=False, meta=False, archives=False) and list_media_files(dir_name, media=False, audio=False, meta=False, archives=True) and extract: + logger.debug('Checking for archives to extract in directory: {0}'.format(dir_name)) + core.extract_files(dir_name) + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + good_files = 0 + num_files = 0 + # Check video files for corruption + for video in list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): + num_files += 1 + if transcoder.is_video_good(video, status): + import_subs(video) + good_files += 1 + if num_files and good_files == num_files: + if status: + logger.info('Status shown as failed from Downloader, but {0} valid video files found. Setting as success.'.format(good_files), section) + status = 0 + elif num_files and good_files < num_files: + logger.info('Status shown as success from Downloader, but corrupt video files found. Setting as failed.', section) + if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': + print('[NZB] MARK=BAD') + if failure_link: + failure_link += '&corrupt=true' + status = 1 + elif client_agent == 'manual': + logger.warning('No media files found in directory {0} to manually process.'.format(dir_name), section) + return ProcessResult( + message='', + status_code=0, # Success (as far as this script is concerned) + ) + else: + logger.warning('No media files found in directory {0}. Processing this as a failed download'.format(dir_name), section) + status = 1 + if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': + print('[NZB] MARK=BAD') + + if status == 0: + if core.TRANSCODE == 1: + result, new_dir_name = transcoder.transcode_directory(dir_name) + if result == 0: + logger.debug('Transcoding succeeded for files in {0}'.format(dir_name), section) + dir_name = new_dir_name + + chmod_directory = int(str(cfg.get('chmodDirectory', '0')), 8) + logger.debug('Config setting \'chmodDirectory\' currently set to {0}'.format(oct(chmod_directory)), section) + if chmod_directory: + logger.info('Attempting to set the octal permission of \'{0}\' on directory \'{1}\''.format(oct(chmod_directory), dir_name), section) + core.rchmod(dir_name, chmod_directory) + else: + logger.error('Transcoding failed for files in {0}'.format(dir_name), section) + return ProcessResult( + message='{0}: Failed to post-process - Transcoding failed'.format(section), + status_code=1, + ) + for video in list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): + if not release and '.cp(tt' not in video and imdbid: + video_name, video_ext = os.path.splitext(video) + video2 = '{0}.cp({1}){2}'.format(video_name, imdbid, video_ext) + if not (client_agent in [core.TORRENT_CLIENTAGENT, 'manual'] and core.USELINK == 'move-sym'): + logger.debug('Renaming: {0} to: {1}'.format(video, video2)) + os.rename(video, video2) + + if not apikey: # If only using Transcoder functions, exit here. + logger.info('No CouchPotato or Radarr apikey entered. Processing completed.') + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + + params = {} + if download_id and release_id: + params['downloader'] = downloader or client_agent + params['download_id'] = download_id + + params['media_folder'] = remote_dir(dir_name) if remote_path else dir_name + + if section == 'CouchPotato': + if method == 'manage': + command = 'manage.update' + params = {} + else: + command = 'renamer.scan' + + url = '{0}{1}'.format(base_url, command) + logger.debug('Opening URL: {0} with PARAMS: {1}'.format(url, params), section) + logger.postprocess('Starting {0} scan for {1}'.format(method, input_name), section) + + if section == 'Radarr': + payload = {'name': 'DownloadedMoviesScan', 'path': params['media_folder'], 'downloadClientId': download_id, 'importMode': import_mode} + if not download_id: + payload.pop('downloadClientId') + logger.debug('Opening URL: {0} with PARAMS: {1}'.format(base_url, payload), section) + logger.postprocess('Starting DownloadedMoviesScan scan for {0}'.format(input_name), section) + + try: + if section == 'CouchPotato': + r = requests.get(url, params=params, verify=False, timeout=(30, 1800)) + else: + r = requests.post(base_url, data=json.dumps(payload), headers=headers, stream=True, verify=False, timeout=(30, 1800)) + except requests.ConnectionError: + logger.error('Unable to open URL', section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + status_code=1, + ) + + result = r.json() + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + status_code=1, + ) + elif section == 'CouchPotato' and result['success']: + logger.postprocess('SUCCESS: Finished {0} scan for folder {1}'.format(method, dir_name), section) + if method == 'manage': + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + elif section == 'Radarr': + logger.postprocess('Radarr response: {0}'.format(result['state'])) + try: + res = json.loads(r.content) + scan_id = int(res['id']) + logger.debug('Scan started with id: {0}'.format(scan_id), section) + started = True + except Exception as e: + logger.warning('No scan id was returned due to: {0}'.format(e), section) + scan_id = None + else: + logger.error('FAILED: {0} scan was unable to finish for folder {1}. exiting!'.format(method, dir_name), + section) + return ProcessResult( + message='{0}: Failed to post-process - Server did not return success'.format(section), + status_code=1, + ) + else: + core.FAILED = True + logger.postprocess('FAILED DOWNLOAD DETECTED FOR {0}'.format(input_name), section) + if failure_link: + report_nzb(failure_link, client_agent) + + if section == 'Radarr': + logger.postprocess('FAILED: The download failed. Sending failed download to {0} for CDH processing'.format(section), section) + return ProcessResult( + message='{0}: Download Failed. Sending back to {0}'.format(section), + status_code=1, # Return as failed to flag this in the downloader. + ) + + if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: + logger.postprocess('Deleting failed files and folder {0}'.format(dir_name), section) + remove_dir(dir_name) + + if not release_id and not media_id: + logger.error('Could not find a downloaded movie in the database matching {0}, exiting!'.format(input_name), + section) + return ProcessResult( + message='{0}: Failed to post-process - Failed download not found in {0}'.format(section), + status_code=1, + ) + + if release_id: + logger.postprocess('Setting failed release {0} to ignored ...'.format(input_name), section) + + url = '{url}release.ignore'.format(url=base_url) + params = {'id': release_id} + + logger.debug('Opening URL: {0} with PARAMS: {1}'.format(url, params), section) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL {0}'.format(url), section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + status_code=1, + ) + + result = r.json() + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + status_code=1, + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + ) + elif result['success']: + logger.postprocess('SUCCESS: {0} has been set to ignored ...'.format(input_name), section) + else: + logger.warning('FAILED: Unable to set {0} to ignored!'.format(input_name), section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to set {1} to ignored'.format(section, input_name), + status_code=1, + ) + + logger.postprocess('Trying to snatch the next highest ranked release.', section) + + url = '{0}movie.searcher.try_next'.format(base_url) + logger.debug('Opening URL: {0}'.format(url), section) + + try: + r = requests.get(url, params={'media_id': media_id}, verify=False, timeout=(30, 600)) + except requests.ConnectionError: + logger.error('Unable to open URL {0}'.format(url), section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + status_code=1, + ) + + result = r.json() + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + status_code=1, + ) + elif result['success']: + logger.postprocess('SUCCESS: Snatched the next highest release ...', section) + return ProcessResult( + message='{0}: Successfully snatched next highest release'.format(section), + status_code=0, + ) + else: + logger.postprocess('SUCCESS: Unable to find a new release to snatch now. CP will keep searching!', section) + return ProcessResult( + status_code=0, + message='{0}: No new release found now. {0} will keep searching'.format(section), + ) + + # Added a release that was not in the wanted list so confirm rename successful by finding this movie media.list. + if not release: + download_id = None # we don't want to filter new releases based on this. + + # we will now check to see if CPS has finished renaming before returning to TorrentToMedia and unpausing. + timeout = time.time() + 60 * wait_for + while time.time() < timeout: # only wait 2 (default) minutes, then return. + logger.postprocess('Checking for status change, please stand by ...', section) + if section == 'CouchPotato': + release = get_release(base_url, imdbid, download_id, release_id) + scan_id = None + else: + release = None + if release: + try: + release_id = list(release.keys())[0] + title = release[release_id]['title'] + release_status_new = release[release_id]['status'] + if release_status_old is None: # we didn't have a release before, but now we do. + logger.postprocess('SUCCESS: Movie {0} has now been added to CouchPotato with release status of [{1}]'.format( + title, str(release_status_new).upper()), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + + if release_status_new != release_status_old: + logger.postprocess('SUCCESS: Release for {0} has now been marked with a status of [{1}]'.format( + title, str(release_status_new).upper()), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + except Exception: + pass + elif scan_id: + url = '{0}/{1}'.format(base_url, scan_id) + command_status = command_complete(url, params, headers, section) + if command_status: + logger.debug('The Scan command return status: {0}'.format(command_status), section) + if command_status in ['completed']: + logger.debug('The Scan command has completed successfully. Renaming was successful.', section) + return [0, '{0}: Successfully post-processed {1}'.format(section, input_name)] + elif command_status in ['failed']: + logger.debug('The Scan command has failed. Renaming was not successful.', section) + # return ProcessResult( + # message='{0}: Failed to post-process {1}'.format(section, input_name), + # status_code=1, + # ) + + if not os.path.isdir(dir_name): + logger.postprocess('SUCCESS: Input Directory [{0}] has been processed and removed'.format( + dir_name), section) + return ProcessResult( + status_code=0, + message='{0}: Successfully post-processed {1}'.format(section, input_name), + ) + + elif not list_media_files(dir_name, media=True, audio=False, meta=False, archives=True): + logger.postprocess('SUCCESS: Input Directory [{0}] has no remaining media files. This has been fully processed.'.format( + dir_name), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + + # pause and let CouchPotatoServer/Radarr catch its breath + time.sleep(10 * wait_for) + + # The status hasn't changed. we have waited wait_for minutes which is more than enough. uTorrent can resume seeding now. + if section == 'Radarr' and completed_download_handling(url2, headers, section=section): + logger.debug('The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {0}.'.format(section), section) + return ProcessResult( + message='{0}: Complete DownLoad Handling is enabled. Passing back to {0}'.format(section), + status_code=status, + ) + logger.warning( + '{0} does not appear to have changed status after {1} minutes, Please check your logs.'.format(input_name, wait_for), + section, + ) + return ProcessResult( + status_code=1, + message='{0}: Failed to post-process - No change in status'.format(section), + ) + + +def get_release(base_url, imdb_id=None, download_id=None, release_id=None): + results = {} + params = {} + + # determine cmd and params to send to CouchPotato to get our results + section = 'movies' + cmd = 'media.list' + if release_id or imdb_id: + section = 'media' + cmd = 'media.get' + params['id'] = release_id or imdb_id + + if not (release_id or imdb_id or download_id): + logger.debug('No information available to filter CP results') + return results + + url = '{0}{1}'.format(base_url, cmd) + logger.debug('Opening URL: {0} with PARAMS: {1}'.format(url, params)) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 60)) + except requests.ConnectionError: + logger.error('Unable to open URL {0}'.format(url)) + return results + + try: + result = r.json() + except ValueError: + # ValueError catches simplejson's JSONDecodeError and json's ValueError + logger.error('CouchPotato returned the following non-json data') + for line in r.iter_lines(): + logger.error('{0}'.format(line)) + return results + + if not result['success']: + if 'error' in result: + logger.error('{0}'.format(result['error'])) + else: + logger.error('no media found for id {0}'.format(params['id'])) + return results + + # Gather release info and return it back, no need to narrow results + if release_id: + try: + cur_id = result[section]['_id'] + results[cur_id] = result[section] + return results + except Exception: + pass + + # Gather release info and proceed with trying to narrow results to one release choice + + movies = result[section] + if not isinstance(movies, list): + movies = [movies] + for movie in movies: + if movie['status'] not in ['active', 'done']: + continue + releases = movie['releases'] + if not releases: + continue + for release in releases: + try: + if release['status'] not in ['snatched', 'downloaded', 'done']: + continue + if download_id: + if download_id.lower() != release['download_info']['id'].lower(): + continue + + cur_id = release['_id'] + results[cur_id] = release + results[cur_id]['title'] = movie['title'] + except Exception: + continue + + # Narrow results by removing old releases by comparing their last_edit field + if len(results) > 1: + for id1, x1 in results.items(): + for id2, x2 in results.items(): + try: + if x2['last_edit'] > x1['last_edit']: + results.pop(id1) + except Exception: + continue + + # Search downloads on clients for a match to try and narrow our results down to 1 + if len(results) > 1: + for cur_id, x in results.items(): + try: + if not find_download(str(x['download_info']['downloader']).lower(), x['download_info']['id']): + results.pop(cur_id) + except Exception: + continue + + return results diff --git a/core/auto_process/music.py b/core/auto_process/music.py new file mode 100644 index 000000000..9c5f70482 --- /dev/null +++ b/core/auto_process/music.py @@ -0,0 +1,272 @@ +# coding=utf-8 + +import json +import os +import time + +import requests + +import core +from core import logger +from core.auto_process.common import command_complete, ProcessResult +from core.scene_exceptions import process_all_exceptions +from core.utils import convert_to_ascii, list_media_files, remote_dir, remove_dir, server_responding + +requests.packages.urllib3.disable_warnings() + + +def process(section, dir_name, input_name=None, status=0, client_agent='manual', input_category=None): + status = int(status) + + cfg = dict(core.CFG[section][input_category]) + + host = cfg['host'] + port = cfg['port'] + apikey = cfg['apikey'] + wait_for = int(cfg['wait_for']) + ssl = int(cfg.get('ssl', 0)) + delete_failed = int(cfg['delete_failed']) + web_root = cfg.get('web_root', '') + remote_path = int(cfg.get('remote_path', 0)) + protocol = 'https://' if ssl else 'http://' + status = int(status) + if status > 0 and core.NOEXTRACTFAILED: + extract = 0 + else: + extract = int(cfg.get('extract', 0)) + + if section == 'Lidarr': + url = '{0}{1}:{2}{3}/api/v1'.format(protocol, host, port, web_root) + else: + url = '{0}{1}:{2}{3}/api'.format(protocol, host, port, web_root) + if not server_responding(url): + logger.error('Server did not respond. Exiting', section) + return ProcessResult( + message='{0}: Failed to post-process - {0} did not respond.'.format(section), + status_code=1, + ) + + if not os.path.isdir(dir_name) and os.path.isfile(dir_name): # If the input directory is a file, assume single file download and split dir/name. + dir_name = os.path.split(os.path.normpath(dir_name))[0] + + specific_path = os.path.join(dir_name, str(input_name)) + clean_name = os.path.splitext(specific_path) + if clean_name[1] == '.nzb': + specific_path = clean_name[0] + if os.path.isdir(specific_path): + dir_name = specific_path + + process_all_exceptions(input_name, dir_name) + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + if not list_media_files(dir_name, media=False, audio=True, meta=False, archives=False) and list_media_files(dir_name, media=False, audio=False, meta=False, archives=True) and extract: + logger.debug('Checking for archives to extract in directory: {0}'.format(dir_name)) + core.extract_files(dir_name) + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + # if listMediaFiles(dir_name, media=False, audio=True, meta=False, archives=False) and status: + # logger.info('Status shown as failed from Downloader, but valid video files found. Setting as successful.', section) + # status = 0 + + if status == 0 and section == 'HeadPhones': + + params = { + 'apikey': apikey, + 'cmd': 'forceProcess', + 'dir': remote_dir(dir_name) if remote_path else dir_name + } + + res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) + if res[0] in [0, 1]: + return res + + params = { + 'apikey': apikey, + 'cmd': 'forceProcess', + 'dir': os.path.split(remote_dir(dir_name))[0] if remote_path else os.path.split(dir_name)[0] + } + + res = force_process(params, url, apikey, input_name, dir_name, section, wait_for) + if res.status_code in [0, 1]: + return res + + # The status hasn't changed. uTorrent can resume seeding now. + logger.warning('The music album does not appear to have changed status after {0} minutes. Please check your Logs'.format(wait_for), section) + return ProcessResult( + message='{0}: Failed to post-process - No change in wanted status'.format(section), + status_code=1, + ) + + elif status == 0 and section == 'Lidarr': + url = '{0}{1}:{2}{3}/api/v1/command'.format(protocol, host, port, web_root) + headers = {'X-Api-Key': apikey} + if remote_path: + logger.debug('remote_path: {0}'.format(remote_dir(dir_name)), section) + data = {'name': 'Rename', 'path': remote_dir(dir_name)} + else: + logger.debug('path: {0}'.format(dir_name), section) + data = {'name': 'Rename', 'path': dir_name} + data = json.dumps(data) + try: + logger.debug('Opening URL: {0} with data: {1}'.format(url, data), section) + r = requests.post(url, data=data, headers=headers, stream=True, verify=False, timeout=(30, 1800)) + except requests.ConnectionError: + logger.error('Unable to open URL: {0}'.format(url), section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + status_code=1, + ) + + success = False + queued = False + started = False + try: + res = json.loads(r.content) + scan_id = int(res['id']) + logger.debug('Scan started with id: {0}'.format(scan_id), section) + started = True + except Exception as e: + logger.warning('No scan id was returned due to: {0}'.format(e), section) + scan_id = None + started = False + return ProcessResult( + message='{0}: Failed to post-process - Unable to start scan'.format(section), + status_code=1, + ) + + n = 0 + params = {} + url = '{0}/{1}'.format(url, scan_id) + while n < 6: # set up wait_for minutes to see if command completes.. + time.sleep(10 * wait_for) + command_status = command_complete(url, params, headers, section) + if command_status and command_status in ['completed', 'failed']: + break + n += 1 + if command_status: + logger.debug('The Scan command return status: {0}'.format(command_status), section) + if not os.path.exists(dir_name): + logger.debug('The directory {0} has been removed. Renaming was successful.'.format(dir_name), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + elif command_status and command_status in ['completed']: + logger.debug('The Scan command has completed successfully. Renaming was successful.', section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + elif command_status and command_status in ['failed']: + logger.debug('The Scan command has failed. Renaming was not successful.', section) + # return ProcessResult( + # message='{0}: Failed to post-process {1}'.format(section, input_name), + # status_code=1, + # ) + else: + logger.debug('The Scan command did not return status completed. Passing back to {0} to attempt complete download handling.'.format(section), section) + return ProcessResult( + message='{0}: Passing back to {0} to attempt Complete Download Handling'.format(section), + status_code=status, + ) + + else: + if section == 'Lidarr': + logger.postprocess('FAILED: The download failed. Sending failed download to {0} for CDH processing'.format(section), section) + return ProcessResult( + message='{0}: Download Failed. Sending back to {0}'.format(section), + status_code=1, # Return as failed to flag this in the downloader. + ) + else: + logger.warning('FAILED DOWNLOAD DETECTED', section) + if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: + logger.postprocess('Deleting failed files and folder {0}'.format(dir_name), section) + remove_dir(dir_name) + return ProcessResult( + message='{0}: Failed to post-process. {0} does not support failed downloads'.format(section), + status_code=1, # Return as failed to flag this in the downloader. + ) + + +def get_status(url, apikey, dir_name): + logger.debug('Attempting to get current status for release:{0}'.format(os.path.basename(dir_name))) + + params = { + 'apikey': apikey, + 'cmd': 'getHistory' + } + + logger.debug('Opening URL: {0} with PARAMS: {1}'.format(url, params)) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.RequestException: + logger.error('Unable to open URL') + return None + + try: + result = r.json() + except ValueError: + # ValueError catches simplejson's JSONDecodeError and json's ValueError + return None + + for album in result: + if os.path.basename(dir_name) == album['FolderName']: + return album['Status'].lower() + + +def force_process(params, url, apikey, input_name, dir_name, section, wait_for): + release_status = get_status(url, apikey, dir_name) + if not release_status: + logger.error('Could not find a status for {0}, is it in the wanted list ?'.format(input_name), section) + + logger.debug('Opening URL: {0} with PARAMS: {1}'.format(url, params), section) + + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 300)) + except requests.ConnectionError: + logger.error('Unable to open URL {0}'.format(url), section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + status_code=1, + ) + + logger.debug('Result: {0}'.format(r.text), section) + + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + status_code=1, + ) + elif r.text == 'OK': + logger.postprocess('SUCCESS: Post-Processing started for {0} in folder {1} ...'.format(input_name, dir_name), section) + else: + logger.error('FAILED: Post-Processing has NOT started for {0} in folder {1}. exiting!'.format(input_name, dir_name), section) + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(section), + status_code=1, + ) + + # we will now wait for this album to be processed before returning to TorrentToMedia and unpausing. + timeout = time.time() + 60 * wait_for + while time.time() < timeout: + current_status = get_status(url, apikey, dir_name) + if current_status is not None and current_status != release_status: # Something has changed. CPS must have processed this movie. + logger.postprocess('SUCCESS: This release is now marked as status [{0}]'.format(current_status), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + if not os.path.isdir(dir_name): + logger.postprocess('SUCCESS: The input directory {0} has been removed Processing must have finished.'.format(dir_name), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + time.sleep(10 * wait_for) + # The status hasn't changed. + return ProcessResult( + message='no change', + status_code=2, + ) diff --git a/core/auto_process/tv.py b/core/auto_process/tv.py new file mode 100644 index 000000000..8abfd0dfb --- /dev/null +++ b/core/auto_process/tv.py @@ -0,0 +1,388 @@ +# coding=utf-8 + +import copy +import errno +import json +import os +import time + +import requests + +import core +from core import logger, transcoder +from core.auto_process.common import command_complete, completed_download_handling, ProcessResult +from core.forks import auto_fork +from core.scene_exceptions import process_all_exceptions +from core.utils import convert_to_ascii, flatten, import_subs, list_media_files, remote_dir, remove_dir, report_nzb, server_responding + +requests.packages.urllib3.disable_warnings() + + +def process(section, dir_name, input_name=None, failed=False, client_agent='manual', download_id=None, input_category=None, failure_link=None): + + cfg = dict(core.CFG[section][input_category]) + + host = cfg['host'] + port = cfg['port'] + ssl = int(cfg.get('ssl', 0)) + web_root = cfg.get('web_root', '') + protocol = 'https://' if ssl else 'http://' + username = cfg.get('username', '') + password = cfg.get('password', '') + apikey = cfg.get('apikey', '') + + if server_responding('{0}{1}:{2}{3}'.format(protocol, host, port, web_root)): + # auto-detect correct fork + fork, fork_params = auto_fork(section, input_category) + elif not username and not apikey: + logger.info('No SickBeard username or Sonarr apikey entered. Performing transcoder functions only') + fork, fork_params = 'None', {} + else: + logger.error('Server did not respond. Exiting', section) + return ProcessResult( + status_code=1, + message='{0}: Failed to post-process - {0} did not respond.'.format(section), + ) + + delete_failed = int(cfg.get('delete_failed', 0)) + nzb_extraction_by = cfg.get('nzbExtractionBy', 'Downloader') + process_method = cfg.get('process_method') + if client_agent == core.TORRENT_CLIENTAGENT and core.USELINK == 'move-sym': + process_method = 'symlink' + remote_path = int(cfg.get('remote_path', 0)) + wait_for = int(cfg.get('wait_for', 2)) + force = int(cfg.get('force', 0)) + delete_on = int(cfg.get('delete_on', 0)) + ignore_subs = int(cfg.get('ignore_subs', 0)) + status = int(failed) + if status > 0 and core.NOEXTRACTFAILED: + extract = 0 + else: + extract = int(cfg.get('extract', 0)) + # get importmode, default to 'Move' for consistency with legacy + import_mode = cfg.get('importMode', 'Move') + + if not os.path.isdir(dir_name) and os.path.isfile(dir_name): # If the input directory is a file, assume single file download and split dir/name. + dir_name = os.path.split(os.path.normpath(dir_name))[0] + + specific_path = os.path.join(dir_name, str(input_name)) + clean_name = os.path.splitext(specific_path) + if clean_name[1] == '.nzb': + specific_path = clean_name[0] + if os.path.isdir(specific_path): + dir_name = specific_path + + # Attempt to create the directory if it doesn't exist and ignore any + # error stating that it already exists. This fixes a bug where SickRage + # won't process the directory because it doesn't exist. + try: + os.makedirs(dir_name) # Attempt to create the directory + except OSError as e: + # Re-raise the error if it wasn't about the directory not existing + if e.errno != errno.EEXIST: + raise + + if 'process_method' not in fork_params or (client_agent in ['nzbget', 'sabnzbd'] and nzb_extraction_by != 'Destination'): + if input_name: + process_all_exceptions(input_name, dir_name) + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + # Now check if tv files exist in destination. + if not list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): + if list_media_files(dir_name, media=False, audio=False, meta=False, archives=True) and extract: + logger.debug('Checking for archives to extract in directory: {0}'.format(dir_name)) + core.extract_files(dir_name) + input_name, dir_name = convert_to_ascii(input_name, dir_name) + + if list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): # Check that a video exists. if not, assume failed. + flatten(dir_name) + + # Check video files for corruption + good_files = 0 + num_files = 0 + for video in list_media_files(dir_name, media=True, audio=False, meta=False, archives=False): + num_files += 1 + if transcoder.is_video_good(video, status): + good_files += 1 + import_subs(video) + if num_files > 0: + if good_files == num_files and not status == 0: + logger.info('Found Valid Videos. Setting status Success') + status = 0 + failed = 0 + if good_files < num_files and status == 0: + logger.info('Found corrupt videos. Setting status Failed') + status = 1 + failed = 1 + if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': + print('[NZB] MARK=BAD') + if failure_link: + failure_link += '&corrupt=true' + elif client_agent == 'manual': + logger.warning('No media files found in directory {0} to manually process.'.format(dir_name), section) + return ProcessResult( + message='', + status_code=0, # Success (as far as this script is concerned) + ) + elif nzb_extraction_by == 'Destination': + logger.info('Check for media files ignored because nzbExtractionBy is set to Destination.') + if int(failed) == 0: + logger.info('Setting Status Success.') + status = 0 + failed = 0 + else: + logger.info('Downloader reported an error during download or verification. Processing this as a failed download.') + status = 1 + failed = 1 + else: + logger.warning('No media files found in directory {0}. Processing this as a failed download'.format(dir_name), section) + status = 1 + failed = 1 + if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0': + print('[NZB] MARK=BAD') + + if status == 0 and core.TRANSCODE == 1: # only transcode successful downloads + result, new_dir_name = transcoder.transcode_directory(dir_name) + if result == 0: + logger.debug('SUCCESS: Transcoding succeeded for files in {0}'.format(dir_name), section) + dir_name = new_dir_name + + chmod_directory = int(str(cfg.get('chmodDirectory', '0')), 8) + logger.debug('Config setting \'chmodDirectory\' currently set to {0}'.format(oct(chmod_directory)), section) + if chmod_directory: + logger.info('Attempting to set the octal permission of \'{0}\' on directory \'{1}\''.format(oct(chmod_directory), dir_name), section) + core.rchmod(dir_name, chmod_directory) + else: + logger.error('FAILED: Transcoding failed for files in {0}'.format(dir_name), section) + return ProcessResult( + message='{0}: Failed to post-process - Transcoding failed'.format(section), + status_code=1, + ) + + # configure SB params to pass + fork_params['quiet'] = 1 + fork_params['proc_type'] = 'manual' + if input_name is not None: + fork_params['nzbName'] = input_name + + for param in copy.copy(fork_params): + if param == 'failed': + fork_params[param] = failed + del fork_params['proc_type'] + if 'type' in fork_params: + del fork_params['type'] + + if param == 'return_data': + fork_params[param] = 0 + del fork_params['quiet'] + + if param == 'type': + fork_params[param] = 'manual' + if 'proc_type' in fork_params: + del fork_params['proc_type'] + + if param in ['dir_name', 'dir', 'proc_dir', 'process_directory', 'path']: + fork_params[param] = dir_name + if remote_path: + fork_params[param] = remote_dir(dir_name) + + if param == 'process_method': + if process_method: + fork_params[param] = process_method + else: + del fork_params[param] + + if param in ['force', 'force_replace']: + if force: + fork_params[param] = force + else: + del fork_params[param] + + if param in ['delete_on', 'delete']: + if delete_on: + fork_params[param] = delete_on + else: + del fork_params[param] + + if param == 'ignore_subs': + if ignore_subs: + fork_params[param] = ignore_subs + else: + del fork_params[param] + + if param == 'force_next': + fork_params[param] = 1 + + # delete any unused params so we don't pass them to SB by mistake + [fork_params.pop(k) for k, v in fork_params.items() if v is None] + + if status == 0: + if section == 'NzbDrone' and not apikey: + logger.info('No Sonarr apikey entered. Processing completed.') + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + logger.postprocess('SUCCESS: The download succeeded, sending a post-process request', section) + else: + core.FAILED = True + if failure_link: + report_nzb(failure_link, client_agent) + if 'failed' in fork_params: + logger.postprocess('FAILED: The download failed. Sending \'failed\' process request to {0} branch'.format(fork), section) + elif section == 'NzbDrone': + logger.postprocess('FAILED: The download failed. Sending failed download to {0} for CDH processing'.format(fork), section) + return ProcessResult( + message='{0}: Download Failed. Sending back to {0}'.format(section), + status_code=1, # Return as failed to flag this in the downloader. + ) + else: + logger.postprocess('FAILED: The download failed. {0} branch does not handle failed downloads. Nothing to process'.format(fork), section) + if delete_failed and os.path.isdir(dir_name) and not os.path.dirname(dir_name) == dir_name: + logger.postprocess('Deleting failed files and folder {0}'.format(dir_name), section) + remove_dir(dir_name) + return ProcessResult( + message='{0}: Failed to post-process. {0} does not support failed downloads'.format(section), + status_code=1, # Return as failed to flag this in the downloader. + ) + + url = None + if section == 'SickBeard': + if apikey: + url = '{0}{1}:{2}{3}/api/{4}/?cmd=postprocess'.format(protocol, host, port, web_root, apikey) + elif fork == 'Stheno': + url = "{0}{1}:{2}{3}/home/postprocess/process_episode".format(protocol, host, port, web_root) + else: + url = '{0}{1}:{2}{3}/home/postprocess/processEpisode'.format(protocol, host, port, web_root) + elif section == 'NzbDrone': + url = '{0}{1}:{2}{3}/api/command'.format(protocol, host, port, web_root) + url2 = '{0}{1}:{2}{3}/api/config/downloadClient'.format(protocol, host, port, web_root) + headers = {'X-Api-Key': apikey} + # params = {'sortKey': 'series.title', 'page': 1, 'pageSize': 1, 'sortDir': 'asc'} + if remote_path: + logger.debug('remote_path: {0}'.format(remote_dir(dir_name)), section) + data = {'name': 'DownloadedEpisodesScan', 'path': remote_dir(dir_name), 'downloadClientId': download_id, 'importMode': import_mode} + else: + logger.debug('path: {0}'.format(dir_name), section) + data = {'name': 'DownloadedEpisodesScan', 'path': dir_name, 'downloadClientId': download_id, 'importMode': import_mode} + if not download_id: + data.pop('downloadClientId') + data = json.dumps(data) + + try: + if section == 'SickBeard': + logger.debug('Opening URL: {0} with params: {1}'.format(url, fork_params), section) + s = requests.Session() + if not apikey and username and password: + login = '{0}{1}:{2}{3}/login'.format(protocol, host, port, web_root) + login_params = {'username': username, 'password': password} + r = s.get(login, verify=False, timeout=(30, 60)) + if r.status_code == 401 and r.cookies.get('_xsrf'): + login_params['_xsrf'] = r.cookies.get('_xsrf') + s.post(login, data=login_params, stream=True, verify=False, timeout=(30, 60)) + r = s.get(url, auth=(username, password), params=fork_params, stream=True, verify=False, timeout=(30, 1800)) + elif section == 'NzbDrone': + logger.debug('Opening URL: {0} with data: {1}'.format(url, data), section) + r = requests.post(url, data=data, headers=headers, stream=True, verify=False, timeout=(30, 1800)) + except requests.ConnectionError: + logger.error('Unable to open URL: {0}'.format(url), section) + return ProcessResult( + message='{0}: Failed to post-process - Unable to connect to {0}'.format(section), + status_code=1, + ) + + if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]: + logger.error('Server returned status {0}'.format(r.status_code), section) + return ProcessResult( + message='{0}: Failed to post-process - Server returned status {1}'.format(section, r.status_code), + status_code=1, + ) + + success = False + queued = False + started = False + if section == 'SickBeard': + if apikey: + if r.json()['result'] == 'success': + success = True + else: + for line in r.iter_lines(): + if line: + line = line.decode('utf-8') + logger.postprocess('{0}'.format(line), section) + if 'Moving file from' in line: + input_name = os.path.split(line)[1] + if 'added to the queue' in line: + queued = True + if 'Processing succeeded' in line or 'Successfully processed' in line: + success = True + + if queued: + time.sleep(60) + elif section == 'NzbDrone': + try: + res = json.loads(r.content) + scan_id = int(res['id']) + logger.debug('Scan started with id: {0}'.format(scan_id), section) + started = True + except Exception as e: + logger.warning('No scan id was returned due to: {0}'.format(e), section) + scan_id = None + started = False + + if status != 0 and delete_failed and not os.path.dirname(dir_name) == dir_name: + logger.postprocess('Deleting failed files and folder {0}'.format(dir_name), section) + remove_dir(dir_name) + + if success: + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + elif section == 'NzbDrone' and started: + n = 0 + params = {} + url = '{0}/{1}'.format(url, scan_id) + while n < 6: # set up wait_for minutes to see if command completes.. + time.sleep(10 * wait_for) + command_status = command_complete(url, params, headers, section) + if command_status and command_status in ['completed', 'failed']: + break + n += 1 + if command_status: + logger.debug('The Scan command return status: {0}'.format(command_status), section) + if not os.path.exists(dir_name): + logger.debug('The directory {0} has been removed. Renaming was successful.'.format(dir_name), section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + elif command_status and command_status in ['completed']: + logger.debug('The Scan command has completed successfully. Renaming was successful.', section) + return ProcessResult( + message='{0}: Successfully post-processed {1}'.format(section, input_name), + status_code=0, + ) + elif command_status and command_status in ['failed']: + logger.debug('The Scan command has failed. Renaming was not successful.', section) + # return ProcessResult( + # message='{0}: Failed to post-process {1}'.format(section, input_name), + # status_code=1, + # ) + if completed_download_handling(url2, headers, section=section): + logger.debug('The Scan command did not return status completed, but complete Download Handling is enabled. Passing back to {0}.'.format(section), section) + return ProcessResult( + message='{0}: Complete DownLoad Handling is enabled. Passing back to {0}'.format(section), + status_code=status, + ) + else: + logger.warning('The Scan command did not return a valid status. Renaming was not successful.', section) + return ProcessResult( + message='{0}: Failed to post-process {1}'.format(section, input_name), + status_code=1, + ) + else: + return ProcessResult( + message='{0}: Failed to post-process - Returned log from {0} was not as expected.'.format(section), + status_code=1, # We did not receive Success confirmation. + ) diff --git a/core/configuration.py b/core/configuration.py new file mode 100644 index 000000000..31ea48524 --- /dev/null +++ b/core/configuration.py @@ -0,0 +1,535 @@ +# coding=utf-8 + +import copy +import os +import shutil +from itertools import chain + +import configobj +from six import iteritems + +import core +from core import logger + + +class Section(configobj.Section, object): + def isenabled(section): + # checks if subsection enabled, returns true/false if subsection specified otherwise returns true/false in {} + if not section.sections: + try: + value = list(ConfigObj.find_key(section, 'enabled'))[0] + except Exception: + value = 0 + if int(value) == 1: + return section + else: + to_return = copy.deepcopy(section) + for section_name, subsections in to_return.items(): + for subsection in subsections: + try: + value = list(ConfigObj.find_key(subsections, 'enabled'))[0] + except Exception: + value = 0 + + if int(value) != 1: + del to_return[section_name][subsection] + + # cleanout empty sections and subsections + for key in [k for (k, v) in to_return.items() if not v]: + del to_return[key] + + return to_return + + def findsection(section, key): + to_return = copy.deepcopy(section) + for subsection in to_return: + try: + value = list(ConfigObj.find_key(to_return[subsection], key))[0] + except Exception: + value = None + + if not value: + del to_return[subsection] + else: + for category in to_return[subsection]: + if category != key: + del to_return[subsection][category] + + # cleanout empty sections and subsections + for key in [k for (k, v) in to_return.items() if not v]: + del to_return[key] + + return to_return + + def __getitem__(self, key): + if key in self.keys(): + return dict.__getitem__(self, key) + + to_return = copy.deepcopy(self) + for section, subsections in to_return.items(): + if section in key: + continue + if isinstance(subsections, Section) and subsections.sections: + for subsection, options in subsections.items(): + if subsection in key: + continue + if key in options: + return options[key] + + del subsections[subsection] + else: + if section not in key: + del to_return[section] + + # cleanout empty sections and subsections + for key in [k for (k, v) in to_return.items() if not v]: + del to_return[key] + + return to_return + + +class ConfigObj(configobj.ConfigObj, Section): + def __init__(self, *args, **kw): + if len(args) == 0: + args = (core.CONFIG_FILE,) + super(configobj.ConfigObj, self).__init__(*args, **kw) + self.interpolation = False + + @staticmethod + def find_key(node, kv): + if isinstance(node, list): + for i in node: + for x in ConfigObj.find_key(i, kv): + yield x + elif isinstance(node, dict): + if kv in node: + yield node[kv] + for j in node.values(): + for x in ConfigObj.find_key(j, kv): + yield x + + @staticmethod + def migrate(): + global CFG_NEW, CFG_OLD + CFG_NEW = None + CFG_OLD = None + + try: + # check for autoProcessMedia.cfg and create if it does not exist + if not os.path.isfile(core.CONFIG_FILE): + shutil.copyfile(core.CONFIG_SPEC_FILE, core.CONFIG_FILE) + CFG_OLD = config(core.CONFIG_FILE) + except Exception as error: + logger.debug('Error {msg} when copying to .cfg'.format(msg=error)) + + try: + # check for autoProcessMedia.cfg.spec and create if it does not exist + if not os.path.isfile(core.CONFIG_SPEC_FILE): + shutil.copyfile(core.CONFIG_FILE, core.CONFIG_SPEC_FILE) + CFG_NEW = config(core.CONFIG_SPEC_FILE) + except Exception as error: + logger.debug('Error {msg} when copying to .spec'.format(msg=error)) + + # check for autoProcessMedia.cfg and autoProcessMedia.cfg.spec and if they don't exist return and fail + if CFG_NEW is None or CFG_OLD is None: + return False + + subsections = {} + # gather all new-style and old-style sub-sections + for newsection, newitems in CFG_NEW.items(): + if CFG_NEW[newsection].sections: + subsections.update({newsection: CFG_NEW[newsection].sections}) + for section, items in CFG_OLD.items(): + if CFG_OLD[section].sections: + subsections.update({section: CFG_OLD[section].sections}) + for option, value in CFG_OLD[section].items(): + if option in ['category', 'cpsCategory', 'sbCategory', 'hpCategory', 'mlCategory', 'gzCategory', 'raCategory', 'ndCategory']: + if not isinstance(value, list): + value = [value] + + # add subsection + subsections.update({section: value}) + CFG_OLD[section].pop(option) + continue + + def cleanup_values(values, section): + for option, value in iteritems(values): + if section in ['CouchPotato']: + if option == ['outputDirectory']: + CFG_NEW['Torrent'][option] = os.path.split(os.path.normpath(value))[0] + values.pop(option) + if section in ['CouchPotato', 'HeadPhones', 'Gamez', 'Mylar']: + if option in ['username', 'password']: + values.pop(option) + if section in ['SickBeard', 'Mylar']: + if option == 'wait_for': # remove old format + values.pop(option) + if section in ['SickBeard', 'NzbDrone']: + if option == 'failed_fork': # change this old format + values['failed'] = 'auto' + values.pop(option) + if option == 'outputDirectory': # move this to new location format + CFG_NEW['Torrent'][option] = os.path.split(os.path.normpath(value))[0] + values.pop(option) + if section in ['Torrent']: + if option in ['compressedExtensions', 'mediaExtensions', 'metaExtensions', 'minSampleSize']: + CFG_NEW['Extensions'][option] = value + values.pop(option) + if option == 'useLink': # Sym links supported now as well. + if value in ['1', 1]: + value = 'hard' + elif value in ['0', 0]: + value = 'no' + values[option] = value + if option == 'forceClean': + CFG_NEW['General']['force_clean'] = value + values.pop(option) + if section in ['Transcoder']: + if option in ['niceness']: + CFG_NEW['Posix'][option] = value + values.pop(option) + if option == 'remote_path': + if value and value not in ['0', '1', 0, 1]: + value = 1 + elif not value: + value = 0 + values[option] = value + # remove any options that we no longer need so they don't migrate into our new config + if not list(ConfigObj.find_key(CFG_NEW, option)): + try: + values.pop(option) + except Exception: + pass + + return values + + def process_section(section, subsections=None): + if subsections: + for subsection in subsections: + if subsection in CFG_OLD.sections: + values = cleanup_values(CFG_OLD[subsection], section) + if subsection not in CFG_NEW[section].sections: + CFG_NEW[section][subsection] = {} + for option, value in values.items(): + CFG_NEW[section][subsection][option] = value + elif subsection in CFG_OLD[section].sections: + values = cleanup_values(CFG_OLD[section][subsection], section) + if subsection not in CFG_NEW[section].sections: + CFG_NEW[section][subsection] = {} + for option, value in values.items(): + CFG_NEW[section][subsection][option] = value + else: + values = cleanup_values(CFG_OLD[section], section) + if section not in CFG_NEW.sections: + CFG_NEW[section] = {} + for option, value in values.items(): + CFG_NEW[section][option] = value + + # convert old-style categories to new-style sub-sections + for section in CFG_OLD.keys(): + subsection = None + if section in list(chain.from_iterable(subsections.values())): + subsection = section + section = ''.join([k for k, v in iteritems(subsections) if subsection in v]) + process_section(section, subsection) + elif section in subsections.keys(): + subsection = subsections[section] + process_section(section, subsection) + elif section in CFG_OLD.keys(): + process_section(section, subsection) + + # create a backup of our old config + CFG_OLD.filename = '{config}.old'.format(config=core.CONFIG_FILE) + CFG_OLD.write() + + # write our new config to autoProcessMedia.cfg + CFG_NEW.filename = core.CONFIG_FILE + CFG_NEW.write() + + return True + + @staticmethod + def addnzbget(): + # load configs into memory + cfg_new = config() + + try: + if 'NZBPO_NDCATEGORY' in os.environ and 'NZBPO_SBCATEGORY' in os.environ: + if os.environ['NZBPO_NDCATEGORY'] == os.environ['NZBPO_SBCATEGORY']: + logger.warning('{x} category is set for SickBeard and Sonarr. ' + 'Please check your config in NZBGet'.format + (x=os.environ['NZBPO_NDCATEGORY'])) + if 'NZBPO_RACATEGORY' in os.environ and 'NZBPO_CPSCATEGORY' in os.environ: + if os.environ['NZBPO_RACATEGORY'] == os.environ['NZBPO_CPSCATEGORY']: + logger.warning('{x} category is set for CouchPotato and Radarr. ' + 'Please check your config in NZBGet'.format + (x=os.environ['NZBPO_RACATEGORY'])) + if 'NZBPO_LICATEGORY' in os.environ and 'NZBPO_HPCATEGORY' in os.environ: + if os.environ['NZBPO_LICATEGORY'] == os.environ['NZBPO_HPCATEGORY']: + logger.warning('{x} category is set for HeadPhones and Lidarr. ' + 'Please check your config in NZBGet'.format + (x=os.environ['NZBPO_LICATEGORY'])) + section = 'Nzb' + key = 'NZBOP_DESTDIR' + if key in os.environ: + option = 'default_downloadDirectory' + value = os.environ[key] + cfg_new[section][option] = value + + section = 'General' + env_keys = ['AUTO_UPDATE', 'CHECK_MEDIA', 'SAFE_MODE', 'NO_EXTRACT_FAILED'] + cfg_keys = ['auto_update', 'check_media', 'safe_mode', 'no_extract_failed'] + for index in range(len(env_keys)): + key = 'NZBPO_{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + cfg_new[section][option] = value + + section = 'Network' + env_keys = ['MOUNTPOINTS'] + cfg_keys = ['mount_points'] + for index in range(len(env_keys)): + key = 'NZBPO_{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + cfg_new[section][option] = value + + section = 'CouchPotato' + env_cat_key = 'NZBPO_CPSCATEGORY' + env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'METHOD', 'DELETE_FAILED', 'REMOTE_PATH', + 'WAIT_FOR', 'WATCH_DIR', 'OMDBAPIKEY'] + cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'method', 'delete_failed', 'remote_path', + 'wait_for', 'watch_dir', 'omdbapikey'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_CPS{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + if os.environ[env_cat_key] in cfg_new['Radarr'].sections: + cfg_new['Radarr'][env_cat_key]['enabled'] = 0 + + section = 'SickBeard' + env_cat_key = 'NZBPO_SBCATEGORY' + env_keys = ['ENABLED', 'HOST', 'PORT', 'APIKEY', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', + 'DELETE_FAILED', 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'REMOTE_PATH', 'PROCESS_METHOD'] + cfg_keys = ['enabled', 'host', 'port', 'apikey', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork', + 'delete_failed', 'Torrent_NoLink', 'nzbExtractionBy', 'remote_path', 'process_method'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_SB{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + if os.environ[env_cat_key] in cfg_new['NzbDrone'].sections: + cfg_new['NzbDrone'][env_cat_key]['enabled'] = 0 + + section = 'HeadPhones' + env_cat_key = 'NZBPO_HPCATEGORY' + env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WAIT_FOR', 'WATCH_DIR', 'REMOTE_PATH', 'DELETE_FAILED'] + cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'wait_for', 'watch_dir', 'remote_path', 'delete_failed'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_HP{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + if os.environ[env_cat_key] in cfg_new['Lidarr'].sections: + cfg_new['Lidarr'][env_cat_key]['enabled'] = 0 + + section = 'Mylar' + env_cat_key = 'NZBPO_MYCATEGORY' + env_keys = ['ENABLED', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'APIKEY', 'SSL', 'WEB_ROOT', 'WATCH_DIR', + 'REMOTE_PATH'] + cfg_keys = ['enabled', 'host', 'port', 'username', 'password', 'apikey', 'ssl', 'web_root', 'watch_dir', + 'remote_path'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_MY{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + + section = 'Gamez' + env_cat_key = 'NZBPO_GZCATEGORY' + env_keys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'LIBRARY', 'REMOTE_PATH'] + cfg_keys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'watch_dir', 'library', 'remote_path'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_GZ{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + + section = 'NzbDrone' + env_cat_key = 'NZBPO_NDCATEGORY' + env_keys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', + 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH', 'IMPORTMODE'] + # new cfgKey added for importMode + cfg_keys = ['enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', + 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path', 'importMode'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_ND{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + if os.environ[env_cat_key] in cfg_new['SickBeard'].sections: + cfg_new['SickBeard'][env_cat_key]['enabled'] = 0 + + section = 'Radarr' + env_cat_key = 'NZBPO_RACATEGORY' + env_keys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', + 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH', 'OMDBAPIKEY', 'IMPORTMODE'] + # new cfgKey added for importMode + cfg_keys = ['enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', + 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path', 'omdbapikey', 'importMode'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_RA{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + if os.environ[env_cat_key] in cfg_new['CouchPotato'].sections: + cfg_new['CouchPotato'][env_cat_key]['enabled'] = 0 + + section = 'Lidarr' + env_cat_key = 'NZBPO_LICATEGORY' + env_keys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', + 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH'] + cfg_keys = ['enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', + 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_LI{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + if os.environ[env_cat_key] in cfg_new['HeadPhones'].sections: + cfg_new['HeadPhones'][env_cat_key]['enabled'] = 0 + + section = 'Extensions' + env_keys = ['COMPRESSEDEXTENSIONS', 'MEDIAEXTENSIONS', 'METAEXTENSIONS'] + cfg_keys = ['compressedExtensions', 'mediaExtensions', 'metaExtensions'] + for index in range(len(env_keys)): + key = 'NZBPO_{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + cfg_new[section][option] = value + + section = 'Posix' + env_keys = ['NICENESS', 'IONICE_CLASS', 'IONICE_CLASSDATA'] + cfg_keys = ['niceness', 'ionice_class', 'ionice_classdata'] + for index in range(len(env_keys)): + key = 'NZBPO_{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + cfg_new[section][option] = value + + section = 'Transcoder' + env_keys = ['TRANSCODE', 'DUPLICATE', 'IGNOREEXTENSIONS', 'OUTPUTFASTSTART', 'OUTPUTVIDEOPATH', + 'PROCESSOUTPUT', 'AUDIOLANGUAGE', 'ALLAUDIOLANGUAGES', 'SUBLANGUAGES', + 'ALLSUBLANGUAGES', 'EMBEDSUBS', 'BURNINSUBTITLE', 'EXTRACTSUBS', 'EXTERNALSUBDIR', + 'OUTPUTDEFAULT', 'OUTPUTVIDEOEXTENSION', 'OUTPUTVIDEOCODEC', 'VIDEOCODECALLOW', + 'OUTPUTVIDEOPRESET', 'OUTPUTVIDEOFRAMERATE', 'OUTPUTVIDEOBITRATE', 'OUTPUTAUDIOCODEC', + 'AUDIOCODECALLOW', 'OUTPUTAUDIOBITRATE', 'OUTPUTQUALITYPERCENT', 'GETSUBS', + 'OUTPUTAUDIOTRACK2CODEC', 'AUDIOCODEC2ALLOW', 'OUTPUTAUDIOTRACK2BITRATE', + 'OUTPUTAUDIOOTHERCODEC', 'AUDIOOTHERCODECALLOW', 'OUTPUTAUDIOOTHERBITRATE', + 'OUTPUTSUBTITLECODEC', 'OUTPUTAUDIOCHANNELS', 'OUTPUTAUDIOTRACK2CHANNELS', + 'OUTPUTAUDIOOTHERCHANNELS', 'OUTPUTVIDEORESOLUTION'] + cfg_keys = ['transcode', 'duplicate', 'ignoreExtensions', 'outputFastStart', 'outputVideoPath', + 'processOutput', 'audioLanguage', 'allAudioLanguages', 'subLanguages', + 'allSubLanguages', 'embedSubs', 'burnInSubtitle', 'extractSubs', 'externalSubDir', + 'outputDefault', 'outputVideoExtension', 'outputVideoCodec', 'VideoCodecAllow', + 'outputVideoPreset', 'outputVideoFramerate', 'outputVideoBitrate', 'outputAudioCodec', + 'AudioCodecAllow', 'outputAudioBitrate', 'outputQualityPercent', 'getSubs', + 'outputAudioTrack2Codec', 'AudioCodec2Allow', 'outputAudioTrack2Bitrate', + 'outputAudioOtherCodec', 'AudioOtherCodecAllow', 'outputAudioOtherBitrate', + 'outputSubtitleCodec', 'outputAudioChannels', 'outputAudioTrack2Channels', + 'outputAudioOtherChannels', 'outputVideoResolution'] + for index in range(len(env_keys)): + key = 'NZBPO_{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + cfg_new[section][option] = value + + section = 'WakeOnLan' + env_keys = ['WAKE', 'HOST', 'PORT', 'MAC'] + cfg_keys = ['wake', 'host', 'port', 'mac'] + for index in range(len(env_keys)): + key = 'NZBPO_WOL{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + cfg_new[section][option] = value + + section = 'UserScript' + env_cat_key = 'NZBPO_USCATEGORY' + env_keys = ['USER_SCRIPT_MEDIAEXTENSIONS', 'USER_SCRIPT_PATH', 'USER_SCRIPT_PARAM', 'USER_SCRIPT_RUNONCE', + 'USER_SCRIPT_SUCCESSCODES', 'USER_SCRIPT_CLEAN', 'USDELAY', 'USREMOTE_PATH'] + cfg_keys = ['user_script_mediaExtensions', 'user_script_path', 'user_script_param', 'user_script_runOnce', + 'user_script_successCodes', 'user_script_clean', 'delay', 'remote_path'] + if env_cat_key in os.environ: + for index in range(len(env_keys)): + key = 'NZBPO_{index}'.format(index=env_keys[index]) + if key in os.environ: + option = cfg_keys[index] + value = os.environ[key] + if os.environ[env_cat_key] not in cfg_new[section].sections: + cfg_new[section][os.environ[env_cat_key]] = {} + cfg_new[section][os.environ[env_cat_key]][option] = value + cfg_new[section][os.environ[env_cat_key]]['enabled'] = 1 + + except Exception as error: + logger.debug('Error {msg} when applying NZBGet config'.format(msg=error)) + + try: + # write our new config to autoProcessMedia.cfg + cfg_new.filename = core.CONFIG_FILE + cfg_new.write() + except Exception as error: + logger.debug('Error {msg} when writing changes to .cfg'.format(msg=error)) + + return cfg_new + + +configobj.Section = Section +configobj.ConfigObj = ConfigObj +config = ConfigObj diff --git a/core/databases.py b/core/databases.py new file mode 100644 index 000000000..07a78a4d5 --- /dev/null +++ b/core/databases.py @@ -0,0 +1,65 @@ +# coding=utf-8 + +from core import logger, main_db +from core.utils import backup_versioned_file + +MIN_DB_VERSION = 1 # oldest db version we support migrating from +MAX_DB_VERSION = 2 + + +def backup_database(version): + logger.info('Backing up database before upgrade') + if not backup_versioned_file(main_db.db_filename(), version): + logger.log_error_and_exit('Database backup failed, abort upgrading database') + else: + logger.info('Proceeding with upgrade') + + +# ====================== +# = Main DB Migrations = +# ====================== +# Add new migrations at the bottom of the list; subclass the previous migration. + +class InitialSchema(main_db.SchemaUpgrade): + def test(self): + no_update = False + if self.has_table('db_version'): + cur_db_version = self.check_db_version() + no_update = not cur_db_version < MAX_DB_VERSION + return no_update + + def execute(self): + if not self.has_table('downloads') and not self.has_table('db_version'): + queries = [ + 'CREATE TABLE db_version (db_version INTEGER);', + 'CREATE TABLE downloads (input_directory TEXT, input_name TEXT, input_hash TEXT, input_id TEXT, client_agent TEXT, status INTEGER, last_update NUMERIC, CONSTRAINT pk_downloadID PRIMARY KEY (input_directory, input_name));', + 'INSERT INTO db_version (db_version) VALUES (2);' + ] + for query in queries: + self.connection.action(query) + + else: + cur_db_version = self.check_db_version() + + if cur_db_version < MIN_DB_VERSION: + logger.log_error_and_exit(u'Your database version ({current}) is too old to migrate ' + u'from what this version of nzbToMedia supports ({min}).' + u'\nPlease remove nzbtomedia.db file to begin fresh.'.format + (current=cur_db_version, min=MIN_DB_VERSION)) + + if cur_db_version > MAX_DB_VERSION: + logger.log_error_and_exit(u'Your database version ({current}) has been incremented ' + u'past what this version of nzbToMedia supports ({max}).' + u'\nIf you have used other forks of nzbToMedia, your database ' + u'may be unusable due to their modifications.'.format + (current=cur_db_version, max=MAX_DB_VERSION)) + if cur_db_version < MAX_DB_VERSION: # We need to upgrade. + queries = [ + 'CREATE TABLE downloads2 (input_directory TEXT, input_name TEXT, input_hash TEXT, input_id TEXT, client_agent TEXT, status INTEGER, last_update NUMERIC, CONSTRAINT pk_downloadID PRIMARY KEY (input_directory, input_name));', + 'INSERT INTO downloads2 SELECT * FROM downloads;', + 'DROP TABLE IF EXISTS downloads;', + 'ALTER TABLE downloads2 RENAME TO downloads;', + 'INSERT INTO db_version (db_version) VALUES (2);' + ] + for query in queries: + self.connection.action(query) diff --git a/core/databases/__init__.py b/core/databases/__init__.py deleted file mode 100644 index 14f97982d..000000000 --- a/core/databases/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding=utf-8 -__all__ = ["mainDB"] diff --git a/core/databases/mainDB.py b/core/databases/mainDB.py deleted file mode 100644 index d79033db4..000000000 --- a/core/databases/mainDB.py +++ /dev/null @@ -1,65 +0,0 @@ -# coding=utf-8 - -from core import logger, nzbToMediaDB -from core.nzbToMediaUtil import backupVersionedFile - -MIN_DB_VERSION = 1 # oldest db version we support migrating from -MAX_DB_VERSION = 2 - - -def backupDatabase(version): - logger.info("Backing up database before upgrade") - if not backupVersionedFile(nzbToMediaDB.dbFilename(), version): - logger.log_error_and_exit("Database backup failed, abort upgrading database") - else: - logger.info("Proceeding with upgrade") - - -# ====================== -# = Main DB Migrations = -# ====================== -# Add new migrations at the bottom of the list; subclass the previous migration. - -class InitialSchema(nzbToMediaDB.SchemaUpgrade): - def test(self): - no_update = False - if self.hasTable("db_version"): - cur_db_version = self.checkDBVersion() - no_update = not cur_db_version < MAX_DB_VERSION - return no_update - - def execute(self): - if not self.hasTable("downloads") and not self.hasTable("db_version"): - queries = [ - "CREATE TABLE db_version (db_version INTEGER);", - "CREATE TABLE downloads (input_directory TEXT, input_name TEXT, input_hash TEXT, input_id TEXT, client_agent TEXT, status INTEGER, last_update NUMERIC, CONSTRAINT pk_downloadID PRIMARY KEY (input_directory, input_name));", - "INSERT INTO db_version (db_version) VALUES (2);" - ] - for query in queries: - self.connection.action(query) - - else: - cur_db_version = self.checkDBVersion() - - if cur_db_version < MIN_DB_VERSION: - logger.log_error_and_exit(u"Your database version ({current}) is too old to migrate " - u"from what this version of nzbToMedia supports ({min})." - u"\nPlease remove nzbtomedia.db file to begin fresh.".format - (current=cur_db_version, min=MIN_DB_VERSION)) - - if cur_db_version > MAX_DB_VERSION: - logger.log_error_and_exit(u"Your database version ({current}) has been incremented " - u"past what this version of nzbToMedia supports ({max})." - u"\nIf you have used other forks of nzbToMedia, your database " - u"may be unusable due to their modifications.".format - (current=cur_db_version, max=MAX_DB_VERSION)) - if cur_db_version < MAX_DB_VERSION: # We need to upgrade. - queries = [ - "CREATE TABLE downloads2 (input_directory TEXT, input_name TEXT, input_hash TEXT, input_id TEXT, client_agent TEXT, status INTEGER, last_update NUMERIC, CONSTRAINT pk_downloadID PRIMARY KEY (input_directory, input_name));", - "INSERT INTO downloads2 SELECT * FROM downloads;", - "DROP TABLE IF EXISTS downloads;", - "ALTER TABLE downloads2 RENAME TO downloads;", - "INSERT INTO db_version (db_version) VALUES (2);" - ] - for query in queries: - self.connection.action(query) diff --git a/core/extractor/__init__.py b/core/extractor/__init__.py index 9bad5790a..b1090ea61 100644 --- a/core/extractor/__init__.py +++ b/core/extractor/__init__.py @@ -1 +1,181 @@ # coding=utf-8 + +import os +import platform +import shutil +import stat +import subprocess +from subprocess import Popen, call +from time import sleep + +import core + + +def extract(file_path, output_destination): + success = 0 + # Using Windows + if platform.system() == 'Windows': + if not os.path.exists(core.SEVENZIP): + core.logger.error('EXTRACTOR: Could not find 7-zip, Exiting') + return False + wscriptlocation = os.path.join(os.environ['WINDIR'], 'system32', 'wscript.exe') + invislocation = os.path.join(core.APP_ROOT, 'core', 'extractor', 'bin', 'invisible.vbs') + cmd_7zip = [wscriptlocation, invislocation, str(core.SHOWEXTRACT), core.SEVENZIP, 'x', '-y'] + ext_7zip = ['.rar', '.zip', '.tar.gz', 'tgz', '.tar.bz2', '.tbz', '.tar.lzma', '.tlz', '.7z', '.xz'] + extract_commands = dict.fromkeys(ext_7zip, cmd_7zip) + # Using unix + else: + required_cmds = ['unrar', 'unzip', 'tar', 'unxz', 'unlzma', '7zr', 'bunzip2'] + # ## Possible future suport: + # gunzip: gz (cmd will delete original archive) + # ## the following do not extract to dest dir + # '.xz': ['xz', '-d --keep'], + # '.lzma': ['xz', '-d --format=lzma --keep'], + # '.bz2': ['bzip2', '-d --keep'], + + extract_commands = { + '.rar': ['unrar', 'x', '-o+', '-y'], + '.tar': ['tar', '-xf'], + '.zip': ['unzip'], + '.tar.gz': ['tar', '-xzf'], '.tgz': ['tar', '-xzf'], + '.tar.bz2': ['tar', '-xjf'], '.tbz': ['tar', '-xjf'], + '.tar.lzma': ['tar', '--lzma', '-xf'], '.tlz': ['tar', '--lzma', '-xf'], + '.tar.xz': ['tar', '--xz', '-xf'], '.txz': ['tar', '--xz', '-xf'], + '.7z': ['7zr', 'x'], + } + # Test command exists and if not, remove + if not os.getenv('TR_TORRENT_DIR'): + devnull = open(os.devnull, 'w') + for cmd in required_cmds: + if call(['which', cmd], stdout=devnull, + stderr=devnull): # note, returns 0 if exists, or 1 if doesn't exist. + for k, v in extract_commands.items(): + if cmd in v[0]: + if not call(['which', '7zr'], stdout=devnull, stderr=devnull): # we do have '7zr' + extract_commands[k] = ['7zr', 'x', '-y'] + elif not call(['which', '7z'], stdout=devnull, stderr=devnull): # we do have '7z' + extract_commands[k] = ['7z', 'x', '-y'] + elif not call(['which', '7za'], stdout=devnull, stderr=devnull): # we do have '7za' + extract_commands[k] = ['7za', 'x', '-y'] + else: + core.logger.error('EXTRACTOR: {cmd} not found, ' + 'disabling support for {feature}'.format + (cmd=cmd, feature=k)) + del extract_commands[k] + devnull.close() + else: + core.logger.warning('EXTRACTOR: Cannot determine which tool to use when called from Transmission') + + if not extract_commands: + core.logger.warning('EXTRACTOR: No archive extracting programs found, plugin will be disabled') + + ext = os.path.splitext(file_path) + cmd = [] + if ext[1] in ('.gz', '.bz2', '.lzma'): + # Check if this is a tar + if os.path.splitext(ext[0])[1] == '.tar': + cmd = extract_commands['.tar{ext}'.format(ext=ext[1])] + elif ext[1] in ('.1', '.01', '.001') and os.path.splitext(ext[0])[1] in ('.rar', '.zip', '.7z'): + cmd = extract_commands[os.path.splitext(ext[0])[1]] + elif ext[1] in ('.cb7', '.cba', '.cbr', '.cbt', '.cbz'): # don't extract these comic book archives. + return False + else: + if ext[1] in extract_commands: + cmd = extract_commands[ext[1]] + else: + core.logger.debug('EXTRACTOR: Unknown file type: {ext}'.format + (ext=ext[1])) + return False + + # Create outputDestination folder + core.make_dir(output_destination) + + if core.PASSWORDSFILE and os.path.isfile(os.path.normpath(core.PASSWORDSFILE)): + passwords = [line.strip() for line in open(os.path.normpath(core.PASSWORDSFILE))] + else: + passwords = [] + + core.logger.info('Extracting {file} to {destination}'.format + (file=file_path, destination=output_destination)) + core.logger.debug('Extracting {cmd} {file} {destination}'.format + (cmd=cmd, file=file_path, destination=output_destination)) + + orig_files = [] + orig_dirs = [] + for directory, subdirs, files in os.walk(output_destination): + for subdir in subdirs: + orig_dirs.append(os.path.join(directory, subdir)) + for file in files: + orig_files.append(os.path.join(directory, file)) + + pwd = os.getcwd() # Get our Present Working Directory + os.chdir(output_destination) # Not all unpack commands accept full paths, so just extract into this directory + devnull = open(os.devnull, 'w') + + try: # now works same for nt and *nix + info = None + cmd.append(file_path) # add filePath to final cmd arg. + if platform.system() == 'Windows': + info = subprocess.STARTUPINFO() + info.dwFlags |= subprocess.STARTF_USESHOWWINDOW + else: + cmd = core.NICENESS + cmd + cmd2 = cmd + cmd2.append('-p-') # don't prompt for password. + p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. + res = p.wait() + if res == 0: # Both Linux and Windows return 0 for successful. + core.logger.info('EXTRACTOR: Extraction was successful for {file} to {destination}'.format + (file=file_path, destination=output_destination)) + success = 1 + elif len(passwords) > 0: + core.logger.info('EXTRACTOR: Attempting to extract with passwords') + for password in passwords: + if password == '': # if edited in windows or otherwise if blank lines. + continue + cmd2 = cmd + # append password here. + passcmd = '-p{pwd}'.format(pwd=password) + cmd2.append(passcmd) + p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. + res = p.wait() + if (res >= 0 and platform == 'Windows') or res == 0: + core.logger.info('EXTRACTOR: Extraction was successful ' + 'for {file} to {destination} using password: {pwd}'.format + (file=file_path, destination=output_destination, pwd=password)) + success = 1 + break + else: + continue + except Exception: + core.logger.error('EXTRACTOR: Extraction failed for {file}. ' + 'Could not call command {cmd}'.format + (file=file_path, cmd=cmd)) + os.chdir(pwd) + return False + + devnull.close() + os.chdir(pwd) # Go back to our Original Working Directory + if success: + # sleep to let files finish writing to disk + sleep(3) + perms = stat.S_IMODE(os.lstat(os.path.split(file_path)[0]).st_mode) + for directory, subdirs, files in os.walk(output_destination): + for subdir in subdirs: + if not os.path.join(directory, subdir) in orig_files: + try: + os.chmod(os.path.join(directory, subdir), perms) + except Exception: + pass + for file in files: + if not os.path.join(directory, file) in orig_files: + try: + shutil.copymode(file_path, os.path.join(directory, file)) + except Exception: + pass + return True + else: + core.logger.error('EXTRACTOR: Extraction failed for {file}. ' + 'Result was {result}'.format + (file=file_path, result=res)) + return False diff --git a/core/extractor/bin/AMD64/7z.dll b/core/extractor/bin/AMD64/7z.dll index cea996e4f..be29515b7 100644 Binary files a/core/extractor/bin/AMD64/7z.dll and b/core/extractor/bin/AMD64/7z.dll differ diff --git a/core/extractor/bin/AMD64/7z.exe b/core/extractor/bin/AMD64/7z.exe index b55fefe6d..337d4b018 100644 Binary files a/core/extractor/bin/AMD64/7z.exe and b/core/extractor/bin/AMD64/7z.exe differ diff --git a/core/extractor/bin/AMD64/license.txt b/core/extractor/bin/AMD64/license.txt index 0be9890ae..9855d1ea5 100644 --- a/core/extractor/bin/AMD64/license.txt +++ b/core/extractor/bin/AMD64/license.txt @@ -3,19 +3,20 @@ License for use and distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 7-Zip Copyright (C) 1999-2012 Igor Pavlov. + 7-Zip Copyright (C) 1999-2018 Igor Pavlov. - Licenses for files are: + The licenses for files are: - 1) 7z.dll: GNU LGPL + unRAR restriction - 2) All other files: GNU LGPL + 1) 7z.dll: + - The "GNU LGPL" as main license for most of the code + - The "GNU LGPL" with "unRAR license restriction" for some code + - The "BSD 3-clause License" for some code + 2) All other files: the "GNU LGPL". - The GNU LGPL + unRAR restriction means that you must follow both - GNU LGPL rules and unRAR restriction rules. + Redistributions in binary form must reproduce related license information from this file. - - Note: - You can use 7-Zip on any computer, including a computer in a commercial + Note: + You can use 7-Zip on any computer, including a computer in a commercial organization. You don't need to register or pay for 7-Zip. @@ -32,25 +33,58 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. - You can receive a copy of the GNU Lesser General Public License from + You can receive a copy of the GNU Lesser General Public License from http://www.gnu.org/ - unRAR restriction - ----------------- - The decompression engine for RAR archives was developed using source + + BSD 3-clause License + -------------------- + + The "BSD 3-clause License" is used for the code in 7z.dll that implements LZFSE data decompression. + That code was derived from the code in the "LZFSE compression library" developed by Apple Inc, + that also uses the "BSD 3-clause License": + + ---- + Copyright (c) 2015-2016, Apple Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ---- + + + + + unRAR license restriction + ------------------------- + + The decompression engine for RAR archives was developed using source code of unRAR program. All copyrights to original unRAR code are owned by Alexander Roshal. The license for original unRAR code has the following restriction: - The unRAR sources cannot be used to re-create the RAR compression algorithm, - which is proprietary. Distribution of modified unRAR sources in separate form + The unRAR sources cannot be used to re-create the RAR compression algorithm, + which is proprietary. Distribution of modified unRAR sources in separate form or as a part of other software is permitted, provided that it is clearly stated in the documentation and source comments that the code may not be used to develop a RAR (WinRAR) compatible archiver. -- - Igor Pavlov \ No newline at end of file + Igor Pavlov diff --git a/core/extractor/bin/invisible.cmd b/core/extractor/bin/invisible.cmd deleted file mode 100755 index ef1e54aec..000000000 --- a/core/extractor/bin/invisible.cmd +++ /dev/null @@ -1 +0,0 @@ -start /B /wait wscript "%~dp0\invisible.vbs" %* \ No newline at end of file diff --git a/core/extractor/bin/invisible.vbs b/core/extractor/bin/invisible.vbs index 1c96e8cc9..01979e2ab 100755 --- a/core/extractor/bin/invisible.vbs +++ b/core/extractor/bin/invisible.vbs @@ -1,15 +1,15 @@ set args = WScript.Arguments num = args.Count -if num = 0 then - WScript.Echo "Usage: [CScript | WScript] invis.vbs aScript.bat " +if num < 2 then + WScript.Echo "Usage: [CScript | WScript] invis.vbs aScript.bat " WScript.Quit 1 end if sargs = "" -if num > 1 then +if num > 2 then sargs = " " - for k = 1 to num - 1 + for k = 2 to num - 1 anArg = args.Item(k) sargs = sargs & anArg & " " next @@ -17,4 +17,5 @@ end if Set WshShell = WScript.CreateObject("WScript.Shell") -WshShell.Run """" & WScript.Arguments(0) & """" & sargs, 0, True \ No newline at end of file +returnValue = WshShell.Run("""" & args(1) & """" & sargs, args(0), True) +WScript.Quit(returnValue) diff --git a/core/extractor/bin/x86/7z.dll b/core/extractor/bin/x86/7z.dll index 2bdaed6a8..5598fe3e8 100644 Binary files a/core/extractor/bin/x86/7z.dll and b/core/extractor/bin/x86/7z.dll differ diff --git a/core/extractor/bin/x86/7z.exe b/core/extractor/bin/x86/7z.exe index 101884c32..77cdcba68 100644 Binary files a/core/extractor/bin/x86/7z.exe and b/core/extractor/bin/x86/7z.exe differ diff --git a/core/extractor/bin/x86/license.txt b/core/extractor/bin/x86/license.txt index 0be9890ae..9855d1ea5 100644 --- a/core/extractor/bin/x86/license.txt +++ b/core/extractor/bin/x86/license.txt @@ -3,19 +3,20 @@ License for use and distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 7-Zip Copyright (C) 1999-2012 Igor Pavlov. + 7-Zip Copyright (C) 1999-2018 Igor Pavlov. - Licenses for files are: + The licenses for files are: - 1) 7z.dll: GNU LGPL + unRAR restriction - 2) All other files: GNU LGPL + 1) 7z.dll: + - The "GNU LGPL" as main license for most of the code + - The "GNU LGPL" with "unRAR license restriction" for some code + - The "BSD 3-clause License" for some code + 2) All other files: the "GNU LGPL". - The GNU LGPL + unRAR restriction means that you must follow both - GNU LGPL rules and unRAR restriction rules. + Redistributions in binary form must reproduce related license information from this file. - - Note: - You can use 7-Zip on any computer, including a computer in a commercial + Note: + You can use 7-Zip on any computer, including a computer in a commercial organization. You don't need to register or pay for 7-Zip. @@ -32,25 +33,58 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. - You can receive a copy of the GNU Lesser General Public License from + You can receive a copy of the GNU Lesser General Public License from http://www.gnu.org/ - unRAR restriction - ----------------- - The decompression engine for RAR archives was developed using source + + BSD 3-clause License + -------------------- + + The "BSD 3-clause License" is used for the code in 7z.dll that implements LZFSE data decompression. + That code was derived from the code in the "LZFSE compression library" developed by Apple Inc, + that also uses the "BSD 3-clause License": + + ---- + Copyright (c) 2015-2016, Apple Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ---- + + + + + unRAR license restriction + ------------------------- + + The decompression engine for RAR archives was developed using source code of unRAR program. All copyrights to original unRAR code are owned by Alexander Roshal. The license for original unRAR code has the following restriction: - The unRAR sources cannot be used to re-create the RAR compression algorithm, - which is proprietary. Distribution of modified unRAR sources in separate form + The unRAR sources cannot be used to re-create the RAR compression algorithm, + which is proprietary. Distribution of modified unRAR sources in separate form or as a part of other software is permitted, provided that it is clearly stated in the documentation and source comments that the code may not be used to develop a RAR (WinRAR) compatible archiver. -- - Igor Pavlov \ No newline at end of file + Igor Pavlov diff --git a/core/extractor/extractor.py b/core/extractor/extractor.py deleted file mode 100644 index ed8871879..000000000 --- a/core/extractor/extractor.py +++ /dev/null @@ -1,179 +0,0 @@ -# coding=utf-8 - -import os -import platform -import shutil -import stat -from time import sleep -import core -from subprocess import call, Popen -import subprocess - - -def extract(filePath, outputDestination): - success = 0 - # Using Windows - if platform.system() == 'Windows': - if not os.path.exists(core.SEVENZIP): - core.logger.error("EXTRACTOR: Could not find 7-zip, Exiting") - return False - invislocation = os.path.join(core.PROGRAM_DIR, 'core', 'extractor', 'bin', 'invisible.cmd') - cmd_7zip = [invislocation, core.SEVENZIP, "x", "-y"] - ext_7zip = [".rar", ".zip", ".tar.gz", "tgz", ".tar.bz2", ".tbz", ".tar.lzma", ".tlz", ".7z", ".xz"] - EXTRACT_COMMANDS = dict.fromkeys(ext_7zip, cmd_7zip) - # Using unix - else: - required_cmds = ["unrar", "unzip", "tar", "unxz", "unlzma", "7zr", "bunzip2"] - # ## Possible future suport: - # gunzip: gz (cmd will delete original archive) - # ## the following do not extract to dest dir - # ".xz": ["xz", "-d --keep"], - # ".lzma": ["xz", "-d --format=lzma --keep"], - # ".bz2": ["bzip2", "-d --keep"], - - EXTRACT_COMMANDS = { - ".rar": ["unrar", "x", "-o+", "-y"], - ".tar": ["tar", "-xf"], - ".zip": ["unzip"], - ".tar.gz": ["tar", "-xzf"], ".tgz": ["tar", "-xzf"], - ".tar.bz2": ["tar", "-xjf"], ".tbz": ["tar", "-xjf"], - ".tar.lzma": ["tar", "--lzma", "-xf"], ".tlz": ["tar", "--lzma", "-xf"], - ".tar.xz": ["tar", "--xz", "-xf"], ".txz": ["tar", "--xz", "-xf"], - ".7z": ["7zr", "x"], - } - # Test command exists and if not, remove - if not os.getenv('TR_TORRENT_DIR'): - devnull = open(os.devnull, 'w') - for cmd in required_cmds: - if call(['which', cmd], stdout=devnull, - stderr=devnull): # note, returns 0 if exists, or 1 if doesn't exist. - for k, v in EXTRACT_COMMANDS.items(): - if cmd in v[0]: - if not call(["which", "7zr"], stdout=devnull, stderr=devnull): # we do have "7zr" - EXTRACT_COMMANDS[k] = ["7zr", "x", "-y"] - elif not call(["which", "7z"], stdout=devnull, stderr=devnull): # we do have "7z" - EXTRACT_COMMANDS[k] = ["7z", "x", "-y"] - elif not call(["which", "7za"], stdout=devnull, stderr=devnull): # we do have "7za" - EXTRACT_COMMANDS[k] = ["7za", "x", "-y"] - else: - core.logger.error("EXTRACTOR: {cmd} not found, " - "disabling support for {feature}".format - (cmd=cmd, feature=k)) - del EXTRACT_COMMANDS[k] - devnull.close() - else: - core.logger.warning("EXTRACTOR: Cannot determine which tool to use when called from Transmission") - - if not EXTRACT_COMMANDS: - core.logger.warning("EXTRACTOR: No archive extracting programs found, plugin will be disabled") - - ext = os.path.splitext(filePath) - cmd = [] - if ext[1] in (".gz", ".bz2", ".lzma"): - # Check if this is a tar - if os.path.splitext(ext[0])[1] == ".tar": - cmd = EXTRACT_COMMANDS[".tar{ext}".format(ext=ext[1])] - elif ext[1] in (".1", ".01", ".001") and os.path.splitext(ext[0])[1] in (".rar", ".zip", ".7z"): - cmd = EXTRACT_COMMANDS[os.path.splitext(ext[0])[1]] - elif ext[1] in (".cb7", ".cba", ".cbr", ".cbt", ".cbz"): # don't extract these comic book archives. - return False - else: - if ext[1] in EXTRACT_COMMANDS: - cmd = EXTRACT_COMMANDS[ext[1]] - else: - core.logger.debug("EXTRACTOR: Unknown file type: {ext}".format - (ext=ext[1])) - return False - - # Create outputDestination folder - core.makeDir(outputDestination) - - if core.PASSWORDSFILE != "" and os.path.isfile(os.path.normpath(core.PASSWORDSFILE)): - passwords = [line.strip() for line in open(os.path.normpath(core.PASSWORDSFILE))] - else: - passwords = [] - - core.logger.info("Extracting {file} to {destination}".format - (file=filePath, destination=outputDestination)) - core.logger.debug("Extracting {cmd} {file} {destination}".format - (cmd=cmd, file=filePath, destination=outputDestination)) - - origFiles = [] - origDirs = [] - for dir, subdirs, files in os.walk(outputDestination): - for subdir in subdirs: - origDirs.append(os.path.join(dir, subdir)) - for file in files: - origFiles.append(os.path.join(dir, file)) - - pwd = os.getcwd() # Get our Present Working Directory - os.chdir(outputDestination) # Not all unpack commands accept full paths, so just extract into this directory - devnull = open(os.devnull, 'w') - - try: # now works same for nt and *nix - info = None - cmd.append(filePath) # add filePath to final cmd arg. - if platform.system() == 'Windows': - info = subprocess.STARTUPINFO() - info.dwFlags |= subprocess.STARTF_USESHOWWINDOW - else: - cmd = core.NICENESS + cmd - cmd2 = cmd - cmd2.append("-p-") # don't prompt for password. - p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. - res = p.wait() - if (res >= 0 and os.name == 'nt') or res == 0: # for windows chp returns process id if successful or -1*Error code. Linux returns 0 for successful. - core.logger.info("EXTRACTOR: Extraction was successful for {file} to {destination}".format - (file=filePath, destination=outputDestination)) - success = 1 - elif len(passwords) > 0: - core.logger.info("EXTRACTOR: Attempting to extract with passwords") - for password in passwords: - if password == "": # if edited in windows or otherwise if blank lines. - continue - cmd2 = cmd - # append password here. - passcmd = "-p{pwd}".format(pwd=password) - cmd2.append(passcmd) - p = Popen(cmd2, stdout=devnull, stderr=devnull, startupinfo=info) # should extract files fine. - res = p.wait() - if (res >= 0 and platform == 'Windows') or res == 0: - core.logger.info("EXTRACTOR: Extraction was successful " - "for {file} to {destination} using password: {pwd}".format - (file=filePath, destination=outputDestination, pwd=password)) - success = 1 - break - else: - continue - except: - core.logger.error("EXTRACTOR: Extraction failed for {file}. " - "Could not call command {cmd}".format - (file=filePath, cmd=cmd)) - os.chdir(pwd) - return False - - devnull.close() - os.chdir(pwd) # Go back to our Original Working Directory - if success: - # sleep to let files finish writing to disk - sleep(3) - perms = stat.S_IMODE(os.lstat(os.path.split(filePath)[0]).st_mode) - for dir, subdirs, files in os.walk(outputDestination): - for subdir in subdirs: - if not os.path.join(dir, subdir) in origFiles: - try: - os.chmod(os.path.join(dir, subdir), perms) - except: - pass - for file in files: - if not os.path.join(dir, file) in origFiles: - try: - shutil.copymode(filePath, os.path.join(dir, file)) - except: - pass - return True - else: - core.logger.error("EXTRACTOR: Extraction failed for {file}. " - "Result was {result}".format - (file=filePath, result=res)) - return False diff --git a/core/forks.py b/core/forks.py new file mode 100644 index 000000000..8f8fcb26d --- /dev/null +++ b/core/forks.py @@ -0,0 +1,114 @@ +# coding=utf-8 + +import requests +from six import iteritems + +import core +from core import logger + + +def auto_fork(section, input_category): + # auto-detect correct section + # config settings + + cfg = dict(core.CFG[section][input_category]) + + host = cfg.get('host') + port = cfg.get('port') + username = cfg.get('username') + password = cfg.get('password') + apikey = cfg.get('apikey') + ssl = int(cfg.get('ssl', 0)) + web_root = cfg.get('web_root', '') + replace = {'sickrage': 'SickRage', 'sickchill': 'SickChill', 'sickgear': 'SickGear', 'medusa': 'Medusa', 'sickbeard-api': 'SickBeard-api', 'stheno': 'Stheno'} + f1 = replace[cfg.get('fork', 'auto')] if cfg.get('fork', 'auto') in replace else cfg.get('fork', 'auto') + try: + fork = f1, core.FORKS[f1] + except KeyError: + fork = 'auto' + protocol = 'https://' if ssl else 'http://' + + detected = False + if section == 'NzbDrone': + logger.info('Attempting to verify {category} fork'.format + (category=input_category)) + url = '{protocol}{host}:{port}{root}/api/rootfolder'.format( + protocol=protocol, host=host, port=port, root=web_root) + headers = {'X-Api-Key': apikey} + try: + r = requests.get(url, headers=headers, stream=True, verify=False) + except requests.ConnectionError: + logger.warning('Could not connect to {0}:{1} to verify fork!'.format(section, input_category)) + + if not r.ok: + logger.warning('Connection to {section}:{category} failed! ' + 'Check your configuration'.format + (section=section, category=input_category)) + + fork = ['default', {}] + + elif fork == 'auto': + params = core.ALL_FORKS + rem_params = [] + logger.info('Attempting to auto-detect {category} fork'.format(category=input_category)) + # define the order to test. Default must be first since the default fork doesn't reject parameters. + # then in order of most unique parameters. + + if apikey: + url = '{protocol}{host}:{port}{root}/api/{apikey}/?cmd=help&subject=postprocess'.format( + protocol=protocol, host=host, port=port, root=web_root, apikey=apikey) + else: + url = '{protocol}{host}:{port}{root}/home/postprocess/'.format( + protocol=protocol, host=host, port=port, root=web_root) + + # attempting to auto-detect fork + try: + s = requests.Session() + if not apikey and username and password: + login = '{protocol}{host}:{port}{root}/login'.format( + protocol=protocol, host=host, port=port, root=web_root) + login_params = {'username': username, 'password': password} + r = s.get(login, verify=False, timeout=(30, 60)) + if r.status_code == 401 and r.cookies.get('_xsrf'): + login_params['_xsrf'] = r.cookies.get('_xsrf') + s.post(login, data=login_params, stream=True, verify=False) + r = s.get(url, auth=(username, password), verify=False) + except requests.ConnectionError: + logger.info('Could not connect to {section}:{category} to perform auto-fork detection!'.format + (section=section, category=input_category)) + r = [] + if r and r.ok: + if apikey: + optional_parameters = [] + try: + optional_parameters = r.json()['data']['optionalParameters'].keys() + except Exception: + optional_parameters = r.json()['data']['data']['optionalParameters'].keys() + for param in params: + if param not in optional_parameters: + rem_params.append(param) + else: + for param in params: + if 'name="{param}"'.format(param=param) not in r.text: + rem_params.append(param) + for param in rem_params: + params.pop(param) + for fork in sorted(iteritems(core.FORKS), reverse=False): + if params == fork[1]: + detected = True + break + if detected: + logger.info('{section}:{category} fork auto-detection successful ...'.format + (section=section, category=input_category)) + elif rem_params: + logger.info('{section}:{category} fork auto-detection found custom params {params}'.format + (section=section, category=input_category, params=params)) + fork = ['custom', params] + else: + logger.info('{section}:{category} fork auto-detection failed'.format + (section=section, category=input_category)) + fork = core.FORKS.items()[core.FORKS.keys().index(core.FORK_DEFAULT)] + + logger.info('{section}:{category} fork set to {fork}'.format + (section=section, category=input_category, fork=fork[0])) + return fork[0], fork[1] diff --git a/core/gh_api.py b/core/gh_api.py deleted file mode 100644 index f1264c097..000000000 --- a/core/gh_api.py +++ /dev/null @@ -1,66 +0,0 @@ -# coding=utf-8 - -import requests -from six import iteritems - - -class GitHub(object): - """ - Simple api wrapper for the Github API v3. - """ - - def __init__(self, github_repo_user, github_repo, branch='master'): - - self.github_repo_user = github_repo_user - self.github_repo = github_repo - self.branch = branch - - def _access_API(self, path, params=None): - """ - Access the API at the path given and with the optional params given. - """ - - url = 'https://api.github.com/{path}'.format(path='/'.join(path)) - - if params and type(params) is dict: - url += '?{params}'.format(params='&'.join(['{key}={value}'.format(key=k, value=v) - for k, v in iteritems(params)])) - - data = requests.get(url, verify=False) - - if data.ok: - json_data = data.json() - return json_data - else: - return [] - - def commits(self): - """ - Uses the API to get a list of the 100 most recent commits from the specified user/repo/branch, starting from HEAD. - - user: The github username of the person whose repo you're querying - repo: The repo name to query - branch: Optional, the branch name to show commits from - - Returns a deserialized json object containing the commit info. See http://developer.github.com/v3/repos/commits/ - """ - access_API = self._access_API(['repos', self.github_repo_user, self.github_repo, 'commits'], - params={'per_page': 100, 'sha': self.branch}) - return access_API - - def compare(self, base, head, per_page=1): - """ - Uses the API to get a list of compares between base and head. - - user: The github username of the person whose repo you're querying - repo: The repo name to query - base: Start compare from branch - head: Current commit sha or branch name to compare - per_page: number of items per page - - Returns a deserialized json object containing the compare info. See http://developer.github.com/v3/repos/commits/ - """ - access_API = self._access_API( - ['repos', self.github_repo_user, self.github_repo, 'compare', '{base}...{head}'.format(base=base, head=head)], - params={'per_page': per_page}) - return access_API diff --git a/core/github_api.py b/core/github_api.py new file mode 100644 index 000000000..6e44f9f3a --- /dev/null +++ b/core/github_api.py @@ -0,0 +1,56 @@ +# coding=utf-8 + +import requests + + +class GitHub(object): + """ + Simple api wrapper for the Github API v3. + """ + + def __init__(self, github_repo_user, github_repo, branch='master'): + + self.github_repo_user = github_repo_user + self.github_repo = github_repo + self.branch = branch + + def _access_api(self, path, params=None): + """ + Access the API at the path given and with the optional params given. + """ + url = 'https://api.github.com/{path}'.format(path='/'.join(path)) + data = requests.get(url, params=params, verify=False) + return data.json() if data.ok else [] + + def commits(self): + """ + Uses the API to get a list of the 100 most recent commits from the specified user/repo/branch, starting from HEAD. + + user: The github username of the person whose repo you're querying + repo: The repo name to query + branch: Optional, the branch name to show commits from + + Returns a deserialized json object containing the commit info. See http://developer.github.com/v3/repos/commits/ + """ + return self._access_api( + ['repos', self.github_repo_user, self.github_repo, 'commits'], + params={'per_page': 100, 'sha': self.branch}, + ) + + def compare(self, base, head, per_page=1): + """ + Uses the API to get a list of compares between base and head. + + user: The github username of the person whose repo you're querying + repo: The repo name to query + base: Start compare from branch + head: Current commit sha or branch name to compare + per_page: number of items per page + + Returns a deserialized json object containing the compare info. See http://developer.github.com/v3/repos/commits/ + """ + return self._access_api( + ['repos', self.github_repo_user, self.github_repo, 'compare', + '{base}...{head}'.format(base=base, head=head)], + params={'per_page': per_page}, + ) diff --git a/core/linktastic/__init__.py b/core/linktastic/__init__.py deleted file mode 100644 index 9bad5790a..000000000 --- a/core/linktastic/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding=utf-8 diff --git a/core/linktastic/linktastic.py b/core/linktastic/linktastic.py deleted file mode 100644 index 95d2f8c65..000000000 --- a/core/linktastic/linktastic.py +++ /dev/null @@ -1,123 +0,0 @@ -# coding=utf-8 -# Linktastic Module -# - A python2/3 compatible module that can create hardlinks/symlinks on windows-based systems -# -# Linktastic is distributed under the MIT License. The follow are the terms and conditions of using Linktastic. -# -# The MIT License (MIT) -# Copyright (c) 2012 Solipsis Development -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and -# associated documentation files (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial -# portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT -# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import subprocess -from subprocess import CalledProcessError -import os - -if os.name == 'nt': - info = subprocess.STARTUPINFO() - info.dwFlags |= subprocess.STARTF_USESHOWWINDOW - - -# Prevent spaces from messing with us! -def _escape_param(param): - return '"{0}"'.format(param) - - -# Private function to create link on nt-based systems -def _link_windows(src, dest): - try: - subprocess.check_output( - 'cmd /C mklink /H {0} {1}'.format(_escape_param(dest), _escape_param(src)), - stderr=subprocess.STDOUT, startupinfo=info) - except CalledProcessError as err: - - raise IOError(err.output.decode('utf-8')) - - # TODO, find out what kind of messages Windows sends us from mklink - # print(stdout) - # assume if they ret-coded 0 we're good - - -def _symlink_windows(src, dest): - try: - subprocess.check_output( - 'cmd /C mklink {0} {1}'.format(_escape_param(dest), _escape_param(src)), - stderr=subprocess.STDOUT, startupinfo=info) - except CalledProcessError as err: - raise IOError(err.output.decode('utf-8')) - - # TODO, find out what kind of messages Windows sends us from mklink - # print(stdout) - # assume if they ret-coded 0 we're good - - -def _dirlink_windows(src, dest): - try: - subprocess.check_output( - 'cmd /C mklink /J {0} {1}'.format(_escape_param(dest), _escape_param(src)), - stderr=subprocess.STDOUT, startupinfo=info) - except CalledProcessError as err: - raise IOError(err.output.decode('utf-8')) - - # TODO, find out what kind of messages Windows sends us from mklink - # print(stdout) - # assume if they ret-coded 0 we're good - - -def _junctionlink_windows(src, dest): - try: - subprocess.check_output( - 'cmd /C mklink /D {0} {1}'.format(_escape_param(dest), _escape_param(src)), - stderr=subprocess.STDOUT, startupinfo=info) - except CalledProcessError as err: - raise IOError(err.output.decode('utf-8')) - - # TODO, find out what kind of messages Windows sends us from mklink - # print(stdout) - # assume if they ret-coded 0 we're good - - -# Create a hard link to src named as dest -# This version of link, unlike os.link, supports nt systems as well -def link(src, dest): - if os.name == 'nt': - _link_windows(src, dest) - else: - os.link(src, dest) - - -# Create a symlink to src named as dest, but don't fail if you're on nt -def symlink(src, dest): - if os.name == 'nt': - _symlink_windows(src, dest) - else: - os.symlink(src, dest) - - -# Create a symlink to src named as dest, but don't fail if you're on nt -def dirlink(src, dest): - if os.name == 'nt': - _dirlink_windows(src, dest) - else: - os.symlink(src, dest) - - -# Create a symlink to src named as dest, but don't fail if you're on nt -def junctionlink(src, dest): - if os.name == 'nt': - _junctionlink_windows(src, dest) - else: - os.symlink(src, dest) diff --git a/core/logger.py b/core/logger.py index 5a555bf2a..3305a96e4 100644 --- a/core/logger.py +++ b/core/logger.py @@ -1,10 +1,10 @@ # coding=utf-8 -from __future__ import with_statement +import logging import os import sys import threading -import logging + import core # number of log files to keep @@ -58,10 +58,10 @@ def close_log(self, handler=None): handler.flush() handler.close() - def initLogging(self, consoleLogging=True): + def init_logging(self, console_logging=True): - if consoleLogging: - self.console_logging = consoleLogging + if console_logging: + self.console_logging = console_logging old_handler = None @@ -180,7 +180,7 @@ def _rotate_logs(self): pp_logger.addHandler(new_file_handler) db_logger.addHandler(new_file_handler) - def log(self, toLog, logLevel=MESSAGE, section='MAIN'): + def log(self, to_log, log_level=MESSAGE, section='MAIN'): with self.log_lock: @@ -193,9 +193,9 @@ def log(self, toLog, logLevel=MESSAGE, section='MAIN'): self.writes_since_check += 1 try: - message = u"{0}: {1}".format(section.upper(), toLog) + message = u'{0}: {1}'.format(section.upper(), to_log) except UnicodeError: - message = u"{0}: Message contains non-utf-8 string".format(section.upper()) + message = u'{0}: Message contains non-utf-8 string'.format(section.upper()) out_line = message @@ -206,22 +206,22 @@ def log(self, toLog, logLevel=MESSAGE, section='MAIN'): setattr(db_logger, 'db', lambda *args: db_logger.log(DB, *args)) try: - if logLevel == DEBUG: + if log_level == DEBUG: if core.LOG_DEBUG == 1: ntm_logger.debug(out_line) - elif logLevel == MESSAGE: + elif log_level == MESSAGE: ntm_logger.info(out_line) - elif logLevel == WARNING: + elif log_level == WARNING: ntm_logger.warning(out_line) - elif logLevel == ERROR: + elif log_level == ERROR: ntm_logger.error(out_line) - elif logLevel == POSTPROCESS: + elif log_level == POSTPROCESS: pp_logger.postprocess(out_line) - elif logLevel == DB: + elif log_level == DB: if core.LOG_DB == 1: db_logger.db(out_line) else: - ntm_logger.info(logLevel, out_line) + ntm_logger.info(log_level, out_line) except ValueError: pass @@ -249,32 +249,32 @@ def format(self, record): ntm_log_instance = NTMRotatingLogHandler(core.LOG_FILE, NUM_LOGS, LOG_SIZE) -def log(toLog, logLevel=MESSAGE, section='MAIN'): - ntm_log_instance.log(toLog, logLevel, section) +def log(to_log, log_level=MESSAGE, section='MAIN'): + ntm_log_instance.log(to_log, log_level, section) -def info(toLog, section='MAIN'): - log(toLog, MESSAGE, section) +def info(to_log, section='MAIN'): + log(to_log, MESSAGE, section) -def error(toLog, section='MAIN'): - log(toLog, ERROR, section) +def error(to_log, section='MAIN'): + log(to_log, ERROR, section) -def warning(toLog, section='MAIN'): - log(toLog, WARNING, section) +def warning(to_log, section='MAIN'): + log(to_log, WARNING, section) -def debug(toLog, section='MAIN'): - log(toLog, DEBUG, section) +def debug(to_log, section='MAIN'): + log(to_log, DEBUG, section) -def postprocess(toLog, section='POSTPROCESS'): - log(toLog, POSTPROCESS, section) +def postprocess(to_log, section='POSTPROCESS'): + log(to_log, POSTPROCESS, section) -def db(toLog, section='DB'): - log(toLog, DB, section) +def db(to_log, section='DB'): + log(to_log, DB, section) def log_error_and_exit(error_msg): diff --git a/core/main_db.py b/core/main_db.py new file mode 100644 index 000000000..6d2b6b955 --- /dev/null +++ b/core/main_db.py @@ -0,0 +1,291 @@ +# coding=utf-8 + +from __future__ import print_function + +import re +import sqlite3 +import time + +from six import text_type + +import core +from core import logger + + +def db_filename(filename='nzbtomedia.db', suffix=None): + """ + @param filename: The sqlite database filename to use. If not specified, + will be made to be nzbtomedia.db + @param suffix: The suffix to append to the filename. A '.' will be added + automatically, i.e. suffix='v0' will make dbfile.db.v0 + @return: the correct location of the database file. + """ + if suffix: + filename = '{0}.{1}'.format(filename, suffix) + return core.os.path.join(core.APP_ROOT, filename) + + +class DBConnection(object): + def __init__(self, filename='nzbtomedia.db', suffix=None, row_type=None): + + self.filename = filename + self.connection = sqlite3.connect(db_filename(filename), 20) + if row_type == 'dict': + self.connection.row_factory = self._dict_factory + else: + self.connection.row_factory = sqlite3.Row + + def check_db_version(self): + result = None + try: + result = self.select('SELECT db_version FROM db_version') + except sqlite3.OperationalError as e: + if 'no such table: db_version' in e.args[0]: + return 0 + + if result: + return int(result[0]['db_version']) + else: + return 0 + + def fetch(self, query, args=None): + if query is None: + return + + sql_result = None + attempt = 0 + + while attempt < 5: + try: + if args is None: + logger.log('{name}: {query}'.format(name=self.filename, query=query), logger.DB) + cursor = self.connection.cursor() + cursor.execute(query) + sql_result = cursor.fetchone()[0] + else: + logger.log('{name}: {query} with args {args}'.format + (name=self.filename, query=query, args=args), logger.DB) + cursor = self.connection.cursor() + cursor.execute(query, args) + sql_result = cursor.fetchone()[0] + + # get out of the connection attempt loop since we were successful + break + except sqlite3.OperationalError as error: + if 'unable to open database file' in error.args[0] or 'database is locked' in error.args[0]: + logger.log(u'DB error: {msg}'.format(msg=error), logger.WARNING) + attempt += 1 + time.sleep(1) + else: + logger.log(u'DB error: {msg}'.format(msg=error), logger.ERROR) + raise + except sqlite3.DatabaseError as error: + logger.log(u'Fatal error executing query: {msg}'.format(msg=error), logger.ERROR) + raise + + return sql_result + + def mass_action(self, querylist, log_transaction=False): + if querylist is None: + return + + sql_result = [] + attempt = 0 + + while attempt < 5: + try: + for qu in querylist: + if len(qu) == 1: + if log_transaction: + logger.log(qu[0], logger.DEBUG) + sql_result.append(self.connection.execute(qu[0])) + elif len(qu) > 1: + if log_transaction: + logger.log(u'{query} with args {args}'.format(query=qu[0], args=qu[1]), logger.DEBUG) + sql_result.append(self.connection.execute(qu[0], qu[1])) + self.connection.commit() + logger.log(u'Transaction with {x} query\'s executed'.format(x=len(querylist)), logger.DEBUG) + return sql_result + except sqlite3.OperationalError as error: + sql_result = [] + if self.connection: + self.connection.rollback() + if 'unable to open database file' in error.args[0] or 'database is locked' in error.args[0]: + logger.log(u'DB error: {msg}'.format(msg=error), logger.WARNING) + attempt += 1 + time.sleep(1) + else: + logger.log(u'DB error: {msg}'.format(msg=error), logger.ERROR) + raise + except sqlite3.DatabaseError as error: + if self.connection: + self.connection.rollback() + logger.log(u'Fatal error executing query: {msg}'.format(msg=error), logger.ERROR) + raise + + return sql_result + + def action(self, query, args=None): + if query is None: + return + + sql_result = None + attempt = 0 + + while attempt < 5: + try: + if args is None: + logger.log(u'{name}: {query}'.format(name=self.filename, query=query), logger.DB) + sql_result = self.connection.execute(query) + else: + logger.log(u'{name}: {query} with args {args}'.format + (name=self.filename, query=query, args=args), logger.DB) + sql_result = self.connection.execute(query, args) + self.connection.commit() + # get out of the connection attempt loop since we were successful + break + except sqlite3.OperationalError as error: + if 'unable to open database file' in error.args[0] or 'database is locked' in error.args[0]: + logger.log(u'DB error: {msg}'.format(msg=error), logger.WARNING) + attempt += 1 + time.sleep(1) + else: + logger.log(u'DB error: {msg}'.format(msg=error), logger.ERROR) + raise + except sqlite3.DatabaseError as error: + logger.log(u'Fatal error executing query: {msg}'.format(msg=error), logger.ERROR) + raise + + return sql_result + + def select(self, query, args=None): + + sql_results = self.action(query, args).fetchall() + + if sql_results is None: + return [] + + return sql_results + + def upsert(self, table_name, value_dict, key_dict): + + def gen_params(my_dict): + return [ + '{key} = ?'.format(key=k) + for k in my_dict.keys() + ] + + changes_before = self.connection.total_changes + items = list(value_dict.values()) + list(key_dict.values()) + self.action( + 'UPDATE {table} ' + 'SET {params} ' + 'WHERE {conditions}'.format( + table=table_name, + params=', '.join(gen_params(value_dict)), + conditions=' AND '.join(gen_params(key_dict)) + ), + items + ) + + if self.connection.total_changes == changes_before: + self.action( + 'INSERT OR IGNORE INTO {table} ({columns}) ' + 'VALUES ({values})'.format( + table=table_name, + columns=', '.join(map(text_type, value_dict.keys())), + values=', '.join(['?'] * len(value_dict.values())) + ), + list(value_dict.values()) + ) + + def table_info(self, table_name): + # FIXME ? binding is not supported here, but I cannot find a way to escape a string manually + cursor = self.connection.execute('PRAGMA table_info({0})'.format(table_name)) + columns = {} + for column in cursor: + columns[column['name']] = {'type': column['type']} + return columns + + # http://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query + def _dict_factory(self, cursor, row): + d = {} + for idx, col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d + + +def sanity_check_database(connection, sanity_check): + sanity_check(connection).check() + + +class DBSanityCheck(object): + def __init__(self, connection): + self.connection = connection + + def check(self): + pass + + +# =============== +# = Upgrade API = +# =============== + +def upgrade_database(connection, schema): + logger.log(u'Checking database structure...', logger.MESSAGE) + _process_upgrade(connection, schema) + + +def pretty_name(class_name): + return ' '.join([x.group() for x in re.finditer('([A-Z])([a-z0-9]+)', class_name)]) + + +def _process_upgrade(connection, upgrade_class): + instance = upgrade_class(connection) + logger.log(u'Checking {name} database upgrade'.format + (name=pretty_name(upgrade_class.__name__)), logger.DEBUG) + if not instance.test(): + logger.log(u'Database upgrade required: {name}'.format + (name=pretty_name(upgrade_class.__name__)), logger.MESSAGE) + try: + instance.execute() + except sqlite3.DatabaseError as error: + print(u'Error in {name}: {msg}'.format + (name=upgrade_class.__name__, msg=error)) + raise + logger.log(u'{name} upgrade completed'.format + (name=upgrade_class.__name__), logger.DEBUG) + else: + logger.log(u'{name} upgrade not required'.format + (name=upgrade_class.__name__), logger.DEBUG) + + for upgradeSubClass in upgrade_class.__subclasses__(): + _process_upgrade(connection, upgradeSubClass) + + +# Base migration class. All future DB changes should be subclassed from this class +class SchemaUpgrade(object): + def __init__(self, connection): + self.connection = connection + + def has_table(self, table_name): + return len(self.connection.action('SELECT 1 FROM sqlite_master WHERE name = ?;', (table_name,)).fetchall()) > 0 + + def has_column(self, table_name, column): + return column in self.connection.table_info(table_name) + + def add_column(self, table, column, data_type='NUMERIC', default=0): + self.connection.action('ALTER TABLE {0} ADD {1} {2}'.format(table, column, data_type)) + self.connection.action('UPDATE {0} SET {1} = ?'.format(table, column), (default,)) + + def check_db_version(self): + result = self.connection.select('SELECT db_version FROM db_version') + if result: + return int(result[-1]['db_version']) + else: + return 0 + + def inc_db_version(self): + new_version = self.check_db_version() + 1 + self.connection.action('UPDATE db_version SET db_version = ?', [new_version]) + return new_version diff --git a/core/nzbToMediaAutoFork.py b/core/nzbToMediaAutoFork.py deleted file mode 100644 index a4bb92071..000000000 --- a/core/nzbToMediaAutoFork.py +++ /dev/null @@ -1,115 +0,0 @@ -# coding=utf-8 - -import requests - -from six import iteritems - -import core -from core import logger - - -def autoFork(section, inputCategory): - # auto-detect correct section - # config settings - - cfg = dict(core.CFG[section][inputCategory]) - - host = cfg.get("host") - port = cfg.get("port") - username = cfg.get("username") - password = cfg.get("password") - apikey = cfg.get("apikey") - ssl = int(cfg.get("ssl", 0)) - web_root = cfg.get("web_root", "") - replace = {'sickrage':'SickRage', 'sickchill':'SickChill', 'sickgear':'SickGear', 'medusa':'Medusa', 'sickbeard-api':'SickBeard-api'} - f1 = replace[cfg.get("fork", "auto")] if cfg.get("fork", "auto") in replace else cfg.get("fork", "auto") - try: - fork = core.FORKS.items()[core.FORKS.keys().index(f1)] - except: - fork = "auto" - protocol = "https://" if ssl else "http://" - - detected = False - if section == "NzbDrone": - logger.info("Attempting to verify {category} fork".format - (category=inputCategory)) - url = "{protocol}{host}:{port}{root}/api/rootfolder".format( - protocol=protocol, host=host, port=port, root=web_root) - headers = {"X-Api-Key": apikey} - try: - r = requests.get(url, headers=headers, stream=True, verify=False) - except requests.ConnectionError: - logger.warning("Could not connect to {0}:{1} to verify fork!".format(section, inputCategory)) - - if not r.ok: - logger.warning("Connection to {section}:{category} failed! " - "Check your configuration".format - (section=section, category=inputCategory)) - - fork = ['default', {}] - - elif fork == "auto": - params = core.ALL_FORKS - rem_params = [] - logger.info("Attempting to auto-detect {category} fork".format(category=inputCategory)) - # define the order to test. Default must be first since the default fork doesn't reject parameters. - # then in order of most unique parameters. - - if apikey: - url = "{protocol}{host}:{port}{root}/api/{apikey}/?cmd=help&subject=postprocess".format( - protocol=protocol, host=host, port=port, root=web_root, apikey=apikey) - else: - url = "{protocol}{host}:{port}{root}/home/postprocess/".format( - protocol=protocol, host=host, port=port, root=web_root) - - # attempting to auto-detect fork - try: - s = requests.Session() - if not apikey and username and password: - login = "{protocol}{host}:{port}{root}/login".format( - protocol=protocol, host=host, port=port, root=web_root) - login_params = {'username': username, 'password': password} - r = s.get(login, verify=False, timeout=(30,60)) - if r.status_code == 401 and r.cookies.get('_xsrf'): - login_params['_xsrf'] = r.cookies.get('_xsrf') - s.post(login, data=login_params, stream=True, verify=False) - r = s.get(url, auth=(username, password), verify=False) - except requests.ConnectionError: - logger.info("Could not connect to {section}:{category} to perform auto-fork detection!".format - (section=section, category=inputCategory)) - r = [] - if r and r.ok: - if apikey: - optionalParameters = [] - try: - optionalParameters = r.json()['data']['optionalParameters'].keys() - except: - optionalParameters = r.json()['data']['data']['optionalParameters'].keys() - for param in params: - if param not in optionalParameters: - rem_params.append(param) - else: - for param in params: - if 'name="{param}"'.format(param=param) not in r.text: - rem_params.append(param) - for param in rem_params: - params.pop(param) - for fork in sorted(iteritems(core.FORKS), reverse=False): - if params == fork[1]: - detected = True - break - if detected: - logger.info("{section}:{category} fork auto-detection successful ...".format - (section=section, category=inputCategory)) - elif rem_params: - logger.info("{section}:{category} fork auto-detection found custom params {params}".format - (section=section, category=inputCategory, params=params)) - fork = ['custom', params] - else: - logger.info("{section}:{category} fork auto-detection failed".format - (section=section, category=inputCategory)) - fork = core.FORKS.items()[core.FORKS.keys().index(core.FORK_DEFAULT)] - - logger.info("{section}:{category} fork set to {fork}".format - (section=section, category=inputCategory, fork=fork[0])) - return fork[0], fork[1] diff --git a/core/nzbToMediaConfig.py b/core/nzbToMediaConfig.py deleted file mode 100644 index 17738ffe2..000000000 --- a/core/nzbToMediaConfig.py +++ /dev/null @@ -1,534 +0,0 @@ -# coding=utf-8 - -from six import iteritems -import os -import shutil -import copy -import core -from configobj import * -from core import logger - -from itertools import chain - - -class Section(configobj.Section, object): - def isenabled(section): - # checks if subsection enabled, returns true/false if subsection specified otherwise returns true/false in {} - if not section.sections: - try: - value = list(ConfigObj.find_key(section, 'enabled'))[0] - except: - value = 0 - if int(value) == 1: - return section - else: - to_return = copy.deepcopy(section) - for section_name, subsections in to_return.items(): - for subsection in subsections: - try: - value = list(ConfigObj.find_key(subsections, 'enabled'))[0] - except: - value = 0 - - if int(value) != 1: - del to_return[section_name][subsection] - - # cleanout empty sections and subsections - for key in [k for (k, v) in to_return.items() if not v]: - del to_return[key] - - return to_return - - def findsection(section, key): - to_return = copy.deepcopy(section) - for subsection in to_return: - try: - value = list(ConfigObj.find_key(to_return[subsection], key))[0] - except: - value = None - - if not value: - del to_return[subsection] - else: - for category in to_return[subsection]: - if category != key: - del to_return[subsection][category] - - # cleanout empty sections and subsections - for key in [k for (k, v) in to_return.items() if not v]: - del to_return[key] - - return to_return - - def __getitem__(self, key): - if key in self.keys(): - return dict.__getitem__(self, key) - - to_return = copy.deepcopy(self) - for section, subsections in to_return.items(): - if section in key: - continue - if isinstance(subsections, Section) and subsections.sections: - for subsection, options in subsections.items(): - if subsection in key: - continue - if key in options: - return options[key] - - del subsections[subsection] - else: - if section not in key: - del to_return[section] - - # cleanout empty sections and subsections - for key in [k for (k, v) in to_return.items() if not v]: - del to_return[key] - - return to_return - - -class ConfigObj(configobj.ConfigObj, Section): - def __init__(self, *args, **kw): - if len(args) == 0: - args = (core.CONFIG_FILE,) - super(configobj.ConfigObj, self).__init__(*args, **kw) - self.interpolation = False - - @staticmethod - def find_key(node, kv): - if isinstance(node, list): - for i in node: - for x in ConfigObj.find_key(i, kv): - yield x - elif isinstance(node, dict): - if kv in node: - yield node[kv] - for j in node.values(): - for x in ConfigObj.find_key(j, kv): - yield x - - @staticmethod - def migrate(): - global CFG_NEW, CFG_OLD - CFG_NEW = None - CFG_OLD = None - - try: - # check for autoProcessMedia.cfg and create if it does not exist - if not os.path.isfile(core.CONFIG_FILE): - shutil.copyfile(core.CONFIG_SPEC_FILE, core.CONFIG_FILE) - CFG_OLD = config(core.CONFIG_FILE) - except Exception as error: - logger.debug("Error {msg} when copying to .cfg".format(msg=error)) - - try: - # check for autoProcessMedia.cfg.spec and create if it does not exist - if not os.path.isfile(core.CONFIG_SPEC_FILE): - shutil.copyfile(core.CONFIG_FILE, core.CONFIG_SPEC_FILE) - CFG_NEW = config(core.CONFIG_SPEC_FILE) - except Exception as error: - logger.debug("Error {msg} when copying to .spec".format(msg=error)) - - # check for autoProcessMedia.cfg and autoProcessMedia.cfg.spec and if they don't exist return and fail - if CFG_NEW is None or CFG_OLD is None: - return False - - subsections = {} - # gather all new-style and old-style sub-sections - for newsection, newitems in CFG_NEW.items(): - if CFG_NEW[newsection].sections: - subsections.update({newsection: CFG_NEW[newsection].sections}) - for section, items in CFG_OLD.items(): - if CFG_OLD[section].sections: - subsections.update({section: CFG_OLD[section].sections}) - for option, value in CFG_OLD[section].items(): - if option in ["category", "cpsCategory", "sbCategory", "hpCategory", "mlCategory", "gzCategory", "raCategory", "ndCategory"]: - if not isinstance(value, list): - value = [value] - - # add subsection - subsections.update({section: value}) - CFG_OLD[section].pop(option) - continue - - def cleanup_values(values, section): - for option, value in iteritems(values): - if section in ['CouchPotato']: - if option == ['outputDirectory']: - CFG_NEW['Torrent'][option] = os.path.split(os.path.normpath(value))[0] - values.pop(option) - if section in ['CouchPotato', 'HeadPhones', 'Gamez', 'Mylar']: - if option in ['username', 'password']: - values.pop(option) - if section in ["SickBeard", "Mylar"]: - if option == "wait_for": # remove old format - values.pop(option) - if section in ["SickBeard", "NzbDrone"]: - if option == "failed_fork": # change this old format - values['failed'] = 'auto' - values.pop(option) - if option == "outputDirectory": # move this to new location format - CFG_NEW['Torrent'][option] = os.path.split(os.path.normpath(value))[0] - values.pop(option) - if section in ["Torrent"]: - if option in ["compressedExtensions", "mediaExtensions", "metaExtensions", "minSampleSize"]: - CFG_NEW['Extensions'][option] = value - values.pop(option) - if option == "useLink": # Sym links supported now as well. - if value in ['1', 1]: - value = 'hard' - elif value in ['0', 0]: - value = 'no' - values[option] = value - if option == "forceClean": - CFG_NEW['General']['force_clean'] = value - values.pop(option) - if section in ["Transcoder"]: - if option in ["niceness"]: - CFG_NEW['Posix'][option] = value - values.pop(option) - if option == "remote_path": - if value and value not in ['0', '1', 0, 1]: - value = 1 - elif not value: - value = 0 - values[option] = value - # remove any options that we no longer need so they don't migrate into our new config - if not list(ConfigObj.find_key(CFG_NEW, option)): - try: - values.pop(option) - except: - pass - - return values - - def process_section(section, subsections=None): - if subsections: - for subsection in subsections: - if subsection in CFG_OLD.sections: - values = cleanup_values(CFG_OLD[subsection], section) - if subsection not in CFG_NEW[section].sections: - CFG_NEW[section][subsection] = {} - for option, value in values.items(): - CFG_NEW[section][subsection][option] = value - elif subsection in CFG_OLD[section].sections: - values = cleanup_values(CFG_OLD[section][subsection], section) - if subsection not in CFG_NEW[section].sections: - CFG_NEW[section][subsection] = {} - for option, value in values.items(): - CFG_NEW[section][subsection][option] = value - else: - values = cleanup_values(CFG_OLD[section], section) - if section not in CFG_NEW.sections: - CFG_NEW[section] = {} - for option, value in values.items(): - CFG_NEW[section][option] = value - - # convert old-style categories to new-style sub-sections - for section in CFG_OLD.keys(): - subsection = None - if section in list(chain.from_iterable(subsections.values())): - subsection = section - section = ''.join([k for k, v in iteritems(subsections) if subsection in v]) - process_section(section, subsection) - elif section in subsections.keys(): - subsection = subsections[section] - process_section(section, subsection) - elif section in CFG_OLD.keys(): - process_section(section, subsection) - - # create a backup of our old config - CFG_OLD.filename ="{config}.old".format(config=core.CONFIG_FILE) - CFG_OLD.write() - - # write our new config to autoProcessMedia.cfg - CFG_NEW.filename = core.CONFIG_FILE - CFG_NEW.write() - - return True - - @staticmethod - def addnzbget(): - # load configs into memory - CFG_NEW = config() - - try: - if 'NZBPO_NDCATEGORY' in os.environ and 'NZBPO_SBCATEGORY' in os.environ: - if os.environ['NZBPO_NDCATEGORY'] == os.environ['NZBPO_SBCATEGORY']: - logger.warning("{x} category is set for SickBeard and Sonarr. " - "Please check your config in NZBGet".format - (x=os.environ['NZBPO_NDCATEGORY'])) - if 'NZBPO_RACATEGORY' in os.environ and 'NZBPO_CPSCATEGORY' in os.environ: - if os.environ['NZBPO_RACATEGORY'] == os.environ['NZBPO_CPSCATEGORY']: - logger.warning("{x} category is set for CouchPotato and Radarr. " - "Please check your config in NZBGet".format - (x=os.environ['NZBPO_RACATEGORY'])) - if 'NZBPO_LICATEGORY' in os.environ and 'NZBPO_HPCATEGORY' in os.environ: - if os.environ['NZBPO_LICATEGORY'] == os.environ['NZBPO_HPCATEGORY']: - logger.warning("{x} category is set for HeadPhones and Lidarr. " - "Please check your config in NZBGet".format - (x=os.environ['NZBPO_LICATEGORY'])) - section = "Nzb" - key = 'NZBOP_DESTDIR' - if key in os.environ: - option = 'default_downloadDirectory' - value = os.environ[key] - CFG_NEW[section][option] = value - - section = "General" - envKeys = ['AUTO_UPDATE', 'CHECK_MEDIA', 'SAFE_MODE', 'NO_EXTRACT_FAILED'] - cfgKeys = ['auto_update', 'check_media', 'safe_mode', 'no_extract_failed'] - for index in range(len(envKeys)): - key = 'NZBPO_{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - CFG_NEW[section][option] = value - - section = "Network" - envKeys = ['MOUNTPOINTS'] - cfgKeys = ['mount_points'] - for index in range(len(envKeys)): - key = 'NZBPO_{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - CFG_NEW[section][option] = value - - section = "CouchPotato" - envCatKey = 'NZBPO_CPSCATEGORY' - envKeys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'METHOD', 'DELETE_FAILED', 'REMOTE_PATH', - 'WAIT_FOR', 'WATCH_DIR', 'OMDBAPIKEY'] - cfgKeys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'method', 'delete_failed', 'remote_path', - 'wait_for', 'watch_dir', 'omdbapikey'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_CPS{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - if os.environ[envCatKey] in CFG_NEW['Radarr'].sections: - CFG_NEW['Radarr'][envCatKey]['enabled'] = 0 - - section = "SickBeard" - envCatKey = 'NZBPO_SBCATEGORY' - envKeys = ['ENABLED', 'HOST', 'PORT', 'APIKEY', 'USERNAME', 'PASSWORD', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', - 'DELETE_FAILED', 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'REMOTE_PATH', 'PROCESS_METHOD'] - cfgKeys = ['enabled', 'host', 'port', 'apikey', 'username', 'password', 'ssl', 'web_root', 'watch_dir', 'fork', - 'delete_failed', 'Torrent_NoLink', 'nzbExtractionBy', 'remote_path', 'process_method'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_SB{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - if os.environ[envCatKey] in CFG_NEW['NzbDrone'].sections: - CFG_NEW['NzbDrone'][envCatKey]['enabled'] = 0 - - section = "HeadPhones" - envCatKey = 'NZBPO_HPCATEGORY' - envKeys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WAIT_FOR', 'WATCH_DIR', 'REMOTE_PATH', 'DELETE_FAILED'] - cfgKeys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'wait_for', 'watch_dir', 'remote_path', 'delete_failed'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_HP{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - if os.environ[envCatKey] in CFG_NEW['Lidarr'].sections: - CFG_NEW['Lidarr'][envCatKey]['enabled'] = 0 - - section = "Mylar" - envCatKey = 'NZBPO_MYCATEGORY' - envKeys = ['ENABLED', 'HOST', 'PORT', 'USERNAME', 'PASSWORD', 'APIKEY', 'SSL', 'WEB_ROOT', 'WATCH_DIR', - 'REMOTE_PATH'] - cfgKeys = ['enabled', 'host', 'port', 'username', 'password', 'apikey', 'ssl', 'web_root', 'watch_dir', - 'remote_path'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_MY{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - - section = "Gamez" - envCatKey = 'NZBPO_GZCATEGORY' - envKeys = ['ENABLED', 'APIKEY', 'HOST', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'LIBRARY', 'REMOTE_PATH'] - cfgKeys = ['enabled', 'apikey', 'host', 'port', 'ssl', 'web_root', 'watch_dir', 'library', 'remote_path'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_GZ{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - - section = "NzbDrone" - envCatKey = 'NZBPO_NDCATEGORY' - envKeys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', - 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH', 'IMPORTMODE'] - #new cfgKey added for importMode - cfgKeys = ['enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', - 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path','importMode'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_ND{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - if os.environ[envCatKey] in CFG_NEW['SickBeard'].sections: - CFG_NEW['SickBeard'][envCatKey]['enabled'] = 0 - - section = "Radarr" - envCatKey = 'NZBPO_RACATEGORY' - envKeys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', - 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH', 'OMDBAPIKEY', 'IMPORTMODE'] - #new cfgKey added for importMode - cfgKeys = ['enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', - 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path', 'omdbapikey','importMode'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_RA{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - if os.environ[envCatKey] in CFG_NEW['CouchPotato'].sections: - CFG_NEW['CouchPotato'][envCatKey]['enabled'] = 0 - - section = "Lidarr" - envCatKey = 'NZBPO_LICATEGORY' - envKeys = ['ENABLED', 'HOST', 'APIKEY', 'PORT', 'SSL', 'WEB_ROOT', 'WATCH_DIR', 'FORK', 'DELETE_FAILED', - 'TORRENT_NOLINK', 'NZBEXTRACTIONBY', 'WAIT_FOR', 'DELETE_FAILED', 'REMOTE_PATH'] - cfgKeys = ['enabled', 'host', 'apikey', 'port', 'ssl', 'web_root', 'watch_dir', 'fork', 'delete_failed', - 'Torrent_NoLink', 'nzbExtractionBy', 'wait_for', 'delete_failed', 'remote_path'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_LI{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - if os.environ[envCatKey] in CFG_NEW['HeadPhones'].sections: - CFG_NEW['HeadPhones'][envCatKey]['enabled'] = 0 - - section = "Extensions" - envKeys = ['COMPRESSEDEXTENSIONS', 'MEDIAEXTENSIONS', 'METAEXTENSIONS'] - cfgKeys = ['compressedExtensions', 'mediaExtensions', 'metaExtensions'] - for index in range(len(envKeys)): - key = 'NZBPO_{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - CFG_NEW[section][option] = value - - section = "Posix" - envKeys = ['NICENESS', 'IONICE_CLASS', 'IONICE_CLASSDATA'] - cfgKeys = ['niceness', 'ionice_class', 'ionice_classdata'] - for index in range(len(envKeys)): - key = 'NZBPO_{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - CFG_NEW[section][option] = value - - section = "Transcoder" - envKeys = ['TRANSCODE', 'DUPLICATE', 'IGNOREEXTENSIONS', 'OUTPUTFASTSTART', 'OUTPUTVIDEOPATH', - 'PROCESSOUTPUT', 'AUDIOLANGUAGE', 'ALLAUDIOLANGUAGES', 'SUBLANGUAGES', - 'ALLSUBLANGUAGES', 'EMBEDSUBS', 'BURNINSUBTITLE', 'EXTRACTSUBS', 'EXTERNALSUBDIR', - 'OUTPUTDEFAULT', 'OUTPUTVIDEOEXTENSION', 'OUTPUTVIDEOCODEC', 'VIDEOCODECALLOW', - 'OUTPUTVIDEOPRESET', 'OUTPUTVIDEOFRAMERATE', 'OUTPUTVIDEOBITRATE', 'OUTPUTAUDIOCODEC', - 'AUDIOCODECALLOW', 'OUTPUTAUDIOBITRATE', 'OUTPUTQUALITYPERCENT', 'GETSUBS', - 'OUTPUTAUDIOTRACK2CODEC', 'AUDIOCODEC2ALLOW', 'OUTPUTAUDIOTRACK2BITRATE', - 'OUTPUTAUDIOOTHERCODEC', 'AUDIOOTHERCODECALLOW', 'OUTPUTAUDIOOTHERBITRATE', - 'OUTPUTSUBTITLECODEC', 'OUTPUTAUDIOCHANNELS', 'OUTPUTAUDIOTRACK2CHANNELS', - 'OUTPUTAUDIOOTHERCHANNELS','OUTPUTVIDEORESOLUTION'] - cfgKeys = ['transcode', 'duplicate', 'ignoreExtensions', 'outputFastStart', 'outputVideoPath', - 'processOutput', 'audioLanguage', 'allAudioLanguages', 'subLanguages', - 'allSubLanguages', 'embedSubs', 'burnInSubtitle', 'extractSubs', 'externalSubDir', - 'outputDefault', 'outputVideoExtension', 'outputVideoCodec', 'VideoCodecAllow', - 'outputVideoPreset', 'outputVideoFramerate', 'outputVideoBitrate', 'outputAudioCodec', - 'AudioCodecAllow', 'outputAudioBitrate', 'outputQualityPercent', 'getSubs', - 'outputAudioTrack2Codec', 'AudioCodec2Allow', 'outputAudioTrack2Bitrate', - 'outputAudioOtherCodec', 'AudioOtherCodecAllow', 'outputAudioOtherBitrate', - 'outputSubtitleCodec', 'outputAudioChannels', 'outputAudioTrack2Channels', - 'outputAudioOtherChannels', 'outputVideoResolution'] - for index in range(len(envKeys)): - key = 'NZBPO_{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - CFG_NEW[section][option] = value - - section = "WakeOnLan" - envKeys = ['WAKE', 'HOST', 'PORT', 'MAC'] - cfgKeys = ['wake', 'host', 'port', 'mac'] - for index in range(len(envKeys)): - key = 'NZBPO_WOL{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - CFG_NEW[section][option] = value - - section = "UserScript" - envCatKey = 'NZBPO_USCATEGORY' - envKeys = ['USER_SCRIPT_MEDIAEXTENSIONS', 'USER_SCRIPT_PATH', 'USER_SCRIPT_PARAM', 'USER_SCRIPT_RUNONCE', - 'USER_SCRIPT_SUCCESSCODES', 'USER_SCRIPT_CLEAN', 'USDELAY', 'USREMOTE_PATH'] - cfgKeys = ['user_script_mediaExtensions', 'user_script_path', 'user_script_param', 'user_script_runOnce', - 'user_script_successCodes', 'user_script_clean', 'delay', 'remote_path'] - if envCatKey in os.environ: - for index in range(len(envKeys)): - key = 'NZBPO_{index}'.format(index=envKeys[index]) - if key in os.environ: - option = cfgKeys[index] - value = os.environ[key] - if os.environ[envCatKey] not in CFG_NEW[section].sections: - CFG_NEW[section][os.environ[envCatKey]] = {} - CFG_NEW[section][os.environ[envCatKey]][option] = value - CFG_NEW[section][os.environ[envCatKey]]['enabled'] = 1 - - except Exception as error: - logger.debug("Error {msg} when applying NZBGet config".format(msg=error)) - - try: - # write our new config to autoProcessMedia.cfg - CFG_NEW.filename = core.CONFIG_FILE - CFG_NEW.write() - except Exception as error: - logger.debug("Error {msg} when writing changes to .cfg".format(msg=error)) - - return CFG_NEW - - -configobj.Section = Section -configobj.ConfigObj = ConfigObj -config = ConfigObj diff --git a/core/nzbToMediaDB.py b/core/nzbToMediaDB.py deleted file mode 100644 index de9e3c916..000000000 --- a/core/nzbToMediaDB.py +++ /dev/null @@ -1,284 +0,0 @@ -# coding=utf-8 - -from __future__ import print_function, with_statement - -import re -import sqlite3 -import time - -import core -from core import logger - - -def dbFilename(filename="nzbtomedia.db", suffix=None): - """ - @param filename: The sqlite database filename to use. If not specified, - will be made to be nzbtomedia.db - @param suffix: The suffix to append to the filename. A '.' will be added - automatically, i.e. suffix='v0' will make dbfile.db.v0 - @return: the correct location of the database file. - """ - if suffix: - filename = "{0}.{1}".format(filename, suffix) - return core.os.path.join(core.PROGRAM_DIR, filename) - - -class DBConnection(object): - def __init__(self, filename="nzbtomedia.db", suffix=None, row_type=None): - - self.filename = filename - self.connection = sqlite3.connect(dbFilename(filename), 20) - if row_type == "dict": - self.connection.row_factory = self._dict_factory - else: - self.connection.row_factory = sqlite3.Row - - def checkDBVersion(self): - result = None - try: - result = self.select("SELECT db_version FROM db_version") - except sqlite3.OperationalError as e: - if "no such table: db_version" in e.args[0]: - return 0 - - if result: - return int(result[0]["db_version"]) - else: - return 0 - - def fetch(self, query, args=None): - if query is None: - return - - sqlResult = None - attempt = 0 - - while attempt < 5: - try: - if args is None: - logger.log("{name}: {query}".format(name=self.filename, query=query), logger.DB) - cursor = self.connection.cursor() - cursor.execute(query) - sqlResult = cursor.fetchone()[0] - else: - logger.log("{name}: {query} with args {args}".format - (name=self.filename, query=query, args=args), logger.DB) - cursor = self.connection.cursor() - cursor.execute(query, args) - sqlResult = cursor.fetchone()[0] - - # get out of the connection attempt loop since we were successful - break - except sqlite3.OperationalError as error: - if "unable to open database file" in error.args[0] or "database is locked" in error.args[0]: - logger.log(u"DB error: {msg}".format(msg=error), logger.WARNING) - attempt += 1 - time.sleep(1) - else: - logger.log(u"DB error: {msg}".format(msg=error), logger.ERROR) - raise - except sqlite3.DatabaseError as error: - logger.log(u"Fatal error executing query: {msg}".format(msg=error), logger.ERROR) - raise - - return sqlResult - - def mass_action(self, querylist, logTransaction=False): - if querylist is None: - return - - sqlResult = [] - attempt = 0 - - while attempt < 5: - try: - for qu in querylist: - if len(qu) == 1: - if logTransaction: - logger.log(qu[0], logger.DEBUG) - sqlResult.append(self.connection.execute(qu[0])) - elif len(qu) > 1: - if logTransaction: - logger.log(u"{query} with args {args}".format(query=qu[0], args=qu[1]), logger.DEBUG) - sqlResult.append(self.connection.execute(qu[0], qu[1])) - self.connection.commit() - logger.log(u"Transaction with {x} query's executed".format(x=len(querylist)), logger.DEBUG) - return sqlResult - except sqlite3.OperationalError as error: - sqlResult = [] - if self.connection: - self.connection.rollback() - if "unable to open database file" in error.args[0] or "database is locked" in error.args[0]: - logger.log(u"DB error: {msg}".format(msg=error), logger.WARNING) - attempt += 1 - time.sleep(1) - else: - logger.log(u"DB error: {msg}".format(msg=error), logger.ERROR) - raise - except sqlite3.DatabaseError as error: - if self.connection: - self.connection.rollback() - logger.log(u"Fatal error executing query: {msg}".format(msg=error), logger.ERROR) - raise - - return sqlResult - - def action(self, query, args=None): - if query is None: - return - - sqlResult = None - attempt = 0 - - while attempt < 5: - try: - if args is None: - logger.log(u"{name}: {query}".format(name=self.filename, query=query), logger.DB) - sqlResult = self.connection.execute(query) - else: - logger.log(u"{name}: {query} with args {args}".format - (name=self.filename, query=query, args=args), logger.DB) - sqlResult = self.connection.execute(query, args) - self.connection.commit() - # get out of the connection attempt loop since we were successful - break - except sqlite3.OperationalError as error: - if "unable to open database file" in error.args[0] or "database is locked" in error.args[0]: - logger.log(u"DB error: {msg}".format(msg=error), logger.WARNING) - attempt += 1 - time.sleep(1) - else: - logger.log(u"DB error: {msg}".format(msg=error), logger.ERROR) - raise - except sqlite3.DatabaseError as error: - logger.log(u"Fatal error executing query: {msg}".format(msg=error), logger.ERROR) - raise - - return sqlResult - - def select(self, query, args=None): - - sqlResults = self.action(query, args).fetchall() - - if sqlResults is None: - return [] - - return sqlResults - - def upsert(self, tableName, valueDict, keyDict): - - changesBefore = self.connection.total_changes - - genParams = lambda myDict: ["{key} = ?".format(key=k) for k in myDict.keys()] - - self.action( - "UPDATE {table} " - "SET {params} " - "WHERE {conditions}".format( - table=tableName, - params=", ".join(genParams(valueDict)), - conditions=" AND ".join(genParams(keyDict))), - valueDict.values() + keyDict.values() - ) - - if self.connection.total_changes == changesBefore: - self.action( - "INSERT OR IGNORE INTO {table} ({columns}) " - "VALUES ({values})".format( - table=tableName, - columns=", ".join(valueDict.keys() + keyDict.keys()), - values=", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) - ) - , valueDict.values() + keyDict.values() - ) - - def tableInfo(self, tableName): - # FIXME ? binding is not supported here, but I cannot find a way to escape a string manually - cursor = self.connection.execute("PRAGMA table_info({0})".format(tableName)) - columns = {} - for column in cursor: - columns[column['name']] = {'type': column['type']} - return columns - - # http://stackoverflow.com/questions/3300464/how-can-i-get-dict-from-sqlite-query - def _dict_factory(self, cursor, row): - d = {} - for idx, col in enumerate(cursor.description): - d[col[0]] = row[idx] - return d - - -def sanityCheckDatabase(connection, sanity_check): - sanity_check(connection).check() - - -class DBSanityCheck(object): - def __init__(self, connection): - self.connection = connection - - def check(self): - pass - - -# =============== -# = Upgrade API = -# =============== - -def upgradeDatabase(connection, schema): - logger.log(u"Checking database structure...", logger.MESSAGE) - _processUpgrade(connection, schema) - - -def prettyName(class_name): - return ' '.join([x.group() for x in re.finditer("([A-Z])([a-z0-9]+)", class_name)]) - - -def _processUpgrade(connection, upgradeClass): - instance = upgradeClass(connection) - logger.log(u"Checking {name} database upgrade".format - (name=prettyName(upgradeClass.__name__)), logger.DEBUG) - if not instance.test(): - logger.log(u"Database upgrade required: {name}".format - (name=prettyName(upgradeClass.__name__)), logger.MESSAGE) - try: - instance.execute() - except sqlite3.DatabaseError as error: - print(u"Error in {name}: {msg}".format - (name=upgradeClass.__name__, msg=error)) - raise - logger.log(u"{name} upgrade completed".format - (name=upgradeClass.__name__), logger.DEBUG) - else: - logger.log(u"{name} upgrade not required".format - (name=upgradeClass.__name__), logger.DEBUG) - - for upgradeSubClass in upgradeClass.__subclasses__(): - _processUpgrade(connection, upgradeSubClass) - - -# Base migration class. All future DB changes should be subclassed from this class -class SchemaUpgrade(object): - def __init__(self, connection): - self.connection = connection - - def hasTable(self, tableName): - return len(self.connection.action("SELECT 1 FROM sqlite_master WHERE name = ?;", (tableName,)).fetchall()) > 0 - - def hasColumn(self, tableName, column): - return column in self.connection.tableInfo(tableName) - - def addColumn(self, table, column, type="NUMERIC", default=0): - self.connection.action("ALTER TABLE {0} ADD {1} {2}".format(table, column, type)) - self.connection.action("UPDATE {0} SET {1} = ?".format(table, column), (default,)) - - def checkDBVersion(self): - result = self.connection.select("SELECT db_version FROM db_version") - if result: - return int(result[-1]["db_version"]) - else: - return 0 - - def incDBVersion(self): - new_version = self.checkDBVersion() + 1 - self.connection.action("UPDATE db_version SET db_version = ?", [new_version]) - return new_version diff --git a/core/nzbToMediaSceneExceptions.py b/core/nzbToMediaSceneExceptions.py deleted file mode 100644 index 21289d122..000000000 --- a/core/nzbToMediaSceneExceptions.py +++ /dev/null @@ -1,186 +0,0 @@ -# coding=utf-8 -import os -import re -import core -import shlex -import platform -import subprocess -from core import logger -from core.nzbToMediaUtil import listMediaFiles - -reverse_list = [r"\.\d{2}e\d{2}s\.", r"\.[pi]0801\.", r"\.p027\.", r"\.[pi]675\.", r"\.[pi]084\.", r"\.p063\.", - r"\b[45]62[xh]\.", r"\.yarulb\.", r"\.vtd[hp]\.", - r"\.ld[.-]?bew\.", r"\.pir.?(dov|dvd|bew|db|rb)\.", r"\brdvd\.", r"\.vts\.", r"\.reneercs\.", - r"\.dcv\.", r"\b(pir|mac)dh\b", r"\.reporp\.", r"\.kcaper\.", - r"\.lanretni\.", r"\b3ca\b", r"\.cstn\."] -reverse_pattern = re.compile('|'.join(reverse_list), flags=re.IGNORECASE) -season_pattern = re.compile(r"(.*\.\d{2}e\d{2}s\.)(.*)", flags=re.IGNORECASE) -word_pattern = re.compile(r"([^A-Z0-9]*[A-Z0-9]+)") -media_list = [r"\.s\d{2}e\d{2}\.", r"\.1080[pi]\.", r"\.720p\.", r"\.576[pi]", r"\.480[pi]\.", r"\.360p\.", - r"\.[xh]26[45]\b", r"\.bluray\.", r"\.[hp]dtv\.", - r"\.web[.-]?dl\.", r"\.(vod|dvd|web|bd|br).?rip\.", r"\.dvdr\b", r"\.stv\.", r"\.screener\.", r"\.vcd\.", - r"\bhd(cam|rip)\b", r"\.proper\.", r"\.repack\.", - r"\.internal\.", r"\bac3\b", r"\.ntsc\.", r"\.pal\.", r"\.secam\.", r"\bdivx\b", r"\bxvid\b"] -media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) -garbage_name = re.compile(r"^[a-zA-Z0-9]*$") -char_replace = [[r"(\w)1\.(\w)", r"\1i\2"] - ] - - -def process_all_exceptions(name, dirname): - par2(dirname) - rename_script(dirname) - for filename in listMediaFiles(dirname): - newfilename = None - parentDir = os.path.dirname(filename) - head, fileExtension = os.path.splitext(os.path.basename(filename)) - if reverse_pattern.search(head) is not None: - exception = reverse_filename - elif garbage_name.search(head) is not None: - exception = replace_filename - else: - exception = None - newfilename = filename - if not newfilename: - newfilename = exception(filename, parentDir, name) - if core.GROUPS: - newfilename = strip_groups(newfilename) - if newfilename != filename: - rename_file(filename, newfilename) - - -def strip_groups(filename): - if not core.GROUPS: - return filename - dirname, file = os.path.split(filename) - head, fileExtension = os.path.splitext(file) - newname = head.replace(' ', '.') - for group in core.GROUPS: - newname = newname.replace(group, '') - newname = newname.replace('[]', '') - newfile = newname + fileExtension - newfilePath = os.path.join(dirname, newfile) - return newfilePath - - -def rename_file(filename, newfilePath): - if os.path.isfile(newfilePath): - newfilePath = os.path.splitext(newfilePath)[0] + ".NTM" + os.path.splitext(newfilePath)[1] - logger.debug("Replacing file name {old} with download name {new}".format - (old=filename, new=newfilePath), "EXCEPTION") - try: - os.rename(filename, newfilePath) - except Exception as error: - logger.error("Unable to rename file due to: {error}".format(error=error), "EXCEPTION") - - -def replace_filename(filename, dirname, name): - head, fileExtension = os.path.splitext(os.path.basename(filename)) - if media_pattern.search(os.path.basename(dirname).replace(' ', '.')) is not None: - newname = os.path.basename(dirname).replace(' ', '.') - logger.debug("Replacing file name {old} with directory name {new}".format(old=head, new=newname), "EXCEPTION") - elif media_pattern.search(name.replace(' ', '.').lower()) is not None: - newname = name.replace(' ', '.') - logger.debug("Replacing file name {old} with download name {new}".format - (old=head, new=newname), "EXCEPTION") - else: - logger.warning("No name replacement determined for {name}".format(name=head), "EXCEPTION") - newname = name - newfile = newname + fileExtension - newfilePath = os.path.join(dirname, newfile) - return newfilePath - - -def reverse_filename(filename, dirname, name): - head, fileExtension = os.path.splitext(os.path.basename(filename)) - na_parts = season_pattern.search(head) - if na_parts is not None: - word_p = word_pattern.findall(na_parts.group(2)) - if word_p: - new_words = "" - for wp in word_p: - if wp[0] == ".": - new_words += "." - new_words += re.sub(r"\W", "", wp) - else: - new_words = na_parts.group(2) - for cr in char_replace: - new_words = re.sub(cr[0], cr[1], new_words) - newname = new_words[::-1] + na_parts.group(1)[::-1] - else: - newname = head[::-1].title() - newname = newname.replace(' ', '.') - logger.debug("Reversing filename {old} to {new}".format - (old=head, new=newname), "EXCEPTION") - newfile = newname + fileExtension - newfilePath = os.path.join(dirname, newfile) - return newfilePath - - -def rename_script(dirname): - rename_file = "" - for dir, dirs, files in os.walk(dirname): - for file in files: - if re.search('(rename\S*\.(sh|bat)$)', file, re.IGNORECASE): - rename_file = os.path.join(dir, file) - dirname = dir - break - if rename_file: - rename_lines = [line.strip() for line in open(rename_file)] - for line in rename_lines: - if re.search('^(mv|Move)', line, re.IGNORECASE): - cmd = shlex.split(line)[1:] - else: - continue - if len(cmd) == 2 and os.path.isfile(os.path.join(dirname, cmd[0])): - orig = os.path.join(dirname, cmd[0]) - dest = os.path.join(dirname, cmd[1].split('\\')[-1].split('/')[-1]) - if os.path.isfile(dest): - continue - logger.debug("Renaming file {source} to {destination}".format - (source=orig, destination=dest), "EXCEPTION") - try: - os.rename(orig, dest) - except Exception as error: - logger.error("Unable to rename file due to: {error}".format(error=error), "EXCEPTION") - -def par2(dirname): - newlist = [] - sofar = 0 - parfile = "" - objects = [] - if os.path.exists(dirname): - objects = os.listdir(dirname) - for item in objects: - if item.endswith(".par2"): - size = os.path.getsize(os.path.join(dirname, item)) - if size > sofar: - sofar = size - parfile = item - if core.PAR2CMD and parfile: - pwd = os.getcwd() # Get our Present Working Directory - os.chdir(dirname) # set directory to run par on. - if platform.system() == 'Windows': - bitbucket = open('NUL') - else: - bitbucket = open('/dev/null') - logger.info("Running par2 on file {0}.".format(parfile), "PAR2") - command = [core.PAR2CMD, 'r', parfile, "*"] - cmd = "" - for item in command: - cmd = "{cmd} {item}".format(cmd=cmd, item=item) - logger.debug("calling command:{0}".format(cmd), "PAR2") - try: - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) - proc.communicate() - result = proc.returncode - except: - logger.error("par2 file processing for {0} has failed".format(parfile), "PAR2") - if result == 0: - logger.info("par2 file processing succeeded", "PAR2") - os.chdir(pwd) - bitbucket.close() - -# dict for custom groups -# we can add more to this list -# _customgroups = {'Q o Q': process_qoq, '-ECI': process_eci} diff --git a/core/nzbToMediaUserScript.py b/core/nzbToMediaUserScript.py deleted file mode 100644 index 679521f1d..000000000 --- a/core/nzbToMediaUserScript.py +++ /dev/null @@ -1,116 +0,0 @@ -# coding=utf-8 -import os -import core -from subprocess import Popen -from core.transcoder import transcoder -from core.nzbToMediaUtil import import_subs, listMediaFiles, rmDir -from core import logger - - -def external_script(outputDestination, torrentName, torrentLabel, settings): - final_result = 0 # start at 0. - num_files = 0 - try: - core.USER_SCRIPT_MEDIAEXTENSIONS = settings["user_script_mediaExtensions"].lower() - if isinstance(core.USER_SCRIPT_MEDIAEXTENSIONS, str): - core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.split(',') - except: - core.USER_SCRIPT_MEDIAEXTENSIONS = [] - - core.USER_SCRIPT = settings.get("user_script_path") - - if not core.USER_SCRIPT or core.USER_SCRIPT == "None": # do nothing and return success. - return [0, ""] - try: - core.USER_SCRIPT_PARAM = settings["user_script_param"] - if isinstance(core.USER_SCRIPT_PARAM, str): - core.USER_SCRIPT_PARAM = core.USER_SCRIPT_PARAM.split(',') - except: - core.USER_SCRIPT_PARAM = [] - try: - core.USER_SCRIPT_SUCCESSCODES = settings["user_script_successCodes"] - if isinstance(core.USER_SCRIPT_SUCCESSCODES, str): - core.USER_SCRIPT_SUCCESSCODES = core.USER_SCRIPT_SUCCESSCODES.split(',') - except: - core.USER_SCRIPT_SUCCESSCODES = 0 - - core.USER_SCRIPT_CLEAN = int(settings.get("user_script_clean", 1)) - core.USER_SCRIPT_RUNONCE = int(settings.get("user_script_runOnce", 1)) - - if core.CHECK_MEDIA: - for video in listMediaFiles(outputDestination, media=True, audio=False, meta=False, archives=False): - if transcoder.isVideoGood(video, 0): - import_subs(video) - else: - logger.info("Corrupt video file found {0}. Deleting.".format(video), "USERSCRIPT") - os.unlink(video) - - for dirpath, dirnames, filenames in os.walk(outputDestination): - for file in filenames: - - filePath = core.os.path.join(dirpath, file) - fileName, fileExtension = os.path.splitext(file) - - if fileExtension in core.USER_SCRIPT_MEDIAEXTENSIONS or "all" in core.USER_SCRIPT_MEDIAEXTENSIONS: - num_files += 1 - if core.USER_SCRIPT_RUNONCE == 1 and num_files > 1: # we have already run once, so just continue to get number of files. - continue - command = [core.USER_SCRIPT] - for param in core.USER_SCRIPT_PARAM: - if param == "FN": - command.append('{0}'.format(file)) - continue - elif param == "FP": - command.append('{0}'.format(filePath)) - continue - elif param == "TN": - command.append('{0}'.format(torrentName)) - continue - elif param == "TL": - command.append('{0}'.format(torrentLabel)) - continue - elif param == "DN": - if core.USER_SCRIPT_RUNONCE == 1: - command.append('{0}'.format(outputDestination)) - else: - command.append('{0}'.format(dirpath)) - continue - else: - command.append(param) - continue - cmd = "" - for item in command: - cmd = "{cmd} {item}".format(cmd=cmd, item=item) - logger.info("Running script {cmd} on file {path}.".format(cmd=cmd, path=filePath), "USERSCRIPT") - try: - p = Popen(command) - res = p.wait() - if str(res) in core.USER_SCRIPT_SUCCESSCODES: # Linux returns 0 for successful. - logger.info("UserScript {0} was successfull".format(command[0])) - result = 0 - else: - logger.error("UserScript {0} has failed with return code: {1}".format(command[0], res), "USERSCRIPT") - logger.info( - "If the UserScript completed successfully you should add {0} to the user_script_successCodes".format( - res), "USERSCRIPT") - result = int(1) - except: - logger.error("UserScript {0} has failed".format(command[0]), "USERSCRIPT") - result = int(1) - final_result += result - - num_files_new = 0 - for dirpath, dirnames, filenames in os.walk(outputDestination): - for file in filenames: - fileName, fileExtension = os.path.splitext(file) - - if fileExtension in core.USER_SCRIPT_MEDIAEXTENSIONS or core.USER_SCRIPT_MEDIAEXTENSIONS == "ALL": - num_files_new += 1 - - if core.USER_SCRIPT_CLEAN == int(1) and num_files_new == 0 and final_result == 0: - logger.info("All files have been processed. Cleaning outputDirectory {0}".format(outputDestination)) - rmDir(outputDestination) - elif core.USER_SCRIPT_CLEAN == int(1) and num_files_new != 0: - logger.info("{0} files were processed, but {1} still remain. outputDirectory will not be cleaned.".format( - num_files, num_files_new)) - return [final_result, ''] diff --git a/core/nzbToMediaUtil.py b/core/nzbToMediaUtil.py deleted file mode 100644 index 731c0856c..000000000 --- a/core/nzbToMediaUtil.py +++ /dev/null @@ -1,1390 +0,0 @@ -# coding=utf-8 - -from __future__ import print_function, unicode_literals -from six import text_type -import os -import re -import socket -import stat -import struct -import shutil -import time -import datetime -import platform -import guessit -import beets -import requests -import core -from babelfish import Language -import subliminal - -from core.extractor import extractor -from core.linktastic import linktastic -from core.synchronousdeluge.client import DelugeClient -from core.utorrent.client import UTorrentClient -from core.transmissionrpc.client import Client as TransmissionClient -from core.qbittorrent.client import Client as qBittorrentClient -from core import logger, nzbToMediaDB - -requests.packages.urllib3.disable_warnings() - -# Monkey Patch shutil.copyfileobj() to adjust the buffer length to 512KB rather than 4KB -shutil.copyfileobjOrig = shutil.copyfileobj -def copyfileobjFast(fsrc, fdst, length=512*1024): - shutil.copyfileobjOrig(fsrc, fdst, length=length) -shutil.copyfileobj = copyfileobjFast - -def reportNzb(failure_link, clientAgent): - # Contact indexer site - logger.info("Sending failure notification to indexer site") - if clientAgent == 'nzbget': - headers = {'User-Agent': 'NZBGet / nzbToMedia.py'} - elif clientAgent == 'sabnzbd': - headers = {'User-Agent': 'SABnzbd / nzbToMedia.py'} - else: - return - try: - requests.post(failure_link, headers=headers, timeout=(30, 300)) - except Exception as e: - logger.error("Unable to open URL {0} due to {1}".format(failure_link, e)) - return - - -def sanitizeName(name): - """ - >>> sanitizeName('a/b/c') - 'a-b-c' - >>> sanitizeName('abc') - 'abc' - >>> sanitizeName('a"b') - 'ab' - >>> sanitizeName('.a.b..') - 'a.b' - """ - - # remove bad chars from the filename - name = re.sub(r'[\\\/*]', '-', name) - name = re.sub(r'[:"<>|?]', '', name) - - # remove leading/trailing periods and spaces - name = name.strip(' .') - try: - name = name.encode(core.SYS_ENCODING) - except: - pass - - return name - - -def makeDir(path): - if not os.path.isdir(path): - try: - os.makedirs(path) - except Exception: - return False - return True - - -def remoteDir(path): - if not core.REMOTEPATHS: - return path - for local, remote in core.REMOTEPATHS: - if local in path: - base_dirs = path.replace(local, "").split(os.sep) - if '/' in remote: - remote_sep = '/' - else: - remote_sep = '\\' - new_path = remote_sep.join([remote] + base_dirs) - new_path = re.sub(r'(\S)(\\+)', r'\1\\', new_path) - new_path = re.sub(r'(\/+)', r'/', new_path) - new_path = re.sub(r'([\/\\])$', r'', new_path) - return new_path - return path - - -def category_search(inputDirectory, inputName, inputCategory, root, categories): - tordir = False - - try: - inputName = inputName.encode(core.SYS_ENCODING) - except: - pass - try: - inputDirectory = inputDirectory.encode(core.SYS_ENCODING) - except: - pass - - if inputDirectory is None: # =Nothing to process here. - return inputDirectory, inputName, inputCategory, root - - pathlist = os.path.normpath(inputDirectory).split(os.sep) - - if inputCategory and inputCategory in pathlist: - logger.debug("SEARCH: Found the Category: {0} in directory structure".format(inputCategory)) - elif inputCategory: - logger.debug("SEARCH: Could not find the category: {0} in the directory structure".format(inputCategory)) - else: - try: - inputCategory = list(set(pathlist) & set(categories))[-1] # assume last match is most relevant category. - logger.debug("SEARCH: Found Category: {0} in directory structure".format(inputCategory)) - except IndexError: - inputCategory = "" - logger.debug("SEARCH: Could not find a category in the directory structure") - if not os.path.isdir(inputDirectory) and os.path.isfile(inputDirectory): # If the input directory is a file - if not inputName: - inputName = os.path.split(os.path.normpath(inputDirectory))[1] - return inputDirectory, inputName, inputCategory, root - - if inputCategory and os.path.isdir(os.path.join(inputDirectory, inputCategory)): - logger.info( - "SEARCH: Found category directory {0} in input directory directory {1}".format(inputCategory, inputDirectory)) - inputDirectory = os.path.join(inputDirectory, inputCategory) - logger.info("SEARCH: Setting inputDirectory to {0}".format(inputDirectory)) - if inputName and os.path.isdir(os.path.join(inputDirectory, inputName)): - logger.info("SEARCH: Found torrent directory {0} in input directory directory {1}".format(inputName, inputDirectory)) - inputDirectory = os.path.join(inputDirectory, inputName) - logger.info("SEARCH: Setting inputDirectory to {0}".format(inputDirectory)) - tordir = True - elif inputName and os.path.isdir(os.path.join(inputDirectory, sanitizeName(inputName))): - logger.info("SEARCH: Found torrent directory {0} in input directory directory {1}".format( - sanitizeName(inputName), inputDirectory)) - inputDirectory = os.path.join(inputDirectory, sanitizeName(inputName)) - logger.info("SEARCH: Setting inputDirectory to {0}".format(inputDirectory)) - tordir = True - elif inputName and os.path.isfile(os.path.join(inputDirectory, inputName)): - logger.info("SEARCH: Found torrent file {0} in input directory directory {1}".format(inputName, inputDirectory)) - inputDirectory = os.path.join(inputDirectory, inputName) - logger.info("SEARCH: Setting inputDirectory to {0}".format(inputDirectory)) - tordir = True - elif inputName and os.path.isfile(os.path.join(inputDirectory, sanitizeName(inputName))): - logger.info("SEARCH: Found torrent file {0} in input directory directory {1}".format( - sanitizeName(inputName), inputDirectory)) - inputDirectory = os.path.join(inputDirectory, sanitizeName(inputName)) - logger.info("SEARCH: Setting inputDirectory to {0}".format(inputDirectory)) - tordir = True - - imdbid = [item for item in pathlist if '.cp(tt' in item] # This looks for the .cp(tt imdb id in the path. - if imdbid and '.cp(tt' not in inputName: - inputName = imdbid[0] # This ensures the imdb id is preserved and passed to CP - tordir = True - - if inputCategory and not tordir: - try: - index = pathlist.index(inputCategory) - if index + 1 < len(pathlist): - tordir = True - logger.info("SEARCH: Found a unique directory {0} in the category directory".format - (pathlist[index + 1])) - if not inputName: - inputName = pathlist[index + 1] - except ValueError: - pass - - if inputName and not tordir: - if inputName in pathlist or sanitizeName(inputName) in pathlist: - logger.info("SEARCH: Found torrent directory {0} in the directory structure".format(inputName)) - tordir = True - else: - root = 1 - if not tordir: - root = 2 - - if root > 0: - logger.info("SEARCH: Could not find a unique directory for this download. Assume a common directory.") - logger.info("SEARCH: We will try and determine which files to process, individually") - - return inputDirectory, inputName, inputCategory, root - - -def getDirSize(inputPath): - from functools import partial - prepend = partial(os.path.join, inputPath) - return sum( - [(os.path.getsize(f) if os.path.isfile(f) else getDirSize(f)) for f in map(prepend, os.listdir(unicode(inputPath)))]) - - -def is_minSize(inputName, minSize): - fileName, fileExt = os.path.splitext(os.path.basename(inputName)) - - # audio files we need to check directory size not file size - inputSize = os.path.getsize(inputName) - if fileExt in core.AUDIOCONTAINER: - try: - inputSize = getDirSize(os.path.dirname(inputName)) - except: - logger.error("Failed to get file size for {0}".format(inputName), 'MINSIZE') - return True - - # Ignore files under a certain size - if inputSize > minSize * 1048576: - return True - - -def is_sample(inputName): - # Ignore 'sample' in files - if re.search('(^|[\W_])sample\d*[\W_]', inputName.lower()): - return True - - -def copy_link(src, targetLink, useLink): - logger.info("MEDIAFILE: [{0}]".format(os.path.basename(targetLink)), 'COPYLINK') - logger.info("SOURCE FOLDER: [{0}]".format(os.path.dirname(src)), 'COPYLINK') - logger.info("TARGET FOLDER: [{0}]".format(os.path.dirname(targetLink)), 'COPYLINK') - - if src != targetLink and os.path.exists(targetLink): - logger.info("MEDIAFILE already exists in the TARGET folder, skipping ...", 'COPYLINK') - return True - elif src == targetLink and os.path.isfile(targetLink) and os.path.isfile(src): - logger.info("SOURCE AND TARGET files are the same, skipping ...", 'COPYLINK') - return True - elif src == os.path.dirname(targetLink): - logger.info("SOURCE AND TARGET folders are the same, skipping ...", 'COPYLINK') - return True - - makeDir(os.path.dirname(targetLink)) - try: - if useLink == 'dir': - logger.info("Directory linking SOURCE FOLDER -> TARGET FOLDER", 'COPYLINK') - linktastic.dirlink(src, targetLink) - return True - if useLink == 'junction': - logger.info("Directory junction linking SOURCE FOLDER -> TARGET FOLDER", 'COPYLINK') - linktastic.dirlink(src, targetLink) - return True - elif useLink == "hard": - logger.info("Hard linking SOURCE MEDIAFILE -> TARGET FOLDER", 'COPYLINK') - linktastic.link(src, targetLink) - return True - elif useLink == "sym": - logger.info("Sym linking SOURCE MEDIAFILE -> TARGET FOLDER", 'COPYLINK') - linktastic.symlink(src, targetLink) - return True - elif useLink == "move-sym": - logger.info("Sym linking SOURCE MEDIAFILE -> TARGET FOLDER", 'COPYLINK') - shutil.move(src, targetLink) - linktastic.symlink(targetLink, src) - return True - elif useLink == "move": - logger.info("Moving SOURCE MEDIAFILE -> TARGET FOLDER", 'COPYLINK') - shutil.move(src, targetLink) - return True - except Exception as e: - logger.warning("Error: {0}, copying instead ... ".format(e), 'COPYLINK') - - logger.info("Copying SOURCE MEDIAFILE -> TARGET FOLDER", 'COPYLINK') - shutil.copy(src, targetLink) - - return True - - -def replace_links(link): - n = 0 - target = link - if os.name == 'nt': - import jaraco - if not jaraco.windows.filesystem.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while jaraco.windows.filesystem.islink(target): - target = jaraco.windows.filesystem.readlink(target) - n = n + 1 - else: - if not os.path.islink(link): - logger.debug('{0} is not a link'.format(link)) - return - while os.path.islink(target): - target = os.readlink(target) - n = n + 1 - if n > 1: - logger.info("Changing sym-link: {0} to point directly to file: {1}".format(link, target), 'COPYLINK') - os.unlink(link) - linktastic.symlink(target, link) - - -def flatten(outputDestination): - logger.info("FLATTEN: Flattening directory: {0}".format(outputDestination)) - for outputFile in listMediaFiles(outputDestination): - dirPath = os.path.dirname(outputFile) - fileName = os.path.basename(outputFile) - - if dirPath == outputDestination: - continue - - target = os.path.join(outputDestination, fileName) - - try: - shutil.move(outputFile, target) - except: - logger.error("Could not flatten {0}".format(outputFile), 'FLATTEN') - - removeEmptyFolders(outputDestination) # Cleanup empty directories - - -def removeEmptyFolders(path, removeRoot=True): - """Function to remove empty folders""" - if not os.path.isdir(path): - return - - # remove empty subfolders - logger.debug("Checking for empty folders in:{0}".format(path)) - files = os.listdir(unicode(path)) - if len(files): - for f in files: - fullpath = os.path.join(path, f) - if os.path.isdir(fullpath): - removeEmptyFolders(fullpath) - - # if folder empty, delete it - files = os.listdir(unicode(path)) - if len(files) == 0 and removeRoot: - logger.debug("Removing empty folder:{}".format(path)) - os.rmdir(path) - - -def rmReadOnly(filename): - if os.path.isfile(filename): - # check first the read-only attribute - file_attribute = os.stat(filename)[0] - if not file_attribute & stat.S_IWRITE: - # File is read-only, so make it writeable - logger.debug('Read only mode on file {name}. Attempting to make it writeable'.format - (name=filename)) - try: - os.chmod(filename, stat.S_IWRITE) - except: - logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) - - -# Wake function -def WakeOnLan(ethernet_address): - addr_byte = ethernet_address.split(':') - hw_addr = struct.pack(b'BBBBBB', int(addr_byte[0], 16), - int(addr_byte[1], 16), - int(addr_byte[2], 16), - int(addr_byte[3], 16), - int(addr_byte[4], 16), - int(addr_byte[5], 16)) - - # Build the Wake-On-LAN "Magic Packet"... - - msg = b'\xff' * 6 + hw_addr * 16 - - # ...and send it to the broadcast address using UDP - - ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ss.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - ss.sendto(msg, ('', 9)) - ss.close() - - -# Test Connection function -def TestCon(host, port): - try: - socket.create_connection((host, port)) - return "Up" - except: - return "Down" - - -def WakeUp(): - host = core.CFG["WakeOnLan"]["host"] - port = int(core.CFG["WakeOnLan"]["port"]) - mac = core.CFG["WakeOnLan"]["mac"] - - i = 1 - while TestCon(host, port) == "Down" and i < 4: - logger.info(("Sending WakeOnLan Magic Packet for mac: {0}".format(mac))) - WakeOnLan(mac) - time.sleep(20) - i = i + 1 - - if TestCon(host, port) == "Down": # final check. - logger.warning("System with mac: {0} has not woken after 3 attempts. " - "Continuing with the rest of the script.".format(mac)) - else: - logger.info("System with mac: {0} has been woken. Continuing with the rest of the script.".format(mac)) - - -def CharReplace(Name): - # Special character hex range: - # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) - # UTF-8: 1st hex code 0xC2-0xC3 followed by a 2nd hex code 0xA1-0xFF - # ISO-8859-15: 0xA6-0xFF - # The function will detect if Name contains a special character - # If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding - encoded = False - encoding = None - if isinstance(Name, unicode): - return encoded, Name.encode(core.SYS_ENCODING) - for Idx in range(len(Name)): - # /!\ detection is done 2char by 2char for UTF-8 special character - if (len(Name) != 1) & (Idx < (len(Name) - 1)): - # Detect UTF-8 - if ((Name[Idx] == '\xC2') | (Name[Idx] == '\xC3')) & ( - (Name[Idx + 1] >= '\xA0') & (Name[Idx + 1] <= '\xFF')): - encoding = 'utf-8' - break - # Detect CP850 - elif (Name[Idx] >= '\x80') & (Name[Idx] <= '\xA5'): - encoding = 'cp850' - break - # Detect ISO-8859-15 - elif (Name[Idx] >= '\xA6') & (Name[Idx] <= '\xFF'): - encoding = 'iso-8859-15' - break - else: - # Detect CP850 - if (Name[Idx] >= '\x80') & (Name[Idx] <= '\xA5'): - encoding = 'cp850' - break - # Detect ISO-8859-15 - elif (Name[Idx] >= '\xA6') & (Name[Idx] <= '\xFF'): - encoding = 'iso-8859-15' - break - if encoding and not encoding == core.SYS_ENCODING: - encoded = True - Name = Name.decode(encoding).encode(core.SYS_ENCODING) - return encoded, Name - - -def convert_to_ascii(inputName, dirName): - ascii_convert = int(core.CFG["ASCII"]["convert"]) - if ascii_convert == 0 or os.name == 'nt': # just return if we don't want to convert or on windows os and "\" is replaced!. - return inputName, dirName - - encoded, inputName = CharReplace(inputName) - - dir, base = os.path.split(dirName) - if not base: # ended with "/" - dir, base = os.path.split(dir) - - encoded, base2 = CharReplace(base) - if encoded: - dirName = os.path.join(dir, base2) - logger.info("Renaming directory to: {0}.".format(base2), 'ENCODER') - os.rename(os.path.join(dir, base), dirName) - if 'NZBOP_SCRIPTDIR' in os.environ: - print("[NZB] DIRECTORY={0}".format(dirName)) - - for dirname, dirnames, filenames in os.walk(dirName, topdown=False): - for subdirname in dirnames: - encoded, subdirname2 = CharReplace(subdirname) - if encoded: - logger.info("Renaming directory to: {0}.".format(subdirname2), 'ENCODER') - os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) - - for dirname, dirnames, filenames in os.walk(dirName): - for filename in filenames: - encoded, filename2 = CharReplace(filename) - if encoded: - logger.info("Renaming file to: {0}.".format(filename2), 'ENCODER') - os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2)) - - return inputName, dirName - - -def parse_other(args): - return os.path.normpath(args[1]), '', '', '', '' - - -def parse_rtorrent(args): - # rtorrent usage: system.method.set_key = event.download.finished,TorrentToMedia, - # "execute={/path/to/nzbToMedia/TorrentToMedia.py,\"$d.get_base_path=\",\"$d.get_name=\",\"$d.get_custom1=\",\"$d.get_hash=\"}" - inputDirectory = os.path.normpath(args[1]) - try: - inputName = args[2] - except: - inputName = '' - try: - inputCategory = args[3] - except: - inputCategory = '' - try: - inputHash = args[4] - except: - inputHash = '' - try: - inputID = args[4] - except: - inputID = '' - - return inputDirectory, inputName, inputCategory, inputHash, inputID - - -def parse_utorrent(args): - # uTorrent usage: call TorrentToMedia.py "%D" "%N" "%L" "%I" - inputDirectory = os.path.normpath(args[1]) - inputName = args[2] - try: - inputCategory = args[3] - except: - inputCategory = '' - try: - inputHash = args[4] - except: - inputHash = '' - try: - inputID = args[4] - except: - inputID = '' - - return inputDirectory, inputName, inputCategory, inputHash, inputID - - -def parse_deluge(args): - # Deluge usage: call TorrentToMedia.py TORRENT_ID TORRENT_NAME TORRENT_DIR - inputDirectory = os.path.normpath(args[3]) - inputName = args[2] - inputHash = args[1] - inputID = args[1] - try: - inputCategory = core.TORRENT_CLASS.core.get_torrent_status(inputID, ['label']).get()['label'] - except: - inputCategory = '' - return inputDirectory, inputName, inputCategory, inputHash, inputID - - -def parse_transmission(args): - # Transmission usage: call TorrenToMedia.py (%TR_TORRENT_DIR% %TR_TORRENT_NAME% is passed on as environmental variables) - inputDirectory = os.path.normpath(os.getenv('TR_TORRENT_DIR')) - inputName = os.getenv('TR_TORRENT_NAME') - inputCategory = '' # We dont have a category yet - inputHash = os.getenv('TR_TORRENT_HASH') - inputID = os.getenv('TR_TORRENT_ID') - return inputDirectory, inputName, inputCategory, inputHash, inputID - - -def parse_vuze(args): - # vuze usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py "%D%N%L%I%K%F" - try: - input = args[1].split(',') - except: - input = [] - try: - inputDirectory = os.path.normpath(input[0]) - except: - inputDirectory = '' - try: - inputName = input[1] - except: - inputName = '' - try: - inputCategory = input[2] - except: - inputCategory = '' - try: - inputHash = input[3] - except: - inputHash = '' - try: - inputID = input[3] - except: - inputID = '' - try: - if input[4] == 'single': - inputName = input[5] - except: - pass - - return inputDirectory, inputName, inputCategory, inputHash, inputID - -def parse_qbittorrent(args): - # qbittorrent usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py "%D|%N|%L|%I" - try: - input = args[1].split('|') - except: - input = [] - try: - inputDirectory = os.path.normpath(input[0].replace('"','')) - except: - inputDirectory = '' - try: - inputName = input[1].replace('"','') - except: - inputName = '' - try: - inputCategory = input[2].replace('"','') - except: - inputCategory = '' - try: - inputHash = input[3].replace('"','') - except: - inputHash = '' - try: - inputID = input[3].replace('"','') - except: - inputID = '' - - return inputDirectory, inputName, inputCategory, inputHash, inputID - -def parse_args(clientAgent, args): - clients = { - 'other': parse_other, - 'rtorrent': parse_rtorrent, - 'utorrent': parse_utorrent, - 'deluge': parse_deluge, - 'transmission': parse_transmission, - 'qbittorrent': parse_qbittorrent, - 'vuze': parse_vuze, - } - - try: - return clients[clientAgent](args) - except: - return None, None, None, None, None - - -def getDirs(section, subsection, link='hard'): - to_return = [] - - def processDir(path): - folders = [] - - logger.info("Searching {0} for mediafiles to post-process ...".format(path)) - sync = [o for o in os.listdir(unicode(path)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] - # search for single files and move them into their own folder for post-processing - for mediafile in [os.path.join(path, o) for o in os.listdir(unicode(path)) if - os.path.isfile(os.path.join(path, o))]: - if len(sync) > 0: - break - if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: - continue - try: - logger.debug("Found file {0} in root directory {1}.".format(os.path.split(mediafile)[1], path)) - newPath = None - fileExt = os.path.splitext(mediafile)[1] - try: - if fileExt in core.AUDIOCONTAINER: - f = beets.mediafile.MediaFile(mediafile) - - # get artist and album info - artist = f.artist - album = f.album - - # create new path - newPath = os.path.join(path, "{0} - {1}".format(sanitizeName(artist), sanitizeName(album))) - elif fileExt in core.MEDIACONTAINER: - f = guessit.guessit(mediafile) - - # get title - title = f.get('series') or f.get('title') - - if not title: - title = os.path.splitext(os.path.basename(mediafile))[0] - - newPath = os.path.join(path, sanitizeName(title)) - except Exception as e: - logger.error("Exception parsing name for media file: {0}: {1}".format(os.path.split(mediafile)[1], e)) - - if not newPath: - title = os.path.splitext(os.path.basename(mediafile))[0] - newPath = os.path.join(path, sanitizeName(title)) - - try: - newPath = newPath.encode(core.SYS_ENCODING) - except: - pass - - # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). - if os.path.isfile(newPath): - newPath2 = os.path.join(os.path.join(os.path.split(newPath)[0], 'new'), os.path.split(newPath)[1]) - newPath = newPath2 - - # create new path if it does not exist - if not os.path.exists(newPath): - makeDir(newPath) - - newfile = os.path.join(newPath, sanitizeName(os.path.split(mediafile)[1])) - try: - newfile = newfile.encode(core.SYS_ENCODING) - except: - pass - - # link file to its new path - copy_link(mediafile, newfile, link) - except Exception as e: - logger.error("Failed to move {0} to its own directory: {1}".format(os.path.split(mediafile)[1], e)) - - # removeEmptyFolders(path, removeRoot=False) - - if os.listdir(unicode(path)): - for dir in [os.path.join(path, o) for o in os.listdir(unicode(path)) if - os.path.isdir(os.path.join(path, o))]: - sync = [o for o in os.listdir(unicode(dir)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] - if len(sync) > 0 or len(os.listdir(unicode(dir))) == 0: - continue - folders.extend([dir]) - return folders - - try: - watch_dir = os.path.join(core.CFG[section][subsection]["watch_dir"], subsection) - if os.path.exists(watch_dir): - to_return.extend(processDir(watch_dir)) - elif os.path.exists(core.CFG[section][subsection]["watch_dir"]): - to_return.extend(processDir(core.CFG[section][subsection]["watch_dir"])) - except Exception as e: - logger.error("Failed to add directories from {0} for post-processing: {1}".format - (core.CFG[section][subsection]["watch_dir"], e)) - - if core.USELINK == 'move': - try: - outputDirectory = os.path.join(core.OUTPUTDIRECTORY, subsection) - if os.path.exists(outputDirectory): - to_return.extend(processDir(outputDirectory)) - except Exception as e: - logger.error("Failed to add directories from {0} for post-processing: {1}".format(core.OUTPUTDIRECTORY, e)) - - if not to_return: - logger.debug("No directories identified in {0}:{1} for post-processing".format(section, subsection)) - - return list(set(to_return)) - - -def onerror(func, path, exc_info): - """ - Error handler for ``shutil.rmtree``. - - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - - If the error is for another reason it re-raises the error. - - Usage : ``shutil.rmtree(path, onerror=onerror)`` - """ - if not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise Exception - - -def rmDir(dirName): - logger.info("Deleting {0}".format(dirName)) - try: - shutil.rmtree(unicode(dirName), onerror=onerror) - except: - logger.error("Unable to delete folder {0}".format(dirName)) - - -def cleanDir(path, section, subsection): - cfg = dict(core.CFG[section][subsection]) - if not os.path.exists(path): - logger.info('Directory {0} has been processed and removed ...'.format(path), 'CLEANDIR') - return - if core.FORCE_CLEAN and not core.FAILED: - logger.info('Doing Forceful Clean of {0}'.format(path), 'CLEANDIR') - rmDir(path) - return - minSize = int(cfg.get('minSize', 0)) - delete_ignored = int(cfg.get('delete_ignored', 0)) - try: - num_files = len(listMediaFiles(path, minSize=minSize, delete_ignored=delete_ignored)) - except: - num_files = 'unknown' - if num_files > 0: - logger.info( - "Directory {0} still contains {1} unprocessed file(s), skipping ...".format(path, num_files), - 'CLEANDIRS') - return - - logger.info("Directory {0} has been processed, removing ...".format(path), 'CLEANDIRS') - try: - shutil.rmtree(path, onerror=onerror) - except: - logger.error("Unable to delete directory {0}".format(path)) - - -def create_torrent_class(clientAgent): - # Hardlink solution for Torrents - tc = None - - if clientAgent == 'utorrent': - try: - logger.debug("Connecting to {0}: {1}".format(clientAgent, core.UTORRENTWEBUI)) - tc = UTorrentClient(core.UTORRENTWEBUI, core.UTORRENTUSR, core.UTORRENTPWD) - except: - logger.error("Failed to connect to uTorrent") - - if clientAgent == 'transmission': - try: - logger.debug("Connecting to {0}: http://{1}:{2}".format( - clientAgent, core.TRANSMISSIONHOST, core.TRANSMISSIONPORT)) - tc = TransmissionClient(core.TRANSMISSIONHOST, core.TRANSMISSIONPORT, - core.TRANSMISSIONUSR, - core.TRANSMISSIONPWD) - except: - logger.error("Failed to connect to Transmission") - - if clientAgent == 'deluge': - try: - logger.debug("Connecting to {0}: http://{1}:{2}".format(clientAgent, core.DELUGEHOST, core.DELUGEPORT)) - tc = DelugeClient() - tc.connect(host=core.DELUGEHOST, port=core.DELUGEPORT, username=core.DELUGEUSR, - password=core.DELUGEPWD) - except: - logger.error("Failed to connect to Deluge") - - if clientAgent == 'qbittorrent': - try: - logger.debug("Connecting to {0}: http://{1}:{2}".format(clientAgent, core.QBITTORRENTHOST, core.QBITTORRENTPORT)) - tc = qBittorrentClient("http://{0}:{1}/".format(core.QBITTORRENTHOST, core.QBITTORRENTPORT)) - tc.login(core.QBITTORRENTUSR, core.QBITTORRENTPWD) - except: - logger.error("Failed to connect to qBittorrent") - - return tc - - -def pause_torrent(clientAgent, inputHash, inputID, inputName): - logger.debug("Stopping torrent {0} in {1} while processing".format(inputName, clientAgent)) - try: - if clientAgent == 'utorrent' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.stop(inputHash) - if clientAgent == 'transmission' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.stop_torrent(inputID) - if clientAgent == 'deluge' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.core.pause_torrent([inputID]) - if clientAgent == 'qbittorrent' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.pause(inputHash) - time.sleep(5) - except: - logger.warning("Failed to stop torrent {0} in {1}".format(inputName, clientAgent)) - - -def resume_torrent(clientAgent, inputHash, inputID, inputName): - if not core.TORRENT_RESUME == 1: - return - logger.debug("Starting torrent {0} in {1}".format(inputName, clientAgent)) - try: - if clientAgent == 'utorrent' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.start(inputHash) - if clientAgent == 'transmission' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.start_torrent(inputID) - if clientAgent == 'deluge' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.core.resume_torrent([inputID]) - if clientAgent == 'qbittorrent' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.resume(inputHash) - time.sleep(5) - except: - logger.warning("Failed to start torrent {0} in {1}".format(inputName, clientAgent)) - - -def remove_torrent(clientAgent, inputHash, inputID, inputName): - if core.DELETE_ORIGINAL == 1 or core.USELINK == 'move': - logger.debug("Deleting torrent {0} from {1}".format(inputName, clientAgent)) - try: - if clientAgent == 'utorrent' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.removedata(inputHash) - core.TORRENT_CLASS.remove(inputHash) - if clientAgent == 'transmission' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.remove_torrent(inputID, True) - if clientAgent == 'deluge' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.core.remove_torrent(inputID, True) - if clientAgent == 'qbittorrent' and core.TORRENT_CLASS != "": - core.TORRENT_CLASS.delete_permanently(inputHash) - time.sleep(5) - except: - logger.warning("Failed to delete torrent {0} in {1}".format(inputName, clientAgent)) - else: - resume_torrent(clientAgent, inputHash, inputID, inputName) - - -def find_download(clientAgent, download_id): - logger.debug("Searching for Download on {0} ...".format(clientAgent)) - if clientAgent == 'utorrent': - torrents = core.TORRENT_CLASS.list()[1]['torrents'] - for torrent in torrents: - if download_id in torrent: - return True - if clientAgent == 'transmission': - torrents = core.TORRENT_CLASS.get_torrents() - for torrent in torrents: - hash = torrent.hashString - if hash == download_id: - return True - if clientAgent == 'deluge': - return False - if clientAgent == 'qbittorrent': - torrents = core.TORRENT_CLASS.torrents() - for torrent in torrents: - if torrent['hash'] == download_id: - return True - if clientAgent == 'sabnzbd': - if "http" in core.SABNZBDHOST: - baseURL = "{0}:{1}/api".format(core.SABNZBDHOST, core.SABNZBDPORT) - else: - baseURL = "http://{0}:{1}/api".format(core.SABNZBDHOST, core.SABNZBDPORT) - url = baseURL - params = { - 'apikey': core.SABNZBDAPIKEY, - 'mode': "get_files", - 'output': 'json', - 'value': download_id, - } - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error("Unable to open URL") - return False # failure - - result = r.json() - if result['files']: - return True - return False - - -def get_nzoid(inputName): - nzoid = None - slots = [] - logger.debug("Searching for nzoid from SAbnzbd ...") - if "http" in core.SABNZBDHOST: - baseURL = "{0}:{1}/api".format(core.SABNZBDHOST, core.SABNZBDPORT) - else: - baseURL = "http://{0}:{1}/api".format(core.SABNZBDHOST, core.SABNZBDPORT) - url = baseURL - params = { - 'apikey': core.SABNZBDAPIKEY, - 'mode': "queue", - 'output': 'json', - } - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error("Unable to open URL") - return nzoid # failure - try: - result = r.json() - cleanName = os.path.splitext(os.path.split(inputName)[1])[0] - slots.extend([(slot['nzo_id'], slot['filename']) for slot in result['queue']['slots']]) - except: - logger.warning("Data from SABnzbd queue could not be parsed") - params['mode'] = "history" - try: - r = requests.get(url, params=params, verify=False, timeout=(30, 120)) - except requests.ConnectionError: - logger.error("Unable to open URL") - return nzoid # failure - try: - result = r.json() - cleanName = os.path.splitext(os.path.split(inputName)[1])[0] - slots.extend([(slot['nzo_id'], slot['name']) for slot in result['history']['slots']]) - except: - logger.warning("Data from SABnzbd history could not be parsed") - try: - for nzo_id, name in slots: - if name in [inputName, cleanName]: - nzoid = nzo_id - logger.debug("Found nzoid: {0}".format(nzoid)) - break - except: - logger.warning("Data from SABnzbd could not be parsed") - return nzoid - - -def cleanFileName(filename): - """Cleans up nzb name by removing any . and _ - characters, along with any trailing hyphens. - - Is basically equivalent to replacing all _ and . with a - space, but handles decimal numbers in string, for example: - """ - - filename = re.sub("(\D)\.(?!\s)(\D)", "\\1 \\2", filename) - filename = re.sub("(\d)\.(\d{4})", "\\1 \\2", filename) # if it ends in a year then don't keep the dot - filename = re.sub("(\D)\.(?!\s)", "\\1 ", filename) - filename = re.sub("\.(?!\s)(\D)", " \\1", filename) - filename = filename.replace("_", " ") - filename = re.sub("-$", "", filename) - filename = re.sub("^\[.*\]", "", filename) - return filename.strip() - - -def is_archive_file(filename): - """Check if the filename is allowed for the Archive""" - for regext in core.COMPRESSEDCONTAINER: - if regext.search(filename): - return regext.split(filename)[0] - return False - - -def isMediaFile(mediafile, media=True, audio=True, meta=True, archives=True, other=False, otherext=[]): - fileName, fileExt = os.path.splitext(mediafile) - - try: - # ignore MAC OS's "resource fork" files - if fileName.startswith('._'): - return False - except: - pass - if (media and fileExt.lower() in core.MEDIACONTAINER) \ - or (audio and fileExt.lower() in core.AUDIOCONTAINER) \ - or (meta and fileExt.lower() in core.METACONTAINER) \ - or (archives and is_archive_file(mediafile)) \ - or (other and (fileExt.lower() in otherext or 'all' in otherext)): - return True - else: - return False - - -def listMediaFiles(path, minSize=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=[]): - files = [] - if not os.path.isdir(path): - if os.path.isfile(path): # Single file downloads. - curFile = os.path.split(path)[1] - if isMediaFile(curFile, media, audio, meta, archives, other, otherext): - # Optionally ignore sample files - if is_sample(path) or not is_minSize(path, minSize): - if delete_ignored == 1: - try: - os.unlink(path) - logger.debug('Ignored file {0} has been removed ...'.format - (curFile)) - except: - pass - else: - files.append(path) - - return files - - for curFile in os.listdir(unicode(path)): - fullCurFile = os.path.join(path, curFile) - - # if it's a folder do it recursively - if os.path.isdir(fullCurFile) and not curFile.startswith('.'): - files += listMediaFiles(fullCurFile, minSize, delete_ignored, media, audio, meta, archives, other, otherext) - - elif isMediaFile(curFile, media, audio, meta, archives, other, otherext): - # Optionally ignore sample files - if is_sample(fullCurFile) or not is_minSize(fullCurFile, minSize): - if delete_ignored == 1: - try: - os.unlink(fullCurFile) - logger.debug('Ignored file {0} has been removed ...'.format - (curFile)) - except: - pass - continue - - files.append(fullCurFile) - - return sorted(files, key=len) - - -def find_imdbid(dirName, inputName, omdbApiKey): - imdbid = None - - logger.info('Attemping imdbID lookup for {0}'.format(inputName)) - - # find imdbid in dirName - logger.info('Searching folder and file names for imdbID ...') - m = re.search('(tt\d{7})', dirName + inputName) - if m: - imdbid = m.group(1) - logger.info("Found imdbID [{0}]".format(imdbid)) - return imdbid - if os.path.isdir(dirName): - for file in os.listdir(unicode(dirName)): - m = re.search('(tt\d{7})', file) - if m: - imdbid = m.group(1) - logger.info("Found imdbID [{0}] via file name".format(imdbid)) - return imdbid - if 'NZBPR__DNZB_MOREINFO' in os.environ: - dnzb_more_info = os.environ.get('NZBPR__DNZB_MOREINFO', '') - if dnzb_more_info != '': - regex = re.compile(r'^http://www.imdb.com/title/(tt[0-9]+)/$', re.IGNORECASE) - m = regex.match(dnzb_more_info) - if m: - imdbid = m.group(1) - logger.info("Found imdbID [{0}] from DNZB-MoreInfo".format(imdbid)) - return imdbid - logger.info('Searching IMDB for imdbID ...') - try: - guess = guessit.guessit(inputName) - except: - guess = None - if guess: - # Movie Title - title = None - if 'title' in guess: - title = guess['title'] - - # Movie Year - year = None - if 'year' in guess: - year = guess['year'] - - url = "http://www.omdbapi.com" - - if not omdbApiKey: - logger.info("Unable to determine imdbID: No api key provided for ombdapi.com.") - return - - logger.debug("Opening URL: {0}".format(url)) - - try: - r = requests.get(url, params={'apikey': omdbApiKey, 'y': year, 't': title}, - verify=False, timeout=(60, 300)) - except requests.ConnectionError: - logger.error("Unable to open URL {0}".format(url)) - return - - try: - results = r.json() - except: - logger.error("No json data returned from omdbapi.com") - - try: - imdbid = results['imdbID'] - except: - logger.error("No imdbID returned from omdbapi.com") - - if imdbid: - logger.info("Found imdbID [{0}]".format(imdbid)) - return imdbid - - logger.warning('Unable to find a imdbID for {0}'.format(inputName)) - return imdbid - - -def extractFiles(src, dst=None, keep_archive=None): - extracted_folder = [] - extracted_archive = [] - - for inputFile in listMediaFiles(src, media=False, audio=False, meta=False, archives=True): - dirPath = os.path.dirname(inputFile) - fullFileName = os.path.basename(inputFile) - archiveName = os.path.splitext(fullFileName)[0] - archiveName = re.sub(r"part[0-9]+", "", archiveName) - - if dirPath in extracted_folder and archiveName in extracted_archive: - continue # no need to extract this, but keep going to look for other archives and sub directories. - - try: - if extractor.extract(inputFile, dst or dirPath): - extracted_folder.append(dirPath) - extracted_archive.append(archiveName) - except Exception: - logger.error("Extraction failed for: {0}".format(fullFileName)) - - for folder in extracted_folder: - for inputFile in listMediaFiles(folder, media=False, audio=False, meta=False, archives=True): - fullFileName = os.path.basename(inputFile) - archiveName = os.path.splitext(fullFileName)[0] - archiveName = re.sub(r"part[0-9]+", "", archiveName) - if archiveName not in extracted_archive or keep_archive: - continue # don't remove if we haven't extracted this archive, or if we want to preserve them. - logger.info("Removing extracted archive {0} from folder {1} ...".format(fullFileName, folder)) - try: - if not os.access(inputFile, os.W_OK): - os.chmod(inputFile, stat.S_IWUSR) - os.remove(inputFile) - time.sleep(1) - except Exception as e: - logger.error("Unable to remove file {0} due to: {1}".format(inputFile, e)) - - -def import_subs(filename): - if not core.GETSUBS: - return - try: - subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) - except: - pass - - languages = set() - for item in core.SLANGUAGES: - try: - languages.add(Language(item)) - except: - pass - if not languages: - return - - logger.info("Attempting to download subtitles for {0}".format(filename), 'SUBTITLES') - try: - video = subliminal.scan_video(filename) - subtitles = subliminal.download_best_subtitles({video}, languages) - subliminal.save_subtitles(video, subtitles[video]) - except Exception as e: - logger.error("Failed to download subtitles for {0} due to: {1}".format(filename, e), 'SUBTITLES') - - -def server_responding(baseURL): - logger.debug("Attempting to connect to server at {0}".format(baseURL), 'SERVER') - try: - requests.get(baseURL, timeout=(60, 120), verify=False) - logger.debug("Server responded at {0}".format(baseURL), 'SERVER') - return True - except (requests.ConnectionError, requests.exceptions.Timeout): - logger.error("Server failed to respond at {0}".format(baseURL), 'SERVER') - return False - - -def plex_update(category): - if core.FAILED: - return - url = '{scheme}://{host}:{port}/library/sections/'.format( - scheme='https' if core.PLEXSSL else 'http', - host=core.PLEXHOST, - port=core.PLEXPORT, - ) - section = None - if not core.PLEXSEC: - return - logger.debug("Attempting to update Plex Library for category {0}.".format(category), 'PLEX') - for item in core.PLEXSEC: - if item[0] == category: - section = item[1] - - if section: - url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEXTOKEN) - requests.get(url, timeout=(60, 120), verify=False) - logger.debug("Plex Library has been refreshed.", 'PLEX') - else: - logger.debug("Could not identify section for plex update", 'PLEX') - - -def backupVersionedFile(old_file, version): - numTries = 0 - - new_file = '{old}.v{version}'.format(old=old_file, version=version) - - while not os.path.isfile(new_file): - if not os.path.isfile(old_file): - logger.log(u"Not creating backup, {file} doesn't exist".format(file=old_file), logger.DEBUG) - break - - try: - logger.log(u"Trying to back up {old} to {new]".format(old=old_file, new=new_file), logger.DEBUG) - shutil.copy(old_file, new_file) - logger.log(u"Backup done", logger.DEBUG) - break - except Exception as error: - logger.log(u"Error while trying to back up {old} to {new} : {msg}".format - (old=old_file, new=new_file, msg=error), logger.WARNING) - numTries += 1 - time.sleep(1) - logger.log(u"Trying again.", logger.DEBUG) - - if numTries >= 10: - logger.log(u"Unable to back up {old} to {new} please do it manually.".format(old=old_file, new=new_file), logger.ERROR) - return False - - return True - - -def update_downloadInfoStatus(inputName, status): - logger.db("Updating status of our download {0} in the DB to {1}".format(inputName, status)) - - myDB = nzbToMediaDB.DBConnection() - myDB.action("UPDATE downloads SET status=?, last_update=? WHERE input_name=?", - [status, datetime.date.today().toordinal(), text_type(inputName)]) - - -def get_downloadInfo(inputName, status): - logger.db("Getting download info for {0} from the DB".format(inputName)) - - myDB = nzbToMediaDB.DBConnection() - sqlResults = myDB.select("SELECT * FROM downloads WHERE input_name=? AND status=?", - [text_type(inputName), status]) - - return sqlResults - - -class RunningProcess(object): - """ Limits application to single instance """ - - def __init__(self): - if platform.system() == 'Windows': - self.process = WindowsProcess() - else: - self.process = PosixProcess() - - def alreadyrunning(self): - return self.process.alreadyrunning() - - # def __del__(self): - # self.process.__del__() - - -class WindowsProcess(object): - def __init__(self): - self.mutexname = "nzbtomedia_{pid}".format(pid=core.PID_FILE.replace('\\', '/')) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}" - if platform.system() == 'Windows': - from win32event import CreateMutex - from win32api import CloseHandle, GetLastError - from winerror import ERROR_ALREADY_EXISTS - self.CreateMutex = CreateMutex - self.CloseHandle = CloseHandle - self.GetLastError = GetLastError - self.ERROR_ALREADY_EXISTS = ERROR_ALREADY_EXISTS - - def alreadyrunning(self): - self.mutex = self.CreateMutex(None, 0, self.mutexname) - self.lasterror = self.GetLastError() - if self.lasterror == self.ERROR_ALREADY_EXISTS: - self.CloseHandle(self.mutex) - return True - else: - return False - - def __del__(self): - if self.mutex: - self.CloseHandle(self.mutex) - - -class PosixProcess(object): - def __init__(self): - self.pidpath = core.PID_FILE - self.lock_socket = None - - def alreadyrunning(self): - try: - self.lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - self.lock_socket.bind('\0{path}'.format(path=self.pidpath)) - self.lasterror = False - return self.lasterror - except socket.error as e: - if "Address already in use" in e: - self.lasterror = True - return self.lasterror - except AttributeError: - pass - if os.path.exists(self.pidpath): - # Make sure it is not a "stale" pidFile - try: - pid = int(open(self.pidpath, 'r').read().strip()) - except: - pid = None - # Check list of running pids, if not running it is stale so overwrite - if isinstance(pid, int): - try: - os.kill(pid, 0) - self.lasterror = True - except OSError: - self.lasterror = False - else: - self.lasterror = False - else: - self.lasterror = False - - if not self.lasterror: - # Write my pid into pidFile to keep multiple copies of program from running - try: - fp = open(self.pidpath, 'w') - fp.write(str(os.getpid())) - fp.close() - except: - pass - - return self.lasterror - - def __del__(self): - if not self.lasterror: - if self.lock_socket: - self.lock_socket.close() - if os.path.isfile(self.pidpath): - os.unlink(self.pidpath) diff --git a/core/qbittorrent/__init__.py b/core/qbittorrent/__init__.py deleted file mode 100644 index bf893c064..000000000 --- a/core/qbittorrent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding=utf-8 \ No newline at end of file diff --git a/core/scene_exceptions.py b/core/scene_exceptions.py new file mode 100644 index 000000000..5d830012e --- /dev/null +++ b/core/scene_exceptions.py @@ -0,0 +1,189 @@ +# coding=utf-8 + +import os +import platform +import re +import shlex +import subprocess + +import core +from core import logger +from core.utils import list_media_files + +reverse_list = [r'\.\d{2}e\d{2}s\.', r'\.[pi]0801\.', r'\.p027\.', r'\.[pi]675\.', r'\.[pi]084\.', r'\.p063\.', + r'\b[45]62[xh]\.', r'\.yarulb\.', r'\.vtd[hp]\.', + r'\.ld[.-]?bew\.', r'\.pir.?(dov|dvd|bew|db|rb)\.', r'\brdvd\.', r'\.vts\.', r'\.reneercs\.', + r'\.dcv\.', r'\b(pir|mac)dh\b', r'\.reporp\.', r'\.kcaper\.', + r'\.lanretni\.', r'\b3ca\b', r'\.cstn\.'] +reverse_pattern = re.compile('|'.join(reverse_list), flags=re.IGNORECASE) +season_pattern = re.compile(r'(.*\.\d{2}e\d{2}s\.)(.*)', flags=re.IGNORECASE) +word_pattern = re.compile(r'([^A-Z0-9]*[A-Z0-9]+)') +media_list = [r'\.s\d{2}e\d{2}\.', r'\.1080[pi]\.', r'\.720p\.', r'\.576[pi]', r'\.480[pi]\.', r'\.360p\.', + r'\.[xh]26[45]\b', r'\.bluray\.', r'\.[hp]dtv\.', + r'\.web[.-]?dl\.', r'\.(vod|dvd|web|bd|br).?rip\.', r'\.dvdr\b', r'\.stv\.', r'\.screener\.', r'\.vcd\.', + r'\bhd(cam|rip)\b', r'\.proper\.', r'\.repack\.', + r'\.internal\.', r'\bac3\b', r'\.ntsc\.', r'\.pal\.', r'\.secam\.', r'\bdivx\b', r'\bxvid\b'] +media_pattern = re.compile('|'.join(media_list), flags=re.IGNORECASE) +garbage_name = re.compile(r'^[a-zA-Z0-9]*$') +char_replace = [[r'(\w)1\.(\w)', r'\1i\2'] + ] + + +def process_all_exceptions(name, dirname): + par2(dirname) + rename_script(dirname) + for filename in list_media_files(dirname): + newfilename = None + parent_dir = os.path.dirname(filename) + head, file_extension = os.path.splitext(os.path.basename(filename)) + if reverse_pattern.search(head) is not None: + exception = reverse_filename + elif garbage_name.search(head) is not None: + exception = replace_filename + else: + exception = None + newfilename = filename + if not newfilename: + newfilename = exception(filename, parent_dir, name) + if core.GROUPS: + newfilename = strip_groups(newfilename) + if newfilename != filename: + rename_file(filename, newfilename) + + +def strip_groups(filename): + if not core.GROUPS: + return filename + dirname, file = os.path.split(filename) + head, file_extension = os.path.splitext(file) + newname = head.replace(' ', '.') + for group in core.GROUPS: + newname = newname.replace(group, '') + newname = newname.replace('[]', '') + newfile = newname + file_extension + newfile_path = os.path.join(dirname, newfile) + return newfile_path + + +def rename_file(filename, newfile_path): + if os.path.isfile(newfile_path): + newfile_path = os.path.splitext(newfile_path)[0] + '.NTM' + os.path.splitext(newfile_path)[1] + logger.debug('Replacing file name {old} with download name {new}'.format + (old=filename, new=newfile_path), 'EXCEPTION') + try: + os.rename(filename, newfile_path) + except Exception as error: + logger.error('Unable to rename file due to: {error}'.format(error=error), 'EXCEPTION') + + +def replace_filename(filename, dirname, name): + head, file_extension = os.path.splitext(os.path.basename(filename)) + if media_pattern.search(os.path.basename(dirname).replace(' ', '.')) is not None: + newname = os.path.basename(dirname).replace(' ', '.') + logger.debug('Replacing file name {old} with directory name {new}'.format(old=head, new=newname), 'EXCEPTION') + elif media_pattern.search(name.replace(' ', '.').lower()) is not None: + newname = name.replace(' ', '.') + logger.debug('Replacing file name {old} with download name {new}'.format + (old=head, new=newname), 'EXCEPTION') + else: + logger.warning('No name replacement determined for {name}'.format(name=head), 'EXCEPTION') + newname = name + newfile = newname + file_extension + newfile_path = os.path.join(dirname, newfile) + return newfile_path + + +def reverse_filename(filename, dirname, name): + head, file_extension = os.path.splitext(os.path.basename(filename)) + na_parts = season_pattern.search(head) + if na_parts is not None: + word_p = word_pattern.findall(na_parts.group(2)) + if word_p: + new_words = '' + for wp in word_p: + if wp[0] == '.': + new_words += '.' + new_words += re.sub(r'\W', '', wp) + else: + new_words = na_parts.group(2) + for cr in char_replace: + new_words = re.sub(cr[0], cr[1], new_words) + newname = new_words[::-1] + na_parts.group(1)[::-1] + else: + newname = head[::-1].title() + newname = newname.replace(' ', '.') + logger.debug('Reversing filename {old} to {new}'.format + (old=head, new=newname), 'EXCEPTION') + newfile = newname + file_extension + newfile_path = os.path.join(dirname, newfile) + return newfile_path + + +def rename_script(dirname): + rename_file = '' + for directory, directories, files in os.walk(dirname): + for file in files: + if re.search(r'(rename\S*\.(sh|bat)$)', file, re.IGNORECASE): + rename_file = os.path.join(directory, file) + dirname = directory + break + if rename_file: + rename_lines = [line.strip() for line in open(rename_file)] + for line in rename_lines: + if re.search('^(mv|Move)', line, re.IGNORECASE): + cmd = shlex.split(line)[1:] + else: + continue + if len(cmd) == 2 and os.path.isfile(os.path.join(dirname, cmd[0])): + orig = os.path.join(dirname, cmd[0]) + dest = os.path.join(dirname, cmd[1].split('\\')[-1].split('/')[-1]) + if os.path.isfile(dest): + continue + logger.debug('Renaming file {source} to {destination}'.format + (source=orig, destination=dest), 'EXCEPTION') + try: + os.rename(orig, dest) + except Exception as error: + logger.error('Unable to rename file due to: {error}'.format(error=error), 'EXCEPTION') + + +def par2(dirname): + newlist = [] + sofar = 0 + parfile = '' + objects = [] + if os.path.exists(dirname): + objects = os.listdir(dirname) + for item in objects: + if item.endswith('.par2'): + size = os.path.getsize(os.path.join(dirname, item)) + if size > sofar: + sofar = size + parfile = item + if core.PAR2CMD and parfile: + pwd = os.getcwd() # Get our Present Working Directory + os.chdir(dirname) # set directory to run par on. + if platform.system() == 'Windows': + bitbucket = open('NUL') + else: + bitbucket = open('/dev/null') + logger.info('Running par2 on file {0}.'.format(parfile), 'PAR2') + command = [core.PAR2CMD, 'r', parfile, '*'] + cmd = '' + for item in command: + cmd = '{cmd} {item}'.format(cmd=cmd, item=item) + logger.debug('calling command:{0}'.format(cmd), 'PAR2') + try: + proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) + proc.communicate() + result = proc.returncode + except Exception: + logger.error('par2 file processing for {0} has failed'.format(parfile), 'PAR2') + if result == 0: + logger.info('par2 file processing succeeded', 'PAR2') + os.chdir(pwd) + bitbucket.close() + +# dict for custom groups +# we can add more to this list +# _customgroups = {'Q o Q': process_qoq, '-ECI': process_eci} diff --git a/core/synchronousdeluge/__init__.py b/core/synchronousdeluge/__init__.py deleted file mode 100644 index 9d4d8c77d..000000000 --- a/core/synchronousdeluge/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# coding=utf-8 -"""A synchronous implementation of the Deluge RPC protocol - based on gevent-deluge by Christopher Rosell. - - https://github.com/chrippa/gevent-deluge - -Example usage: - - from synchronousdeluge import DelgueClient - - client = DelugeClient() - client.connect() - - # Wait for value - download_location = client.core.get_config_value("download_location").get() -""" - -from core.synchronousdeluge.exceptions import DelugeRPCError - - -__title__ = "synchronous-deluge" -__version__ = "0.1" -__author__ = "Christian Dale" diff --git a/core/synchronousdeluge/client.py b/core/synchronousdeluge/client.py deleted file mode 100644 index cecb2a887..000000000 --- a/core/synchronousdeluge/client.py +++ /dev/null @@ -1,159 +0,0 @@ -# coding=utf-8 -import os -import platform - -from collections import defaultdict -from itertools import imap -from .exceptions import DelugeRPCError -from .protocol import DelugeRPCRequest, DelugeRPCResponse -from .transfer import DelugeTransfer - -__all__ = ["DelugeClient"] - -RPC_RESPONSE = 1 -RPC_ERROR = 2 -RPC_EVENT = 3 - - -class DelugeClient(object): - def __init__(self): - """A deluge client session.""" - self.transfer = DelugeTransfer() - self.modules = [] - self._request_counter = 0 - - def _get_local_auth(self): - username = password = "" - if platform.system() in ('Windows', 'Microsoft'): - appDataPath = os.environ.get("APPDATA") - if not appDataPath: - import _winreg - hkey = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, - "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders") - appDataReg = _winreg.QueryValueEx(hkey, "AppData") - appDataPath = appDataReg[0] - _winreg.CloseKey(hkey) - - auth_file = os.path.join(appDataPath, "deluge", "auth") - else: - from xdg.BaseDirectory import save_config_path - try: - auth_file = os.path.join(save_config_path("deluge"), "auth") - except OSError: - return username, password - - if os.path.exists(auth_file): - for line in open(auth_file): - if line.startswith("#"): - # This is a comment line - continue - line = line.strip() - try: - lsplit = line.split(":") - except Exception: - continue - - if len(lsplit) == 2: - username, password = lsplit - elif len(lsplit) == 3: - username, password, level = lsplit - else: - continue - - if username == "localclient": - return username, password - - return "", "" - - def _create_module_method(self, module, method): - fullname = "{0}.{1}".format(module, method) - - def func(obj, *args, **kwargs): - return self.remote_call(fullname, *args, **kwargs) - - func.__name__ = method - - return func - - def _introspect(self): - self.modules = [] - - methods = self.remote_call("daemon.get_method_list").get() - methodmap = defaultdict(dict) - splitter = lambda v: v.split(".") - - for module, method in imap(splitter, methods): - methodmap[module][method] = self._create_module_method(module, method) - - for module, methods in methodmap.items(): - clsname = "DelugeModule{0}".format(module.capitalize()) - cls = type(clsname, (), methods) - setattr(self, module, cls()) - self.modules.append(module) - - def remote_call(self, method, *args, **kwargs): - req = DelugeRPCRequest(self._request_counter, method, *args, **kwargs) - message = next(self.transfer.send_request(req)) - - response = DelugeRPCResponse() - - if not isinstance(message, tuple): - return - - if len(message) < 3: - return - - message_type = message[0] - - # if message_type == RPC_EVENT: - # event = message[1] - # values = message[2] - # - # if event in self._event_handlers: - # for handler in self._event_handlers[event]: - # gevent.spawn(handler, *values) - # - # elif message_type in (RPC_RESPONSE, RPC_ERROR): - if message_type in (RPC_RESPONSE, RPC_ERROR): - request_id = message[1] - value = message[2] - - if request_id == self._request_counter: - if message_type == RPC_RESPONSE: - response.set(value) - elif message_type == RPC_ERROR: - err = DelugeRPCError(*value) - response.set_exception(err) - - self._request_counter += 1 - return response - - def connect(self, host="127.0.0.1", port=58846, username="", password=""): - """Connects to a daemon process. - - :param host: str, the hostname of the daemon - :param port: int, the port of the daemon - :param username: str, the username to login with - :param password: str, the password to login with - """ - - # Connect transport - self.transfer.connect((host, port)) - - # Attempt to fetch local auth info if needed - if not username and host in ("127.0.0.1", "localhost"): - username, password = self._get_local_auth() - - # Authenticate - self.remote_call("daemon.login", username, password).get() - - # Introspect available methods - self._introspect() - - @property - def connected(self): - return self.transfer.connected - - def disconnect(self): - """Disconnects from the daemon.""" - self.transfer.disconnect() diff --git a/core/synchronousdeluge/exceptions.py b/core/synchronousdeluge/exceptions.py deleted file mode 100644 index 95bf7f04f..000000000 --- a/core/synchronousdeluge/exceptions.py +++ /dev/null @@ -1,12 +0,0 @@ -# coding=utf-8 -__all__ = ["DelugeRPCError"] - - -class DelugeRPCError(Exception): - def __init__(self, name, msg, traceback): - self.name = name - self.msg = msg - self.traceback = traceback - - def __str__(self): - return "{0}: {1}: {2}".format(self.__class__.__name__, self.name, self.msg) diff --git a/core/synchronousdeluge/rencode.py b/core/synchronousdeluge/rencode.py deleted file mode 100644 index 8ab013752..000000000 --- a/core/synchronousdeluge/rencode.py +++ /dev/null @@ -1,487 +0,0 @@ -# coding=utf-8 -""" -rencode -- Web safe object pickling/unpickling. - -Public domain, Connelly Barnes 2006-2007. - -The rencode module is a modified version of bencode from the -BitTorrent project. For complex, heterogeneous data structures with -many small elements, r-encodings take up significantly less space than -b-encodings: - - >>> len(rencode.dumps({'a': 0, 'b': [1, 2], 'c': 99})) - 13 - >>> len(bencode.bencode({'a': 0, 'b': [1, 2], 'c': 99})) - 26 - -The rencode format is not standardized, and may change with different -rencode module versions, so you should check that you are using the -same rencode version throughout your project. -""" - -import struct -from threading import Lock -from six import PY3 - -if PY3: - long = int - -__version__ = '1.0.1' -__all__ = ['dumps', 'loads'] - -# Original bencode module by Petru Paler, et al. -# -# Modifications by Connelly Barnes: -# -# - Added support for floats (sent as 32-bit or 64-bit in network -# order), bools, None. -# - Allowed dict keys to be of any serializable type. -# - Lists/tuples are always decoded as tuples (thus, tuples can be -# used as dict keys). -# - Embedded extra information in the 'typecodes' to save some space. -# - Added a restriction on integer length, so that malicious hosts -# cannot pass us large integers which take a long time to decode. -# -# Licensed by Bram Cohen under the "MIT license": -# -# "Copyright (C) 2001-2002 Bram Cohen -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# The Software is provided "AS IS", without warranty of any kind, -# express or implied, including but not limited to the warranties of -# merchantability, fitness for a particular purpose and -# noninfringement. In no event shall the authors or copyright holders -# be liable for any claim, damages or other liability, whether in an -# action of contract, tort or otherwise, arising from, out of or in -# connection with the Software or the use or other dealings in the -# Software." -# -# (The rencode module is licensed under the above license as well). -# - -# Default number of bits for serialized floats, either 32 or 64 (also a parameter for dumps()). -DEFAULT_FLOAT_BITS = 32 - -# Maximum length of integer when written as base 10 string. -MAX_INT_LENGTH = 64 - -# The bencode 'typecodes' such as i, d, etc have been extended and -# relocated on the base-256 character set. -CHR_LIST = chr(59) -CHR_DICT = chr(60) -CHR_INT = chr(61) -CHR_INT1 = chr(62) -CHR_INT2 = chr(63) -CHR_INT4 = chr(64) -CHR_INT8 = chr(65) -CHR_FLOAT32 = chr(66) -CHR_FLOAT64 = chr(44) -CHR_TRUE = chr(67) -CHR_FALSE = chr(68) -CHR_NONE = chr(69) -CHR_TERM = chr(127) - -# Positive integers with value embedded in typecode. -INT_POS_FIXED_START = 0 -INT_POS_FIXED_COUNT = 44 - -# Dictionaries with length embedded in typecode. -DICT_FIXED_START = 102 -DICT_FIXED_COUNT = 25 - -# Negative integers with value embedded in typecode. -INT_NEG_FIXED_START = 70 -INT_NEG_FIXED_COUNT = 32 - -# Strings with length embedded in typecode. -STR_FIXED_START = 128 -STR_FIXED_COUNT = 64 - -# Lists with length embedded in typecode. -LIST_FIXED_START = STR_FIXED_START + STR_FIXED_COUNT -LIST_FIXED_COUNT = 64 - - -def decode_int(x, f): - f += 1 - newf = x.index(CHR_TERM, f) - if newf - f >= MAX_INT_LENGTH: - raise ValueError('overflow') - try: - n = int(x[f:newf]) - except (OverflowError, ValueError): - n = long(x[f:newf]) - if x[f] == '-': - if x[f + 1] == '0': - raise ValueError - elif x[f] == '0' and newf != f + 1: - raise ValueError - return n, newf + 1 - - -def decode_intb(x, f): - f += 1 - return struct.unpack('!b', x[f:f + 1])[0], f + 1 - - -def decode_inth(x, f): - f += 1 - return struct.unpack('!h', x[f:f + 2])[0], f + 2 - - -def decode_intl(x, f): - f += 1 - return struct.unpack('!l', x[f:f + 4])[0], f + 4 - - -def decode_intq(x, f): - f += 1 - return struct.unpack('!q', x[f:f + 8])[0], f + 8 - - -def decode_float32(x, f): - f += 1 - n = struct.unpack('!f', x[f:f + 4])[0] - return n, f + 4 - - -def decode_float64(x, f): - f += 1 - n = struct.unpack('!d', x[f:f + 8])[0] - return n, f + 8 - - -def decode_string(x, f): - colon = x.index(':', f) - try: - n = int(x[f:colon]) - except (OverflowError, ValueError): - n = long(x[f:colon]) - if x[f] == '0' and colon != f + 1: - raise ValueError - colon += 1 - s = x[colon:colon + n] - try: - t = s.decode("utf8") - if len(t) != len(s): - s = t - except UnicodeDecodeError: - pass - return s, colon + n - - -def decode_list(x, f): - r, f = [], f + 1 - while x[f] != CHR_TERM: - v, f = decode_func[x[f]](x, f) - r.append(v) - return tuple(r), f + 1 - - -def decode_dict(x, f): - r, f = {}, f + 1 - while x[f] != CHR_TERM: - k, f = decode_func[x[f]](x, f) - r[k], f = decode_func[x[f]](x, f) - return r, f + 1 - - -def decode_true(x, f): - return True, f + 1 - - -def decode_false(x, f): - return False, f + 1 - - -def decode_none(x, f): - return None, f + 1 - - -decode_func = { - '0': decode_string, - '1': decode_string, - '2': decode_string, - '3': decode_string, - '4': decode_string, - '5': decode_string, - '6': decode_string, - '7': decode_string, - '8': decode_string, - '9': decode_string, - CHR_LIST: decode_list, - CHR_DICT: decode_dict, - CHR_INT: decode_int, - CHR_INT1: decode_intb, - CHR_INT2: decode_inth, - CHR_INT4: decode_intl, - CHR_INT8: decode_intq, - CHR_FLOAT32: decode_float32, - CHR_FLOAT64: decode_float64, - CHR_TRUE: decode_true, - CHR_FALSE: decode_false, - CHR_NONE: decode_none, -} - - -def make_fixed_length_string_decoders(): - def make_decoder(slen): - def f(x, f): - s = x[f + 1:f + 1 + slen] - try: - t = s.decode("utf8") - if len(t) != len(s): - s = t - except UnicodeDecodeError: - pass - return s, f + 1 + slen - - return f - - for i in range(STR_FIXED_COUNT): - decode_func[chr(STR_FIXED_START + i)] = make_decoder(i) - - -make_fixed_length_string_decoders() - - -def make_fixed_length_list_decoders(): - def make_decoder(slen): - def f(x, f): - r, f = [], f + 1 - for i in range(slen): - v, f = decode_func[x[f]](x, f) - r.append(v) - return tuple(r), f - - return f - - for i in range(LIST_FIXED_COUNT): - decode_func[chr(LIST_FIXED_START + i)] = make_decoder(i) - - -make_fixed_length_list_decoders() - - -def make_fixed_length_int_decoders(): - def make_decoder(j): - def f(x, f): - return j, f + 1 - - return f - - for i in range(INT_POS_FIXED_COUNT): - decode_func[chr(INT_POS_FIXED_START + i)] = make_decoder(i) - for i in range(INT_NEG_FIXED_COUNT): - decode_func[chr(INT_NEG_FIXED_START + i)] = make_decoder(-1 - i) - - -make_fixed_length_int_decoders() - - -def make_fixed_length_dict_decoders(): - def make_decoder(slen): - def f(x, f): - r, f = {}, f + 1 - for j in range(slen): - k, f = decode_func[x[f]](x, f) - r[k], f = decode_func[x[f]](x, f) - return r, f - - return f - - for i in range(DICT_FIXED_COUNT): - decode_func[chr(DICT_FIXED_START + i)] = make_decoder(i) - - -make_fixed_length_dict_decoders() - - -def encode_dict(x, r): - r.append(CHR_DICT) - for k, v in x.items(): - encode_func[type(k)](k, r) - encode_func[type(v)](v, r) - r.append(CHR_TERM) - - -def loads(x): - try: - r, l = decode_func[x[0]](x, 0) - except (IndexError, KeyError): - raise ValueError - if l != len(x): - raise ValueError - return r - - -from types import StringType, IntType, LongType, DictType, ListType, TupleType, FloatType, NoneType, UnicodeType - - -def encode_int(x, r): - if 0 <= x < INT_POS_FIXED_COUNT: - r.append(chr(INT_POS_FIXED_START + x)) - elif -INT_NEG_FIXED_COUNT <= x < 0: - r.append(chr(INT_NEG_FIXED_START - 1 - x)) - elif -128 <= x < 128: - r.extend((CHR_INT1, struct.pack('!b', x))) - elif -32768 <= x < 32768: - r.extend((CHR_INT2, struct.pack('!h', x))) - elif -2147483648 <= x < 2147483648: - r.extend((CHR_INT4, struct.pack('!l', x))) - elif -9223372036854775808 <= x < 9223372036854775808: - r.extend((CHR_INT8, struct.pack('!q', x))) - else: - s = str(x) - if len(s) >= MAX_INT_LENGTH: - raise ValueError('overflow') - r.extend((CHR_INT, s, CHR_TERM)) - - -def encode_float32(x, r): - r.extend((CHR_FLOAT32, struct.pack('!f', x))) - - -def encode_float64(x, r): - r.extend((CHR_FLOAT64, struct.pack('!d', x))) - - -def encode_bool(x, r): - r.extend({False: CHR_FALSE, True: CHR_TRUE}[bool(x)]) - - -def encode_none(x, r): - r.extend(CHR_NONE) - - -def encode_string(x, r): - if len(x) < STR_FIXED_COUNT: - r.extend((chr(STR_FIXED_START + len(x)), x)) - else: - r.extend((str(len(x)), ':', x)) - - -def encode_unicode(x, r): - encode_string(x.encode("utf8"), r) - - -def encode_list(x, r): - if len(x) < LIST_FIXED_COUNT: - r.append(chr(LIST_FIXED_START + len(x))) - for i in x: - encode_func[type(i)](i, r) - else: - r.append(CHR_LIST) - for i in x: - encode_func[type(i)](i, r) - r.append(CHR_TERM) - - -def encode_dict(x, r): - if len(x) < DICT_FIXED_COUNT: - r.append(chr(DICT_FIXED_START + len(x))) - for k, v in x.items(): - encode_func[type(k)](k, r) - encode_func[type(v)](v, r) - else: - r.append(CHR_DICT) - for k, v in x.items(): - encode_func[type(k)](k, r) - encode_func[type(v)](v, r) - r.append(CHR_TERM) - - -encode_func = { - IntType: encode_int, - LongType: encode_int, - StringType: encode_string, - ListType: encode_list, - TupleType: encode_list, - DictType: encode_dict, - NoneType: encode_none, - UnicodeType: encode_unicode, -} - -lock = Lock() - -try: - from types import BooleanType - - encode_func[BooleanType] = encode_bool -except ImportError: - pass - - -def dumps(x, float_bits=DEFAULT_FLOAT_BITS): - """ - Dump data structure to str. - - Here float_bits is either 32 or 64. - """ - lock.acquire() - try: - if float_bits == 32: - encode_func[FloatType] = encode_float32 - elif float_bits == 64: - encode_func[FloatType] = encode_float64 - else: - raise ValueError('Float bits ({0:d}) is not 32 or 64'.format(float_bits)) - r = [] - encode_func[type(x)](x, r) - finally: - lock.release() - return ''.join(r) - - -def test(): - f1 = struct.unpack('!f', struct.pack('!f', 25.5))[0] - f2 = struct.unpack('!f', struct.pack('!f', 29.3))[0] - f3 = struct.unpack('!f', struct.pack('!f', -0.6))[0] - L = (({'a': 15, 'bb': f1, 'ccc': f2, '': (f3, (), False, True, '')}, ('a', 10 ** 20), tuple(range(-100000, 100000)), - 'b' * 31, 'b' * 62, 'b' * 64, 2 ** 30, 2 ** 33, 2 ** 62, 2 ** 64, 2 ** 30, 2 ** 33, 2 ** 62, 2 ** 64, False, - False, True, -1, 2, 0),) - assert loads(dumps(L)) == L - d = dict(zip(range(-100000, 100000), range(-100000, 100000))) - d.update({'a': 20, 20: 40, 40: 41, f1: f2, f2: f3, f3: False, False: True, True: False}) - L = (d, {}, {5: 6}, {7: 7, True: 8}, {9: 10, 22: 39, 49: 50, 44: ''}) - assert loads(dumps(L)) == L - L = ('', 'a' * 10, 'a' * 100, 'a' * 1000, 'a' * 10000, 'a' * 100000, 'a' * 1000000, 'a' * 10000000) - assert loads(dumps(L)) == L - L = tuple([dict(zip(range(n), range(n))) for n in range(100)]) + ('b',) - assert loads(dumps(L)) == L - L = tuple([dict(zip(range(n), range(-n, 0))) for n in range(100)]) + ('b',) - assert loads(dumps(L)) == L - L = tuple([tuple(range(n)) for n in range(100)]) + ('b',) - assert loads(dumps(L)) == L - L = tuple(['a' * n for n in range(1000)]) + ('b',) - assert loads(dumps(L)) == L - L = tuple(['a' * n for n in range(1000)]) + (None, True, None) - assert loads(dumps(L)) == L - assert loads(dumps(None)) is None - assert loads(dumps({None: None})) == {None: None} - assert 1e-10 < abs(loads(dumps(1.1)) - 1.1) < 1e-6 - assert 1e-10 < abs(loads(dumps(1.1, 32)) - 1.1) < 1e-6 - assert abs(loads(dumps(1.1, 64)) - 1.1) < 1e-12 - assert loads(dumps(u"Hello World!!")) - - -try: - import psyco - - psyco.bind(dumps) - psyco.bind(loads) -except ImportError: - pass - -if __name__ == '__main__': - test() diff --git a/core/transcoder.py b/core/transcoder.py new file mode 100644 index 000000000..38973a337 --- /dev/null +++ b/core/transcoder.py @@ -0,0 +1,830 @@ +# coding=utf-8 + +import errno +import json +import os +import platform +import re +import shutil +import subprocess + +from babelfish import Language +from six import iteritems, string_types, text_type + +import core +from core import logger +from core.utils import make_dir + +__author__ = 'Justin' + + +def is_video_good(videofile, status): + file_name_ext = os.path.basename(videofile) + file_name, file_ext = os.path.splitext(file_name_ext) + disable = False + if file_ext not in core.MEDIACONTAINER or not core.FFPROBE or not core.CHECK_MEDIA or file_ext in ['.iso'] or (status > 0 and core.NOEXTRACTFAILED): + disable = True + else: + test_details, res = get_video_details(core.TEST_FILE) + if res != 0 or test_details.get('error'): + disable = True + logger.info('DISABLED: ffprobe failed to analyse test file. Stopping corruption check.', 'TRANSCODER') + if test_details.get('streams'): + vid_streams = [item for item in test_details['streams'] if 'codec_type' in item and item['codec_type'] == 'video'] + aud_streams = [item for item in test_details['streams'] if 'codec_type' in item and item['codec_type'] == 'audio'] + if not (len(vid_streams) > 0 and len(aud_streams) > 0): + disable = True + logger.info('DISABLED: ffprobe failed to analyse streams from test file. Stopping corruption check.', + 'TRANSCODER') + if disable: + if status: # if the download was 'failed', assume bad. If it was successful, assume good. + return False + else: + return True + + logger.info('Checking [{0}] for corruption, please stand by ...'.format(file_name_ext), 'TRANSCODER') + video_details, result = get_video_details(videofile) + + if result != 0: + logger.error('FAILED: [{0}] is corrupted!'.format(file_name_ext), 'TRANSCODER') + return False + if video_details.get('error'): + logger.info('FAILED: [{0}] returned error [{1}].'.format(file_name_ext, video_details.get('error')), 'TRANSCODER') + return False + if video_details.get('streams'): + video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video'] + audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio'] + if len(video_streams) > 0 and len(audio_streams) > 0: + logger.info('SUCCESS: [{0}] has no corruption.'.format(file_name_ext), 'TRANSCODER') + return True + else: + logger.info('FAILED: [{0}] has {1} video streams and {2} audio streams. ' + 'Assume corruption.'.format + (file_name_ext, len(video_streams), len(audio_streams)), 'TRANSCODER') + return False + + +def zip_out(file, img, bitbucket): + procin = None + cmd = [core.SEVENZIP, '-so', 'e', img, file] + try: + procin = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + except Exception: + logger.error('Extracting [{0}] has failed'.format(file), 'TRANSCODER') + return procin + + +def get_video_details(videofile, img=None, bitbucket=None): + video_details = {} + result = 1 + file = videofile + if not core.FFPROBE: + return video_details, result + if 'avprobe' in core.FFPROBE: + print_format = '-of' + else: + print_format = '-print_format' + try: + if img: + videofile = '-' + command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', '-show_error', + videofile] + print_cmd(command) + if img: + procin = zip_out(file, img, bitbucket) + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) + procin.stdout.close() + else: + proc = subprocess.Popen(command, stdout=subprocess.PIPE) + out, err = proc.communicate() + result = proc.returncode + video_details = json.loads(out) + except Exception: + pass + if not video_details: + try: + command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', videofile] + if img: + procin = zip_out(file, img) + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) + procin.stdout.close() + else: + proc = subprocess.Popen(command, stdout=subprocess.PIPE) + out, err = proc.communicate() + result = proc.returncode + video_details = json.loads(out) + except Exception: + logger.error('Checking [{0}] has failed'.format(file), 'TRANSCODER') + return video_details, result + + +def build_commands(file, new_dir, movie_name, bitbucket): + if isinstance(file, string_types): + input_file = file + if 'concat:' in file: + file = file.split('|')[0].replace('concat:', '') + video_details, result = get_video_details(file) + directory, name = os.path.split(file) + name, ext = os.path.splitext(name) + check = re.match('VTS_([0-9][0-9])_[0-9]+', name) + if check and core.CONCAT: + name = movie_name + elif check: + name = ('{0}.cd{1}'.format(movie_name, check.groups()[0])) + elif core.CONCAT and re.match('(.+)[cC][dD][0-9]', name): + name = re.sub('([ ._=:-]+[cC][dD][0-9])', '', name) + if ext == core.VEXTENSION and new_dir == directory: # we need to change the name to prevent overwriting itself. + core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext' + else: + img, data = next(iteritems(file)) + name = data['name'] + video_details, result = get_video_details(data['files'][0], img, bitbucket) + input_file = '-' + file = '-' + + newfile_path = os.path.normpath(os.path.join(new_dir, name) + core.VEXTENSION) + + map_cmd = [] + video_cmd = [] + audio_cmd = [] + audio_cmd2 = [] + sub_cmd = [] + meta_cmd = [] + other_cmd = [] + + if not video_details or not video_details.get( + 'streams'): # we couldn't read streams with ffprobe. Set defaults to try transcoding. + video_streams = [] + audio_streams = [] + sub_streams = [] + + map_cmd.extend(['-map', '0']) + if core.VCODEC: + video_cmd.extend(['-c:v', core.VCODEC]) + if core.VCODEC == 'libx264' and core.VPRESET: + video_cmd.extend(['-pre', core.VPRESET]) + else: + video_cmd.extend(['-c:v', 'copy']) + if core.VFRAMERATE: + video_cmd.extend(['-r', str(core.VFRAMERATE)]) + if core.VBITRATE: + video_cmd.extend(['-b:v', str(core.VBITRATE)]) + if core.VRESOLUTION: + video_cmd.extend(['-vf', 'scale={vres}'.format(vres=core.VRESOLUTION)]) + if core.VPRESET: + video_cmd.extend(['-preset', core.VPRESET]) + if core.VCRF: + video_cmd.extend(['-crf', str(core.VCRF)]) + if core.VLEVEL: + video_cmd.extend(['-level', str(core.VLEVEL)]) + + if core.ACODEC: + audio_cmd.extend(['-c:a', core.ACODEC]) + if core.ACODEC in ['aac', + 'dts']: # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg + audio_cmd.extend(['-strict', '-2']) + else: + audio_cmd.extend(['-c:a', 'copy']) + if core.ACHANNELS: + audio_cmd.extend(['-ac', str(core.ACHANNELS)]) + if core.ABITRATE: + audio_cmd.extend(['-b:a', str(core.ABITRATE)]) + if core.OUTPUTQUALITYPERCENT: + audio_cmd.extend(['-q:a', str(core.OUTPUTQUALITYPERCENT)]) + + if core.SCODEC and core.ALLOWSUBS: + sub_cmd.extend(['-c:s', core.SCODEC]) + elif core.ALLOWSUBS: # Not every subtitle codec can be used for every video container format! + sub_cmd.extend(['-c:s', 'copy']) + else: # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options + sub_cmd.extend(['-sn']) # Don't copy the subtitles over + + if core.OUTPUTFASTSTART: + other_cmd.extend(['-movflags', '+faststart']) + + else: + video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video'] + audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio'] + sub_streams = [item for item in video_details['streams'] if item['codec_type'] == 'subtitle'] + if core.VEXTENSION not in ['.mkv', '.mpegts']: + sub_streams = [item for item in video_details['streams'] if + item['codec_type'] == 'subtitle' and item['codec_name'] != 'hdmv_pgs_subtitle' and item[ + 'codec_name'] != 'pgssub'] + + for video in video_streams: + codec = video['codec_name'] + fr = video.get('avg_frame_rate', 0) + width = video.get('width', 0) + height = video.get('height', 0) + scale = core.VRESOLUTION + if codec in core.VCODEC_ALLOW or not core.VCODEC: + video_cmd.extend(['-c:v', 'copy']) + else: + video_cmd.extend(['-c:v', core.VCODEC]) + if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001): + video_cmd.extend(['-r', str(core.VFRAMERATE)]) + if scale: + w_scale = width / float(scale.split(':')[0]) + h_scale = height / float(scale.split(':')[1]) + if w_scale > h_scale: # widescreen, Scale by width only. + scale = '{width}:{height}'.format( + width=scale.split(':')[0], + height=int((height / w_scale) / 2) * 2, + ) + if w_scale > 1: + video_cmd.extend(['-vf', 'scale={width}'.format(width=scale)]) + else: # lower or matching ratio, scale by height only. + scale = '{width}:{height}'.format( + width=int((width / h_scale) / 2) * 2, + height=scale.split(':')[1], + ) + if h_scale > 1: + video_cmd.extend(['-vf', 'scale={height}'.format(height=scale)]) + if core.VBITRATE: + video_cmd.extend(['-b:v', str(core.VBITRATE)]) + if core.VPRESET: + video_cmd.extend(['-preset', core.VPRESET]) + if core.VCRF: + video_cmd.extend(['-crf', str(core.VCRF)]) + if core.VLEVEL: + video_cmd.extend(['-level', str(core.VLEVEL)]) + no_copy = ['-vf', '-r', '-crf', '-level', '-preset', '-b:v'] + if video_cmd[1] == 'copy' and any(i in video_cmd for i in no_copy): + video_cmd[1] = core.VCODEC + if core.VCODEC == 'copy': # force copy. therefore ignore all other video transcoding. + video_cmd = ['-c:v', 'copy'] + map_cmd.extend(['-map', '0:{index}'.format(index=video['index'])]) + break # Only one video needed + + used_audio = 0 + a_mapped = [] + commentary = [] + if audio_streams: + for i, val in reversed(list(enumerate(audio_streams))): + try: + if 'Commentary' in val.get('tags').get('title'): # Split out commentry tracks. + commentary.append(val) + del audio_streams[i] + except Exception: + continue + try: + audio1 = [item for item in audio_streams if item['tags']['language'] == core.ALANGUAGE] + except Exception: # no language tags. Assume only 1 language. + audio1 = audio_streams + try: + audio2 = [item for item in audio1 if item['codec_name'] in core.ACODEC_ALLOW] + except Exception: + audio2 = [] + try: + audio3 = [item for item in audio_streams if item['tags']['language'] != core.ALANGUAGE] + except Exception: + audio3 = [] + try: + audio4 = [item for item in audio3 if item['codec_name'] in core.ACODEC_ALLOW] + except Exception: + audio4 = [] + + if audio2: # right (or only) language and codec... + map_cmd.extend(['-map', '0:{index}'.format(index=audio2[0]['index'])]) + a_mapped.extend([audio2[0]['index']]) + bitrate = int(float(audio2[0].get('bit_rate', 0))) / 1000 + channels = int(float(audio2[0].get('channels', 0))) + audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy']) + elif audio1: # right (or only) language, wrong codec. + map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]['index'])]) + a_mapped.extend([audio1[0]['index']]) + bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000 + channels = int(float(audio1[0].get('channels', 0))) + audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy']) + elif audio4: # wrong language, right codec. + map_cmd.extend(['-map', '0:{index}'.format(index=audio4[0]['index'])]) + a_mapped.extend([audio4[0]['index']]) + bitrate = int(float(audio4[0].get('bit_rate', 0))) / 1000 + channels = int(float(audio4[0].get('channels', 0))) + audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy']) + elif audio3: # wrong language, wrong codec. just pick the default audio track + map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]['index'])]) + a_mapped.extend([audio3[0]['index']]) + bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000 + channels = int(float(audio3[0].get('channels', 0))) + audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy']) + + if core.ACHANNELS and channels and channels > core.ACHANNELS: + audio_cmd.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS)]) + if audio_cmd[1] == 'copy': + audio_cmd[1] = core.ACODEC + if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1): + audio_cmd.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE)]) + if audio_cmd[1] == 'copy': + audio_cmd[1] = core.ACODEC + if core.OUTPUTQUALITYPERCENT: + audio_cmd.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)]) + if audio_cmd[1] == 'copy': + audio_cmd[1] = core.ACODEC + if audio_cmd[1] in ['aac', 'dts']: + audio_cmd[2:2] = ['-strict', '-2'] + + if core.ACODEC2_ALLOW: + used_audio += 1 + try: + audio5 = [item for item in audio1 if item['codec_name'] in core.ACODEC2_ALLOW] + except Exception: + audio5 = [] + try: + audio6 = [item for item in audio3 if item['codec_name'] in core.ACODEC2_ALLOW] + except Exception: + audio6 = [] + if audio5: # right language and codec. + map_cmd.extend(['-map', '0:{index}'.format(index=audio5[0]['index'])]) + a_mapped.extend([audio5[0]['index']]) + bitrate = int(float(audio5[0].get('bit_rate', 0))) / 1000 + channels = int(float(audio5[0].get('channels', 0))) + audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) + elif audio1: # right language wrong codec. + map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]['index'])]) + a_mapped.extend([audio1[0]['index']]) + bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000 + channels = int(float(audio1[0].get('channels', 0))) + if core.ACODEC2: + audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2]) + else: + audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) + elif audio6: # wrong language, right codec + map_cmd.extend(['-map', '0:{index}'.format(index=audio6[0]['index'])]) + a_mapped.extend([audio6[0]['index']]) + bitrate = int(float(audio6[0].get('bit_rate', 0))) / 1000 + channels = int(float(audio6[0].get('channels', 0))) + audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) + elif audio3: # wrong language, wrong codec just pick the default audio track + map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]['index'])]) + a_mapped.extend([audio3[0]['index']]) + bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000 + channels = int(float(audio3[0].get('channels', 0))) + if core.ACODEC2: + audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2]) + else: + audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) + + if core.ACHANNELS2 and channels and channels > core.ACHANNELS2: + audio_cmd2.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS2)]) + if audio_cmd2[1] == 'copy': + audio_cmd2[1] = core.ACODEC2 + if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1): + audio_cmd2.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE2)]) + if audio_cmd2[1] == 'copy': + audio_cmd2[1] = core.ACODEC2 + if core.OUTPUTQUALITYPERCENT: + audio_cmd2.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)]) + if audio_cmd2[1] == 'copy': + audio_cmd2[1] = core.ACODEC2 + if audio_cmd2[1] in ['aac', 'dts']: + audio_cmd2[2:2] = ['-strict', '-2'] + + if a_mapped[1] == a_mapped[0] and audio_cmd2[1:] == audio_cmd[1:]: # check for duplicate output track. + del map_cmd[-2:] + else: + audio_cmd.extend(audio_cmd2) + + if core.AINCLUDE and core.ACODEC3: + audio_streams.extend(commentary) # add commentry tracks back here. + for audio in audio_streams: + if audio['index'] in a_mapped: + continue + used_audio += 1 + map_cmd.extend(['-map', '0:{index}'.format(index=audio['index'])]) + audio_cmd3 = [] + bitrate = int(float(audio.get('bit_rate', 0))) / 1000 + channels = int(float(audio.get('channels', 0))) + if audio['codec_name'] in core.ACODEC3_ALLOW: + audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy']) + else: + if core.ACODEC3: + audio_cmd3.extend(['-c:a:{0}'.format(used_audio), core.ACODEC3]) + else: + audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy']) + + if core.ACHANNELS3 and channels and channels > core.ACHANNELS3: + audio_cmd3.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS3)]) + if audio_cmd3[1] == 'copy': + audio_cmd3[1] = core.ACODEC3 + if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1): + audio_cmd3.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE3)]) + if audio_cmd3[1] == 'copy': + audio_cmd3[1] = core.ACODEC3 + if core.OUTPUTQUALITYPERCENT > 0: + audio_cmd3.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)]) + if audio_cmd3[1] == 'copy': + audio_cmd3[1] = core.ACODEC3 + if audio_cmd3[1] in ['aac', 'dts']: + audio_cmd3[2:2] = ['-strict', '-2'] + audio_cmd.extend(audio_cmd3) + + s_mapped = [] + burnt = 0 + n = 0 + for lan in core.SLANGUAGES: + try: + subs1 = [item for item in sub_streams if item['tags']['language'] == lan] + except Exception: + subs1 = [] + if core.BURN and not subs1 and not burnt and os.path.isfile(file): + for subfile in get_subs(file): + if lan in os.path.split(subfile)[1]: + video_cmd.extend(['-vf', 'subtitles={subs}'.format(subs=subfile)]) + burnt = 1 + for sub in subs1: + if core.BURN and not burnt and os.path.isfile(input_file): + subloc = 0 + for index in range(len(sub_streams)): + if sub_streams[index]['index'] == sub['index']: + subloc = index + break + video_cmd.extend(['-vf', 'subtitles={sub}:si={loc}'.format(sub=input_file, loc=subloc)]) + burnt = 1 + if not core.ALLOWSUBS: + break + if sub['codec_name'] in ['dvd_subtitle', 'VobSub'] and core.SCODEC == 'mov_text': # We can't convert these. + continue + map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])]) + s_mapped.extend([sub['index']]) + + if core.SINCLUDE: + for sub in sub_streams: + if not core.ALLOWSUBS: + break + if sub['index'] in s_mapped: + continue + if sub['codec_name'] in ['dvd_subtitle', 'VobSub'] and core.SCODEC == 'mov_text': # We can't convert these. + continue + map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])]) + s_mapped.extend([sub['index']]) + + if core.OUTPUTFASTSTART: + other_cmd.extend(['-movflags', '+faststart']) + + command = [core.FFMPEG, '-loglevel', 'warning'] + + if core.HWACCEL: + command.extend(['-hwaccel', 'auto']) + if core.GENERALOPTS: + command.extend(core.GENERALOPTS) + + command.extend(['-i', input_file]) + + if core.SEMBED and os.path.isfile(file): + for subfile in get_subs(file): + sub_details, result = get_video_details(subfile) + if not sub_details or not sub_details.get('streams'): + continue + if core.SCODEC == 'mov_text': + subcode = [stream['codec_name'] for stream in sub_details['streams']] + if set(subcode).intersection(['dvd_subtitle', 'VobSub']): # We can't convert these. + continue + command.extend(['-i', subfile]) + lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split('-')[0] + lan = text_type(lan) + metlan = None + try: + if len(lan) == 3: + metlan = Language(lan) + if len(lan) == 2: + metlan = Language.fromalpha2(lan) + except Exception: + pass + if metlan: + meta_cmd.extend(['-metadata:s:s:{x}'.format(x=len(s_mapped) + n), + 'language={lang}'.format(lang=metlan.alpha3)]) + n += 1 + map_cmd.extend(['-map', '{x}:0'.format(x=n)]) + + if not core.ALLOWSUBS or (not s_mapped and not n): + sub_cmd.extend(['-sn']) + else: + if core.SCODEC: + sub_cmd.extend(['-c:s', core.SCODEC]) + else: + sub_cmd.extend(['-c:s', 'copy']) + + command.extend(map_cmd) + command.extend(video_cmd) + command.extend(audio_cmd) + command.extend(sub_cmd) + command.extend(meta_cmd) + command.extend(other_cmd) + command.append(newfile_path) + if platform.system() != 'Windows': + command = core.NICENESS + command + return command + + +def get_subs(file): + filepaths = [] + sub_ext = ['.srt', '.sub', '.idx'] + name = os.path.splitext(os.path.split(file)[1])[0] + path = os.path.split(file)[0] + for directory, directories, filenames in os.walk(path): + for filename in filenames: + filepaths.extend([os.path.join(directory, filename)]) + subfiles = [item for item in filepaths if os.path.splitext(item)[1] in sub_ext and name in item] + return subfiles + + +def extract_subs(file, newfile_path, bitbucket): + video_details, result = get_video_details(file) + if not video_details: + return + + if core.SUBSDIR: + subdir = core.SUBSDIR + else: + subdir = os.path.split(newfile_path)[0] + name = os.path.splitext(os.path.split(newfile_path)[1])[0] + + try: + sub_streams = [item for item in video_details['streams'] if + item['codec_type'] == 'subtitle' and item['tags']['language'] in core.SLANGUAGES and item[ + 'codec_name'] != 'hdmv_pgs_subtitle' and item['codec_name'] != 'pgssub'] + except Exception: + sub_streams = [item for item in video_details['streams'] if + item['codec_type'] == 'subtitle' and item['codec_name'] != 'hdmv_pgs_subtitle' and item[ + 'codec_name'] != 'pgssub'] + num = len(sub_streams) + for n in range(num): + sub = sub_streams[n] + idx = sub['index'] + lan = sub.get('tags', {}).get('language', 'unk') + + if num == 1: + output_file = os.path.join(subdir, '{0}.srt'.format(name)) + if os.path.isfile(output_file): + output_file = os.path.join(subdir, '{0}.{1}.srt'.format(name, n)) + else: + output_file = os.path.join(subdir, '{0}.{1}.srt'.format(name, lan)) + if os.path.isfile(output_file): + output_file = os.path.join(subdir, '{0}.{1}.{2}.srt'.format(name, lan, n)) + + command = [core.FFMPEG, '-loglevel', 'warning', '-i', file, '-vn', '-an', + '-codec:{index}'.format(index=idx), 'srt', output_file] + if platform.system() != 'Windows': + command = core.NICENESS + command + + logger.info('Extracting {0} subtitle from: {1}'.format(lan, file)) + print_cmd(command) + result = 1 # set result to failed in case call fails. + try: + proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) + proc.communicate() + result = proc.returncode + except Exception: + logger.error('Extracting subtitle has failed') + + if result == 0: + try: + shutil.copymode(file, output_file) + except Exception: + pass + logger.info('Extracting {0} subtitle from {1} has succeeded'.format(lan, file)) + else: + logger.error('Extracting subtitles has failed') + + +def process_list(it, new_dir, bitbucket): + rem_list = [] + new_list = [] + combine = [] + vts_path = None + success = True + for item in it: + ext = os.path.splitext(item)[1].lower() + if ext in ['.iso', '.bin', '.img'] and ext not in core.IGNOREEXTENSIONS: + logger.debug('Attempting to rip disk image: {0}'.format(item), 'TRANSCODER') + new_list.extend(rip_iso(item, new_dir, bitbucket)) + rem_list.append(item) + elif re.match('.+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', item) and '.vob' not in core.IGNOREEXTENSIONS: + logger.debug('Found VIDEO_TS image file: {0}'.format(item), 'TRANSCODER') + if not vts_path: + try: + vts_path = re.match('(.+VIDEO_TS)', item).groups()[0] + except Exception: + vts_path = os.path.split(item)[0] + rem_list.append(item) + elif re.match('.+VIDEO_TS.', item) or re.match('.+VTS_[0-9][0-9]_[0-9].', item): + rem_list.append(item) + elif core.CONCAT and re.match('.+[cC][dD][0-9].', item): + rem_list.append(item) + combine.append(item) + else: + continue + if vts_path: + new_list.extend(combine_vts(vts_path)) + if combine: + new_list.extend(combine_cd(combine)) + for file in new_list: + if isinstance(file, string_types) and 'concat:' not in file and not os.path.isfile(file): + success = False + break + if success and new_list: + it.extend(new_list) + for item in rem_list: + it.remove(item) + logger.debug('Successfully extracted .vob file {0} from disk image'.format(new_list[0]), 'TRANSCODER') + elif new_list and not success: + new_list = [] + rem_list = [] + logger.error('Failed extracting .vob files from disk image. Stopping transcoding.', 'TRANSCODER') + return it, rem_list, new_list, success + + +def rip_iso(item, new_dir, bitbucket): + new_files = [] + failure_dir = 'failure' + # Mount the ISO in your OS and call combineVTS. + if not core.SEVENZIP: + logger.error('No 7zip installed. Can\'t extract image file {0}'.format(item), 'TRANSCODER') + new_files = [failure_dir] + return new_files + cmd = [core.SEVENZIP, 'l', item] + try: + logger.debug('Attempting to extract .vob from image file {0}'.format(item), 'TRANSCODER') + print_cmd(cmd) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) + out, err = proc.communicate() + file_list = [re.match(r'.+(VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])', line).groups()[0] for line in + out.splitlines() if re.match(r'.+VIDEO_TS[/\\]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]', line)] + combined = [] + for n in range(99): + concat = [] + m = 1 + while True: + vts_name = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m) + if vts_name in file_list: + concat.append(vts_name) + m += 1 + else: + break + if not concat: + break + if core.CONCAT: + combined.extend(concat) + continue + name = '{name}.cd{x}'.format( + name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1 + ) + new_files.append({item: {'name': name, 'files': concat}}) + if core.CONCAT: + name = os.path.splitext(os.path.split(item)[1])[0] + new_files.append({item: {'name': name, 'files': combined}}) + if not new_files: + logger.error('No VIDEO_TS folder found in image file {0}'.format(item), 'TRANSCODER') + new_files = [failure_dir] + except Exception: + logger.error('Failed to extract from image file {0}'.format(item), 'TRANSCODER') + new_files = [failure_dir] + return new_files + + +def combine_vts(vts_path): + new_files = [] + combined = '' + for n in range(99): + concat = '' + m = 1 + while True: + vts_name = 'VTS_{0:02d}_{1:d}.VOB'.format(n + 1, m) + if os.path.isfile(os.path.join(vts_path, vts_name)): + concat += '{file}|'.format(file=os.path.join(vts_path, vts_name)) + m += 1 + else: + break + if not concat: + break + if core.CONCAT: + combined += '{files}|'.format(files=concat) + continue + new_files.append('concat:{0}'.format(concat[:-1])) + if core.CONCAT: + new_files.append('concat:{0}'.format(combined[:-1])) + return new_files + + +def combine_cd(combine): + new_files = [] + for item in set([re.match('(.+)[cC][dD][0-9].', item).groups()[0] for item in combine]): + concat = '' + for n in range(99): + files = [file for file in combine if + n + 1 == int(re.match('.+[cC][dD]([0-9]+).', file).groups()[0]) and item in file] + if files: + concat += '{file}|'.format(file=files[0]) + else: + break + if concat: + new_files.append('concat:{0}'.format(concat[:-1])) + return new_files + + +def print_cmd(command): + cmd = '' + for item in command: + cmd = '{cmd} {item}'.format(cmd=cmd, item=item) + logger.debug('calling command:{0}'.format(cmd)) + + +def transcode_directory(dir_name): + if not core.FFMPEG: + return 1, dir_name + logger.info('Checking for files to be transcoded') + final_result = 0 # initialize as successful + if core.OUTPUTVIDEOPATH: + new_dir = core.OUTPUTVIDEOPATH + make_dir(new_dir) + name = os.path.splitext(os.path.split(dir_name)[1])[0] + new_dir = os.path.join(new_dir, name) + make_dir(new_dir) + else: + new_dir = dir_name + if platform.system() == 'Windows': + bitbucket = open('NUL') + else: + bitbucket = open('/dev/null') + movie_name = os.path.splitext(os.path.split(dir_name)[1])[0] + file_list = core.list_media_files(dir_name, media=True, audio=False, meta=False, archives=False) + file_list, rem_list, new_list, success = process_list(file_list, new_dir, bitbucket) + if not success: + bitbucket.close() + return 1, dir_name + + for file in file_list: + if isinstance(file, string_types) and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS: + continue + command = build_commands(file, new_dir, movie_name, bitbucket) + newfile_path = command[-1] + + # transcoding files may remove the original file, so make sure to extract subtitles first + if core.SEXTRACT and isinstance(file, string_types): + extract_subs(file, newfile_path, bitbucket) + + try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason) + os.remove(newfile_path) + except OSError as e: + if e.errno != errno.ENOENT: # Ignore the error if it's just telling us that the file doesn't exist + logger.debug('Error when removing transcoding target: {0}'.format(e)) + except Exception as e: + logger.debug('Error when removing transcoding target: {0}'.format(e)) + + logger.info('Transcoding video: {0}'.format(newfile_path)) + print_cmd(command) + result = 1 # set result to failed in case call fails. + try: + if isinstance(file, string_types): + proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) + else: + img, data = next(iteritems(file)) + proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket, stdin=subprocess.PIPE) + for vob in data['files']: + procin = zip_out(vob, img, bitbucket) + if procin: + shutil.copyfileobj(procin.stdout, proc.stdin) + procin.stdout.close() + proc.communicate() + result = proc.returncode + except Exception: + logger.error('Transcoding of video {0} has failed'.format(newfile_path)) + + if core.SUBSDIR and result == 0 and isinstance(file, string_types): + for sub in get_subs(file): + name = os.path.splitext(os.path.split(file)[1])[0] + subname = os.path.split(sub)[1] + newname = os.path.splitext(os.path.split(newfile_path)[1])[0] + newpath = os.path.join(core.SUBSDIR, subname.replace(name, newname)) + if not os.path.isfile(newpath): + os.rename(sub, newpath) + + if result == 0: + try: + shutil.copymode(file, newfile_path) + except Exception: + pass + logger.info('Transcoding of video to {0} succeeded'.format(newfile_path)) + if os.path.isfile(newfile_path) and (file in new_list or not core.DUPLICATE): + try: + os.unlink(file) + except Exception: + pass + else: + logger.error('Transcoding of video to {0} failed with result {1}'.format(newfile_path, result)) + # this will be 0 (successful) it all are successful, else will return a positive integer for failure. + final_result = final_result + result + if final_result == 0 and not core.DUPLICATE: + for file in rem_list: + try: + os.unlink(file) + except Exception: + pass + if not os.listdir(text_type(new_dir)): # this is an empty directory and we didn't transcode into it. + os.rmdir(new_dir) + new_dir = dir_name + if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB + new_dir = dir_name + bitbucket.close() + return final_result, new_dir diff --git a/core/transcoder/__init__.py b/core/transcoder/__init__.py deleted file mode 100644 index b1629751e..000000000 --- a/core/transcoder/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding=utf-8 -__author__ = 'Justin' diff --git a/core/transcoder/transcoder.py b/core/transcoder/transcoder.py deleted file mode 100644 index 19427aa62..000000000 --- a/core/transcoder/transcoder.py +++ /dev/null @@ -1,824 +0,0 @@ -# coding=utf-8 - -from six import iteritems -import errno -import os -import platform -import subprocess -import core -import json -import shutil -import re -from core import logger -from core.nzbToMediaUtil import makeDir -from babelfish import Language - - -def isVideoGood(videofile, status): - fileNameExt = os.path.basename(videofile) - fileName, fileExt = os.path.splitext(fileNameExt) - disable = False - if fileExt not in core.MEDIACONTAINER or not core.FFPROBE or not core.CHECK_MEDIA or fileExt in ['.iso']: - disable = True - else: - test_details, res = getVideoDetails(core.TEST_FILE) - if res != 0 or test_details.get("error"): - disable = True - logger.info("DISABLED: ffprobe failed to analyse test file. Stopping corruption check.", 'TRANSCODER') - if test_details.get("streams"): - vidStreams = [item for item in test_details["streams"] if "codec_type" in item and item["codec_type"] == "video"] - audStreams = [item for item in test_details["streams"] if "codec_type" in item and item["codec_type"] == "audio"] - if not (len(vidStreams) > 0 and len(audStreams) > 0): - disable = True - logger.info("DISABLED: ffprobe failed to analyse streams from test file. Stopping corruption check.", - 'TRANSCODER') - if disable: - if status: # if the download was "failed", assume bad. If it was successful, assume good. - return False - else: - return True - - logger.info('Checking [{0}] for corruption, please stand by ...'.format(fileNameExt), 'TRANSCODER') - video_details, result = getVideoDetails(videofile) - - if result != 0: - logger.error("FAILED: [{0}] is corrupted!".format(fileNameExt), 'TRANSCODER') - return False - if video_details.get("error"): - logger.info("FAILED: [{0}] returned error [{1}].".format(fileNameExt, video_details.get("error")), 'TRANSCODER') - return False - if video_details.get("streams"): - videoStreams = [item for item in video_details["streams"] if item["codec_type"] == "video"] - audioStreams = [item for item in video_details["streams"] if item["codec_type"] == "audio"] - if len(videoStreams) > 0 and len(audioStreams) > 0: - logger.info("SUCCESS: [{0}] has no corruption.".format(fileNameExt), 'TRANSCODER') - return True - else: - logger.info("FAILED: [{0}] has {1} video streams and {2} audio streams. " - "Assume corruption.".format - (fileNameExt, len(videoStreams), len(audioStreams)), 'TRANSCODER') - return False - - -def zip_out(file, img, bitbucket): - procin = None - cmd = [core.SEVENZIP, '-so', 'e', img, file] - try: - procin = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) - except: - logger.error("Extracting [{0}] has failed".format(file), 'TRANSCODER') - return procin - - -def getVideoDetails(videofile, img=None, bitbucket=None): - video_details = {} - result = 1 - file = videofile - if not core.FFPROBE: - return video_details, result - if 'avprobe' in core.FFPROBE: - print_format = '-of' - else: - print_format = '-print_format' - try: - if img: - videofile = '-' - command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', '-show_error', - videofile] - print_cmd(command) - if img: - procin = zip_out(file, img, bitbucket) - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) - procin.stdout.close() - else: - proc = subprocess.Popen(command, stdout=subprocess.PIPE) - out, err = proc.communicate() - result = proc.returncode - video_details = json.loads(out) - except: - pass - if not video_details: - try: - command = [core.FFPROBE, '-v', 'quiet', print_format, 'json', '-show_format', '-show_streams', videofile] - if img: - procin = zip_out(file, img) - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=procin.stdout) - procin.stdout.close() - else: - proc = subprocess.Popen(command, stdout=subprocess.PIPE) - out, err = proc.communicate() - result = proc.returncode - video_details = json.loads(out) - except: - logger.error("Checking [{0}] has failed".format(file), 'TRANSCODER') - return video_details, result - - -def buildCommands(file, newDir, movieName, bitbucket): - if isinstance(file, basestring): - inputFile = file - if 'concat:' in file: - file = file.split('|')[0].replace('concat:', '') - video_details, result = getVideoDetails(file) - dir, name = os.path.split(file) - name, ext = os.path.splitext(name) - check = re.match("VTS_([0-9][0-9])_[0-9]+", name) - if check and core.CONCAT: - name = movieName - elif check: - name = ('{0}.cd{1}'.format(movieName, check.groups()[0])) - elif core.CONCAT and re.match("(.+)[cC][dD][0-9]", name): - name = re.sub("([\ \.\-\_\=\:]+[cC][dD][0-9])", "", name) - if ext == core.VEXTENSION and newDir == dir: # we need to change the name to prevent overwriting itself. - core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext' - else: - img, data = iteritems(file).next() - name = data['name'] - video_details, result = getVideoDetails(data['files'][0], img, bitbucket) - inputFile = '-' - file = '-' - - newfilePath = os.path.normpath(os.path.join(newDir, name) + core.VEXTENSION) - - map_cmd = [] - video_cmd = [] - audio_cmd = [] - audio_cmd2 = [] - sub_cmd = [] - meta_cmd = [] - other_cmd = [] - - if not video_details or not video_details.get( - "streams"): # we couldn't read streams with ffprobe. Set defaults to try transcoding. - videoStreams = [] - audioStreams = [] - subStreams = [] - - map_cmd.extend(['-map', '0']) - if core.VCODEC: - video_cmd.extend(['-c:v', core.VCODEC]) - if core.VCODEC == 'libx264' and core.VPRESET: - video_cmd.extend(['-pre', core.VPRESET]) - else: - video_cmd.extend(['-c:v', 'copy']) - if core.VFRAMERATE: - video_cmd.extend(['-r', str(core.VFRAMERATE)]) - if core.VBITRATE: - video_cmd.extend(['-b:v', str(core.VBITRATE)]) - if core.VRESOLUTION: - video_cmd.extend(['-vf', 'scale={vres}'.format(vres=core.VRESOLUTION)]) - if core.VPRESET: - video_cmd.extend(['-preset', core.VPRESET]) - if core.VCRF: - video_cmd.extend(['-crf', str(core.VCRF)]) - if core.VLEVEL: - video_cmd.extend(['-level', str(core.VLEVEL)]) - - if core.ACODEC: - audio_cmd.extend(['-c:a', core.ACODEC]) - if core.ACODEC in ['aac', - 'dts']: # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg - audio_cmd.extend(['-strict', '-2']) - else: - audio_cmd.extend(['-c:a', 'copy']) - if core.ACHANNELS: - audio_cmd.extend(['-ac', str(core.ACHANNELS)]) - if core.ABITRATE: - audio_cmd.extend(['-b:a', str(core.ABITRATE)]) - if core.OUTPUTQUALITYPERCENT: - audio_cmd.extend(['-q:a', str(core.OUTPUTQUALITYPERCENT)]) - - if core.SCODEC and core.ALLOWSUBS: - sub_cmd.extend(['-c:s', core.SCODEC]) - elif core.ALLOWSUBS: # Not every subtitle codec can be used for every video container format! - sub_cmd.extend(['-c:s', 'copy']) - else: # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options - sub_cmd.extend(['-sn']) # Don't copy the subtitles over - - if core.OUTPUTFASTSTART: - other_cmd.extend(['-movflags', '+faststart']) - - else: - videoStreams = [item for item in video_details["streams"] if item["codec_type"] == "video"] - audioStreams = [item for item in video_details["streams"] if item["codec_type"] == "audio"] - subStreams = [item for item in video_details["streams"] if item["codec_type"] == "subtitle"] - if core.VEXTENSION not in ['.mkv', '.mpegts']: - subStreams = [item for item in video_details["streams"] if - item["codec_type"] == "subtitle" and item["codec_name"] != "hdmv_pgs_subtitle" and item[ - "codec_name"] != "pgssub"] - - for video in videoStreams: - codec = video["codec_name"] - fr = video.get("avg_frame_rate", 0) - width = video.get("width", 0) - height = video.get("height", 0) - scale = core.VRESOLUTION - if codec in core.VCODEC_ALLOW or not core.VCODEC: - video_cmd.extend(['-c:v', 'copy']) - else: - video_cmd.extend(['-c:v', core.VCODEC]) - if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001): - video_cmd.extend(['-r', str(core.VFRAMERATE)]) - if scale: - w_scale = width / float(scale.split(':')[0]) - h_scale = height / float(scale.split(':')[1]) - if w_scale > h_scale: # widescreen, Scale by width only. - scale = "{width}:{height}".format( - width=scale.split(':')[0], - height=int((height / w_scale) / 2) * 2, - ) - if w_scale > 1: - video_cmd.extend(['-vf', 'scale={width}'.format(width=scale)]) - else: # lower or matching ratio, scale by height only. - scale = "{width}:{height}".format( - width=int((width / h_scale) / 2) * 2, - height=scale.split(':')[1], - ) - if h_scale > 1: - video_cmd.extend(['-vf', 'scale={height}'.format(height=scale)]) - if core.VBITRATE: - video_cmd.extend(['-b:v', str(core.VBITRATE)]) - if core.VPRESET: - video_cmd.extend(['-preset', core.VPRESET]) - if core.VCRF: - video_cmd.extend(['-crf', str(core.VCRF)]) - if core.VLEVEL: - video_cmd.extend(['-level', str(core.VLEVEL)]) - no_copy = ['-vf', '-r', '-crf', '-level', '-preset', '-b:v'] - if video_cmd[1] == 'copy' and any(i in video_cmd for i in no_copy): - video_cmd[1] = core.VCODEC - if core.VCODEC == 'copy': # force copy. therefore ignore all other video transcoding. - video_cmd = ['-c:v', 'copy'] - map_cmd.extend(['-map', '0:{index}'.format(index=video["index"])]) - break # Only one video needed - - used_audio = 0 - a_mapped = [] - commentary = [] - if audioStreams: - for i, val in reversed(list(enumerate(audioStreams))): - try: - if "Commentary" in val.get("tags").get("title"): # Split out commentry tracks. - commentary.append(val) - del audioStreams[i] - except: - continue - try: - audio1 = [item for item in audioStreams if item["tags"]["language"] == core.ALANGUAGE] - except: # no language tags. Assume only 1 language. - audio1 = audioStreams - try: - audio2 = [item for item in audio1 if item["codec_name"] in core.ACODEC_ALLOW] - except: - audio2 = [] - try: - audio3 = [item for item in audioStreams if item["tags"]["language"] != core.ALANGUAGE] - except: - audio3 = [] - try: - audio4 = [item for item in audio3 if item["codec_name"] in core.ACODEC_ALLOW] - except: - audio4 = [] - - if audio2: # right (or only) language and codec... - map_cmd.extend(['-map', '0:{index}'.format(index=audio2[0]["index"])]) - a_mapped.extend([audio2[0]["index"]]) - bitrate = int(float(audio2[0].get("bit_rate", 0))) / 1000 - channels = int(float(audio2[0].get("channels", 0))) - audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy']) - elif audio1: # right (or only) language, wrong codec. - map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]["index"])]) - a_mapped.extend([audio1[0]["index"]]) - bitrate = int(float(audio1[0].get("bit_rate", 0))) / 1000 - channels = int(float(audio1[0].get("channels", 0))) - audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy']) - elif audio4: # wrong language, right codec. - map_cmd.extend(['-map', '0:{index}'.format(index=audio4[0]["index"])]) - a_mapped.extend([audio4[0]["index"]]) - bitrate = int(float(audio4[0].get("bit_rate", 0))) / 1000 - channels = int(float(audio4[0].get("channels", 0))) - audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy']) - elif audio3: # wrong language, wrong codec. just pick the default audio track - map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]["index"])]) - a_mapped.extend([audio3[0]["index"]]) - bitrate = int(float(audio3[0].get("bit_rate", 0))) / 1000 - channels = int(float(audio3[0].get("channels", 0))) - audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy']) - - if core.ACHANNELS and channels and channels > core.ACHANNELS: - audio_cmd.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS)]) - if audio_cmd[1] == 'copy': - audio_cmd[1] = core.ACODEC - if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1): - audio_cmd.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE)]) - if audio_cmd[1] == 'copy': - audio_cmd[1] = core.ACODEC - if core.OUTPUTQUALITYPERCENT: - audio_cmd.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)]) - if audio_cmd[1] == 'copy': - audio_cmd[1] = core.ACODEC - if audio_cmd[1] in ['aac', 'dts']: - audio_cmd[2:2] = ['-strict', '-2'] - - if core.ACODEC2_ALLOW: - used_audio += 1 - try: - audio5 = [item for item in audio1 if item["codec_name"] in core.ACODEC2_ALLOW] - except: - audio5 = [] - try: - audio6 = [item for item in audio3 if item["codec_name"] in core.ACODEC2_ALLOW] - except: - audio6 = [] - if audio5: # right language and codec. - map_cmd.extend(['-map', '0:{index}'.format(index=audio5[0]["index"])]) - a_mapped.extend([audio5[0]["index"]]) - bitrate = int(float(audio5[0].get("bit_rate", 0))) / 1000 - channels = int(float(audio5[0].get("channels", 0))) - audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) - elif audio1: # right language wrong codec. - map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]["index"])]) - a_mapped.extend([audio1[0]["index"]]) - bitrate = int(float(audio1[0].get("bit_rate", 0))) / 1000 - channels = int(float(audio1[0].get("channels", 0))) - if core.ACODEC2: - audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2]) - else: - audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) - elif audio6: # wrong language, right codec - map_cmd.extend(['-map', '0:{index}'.format(index=audio6[0]["index"])]) - a_mapped.extend([audio6[0]["index"]]) - bitrate = int(float(audio6[0].get("bit_rate", 0))) / 1000 - channels = int(float(audio6[0].get("channels", 0))) - audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) - elif audio3: # wrong language, wrong codec just pick the default audio track - map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]["index"])]) - a_mapped.extend([audio3[0]["index"]]) - bitrate = int(float(audio3[0].get("bit_rate", 0))) / 1000 - channels = int(float(audio3[0].get("channels", 0))) - if core.ACODEC2: - audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2]) - else: - audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) - - if core.ACHANNELS2 and channels and channels > core.ACHANNELS2: - audio_cmd2.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS2)]) - if audio_cmd2[1] == 'copy': - audio_cmd2[1] = core.ACODEC2 - if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1): - audio_cmd2.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE2)]) - if audio_cmd2[1] == 'copy': - audio_cmd2[1] = core.ACODEC2 - if core.OUTPUTQUALITYPERCENT: - audio_cmd2.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)]) - if audio_cmd2[1] == 'copy': - audio_cmd2[1] = core.ACODEC2 - if audio_cmd2[1] in ['aac', 'dts']: - audio_cmd2[2:2] = ['-strict', '-2'] - - if a_mapped[1] == a_mapped[0] and audio_cmd2[1:] == audio_cmd[1:]: #check for duplicate output track. - del map_cmd[-2:] - else: - audio_cmd.extend(audio_cmd2) - - if core.AINCLUDE and core.ACODEC3: - audioStreams.extend(commentary) #add commentry tracks back here. - for audio in audioStreams: - if audio["index"] in a_mapped: - continue - used_audio += 1 - map_cmd.extend(['-map', '0:{index}'.format(index=audio["index"])]) - audio_cmd3 = [] - bitrate = int(float(audio.get("bit_rate", 0))) / 1000 - channels = int(float(audio.get("channels", 0))) - if audio["codec_name"] in core.ACODEC3_ALLOW: - audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy']) - else: - if core.ACODEC3: - audio_cmd3.extend(['-c:a:{0}'.format(used_audio), core.ACODEC3]) - else: - audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy']) - - if core.ACHANNELS3 and channels and channels > core.ACHANNELS3: - audio_cmd3.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS3)]) - if audio_cmd3[1] == 'copy': - audio_cmd3[1] = core.ACODEC3 - if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1): - audio_cmd3.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE3)]) - if audio_cmd3[1] == 'copy': - audio_cmd3[1] = core.ACODEC3 - if core.OUTPUTQUALITYPERCENT > 0: - audio_cmd3.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)]) - if audio_cmd3[1] == 'copy': - audio_cmd3[1] = core.ACODEC3 - if audio_cmd3[1] in ['aac', 'dts']: - audio_cmd3[2:2] = ['-strict', '-2'] - audio_cmd.extend(audio_cmd3) - - s_mapped = [] - burnt = 0 - n = 0 - for lan in core.SLANGUAGES: - try: - subs1 = [item for item in subStreams if item["tags"]["language"] == lan] - except: - subs1 = [] - if core.BURN and not subs1 and not burnt and os.path.isfile(file): - for subfile in get_subs(file): - if lan in os.path.split(subfile)[1]: - video_cmd.extend(['-vf', 'subtitles={subs}'.format(subs=subfile)]) - burnt = 1 - for sub in subs1: - if core.BURN and not burnt and os.path.isfile(inputFile): - subloc = 0 - for index in range(len(subStreams)): - if subStreams[index]["index"] == sub["index"]: - subloc = index - break - video_cmd.extend(['-vf', 'subtitles={sub}:si={loc}'.format(sub=inputFile, loc=subloc)]) - burnt = 1 - if not core.ALLOWSUBS: - break - if sub["codec_name"] in ["dvd_subtitle", "VobSub"] and core.SCODEC == "mov_text": # We can't convert these. - continue - map_cmd.extend(['-map', '0:{index}'.format(index=sub["index"])]) - s_mapped.extend([sub["index"]]) - - if core.SINCLUDE: - for sub in subStreams: - if not core.ALLOWSUBS: - break - if sub["index"] in s_mapped: - continue - if sub["codec_name"] in ["dvd_subtitle", "VobSub"] and core.SCODEC == "mov_text": # We can't convert these. - continue - map_cmd.extend(['-map', '0:{index}'.format(index=sub["index"])]) - s_mapped.extend([sub["index"]]) - - if core.OUTPUTFASTSTART: - other_cmd.extend(['-movflags', '+faststart']) - - command = [core.FFMPEG, '-loglevel', 'warning'] - - if core.HWACCEL: - command.extend(['-hwaccel', 'auto']) - if core.GENERALOPTS: - command.extend(core.GENERALOPTS) - - command.extend(['-i', inputFile]) - - if core.SEMBED and os.path.isfile(file): - for subfile in get_subs(file): - sub_details, result = getVideoDetails(subfile) - if not sub_details or not sub_details.get("streams"): - continue - if core.SCODEC == "mov_text": - subcode = [stream["codec_name"] for stream in sub_details["streams"]] - if set(subcode).intersection(["dvd_subtitle", "VobSub"]): # We can't convert these. - continue - command.extend(['-i', subfile]) - lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split('-')[0] - metlan = None - try: - if len(lan) == 3: - metlan = Language(lan) - if len(lan) == 2: - metlan = Language.fromalpha2(lan) - except: pass - if metlan: - meta_cmd.extend(['-metadata:s:s:{x}'.format(x=len(s_mapped) + n), - 'language={lang}'.format(lang=metlan.alpha3)]) - n += 1 - map_cmd.extend(['-map', '{x}:0'.format(x=n)]) - - if not core.ALLOWSUBS or (not s_mapped and not n): - sub_cmd.extend(['-sn']) - else: - if core.SCODEC: - sub_cmd.extend(['-c:s', core.SCODEC]) - else: - sub_cmd.extend(['-c:s', 'copy']) - - command.extend(map_cmd) - command.extend(video_cmd) - command.extend(audio_cmd) - command.extend(sub_cmd) - command.extend(meta_cmd) - command.extend(other_cmd) - command.append(newfilePath) - if platform.system() != 'Windows': - command = core.NICENESS + command - return command - - -def get_subs(file): - filepaths = [] - subExt = ['.srt', '.sub', '.idx'] - name = os.path.splitext(os.path.split(file)[1])[0] - dir = os.path.split(file)[0] - for dirname, dirs, filenames in os.walk(dir): - for filename in filenames: - filepaths.extend([os.path.join(dirname, filename)]) - subfiles = [item for item in filepaths if os.path.splitext(item)[1] in subExt and name in item] - return subfiles - - -def extract_subs(file, newfilePath, bitbucket): - video_details, result = getVideoDetails(file) - if not video_details: - return - - if core.SUBSDIR: - subdir = core.SUBSDIR - else: - subdir = os.path.split(newfilePath)[0] - name = os.path.splitext(os.path.split(newfilePath)[1])[0] - - try: - subStreams = [item for item in video_details["streams"] if - item["codec_type"] == "subtitle" and item["tags"]["language"] in core.SLANGUAGES and item[ - "codec_name"] != "hdmv_pgs_subtitle" and item["codec_name"] != "pgssub"] - except: - subStreams = [item for item in video_details["streams"] if - item["codec_type"] == "subtitle" and item["codec_name"] != "hdmv_pgs_subtitle" and item[ - "codec_name"] != "pgssub"] - num = len(subStreams) - for n in range(num): - sub = subStreams[n] - idx = sub["index"] - lan = sub.get("tags", {}).get("language", "unk") - - if num == 1: - outputFile = os.path.join(subdir, "{0}.srt".format(name)) - if os.path.isfile(outputFile): - outputFile = os.path.join(subdir, "{0}.{1}.srt".format(name, n)) - else: - outputFile = os.path.join(subdir, "{0}.{1}.srt".format(name, lan)) - if os.path.isfile(outputFile): - outputFile = os.path.join(subdir, "{0}.{1}.{2}.srt".format(name, lan, n)) - - command = [core.FFMPEG, '-loglevel', 'warning', '-i', file, '-vn', '-an', - '-codec:{index}'.format(index=idx), 'srt', outputFile] - if platform.system() != 'Windows': - command = core.NICENESS + command - - logger.info("Extracting {0} subtitle from: {1}".format(lan, file)) - print_cmd(command) - result = 1 # set result to failed in case call fails. - try: - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) - proc.communicate() - result = proc.returncode - except: - logger.error("Extracting subtitle has failed") - - if result == 0: - try: - shutil.copymode(file, outputFile) - except: - pass - logger.info("Extracting {0} subtitle from {1} has succeeded".format(lan, file)) - else: - logger.error("Extracting subtitles has failed") - - -def processList(List, newDir, bitbucket): - remList = [] - newList = [] - combine = [] - vtsPath = None - success = True - for item in List: - ext = os.path.splitext(item)[1].lower() - if ext in ['.iso', '.bin', '.img'] and ext not in core.IGNOREEXTENSIONS: - logger.debug("Attempting to rip disk image: {0}".format(item), "TRANSCODER") - newList.extend(ripISO(item, newDir, bitbucket)) - remList.append(item) - elif re.match(".+VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]", item) and '.vob' not in core.IGNOREEXTENSIONS: - logger.debug("Found VIDEO_TS image file: {0}".format(item), "TRANSCODER") - if not vtsPath: - try: - vtsPath = re.match("(.+VIDEO_TS)", item).groups()[0] - except: - vtsPath = os.path.split(item)[0] - remList.append(item) - elif re.match(".+VIDEO_TS.", item) or re.match(".+VTS_[0-9][0-9]_[0-9].", item): - remList.append(item) - elif core.CONCAT and re.match(".+[cC][dD][0-9].", item): - remList.append(item) - combine.append(item) - else: - continue - if vtsPath: - newList.extend(combineVTS(vtsPath)) - if combine: - newList.extend(combineCD(combine)) - for file in newList: - if isinstance(file, basestring) and 'concat:' not in file and not os.path.isfile(file): - success = False - break - if success and newList: - List.extend(newList) - for item in remList: - List.remove(item) - logger.debug("Successfully extracted .vob file {0} from disk image".format(newList[0]), "TRANSCODER") - elif newList and not success: - newList = [] - remList = [] - logger.error("Failed extracting .vob files from disk image. Stopping transcoding.", "TRANSCODER") - return List, remList, newList, success - - -def ripISO(item, newDir, bitbucket): - newFiles = [] - failure_dir = 'failure' - # Mount the ISO in your OS and call combineVTS. - if not core.SEVENZIP: - logger.error("No 7zip installed. Can't extract image file {0}".format(item), "TRANSCODER") - newFiles = [failure_dir] - return newFiles - cmd = [core.SEVENZIP, 'l', item] - try: - logger.debug("Attempting to extract .vob from image file {0}".format(item), "TRANSCODER") - print_cmd(cmd) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=bitbucket) - out, err = proc.communicate() - fileList = [re.match(".+(VIDEO_TS[\\\/]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb])", line).groups()[0] for line in - out.splitlines() if re.match(".+VIDEO_TS[\\\/]VTS_[0-9][0-9]_[0-9].[Vv][Oo][Bb]", line)] - combined = [] - for n in range(99): - concat = [] - m = 1 - while True: - vtsName = 'VIDEO_TS{0}VTS_{1:02d}_{2:d}.VOB'.format(os.sep, n + 1, m) - if vtsName in fileList: - concat.append(vtsName) - m += 1 - else: - break - if not concat: - break - if core.CONCAT: - combined.extend(concat) - continue - name = '{name}.cd{x}'.format( - name=os.path.splitext(os.path.split(item)[1])[0], x=n + 1 - ) - newFiles.append({item: {'name': name, 'files': concat}}) - if core.CONCAT: - name = os.path.splitext(os.path.split(item)[1])[0] - newFiles.append({item: {'name': name, 'files': combined}}) - if not newFiles: - logger.error("No VIDEO_TS folder found in image file {0}".format(item), "TRANSCODER") - newFiles = [failure_dir] - except: - logger.error("Failed to extract from image file {0}".format(item), "TRANSCODER") - newFiles = [failure_dir] - return newFiles - - -def combineVTS(vtsPath): - newFiles = [] - combined = '' - for n in range(99): - concat = '' - m = 1 - while True: - vtsName = 'VTS_{0:02d}_{1:d}.VOB'.format(n + 1, m) - if os.path.isfile(os.path.join(vtsPath, vtsName)): - concat += '{file}|'.format(file=os.path.join(vtsPath, vtsName)) - m += 1 - else: - break - if not concat: - break - if core.CONCAT: - combined += '{files}|'.format(files=concat) - continue - newFiles.append('concat:{0}'.format(concat[:-1])) - if core.CONCAT: - newFiles.append('concat:{0}'.format(combined[:-1])) - return newFiles - - -def combineCD(combine): - newFiles = [] - for item in set([re.match("(.+)[cC][dD][0-9].", item).groups()[0] for item in combine]): - concat = '' - for n in range(99): - files = [file for file in combine if - n + 1 == int(re.match(".+[cC][dD]([0-9]+).", file).groups()[0]) and item in file] - if files: - concat += '{file}|'.format(file=files[0]) - else: - break - if concat: - newFiles.append('concat:{0}'.format(concat[:-1])) - return newFiles - - -def print_cmd(command): - cmd = "" - for item in command: - cmd = "{cmd} {item}".format(cmd=cmd, item=item) - logger.debug("calling command:{0}".format(cmd)) - - -def Transcode_directory(dirName): - if not core.FFMPEG: - return 1, dirName - logger.info("Checking for files to be transcoded") - final_result = 0 # initialize as successful - if core.OUTPUTVIDEOPATH: - newDir = core.OUTPUTVIDEOPATH - makeDir(newDir) - name = os.path.splitext(os.path.split(dirName)[1])[0] - newDir = os.path.join(newDir, name) - makeDir(newDir) - else: - newDir = dirName - if platform.system() == 'Windows': - bitbucket = open('NUL') - else: - bitbucket = open('/dev/null') - movieName = os.path.splitext(os.path.split(dirName)[1])[0] - List = core.listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False) - List, remList, newList, success = processList(List, newDir, bitbucket) - if not success: - bitbucket.close() - return 1, dirName - - for file in List: - if isinstance(file, basestring) and os.path.splitext(file)[1] in core.IGNOREEXTENSIONS: - continue - command = buildCommands(file, newDir, movieName, bitbucket) - newfilePath = command[-1] - - # transcoding files may remove the original file, so make sure to extract subtitles first - if core.SEXTRACT and isinstance(file, basestring): - extract_subs(file, newfilePath, bitbucket) - - try: # Try to remove the file that we're transcoding to just in case. (ffmpeg will return an error if it already exists for some reason) - os.remove(newfilePath) - except OSError as e: - if e.errno != errno.ENOENT: # Ignore the error if it's just telling us that the file doesn't exist - logger.debug("Error when removing transcoding target: {0}".format(e)) - except Exception as e: - logger.debug("Error when removing transcoding target: {0}".format(e)) - - logger.info("Transcoding video: {0}".format(newfilePath)) - print_cmd(command) - result = 1 # set result to failed in case call fails. - try: - if isinstance(file, basestring): - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket) - else: - img, data = iteritems(file).next() - proc = subprocess.Popen(command, stdout=bitbucket, stderr=bitbucket, stdin=subprocess.PIPE) - for vob in data['files']: - procin = zip_out(vob, img, bitbucket) - if procin: - shutil.copyfileobj(procin.stdout, proc.stdin) - procin.stdout.close() - proc.communicate() - result = proc.returncode - except: - logger.error("Transcoding of video {0} has failed".format(newfilePath)) - - if core.SUBSDIR and result == 0 and isinstance(file, basestring): - for sub in get_subs(file): - name = os.path.splitext(os.path.split(file)[1])[0] - subname = os.path.split(sub)[1] - newname = os.path.splitext(os.path.split(newfilePath)[1])[0] - newpath = os.path.join(core.SUBSDIR, subname.replace(name, newname)) - if not os.path.isfile(newpath): - os.rename(sub, newpath) - - if result == 0: - try: - shutil.copymode(file, newfilePath) - except: - pass - logger.info("Transcoding of video to {0} succeeded".format(newfilePath)) - if os.path.isfile(newfilePath) and (file in newList or not core.DUPLICATE): - try: - os.unlink(file) - except: - pass - else: - logger.error("Transcoding of video to {0} failed with result {1}".format(newfilePath, result)) - # this will be 0 (successful) it all are successful, else will return a positive integer for failure. - final_result = final_result + result - if final_result == 0 and not core.DUPLICATE: - for file in remList: - try: - os.unlink(file) - except: - pass - if not os.listdir(unicode(newDir)): # this is an empty directory and we didn't transcode into it. - os.rmdir(newDir) - newDir = dirName - if not core.PROCESSOUTPUT and core.DUPLICATE: # We postprocess the original files to CP/SB - newDir = dirName - bitbucket.close() - return final_result, newDir diff --git a/core/transmissionrpc/__init__.py b/core/transmissionrpc/__init__.py deleted file mode 100644 index c0ced381e..000000000 --- a/core/transmissionrpc/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2008-2013 Erik Svensson -# Licensed under the MIT license. - -from core.transmissionrpc.constants import DEFAULT_PORT, DEFAULT_TIMEOUT, PRIORITY, RATIO_LIMIT, LOGGER -from core.transmissionrpc.error import TransmissionError, HTTPHandlerError -from core.transmissionrpc.httphandler import HTTPHandler, DefaultHTTPHandler -from core.transmissionrpc.torrent import Torrent -from core.transmissionrpc.session import Session -from core.transmissionrpc.client import Client -from core.transmissionrpc.utils import add_stdout_logger, add_file_logger - -__author__ = 'Erik Svensson ' -__version_major__ = 0 -__version_minor__ = 11 -__version__ = '{0}.{1}'.format(__version_major__, __version_minor__) -__copyright__ = 'Copyright (c) 2008-2013 Erik Svensson' -__license__ = 'MIT' diff --git a/core/transmissionrpc/client.py b/core/transmissionrpc/client.py deleted file mode 100644 index 66353762c..000000000 --- a/core/transmissionrpc/client.py +++ /dev/null @@ -1,933 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2008-2013 Erik Svensson -# Licensed under the MIT license. - -import re -import time -import operator -import warnings -import os -import base64 -import json - -from core.transmissionrpc.constants import DEFAULT_PORT, DEFAULT_TIMEOUT -from core.transmissionrpc.error import TransmissionError, HTTPHandlerError -from core.transmissionrpc.utils import LOGGER, get_arguments, make_rpc_name, argument_value_convert, rpc_bool -from core.transmissionrpc.httphandler import DefaultHTTPHandler -from core.transmissionrpc.torrent import Torrent -from core.transmissionrpc.session import Session -from six import PY3, integer_types, string_types, iteritems - -from six.moves.urllib_parse import urlparse -from six.moves.urllib_request import urlopen - - -def debug_httperror(error): - """ - Log the Transmission RPC HTTP error. - """ - try: - data = json.loads(error.data) - except ValueError: - data = error.data - LOGGER.debug( - json.dumps( - { - 'response': { - 'url': error.url, - 'code': error.code, - 'msg': error.message, - 'headers': error.headers, - 'data': data, - } - }, - indent=2 - ) - ) - - -def parse_torrent_id(arg): - """Parse an torrent id or torrent hashString.""" - torrent_id = None - if isinstance(arg, integer_types): - # handle index - torrent_id = int(arg) - elif isinstance(arg, float): - torrent_id = int(arg) - if torrent_id != arg: - torrent_id = None - elif isinstance(arg, string_types): - try: - torrent_id = int(arg) - if torrent_id >= 2 ** 31: - torrent_id = None - except (ValueError, TypeError): - pass - if torrent_id is None: - # handle hashes - try: - int(arg, 16) - torrent_id = arg - except (ValueError, TypeError): - pass - return torrent_id - - -def parse_torrent_ids(args): - """ - Take things and make them valid torrent identifiers - """ - ids = [] - - if args is None: - pass - elif isinstance(args, string_types): - for item in re.split('[ ,]+', args): - if len(item) == 0: - continue - addition = None - torrent_id = parse_torrent_id(item) - if torrent_id is not None: - addition = [torrent_id] - if not addition: - # handle index ranges i.e. 5:10 - match = re.match('^(\d+):(\d+)$', item) - if match: - try: - idx_from = int(match.group(1)) - idx_to = int(match.group(2)) - addition = list(range(idx_from, idx_to + 1)) - except ValueError: - pass - if not addition: - raise ValueError('Invalid torrent id, {item!r}'.format(item=item)) - ids.extend(addition) - elif isinstance(args, (list, tuple)): - for item in args: - ids.extend(parse_torrent_ids(item)) - else: - torrent_id = parse_torrent_id(args) - if torrent_id is None: - raise ValueError('Invalid torrent id') - else: - ids = [torrent_id] - return ids - - -""" -Torrent ids - -Many functions in Client takes torrent id. A torrent id can either be id or -hashString. When supplying multiple id's it is possible to use a list mixed -with both id and hashString. - -Timeouts - -Since most methods results in HTTP requests against Transmission, it is -possible to provide a argument called ``timeout``. Timeout is only effective -when using Python 2.6 or later and the default timeout is 30 seconds. -""" - - -class Client(object): - """ - Client is the class handling the Transmission JSON-RPC client protocol. - """ - - def __init__(self, address='localhost', port=DEFAULT_PORT, user=None, password=None, http_handler=None, - timeout=None): - if isinstance(timeout, (integer_types, float)): - self._query_timeout = float(timeout) - else: - self._query_timeout = DEFAULT_TIMEOUT - urlo = urlparse(address) - if not urlo.scheme: - self.url = 'http://{host}:{port}/transmission/rpc/'.format(host=address, port=port) - else: - if urlo.port: - self.url = '{url.scheme}://{url.hostname}:{url.port}{url.path}'.format(url=urlo) - else: - self.url = '{url.scheme}://{url.hostname}{url.path}'.format(url=urlo) - LOGGER.info('Using custom URL {url!r}.'.format(url=self.url)) - if urlo.username and urlo.password: - user = urlo.username - password = urlo.password - elif urlo.username or urlo.password: - LOGGER.warning('Either user or password missing, not using authentication.') - if http_handler is None: - self.http_handler = DefaultHTTPHandler() - else: - if hasattr(http_handler, 'set_authentication') and hasattr(http_handler, 'request'): - self.http_handler = http_handler - else: - raise ValueError('Invalid HTTP handler.') - if user and password: - self.http_handler.set_authentication(self.url, user, password) - elif user or password: - LOGGER.warning('Either user or password missing, not using authentication.') - self._sequence = 0 - self.session = None - self.session_id = 0 - self.server_version = None - self.protocol_version = None - self.get_session() - self.torrent_get_arguments = get_arguments('torrent-get' - , self.rpc_version) - - def _get_timeout(self): - """ - Get current timeout for HTTP queries. - """ - return self._query_timeout - - def _set_timeout(self, value): - """ - Set timeout for HTTP queries. - """ - self._query_timeout = float(value) - - def _del_timeout(self): - """ - Reset the HTTP query timeout to the default. - """ - self._query_timeout = DEFAULT_TIMEOUT - - timeout = property(_get_timeout, _set_timeout, _del_timeout, doc="HTTP query timeout.") - - def _http_query(self, query, timeout=None): - """ - Query Transmission through HTTP. - """ - headers = {'x-transmission-session-id': str(self.session_id)} - result = {} - request_count = 0 - if timeout is None: - timeout = self._query_timeout - while True: - LOGGER.debug( - json.dumps({'url': self.url, 'headers': headers, 'query': query, 'timeout': timeout}, indent=2)) - try: - result = self.http_handler.request(self.url, query, headers, timeout) - break - except HTTPHandlerError as error: - if error.code == 409: - LOGGER.info('Server responded with 409, trying to set session-id.') - if request_count > 1: - raise TransmissionError('Session ID negotiation failed.', error) - session_id = None - for key in list(error.headers.keys()): - if key.lower() == 'x-transmission-session-id': - session_id = error.headers[key] - self.session_id = session_id - headers = {'x-transmission-session-id': str(self.session_id)} - if session_id is None: - debug_httperror(error) - raise TransmissionError('Unknown conflict.', error) - else: - debug_httperror(error) - raise TransmissionError('Request failed.', error) - request_count += 1 - return result - - def _request(self, method, arguments=None, ids=None, require_ids=False, timeout=None): - """ - Send json-rpc request to Transmission using http POST - """ - if not isinstance(method, string_types): - raise ValueError('request takes method as string') - if arguments is None: - arguments = {} - if not isinstance(arguments, dict): - raise ValueError('request takes arguments as dict') - ids = parse_torrent_ids(ids) - if len(ids) > 0: - arguments['ids'] = ids - elif require_ids: - raise ValueError('request require ids') - - query = json.dumps({'tag': self._sequence, 'method': method, 'arguments': arguments}) - self._sequence += 1 - start = time.time() - http_data = self._http_query(query, timeout) - elapsed = time.time() - start - LOGGER.info('http request took {time:.3f} s'.format(time=elapsed)) - - try: - data = json.loads(http_data) - except ValueError as error: - LOGGER.error('Error: {msg}'.format(msg=error)) - LOGGER.error('Request: {request!r}'.format(request=query)) - LOGGER.error('HTTP data: {data!r}'.format(data=http_data)) - raise - - LOGGER.debug(json.dumps(data, indent=2)) - if 'result' in data: - if data['result'] != 'success': - raise TransmissionError('Query failed with result {result!r}.'.format(result=data['result'])) - else: - raise TransmissionError('Query failed without result.') - - results = {} - if method == 'torrent-get': - for item in data['arguments']['torrents']: - results[item['id']] = Torrent(self, item) - if self.protocol_version == 2 and 'peers' not in item: - self.protocol_version = 1 - elif method == 'torrent-add': - item = None - if 'torrent-added' in data['arguments']: - item = data['arguments']['torrent-added'] - elif 'torrent-duplicate' in data['arguments']: - item = data['arguments']['torrent-duplicate'] - if item: - results[item['id']] = Torrent(self, item) - else: - raise TransmissionError('Invalid torrent-add response.') - elif method == 'session-get': - self._update_session(data['arguments']) - elif method == 'session-stats': - # older versions of T has the return data in "session-stats" - if 'session-stats' in data['arguments']: - self._update_session(data['arguments']['session-stats']) - else: - self._update_session(data['arguments']) - elif method in ('port-test', 'blocklist-update', 'free-space', 'torrent-rename-path'): - results = data['arguments'] - else: - return None - - return results - - def _update_session(self, data): - """ - Update session data. - """ - if self.session: - self.session.from_request(data) - else: - self.session = Session(self, data) - - def _update_server_version(self): - """Decode the Transmission version string, if available.""" - if self.server_version is None: - version_major = 1 - version_minor = 30 - version_changeset = 0 - version_parser = re.compile('(\d).(\d+) \((\d+)\)') - if hasattr(self.session, 'version'): - match = version_parser.match(self.session.version) - if match: - version_major = int(match.group(1)) - version_minor = int(match.group(2)) - version_changeset = match.group(3) - self.server_version = (version_major, version_minor, version_changeset) - - @property - def rpc_version(self): - """ - Get the Transmission RPC version. Trying to deduct if the server don't have a version value. - """ - if self.protocol_version is None: - # Ugly fix for 2.20 - 2.22 reporting rpc-version 11, but having new arguments - if self.server_version and (self.server_version[0] == 2 and self.server_version[1] in [20, 21, 22]): - self.protocol_version = 12 - # Ugly fix for 2.12 reporting rpc-version 10, but having new arguments - elif self.server_version and (self.server_version[0] == 2 and self.server_version[1] == 12): - self.protocol_version = 11 - elif hasattr(self.session, 'rpc_version'): - self.protocol_version = self.session.rpc_version - elif hasattr(self.session, 'version'): - self.protocol_version = 3 - else: - self.protocol_version = 2 - return self.protocol_version - - def _rpc_version_warning(self, version): - """ - Add a warning to the log if the Transmission RPC version is lower then the provided version. - """ - if self.rpc_version < version: - LOGGER.warning('Using feature not supported by server. ' - 'RPC version for server {x}, feature introduced in {y}.'.format - (x=self.rpc_version, y=version)) - - def add_torrent(self, torrent, timeout=None, **kwargs): - """ - Add torrent to transfers list. Takes a uri to a torrent or base64 encoded torrent data in ``torrent``. - Additional arguments are: - - ===================== ===== =========== ============================================================= - Argument RPC Replaced by Description - ===================== ===== =========== ============================================================= - ``bandwidthPriority`` 8 - Priority for this transfer. - ``cookies`` 13 - One or more HTTP cookie(s). - ``download_dir`` 1 - The directory where the downloaded contents will be saved in. - ``files_unwanted`` 1 - A list of file id's that shouldn't be downloaded. - ``files_wanted`` 1 - A list of file id's that should be downloaded. - ``paused`` 1 - If True, does not start the transfer when added. - ``peer_limit`` 1 - Maximum number of peers allowed. - ``priority_high`` 1 - A list of file id's that should have high priority. - ``priority_low`` 1 - A list of file id's that should have low priority. - ``priority_normal`` 1 - A list of file id's that should have normal priority. - ===================== ===== =========== ============================================================= - - Returns a Torrent object with the fields. - """ - if torrent is None: - raise ValueError('add_torrent requires data or a URI.') - torrent_data = None - parsed_uri = urlparse(torrent) - if parsed_uri.scheme in ['ftp', 'ftps', 'http', 'https']: - # there has been some problem with T's built in torrent fetcher, - # use a python one instead - torrent_file = urlopen(torrent) - torrent_data = torrent_file.read() - torrent_data = base64.b64encode(torrent_data).decode('utf-8') - if parsed_uri.scheme in ['file']: - filepath = torrent - # uri decoded different on linux / windows ? - if len(parsed_uri.path) > 0: - filepath = parsed_uri.path - elif len(parsed_uri.netloc) > 0: - filepath = parsed_uri.netloc - torrent_file = open(filepath, 'rb') - torrent_data = torrent_file.read() - torrent_data = base64.b64encode(torrent_data).decode('utf-8') - if not torrent_data: - if torrent.endswith('.torrent') or torrent.startswith('magnet:'): - torrent_data = None - else: - might_be_base64 = False - try: - # check if this is base64 data - if PY3: - base64.b64decode(torrent.encode('utf-8')) - else: - base64.b64decode(torrent) - might_be_base64 = True - except Exception: - pass - if might_be_base64: - torrent_data = torrent - - args = {'metainfo': torrent_data} if torrent_data else {'filename': torrent} - for key, value in iteritems(kwargs): - argument = make_rpc_name(key) - (arg, val) = argument_value_convert('torrent-add', argument, value, self.rpc_version) - args[arg] = val - return list(self._request('torrent-add', args, timeout=timeout).values())[0] - - def add(self, data, timeout=None, **kwargs): - """ - - .. WARNING:: - Deprecated, please use add_torrent. - """ - args = {} - if data: - args = {'metainfo': data} - elif 'metainfo' not in kwargs and 'filename' not in kwargs: - raise ValueError('No torrent data or torrent uri.') - for key, value in iteritems(kwargs): - argument = make_rpc_name(key) - (arg, val) = argument_value_convert('torrent-add', argument, value, self.rpc_version) - args[arg] = val - warnings.warn('add has been deprecated, please use add_torrent instead.', DeprecationWarning) - return self._request('torrent-add', args, timeout=timeout) - - def add_uri(self, uri, **kwargs): - """ - - .. WARNING:: - Deprecated, please use add_torrent. - """ - if uri is None: - raise ValueError('add_uri requires a URI.') - # there has been some problem with T's built in torrent fetcher, - # use a python one instead - parsed_uri = urlparse(uri) - torrent_data = None - if parsed_uri.scheme in ['ftp', 'ftps', 'http', 'https']: - torrent_file = urlopen(uri) - torrent_data = torrent_file.read() - torrent_data = base64.b64encode(torrent_data).decode('utf-8') - if parsed_uri.scheme in ['file']: - filepath = uri - # uri decoded different on linux / windows ? - if len(parsed_uri.path) > 0: - filepath = parsed_uri.path - elif len(parsed_uri.netloc) > 0: - filepath = parsed_uri.netloc - torrent_file = open(filepath, 'rb') - torrent_data = torrent_file.read() - torrent_data = base64.b64encode(torrent_data).decode('utf-8') - warnings.warn('add_uri has been deprecated, please use add_torrent instead.', DeprecationWarning) - if torrent_data: - return self.add(torrent_data, **kwargs) - else: - return self.add(None, filename=uri, **kwargs) - - def remove_torrent(self, ids, delete_data=False, timeout=None): - """ - remove torrent(s) with provided id(s). Local data is removed if - delete_data is True, otherwise not. - """ - self._rpc_version_warning(3) - self._request('torrent-remove', - {'delete-local-data': rpc_bool(delete_data)}, ids, True, timeout=timeout) - - def remove(self, ids, delete_data=False, timeout=None): - """ - - .. WARNING:: - Deprecated, please use remove_torrent. - """ - warnings.warn('remove has been deprecated, please use remove_torrent instead.', DeprecationWarning) - self.remove_torrent(ids, delete_data, timeout) - - def start_torrent(self, ids, bypass_queue=False, timeout=None): - """Start torrent(s) with provided id(s)""" - method = 'torrent-start' - if bypass_queue and self.rpc_version >= 14: - method = 'torrent-start-now' - self._request(method, {}, ids, True, timeout=timeout) - - def start(self, ids, bypass_queue=False, timeout=None): - """ - - .. WARNING:: - Deprecated, please use start_torrent. - """ - warnings.warn('start has been deprecated, please use start_torrent instead.', DeprecationWarning) - self.start_torrent(ids, bypass_queue, timeout) - - def start_all(self, bypass_queue=False, timeout=None): - """Start all torrents respecting the queue order""" - torrent_list = self.get_torrents() - method = 'torrent-start' - if self.rpc_version >= 14: - if bypass_queue: - method = 'torrent-start-now' - torrent_list = sorted(torrent_list, key=operator.attrgetter('queuePosition')) - ids = [x.id for x in torrent_list] - self._request(method, {}, ids, True, timeout=timeout) - - def stop_torrent(self, ids, timeout=None): - """stop torrent(s) with provided id(s)""" - self._request('torrent-stop', {}, ids, True, timeout=timeout) - - def stop(self, ids, timeout=None): - """ - - .. WARNING:: - Deprecated, please use stop_torrent. - """ - warnings.warn('stop has been deprecated, please use stop_torrent instead.', DeprecationWarning) - self.stop_torrent(ids, timeout) - - def verify_torrent(self, ids, timeout=None): - """verify torrent(s) with provided id(s)""" - self._request('torrent-verify', {}, ids, True, timeout=timeout) - - def verify(self, ids, timeout=None): - """ - - .. WARNING:: - Deprecated, please use verify_torrent. - """ - warnings.warn('verify has been deprecated, please use verify_torrent instead.', DeprecationWarning) - self.verify_torrent(ids, timeout) - - def reannounce_torrent(self, ids, timeout=None): - """Reannounce torrent(s) with provided id(s)""" - self._rpc_version_warning(5) - self._request('torrent-reannounce', {}, ids, True, timeout=timeout) - - def reannounce(self, ids, timeout=None): - """ - - .. WARNING:: - Deprecated, please use reannounce_torrent. - """ - warnings.warn('reannounce has been deprecated, please use reannounce_torrent instead.', DeprecationWarning) - self.reannounce_torrent(ids, timeout) - - def get_torrent(self, torrent_id, arguments=None, timeout=None): - """ - Get information for torrent with provided id. - ``arguments`` contains a list of field names to be returned, when None - all fields are requested. See the Torrent class for more information. - - Returns a Torrent object with the requested fields. - """ - if not arguments: - arguments = self.torrent_get_arguments - torrent_id = parse_torrent_id(torrent_id) - if torrent_id is None: - raise ValueError("Invalid id") - result = self._request('torrent-get', {'fields': arguments}, torrent_id, require_ids=True, timeout=timeout) - if torrent_id in result: - return result[torrent_id] - else: - for torrent in result.values(): - if torrent.hashString == torrent_id: - return torrent - raise KeyError("Torrent not found in result") - - def get_torrents(self, ids=None, arguments=None, timeout=None): - """ - Get information for torrents with provided ids. For more information see get_torrent. - - Returns a list of Torrent object. - """ - if not arguments: - arguments = self.torrent_get_arguments - return list(self._request('torrent-get', {'fields': arguments}, ids, timeout=timeout).values()) - - def info(self, ids=None, arguments=None, timeout=None): - """ - - .. WARNING:: - Deprecated, please use get_torrent or get_torrents. Please note that the return argument has changed in - the new methods. info returns a dictionary indexed by torrent id. - """ - warnings.warn('info has been deprecated, please use get_torrent or get_torrents instead.', DeprecationWarning) - if not arguments: - arguments = self.torrent_get_arguments - return self._request('torrent-get', {'fields': arguments}, ids, timeout=timeout) - - def list(self, timeout=None): - """ - - .. WARNING:: - Deprecated, please use get_torrent or get_torrents. Please note that the return argument has changed in - the new methods. list returns a dictionary indexed by torrent id. - """ - warnings.warn('list has been deprecated, please use get_torrent or get_torrents instead.', DeprecationWarning) - fields = ['id', 'hashString', 'name', 'sizeWhenDone', 'leftUntilDone', - 'eta', 'status', 'rateUpload', 'rateDownload', 'uploadedEver', - 'downloadedEver', 'uploadRatio', 'queuePosition'] - return self._request('torrent-get', {'fields': fields}, timeout=timeout) - - def get_files(self, ids=None, timeout=None): - """ - Get list of files for provided torrent id(s). If ids is empty, - information for all torrents are fetched. This function returns a dictionary - for each requested torrent id holding the information about the files. - - :: - - { - : { - : { - 'name': , - 'size': , - 'completed': , - 'priority': , - 'selected': - } - - ... - } - - ... - } - """ - fields = ['id', 'name', 'hashString', 'files', 'priorities', 'wanted'] - request_result = self._request('torrent-get', {'fields': fields}, ids, timeout=timeout) - result = {} - for tid, torrent in iteritems(request_result): - result[tid] = torrent.files() - return result - - def set_files(self, items, timeout=None): - """ - Set file properties. Takes a dictionary with similar contents as the result - of `get_files`. - - :: - - { - : { - : { - 'priority': , - 'selected': - } - - ... - } - - ... - } - """ - if not isinstance(items, dict): - raise ValueError('Invalid file description') - for tid, files in iteritems(items): - if not isinstance(files, dict): - continue - wanted = [] - unwanted = [] - high = [] - normal = [] - low = [] - for fid, file_desc in iteritems(files): - if not isinstance(file_desc, dict): - continue - if 'selected' in file_desc and file_desc['selected']: - wanted.append(fid) - else: - unwanted.append(fid) - if 'priority' in file_desc: - if file_desc['priority'] == 'high': - high.append(fid) - elif file_desc['priority'] == 'normal': - normal.append(fid) - elif file_desc['priority'] == 'low': - low.append(fid) - args = { - 'timeout': timeout - } - if len(high) > 0: - args['priority_high'] = high - if len(normal) > 0: - args['priority_normal'] = normal - if len(low) > 0: - args['priority_low'] = low - if len(wanted) > 0: - args['files_wanted'] = wanted - if len(unwanted) > 0: - args['files_unwanted'] = unwanted - self.change_torrent([tid], **args) - - def change_torrent(self, ids, timeout=None, **kwargs): - """ - Change torrent parameters for the torrent(s) with the supplied id's. The - parameters are: - - ============================ ===== =============== ======================================================================================= - Argument RPC Replaced by Description - ============================ ===== =============== ======================================================================================= - ``bandwidthPriority`` 5 - Priority for this transfer. - ``downloadLimit`` 5 - Set the speed limit for download in Kib/s. - ``downloadLimited`` 5 - Enable download speed limiter. - ``files_unwanted`` 1 - A list of file id's that shouldn't be downloaded. - ``files_wanted`` 1 - A list of file id's that should be downloaded. - ``honorsSessionLimits`` 5 - Enables or disables the transfer to honour the upload limit set in the session. - ``location`` 1 - Local download location. - ``peer_limit`` 1 - The peer limit for the torrents. - ``priority_high`` 1 - A list of file id's that should have high priority. - ``priority_low`` 1 - A list of file id's that should have normal priority. - ``priority_normal`` 1 - A list of file id's that should have low priority. - ``queuePosition`` 14 - Position of this transfer in its queue. - ``seedIdleLimit`` 10 - Seed inactivity limit in minutes. - ``seedIdleMode`` 10 - Seed inactivity mode. 0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit. - ``seedRatioLimit`` 5 - Seeding ratio. - ``seedRatioMode`` 5 - Which ratio to use. 0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit. - ``speed_limit_down`` 1 - 5 downloadLimit Set the speed limit for download in Kib/s. - ``speed_limit_down_enabled`` 1 - 5 downloadLimited Enable download speed limiter. - ``speed_limit_up`` 1 - 5 uploadLimit Set the speed limit for upload in Kib/s. - ``speed_limit_up_enabled`` 1 - 5 uploadLimited Enable upload speed limiter. - ``trackerAdd`` 10 - Array of string with announce URLs to add. - ``trackerRemove`` 10 - Array of ids of trackers to remove. - ``trackerReplace`` 10 - Array of (id, url) tuples where the announce URL should be replaced. - ``uploadLimit`` 5 - Set the speed limit for upload in Kib/s. - ``uploadLimited`` 5 - Enable upload speed limiter. - ============================ ===== =============== ======================================================================================= - - .. NOTE:: - transmissionrpc will try to automatically fix argument errors. - """ - args = {} - for key, value in iteritems(kwargs): - argument = make_rpc_name(key) - (arg, val) = argument_value_convert('torrent-set', argument, value, self.rpc_version) - args[arg] = val - - if len(args) > 0: - self._request('torrent-set', args, ids, True, timeout=timeout) - else: - ValueError("No arguments to set") - - def change(self, ids, timeout=None, **kwargs): - """ - - .. WARNING:: - Deprecated, please use change_torrent. - """ - warnings.warn('change has been deprecated, please use change_torrent instead.', DeprecationWarning) - self.change_torrent(ids, timeout, **kwargs) - - def move_torrent_data(self, ids, location, timeout=None): - """Move torrent data to the new location.""" - self._rpc_version_warning(6) - args = {'location': location, 'move': True} - self._request('torrent-set-location', args, ids, True, timeout=timeout) - - def move(self, ids, location, timeout=None): - """ - - .. WARNING:: - Deprecated, please use move_torrent_data. - """ - warnings.warn('move has been deprecated, please use move_torrent_data instead.', DeprecationWarning) - self.move_torrent_data(ids, location, timeout) - - def locate_torrent_data(self, ids, location, timeout=None): - """Locate torrent data at the provided location.""" - self._rpc_version_warning(6) - args = {'location': location, 'move': False} - self._request('torrent-set-location', args, ids, True, timeout=timeout) - - def locate(self, ids, location, timeout=None): - """ - - .. WARNING:: - Deprecated, please use locate_torrent_data. - """ - warnings.warn('locate has been deprecated, please use locate_torrent_data instead.', DeprecationWarning) - self.locate_torrent_data(ids, location, timeout) - - def rename_torrent_path(self, torrent_id, location, name, timeout=None): - """ - Rename directory and/or files for torrent. - Remember to use get_torrent or get_torrents to update your file information. - """ - self._rpc_version_warning(15) - torrent_id = parse_torrent_id(torrent_id) - if torrent_id is None: - raise ValueError("Invalid id") - dirname = os.path.dirname(name) - if len(dirname) > 0: - raise ValueError("Target name cannot contain a path delimiter") - args = {'path': location, 'name': name} - result = self._request('torrent-rename-path', args, torrent_id, True, timeout=timeout) - return result['path'], result['name'] - - def queue_top(self, ids, timeout=None): - """Move transfer to the top of the queue.""" - self._rpc_version_warning(14) - self._request('queue-move-top', ids=ids, require_ids=True, timeout=timeout) - - def queue_bottom(self, ids, timeout=None): - """Move transfer to the bottom of the queue.""" - self._rpc_version_warning(14) - self._request('queue-move-bottom', ids=ids, require_ids=True, timeout=timeout) - - def queue_up(self, ids, timeout=None): - """Move transfer up in the queue.""" - self._rpc_version_warning(14) - self._request('queue-move-up', ids=ids, require_ids=True, timeout=timeout) - - def queue_down(self, ids, timeout=None): - """Move transfer down in the queue.""" - self._rpc_version_warning(14) - self._request('queue-move-down', ids=ids, require_ids=True, timeout=timeout) - - def get_session(self, timeout=None): - """ - Get session parameters. See the Session class for more information. - """ - self._request('session-get', timeout=timeout) - self._update_server_version() - return self.session - - def set_session(self, timeout=None, **kwargs): - """ - Set session parameters. The parameters are: - - ================================ ===== ================= ========================================================================================================================== - Argument RPC Replaced by Description - ================================ ===== ================= ========================================================================================================================== - ``alt_speed_down`` 5 - Alternate session download speed limit (in Kib/s). - ``alt_speed_enabled`` 5 - Enables alternate global download speed limiter. - ``alt_speed_time_begin`` 5 - Time when alternate speeds should be enabled. Minutes after midnight. - ``alt_speed_time_day`` 5 - Enables alternate speeds scheduling these days. - ``alt_speed_time_enabled`` 5 - Enables alternate speeds scheduling. - ``alt_speed_time_end`` 5 - Time when alternate speeds should be disabled. Minutes after midnight. - ``alt_speed_up`` 5 - Alternate session upload speed limit (in Kib/s). - ``blocklist_enabled`` 5 - Enables the block list - ``blocklist_url`` 11 - Location of the block list. Updated with blocklist-update. - ``cache_size_mb`` 10 - The maximum size of the disk cache in MB - ``dht_enabled`` 6 - Enables DHT. - ``download_dir`` 1 - Set the session download directory. - ``download_queue_enabled`` 14 - Enables download queue. - ``download_queue_size`` 14 - Number of slots in the download queue. - ``encryption`` 1 - Set the session encryption mode, one of ``required``, ``preferred`` or ``tolerated``. - ``idle_seeding_limit`` 10 - The default seed inactivity limit in minutes. - ``idle_seeding_limit_enabled`` 10 - Enables the default seed inactivity limit - ``incomplete_dir`` 7 - The path to the directory of incomplete transfer data. - ``incomplete_dir_enabled`` 7 - Enables the incomplete transfer data directory. Otherwise data for incomplete transfers are stored in the download target. - ``lpd_enabled`` 9 - Enables local peer discovery for public torrents. - ``peer_limit`` 1 - 5 peer-limit-global Maximum number of peers. - ``peer_limit_global`` 5 - Maximum number of peers. - ``peer_limit_per_torrent`` 5 - Maximum number of peers per transfer. - ``peer_port`` 5 - Peer port. - ``peer_port_random_on_start`` 5 - Enables randomized peer port on start of Transmission. - ``pex_allowed`` 1 - 5 pex-enabled Allowing PEX in public torrents. - ``pex_enabled`` 5 - Allowing PEX in public torrents. - ``port`` 1 - 5 peer-port Peer port. - ``port_forwarding_enabled`` 1 - Enables port forwarding. - ``queue_stalled_enabled`` 14 - Enable tracking of stalled transfers. - ``queue_stalled_minutes`` 14 - Number of minutes of idle that marks a transfer as stalled. - ``rename_partial_files`` 8 - Appends ".part" to incomplete files - ``script_torrent_done_enabled`` 9 - Whether or not to call the "done" script. - ``script_torrent_done_filename`` 9 - Filename of the script to run when the transfer is done. - ``seed_queue_enabled`` 14 - Enables upload queue. - ``seed_queue_size`` 14 - Number of slots in the upload queue. - ``seedRatioLimit`` 5 - Seed ratio limit. 1.0 means 1:1 download and upload ratio. - ``seedRatioLimited`` 5 - Enables seed ration limit. - ``speed_limit_down`` 1 - Download speed limit (in Kib/s). - ``speed_limit_down_enabled`` 1 - Enables download speed limiting. - ``speed_limit_up`` 1 - Upload speed limit (in Kib/s). - ``speed_limit_up_enabled`` 1 - Enables upload speed limiting. - ``start_added_torrents`` 9 - Added torrents will be started right away. - ``trash_original_torrent_files`` 9 - The .torrent file of added torrents will be deleted. - ``utp_enabled`` 13 - Enables Micro Transport Protocol (UTP). - ================================ ===== ================= ========================================================================================================================== - - .. NOTE:: - transmissionrpc will try to automatically fix argument errors. - """ - args = {} - for key, value in iteritems(kwargs): - if key == 'encryption' and value not in ['required', 'preferred', 'tolerated']: - raise ValueError('Invalid encryption value') - argument = make_rpc_name(key) - (arg, val) = argument_value_convert('session-set', argument, value, self.rpc_version) - args[arg] = val - if len(args) > 0: - self._request('session-set', args, timeout=timeout) - - def blocklist_update(self, timeout=None): - """Update block list. Returns the size of the block list.""" - self._rpc_version_warning(5) - result = self._request('blocklist-update', timeout=timeout) - if 'blocklist-size' in result: - return result['blocklist-size'] - return None - - def port_test(self, timeout=None): - """ - Tests to see if your incoming peer port is accessible from the - outside world. - """ - self._rpc_version_warning(5) - result = self._request('port-test', timeout=timeout) - if 'port-is-open' in result: - return result['port-is-open'] - return None - - def free_space(self, path, timeout=None): - """ - Get the ammount of free space (in bytes) at the provided location. - """ - self._rpc_version_warning(15) - result = self._request('free-space', {'path': path}, timeout=timeout) - if result['path'] == path: - return result['size-bytes'] - return None - - def session_stats(self, timeout=None): - """Get session statistics""" - self._request('session-stats', timeout=timeout) - return self.session diff --git a/core/transmissionrpc/constants.py b/core/transmissionrpc/constants.py deleted file mode 100644 index 78e61dd5f..000000000 --- a/core/transmissionrpc/constants.py +++ /dev/null @@ -1,328 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2008-2013 Erik Svensson -# Licensed under the MIT license. - -import logging - -from core.transmissionrpc.six import iteritems - -LOGGER = logging.getLogger('transmissionrpc') -LOGGER.setLevel(logging.ERROR) - - -def mirror_dict(source): - """ - Creates a dictionary with all values as keys and all keys as values. - """ - source.update(dict((value, key) for key, value in iteritems(source))) - return source - - -DEFAULT_PORT = 9091 - -DEFAULT_TIMEOUT = 30.0 - -TR_PRI_LOW = -1 -TR_PRI_NORMAL = 0 -TR_PRI_HIGH = 1 - -PRIORITY = mirror_dict({ - 'low': TR_PRI_LOW, - 'normal': TR_PRI_NORMAL, - 'high': TR_PRI_HIGH -}) - -TR_RATIOLIMIT_GLOBAL = 0 # follow the global settings -TR_RATIOLIMIT_SINGLE = 1 # override the global settings, seeding until a certain ratio -TR_RATIOLIMIT_UNLIMITED = 2 # override the global settings, seeding regardless of ratio - -RATIO_LIMIT = mirror_dict({ - 'global': TR_RATIOLIMIT_GLOBAL, - 'single': TR_RATIOLIMIT_SINGLE, - 'unlimited': TR_RATIOLIMIT_UNLIMITED -}) - -TR_IDLELIMIT_GLOBAL = 0 # follow the global settings -TR_IDLELIMIT_SINGLE = 1 # override the global settings, seeding until a certain idle time -TR_IDLELIMIT_UNLIMITED = 2 # override the global settings, seeding regardless of activity - -IDLE_LIMIT = mirror_dict({ - 'global': TR_RATIOLIMIT_GLOBAL, - 'single': TR_RATIOLIMIT_SINGLE, - 'unlimited': TR_RATIOLIMIT_UNLIMITED -}) - -# A note on argument maps -# These maps are used to verify *-set methods. The information is structured in -# a tree. -# set +- - [, , , , , ] -# | +- - [, , , , , ] -# | -# get +- - [, , , , , ] -# +- - [, , , , , ] - -# Arguments for torrent methods -TORRENT_ARGS = { - 'get': { - 'activityDate': ('number', 1, None, None, None, 'Last time of upload or download activity.'), - 'addedDate': ('number', 1, None, None, None, 'The date when this torrent was first added.'), - 'announceResponse': ('string', 1, 7, None, None, 'The announce message from the tracker.'), - 'announceURL': ('string', 1, 7, None, None, 'Current announce URL.'), - 'bandwidthPriority': ('number', 5, None, None, None, 'Bandwidth priority. Low (-1), Normal (0) or High (1).'), - 'comment': ('string', 1, None, None, None, 'Torrent comment.'), - 'corruptEver': ('number', 1, None, None, None, 'Number of bytes of corrupt data downloaded.'), - 'creator': ('string', 1, None, None, None, 'Torrent creator.'), - 'dateCreated': ('number', 1, None, None, None, 'Torrent creation date.'), - 'desiredAvailable': ('number', 1, None, None, None, 'Number of bytes avalable and left to be downloaded.'), - 'doneDate': ('number', 1, None, None, None, 'The date when the torrent finished downloading.'), - 'downloadDir': ('string', 4, None, None, None, 'The directory path where the torrent is downloaded to.'), - 'downloadedEver': ('number', 1, None, None, None, 'Number of bytes of good data downloaded.'), - 'downloaders': ('number', 4, 7, None, None, 'Number of downloaders.'), - 'downloadLimit': ('number', 1, None, None, None, 'Download limit in Kbps.'), - 'downloadLimited': ('boolean', 5, None, None, None, 'Download limit is enabled'), - 'downloadLimitMode': ( - 'number', 1, 5, None, None, 'Download limit mode. 0 means global, 1 means signle, 2 unlimited.'), - 'error': ('number', 1, None, None, None, - 'Kind of error. 0 means OK, 1 means tracker warning, 2 means tracker error, 3 means local error.'), - 'errorString': ('number', 1, None, None, None, 'Error message.'), - 'eta': ('number', 1, None, None, None, - 'Estimated number of seconds left when downloading or seeding. -1 means not available and -2 means unknown.'), - 'etaIdle': ('number', 15, None, None, None, - 'Estimated number of seconds left until the idle time limit is reached. -1 means not available and -2 means unknown.'), - 'files': ( - 'array', 1, None, None, None, 'Array of file object containing key, bytesCompleted, length and name.'), - 'fileStats': ( - 'array', 5, None, None, None, 'Aray of file statistics containing bytesCompleted, wanted and priority.'), - 'hashString': ('string', 1, None, None, None, 'Hashstring unique for the torrent even between sessions.'), - 'haveUnchecked': ('number', 1, None, None, None, 'Number of bytes of partial pieces.'), - 'haveValid': ('number', 1, None, None, None, 'Number of bytes of checksum verified data.'), - 'honorsSessionLimits': ('boolean', 5, None, None, None, 'True if session upload limits are honored'), - 'id': ('number', 1, None, None, None, 'Session unique torrent id.'), - 'isFinished': ('boolean', 9, None, None, None, 'True if the torrent is finished. Downloaded and seeded.'), - 'isPrivate': ('boolean', 1, None, None, None, 'True if the torrent is private.'), - 'isStalled': ('boolean', 14, None, None, None, 'True if the torrent has stalled (been idle for a long time).'), - 'lastAnnounceTime': ('number', 1, 7, None, None, 'The time of the last announcement.'), - 'lastScrapeTime': ('number', 1, 7, None, None, 'The time af the last successful scrape.'), - 'leechers': ('number', 1, 7, None, None, 'Number of leechers.'), - 'leftUntilDone': ('number', 1, None, None, None, 'Number of bytes left until the download is done.'), - 'magnetLink': ('string', 7, None, None, None, 'The magnet link for this torrent.'), - 'manualAnnounceTime': ('number', 1, None, None, None, 'The time until you manually ask for more peers.'), - 'maxConnectedPeers': ('number', 1, None, None, None, 'Maximum of connected peers.'), - 'metadataPercentComplete': ('number', 7, None, None, None, 'Download progress of metadata. 0.0 to 1.0.'), - 'name': ('string', 1, None, None, None, 'Torrent name.'), - 'nextAnnounceTime': ('number', 1, 7, None, None, 'Next announce time.'), - 'nextScrapeTime': ('number', 1, 7, None, None, 'Next scrape time.'), - 'peer-limit': ('number', 5, None, None, None, 'Maximum number of peers.'), - 'peers': ('array', 2, None, None, None, 'Array of peer objects.'), - 'peersConnected': ('number', 1, None, None, None, 'Number of peers we are connected to.'), - 'peersFrom': ( - 'object', 1, None, None, None, 'Object containing download peers counts for different peer types.'), - 'peersGettingFromUs': ('number', 1, None, None, None, 'Number of peers we are sending data to.'), - 'peersKnown': ('number', 1, 13, None, None, 'Number of peers that the tracker knows.'), - 'peersSendingToUs': ('number', 1, None, None, None, 'Number of peers sending to us'), - 'percentDone': ('double', 5, None, None, None, 'Download progress of selected files. 0.0 to 1.0.'), - 'pieces': ('string', 5, None, None, None, 'String with base64 encoded bitfield indicating finished pieces.'), - 'pieceCount': ('number', 1, None, None, None, 'Number of pieces.'), - 'pieceSize': ('number', 1, None, None, None, 'Number of bytes in a piece.'), - 'priorities': ('array', 1, None, None, None, 'Array of file priorities.'), - 'queuePosition': ('number', 14, None, None, None, 'The queue position.'), - 'rateDownload': ('number', 1, None, None, None, 'Download rate in bps.'), - 'rateUpload': ('number', 1, None, None, None, 'Upload rate in bps.'), - 'recheckProgress': ('double', 1, None, None, None, 'Progress of recheck. 0.0 to 1.0.'), - 'secondsDownloading': ('number', 15, None, None, None, ''), - 'secondsSeeding': ('number', 15, None, None, None, ''), - 'scrapeResponse': ('string', 1, 7, None, None, 'Scrape response message.'), - 'scrapeURL': ('string', 1, 7, None, None, 'Current scrape URL'), - 'seeders': ('number', 1, 7, None, None, 'Number of seeders reported by the tracker.'), - 'seedIdleLimit': ('number', 10, None, None, None, 'Idle limit in minutes.'), - 'seedIdleMode': ('number', 10, None, None, None, 'Use global (0), torrent (1), or unlimited (2) limit.'), - 'seedRatioLimit': ('double', 5, None, None, None, 'Seed ratio limit.'), - 'seedRatioMode': ('number', 5, None, None, None, 'Use global (0), torrent (1), or unlimited (2) limit.'), - 'sizeWhenDone': ('number', 1, None, None, None, 'Size of the torrent download in bytes.'), - 'startDate': ('number', 1, None, None, None, 'The date when the torrent was last started.'), - 'status': ('number', 1, None, None, None, 'Current status, see source'), - 'swarmSpeed': ('number', 1, 7, None, None, 'Estimated speed in Kbps in the swarm.'), - 'timesCompleted': ('number', 1, 7, None, None, 'Number of successful downloads reported by the tracker.'), - 'trackers': ('array', 1, None, None, None, 'Array of tracker objects.'), - 'trackerStats': ('object', 7, None, None, None, 'Array of object containing tracker statistics.'), - 'totalSize': ('number', 1, None, None, None, 'Total size of the torrent in bytes'), - 'torrentFile': ('string', 5, None, None, None, 'Path to .torrent file.'), - 'uploadedEver': ('number', 1, None, None, None, 'Number of bytes uploaded, ever.'), - 'uploadLimit': ('number', 1, None, None, None, 'Upload limit in Kbps'), - 'uploadLimitMode': ( - 'number', 1, 5, None, None, 'Upload limit mode. 0 means global, 1 means signle, 2 unlimited.'), - 'uploadLimited': ('boolean', 5, None, None, None, 'Upload limit enabled.'), - 'uploadRatio': ('double', 1, None, None, None, 'Seed ratio.'), - 'wanted': ('array', 1, None, None, None, 'Array of booleans indicated wanted files.'), - 'webseeds': ('array', 1, None, None, None, 'Array of webseeds objects'), - 'webseedsSendingToUs': ('number', 1, None, None, None, 'Number of webseeds seeding to us.'), - }, - 'set': { - 'bandwidthPriority': ('number', 5, None, None, None, 'Priority for this transfer.'), - 'downloadLimit': ('number', 5, None, 'speed-limit-down', None, 'Set the speed limit for download in Kib/s.'), - 'downloadLimited': ('boolean', 5, None, 'speed-limit-down-enabled', None, 'Enable download speed limiter.'), - 'files-wanted': ('array', 1, None, None, None, "A list of file id's that should be downloaded."), - 'files-unwanted': ('array', 1, None, None, None, "A list of file id's that shouldn't be downloaded."), - 'honorsSessionLimits': ('boolean', 5, None, None, None, - "Enables or disables the transfer to honour the upload limit set in the session."), - 'location': ('array', 1, None, None, None, 'Local download location.'), - 'peer-limit': ('number', 1, None, None, None, 'The peer limit for the torrents.'), - 'priority-high': ('array', 1, None, None, None, "A list of file id's that should have high priority."), - 'priority-low': ('array', 1, None, None, None, "A list of file id's that should have normal priority."), - 'priority-normal': ('array', 1, None, None, None, "A list of file id's that should have low priority."), - 'queuePosition': ('number', 14, None, None, None, 'Position of this transfer in its queue.'), - 'seedIdleLimit': ('number', 10, None, None, None, 'Seed inactivity limit in minutes.'), - 'seedIdleMode': ('number', 10, None, None, None, - 'Seed inactivity mode. 0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.'), - 'seedRatioLimit': ('double', 5, None, None, None, 'Seeding ratio.'), - 'seedRatioMode': ('number', 5, None, None, None, - 'Which ratio to use. 0 = Use session limit, 1 = Use transfer limit, 2 = Disable limit.'), - 'speed-limit-down': ('number', 1, 5, None, 'downloadLimit', 'Set the speed limit for download in Kib/s.'), - 'speed-limit-down-enabled': ('boolean', 1, 5, None, 'downloadLimited', 'Enable download speed limiter.'), - 'speed-limit-up': ('number', 1, 5, None, 'uploadLimit', 'Set the speed limit for upload in Kib/s.'), - 'speed-limit-up-enabled': ('boolean', 1, 5, None, 'uploadLimited', 'Enable upload speed limiter.'), - 'trackerAdd': ('array', 10, None, None, None, 'Array of string with announce URLs to add.'), - 'trackerRemove': ('array', 10, None, None, None, 'Array of ids of trackers to remove.'), - 'trackerReplace': ( - 'array', 10, None, None, None, 'Array of (id, url) tuples where the announce URL should be replaced.'), - 'uploadLimit': ('number', 5, None, 'speed-limit-up', None, 'Set the speed limit for upload in Kib/s.'), - 'uploadLimited': ('boolean', 5, None, 'speed-limit-up-enabled', None, 'Enable upload speed limiter.'), - }, - 'add': { - 'bandwidthPriority': ('number', 8, None, None, None, 'Priority for this transfer.'), - 'download-dir': ( - 'string', 1, None, None, None, 'The directory where the downloaded contents will be saved in.'), - 'cookies': ('string', 13, None, None, None, 'One or more HTTP cookie(s).'), - 'filename': ('string', 1, None, None, None, "A file path or URL to a torrent file or a magnet link."), - 'files-wanted': ('array', 1, None, None, None, "A list of file id's that should be downloaded."), - 'files-unwanted': ('array', 1, None, None, None, "A list of file id's that shouldn't be downloaded."), - 'metainfo': ('string', 1, None, None, None, 'The content of a torrent file, base64 encoded.'), - 'paused': ('boolean', 1, None, None, None, 'If True, does not start the transfer when added.'), - 'peer-limit': ('number', 1, None, None, None, 'Maximum number of peers allowed.'), - 'priority-high': ('array', 1, None, None, None, "A list of file id's that should have high priority."), - 'priority-low': ('array', 1, None, None, None, "A list of file id's that should have low priority."), - 'priority-normal': ('array', 1, None, None, None, "A list of file id's that should have normal priority."), - } -} - -# Arguments for session methods -SESSION_ARGS = { - 'get': { - "alt-speed-down": ('number', 5, None, None, None, 'Alternate session download speed limit (in Kib/s).'), - "alt-speed-enabled": ( - 'boolean', 5, None, None, None, 'True if alternate global download speed limiter is ebabled.'), - "alt-speed-time-begin": ( - 'number', 5, None, None, None, 'Time when alternate speeds should be enabled. Minutes after midnight.'), - "alt-speed-time-enabled": ('boolean', 5, None, None, None, 'True if alternate speeds scheduling is enabled.'), - "alt-speed-time-end": ( - 'number', 5, None, None, None, 'Time when alternate speeds should be disabled. Minutes after midnight.'), - "alt-speed-time-day": ('number', 5, None, None, None, 'Days alternate speeds scheduling is enabled.'), - "alt-speed-up": ('number', 5, None, None, None, 'Alternate session upload speed limit (in Kib/s)'), - "blocklist-enabled": ('boolean', 5, None, None, None, 'True when blocklist is enabled.'), - "blocklist-size": ('number', 5, None, None, None, 'Number of rules in the blocklist'), - "blocklist-url": ('string', 11, None, None, None, 'Location of the block list. Updated with blocklist-update.'), - "cache-size-mb": ('number', 10, None, None, None, 'The maximum size of the disk cache in MB'), - "config-dir": ('string', 8, None, None, None, 'location of transmissions configuration directory'), - "dht-enabled": ('boolean', 6, None, None, None, 'True if DHT enabled.'), - "download-dir": ('string', 1, None, None, None, 'The download directory.'), - "download-dir-free-space": ('number', 12, None, None, None, 'Free space in the download directory, in bytes'), - "download-queue-size": ('number', 14, None, None, None, 'Number of slots in the download queue.'), - "download-queue-enabled": ('boolean', 14, None, None, None, 'True if the download queue is enabled.'), - "encryption": ( - 'string', 1, None, None, None, 'Encryption mode, one of ``required``, ``preferred`` or ``tolerated``.'), - "idle-seeding-limit": ('number', 10, None, None, None, 'Seed inactivity limit in minutes.'), - "idle-seeding-limit-enabled": ('boolean', 10, None, None, None, 'True if the seed activity limit is enabled.'), - "incomplete-dir": ( - 'string', 7, None, None, None, 'The path to the directory for incomplete torrent transfer data.'), - "incomplete-dir-enabled": ('boolean', 7, None, None, None, 'True if the incomplete dir is enabled.'), - "lpd-enabled": ('boolean', 9, None, None, None, 'True if local peer discovery is enabled.'), - "peer-limit": ('number', 1, 5, None, 'peer-limit-global', 'Maximum number of peers.'), - "peer-limit-global": ('number', 5, None, 'peer-limit', None, 'Maximum number of peers.'), - "peer-limit-per-torrent": ('number', 5, None, None, None, 'Maximum number of peers per transfer.'), - "pex-allowed": ('boolean', 1, 5, None, 'pex-enabled', 'True if PEX is allowed.'), - "pex-enabled": ('boolean', 5, None, 'pex-allowed', None, 'True if PEX is enabled.'), - "port": ('number', 1, 5, None, 'peer-port', 'Peer port.'), - "peer-port": ('number', 5, None, 'port', None, 'Peer port.'), - "peer-port-random-on-start": ( - 'boolean', 5, None, None, None, 'Enables randomized peer port on start of Transmission.'), - "port-forwarding-enabled": ('boolean', 1, None, None, None, 'True if port forwarding is enabled.'), - "queue-stalled-minutes": ( - 'number', 14, None, None, None, 'Number of minutes of idle that marks a transfer as stalled.'), - "queue-stalled-enabled": ('boolean', 14, None, None, None, 'True if stalled tracking of transfers is enabled.'), - "rename-partial-files": ('boolean', 8, None, None, None, 'True if ".part" is appended to incomplete files'), - "rpc-version": ('number', 4, None, None, None, 'Transmission RPC API Version.'), - "rpc-version-minimum": ('number', 4, None, None, None, 'Minimum accepted RPC API Version.'), - "script-torrent-done-enabled": ('boolean', 9, None, None, None, 'True if the done script is enabled.'), - "script-torrent-done-filename": ( - 'string', 9, None, None, None, 'Filename of the script to run when the transfer is done.'), - "seedRatioLimit": ('double', 5, None, None, None, 'Seed ratio limit. 1.0 means 1:1 download and upload ratio.'), - "seedRatioLimited": ('boolean', 5, None, None, None, 'True if seed ration limit is enabled.'), - "seed-queue-size": ('number', 14, None, None, None, 'Number of slots in the upload queue.'), - "seed-queue-enabled": ('boolean', 14, None, None, None, 'True if upload queue is enabled.'), - "speed-limit-down": ('number', 1, None, None, None, 'Download speed limit (in Kib/s).'), - "speed-limit-down-enabled": ('boolean', 1, None, None, None, 'True if the download speed is limited.'), - "speed-limit-up": ('number', 1, None, None, None, 'Upload speed limit (in Kib/s).'), - "speed-limit-up-enabled": ('boolean', 1, None, None, None, 'True if the upload speed is limited.'), - "start-added-torrents": ('boolean', 9, None, None, None, 'When true uploaded torrents will start right away.'), - "trash-original-torrent-files": ( - 'boolean', 9, None, None, None, 'When true added .torrent files will be deleted.'), - 'units': ('object', 10, None, None, None, 'An object containing units for size and speed.'), - 'utp-enabled': ('boolean', 13, None, None, None, 'True if Micro Transport Protocol (UTP) is enabled.'), - "version": ('string', 3, None, None, None, 'Transmission version.'), - }, - 'set': { - "alt-speed-down": ('number', 5, None, None, None, 'Alternate session download speed limit (in Kib/s).'), - "alt-speed-enabled": ('boolean', 5, None, None, None, 'Enables alternate global download speed limiter.'), - "alt-speed-time-begin": ( - 'number', 5, None, None, None, 'Time when alternate speeds should be enabled. Minutes after midnight.'), - "alt-speed-time-enabled": ('boolean', 5, None, None, None, 'Enables alternate speeds scheduling.'), - "alt-speed-time-end": ( - 'number', 5, None, None, None, 'Time when alternate speeds should be disabled. Minutes after midnight.'), - "alt-speed-time-day": ('number', 5, None, None, None, 'Enables alternate speeds scheduling these days.'), - "alt-speed-up": ('number', 5, None, None, None, 'Alternate session upload speed limit (in Kib/s).'), - "blocklist-enabled": ('boolean', 5, None, None, None, 'Enables the block list'), - "blocklist-url": ('string', 11, None, None, None, 'Location of the block list. Updated with blocklist-update.'), - "cache-size-mb": ('number', 10, None, None, None, 'The maximum size of the disk cache in MB'), - "dht-enabled": ('boolean', 6, None, None, None, 'Enables DHT.'), - "download-dir": ('string', 1, None, None, None, 'Set the session download directory.'), - "download-queue-size": ('number', 14, None, None, None, 'Number of slots in the download queue.'), - "download-queue-enabled": ('boolean', 14, None, None, None, 'Enables download queue.'), - "encryption": ('string', 1, None, None, None, - 'Set the session encryption mode, one of ``required``, ``preferred`` or ``tolerated``.'), - "idle-seeding-limit": ('number', 10, None, None, None, 'The default seed inactivity limit in minutes.'), - "idle-seeding-limit-enabled": ('boolean', 10, None, None, None, 'Enables the default seed inactivity limit'), - "incomplete-dir": ('string', 7, None, None, None, 'The path to the directory of incomplete transfer data.'), - "incomplete-dir-enabled": ('boolean', 7, None, None, None, - 'Enables the incomplete transfer data directory. Otherwise data for incomplete transfers are stored in the download target.'), - "lpd-enabled": ('boolean', 9, None, None, None, 'Enables local peer discovery for public torrents.'), - "peer-limit": ('number', 1, 5, None, 'peer-limit-global', 'Maximum number of peers.'), - "peer-limit-global": ('number', 5, None, 'peer-limit', None, 'Maximum number of peers.'), - "peer-limit-per-torrent": ('number', 5, None, None, None, 'Maximum number of peers per transfer.'), - "pex-allowed": ('boolean', 1, 5, None, 'pex-enabled', 'Allowing PEX in public torrents.'), - "pex-enabled": ('boolean', 5, None, 'pex-allowed', None, 'Allowing PEX in public torrents.'), - "port": ('number', 1, 5, None, 'peer-port', 'Peer port.'), - "peer-port": ('number', 5, None, 'port', None, 'Peer port.'), - "peer-port-random-on-start": ( - 'boolean', 5, None, None, None, 'Enables randomized peer port on start of Transmission.'), - "port-forwarding-enabled": ('boolean', 1, None, None, None, 'Enables port forwarding.'), - "rename-partial-files": ('boolean', 8, None, None, None, 'Appends ".part" to incomplete files'), - "queue-stalled-minutes": ( - 'number', 14, None, None, None, 'Number of minutes of idle that marks a transfer as stalled.'), - "queue-stalled-enabled": ('boolean', 14, None, None, None, 'Enable tracking of stalled transfers.'), - "script-torrent-done-enabled": ('boolean', 9, None, None, None, 'Whether or not to call the "done" script.'), - "script-torrent-done-filename": ( - 'string', 9, None, None, None, 'Filename of the script to run when the transfer is done.'), - "seed-queue-size": ('number', 14, None, None, None, 'Number of slots in the upload queue.'), - "seed-queue-enabled": ('boolean', 14, None, None, None, 'Enables upload queue.'), - "seedRatioLimit": ('double', 5, None, None, None, 'Seed ratio limit. 1.0 means 1:1 download and upload ratio.'), - "seedRatioLimited": ('boolean', 5, None, None, None, 'Enables seed ration limit.'), - "speed-limit-down": ('number', 1, None, None, None, 'Download speed limit (in Kib/s).'), - "speed-limit-down-enabled": ('boolean', 1, None, None, None, 'Enables download speed limiting.'), - "speed-limit-up": ('number', 1, None, None, None, 'Upload speed limit (in Kib/s).'), - "speed-limit-up-enabled": ('boolean', 1, None, None, None, 'Enables upload speed limiting.'), - "start-added-torrents": ('boolean', 9, None, None, None, 'Added torrents will be started right away.'), - "trash-original-torrent-files": ( - 'boolean', 9, None, None, None, 'The .torrent file of added torrents will be deleted.'), - 'utp-enabled': ('boolean', 13, None, None, None, 'Enables Micro Transport Protocol (UTP).'), - }, -} diff --git a/core/transmissionrpc/error.py b/core/transmissionrpc/error.py deleted file mode 100644 index ecd6bf11c..000000000 --- a/core/transmissionrpc/error.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2008-2013 Erik Svensson -# Licensed under the MIT license. - -from core.transmissionrpc.six import string_types, integer_types - - -class TransmissionError(Exception): - """ - This exception is raised when there has occurred an error related to - communication with Transmission. It is a subclass of Exception. - """ - - def __init__(self, message='', original=None): - Exception.__init__(self) - self.message = message - self.original = original - - def __str__(self): - if self.original: - original_name = type(self.original).__name__ - return '{0} Original exception: {1}, "{2}"'.format(self.message, original_name, str(self.original)) - else: - return self.message - - -class HTTPHandlerError(Exception): - """ - This exception is raised when there has occurred an error related to - the HTTP handler. It is a subclass of Exception. - """ - - def __init__(self, httpurl=None, httpcode=None, httpmsg=None, httpheaders=None, httpdata=None): - Exception.__init__(self) - self.url = '' - self.code = 600 - self.message = '' - self.headers = {} - self.data = '' - if isinstance(httpurl, string_types): - self.url = httpurl - if isinstance(httpcode, integer_types): - self.code = httpcode - if isinstance(httpmsg, string_types): - self.message = httpmsg - if isinstance(httpheaders, dict): - self.headers = httpheaders - if isinstance(httpdata, string_types): - self.data = httpdata - - def __repr__(self): - return ''.format(self.code, self.message) - - def __str__(self): - return 'HTTPHandlerError {0:d}: {1}'.format(self.code, self.message) - - def __unicode__(self): - return 'HTTPHandlerError {0:d}: {1}'.format(self.code, self.message) diff --git a/core/transmissionrpc/utils.py b/core/transmissionrpc/utils.py deleted file mode 100644 index cff602eea..000000000 --- a/core/transmissionrpc/utils.py +++ /dev/null @@ -1,222 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2008-2013 Erik Svensson -# Licensed under the MIT license. - -import constants -import datetime -import logging -import socket -from collections import namedtuple - -from constants import LOGGER -from six import string_types, iteritems - -UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB'] - - -def format_size(size): - """ - Format byte size into IEC prefixes, B, KiB, MiB ... - """ - size = float(size) - i = 0 - while size >= 1024.0 and i < len(UNITS): - i += 1 - size /= 1024.0 - return size, UNITS[i] - - -def format_speed(size): - """ - Format bytes per second speed into IEC prefixes, B/s, KiB/s, MiB/s ... - """ - (size, unit) = format_size(size) - return size, '{unit}/s'.format(unit=unit) - - -def format_timedelta(delta): - """ - Format datetime.timedelta into ::. - """ - minutes, seconds = divmod(delta.seconds, 60) - hours, minutes = divmod(minutes, 60) - return '{0:d} {1:02d}:{2:02d}:{3:02d}'.format(delta.days, hours, minutes, seconds) - - -def format_timestamp(timestamp, utc=False): - """ - Format unix timestamp into ISO date format. - """ - if timestamp > 0: - if utc: - dt_timestamp = datetime.datetime.utcfromtimestamp(timestamp) - else: - dt_timestamp = datetime.datetime.fromtimestamp(timestamp) - return dt_timestamp.isoformat(' ') - else: - return '-' - - -class INetAddressError(Exception): - """ - Error parsing / generating a internet address. - """ - pass - - -def inet_address(address, default_port, default_address='localhost'): - """ - Parse internet address. - """ - addr = address.split(':') - if len(addr) == 1: - try: - port = int(addr[0]) - addr = default_address - except ValueError: - addr = addr[0] - port = default_port - elif len(addr) == 2: - try: - port = int(addr[1]) - except ValueError: - raise INetAddressError('Invalid address "{0}".'.format(address)) - if len(addr[0]) == 0: - addr = default_address - else: - addr = addr[0] - else: - raise INetAddressError('Invalid address "{0}".'.format(address)) - try: - socket.getaddrinfo(addr, port, socket.AF_INET, socket.SOCK_STREAM) - except socket.gaierror: - raise INetAddressError('Cannot look up address "{0}".'.format(address)) - return addr, port - - -def rpc_bool(arg): - """ - Convert between Python boolean and Transmission RPC boolean. - """ - if isinstance(arg, string_types): - try: - arg = bool(int(arg)) - except ValueError: - arg = arg.lower() in ['true', 'yes'] - return 1 if bool(arg) else 0 - - -TR_TYPE_MAP = { - 'number': int, - 'string': str, - 'double': float, - 'boolean': rpc_bool, - 'array': list, - 'object': dict -} - - -def make_python_name(name): - """ - Convert Transmission RPC name to python compatible name. - """ - return name.replace('-', '_') - - -def make_rpc_name(name): - """ - Convert python compatible name to Transmission RPC name. - """ - return name.replace('_', '-') - - -def argument_value_convert(method, argument, value, rpc_version): - """ - Check and fix Transmission RPC issues with regards to methods, arguments and values. - """ - if method in ('torrent-add', 'torrent-get', 'torrent-set'): - args = constants.TORRENT_ARGS[method[-3:]] - elif method in ('session-get', 'session-set'): - args = constants.SESSION_ARGS[method[-3:]] - else: - return ValueError('Method "{0}" not supported'.format(method)) - if argument in args: - info = args[argument] - invalid_version = True - while invalid_version: - invalid_version = False - replacement = None - if rpc_version < info[1]: - invalid_version = True - replacement = info[3] - if info[2] and info[2] <= rpc_version: - invalid_version = True - replacement = info[4] - if invalid_version: - if replacement: - LOGGER.warning( - 'Replacing requested argument "{0}" with "{1}".'.format(argument, replacement)) - argument = replacement - info = args[argument] - else: - raise ValueError( - 'Method "{0}" Argument "{1}" does not exist in version {2:d}.'.format(method, argument, rpc_version)) - return argument, TR_TYPE_MAP[info[0]](value) - else: - raise ValueError('Argument "%s" does not exists for method "%s".', - (argument, method)) - - -def get_arguments(method, rpc_version): - """ - Get arguments for method in specified Transmission RPC version. - """ - if method in ('torrent-add', 'torrent-get', 'torrent-set'): - args = constants.TORRENT_ARGS[method[-3:]] - elif method in ('session-get', 'session-set'): - args = constants.SESSION_ARGS[method[-3:]] - else: - return ValueError('Method "{0}" not supported'.format(method)) - accessible = [] - for argument, info in iteritems(args): - valid_version = True - if rpc_version < info[1]: - valid_version = False - if info[2] and info[2] <= rpc_version: - valid_version = False - if valid_version: - accessible.append(argument) - return accessible - - -def add_stdout_logger(level='debug'): - """ - Add a stdout target for the transmissionrpc logging. - """ - levels = {'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR} - - trpc_logger = logging.getLogger('transmissionrpc') - loghandler = logging.StreamHandler() - if level in list(levels.keys()): - loglevel = levels[level] - trpc_logger.setLevel(loglevel) - loghandler.setLevel(loglevel) - trpc_logger.addHandler(loghandler) - - -def add_file_logger(filepath, level='debug'): - """ - Add a stdout target for the transmissionrpc logging. - """ - levels = {'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR} - - trpc_logger = logging.getLogger('transmissionrpc') - loghandler = logging.FileHandler(filepath, encoding='utf-8') - if level in list(levels.keys()): - loglevel = levels[level] - trpc_logger.setLevel(loglevel) - loghandler.setLevel(loglevel) - trpc_logger.addHandler(loghandler) - - -Field = namedtuple('Field', ['value', 'dirty']) diff --git a/core/user_scripts.py b/core/user_scripts.py new file mode 100644 index 000000000..83f17d606 --- /dev/null +++ b/core/user_scripts.py @@ -0,0 +1,117 @@ +# coding=utf-8 + +import os +from subprocess import Popen + +import core +from core import logger, transcoder +from core.utils import import_subs, list_media_files, remove_dir + + +def external_script(output_destination, torrent_name, torrent_label, settings): + final_result = 0 # start at 0. + num_files = 0 + try: + core.USER_SCRIPT_MEDIAEXTENSIONS = settings['user_script_mediaExtensions'].lower() + if isinstance(core.USER_SCRIPT_MEDIAEXTENSIONS, str): + core.USER_SCRIPT_MEDIAEXTENSIONS = core.USER_SCRIPT_MEDIAEXTENSIONS.split(',') + except Exception: + core.USER_SCRIPT_MEDIAEXTENSIONS = [] + + core.USER_SCRIPT = settings.get('user_script_path') + + if not core.USER_SCRIPT or core.USER_SCRIPT == 'None': # do nothing and return success. + return [0, ''] + try: + core.USER_SCRIPT_PARAM = settings['user_script_param'] + if isinstance(core.USER_SCRIPT_PARAM, str): + core.USER_SCRIPT_PARAM = core.USER_SCRIPT_PARAM.split(',') + except Exception: + core.USER_SCRIPT_PARAM = [] + try: + core.USER_SCRIPT_SUCCESSCODES = settings['user_script_successCodes'] + if isinstance(core.USER_SCRIPT_SUCCESSCODES, str): + core.USER_SCRIPT_SUCCESSCODES = core.USER_SCRIPT_SUCCESSCODES.split(',') + except Exception: + core.USER_SCRIPT_SUCCESSCODES = 0 + + core.USER_SCRIPT_CLEAN = int(settings.get('user_script_clean', 1)) + core.USER_SCRIPT_RUNONCE = int(settings.get('user_script_runOnce', 1)) + + if core.CHECK_MEDIA: + for video in list_media_files(output_destination, media=True, audio=False, meta=False, archives=False): + if transcoder.is_video_good(video, 0): + import_subs(video) + else: + logger.info('Corrupt video file found {0}. Deleting.'.format(video), 'USERSCRIPT') + os.unlink(video) + + for dirpath, dirnames, filenames in os.walk(output_destination): + for file in filenames: + + file_path = core.os.path.join(dirpath, file) + file_name, file_extension = os.path.splitext(file) + + if file_extension in core.USER_SCRIPT_MEDIAEXTENSIONS or 'all' in core.USER_SCRIPT_MEDIAEXTENSIONS: + num_files += 1 + if core.USER_SCRIPT_RUNONCE == 1 and num_files > 1: # we have already run once, so just continue to get number of files. + continue + command = [core.USER_SCRIPT] + for param in core.USER_SCRIPT_PARAM: + if param == 'FN': + command.append('{0}'.format(file)) + continue + elif param == 'FP': + command.append('{0}'.format(file_path)) + continue + elif param == 'TN': + command.append('{0}'.format(torrent_name)) + continue + elif param == 'TL': + command.append('{0}'.format(torrent_label)) + continue + elif param == 'DN': + if core.USER_SCRIPT_RUNONCE == 1: + command.append('{0}'.format(output_destination)) + else: + command.append('{0}'.format(dirpath)) + continue + else: + command.append(param) + continue + cmd = '' + for item in command: + cmd = '{cmd} {item}'.format(cmd=cmd, item=item) + logger.info('Running script {cmd} on file {path}.'.format(cmd=cmd, path=file_path), 'USERSCRIPT') + try: + p = Popen(command) + res = p.wait() + if str(res) in core.USER_SCRIPT_SUCCESSCODES: # Linux returns 0 for successful. + logger.info('UserScript {0} was successfull'.format(command[0])) + result = 0 + else: + logger.error('UserScript {0} has failed with return code: {1}'.format(command[0], res), 'USERSCRIPT') + logger.info( + 'If the UserScript completed successfully you should add {0} to the user_script_successCodes'.format( + res), 'USERSCRIPT') + result = int(1) + except Exception: + logger.error('UserScript {0} has failed'.format(command[0]), 'USERSCRIPT') + result = int(1) + final_result += result + + num_files_new = 0 + for dirpath, dirnames, filenames in os.walk(output_destination): + for file in filenames: + file_name, file_extension = os.path.splitext(file) + + if file_extension in core.USER_SCRIPT_MEDIAEXTENSIONS or core.USER_SCRIPT_MEDIAEXTENSIONS == 'ALL': + num_files_new += 1 + + if core.USER_SCRIPT_CLEAN == int(1) and num_files_new == 0 and final_result == 0: + logger.info('All files have been processed. Cleaning outputDirectory {0}'.format(output_destination)) + remove_dir(output_destination) + elif core.USER_SCRIPT_CLEAN == int(1) and num_files_new != 0: + logger.info('{0} files were processed, but {1} still remain. outputDirectory will not be cleaned.'.format( + num_files, num_files_new)) + return [final_result, ''] diff --git a/core/utils.py b/core/utils.py new file mode 100644 index 000000000..9c8d81cc5 --- /dev/null +++ b/core/utils.py @@ -0,0 +1,1406 @@ +# coding=utf-8 + +from __future__ import print_function, unicode_literals + +import datetime +from functools import partial +import os +import re +import shutil +import socket +import stat +import struct +import time + +from babelfish import Language +import beets +import guessit +import linktastic +from qbittorrent import Client as qBittorrentClient +import requests +from six import text_type +import subliminal +from synchronousdeluge.client import DelugeClient +from transmissionrpc.client import Client as TransmissionClient +from utorrent.client import UTorrentClient + +import core +from core import extractor, logger, main_db + +try: + from win32event import CreateMutex + from win32api import CloseHandle, GetLastError + from winerror import ERROR_ALREADY_EXISTS +except ImportError: + if os.name == 'nt': + raise + +try: + import jaraco +except ImportError: + if os.name == 'nt': + raise + +requests.packages.urllib3.disable_warnings() + +# Monkey Patch shutil.copyfileobj() to adjust the buffer length to 512KB rather than 4KB +shutil.copyfileobjOrig = shutil.copyfileobj + + +def copyfileobj_fast(fsrc, fdst, length=512 * 1024): + shutil.copyfileobjOrig(fsrc, fdst, length=length) + + +shutil.copyfileobj = copyfileobj_fast + + +def report_nzb(failure_link, client_agent): + # Contact indexer site + logger.info('Sending failure notification to indexer site') + if client_agent == 'nzbget': + headers = {'User-Agent': 'NZBGet / nzbToMedia.py'} + elif client_agent == 'sabnzbd': + headers = {'User-Agent': 'SABnzbd / nzbToMedia.py'} + else: + return + try: + requests.post(failure_link, headers=headers, timeout=(30, 300)) + except Exception as e: + logger.error('Unable to open URL {0} due to {1}'.format(failure_link, e)) + return + + +def sanitize_name(name): + """ + >>> sanitize_name('a/b/c') + 'a-b-c' + >>> sanitize_name('abc') + 'abc' + >>> sanitize_name('a"b') + 'ab' + >>> sanitize_name('.a.b..') + 'a.b' + """ + + # remove bad chars from the filename + name = re.sub(r'[\\/*]', '-', name) + name = re.sub(r'[:\'<>|?]', '', name) + + # remove leading/trailing periods and spaces + name = name.strip(' .') + try: + name = name.encode(core.SYS_ENCODING) + except Exception: + pass + + return name + + +def make_dir(path): + if not os.path.isdir(path): + try: + os.makedirs(path) + except Exception: + return False + return True + + +def remote_dir(path): + if not core.REMOTEPATHS: + return path + for local, remote in core.REMOTEPATHS: + if local in path: + base_dirs = path.replace(local, '').split(os.sep) + if '/' in remote: + remote_sep = '/' + else: + remote_sep = '\\' + new_path = remote_sep.join([remote] + base_dirs) + new_path = re.sub(r'(\S)(\\+)', r'\1\\', new_path) + new_path = re.sub(r'(/+)', r'/', new_path) + new_path = re.sub(r'([/\\])$', r'', new_path) + return new_path + return path + + +def category_search(input_directory, input_name, input_category, root, categories): + tordir = False + + try: + input_name = input_name.encode(core.SYS_ENCODING) + except Exception: + pass + try: + input_directory = input_directory.encode(core.SYS_ENCODING) + except Exception: + pass + + if input_directory is None: # =Nothing to process here. + return input_directory, input_name, input_category, root + + pathlist = os.path.normpath(input_directory).split(os.sep) + + if input_category and input_category in pathlist: + logger.debug('SEARCH: Found the Category: {0} in directory structure'.format(input_category)) + elif input_category: + logger.debug('SEARCH: Could not find the category: {0} in the directory structure'.format(input_category)) + else: + try: + input_category = list(set(pathlist) & set(categories))[-1] # assume last match is most relevant category. + logger.debug('SEARCH: Found Category: {0} in directory structure'.format(input_category)) + except IndexError: + input_category = '' + logger.debug('SEARCH: Could not find a category in the directory structure') + if not os.path.isdir(input_directory) and os.path.isfile(input_directory): # If the input directory is a file + if not input_name: + input_name = os.path.split(os.path.normpath(input_directory))[1] + return input_directory, input_name, input_category, root + + if input_category and os.path.isdir(os.path.join(input_directory, input_category)): + logger.info( + 'SEARCH: Found category directory {0} in input directory directory {1}'.format(input_category, input_directory)) + input_directory = os.path.join(input_directory, input_category) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + if input_name and os.path.isdir(os.path.join(input_directory, input_name)): + logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format(input_name, input_directory)) + input_directory = os.path.join(input_directory, input_name) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isdir(os.path.join(input_directory, sanitize_name(input_name))): + logger.info('SEARCH: Found torrent directory {0} in input directory directory {1}'.format( + sanitize_name(input_name), input_directory)) + input_directory = os.path.join(input_directory, sanitize_name(input_name)) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isfile(os.path.join(input_directory, input_name)): + logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format(input_name, input_directory)) + input_directory = os.path.join(input_directory, input_name) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + elif input_name and os.path.isfile(os.path.join(input_directory, sanitize_name(input_name))): + logger.info('SEARCH: Found torrent file {0} in input directory directory {1}'.format( + sanitize_name(input_name), input_directory)) + input_directory = os.path.join(input_directory, sanitize_name(input_name)) + logger.info('SEARCH: Setting input_directory to {0}'.format(input_directory)) + tordir = True + + imdbid = [item for item in pathlist if '.cp(tt' in item] # This looks for the .cp(tt imdb id in the path. + if imdbid and '.cp(tt' not in input_name: + input_name = imdbid[0] # This ensures the imdb id is preserved and passed to CP + tordir = True + + if input_category and not tordir: + try: + index = pathlist.index(input_category) + if index + 1 < len(pathlist): + tordir = True + logger.info('SEARCH: Found a unique directory {0} in the category directory'.format + (pathlist[index + 1])) + if not input_name: + input_name = pathlist[index + 1] + except ValueError: + pass + + if input_name and not tordir: + if input_name in pathlist or sanitize_name(input_name) in pathlist: + logger.info('SEARCH: Found torrent directory {0} in the directory structure'.format(input_name)) + tordir = True + else: + root = 1 + if not tordir: + root = 2 + + if root > 0: + logger.info('SEARCH: Could not find a unique directory for this download. Assume a common directory.') + logger.info('SEARCH: We will try and determine which files to process, individually') + + return input_directory, input_name, input_category, root + + +def get_dir_size(input_path): + prepend = partial(os.path.join, input_path) + return sum([ + (os.path.getsize(f) if os.path.isfile(f) else get_dir_size(f)) + for f in map(prepend, os.listdir(text_type(input_path))) + ]) + + +def is_min_size(input_name, min_size): + file_name, file_ext = os.path.splitext(os.path.basename(input_name)) + + # audio files we need to check directory size not file size + input_size = os.path.getsize(input_name) + if file_ext in core.AUDIOCONTAINER: + try: + input_size = get_dir_size(os.path.dirname(input_name)) + except Exception: + logger.error('Failed to get file size for {0}'.format(input_name), 'MINSIZE') + return True + + # Ignore files under a certain size + if input_size > min_size * 1048576: + return True + + +def is_sample(input_name): + # Ignore 'sample' in files + if re.search('(^|[\\W_])sample\\d*[\\W_]', input_name.lower()): + return True + + +def copy_link(src, target_link, use_link): + logger.info('MEDIAFILE: [{0}]'.format(os.path.basename(target_link)), 'COPYLINK') + logger.info('SOURCE FOLDER: [{0}]'.format(os.path.dirname(src)), 'COPYLINK') + logger.info('TARGET FOLDER: [{0}]'.format(os.path.dirname(target_link)), 'COPYLINK') + + if src != target_link and os.path.exists(target_link): + logger.info('MEDIAFILE already exists in the TARGET folder, skipping ...', 'COPYLINK') + return True + elif src == target_link and os.path.isfile(target_link) and os.path.isfile(src): + logger.info('SOURCE AND TARGET files are the same, skipping ...', 'COPYLINK') + return True + elif src == os.path.dirname(target_link): + logger.info('SOURCE AND TARGET folders are the same, skipping ...', 'COPYLINK') + return True + + make_dir(os.path.dirname(target_link)) + try: + if use_link == 'dir': + logger.info('Directory linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') + linktastic.dirlink(src, target_link) + return True + if use_link == 'junction': + logger.info('Directory junction linking SOURCE FOLDER -> TARGET FOLDER', 'COPYLINK') + linktastic.dirlink(src, target_link) + return True + elif use_link == 'hard': + logger.info('Hard linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + linktastic.link(src, target_link) + return True + elif use_link == 'sym': + logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + linktastic.symlink(src, target_link) + return True + elif use_link == 'move-sym': + logger.info('Sym linking SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.move(src, target_link) + linktastic.symlink(target_link, src) + return True + elif use_link == 'move': + logger.info('Moving SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.move(src, target_link) + return True + except Exception as e: + logger.warning('Error: {0}, copying instead ... '.format(e), 'COPYLINK') + + logger.info('Copying SOURCE MEDIAFILE -> TARGET FOLDER', 'COPYLINK') + shutil.copy(src, target_link) + + return True + + +def replace_links(link): + n = 0 + target = link + if os.name == 'nt': + if not jaraco.windows.filesystem.islink(link): + logger.debug('{0} is not a link'.format(link)) + return + while jaraco.windows.filesystem.islink(target): + target = jaraco.windows.filesystem.readlink(target) + n = n + 1 + else: + if not os.path.islink(link): + logger.debug('{0} is not a link'.format(link)) + return + while os.path.islink(target): + target = os.readlink(target) + n = n + 1 + if n > 1: + logger.info('Changing sym-link: {0} to point directly to file: {1}'.format(link, target), 'COPYLINK') + os.unlink(link) + linktastic.symlink(target, link) + + +def flatten(output_destination): + logger.info('FLATTEN: Flattening directory: {0}'.format(output_destination)) + for outputFile in list_media_files(output_destination): + dir_path = os.path.dirname(outputFile) + file_name = os.path.basename(outputFile) + + if dir_path == output_destination: + continue + + target = os.path.join(output_destination, file_name) + + try: + shutil.move(outputFile, target) + except Exception: + logger.error('Could not flatten {0}'.format(outputFile), 'FLATTEN') + + remove_empty_folders(output_destination) # Cleanup empty directories + + +def remove_empty_folders(path, remove_root=True): + """Function to remove empty folders""" + if not os.path.isdir(path): + return + + # remove empty subfolders + logger.debug('Checking for empty folders in:{0}'.format(path)) + files = os.listdir(text_type(path)) + if len(files): + for f in files: + fullpath = os.path.join(path, f) + if os.path.isdir(fullpath): + remove_empty_folders(fullpath) + + # if folder empty, delete it + files = os.listdir(text_type(path)) + if len(files) == 0 and remove_root: + logger.debug('Removing empty folder:{}'.format(path)) + os.rmdir(path) + + +def remove_read_only(filename): + if os.path.isfile(filename): + # check first the read-only attribute + file_attribute = os.stat(filename)[0] + if not file_attribute & stat.S_IWRITE: + # File is read-only, so make it writeable + logger.debug('Read only mode on file {name}. Attempting to make it writeable'.format + (name=filename)) + try: + os.chmod(filename, stat.S_IWRITE) + except Exception: + logger.warning('Cannot change permissions of {file}'.format(file=filename), logger.WARNING) + + +# Wake function +def wake_on_lan(ethernet_address): + addr_byte = ethernet_address.split(':') + hw_addr = struct.pack(b'BBBBBB', int(addr_byte[0], 16), + int(addr_byte[1], 16), + int(addr_byte[2], 16), + int(addr_byte[3], 16), + int(addr_byte[4], 16), + int(addr_byte[5], 16)) + + # Build the Wake-On-LAN 'Magic Packet'... + + msg = b'\xff' * 6 + hw_addr * 16 + + # ...and send it to the broadcast address using UDP + + ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ss.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + ss.sendto(msg, ('', 9)) + ss.close() + + +# Test Connection function +def test_connection(host, port): + try: + socket.create_connection((host, port)) + return 'Up' + except Exception: + return 'Down' + + +def wake_up(): + host = core.CFG['WakeOnLan']['host'] + port = int(core.CFG['WakeOnLan']['port']) + mac = core.CFG['WakeOnLan']['mac'] + + i = 1 + while test_connection(host, port) == 'Down' and i < 4: + logger.info(('Sending WakeOnLan Magic Packet for mac: {0}'.format(mac))) + wake_on_lan(mac) + time.sleep(20) + i = i + 1 + + if test_connection(host, port) == 'Down': # final check. + logger.warning('System with mac: {0} has not woken after 3 attempts. ' + 'Continuing with the rest of the script.'.format(mac)) + else: + logger.info('System with mac: {0} has been woken. Continuing with the rest of the script.'.format(mac)) + + +def char_replace(name): + # Special character hex range: + # CP850: 0x80-0xA5 (fortunately not used in ISO-8859-15) + # UTF-8: 1st hex code 0xC2-0xC3 followed by a 2nd hex code 0xA1-0xFF + # ISO-8859-15: 0xA6-0xFF + # The function will detect if Name contains a special character + # If there is special character, detects if it is a UTF-8, CP850 or ISO-8859-15 encoding + encoded = False + encoding = None + if isinstance(name, text_type): + return encoded, name.encode(core.SYS_ENCODING) + for Idx in range(len(name)): + # /!\ detection is done 2char by 2char for UTF-8 special character + if (len(name) != 1) & (Idx < (len(name) - 1)): + # Detect UTF-8 + if ((name[Idx] == '\xC2') | (name[Idx] == '\xC3')) & ( + (name[Idx + 1] >= '\xA0') & (name[Idx + 1] <= '\xFF')): + encoding = 'utf-8' + break + # Detect CP850 + elif (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): + encoding = 'cp850' + break + # Detect ISO-8859-15 + elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): + encoding = 'iso-8859-15' + break + else: + # Detect CP850 + if (name[Idx] >= '\x80') & (name[Idx] <= '\xA5'): + encoding = 'cp850' + break + # Detect ISO-8859-15 + elif (name[Idx] >= '\xA6') & (name[Idx] <= '\xFF'): + encoding = 'iso-8859-15' + break + if encoding and not encoding == core.SYS_ENCODING: + encoded = True + name = name.decode(encoding).encode(core.SYS_ENCODING) + return encoded, name + + +def convert_to_ascii(input_name, dir_name): + + ascii_convert = int(core.CFG['ASCII']['convert']) + if ascii_convert == 0 or os.name == 'nt': # just return if we don't want to convert or on windows os and '\' is replaced!. + return input_name, dir_name + + encoded, input_name = char_replace(input_name) + + directory, base = os.path.split(dir_name) + if not base: # ended with '/' + directory, base = os.path.split(directory) + + encoded, base2 = char_replace(base) + if encoded: + dir_name = os.path.join(directory, base2) + logger.info('Renaming directory to: {0}.'.format(base2), 'ENCODER') + os.rename(os.path.join(directory, base), dir_name) + if 'NZBOP_SCRIPTDIR' in os.environ: + print('[NZB] DIRECTORY={0}'.format(dir_name)) + + for dirname, dirnames, filenames in os.walk(dir_name, topdown=False): + for subdirname in dirnames: + encoded, subdirname2 = char_replace(subdirname) + if encoded: + logger.info('Renaming directory to: {0}.'.format(subdirname2), 'ENCODER') + os.rename(os.path.join(dirname, subdirname), os.path.join(dirname, subdirname2)) + + for dirname, dirnames, filenames in os.walk(dir_name): + for filename in filenames: + encoded, filename2 = char_replace(filename) + if encoded: + logger.info('Renaming file to: {0}.'.format(filename2), 'ENCODER') + os.rename(os.path.join(dirname, filename), os.path.join(dirname, filename2)) + + return input_name, dir_name + + +def parse_other(args): + return os.path.normpath(args[1]), '', '', '', '' + + +def parse_rtorrent(args): + # rtorrent usage: system.method.set_key = event.download.finished,TorrentToMedia, + # 'execute={/path/to/nzbToMedia/TorrentToMedia.py,\'$d.get_base_path=\',\'$d.get_name=\',\'$d.get_custom1=\',\'$d.get_hash=\'}' + input_directory = os.path.normpath(args[1]) + try: + input_name = args[2] + except Exception: + input_name = '' + try: + input_category = args[3] + except Exception: + input_category = '' + try: + input_hash = args[4] + except Exception: + input_hash = '' + try: + input_id = args[4] + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_utorrent(args): + # uTorrent usage: call TorrentToMedia.py '%D' '%N' '%L' '%I' + input_directory = os.path.normpath(args[1]) + input_name = args[2] + try: + input_category = args[3] + except Exception: + input_category = '' + try: + input_hash = args[4] + except Exception: + input_hash = '' + try: + input_id = args[4] + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_deluge(args): + # Deluge usage: call TorrentToMedia.py TORRENT_ID TORRENT_NAME TORRENT_DIR + input_directory = os.path.normpath(args[3]) + input_name = args[2] + input_hash = args[1] + input_id = args[1] + try: + input_category = core.TORRENT_CLASS.core.get_torrent_status(input_id, ['label']).get()['label'] + except Exception: + input_category = '' + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_transmission(args): + # Transmission usage: call TorrenToMedia.py (%TR_TORRENT_DIR% %TR_TORRENT_NAME% is passed on as environmental variables) + input_directory = os.path.normpath(os.getenv('TR_TORRENT_DIR')) + input_name = os.getenv('TR_TORRENT_NAME') + input_category = '' # We dont have a category yet + input_hash = os.getenv('TR_TORRENT_HASH') + input_id = os.getenv('TR_TORRENT_ID') + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_vuze(args): + # vuze usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D%N%L%I%K%F' + try: + cur_input = args[1].split(',') + except Exception: + cur_input = [] + try: + input_directory = os.path.normpath(cur_input[0]) + except Exception: + input_directory = '' + try: + input_name = cur_input[1] + except Exception: + input_name = '' + try: + input_category = cur_input[2] + except Exception: + input_category = '' + try: + input_hash = cur_input[3] + except Exception: + input_hash = '' + try: + input_id = cur_input[3] + except Exception: + input_id = '' + try: + if cur_input[4] == 'single': + input_name = cur_input[5] + except Exception: + pass + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_qbittorrent(args): + # qbittorrent usage: C:\full\path\to\nzbToMedia\TorrentToMedia.py '%D|%N|%L|%I' + try: + cur_input = args[1].split('|') + except Exception: + cur_input = [] + try: + input_directory = os.path.normpath(cur_input[0].replace('\'', '')) + except Exception: + input_directory = '' + try: + input_name = cur_input[1].replace('\'', '') + except Exception: + input_name = '' + try: + input_category = cur_input[2].replace('\'', '') + except Exception: + input_category = '' + try: + input_hash = cur_input[3].replace('\'', '') + except Exception: + input_hash = '' + try: + input_id = cur_input[3].replace('\'', '') + except Exception: + input_id = '' + + return input_directory, input_name, input_category, input_hash, input_id + + +def parse_args(client_agent, args): + clients = { + 'other': parse_other, + 'rtorrent': parse_rtorrent, + 'utorrent': parse_utorrent, + 'deluge': parse_deluge, + 'transmission': parse_transmission, + 'qbittorrent': parse_qbittorrent, + 'vuze': parse_vuze, + } + + try: + return clients[client_agent](args) + except Exception: + return None, None, None, None, None + + +def get_dirs(section, subsection, link='hard'): + to_return = [] + + def process_dir(path): + folders = [] + + logger.info('Searching {0} for mediafiles to post-process ...'.format(path)) + sync = [o for o in os.listdir(text_type(path)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] + # search for single files and move them into their own folder for post-processing + for mediafile in [os.path.join(path, o) for o in os.listdir(text_type(path)) if + os.path.isfile(os.path.join(path, o))]: + if len(sync) > 0: + break + if os.path.split(mediafile)[1] in ['Thumbs.db', 'thumbs.db']: + continue + try: + logger.debug('Found file {0} in root directory {1}.'.format(os.path.split(mediafile)[1], path)) + new_path = None + file_ext = os.path.splitext(mediafile)[1] + try: + if file_ext in core.AUDIOCONTAINER: + f = beets.mediafile.MediaFile(mediafile) + + # get artist and album info + artist = f.artist + album = f.album + + # create new path + new_path = os.path.join(path, '{0} - {1}'.format(sanitize_name(artist), sanitize_name(album))) + elif file_ext in core.MEDIACONTAINER: + f = guessit.guessit(mediafile) + + # get title + title = f.get('series') or f.get('title') + + if not title: + title = os.path.splitext(os.path.basename(mediafile))[0] + + new_path = os.path.join(path, sanitize_name(title)) + except Exception as e: + logger.error('Exception parsing name for media file: {0}: {1}'.format(os.path.split(mediafile)[1], e)) + + if not new_path: + title = os.path.splitext(os.path.basename(mediafile))[0] + new_path = os.path.join(path, sanitize_name(title)) + + try: + new_path = new_path.encode(core.SYS_ENCODING) + except Exception: + pass + + # Just fail-safe incase we already have afile with this clean-name (was actually a bug from earlier code, but let's be safe). + if os.path.isfile(new_path): + new_path2 = os.path.join(os.path.join(os.path.split(new_path)[0], 'new'), os.path.split(new_path)[1]) + new_path = new_path2 + + # create new path if it does not exist + if not os.path.exists(new_path): + make_dir(new_path) + + newfile = os.path.join(new_path, sanitize_name(os.path.split(mediafile)[1])) + try: + newfile = newfile.encode(core.SYS_ENCODING) + except Exception: + pass + + # link file to its new path + copy_link(mediafile, newfile, link) + except Exception as e: + logger.error('Failed to move {0} to its own directory: {1}'.format(os.path.split(mediafile)[1], e)) + + # removeEmptyFolders(path, removeRoot=False) + + if os.listdir(text_type(path)): + for directory in [os.path.join(path, o) for o in os.listdir(text_type(path)) if + os.path.isdir(os.path.join(path, o))]: + sync = [o for o in os.listdir(text_type(directory)) if os.path.splitext(o)[1] in ['.!sync', '.bts']] + if len(sync) > 0 or len(os.listdir(text_type(directory))) == 0: + continue + folders.extend([directory]) + return folders + + try: + watch_dir = os.path.join(core.CFG[section][subsection]['watch_dir'], subsection) + if os.path.exists(watch_dir): + to_return.extend(process_dir(watch_dir)) + elif os.path.exists(core.CFG[section][subsection]['watch_dir']): + to_return.extend(process_dir(core.CFG[section][subsection]['watch_dir'])) + except Exception as e: + logger.error('Failed to add directories from {0} for post-processing: {1}'.format + (core.CFG[section][subsection]['watch_dir'], e)) + + if core.USELINK == 'move': + try: + output_directory = os.path.join(core.OUTPUTDIRECTORY, subsection) + if os.path.exists(output_directory): + to_return.extend(process_dir(output_directory)) + except Exception as e: + logger.error('Failed to add directories from {0} for post-processing: {1}'.format(core.OUTPUTDIRECTORY, e)) + + if not to_return: + logger.debug('No directories identified in {0}:{1} for post-processing'.format(section, subsection)) + + return list(set(to_return)) + + +def onerror(func, path, exc_info): + """ + Error handler for ``shutil.rmtree``. + + If the error is due to an access error (read only file) + it attempts to add write permission and then retries. + + If the error is for another reason it re-raises the error. + + Usage : ``shutil.rmtree(path, onerror=onerror)`` + """ + if not os.access(path, os.W_OK): + # Is the error an access error ? + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise Exception + + +def remove_dir(dir_name): + logger.info('Deleting {0}'.format(dir_name)) + try: + shutil.rmtree(text_type(dir_name), onerror=onerror) + except Exception: + logger.error('Unable to delete folder {0}'.format(dir_name)) + + +def clean_dir(path, section, subsection): + cfg = dict(core.CFG[section][subsection]) + if not os.path.exists(path): + logger.info('Directory {0} has been processed and removed ...'.format(path), 'CLEANDIR') + return + if core.FORCE_CLEAN and not core.FAILED: + logger.info('Doing Forceful Clean of {0}'.format(path), 'CLEANDIR') + remove_dir(path) + return + min_size = int(cfg.get('minSize', 0)) + delete_ignored = int(cfg.get('delete_ignored', 0)) + try: + num_files = len(list_media_files(path, min_size=min_size, delete_ignored=delete_ignored)) + except Exception: + num_files = 'unknown' + if num_files > 0: + logger.info( + 'Directory {0} still contains {1} unprocessed file(s), skipping ...'.format(path, num_files), + 'CLEANDIRS') + return + + logger.info('Directory {0} has been processed, removing ...'.format(path), 'CLEANDIRS') + try: + shutil.rmtree(path, onerror=onerror) + except Exception: + logger.error('Unable to delete directory {0}'.format(path)) + + +def create_torrent_class(client_agent): + # Hardlink solution for Torrents + tc = None + + if client_agent == 'utorrent': + try: + logger.debug('Connecting to {0}: {1}'.format(client_agent, core.UTORRENTWEBUI)) + tc = UTorrentClient(core.UTORRENTWEBUI, core.UTORRENTUSR, core.UTORRENTPWD) + except Exception: + logger.error('Failed to connect to uTorrent') + + if client_agent == 'transmission': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format( + client_agent, core.TRANSMISSIONHOST, core.TRANSMISSIONPORT)) + tc = TransmissionClient(core.TRANSMISSIONHOST, core.TRANSMISSIONPORT, + core.TRANSMISSIONUSR, + core.TRANSMISSIONPWD) + except Exception: + logger.error('Failed to connect to Transmission') + + if client_agent == 'deluge': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.DELUGEHOST, core.DELUGEPORT)) + tc = DelugeClient() + tc.connect(host=core.DELUGEHOST, port=core.DELUGEPORT, username=core.DELUGEUSR, + password=core.DELUGEPWD) + except Exception: + logger.error('Failed to connect to Deluge') + + if client_agent == 'qbittorrent': + try: + logger.debug('Connecting to {0}: http://{1}:{2}'.format(client_agent, core.QBITTORRENTHOST, core.QBITTORRENTPORT)) + tc = qBittorrentClient('http://{0}:{1}/'.format(core.QBITTORRENTHOST, core.QBITTORRENTPORT)) + tc.login(core.QBITTORRENTUSR, core.QBITTORRENTPWD) + except Exception: + logger.error('Failed to connect to qBittorrent') + + return tc + + +def pause_torrent(client_agent, input_hash, input_id, input_name): + logger.debug('Stopping torrent {0} in {1} while processing'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.stop(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.stop_torrent(input_id) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.pause_torrent([input_id]) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.pause(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to stop torrent {0} in {1}'.format(input_name, client_agent)) + + +def resume_torrent(client_agent, input_hash, input_id, input_name): + if not core.TORRENT_RESUME == 1: + return + logger.debug('Starting torrent {0} in {1}'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.start(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.start_torrent(input_id) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.resume_torrent([input_id]) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.resume(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to start torrent {0} in {1}'.format(input_name, client_agent)) + + +def remove_torrent(client_agent, input_hash, input_id, input_name): + if core.DELETE_ORIGINAL == 1 or core.USELINK == 'move': + logger.debug('Deleting torrent {0} from {1}'.format(input_name, client_agent)) + try: + if client_agent == 'utorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.removedata(input_hash) + core.TORRENT_CLASS.remove(input_hash) + if client_agent == 'transmission' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.remove_torrent(input_id, True) + if client_agent == 'deluge' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.core.remove_torrent(input_id, True) + if client_agent == 'qbittorrent' and core.TORRENT_CLASS != '': + core.TORRENT_CLASS.delete_permanently(input_hash) + time.sleep(5) + except Exception: + logger.warning('Failed to delete torrent {0} in {1}'.format(input_name, client_agent)) + else: + resume_torrent(client_agent, input_hash, input_id, input_name) + + +def find_download(client_agent, download_id): + logger.debug('Searching for Download on {0} ...'.format(client_agent)) + if client_agent == 'utorrent': + torrents = core.TORRENT_CLASS.list()[1]['torrents'] + for torrent in torrents: + if download_id in torrent: + return True + if client_agent == 'transmission': + torrents = core.TORRENT_CLASS.get_torrents() + for torrent in torrents: + torrent_hash = torrent.hashString + if torrent_hash == download_id: + return True + if client_agent == 'deluge': + return False + if client_agent == 'qbittorrent': + torrents = core.TORRENT_CLASS.torrents() + for torrent in torrents: + if torrent['hash'] == download_id: + return True + if client_agent == 'sabnzbd': + if 'http' in core.SABNZBDHOST: + base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + else: + base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + url = base_url + params = { + 'apikey': core.SABNZBDAPIKEY, + 'mode': 'get_files', + 'output': 'json', + 'value': download_id, + } + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return False # failure + + result = r.json() + if result['files']: + return True + return False + + +def get_nzoid(input_name): + nzoid = None + slots = [] + logger.debug('Searching for nzoid from SAbnzbd ...') + if 'http' in core.SABNZBDHOST: + base_url = '{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + else: + base_url = 'http://{0}:{1}/api'.format(core.SABNZBDHOST, core.SABNZBDPORT) + url = base_url + params = { + 'apikey': core.SABNZBDAPIKEY, + 'mode': 'queue', + 'output': 'json', + } + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return nzoid # failure + try: + result = r.json() + clean_name = os.path.splitext(os.path.split(input_name)[1])[0] + slots.extend([(slot['nzo_id'], slot['filename']) for slot in result['queue']['slots']]) + except Exception: + logger.warning('Data from SABnzbd queue could not be parsed') + params['mode'] = 'history' + try: + r = requests.get(url, params=params, verify=False, timeout=(30, 120)) + except requests.ConnectionError: + logger.error('Unable to open URL') + return nzoid # failure + try: + result = r.json() + clean_name = os.path.splitext(os.path.split(input_name)[1])[0] + slots.extend([(slot['nzo_id'], slot['name']) for slot in result['history']['slots']]) + except Exception: + logger.warning('Data from SABnzbd history could not be parsed') + try: + for nzo_id, name in slots: + if name in [input_name, clean_name]: + nzoid = nzo_id + logger.debug('Found nzoid: {0}'.format(nzoid)) + break + except Exception: + logger.warning('Data from SABnzbd could not be parsed') + return nzoid + + +def clean_file_name(filename): + """Cleans up nzb name by removing any . and _ + characters, along with any trailing hyphens. + + Is basically equivalent to replacing all _ and . with a + space, but handles decimal numbers in string, for example: + """ + + filename = re.sub(r'(\D)\.(?!\s)(\D)', r'\1 \2', filename) + filename = re.sub(r'(\d)\.(\d{4})', r'\1 \2', filename) # if it ends in a year then don't keep the dot + filename = re.sub(r'(\D)\.(?!\s)', r'\1 ', filename) + filename = re.sub(r'\.(?!\s)(\D)', r' \1', filename) + filename = filename.replace('_', ' ') + filename = re.sub('-$', '', filename) + filename = re.sub(r'^\[.*]', '', filename) + return filename.strip() + + +def is_archive_file(filename): + """Check if the filename is allowed for the Archive""" + for regext in core.COMPRESSEDCONTAINER: + if regext.search(filename): + return regext.split(filename)[0] + return False + + +def is_media_file(mediafile, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): + if otherext is None: + otherext = [] + + file_name, file_ext = os.path.splitext(mediafile) + + try: + # ignore MAC OS's 'resource fork' files + if file_name.startswith('._'): + return False + except Exception: + pass + if (media and file_ext.lower() in core.MEDIACONTAINER) \ + or (audio and file_ext.lower() in core.AUDIOCONTAINER) \ + or (meta and file_ext.lower() in core.METACONTAINER) \ + or (archives and is_archive_file(mediafile)) \ + or (other and (file_ext.lower() in otherext or 'all' in otherext)): + return True + else: + return False + + +def list_media_files(path, min_size=0, delete_ignored=0, media=True, audio=True, meta=True, archives=True, other=False, otherext=None): + if otherext is None: + otherext = [] + + files = [] + if not os.path.isdir(path): + if os.path.isfile(path): # Single file downloads. + cur_file = os.path.split(path)[1] + if is_media_file(cur_file, media, audio, meta, archives, other, otherext): + # Optionally ignore sample files + if is_sample(path) or not is_min_size(path, min_size): + if delete_ignored == 1: + try: + os.unlink(path) + logger.debug('Ignored file {0} has been removed ...'.format + (cur_file)) + except Exception: + pass + else: + files.append(path) + + return files + + for cur_file in os.listdir(text_type(path)): + full_cur_file = os.path.join(path, cur_file) + + # if it's a folder do it recursively + if os.path.isdir(full_cur_file) and not cur_file.startswith('.'): + files += list_media_files(full_cur_file, min_size, delete_ignored, media, audio, meta, archives, other, otherext) + + elif is_media_file(cur_file, media, audio, meta, archives, other, otherext): + # Optionally ignore sample files + if is_sample(full_cur_file) or not is_min_size(full_cur_file, min_size): + if delete_ignored == 1: + try: + os.unlink(full_cur_file) + logger.debug('Ignored file {0} has been removed ...'.format + (cur_file)) + except Exception: + pass + continue + + files.append(full_cur_file) + + return sorted(files, key=len) + + +def find_imdbid(dir_name, input_name, omdb_api_key): + imdbid = None + + logger.info('Attemping imdbID lookup for {0}'.format(input_name)) + + # find imdbid in dirName + logger.info('Searching folder and file names for imdbID ...') + m = re.search(r'(tt\d{7})', dir_name + input_name) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}]'.format(imdbid)) + return imdbid + if os.path.isdir(dir_name): + for file in os.listdir(text_type(dir_name)): + m = re.search(r'(tt\d{7})', file) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}] via file name'.format(imdbid)) + return imdbid + if 'NZBPR__DNZB_MOREINFO' in os.environ: + dnzb_more_info = os.environ.get('NZBPR__DNZB_MOREINFO', '') + if dnzb_more_info != '': + regex = re.compile(r'^http://www.imdb.com/title/(tt[0-9]+)/$', re.IGNORECASE) + m = regex.match(dnzb_more_info) + if m: + imdbid = m.group(1) + logger.info('Found imdbID [{0}] from DNZB-MoreInfo'.format(imdbid)) + return imdbid + logger.info('Searching IMDB for imdbID ...') + try: + guess = guessit.guessit(input_name) + except Exception: + guess = None + if guess: + # Movie Title + title = None + if 'title' in guess: + title = guess['title'] + + # Movie Year + year = None + if 'year' in guess: + year = guess['year'] + + url = 'http://www.omdbapi.com' + + if not omdb_api_key: + logger.info('Unable to determine imdbID: No api key provided for ombdapi.com.') + return + + logger.debug('Opening URL: {0}'.format(url)) + + try: + r = requests.get(url, params={'apikey': omdb_api_key, 'y': year, 't': title}, + verify=False, timeout=(60, 300)) + except requests.ConnectionError: + logger.error('Unable to open URL {0}'.format(url)) + return + + try: + results = r.json() + except Exception: + logger.error('No json data returned from omdbapi.com') + + try: + imdbid = results['imdbID'] + except Exception: + logger.error('No imdbID returned from omdbapi.com') + + if imdbid: + logger.info('Found imdbID [{0}]'.format(imdbid)) + return imdbid + + logger.warning('Unable to find a imdbID for {0}'.format(input_name)) + return imdbid + + +def extract_files(src, dst=None, keep_archive=None): + extracted_folder = [] + extracted_archive = [] + + for inputFile in list_media_files(src, media=False, audio=False, meta=False, archives=True): + dir_path = os.path.dirname(inputFile) + full_file_name = os.path.basename(inputFile) + archive_name = os.path.splitext(full_file_name)[0] + archive_name = re.sub(r'part[0-9]+', '', archive_name) + + if dir_path in extracted_folder and archive_name in extracted_archive: + continue # no need to extract this, but keep going to look for other archives and sub directories. + + try: + if extractor.extract(inputFile, dst or dir_path): + extracted_folder.append(dir_path) + extracted_archive.append(archive_name) + except Exception: + logger.error('Extraction failed for: {0}'.format(full_file_name)) + + for folder in extracted_folder: + for inputFile in list_media_files(folder, media=False, audio=False, meta=False, archives=True): + full_file_name = os.path.basename(inputFile) + archive_name = os.path.splitext(full_file_name)[0] + archive_name = re.sub(r'part[0-9]+', '', archive_name) + if archive_name not in extracted_archive or keep_archive: + continue # don't remove if we haven't extracted this archive, or if we want to preserve them. + logger.info('Removing extracted archive {0} from folder {1} ...'.format(full_file_name, folder)) + try: + if not os.access(inputFile, os.W_OK): + os.chmod(inputFile, stat.S_IWUSR) + os.remove(inputFile) + time.sleep(1) + except Exception as e: + logger.error('Unable to remove file {0} due to: {1}'.format(inputFile, e)) + + +def import_subs(filename): + if not core.GETSUBS: + return + try: + subliminal.region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) + except Exception: + pass + + languages = set() + for item in core.SLANGUAGES: + try: + languages.add(Language(item)) + except Exception: + pass + if not languages: + return + + logger.info('Attempting to download subtitles for {0}'.format(filename), 'SUBTITLES') + try: + video = subliminal.scan_video(filename) + subtitles = subliminal.download_best_subtitles({video}, languages) + subliminal.save_subtitles(video, subtitles[video]) + except Exception as e: + logger.error('Failed to download subtitles for {0} due to: {1}'.format(filename, e), 'SUBTITLES') + + +def server_responding(base_url): + logger.debug('Attempting to connect to server at {0}'.format(base_url), 'SERVER') + try: + requests.get(base_url, timeout=(60, 120), verify=False) + logger.debug('Server responded at {0}'.format(base_url), 'SERVER') + return True + except (requests.ConnectionError, requests.exceptions.Timeout): + logger.error('Server failed to respond at {0}'.format(base_url), 'SERVER') + return False + + +def plex_update(category): + if core.FAILED: + return + url = '{scheme}://{host}:{port}/library/sections/'.format( + scheme='https' if core.PLEXSSL else 'http', + host=core.PLEXHOST, + port=core.PLEXPORT, + ) + section = None + if not core.PLEXSEC: + return + logger.debug('Attempting to update Plex Library for category {0}.'.format(category), 'PLEX') + for item in core.PLEXSEC: + if item[0] == category: + section = item[1] + + if section: + url = '{url}{section}/refresh?X-Plex-Token={token}'.format(url=url, section=section, token=core.PLEXTOKEN) + requests.get(url, timeout=(60, 120), verify=False) + logger.debug('Plex Library has been refreshed.', 'PLEX') + else: + logger.debug('Could not identify section for plex update', 'PLEX') + + +def backup_versioned_file(old_file, version): + num_tries = 0 + + new_file = '{old}.v{version}'.format(old=old_file, version=version) + + while not os.path.isfile(new_file): + if not os.path.isfile(old_file): + logger.log(u'Not creating backup, {file} doesn\'t exist'.format(file=old_file), logger.DEBUG) + break + + try: + logger.log(u'Trying to back up {old} to {new]'.format(old=old_file, new=new_file), logger.DEBUG) + shutil.copy(old_file, new_file) + logger.log(u'Backup done', logger.DEBUG) + break + except Exception as error: + logger.log(u'Error while trying to back up {old} to {new} : {msg}'.format + (old=old_file, new=new_file, msg=error), logger.WARNING) + num_tries += 1 + time.sleep(1) + logger.log(u'Trying again.', logger.DEBUG) + + if num_tries >= 10: + logger.log(u'Unable to back up {old} to {new} please do it manually.'.format(old=old_file, new=new_file), logger.ERROR) + return False + + return True + + +def update_download_info_status(input_name, status): + logger.db('Updating status of our download {0} in the DB to {1}'.format(input_name, status)) + + my_db = main_db.DBConnection() + my_db.action('UPDATE downloads SET status=?, last_update=? WHERE input_name=?', + [status, datetime.date.today().toordinal(), text_type(input_name)]) + + +def get_download_info(input_name, status): + logger.db('Getting download info for {0} from the DB'.format(input_name)) + + my_db = main_db.DBConnection() + sql_results = my_db.select('SELECT * FROM downloads WHERE input_name=? AND status=?', + [text_type(input_name), status]) + + return sql_results + + +class WindowsProcess(object): + def __init__(self): + self.mutex = None + self.mutexname = 'nzbtomedia_{pid}'.format(pid=core.PID_FILE.replace('\\', '/')) # {D0E858DF-985E-4907-B7FB-8D732C3FC3B9}' + self.CreateMutex = CreateMutex + self.CloseHandle = CloseHandle + self.GetLastError = GetLastError + self.ERROR_ALREADY_EXISTS = ERROR_ALREADY_EXISTS + + def alreadyrunning(self): + self.mutex = self.CreateMutex(None, 0, self.mutexname) + self.lasterror = self.GetLastError() + if self.lasterror == self.ERROR_ALREADY_EXISTS: + self.CloseHandle(self.mutex) + return True + else: + return False + + def __del__(self): + if self.mutex: + self.CloseHandle(self.mutex) + + +class PosixProcess(object): + def __init__(self): + self.pidpath = core.PID_FILE + self.lock_socket = None + + def alreadyrunning(self): + try: + self.lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self.lock_socket.bind('\0{path}'.format(path=self.pidpath)) + self.lasterror = False + return self.lasterror + except socket.error as e: + if 'Address already in use' in e: + self.lasterror = True + return self.lasterror + except AttributeError: + pass + if os.path.exists(self.pidpath): + # Make sure it is not a 'stale' pidFile + try: + pid = int(open(self.pidpath, 'r').read().strip()) + except Exception: + pid = None + # Check list of running pids, if not running it is stale so overwrite + if isinstance(pid, int): + try: + os.kill(pid, 0) + self.lasterror = True + except OSError: + self.lasterror = False + else: + self.lasterror = False + else: + self.lasterror = False + + if not self.lasterror: + # Write my pid into pidFile to keep multiple copies of program from running + try: + fp = open(self.pidpath, 'w') + fp.write(str(os.getpid())) + fp.close() + except Exception: + pass + + return self.lasterror + + def __del__(self): + if not self.lasterror: + if self.lock_socket: + self.lock_socket.close() + if os.path.isfile(self.pidpath): + os.unlink(self.pidpath) + + +if os.name == 'nt': + RunningProcess = WindowsProcess +else: + RunningProcess = PosixProcess diff --git a/core/utorrent/__init__.py b/core/utorrent/__init__.py deleted file mode 100644 index 9bad5790a..000000000 --- a/core/utorrent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding=utf-8 diff --git a/core/utorrent/client.py b/core/utorrent/client.py deleted file mode 100644 index 2d5a57414..000000000 --- a/core/utorrent/client.py +++ /dev/null @@ -1,146 +0,0 @@ -# coding=utf8 -import urllib -import urllib2 -import urlparse -import cookielib -import re -import StringIO -try: - import json -except ImportError: - import simplejson as json - -from upload import MultiPartForm - -class UTorrentClient(object): - - def __init__(self, base_url, username, password): - self.base_url = base_url - self.username = username - self.password = password - self.opener = self._make_opener('uTorrent', base_url, username, password) - self.token = self._get_token() - #TODO refresh token, when necessary - - def _make_opener(self, realm, base_url, username, password): - '''uTorrent API need HTTP Basic Auth and cookie support for token verify.''' - - auth_handler = urllib2.HTTPBasicAuthHandler() - auth_handler.add_password(realm=realm, - uri=base_url, - user=username, - passwd=password) - opener = urllib2.build_opener(auth_handler) - urllib2.install_opener(opener) - - cookie_jar = cookielib.CookieJar() - cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar) - - handlers = [auth_handler, cookie_handler] - opener = urllib2.build_opener(*handlers) - return opener - - def _get_token(self): - url = urlparse.urljoin(self.base_url, 'token.html') - response = self.opener.open(url) - token_re = "" - match = re.search(token_re, response.read()) - return match.group(1) - - def list(self, **kwargs): - params = [('list', '1')] - params += kwargs.items() - return self._action(params) - - def start(self, *hashes): - params = [('action', 'start'),] - for hash in hashes: - params.append(('hash', hash)) - return self._action(params) - - def stop(self, *hashes): - params = [('action', 'stop'),] - for hash in hashes: - params.append(('hash', hash)) - return self._action(params) - - def pause(self, *hashes): - params = [('action', 'pause'),] - for hash in hashes: - params.append(('hash', hash)) - return self._action(params) - - def forcestart(self, *hashes): - params = [('action', 'forcestart'),] - for hash in hashes: - params.append(('hash', hash)) - return self._action(params) - - def getfiles(self, hash): - params = [('action', 'getfiles'), ('hash', hash)] - return self._action(params) - - def getprops(self, hash): - params = [('action', 'getprops'), ('hash', hash)] - return self._action(params) - - def setprops(self, hash, **kvpairs): - params = [('action', 'setprops'), ('hash', hash)] - for k, v in kvpairs.iteritems(): - params.append( ("s", k) ) - params.append( ("v", v) ) - - return self._action(params) - - def setprio(self, hash, priority, *files): - params = [('action', 'setprio'), ('hash', hash), ('p', str(priority))] - for file_index in files: - params.append(('f', str(file_index))) - - return self._action(params) - - def addfile(self, filename, filepath=None, bytes=None): - params = [('action', 'add-file')] - - form = MultiPartForm() - if filepath is not None: - file_handler = open(filepath,'rb') - else: - file_handler = StringIO.StringIO(bytes) - - form.add_file('torrent_file', filename.encode('utf-8'), file_handler) - - return self._action(params, str(form), form.get_content_type()) - - def addurl(self, url): - params = [('action', 'add-url'), ('s', url)] - self._action(params) - - def remove(self, *hashes): - params = [('action', 'remove'),] - for hash in hashes: - params.append(('hash', hash)) - return self._action(params) - - def removedata(self, *hashes): - params = [('action', 'removedata'),] - for hash in hashes: - params.append(('hash', hash)) - return self._action(params) - - def _action(self, params, body=None, content_type=None): - #about token, see https://github.com/bittorrent/webui/wiki/TokenSystem - url = self.base_url + '?token=' + self.token + '&' + urllib.urlencode(params) - request = urllib2.Request(url) - - if body: - request.add_data(body) - request.add_header('Content-length', len(body)) - if content_type: - request.add_header('Content-type', content_type) - - try: - response = self.opener.open(request) - return response.code, json.loads(response.read()) - except urllib2.HTTPError,e: - raise \ No newline at end of file diff --git a/core/utorrent/upload.py b/core/utorrent/upload.py deleted file mode 100644 index 8b5025327..000000000 --- a/core/utorrent/upload.py +++ /dev/null @@ -1,72 +0,0 @@ -# coding=utf-8 -# code copied from http://www.doughellmann.com/PyMOTW/urllib2/ - -import itertools -import mimetools -import mimetypes -from cStringIO import StringIO -import urllib -import urllib2 - -class MultiPartForm(object): - """Accumulate the data to be used when posting a form.""" - - def __init__(self): - self.form_fields = [] - self.files = [] - self.boundary = mimetools.choose_boundary() - return - - def get_content_type(self): - return 'multipart/form-data; boundary=%s' % self.boundary - - def add_field(self, name, value): - """Add a simple field to the form data.""" - self.form_fields.append((name, value)) - return - - def add_file(self, fieldname, filename, fileHandle, mimetype=None): - """Add a file to be uploaded.""" - body = fileHandle.read() - if mimetype is None: - mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' - self.files.append((fieldname, filename, mimetype, body)) - return - - def __str__(self): - """Return a string representing the form data, including attached files.""" - # Build a list of lists, each containing "lines" of the - # request. Each part is separated by a boundary string. - # Once the list is built, return a string where each - # line is separated by '\r\n'. - parts = [] - part_boundary = '--' + self.boundary - - # Add the form fields - parts.extend( - [ part_boundary, - 'Content-Disposition: form-data; name="%s"' % name, - '', - value, - ] - for name, value in self.form_fields - ) - - # Add the files to upload - parts.extend( - [ part_boundary, - 'Content-Disposition: file; name="%s"; filename="%s"' % \ - (field_name, filename), - 'Content-Type: %s' % content_type, - '', - body, - ] - for field_name, filename, content_type, body in self.files - ) - - # Flatten the list and add closing boundary marker, - # then return CR+LF separated data - flattened = list(itertools.chain(*parts)) - flattened.append('--' + self.boundary + '--') - flattened.append('') - return '\r\n'.join(flattened) \ No newline at end of file diff --git a/core/versionCheck.py b/core/versionCheck.py deleted file mode 100644 index ba6726023..000000000 --- a/core/versionCheck.py +++ /dev/null @@ -1,529 +0,0 @@ -# coding=utf-8 -# Author: Nic Wolfe -# Modified by: echel0n - -import os -import platform -import shutil -import subprocess -import re -import urllib -import tarfile -import stat -import traceback -import gh_api as github - -import cleanup -import core -from core import logger - - -class CheckVersion(object): - """ - Version check class meant to run as a thread object with the SB scheduler. - """ - - def __init__(self): - self.install_type = self.find_install_type() - self.installed_version = None - self.installed_branch = None - - if self.install_type == 'git': - self.updater = GitUpdateManager() - elif self.install_type == 'source': - self.updater = SourceUpdateManager() - else: - self.updater = None - - def run(self): - self.check_for_new_version() - - def find_install_type(self): - """ - Determines how this copy of SB was installed. - - returns: type of installation. Possible values are: - 'win': any compiled windows build - 'git': running from source using git - 'source': running from source without git - """ - - # check if we're a windows build - if os.path.isdir(os.path.join(core.PROGRAM_DIR, u'.git')): - install_type = 'git' - else: - install_type = 'source' - - return install_type - - def check_for_new_version(self, force=False): - """ - Checks the internet for a newer version. - - returns: bool, True for new version or False for no new version. - - force: if true the VERSION_NOTIFY setting will be ignored and a check will be forced - """ - - if not core.VERSION_NOTIFY and not force: - logger.log(u"Version checking is disabled, not checking for the newest version") - return False - - logger.log(u"Checking if {install} needs an update".format(install=self.install_type)) - if not self.updater.need_update(): - core.NEWEST_VERSION_STRING = None - logger.log(u"No update needed") - return False - - self.updater.set_newest_text() - return True - - def update(self): - if self.updater.need_update(): - result = self.updater.update() - cleanup.clean('core', 'libs') - return result - - -class UpdateManager(object): - def get_github_repo_user(self): - return core.GIT_USER - - def get_github_repo(self): - return core.GIT_REPO - - def get_github_branch(self): - return core.GIT_BRANCH - - -class GitUpdateManager(UpdateManager): - def __init__(self): - self._git_path = self._find_working_git() - self.github_repo_user = self.get_github_repo_user() - self.github_repo = self.get_github_repo() - self.branch = self._find_git_branch() - - self._cur_commit_hash = None - self._newest_commit_hash = None - self._num_commits_behind = 0 - self._num_commits_ahead = 0 - - def _git_error(self): - logger.debug( - 'Unable to find your git executable - Set git_path in your autoProcessMedia.cfg OR delete your .git folder and run from source to enable updates.') - - def _find_working_git(self): - test_cmd = 'version' - - if core.GIT_PATH: - main_git = '"{git}"'.format(git=core.GIT_PATH) - else: - main_git = 'git' - - logger.log(u"Checking if we can use git commands: {git} {cmd}".format - (git=main_git, cmd=test_cmd), logger.DEBUG) - output, err, exit_status = self._run_git(main_git, test_cmd) - - if exit_status == 0: - logger.log(u"Using: {git}".format(git=main_git), logger.DEBUG) - return main_git - else: - logger.log(u"Not using: {git}".format(git=main_git), logger.DEBUG) - - # trying alternatives - - alternative_git = [] - - # osx people who start SB from launchd have a broken path, so try a hail-mary attempt for them - if platform.system().lower() == 'darwin': - alternative_git.append('/usr/local/git/bin/git') - - if platform.system().lower() == 'windows': - if main_git != main_git.lower(): - alternative_git.append(main_git.lower()) - - if alternative_git: - logger.log(u"Trying known alternative git locations", logger.DEBUG) - - for cur_git in alternative_git: - logger.log(u"Checking if we can use git commands: {git} {cmd}".format - (git=cur_git, cmd=test_cmd), logger.DEBUG) - output, err, exit_status = self._run_git(cur_git, test_cmd) - - if exit_status == 0: - logger.log(u"Using: {git}".format(git=cur_git), logger.DEBUG) - return cur_git - else: - logger.log(u"Not using: {git}".format(git=cur_git), logger.DEBUG) - - # Still haven't found a working git - logger.debug('Unable to find your git executable - ' - 'Set git_path in your autoProcessMedia.cfg OR ' - 'delete your .git folder and run from source to enable updates.') - - return None - - def _run_git(self, git_path, args): - - output = None - err = None - - if not git_path: - logger.log(u"No git specified, can't use git commands", logger.DEBUG) - exit_status = 1 - return output, err, exit_status - - cmd = '{git} {args}'.format(git=git_path, args=args) - - try: - logger.log(u"Executing {cmd} with your shell in {directory}".format - (cmd=cmd, directory=core.PROGRAM_DIR), logger.DEBUG) - p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - shell=True, cwd=core.PROGRAM_DIR) - output, err = p.communicate() - exit_status = p.returncode - - if output: - output = output.strip() - if core.LOG_GIT: - logger.log(u"git output: {output}".format(output=output), logger.DEBUG) - - except OSError: - logger.log(u"Command {cmd} didn't work".format(cmd=cmd)) - exit_status = 1 - - exit_status = 128 if ('fatal:' in output) or err else exit_status - if exit_status == 0: - logger.log(u"{cmd} : returned successful".format(cmd=cmd), logger.DEBUG) - exit_status = 0 - elif core.LOG_GIT and exit_status in (1, 128): - logger.log(u"{cmd} returned : {output}".format - (cmd=cmd, output=output), logger.DEBUG) - else: - if core.LOG_GIT: - logger.log(u"{cmd} returned : {output}, treat as error for now".format - (cmd=cmd, output=output), logger.DEBUG) - exit_status = 1 - - return output, err, exit_status - - def _find_installed_version(self): - """ - Attempts to find the currently installed version of Sick Beard. - - Uses git show to get commit version. - - Returns: True for success or False for failure - """ - - output, err, exit_status = self._run_git(self._git_path, 'rev-parse HEAD') # @UnusedVariable - - if exit_status == 0 and output: - cur_commit_hash = output.strip() - if not re.match('^[a-z0-9]+$', cur_commit_hash): - logger.log(u"Output doesn't look like a hash, not using it", logger.ERROR) - return False - self._cur_commit_hash = cur_commit_hash - if self._cur_commit_hash: - core.NZBTOMEDIA_VERSION = self._cur_commit_hash - return True - else: - return False - - def _find_git_branch(self): - core.NZBTOMEDIA_BRANCH = self.get_github_branch() - branch_info, err, exit_status = self._run_git(self._git_path, 'symbolic-ref -q HEAD') # @UnusedVariable - if exit_status == 0 and branch_info: - branch = branch_info.strip().replace('refs/heads/', '', 1) - if branch: - core.NZBTOMEDIA_BRANCH = branch - core.GIT_BRANCH = branch - return core.GIT_BRANCH - - def _check_github_for_update(self): - """ - Uses git commands to check if there is a newer version that the provided - commit hash. If there is a newer version it sets _num_commits_behind. - """ - - self._newest_commit_hash = None - self._num_commits_behind = 0 - self._num_commits_ahead = 0 - - # get all new info from github - output, err, exit_status = self._run_git(self._git_path, 'fetch origin') - - if not exit_status == 0: - logger.log(u"Unable to contact github, can't check for update", logger.ERROR) - return - - # get latest commit_hash from remote - output, err, exit_status = self._run_git(self._git_path, 'rev-parse --verify --quiet "@{upstream}"') - - if exit_status == 0 and output: - cur_commit_hash = output.strip() - - if not re.match('^[a-z0-9]+$', cur_commit_hash): - logger.log(u"Output doesn't look like a hash, not using it", logger.DEBUG) - return - - else: - self._newest_commit_hash = cur_commit_hash - else: - logger.log(u"git didn't return newest commit hash", logger.DEBUG) - return - - # get number of commits behind and ahead (option --count not supported git < 1.7.2) - output, err, exit_status = self._run_git(self._git_path, 'rev-list --left-right "@{upstream}"...HEAD') - - if exit_status == 0 and output: - - try: - self._num_commits_behind = int(output.count("<")) - self._num_commits_ahead = int(output.count(">")) - - except: - logger.log(u"git didn't return numbers for behind and ahead, not using it", logger.DEBUG) - return - - logger.log(u"cur_commit = {current} % (newest_commit)= {new}, " - u"num_commits_behind = {x}, num_commits_ahead = {y}".format - (current=self._cur_commit_hash, new=self._newest_commit_hash, - x=self._num_commits_behind, y=self._num_commits_ahead), logger.DEBUG) - - def set_newest_text(self): - if self._num_commits_ahead: - logger.log(u"Local branch is ahead of {branch}. Automatic update not possible.".format - (branch=self.branch), logger.ERROR) - elif self._num_commits_behind: - logger.log(u"There is a newer version available (you're {x} commit{s} behind)".format - (x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE) - else: - return - - def need_update(self): - if not self._find_installed_version(): - logger.error("Unable to determine installed version via git, please check your logs!") - return False - - if not self._cur_commit_hash: - return True - else: - try: - self._check_github_for_update() - except Exception as error: - logger.log(u"Unable to contact github, can't check for update: {msg!r}".format(msg=error), logger.ERROR) - return False - - if self._num_commits_behind > 0: - return True - - return False - - def update(self): - """ - Calls git pull origin in order to update Sick Beard. Returns a bool depending - on the call's success. - """ - - output, err, exit_status = self._run_git(self._git_path, 'pull origin {branch}'.format(branch=self.branch)) # @UnusedVariable - - if exit_status == 0: - return True - - return False - - -class SourceUpdateManager(UpdateManager): - def __init__(self): - self.github_repo_user = self.get_github_repo_user() - self.github_repo = self.get_github_repo() - self.branch = self.get_github_branch() - - self._cur_commit_hash = None - self._newest_commit_hash = None - self._num_commits_behind = 0 - - def _find_installed_version(self): - - version_file = os.path.join(core.PROGRAM_DIR, u'version.txt') - - if not os.path.isfile(version_file): - self._cur_commit_hash = None - return - - try: - with open(version_file, 'r') as fp: - self._cur_commit_hash = fp.read().strip(' \n\r') - except EnvironmentError as error: - logger.log(u"Unable to open 'version.txt': {msg}".format(msg=error), logger.DEBUG) - - if not self._cur_commit_hash: - self._cur_commit_hash = None - else: - core.NZBTOMEDIA_VERSION = self._cur_commit_hash - - def need_update(self): - - self._find_installed_version() - - try: - self._check_github_for_update() - except Exception as error: - logger.log(u"Unable to contact github, can't check for update: {msg!r}".format(msg=error), logger.ERROR) - return False - - if not self._cur_commit_hash or self._num_commits_behind > 0: - return True - - return False - - def _check_github_for_update(self): - """ - Uses pygithub to ask github if there is a newer version that the provided - commit hash. If there is a newer version it sets Sick Beard's version text. - - commit_hash: hash that we're checking against - """ - - self._num_commits_behind = 0 - self._newest_commit_hash = None - - gh = github.GitHub(self.github_repo_user, self.github_repo, self.branch) - - # try to get newest commit hash and commits behind directly by comparing branch and current commit - if self._cur_commit_hash: - branch_compared = gh.compare(base=self.branch, head=self._cur_commit_hash) - - if 'base_commit' in branch_compared: - self._newest_commit_hash = branch_compared['base_commit']['sha'] - - if 'behind_by' in branch_compared: - self._num_commits_behind = int(branch_compared['behind_by']) - - # fall back and iterate over last 100 (items per page in gh_api) commits - if not self._newest_commit_hash: - - for curCommit in gh.commits(): - if not self._newest_commit_hash: - self._newest_commit_hash = curCommit['sha'] - if not self._cur_commit_hash: - break - - if curCommit['sha'] == self._cur_commit_hash: - break - - # when _cur_commit_hash doesn't match anything _num_commits_behind == 100 - self._num_commits_behind += 1 - - logger.log(u"cur_commit = {current} % (newest_commit)= {new}, num_commits_behind = {x}".format - (current=self._cur_commit_hash, new=self._newest_commit_hash, x=self._num_commits_behind), logger.DEBUG) - - def set_newest_text(self): - - # if we're up to date then don't set this - core.NEWEST_VERSION_STRING = None - - if not self._cur_commit_hash: - logger.log(u"Unknown current version number, don't know if we should update or not", logger.ERROR) - elif self._num_commits_behind > 0: - logger.log(u"There is a newer version available (you're {x} commit{s} behind)".format - (x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE) - else: - return - - def update(self): - """ - Downloads the latest source tarball from github and installs it over the existing version. - """ - tar_download_url = 'https://github.com/{org}/{repo}/tarball/{branch}'.format( - org=self.github_repo_user, repo=self.github_repo, branch=self.branch) - version_path = os.path.join(core.PROGRAM_DIR, u'version.txt') - - try: - # prepare the update dir - sb_update_dir = os.path.join(core.PROGRAM_DIR, u'sb-update') - - if os.path.isdir(sb_update_dir): - logger.log(u"Clearing out update folder {dir} before extracting".format(dir=sb_update_dir)) - shutil.rmtree(sb_update_dir) - - logger.log(u"Creating update folder {dir} before extracting".format(dir=sb_update_dir)) - os.makedirs(sb_update_dir) - - # retrieve file - logger.log(u"Downloading update from {url!r}".format(url=tar_download_url)) - tar_download_path = os.path.join(sb_update_dir, u'nzbtomedia-update.tar') - urllib.urlretrieve(tar_download_url, tar_download_path) - - if not os.path.isfile(tar_download_path): - logger.log(u"Unable to retrieve new version from {url}, can't update".format - (url=tar_download_url), logger.ERROR) - return False - - if not tarfile.is_tarfile(tar_download_path): - logger.log(u"Retrieved version from {url} is corrupt, can't update".format - (url=tar_download_url), logger.ERROR) - return False - - # extract to sb-update dir - logger.log(u"Extracting file {path}".format(path=tar_download_path)) - tar = tarfile.open(tar_download_path) - tar.extractall(sb_update_dir) - tar.close() - - # delete .tar.gz - logger.log(u"Deleting file {path}".format(path=tar_download_path)) - os.remove(tar_download_path) - - # find update dir name - update_dir_contents = [x for x in os.listdir(sb_update_dir) if - os.path.isdir(os.path.join(sb_update_dir, x))] - if len(update_dir_contents) != 1: - logger.log(u"Invalid update data, update failed: {0}".format(update_dir_contents), logger.ERROR) - return False - content_dir = os.path.join(sb_update_dir, update_dir_contents[0]) - - # walk temp folder and move files to main folder - logger.log(u"Moving files from {source} to {destination}".format - (source=content_dir, destination=core.PROGRAM_DIR)) - for dirname, dirnames, filenames in os.walk(content_dir): # @UnusedVariable - dirname = dirname[len(content_dir) + 1:] - for curfile in filenames: - old_path = os.path.join(content_dir, dirname, curfile) - new_path = os.path.join(core.PROGRAM_DIR, dirname, curfile) - - # Avoid DLL access problem on WIN32/64 - # These files needing to be updated manually - # or find a way to kill the access from memory - if curfile in ('unrar.dll', 'unrar64.dll'): - try: - os.chmod(new_path, stat.S_IWRITE) - os.remove(new_path) - os.renames(old_path, new_path) - except Exception as error: - logger.log(u"Unable to update {path}: {msg}".format - (path=new_path, msg=error), logger.DEBUG) - os.remove(old_path) # Trash the updated file without moving in new path - continue - - if os.path.isfile(new_path): - os.remove(new_path) - os.renames(old_path, new_path) - - # update version.txt with commit hash - try: - with open(version_path, 'w') as ver_file: - ver_file.write(self._newest_commit_hash) - except EnvironmentError as error: - logger.log(u"Unable to write version file, update not complete: {msg}".format - (msg=error), logger.ERROR) - return False - - except Exception as error: - logger.log(u"Error while trying to update: {msg}".format - (msg=error), logger.ERROR) - logger.log(u"Traceback: {error}".format(error=traceback.format_exc()), logger.DEBUG) - return False - - return True diff --git a/core/version_check.py b/core/version_check.py new file mode 100644 index 000000000..a414f4b92 --- /dev/null +++ b/core/version_check.py @@ -0,0 +1,531 @@ +# coding=utf-8 +# Author: Nic Wolfe +# Modified by: echel0n + +import os +import platform +import re +import shutil +import stat +import subprocess +import tarfile +import traceback + +from six.moves.urllib.request import urlretrieve + +import cleanup +import core +from core import github_api as github, logger + + +class CheckVersion(object): + """ + Version check class meant to run as a thread object with the SB scheduler. + """ + + def __init__(self): + self.install_type = self.find_install_type() + self.installed_version = None + self.installed_branch = None + + if self.install_type == 'git': + self.updater = GitUpdateManager() + elif self.install_type == 'source': + self.updater = SourceUpdateManager() + else: + self.updater = None + + def run(self): + self.check_for_new_version() + + def find_install_type(self): + """ + Determines how this copy of SB was installed. + + returns: type of installation. Possible values are: + 'win': any compiled windows build + 'git': running from source using git + 'source': running from source without git + """ + + # check if we're a windows build + if os.path.isdir(os.path.join(core.APP_ROOT, u'.git')): + install_type = 'git' + else: + install_type = 'source' + + return install_type + + def check_for_new_version(self, force=False): + """ + Checks the internet for a newer version. + + returns: bool, True for new version or False for no new version. + + force: if true the VERSION_NOTIFY setting will be ignored and a check will be forced + """ + + if not core.VERSION_NOTIFY and not force: + logger.log(u'Version checking is disabled, not checking for the newest version') + return False + + logger.log(u'Checking if {install} needs an update'.format(install=self.install_type)) + if not self.updater.need_update(): + core.NEWEST_VERSION_STRING = None + logger.log(u'No update needed') + return False + + self.updater.set_newest_text() + return True + + def update(self): + if self.updater.need_update(): + result = self.updater.update() + cleanup.clean('core', 'libs') + return result + + +class UpdateManager(object): + def get_github_repo_user(self): + return core.GIT_USER + + def get_github_repo(self): + return core.GIT_REPO + + def get_github_branch(self): + return core.GIT_BRANCH + + +class GitUpdateManager(UpdateManager): + def __init__(self): + self._git_path = self._find_working_git() + self.github_repo_user = self.get_github_repo_user() + self.github_repo = self.get_github_repo() + self.branch = self._find_git_branch() + + self._cur_commit_hash = None + self._newest_commit_hash = None + self._num_commits_behind = 0 + self._num_commits_ahead = 0 + + def _git_error(self): + logger.debug( + 'Unable to find your git executable - Set git_path in your autoProcessMedia.cfg OR delete your .git folder and run from source to enable updates.') + + def _find_working_git(self): + test_cmd = 'version' + + if core.GIT_PATH: + main_git = '\'{git}\''.format(git=core.GIT_PATH) + else: + main_git = 'git' + + logger.log(u'Checking if we can use git commands: {git} {cmd}'.format + (git=main_git, cmd=test_cmd), logger.DEBUG) + output, err, exit_status = self._run_git(main_git, test_cmd) + + if exit_status == 0: + logger.log(u'Using: {git}'.format(git=main_git), logger.DEBUG) + return main_git + else: + logger.log(u'Not using: {git}'.format(git=main_git), logger.DEBUG) + + # trying alternatives + + alternative_git = [] + + # osx people who start SB from launchd have a broken path, so try a hail-mary attempt for them + if platform.system().lower() == 'darwin': + alternative_git.append('/usr/local/git/bin/git') + + if platform.system().lower() == 'windows': + if main_git != main_git.lower(): + alternative_git.append(main_git.lower()) + + if alternative_git: + logger.log(u'Trying known alternative git locations', logger.DEBUG) + + for cur_git in alternative_git: + logger.log(u'Checking if we can use git commands: {git} {cmd}'.format + (git=cur_git, cmd=test_cmd), logger.DEBUG) + output, err, exit_status = self._run_git(cur_git, test_cmd) + + if exit_status == 0: + logger.log(u'Using: {git}'.format(git=cur_git), logger.DEBUG) + return cur_git + else: + logger.log(u'Not using: {git}'.format(git=cur_git), logger.DEBUG) + + # Still haven't found a working git + logger.debug('Unable to find your git executable - ' + 'Set git_path in your autoProcessMedia.cfg OR ' + 'delete your .git folder and run from source to enable updates.') + + return None + + def _run_git(self, git_path, args): + + output = None + err = None + + if not git_path: + logger.log(u'No git specified, can\'t use git commands', logger.DEBUG) + exit_status = 1 + return output, err, exit_status + + cmd = '{git} {args}'.format(git=git_path, args=args) + + try: + logger.log(u'Executing {cmd} with your shell in {directory}'.format + (cmd=cmd, directory=core.APP_ROOT), logger.DEBUG) + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + shell=True, cwd=core.APP_ROOT) + output, err = p.communicate() + exit_status = p.returncode + + output = output.decode('utf-8') + + if output: + output = output.strip() + if core.LOG_GIT: + logger.log(u'git output: {output}'.format(output=output), logger.DEBUG) + + except OSError: + logger.log(u'Command {cmd} didn\'t work'.format(cmd=cmd)) + exit_status = 1 + + exit_status = 128 if ('fatal:' in output) or err else exit_status + if exit_status == 0: + logger.log(u'{cmd} : returned successful'.format(cmd=cmd), logger.DEBUG) + exit_status = 0 + elif core.LOG_GIT and exit_status in (1, 128): + logger.log(u'{cmd} returned : {output}'.format + (cmd=cmd, output=output), logger.DEBUG) + else: + if core.LOG_GIT: + logger.log(u'{cmd} returned : {output}, treat as error for now'.format + (cmd=cmd, output=output), logger.DEBUG) + exit_status = 1 + + return output, err, exit_status + + def _find_installed_version(self): + """ + Attempts to find the currently installed version of Sick Beard. + + Uses git show to get commit version. + + Returns: True for success or False for failure + """ + + output, err, exit_status = self._run_git(self._git_path, 'rev-parse HEAD') # @UnusedVariable + + if exit_status == 0 and output: + cur_commit_hash = output.strip() + if not re.match('^[a-z0-9]+$', cur_commit_hash): + logger.log(u'Output doesn\'t look like a hash, not using it', logger.ERROR) + return False + self._cur_commit_hash = cur_commit_hash + if self._cur_commit_hash: + core.NZBTOMEDIA_VERSION = self._cur_commit_hash + return True + else: + return False + + def _find_git_branch(self): + core.NZBTOMEDIA_BRANCH = self.get_github_branch() + branch_info, err, exit_status = self._run_git(self._git_path, 'symbolic-ref -q HEAD') # @UnusedVariable + if exit_status == 0 and branch_info: + branch = branch_info.strip().replace('refs/heads/', '', 1) + if branch: + core.NZBTOMEDIA_BRANCH = branch + core.GIT_BRANCH = branch + return core.GIT_BRANCH + + def _check_github_for_update(self): + """ + Uses git commands to check if there is a newer version that the provided + commit hash. If there is a newer version it sets _num_commits_behind. + """ + + self._newest_commit_hash = None + self._num_commits_behind = 0 + self._num_commits_ahead = 0 + + # get all new info from github + output, err, exit_status = self._run_git(self._git_path, 'fetch origin') + + if not exit_status == 0: + logger.log(u'Unable to contact github, can\'t check for update', logger.ERROR) + return + + # get latest commit_hash from remote + output, err, exit_status = self._run_git(self._git_path, 'rev-parse --verify --quiet \'@{upstream}\'') + + if exit_status == 0 and output: + cur_commit_hash = output.strip() + + if not re.match('^[a-z0-9]+$', cur_commit_hash): + logger.log(u'Output doesn\'t look like a hash, not using it', logger.DEBUG) + return + + else: + self._newest_commit_hash = cur_commit_hash + else: + logger.log(u'git didn\'t return newest commit hash', logger.DEBUG) + return + + # get number of commits behind and ahead (option --count not supported git < 1.7.2) + output, err, exit_status = self._run_git(self._git_path, 'rev-list --left-right \'@{upstream}\'...HEAD') + + if exit_status == 0 and output: + + try: + self._num_commits_behind = int(output.count('<')) + self._num_commits_ahead = int(output.count('>')) + + except Exception: + logger.log(u'git didn\'t return numbers for behind and ahead, not using it', logger.DEBUG) + return + + logger.log(u'cur_commit = {current} % (newest_commit)= {new}, ' + u'num_commits_behind = {x}, num_commits_ahead = {y}'.format + (current=self._cur_commit_hash, new=self._newest_commit_hash, + x=self._num_commits_behind, y=self._num_commits_ahead), logger.DEBUG) + + def set_newest_text(self): + if self._num_commits_ahead: + logger.log(u'Local branch is ahead of {branch}. Automatic update not possible.'.format + (branch=self.branch), logger.ERROR) + elif self._num_commits_behind: + logger.log(u'There is a newer version available (you\'re {x} commit{s} behind)'.format + (x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE) + else: + return + + def need_update(self): + if not self._find_installed_version(): + logger.error('Unable to determine installed version via git, please check your logs!') + return False + + if not self._cur_commit_hash: + return True + else: + try: + self._check_github_for_update() + except Exception as error: + logger.log(u'Unable to contact github, can\'t check for update: {msg!r}'.format(msg=error), logger.ERROR) + return False + + if self._num_commits_behind > 0: + return True + + return False + + def update(self): + """ + Calls git pull origin in order to update Sick Beard. Returns a bool depending + on the call's success. + """ + + output, err, exit_status = self._run_git(self._git_path, 'pull origin {branch}'.format(branch=self.branch)) # @UnusedVariable + + if exit_status == 0: + return True + + return False + + +class SourceUpdateManager(UpdateManager): + def __init__(self): + self.github_repo_user = self.get_github_repo_user() + self.github_repo = self.get_github_repo() + self.branch = self.get_github_branch() + + self._cur_commit_hash = None + self._newest_commit_hash = None + self._num_commits_behind = 0 + + def _find_installed_version(self): + + version_file = os.path.join(core.APP_ROOT, u'version.txt') + + if not os.path.isfile(version_file): + self._cur_commit_hash = None + return + + try: + with open(version_file, 'r') as fp: + self._cur_commit_hash = fp.read().strip(' \n\r') + except EnvironmentError as error: + logger.log(u'Unable to open \'version.txt\': {msg}'.format(msg=error), logger.DEBUG) + + if not self._cur_commit_hash: + self._cur_commit_hash = None + else: + core.NZBTOMEDIA_VERSION = self._cur_commit_hash + + def need_update(self): + + self._find_installed_version() + + try: + self._check_github_for_update() + except Exception as error: + logger.log(u'Unable to contact github, can\'t check for update: {msg!r}'.format(msg=error), logger.ERROR) + return False + + if not self._cur_commit_hash or self._num_commits_behind > 0: + return True + + return False + + def _check_github_for_update(self): + """ + Uses pygithub to ask github if there is a newer version that the provided + commit hash. If there is a newer version it sets Sick Beard's version text. + + commit_hash: hash that we're checking against + """ + + self._num_commits_behind = 0 + self._newest_commit_hash = None + + gh = github.GitHub(self.github_repo_user, self.github_repo, self.branch) + + # try to get newest commit hash and commits behind directly by comparing branch and current commit + if self._cur_commit_hash: + branch_compared = gh.compare(base=self.branch, head=self._cur_commit_hash) + + if 'base_commit' in branch_compared: + self._newest_commit_hash = branch_compared['base_commit']['sha'] + + if 'behind_by' in branch_compared: + self._num_commits_behind = int(branch_compared['behind_by']) + + # fall back and iterate over last 100 (items per page in gh_api) commits + if not self._newest_commit_hash: + + for curCommit in gh.commits(): + if not self._newest_commit_hash: + self._newest_commit_hash = curCommit['sha'] + if not self._cur_commit_hash: + break + + if curCommit['sha'] == self._cur_commit_hash: + break + + # when _cur_commit_hash doesn't match anything _num_commits_behind == 100 + self._num_commits_behind += 1 + + logger.log(u'cur_commit = {current} % (newest_commit)= {new}, num_commits_behind = {x}'.format + (current=self._cur_commit_hash, new=self._newest_commit_hash, x=self._num_commits_behind), logger.DEBUG) + + def set_newest_text(self): + + # if we're up to date then don't set this + core.NEWEST_VERSION_STRING = None + + if not self._cur_commit_hash: + logger.log(u'Unknown current version number, don\'t know if we should update or not', logger.ERROR) + elif self._num_commits_behind > 0: + logger.log(u'There is a newer version available (you\'re {x} commit{s} behind)'.format + (x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE) + else: + return + + def update(self): + """ + Downloads the latest source tarball from github and installs it over the existing version. + """ + tar_download_url = 'https://github.com/{org}/{repo}/tarball/{branch}'.format( + org=self.github_repo_user, repo=self.github_repo, branch=self.branch) + version_path = os.path.join(core.APP_ROOT, u'version.txt') + + try: + # prepare the update dir + sb_update_dir = os.path.join(core.APP_ROOT, u'sb-update') + + if os.path.isdir(sb_update_dir): + logger.log(u'Clearing out update folder {dir} before extracting'.format(dir=sb_update_dir)) + shutil.rmtree(sb_update_dir) + + logger.log(u'Creating update folder {dir} before extracting'.format(dir=sb_update_dir)) + os.makedirs(sb_update_dir) + + # retrieve file + logger.log(u'Downloading update from {url!r}'.format(url=tar_download_url)) + tar_download_path = os.path.join(sb_update_dir, u'nzbtomedia-update.tar') + urlretrieve(tar_download_url, tar_download_path) + + if not os.path.isfile(tar_download_path): + logger.log(u'Unable to retrieve new version from {url}, can\'t update'.format + (url=tar_download_url), logger.ERROR) + return False + + if not tarfile.is_tarfile(tar_download_path): + logger.log(u'Retrieved version from {url} is corrupt, can\'t update'.format + (url=tar_download_url), logger.ERROR) + return False + + # extract to sb-update dir + logger.log(u'Extracting file {path}'.format(path=tar_download_path)) + tar = tarfile.open(tar_download_path) + tar.extractall(sb_update_dir) + tar.close() + + # delete .tar.gz + logger.log(u'Deleting file {path}'.format(path=tar_download_path)) + os.remove(tar_download_path) + + # find update dir name + update_dir_contents = [x for x in os.listdir(sb_update_dir) if + os.path.isdir(os.path.join(sb_update_dir, x))] + if len(update_dir_contents) != 1: + logger.log(u'Invalid update data, update failed: {0}'.format(update_dir_contents), logger.ERROR) + return False + content_dir = os.path.join(sb_update_dir, update_dir_contents[0]) + + # walk temp folder and move files to main folder + logger.log(u'Moving files from {source} to {destination}'.format + (source=content_dir, destination=core.APP_ROOT)) + for dirname, dirnames, filenames in os.walk(content_dir): # @UnusedVariable + dirname = dirname[len(content_dir) + 1:] + for curfile in filenames: + old_path = os.path.join(content_dir, dirname, curfile) + new_path = os.path.join(core.APP_ROOT, dirname, curfile) + + # Avoid DLL access problem on WIN32/64 + # These files needing to be updated manually + # or find a way to kill the access from memory + if curfile in ('unrar.dll', 'unrar64.dll'): + try: + os.chmod(new_path, stat.S_IWRITE) + os.remove(new_path) + os.renames(old_path, new_path) + except Exception as error: + logger.log(u'Unable to update {path}: {msg}'.format + (path=new_path, msg=error), logger.DEBUG) + os.remove(old_path) # Trash the updated file without moving in new path + continue + + if os.path.isfile(new_path): + os.remove(new_path) + os.renames(old_path, new_path) + + # update version.txt with commit hash + try: + with open(version_path, 'w') as ver_file: + ver_file.write(self._newest_commit_hash) + except EnvironmentError as error: + logger.log(u'Unable to write version file, update not complete: {msg}'.format + (msg=error), logger.ERROR) + return False + + except Exception as error: + logger.log(u'Error while trying to update: {msg}'.format + (msg=error), logger.ERROR) + logger.log(u'Traceback: {error}'.format(error=traceback.format_exc()), logger.DEBUG) + return False + + return True diff --git a/libs/__init__.py b/libs/__init__.py index e69de29bb..4405201d8 100644 --- a/libs/__init__.py +++ b/libs/__init__.py @@ -0,0 +1,50 @@ + +import os +import site +import sys + +import libs.util + +LIB_ROOT = libs.util.module_path() + +COMMON = 'common' +CUSTOM = 'custom' +PY2 = 'py2' +WIN = 'win' + +LOADED = {} +MANDATORY = { + COMMON, + CUSTOM, +} +DIRECTORY = { + lib: os.path.join(LIB_ROOT, lib) + for lib in [COMMON, CUSTOM, PY2, WIN] +} + +if sys.platform == 'win32': + MANDATORY.add(WIN) + +if sys.version_info < (3, ): + MANDATORY.add(PY2) + + +def add_libs(name): + if name in MANDATORY and name not in LOADED: + path = libs.util.add_path(DIRECTORY[name]) + if path: + site.addsitedir(path) + LOADED[name] = path + return path + + +def add_all_libs(): + for lib in [COMMON, CUSTOM, PY2, WIN]: + if lib not in MANDATORY: + continue + add_libs(lib) + return is_finished() + + +def is_finished(): + return MANDATORY.issubset(LOADED.keys()) diff --git a/libs/__main__.py b/libs/__main__.py new file mode 100644 index 000000000..767f0d6ed --- /dev/null +++ b/libs/__main__.py @@ -0,0 +1,20 @@ + +import shutil +import os +import time +import libs + +if __name__ == '__main__': + os.chdir(libs.LIB_ROOT) + for lib, directory in libs.DIRECTORY.items(): + if lib == 'custom': + continue + try: + shutil.rmtree(directory) + except FileNotFoundError: + pass + else: + print('Removed', directory) + time.sleep(10) + requirements = 'requirements-{name}.txt'.format(name=lib) + libs.util.install_requirements(requirements, file=True, path=directory) diff --git a/libs/autoload.py b/libs/autoload.py new file mode 100644 index 000000000..232054486 --- /dev/null +++ b/libs/autoload.py @@ -0,0 +1,6 @@ + +import libs + +__all__ = ['completed'] + +completed = libs.add_all_libs() diff --git a/libs/babelfish/__init__.py b/libs/babelfish/__init__.py deleted file mode 100644 index 205254a58..000000000 --- a/libs/babelfish/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (c) 2013 the BabelFish authors. All rights reserved. -# Use of this source code is governed by the 3-clause BSD license -# that can be found in the LICENSE file. -# -__title__ = 'babelfish' -__version__ = '0.5.4-dev' -__author__ = 'Antoine Bertin' -__license__ = 'BSD' -__copyright__ = 'Copyright 2013 the BabelFish authors' - -import sys - -if sys.version_info[0] >= 3: - basestr = str -else: - basestr = basestring - -from .converters import (LanguageConverter, LanguageReverseConverter, LanguageEquivalenceConverter, CountryConverter, - CountryReverseConverter) -from .country import country_converters, COUNTRIES, COUNTRY_MATRIX, Country -from .exceptions import Error, LanguageConvertError, LanguageReverseError, CountryConvertError, CountryReverseError -from .language import language_converters, LANGUAGES, LANGUAGE_MATRIX, Language -from .script import SCRIPTS, SCRIPT_MATRIX, Script diff --git a/libs/babelfish/converters/__init__.py b/libs/babelfish/converters/__init__.py deleted file mode 100644 index 9a0a1bd90..000000000 --- a/libs/babelfish/converters/__init__.py +++ /dev/null @@ -1,280 +0,0 @@ -# Copyright (c) 2013 the BabelFish authors. All rights reserved. -# Use of this source code is governed by the 3-clause BSD license -# that can be found in the LICENSE file. -# -import collections -from pkg_resources import iter_entry_points, EntryPoint -from ..exceptions import LanguageConvertError, LanguageReverseError - - -# from https://github.com/kennethreitz/requests/blob/master/requests/structures.py -class CaseInsensitiveDict(collections.MutableMapping): - """A case-insensitive ``dict``-like object. - - Implements all methods and operations of - ``collections.MutableMapping`` as well as dict's ``copy``. Also - provides ``lower_items``. - - All keys are expected to be strings. The structure remembers the - case of the last key to be set, and ``iter(instance)``, - ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` - will contain case-sensitive keys. However, querying and contains - testing is case insensitive: - - cid = CaseInsensitiveDict() - cid['English'] = 'eng' - cid['ENGLISH'] == 'eng' # True - list(cid) == ['English'] # True - - If the constructor, ``.update``, or equality comparison - operations are given keys that have equal ``.lower()``s, the - behavior is undefined. - - """ - def __init__(self, data=None, **kwargs): - self._store = dict() - if data is None: - data = {} - self.update(data, **kwargs) - - def __setitem__(self, key, value): - # Use the lowercased key for lookups, but store the actual - # key alongside the value. - self._store[key.lower()] = (key, value) - - def __getitem__(self, key): - return self._store[key.lower()][1] - - def __delitem__(self, key): - del self._store[key.lower()] - - def __iter__(self): - return (casedkey for casedkey, mappedvalue in self._store.values()) - - def __len__(self): - return len(self._store) - - def lower_items(self): - """Like iteritems(), but with all lowercase keys.""" - return ( - (lowerkey, keyval[1]) - for (lowerkey, keyval) - in self._store.items() - ) - - def __eq__(self, other): - if isinstance(other, collections.Mapping): - other = CaseInsensitiveDict(other) - else: - return NotImplemented - # Compare insensitively - return dict(self.lower_items()) == dict(other.lower_items()) - - # Copy is required - def copy(self): - return CaseInsensitiveDict(self._store.values()) - - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, dict(self.items())) - - -class LanguageConverter(object): - """A :class:`LanguageConverter` supports converting an alpha3 language code with an - alpha2 country code and a script code into a custom code - - .. attribute:: codes - - Set of possible custom codes - - """ - def convert(self, alpha3, country=None, script=None): - """Convert an alpha3 language code with an alpha2 country code and a script code - into a custom code - - :param string alpha3: ISO-639-3 language code - :param country: ISO-3166 country code, if any - :type country: string or None - :param script: ISO-15924 script code, if any - :type script: string or None - :return: the corresponding custom code - :rtype: string - :raise: :class:`~babelfish.exceptions.LanguageConvertError` - - """ - raise NotImplementedError - - -class LanguageReverseConverter(LanguageConverter): - """A :class:`LanguageConverter` able to reverse a custom code into a alpha3 - ISO-639-3 language code, alpha2 ISO-3166-1 country code and ISO-15924 script code - - """ - def reverse(self, code): - """Reverse a custom code into alpha3, country and script code - - :param string code: custom code to reverse - :return: the corresponding alpha3 ISO-639-3 language code, alpha2 ISO-3166-1 country code and ISO-15924 script code - :rtype: tuple - :raise: :class:`~babelfish.exceptions.LanguageReverseError` - - """ - raise NotImplementedError - - -class LanguageEquivalenceConverter(LanguageReverseConverter): - """A :class:`LanguageEquivalenceConverter` is a utility class that allows you to easily define a - :class:`LanguageReverseConverter` by only specifying the dict from alpha3 to their corresponding symbols. - - You must specify the dict of equivalence as a class variable named SYMBOLS. - - If you also set the class variable CASE_SENSITIVE to ``True`` then the reverse conversion function will be - case-sensitive (it is case-insensitive by default). - - Example:: - - class MyCodeConverter(babelfish.LanguageEquivalenceConverter): - CASE_SENSITIVE = True - SYMBOLS = {'fra': 'mycode1', 'eng': 'mycode2'} - - """ - CASE_SENSITIVE = False - - def __init__(self): - self.codes = set() - self.to_symbol = {} - if self.CASE_SENSITIVE: - self.from_symbol = {} - else: - self.from_symbol = CaseInsensitiveDict() - - for alpha3, symbol in self.SYMBOLS.items(): - self.to_symbol[alpha3] = symbol - self.from_symbol[symbol] = (alpha3, None, None) - self.codes.add(symbol) - - def convert(self, alpha3, country=None, script=None): - try: - return self.to_symbol[alpha3] - except KeyError: - raise LanguageConvertError(alpha3, country, script) - - def reverse(self, code): - try: - return self.from_symbol[code] - except KeyError: - raise LanguageReverseError(code) - - -class CountryConverter(object): - """A :class:`CountryConverter` supports converting an alpha2 country code - into a custom code - - .. attribute:: codes - - Set of possible custom codes - - """ - def convert(self, alpha2): - """Convert an alpha2 country code into a custom code - - :param string alpha2: ISO-3166-1 language code - :return: the corresponding custom code - :rtype: string - :raise: :class:`~babelfish.exceptions.CountryConvertError` - - """ - raise NotImplementedError - - -class CountryReverseConverter(CountryConverter): - """A :class:`CountryConverter` able to reverse a custom code into a alpha2 - ISO-3166-1 country code - - """ - def reverse(self, code): - """Reverse a custom code into alpha2 code - - :param string code: custom code to reverse - :return: the corresponding alpha2 ISO-3166-1 country code - :rtype: string - :raise: :class:`~babelfish.exceptions.CountryReverseError` - - """ - raise NotImplementedError - - -class ConverterManager(object): - """Manager for babelfish converters behaving like a dict with lazy loading - - Loading is done in this order: - - * Entry point converters - * Registered converters - * Internal converters - - .. attribute:: entry_point - - The entry point where to look for converters - - .. attribute:: internal_converters - - Internal converters with entry point syntax - - """ - entry_point = '' - internal_converters = [] - - def __init__(self): - #: Registered converters with entry point syntax - self.registered_converters = [] - - #: Loaded converters - self.converters = {} - - def __getitem__(self, name): - """Get a converter, lazy loading it if necessary""" - if name in self.converters: - return self.converters[name] - for ep in iter_entry_points(self.entry_point): - if ep.name == name: - self.converters[ep.name] = ep.load()() - return self.converters[ep.name] - for ep in (EntryPoint.parse(c) for c in self.registered_converters + self.internal_converters): - if ep.name == name: - self.converters[ep.name] = ep.load(require=False)() - return self.converters[ep.name] - raise KeyError(name) - - def __setitem__(self, name, converter): - """Load a converter""" - self.converters[name] = converter - - def __delitem__(self, name): - """Unload a converter""" - del self.converters[name] - - def __iter__(self): - """Iterator over loaded converters""" - return iter(self.converters) - - def register(self, entry_point): - """Register a converter - - :param string entry_point: converter to register (entry point syntax) - :raise: ValueError if already registered - - """ - if entry_point in self.registered_converters: - raise ValueError('Already registered') - self.registered_converters.insert(0, entry_point) - - def unregister(self, entry_point): - """Unregister a converter - - :param string entry_point: converter to unregister (entry point syntax) - - """ - self.registered_converters.remove(entry_point) - - def __contains__(self, name): - return name in self.converters diff --git a/libs/babelfish/country.py b/libs/babelfish/country.py deleted file mode 100644 index ce32d9b50..000000000 --- a/libs/babelfish/country.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (c) 2013 the BabelFish authors. All rights reserved. -# Use of this source code is governed by the 3-clause BSD license -# that can be found in the LICENSE file. -# -from __future__ import unicode_literals -from collections import namedtuple -from functools import partial -from pkg_resources import resource_stream # @UnresolvedImport -from .converters import ConverterManager -from . import basestr - - -COUNTRIES = {} -COUNTRY_MATRIX = [] - -#: The namedtuple used in the :data:`COUNTRY_MATRIX` -IsoCountry = namedtuple('IsoCountry', ['name', 'alpha2']) - -f = resource_stream('babelfish', 'data/iso-3166-1.txt') -f.readline() -for l in f: - iso_country = IsoCountry(*l.decode('utf-8').strip().split(';')) - COUNTRIES[iso_country.alpha2] = iso_country.name - COUNTRY_MATRIX.append(iso_country) -f.close() - - -class CountryConverterManager(ConverterManager): - """:class:`~babelfish.converters.ConverterManager` for country converters""" - entry_point = 'babelfish.country_converters' - internal_converters = ['name = babelfish.converters.countryname:CountryNameConverter'] - -country_converters = CountryConverterManager() - - -class CountryMeta(type): - """The :class:`Country` metaclass - - Dynamically redirect :meth:`Country.frommycode` to :meth:`Country.fromcode` with the ``mycode`` `converter` - - """ - def __getattr__(cls, name): - if name.startswith('from'): - return partial(cls.fromcode, converter=name[4:]) - return type.__getattribute__(cls, name) - - -class Country(CountryMeta(str('CountryBase'), (object,), {})): - """A country on Earth - - A country is represented by a 2-letter code from the ISO-3166 standard - - :param string country: 2-letter ISO-3166 country code - - """ - def __init__(self, country): - if country not in COUNTRIES: - raise ValueError('%r is not a valid country' % country) - - #: ISO-3166 2-letter country code - self.alpha2 = country - - @classmethod - def fromcode(cls, code, converter): - """Create a :class:`Country` by its `code` using `converter` to - :meth:`~babelfish.converters.CountryReverseConverter.reverse` it - - :param string code: the code to reverse - :param string converter: name of the :class:`~babelfish.converters.CountryReverseConverter` to use - :return: the corresponding :class:`Country` instance - :rtype: :class:`Country` - - """ - return cls(country_converters[converter].reverse(code)) - - def __getstate__(self): - return self.alpha2 - - def __setstate__(self, state): - self.alpha2 = state - - def __getattr__(self, name): - return country_converters[name].convert(self.alpha2) - - def __hash__(self): - return hash(self.alpha2) - - def __eq__(self, other): - if isinstance(other, basestr): - return str(self) == other - if not isinstance(other, Country): - return False - return self.alpha2 == other.alpha2 - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return '' % self - - def __str__(self): - return self.alpha2 diff --git a/libs/babelfish/data/get_files.py b/libs/babelfish/data/get_files.py deleted file mode 100644 index aaa090ccc..000000000 --- a/libs/babelfish/data/get_files.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2013 the BabelFish authors. All rights reserved. -# Use of this source code is governed by the 3-clause BSD license -# that can be found in the LICENSE file. -# -from __future__ import unicode_literals -import os.path -import tempfile -import zipfile -import requests - - -DATA_DIR = os.path.dirname(__file__) - -# iso-3166-1.txt -print('Downloading ISO-3166-1 standard (ISO country codes)...') -with open(os.path.join(DATA_DIR, 'iso-3166-1.txt'), 'w') as f: - r = requests.get('http://www.iso.org/iso/home/standards/country_codes/country_names_and_code_elements_txt.htm') - f.write(r.content.strip()) - -# iso-639-3.tab -print('Downloading ISO-639-3 standard (ISO language codes)...') -with tempfile.TemporaryFile() as f: - r = requests.get('http://www-01.sil.org/iso639-3/iso-639-3_Code_Tables_20130531.zip') - f.write(r.content) - with zipfile.ZipFile(f) as z: - z.extract('iso-639-3.tab', DATA_DIR) - -# iso-15924 -print('Downloading ISO-15924 standard (ISO script codes)...') -with tempfile.TemporaryFile() as f: - r = requests.get('http://www.unicode.org/iso15924/iso15924.txt.zip') - f.write(r.content) - with zipfile.ZipFile(f) as z: - z.extract('iso15924-utf8-20131012.txt', DATA_DIR) - -# opensubtitles supported languages -print('Downloading OpenSubtitles supported languages...') -with open(os.path.join(DATA_DIR, 'opensubtitles_languages.txt'), 'w') as f: - r = requests.get('http://www.opensubtitles.org/addons/export_languages.php') - f.write(r.content) - -print('Done!') diff --git a/libs/backports.functools_lru_cache-1.2.1-py3.5-nspkg.pth b/libs/backports.functools_lru_cache-1.2.1-py3.5-nspkg.pth deleted file mode 100644 index 0b1f79dd5..000000000 --- a/libs/backports.functools_lru_cache-1.2.1-py3.5-nspkg.pth +++ /dev/null @@ -1 +0,0 @@ -import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('backports',));ie = os.path.exists(os.path.join(p,'__init__.py'));m = not ie and sys.modules.setdefault('backports', types.ModuleType('backports'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p) diff --git a/libs/beets/__init__.py b/libs/beets/__init__.py deleted file mode 100644 index 830477a9a..000000000 --- a/libs/beets/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -from __future__ import division, absolute_import, print_function - -import os - -from beets.util import confit - -__version__ = u'1.3.18' -__author__ = u'Adrian Sampson ' - - -class IncludeLazyConfig(confit.LazyConfig): - """A version of Confit's LazyConfig that also merges in data from - YAML files specified in an `include` setting. - """ - def read(self, user=True, defaults=True): - super(IncludeLazyConfig, self).read(user, defaults) - - try: - for view in self['include']: - filename = view.as_filename() - if os.path.isfile(filename): - self.set_file(filename) - except confit.NotFoundError: - pass - - -config = IncludeLazyConfig('beets', __name__) diff --git a/libs/beets/autotag/__init__.py b/libs/beets/autotag/__init__.py deleted file mode 100644 index f8233be61..000000000 --- a/libs/beets/autotag/__init__.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Facilities for automatically determining files' correct metadata. -""" - -from __future__ import division, absolute_import, print_function - -from beets import logging -from beets import config - -# Parts of external interface. -from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch # noqa -from .match import tag_item, tag_album # noqa -from .match import Recommendation # noqa - -# Global logger. -log = logging.getLogger('beets') - - -# Additional utilities for the main interface. - -def apply_item_metadata(item, track_info): - """Set an item's metadata from its matched TrackInfo object. - """ - item.artist = track_info.artist - item.artist_sort = track_info.artist_sort - item.artist_credit = track_info.artist_credit - item.title = track_info.title - item.mb_trackid = track_info.track_id - if track_info.artist_id: - item.mb_artistid = track_info.artist_id - if track_info.data_source: - item.data_source = track_info.data_source - # At the moment, the other metadata is left intact (including album - # and track number). Perhaps these should be emptied? - - -def apply_metadata(album_info, mapping): - """Set the items' metadata to match an AlbumInfo object using a - mapping from Items to TrackInfo objects. - """ - for item, track_info in mapping.iteritems(): - # Album, artist, track count. - if track_info.artist: - item.artist = track_info.artist - else: - item.artist = album_info.artist - item.albumartist = album_info.artist - item.album = album_info.album - - # Artist sort and credit names. - item.artist_sort = track_info.artist_sort or album_info.artist_sort - item.artist_credit = (track_info.artist_credit or - album_info.artist_credit) - item.albumartist_sort = album_info.artist_sort - item.albumartist_credit = album_info.artist_credit - - # Release date. - for prefix in '', 'original_': - if config['original_date'] and not prefix: - # Ignore specific release date. - continue - - for suffix in 'year', 'month', 'day': - key = prefix + suffix - value = getattr(album_info, key) or 0 - - # If we don't even have a year, apply nothing. - if suffix == 'year' and not value: - break - - # Otherwise, set the fetched value (or 0 for the month - # and day if not available). - item[key] = value - - # If we're using original release date for both fields, - # also set item.year = info.original_year, etc. - if config['original_date']: - item[suffix] = value - - # Title. - item.title = track_info.title - - if config['per_disc_numbering']: - # We want to let the track number be zero, but if the medium index - # is not provided we need to fall back to the overall index. - item.track = track_info.medium_index - if item.track is None: - item.track = track_info.index - item.tracktotal = track_info.medium_total or len(album_info.tracks) - else: - item.track = track_info.index - item.tracktotal = len(album_info.tracks) - - # Disc and disc count. - item.disc = track_info.medium - item.disctotal = album_info.mediums - - # MusicBrainz IDs. - item.mb_trackid = track_info.track_id - item.mb_albumid = album_info.album_id - if track_info.artist_id: - item.mb_artistid = track_info.artist_id - else: - item.mb_artistid = album_info.artist_id - item.mb_albumartistid = album_info.artist_id - item.mb_releasegroupid = album_info.releasegroup_id - - # Compilation flag. - item.comp = album_info.va - - # Miscellaneous metadata. - for field in ('albumtype', - 'label', - 'asin', - 'catalognum', - 'script', - 'language', - 'country', - 'albumstatus', - 'albumdisambig', - 'data_source',): - value = getattr(album_info, field) - if value is not None: - item[field] = value - if track_info.disctitle is not None: - item.disctitle = track_info.disctitle - - if track_info.media is not None: - item.media = track_info.media diff --git a/libs/beets/autotag/hooks.py b/libs/beets/autotag/hooks.py deleted file mode 100644 index 3de803899..000000000 --- a/libs/beets/autotag/hooks.py +++ /dev/null @@ -1,612 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Glue between metadata sources and the matching logic.""" -from __future__ import division, absolute_import, print_function - -from collections import namedtuple -import re - -from beets import logging -from beets import plugins -from beets import config -from beets.autotag import mb -from jellyfish import levenshtein_distance -from unidecode import unidecode - -log = logging.getLogger('beets') - - -# Classes used to represent candidate options. - -class AlbumInfo(object): - """Describes a canonical release that may be used to match a release - in the library. Consists of these data members: - - - ``album``: the release title - - ``album_id``: MusicBrainz ID; UUID fragment only - - ``artist``: name of the release's primary artist - - ``artist_id`` - - ``tracks``: list of TrackInfo objects making up the release - - ``asin``: Amazon ASIN - - ``albumtype``: string describing the kind of release - - ``va``: boolean: whether the release has "various artists" - - ``year``: release year - - ``month``: release month - - ``day``: release day - - ``label``: music label responsible for the release - - ``mediums``: the number of discs in this release - - ``artist_sort``: name of the release's artist for sorting - - ``releasegroup_id``: MBID for the album's release group - - ``catalognum``: the label's catalog number for the release - - ``script``: character set used for metadata - - ``language``: human language of the metadata - - ``country``: the release country - - ``albumstatus``: MusicBrainz release status (Official, etc.) - - ``media``: delivery mechanism (Vinyl, etc.) - - ``albumdisambig``: MusicBrainz release disambiguation comment - - ``artist_credit``: Release-specific artist name - - ``data_source``: The original data source (MusicBrainz, Discogs, etc.) - - ``data_url``: The data source release URL. - - The fields up through ``tracks`` are required. The others are - optional and may be None. - """ - def __init__(self, album, album_id, artist, artist_id, tracks, asin=None, - albumtype=None, va=False, year=None, month=None, day=None, - label=None, mediums=None, artist_sort=None, - releasegroup_id=None, catalognum=None, script=None, - language=None, country=None, albumstatus=None, media=None, - albumdisambig=None, artist_credit=None, original_year=None, - original_month=None, original_day=None, data_source=None, - data_url=None): - self.album = album - self.album_id = album_id - self.artist = artist - self.artist_id = artist_id - self.tracks = tracks - self.asin = asin - self.albumtype = albumtype - self.va = va - self.year = year - self.month = month - self.day = day - self.label = label - self.mediums = mediums - self.artist_sort = artist_sort - self.releasegroup_id = releasegroup_id - self.catalognum = catalognum - self.script = script - self.language = language - self.country = country - self.albumstatus = albumstatus - self.media = media - self.albumdisambig = albumdisambig - self.artist_credit = artist_credit - self.original_year = original_year - self.original_month = original_month - self.original_day = original_day - self.data_source = data_source - self.data_url = data_url - - # Work around a bug in python-musicbrainz-ngs that causes some - # strings to be bytes rather than Unicode. - # https://github.com/alastair/python-musicbrainz-ngs/issues/85 - def decode(self, codec='utf8'): - """Ensure that all string attributes on this object, and the - constituent `TrackInfo` objects, are decoded to Unicode. - """ - for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', - 'catalognum', 'script', 'language', 'country', - 'albumstatus', 'albumdisambig', 'artist_credit', 'media']: - value = getattr(self, fld) - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) - - if self.tracks: - for track in self.tracks: - track.decode(codec) - - -class TrackInfo(object): - """Describes a canonical track present on a release. Appears as part - of an AlbumInfo's ``tracks`` list. Consists of these data members: - - - ``title``: name of the track - - ``track_id``: MusicBrainz ID; UUID fragment only - - ``artist``: individual track artist name - - ``artist_id`` - - ``length``: float: duration of the track in seconds - - ``index``: position on the entire release - - ``media``: delivery mechanism (Vinyl, etc.) - - ``medium``: the disc number this track appears on in the album - - ``medium_index``: the track's position on the disc - - ``medium_total``: the number of tracks on the item's disc - - ``artist_sort``: name of the track artist for sorting - - ``disctitle``: name of the individual medium (subtitle) - - ``artist_credit``: Recording-specific artist name - - ``data_source``: The original data source (MusicBrainz, Discogs, etc.) - - ``data_url``: The data source release URL. - - Only ``title`` and ``track_id`` are required. The rest of the fields - may be None. The indices ``index``, ``medium``, and ``medium_index`` - are all 1-based. - """ - def __init__(self, title, track_id, artist=None, artist_id=None, - length=None, index=None, medium=None, medium_index=None, - medium_total=None, artist_sort=None, disctitle=None, - artist_credit=None, data_source=None, data_url=None, - media=None): - self.title = title - self.track_id = track_id - self.artist = artist - self.artist_id = artist_id - self.length = length - self.index = index - self.media = media - self.medium = medium - self.medium_index = medium_index - self.medium_total = medium_total - self.artist_sort = artist_sort - self.disctitle = disctitle - self.artist_credit = artist_credit - self.data_source = data_source - self.data_url = data_url - - # As above, work around a bug in python-musicbrainz-ngs. - def decode(self, codec='utf8'): - """Ensure that all string attributes on this object are decoded - to Unicode. - """ - for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle', - 'artist_credit', 'media']: - value = getattr(self, fld) - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) - - -# Candidate distance scoring. - -# Parameters for string distance function. -# Words that can be moved to the end of a string using a comma. -SD_END_WORDS = ['the', 'a', 'an'] -# Reduced weights for certain portions of the string. -SD_PATTERNS = [ - (r'^the ', 0.1), - (r'[\[\(]?(ep|single)[\]\)]?', 0.0), - (r'[\[\(]?(featuring|feat|ft)[\. :].+', 0.1), - (r'\(.*?\)', 0.3), - (r'\[.*?\]', 0.3), - (r'(, )?(pt\.|part) .+', 0.2), -] -# Replacements to use before testing distance. -SD_REPLACE = [ - (r'&', 'and'), -] - - -def _string_dist_basic(str1, str2): - """Basic edit distance between two strings, ignoring - non-alphanumeric characters and case. Comparisons are based on a - transliteration/lowering to ASCII characters. Normalized by string - length. - """ - assert isinstance(str1, unicode) - assert isinstance(str2, unicode) - str1 = unidecode(str1).decode('ascii') - str2 = unidecode(str2).decode('ascii') - str1 = re.sub(r'[^a-z0-9]', '', str1.lower()) - str2 = re.sub(r'[^a-z0-9]', '', str2.lower()) - if not str1 and not str2: - return 0.0 - return levenshtein_distance(str1, str2) / float(max(len(str1), len(str2))) - - -def string_dist(str1, str2): - """Gives an "intuitive" edit distance between two strings. This is - an edit distance, normalized by the string length, with a number of - tweaks that reflect intuition about text. - """ - if str1 is None and str2 is None: - return 0.0 - if str1 is None or str2 is None: - return 1.0 - - str1 = str1.lower() - str2 = str2.lower() - - # Don't penalize strings that move certain words to the end. For - # example, "the something" should be considered equal to - # "something, the". - for word in SD_END_WORDS: - if str1.endswith(', %s' % word): - str1 = '%s %s' % (word, str1[:-len(word) - 2]) - if str2.endswith(', %s' % word): - str2 = '%s %s' % (word, str2[:-len(word) - 2]) - - # Perform a couple of basic normalizing substitutions. - for pat, repl in SD_REPLACE: - str1 = re.sub(pat, repl, str1) - str2 = re.sub(pat, repl, str2) - - # Change the weight for certain string portions matched by a set - # of regular expressions. We gradually change the strings and build - # up penalties associated with parts of the string that were - # deleted. - base_dist = _string_dist_basic(str1, str2) - penalty = 0.0 - for pat, weight in SD_PATTERNS: - # Get strings that drop the pattern. - case_str1 = re.sub(pat, '', str1) - case_str2 = re.sub(pat, '', str2) - - if case_str1 != str1 or case_str2 != str2: - # If the pattern was present (i.e., it is deleted in the - # the current case), recalculate the distances for the - # modified strings. - case_dist = _string_dist_basic(case_str1, case_str2) - case_delta = max(0.0, base_dist - case_dist) - if case_delta == 0.0: - continue - - # Shift our baseline strings down (to avoid rematching the - # same part of the string) and add a scaled distance - # amount to the penalties. - str1 = case_str1 - str2 = case_str2 - base_dist = case_dist - penalty += weight * case_delta - - return base_dist + penalty - - -class LazyClassProperty(object): - """A decorator implementing a read-only property that is *lazy* in - the sense that the getter is only invoked once. Subsequent accesses - through *any* instance use the cached result. - """ - def __init__(self, getter): - self.getter = getter - self.computed = False - - def __get__(self, obj, owner): - if not self.computed: - self.value = self.getter(owner) - self.computed = True - return self.value - - -class Distance(object): - """Keeps track of multiple distance penalties. Provides a single - weighted distance for all penalties as well as a weighted distance - for each individual penalty. - """ - def __init__(self): - self._penalties = {} - - @LazyClassProperty - def _weights(cls): # noqa - """A dictionary from keys to floating-point weights. - """ - weights_view = config['match']['distance_weights'] - weights = {} - for key in weights_view.keys(): - weights[key] = weights_view[key].as_number() - return weights - - # Access the components and their aggregates. - - @property - def distance(self): - """Return a weighted and normalized distance across all - penalties. - """ - dist_max = self.max_distance - if dist_max: - return self.raw_distance / self.max_distance - return 0.0 - - @property - def max_distance(self): - """Return the maximum distance penalty (normalization factor). - """ - dist_max = 0.0 - for key, penalty in self._penalties.iteritems(): - dist_max += len(penalty) * self._weights[key] - return dist_max - - @property - def raw_distance(self): - """Return the raw (denormalized) distance. - """ - dist_raw = 0.0 - for key, penalty in self._penalties.iteritems(): - dist_raw += sum(penalty) * self._weights[key] - return dist_raw - - def items(self): - """Return a list of (key, dist) pairs, with `dist` being the - weighted distance, sorted from highest to lowest. Does not - include penalties with a zero value. - """ - list_ = [] - for key in self._penalties: - dist = self[key] - if dist: - list_.append((key, dist)) - # Convert distance into a negative float we can sort items in - # ascending order (for keys, when the penalty is equal) and - # still get the items with the biggest distance first. - return sorted( - list_, - key=lambda key_and_dist: (-key_and_dist[1], key_and_dist[0]) - ) - - # Behave like a float. - - def __cmp__(self, other): - return cmp(self.distance, other) - - def __float__(self): - return self.distance - - def __sub__(self, other): - return self.distance - other - - def __rsub__(self, other): - return other - self.distance - - def __unicode__(self): - return "{0:.2f}".format(self.distance) - - # Behave like a dict. - - def __getitem__(self, key): - """Returns the weighted distance for a named penalty. - """ - dist = sum(self._penalties[key]) * self._weights[key] - dist_max = self.max_distance - if dist_max: - return dist / dist_max - return 0.0 - - def __iter__(self): - return iter(self.items()) - - def __len__(self): - return len(self.items()) - - def keys(self): - return [key for key, _ in self.items()] - - def update(self, dist): - """Adds all the distance penalties from `dist`. - """ - if not isinstance(dist, Distance): - raise ValueError( - u'`dist` must be a Distance object, not {0}'.format(type(dist)) - ) - for key, penalties in dist._penalties.iteritems(): - self._penalties.setdefault(key, []).extend(penalties) - - # Adding components. - - def _eq(self, value1, value2): - """Returns True if `value1` is equal to `value2`. `value1` may - be a compiled regular expression, in which case it will be - matched against `value2`. - """ - if isinstance(value1, re._pattern_type): - return bool(value1.match(value2)) - return value1 == value2 - - def add(self, key, dist): - """Adds a distance penalty. `key` must correspond with a - configured weight setting. `dist` must be a float between 0.0 - and 1.0, and will be added to any existing distance penalties - for the same key. - """ - if not 0.0 <= dist <= 1.0: - raise ValueError( - u'`dist` must be between 0.0 and 1.0, not {0}'.format(dist) - ) - self._penalties.setdefault(key, []).append(dist) - - def add_equality(self, key, value, options): - """Adds a distance penalty of 1.0 if `value` doesn't match any - of the values in `options`. If an option is a compiled regular - expression, it will be considered equal if it matches against - `value`. - """ - if not isinstance(options, (list, tuple)): - options = [options] - for opt in options: - if self._eq(opt, value): - dist = 0.0 - break - else: - dist = 1.0 - self.add(key, dist) - - def add_expr(self, key, expr): - """Adds a distance penalty of 1.0 if `expr` evaluates to True, - or 0.0. - """ - if expr: - self.add(key, 1.0) - else: - self.add(key, 0.0) - - def add_number(self, key, number1, number2): - """Adds a distance penalty of 1.0 for each number of difference - between `number1` and `number2`, or 0.0 when there is no - difference. Use this when there is no upper limit on the - difference between the two numbers. - """ - diff = abs(number1 - number2) - if diff: - for i in range(diff): - self.add(key, 1.0) - else: - self.add(key, 0.0) - - def add_priority(self, key, value, options): - """Adds a distance penalty that corresponds to the position at - which `value` appears in `options`. A distance penalty of 0.0 - for the first option, or 1.0 if there is no matching option. If - an option is a compiled regular expression, it will be - considered equal if it matches against `value`. - """ - if not isinstance(options, (list, tuple)): - options = [options] - unit = 1.0 / (len(options) or 1) - for i, opt in enumerate(options): - if self._eq(opt, value): - dist = i * unit - break - else: - dist = 1.0 - self.add(key, dist) - - def add_ratio(self, key, number1, number2): - """Adds a distance penalty for `number1` as a ratio of `number2`. - `number1` is bound at 0 and `number2`. - """ - number = float(max(min(number1, number2), 0)) - if number2: - dist = number / number2 - else: - dist = 0.0 - self.add(key, dist) - - def add_string(self, key, str1, str2): - """Adds a distance penalty based on the edit distance between - `str1` and `str2`. - """ - dist = string_dist(str1, str2) - self.add(key, dist) - - -# Structures that compose all the information for a candidate match. - -AlbumMatch = namedtuple('AlbumMatch', ['distance', 'info', 'mapping', - 'extra_items', 'extra_tracks']) - -TrackMatch = namedtuple('TrackMatch', ['distance', 'info']) - - -# Aggregation of sources. - -def album_for_mbid(release_id): - """Get an AlbumInfo object for a MusicBrainz release ID. Return None - if the ID is not found. - """ - try: - album = mb.album_for_id(release_id) - if album: - plugins.send(u'albuminfo_received', info=album) - return album - except mb.MusicBrainzAPIError as exc: - exc.log(log) - - -def track_for_mbid(recording_id): - """Get a TrackInfo object for a MusicBrainz recording ID. Return None - if the ID is not found. - """ - try: - track = mb.track_for_id(recording_id) - if track: - plugins.send(u'trackinfo_received', info=track) - return track - except mb.MusicBrainzAPIError as exc: - exc.log(log) - - -def albums_for_id(album_id): - """Get a list of albums for an ID.""" - candidates = [album_for_mbid(album_id)] - plugin_albums = plugins.album_for_id(album_id) - for a in plugin_albums: - plugins.send(u'albuminfo_received', info=a) - candidates.extend(plugin_albums) - return filter(None, candidates) - - -def tracks_for_id(track_id): - """Get a list of tracks for an ID.""" - candidates = [track_for_mbid(track_id)] - plugin_tracks = plugins.track_for_id(track_id) - for t in plugin_tracks: - plugins.send(u'trackinfo_received', info=t) - candidates.extend(plugin_tracks) - return filter(None, candidates) - - -def album_candidates(items, artist, album, va_likely): - """Search for album matches. ``items`` is a list of Item objects - that make up the album. ``artist`` and ``album`` are the respective - names (strings), which may be derived from the item list or may be - entered by the user. ``va_likely`` is a boolean indicating whether - the album is likely to be a "various artists" release. - """ - out = [] - - # Base candidates if we have album and artist to match. - if artist and album: - try: - out.extend(mb.match_album(artist, album, len(items))) - except mb.MusicBrainzAPIError as exc: - exc.log(log) - - # Also add VA matches from MusicBrainz where appropriate. - if va_likely and album: - try: - out.extend(mb.match_album(None, album, len(items))) - except mb.MusicBrainzAPIError as exc: - exc.log(log) - - # Candidates from plugins. - out.extend(plugins.candidates(items, artist, album, va_likely)) - - # Notify subscribed plugins about fetched album info - for a in out: - plugins.send(u'albuminfo_received', info=a) - - return out - - -def item_candidates(item, artist, title): - """Search for item matches. ``item`` is the Item to be matched. - ``artist`` and ``title`` are strings and either reflect the item or - are specified by the user. - """ - out = [] - - # MusicBrainz candidates. - if artist and title: - try: - out.extend(mb.match_track(artist, title)) - except mb.MusicBrainzAPIError as exc: - exc.log(log) - - # Plugin candidates. - out.extend(plugins.item_candidates(item, artist, title)) - - # Notify subscribed plugins about fetched track info - for i in out: - plugins.send(u'trackinfo_received', info=i) - - return out diff --git a/libs/beets/autotag/match.py b/libs/beets/autotag/match.py deleted file mode 100644 index cfe184e71..000000000 --- a/libs/beets/autotag/match.py +++ /dev/null @@ -1,501 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Matches existing metadata with canonical information to identify -releases and tracks. -""" - -from __future__ import division, absolute_import, print_function - -import datetime -import re -from munkres import Munkres - -from beets import logging -from beets import plugins -from beets import config -from beets.util import plurality -from beets.autotag import hooks -from beets.util.enumeration import OrderedEnum -from functools import reduce - -# Artist signals that indicate "various artists". These are used at the -# album level to determine whether a given release is likely a VA -# release and also on the track level to to remove the penalty for -# differing artists. -VA_ARTISTS = (u'', u'various artists', u'various', u'va', u'unknown') - -# Global logger. -log = logging.getLogger('beets') - - -# Recommendation enumeration. - -class Recommendation(OrderedEnum): - """Indicates a qualitative suggestion to the user about what should - be done with a given match. - """ - none = 0 - low = 1 - medium = 2 - strong = 3 - - -# Primary matching functionality. - -def current_metadata(items): - """Extract the likely current metadata for an album given a list of its - items. Return two dictionaries: - - The most common value for each field. - - Whether each field's value was unanimous (values are booleans). - """ - assert items # Must be nonempty. - - likelies = {} - consensus = {} - fields = ['artist', 'album', 'albumartist', 'year', 'disctotal', - 'mb_albumid', 'label', 'catalognum', 'country', 'media', - 'albumdisambig'] - for field in fields: - values = [item[field] for item in items if item] - likelies[field], freq = plurality(values) - consensus[field] = (freq == len(values)) - - # If there's an album artist consensus, use this for the artist. - if consensus['albumartist'] and likelies['albumartist']: - likelies['artist'] = likelies['albumartist'] - - return likelies, consensus - - -def assign_items(items, tracks): - """Given a list of Items and a list of TrackInfo objects, find the - best mapping between them. Returns a mapping from Items to TrackInfo - objects, a set of extra Items, and a set of extra TrackInfo - objects. These "extra" objects occur when there is an unequal number - of objects of the two types. - """ - # Construct the cost matrix. - costs = [] - for item in items: - row = [] - for i, track in enumerate(tracks): - row.append(track_distance(item, track)) - costs.append(row) - - # Find a minimum-cost bipartite matching. - matching = Munkres().compute(costs) - - # Produce the output matching. - mapping = dict((items[i], tracks[j]) for (i, j) in matching) - extra_items = list(set(items) - set(mapping.keys())) - extra_items.sort(key=lambda i: (i.disc, i.track, i.title)) - extra_tracks = list(set(tracks) - set(mapping.values())) - extra_tracks.sort(key=lambda t: (t.index, t.title)) - return mapping, extra_items, extra_tracks - - -def track_index_changed(item, track_info): - """Returns True if the item and track info index is different. Tolerates - per disc and per release numbering. - """ - return item.track not in (track_info.medium_index, track_info.index) - - -def track_distance(item, track_info, incl_artist=False): - """Determines the significance of a track metadata change. Returns a - Distance object. `incl_artist` indicates that a distance component should - be included for the track artist (i.e., for various-artist releases). - """ - dist = hooks.Distance() - - # Length. - if track_info.length: - diff = abs(item.length - track_info.length) - \ - config['match']['track_length_grace'].as_number() - dist.add_ratio('track_length', diff, - config['match']['track_length_max'].as_number()) - - # Title. - dist.add_string('track_title', item.title, track_info.title) - - # Artist. Only check if there is actually an artist in the track data. - if incl_artist and track_info.artist and \ - item.artist.lower() not in VA_ARTISTS: - dist.add_string('track_artist', item.artist, track_info.artist) - - # Track index. - if track_info.index and item.track: - dist.add_expr('track_index', track_index_changed(item, track_info)) - - # Track ID. - if item.mb_trackid: - dist.add_expr('track_id', item.mb_trackid != track_info.track_id) - - # Plugins. - dist.update(plugins.track_distance(item, track_info)) - - return dist - - -def distance(items, album_info, mapping): - """Determines how "significant" an album metadata change would be. - Returns a Distance object. `album_info` is an AlbumInfo object - reflecting the album to be compared. `items` is a sequence of all - Item objects that will be matched (order is not important). - `mapping` is a dictionary mapping Items to TrackInfo objects; the - keys are a subset of `items` and the values are a subset of - `album_info.tracks`. - """ - likelies, _ = current_metadata(items) - - dist = hooks.Distance() - - # Artist, if not various. - if not album_info.va: - dist.add_string('artist', likelies['artist'], album_info.artist) - - # Album. - dist.add_string('album', likelies['album'], album_info.album) - - # Current or preferred media. - if album_info.media: - # Preferred media options. - patterns = config['match']['preferred']['media'].as_str_seq() - options = [re.compile(r'(\d+x)?(%s)' % pat, re.I) for pat in patterns] - if options: - dist.add_priority('media', album_info.media, options) - # Current media. - elif likelies['media']: - dist.add_equality('media', album_info.media, likelies['media']) - - # Mediums. - if likelies['disctotal'] and album_info.mediums: - dist.add_number('mediums', likelies['disctotal'], album_info.mediums) - - # Prefer earliest release. - if album_info.year and config['match']['preferred']['original_year']: - # Assume 1889 (earliest first gramophone discs) if we don't know the - # original year. - original = album_info.original_year or 1889 - diff = abs(album_info.year - original) - diff_max = abs(datetime.date.today().year - original) - dist.add_ratio('year', diff, diff_max) - # Year. - elif likelies['year'] and album_info.year: - if likelies['year'] in (album_info.year, album_info.original_year): - # No penalty for matching release or original year. - dist.add('year', 0.0) - elif album_info.original_year: - # Prefer matchest closest to the release year. - diff = abs(likelies['year'] - album_info.year) - diff_max = abs(datetime.date.today().year - - album_info.original_year) - dist.add_ratio('year', diff, diff_max) - else: - # Full penalty when there is no original year. - dist.add('year', 1.0) - - # Preferred countries. - patterns = config['match']['preferred']['countries'].as_str_seq() - options = [re.compile(pat, re.I) for pat in patterns] - if album_info.country and options: - dist.add_priority('country', album_info.country, options) - # Country. - elif likelies['country'] and album_info.country: - dist.add_string('country', likelies['country'], album_info.country) - - # Label. - if likelies['label'] and album_info.label: - dist.add_string('label', likelies['label'], album_info.label) - - # Catalog number. - if likelies['catalognum'] and album_info.catalognum: - dist.add_string('catalognum', likelies['catalognum'], - album_info.catalognum) - - # Disambiguation. - if likelies['albumdisambig'] and album_info.albumdisambig: - dist.add_string('albumdisambig', likelies['albumdisambig'], - album_info.albumdisambig) - - # Album ID. - if likelies['mb_albumid']: - dist.add_equality('album_id', likelies['mb_albumid'], - album_info.album_id) - - # Tracks. - dist.tracks = {} - for item, track in mapping.iteritems(): - dist.tracks[track] = track_distance(item, track, album_info.va) - dist.add('tracks', dist.tracks[track].distance) - - # Missing tracks. - for i in range(len(album_info.tracks) - len(mapping)): - dist.add('missing_tracks', 1.0) - - # Unmatched tracks. - for i in range(len(items) - len(mapping)): - dist.add('unmatched_tracks', 1.0) - - # Plugins. - dist.update(plugins.album_distance(items, album_info, mapping)) - - return dist - - -def match_by_id(items): - """If the items are tagged with a MusicBrainz album ID, returns an - AlbumInfo object for the corresponding album. Otherwise, returns - None. - """ - # Is there a consensus on the MB album ID? - albumids = [item.mb_albumid for item in items if item.mb_albumid] - if not albumids: - log.debug(u'No album IDs found.') - return None - - # If all album IDs are equal, look up the album. - if bool(reduce(lambda x, y: x if x == y else (), albumids)): - albumid = albumids[0] - log.debug(u'Searching for discovered album ID: {0}', albumid) - return hooks.album_for_mbid(albumid) - else: - log.debug(u'No album ID consensus.') - - -def _recommendation(results): - """Given a sorted list of AlbumMatch or TrackMatch objects, return a - recommendation based on the results' distances. - - If the recommendation is higher than the configured maximum for - an applied penalty, the recommendation will be downgraded to the - configured maximum for that penalty. - """ - if not results: - # No candidates: no recommendation. - return Recommendation.none - - # Basic distance thresholding. - min_dist = results[0].distance - if min_dist < config['match']['strong_rec_thresh'].as_number(): - # Strong recommendation level. - rec = Recommendation.strong - elif min_dist <= config['match']['medium_rec_thresh'].as_number(): - # Medium recommendation level. - rec = Recommendation.medium - elif len(results) == 1: - # Only a single candidate. - rec = Recommendation.low - elif results[1].distance - min_dist >= \ - config['match']['rec_gap_thresh'].as_number(): - # Gap between first two candidates is large. - rec = Recommendation.low - else: - # No conclusion. Return immediately. Can't be downgraded any further. - return Recommendation.none - - # Downgrade to the max rec if it is lower than the current rec for an - # applied penalty. - keys = set(min_dist.keys()) - if isinstance(results[0], hooks.AlbumMatch): - for track_dist in min_dist.tracks.values(): - keys.update(track_dist.keys()) - max_rec_view = config['match']['max_rec'] - for key in keys: - if key in max_rec_view.keys(): - max_rec = max_rec_view[key].as_choice({ - 'strong': Recommendation.strong, - 'medium': Recommendation.medium, - 'low': Recommendation.low, - 'none': Recommendation.none, - }) - rec = min(rec, max_rec) - - return rec - - -def _add_candidate(items, results, info): - """Given a candidate AlbumInfo object, attempt to add the candidate - to the output dictionary of AlbumMatch objects. This involves - checking the track count, ordering the items, checking for - duplicates, and calculating the distance. - """ - log.debug(u'Candidate: {0} - {1}', info.artist, info.album) - - # Discard albums with zero tracks. - if not info.tracks: - log.debug(u'No tracks.') - return - - # Don't duplicate. - if info.album_id in results: - log.debug(u'Duplicate.') - return - - # Discard matches without required tags. - for req_tag in config['match']['required'].as_str_seq(): - if getattr(info, req_tag) is None: - log.debug(u'Ignored. Missing required tag: {0}', req_tag) - return - - # Find mapping between the items and the track info. - mapping, extra_items, extra_tracks = assign_items(items, info.tracks) - - # Get the change distance. - dist = distance(items, info, mapping) - - # Skip matches with ignored penalties. - penalties = [key for key, _ in dist] - for penalty in config['match']['ignored'].as_str_seq(): - if penalty in penalties: - log.debug(u'Ignored. Penalty: {0}', penalty) - return - - log.debug(u'Success. Distance: {0}', dist) - results[info.album_id] = hooks.AlbumMatch(dist, info, mapping, - extra_items, extra_tracks) - - -def tag_album(items, search_artist=None, search_album=None, - search_ids=[]): - """Return a tuple of a artist name, an album name, a list of - `AlbumMatch` candidates from the metadata backend, and a - `Recommendation`. - - The artist and album are the most common values of these fields - among `items`. - - The `AlbumMatch` objects are generated by searching the metadata - backends. By default, the metadata of the items is used for the - search. This can be customized by setting the parameters. - `search_ids` is a list of metadata backend IDs: if specified, - it will restrict the candidates to those IDs, ignoring - `search_artist` and `search album`. The `mapping` field of the - album has the matched `items` as keys. - - The recommendation is calculated from the match quality of the - candidates. - """ - # Get current metadata. - likelies, consensus = current_metadata(items) - cur_artist = likelies['artist'] - cur_album = likelies['album'] - log.debug(u'Tagging {0} - {1}', cur_artist, cur_album) - - # The output result (distance, AlbumInfo) tuples (keyed by MB album - # ID). - candidates = {} - - # Search by explicit ID. - if search_ids: - search_cands = [] - for search_id in search_ids: - log.debug(u'Searching for album ID: {0}', search_id) - search_cands.extend(hooks.albums_for_id(search_id)) - - # Use existing metadata or text search. - else: - # Try search based on current ID. - id_info = match_by_id(items) - if id_info: - _add_candidate(items, candidates, id_info) - rec = _recommendation(candidates.values()) - log.debug(u'Album ID match recommendation is {0}', rec) - if candidates and not config['import']['timid']: - # If we have a very good MBID match, return immediately. - # Otherwise, this match will compete against metadata-based - # matches. - if rec == Recommendation.strong: - log.debug(u'ID match.') - return cur_artist, cur_album, candidates.values(), rec - - # Search terms. - if not (search_artist and search_album): - # No explicit search terms -- use current metadata. - search_artist, search_album = cur_artist, cur_album - log.debug(u'Search terms: {0} - {1}', search_artist, search_album) - - # Is this album likely to be a "various artist" release? - va_likely = ((not consensus['artist']) or - (search_artist.lower() in VA_ARTISTS) or - any(item.comp for item in items)) - log.debug(u'Album might be VA: {0}', va_likely) - - # Get the results from the data sources. - search_cands = hooks.album_candidates(items, search_artist, - search_album, va_likely) - - log.debug(u'Evaluating {0} candidates.', len(search_cands)) - for info in search_cands: - _add_candidate(items, candidates, info) - - # Sort and get the recommendation. - candidates = sorted(candidates.itervalues()) - rec = _recommendation(candidates) - return cur_artist, cur_album, candidates, rec - - -def tag_item(item, search_artist=None, search_title=None, - search_ids=[]): - """Attempts to find metadata for a single track. Returns a - `(candidates, recommendation)` pair where `candidates` is a list of - TrackMatch objects. `search_artist` and `search_title` may be used - to override the current metadata for the purposes of the MusicBrainz - title. `search_ids` may be used for restricting the search to a list - of metadata backend IDs. - """ - # Holds candidates found so far: keys are MBIDs; values are - # (distance, TrackInfo) pairs. - candidates = {} - - # First, try matching by MusicBrainz ID. - trackids = search_ids or filter(None, [item.mb_trackid]) - if trackids: - for trackid in trackids: - log.debug(u'Searching for track ID: {0}', trackid) - for track_info in hooks.tracks_for_id(trackid): - dist = track_distance(item, track_info, incl_artist=True) - candidates[track_info.track_id] = \ - hooks.TrackMatch(dist, track_info) - # If this is a good match, then don't keep searching. - rec = _recommendation(sorted(candidates.itervalues())) - if rec == Recommendation.strong and \ - not config['import']['timid']: - log.debug(u'Track ID match.') - return sorted(candidates.itervalues()), rec - - # If we're searching by ID, don't proceed. - if search_ids: - if candidates: - return sorted(candidates.itervalues()), rec - else: - return [], Recommendation.none - - # Search terms. - if not (search_artist and search_title): - search_artist, search_title = item.artist, item.title - log.debug(u'Item search terms: {0} - {1}', search_artist, search_title) - - # Get and evaluate candidate metadata. - for track_info in hooks.item_candidates(item, search_artist, search_title): - dist = track_distance(item, track_info, incl_artist=True) - candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info) - - # Sort by distance and return with recommendation. - log.debug(u'Found {0} candidates.', len(candidates)) - candidates = sorted(candidates.itervalues()) - rec = _recommendation(candidates) - return candidates, rec diff --git a/libs/beets/dbcore/types.py b/libs/beets/dbcore/types.py deleted file mode 100644 index 2726969dd..000000000 --- a/libs/beets/dbcore/types.py +++ /dev/null @@ -1,211 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Representation of type information for DBCore model fields. -""" -from __future__ import division, absolute_import, print_function - -from . import query -from beets.util import str2bool - - -# Abstract base. - -class Type(object): - """An object encapsulating the type of a model field. Includes - information about how to store, query, format, and parse a given - field. - """ - - sql = u'TEXT' - """The SQLite column type for the value. - """ - - query = query.SubstringQuery - """The `Query` subclass to be used when querying the field. - """ - - model_type = unicode - """The Python type that is used to represent the value in the model. - - The model is guaranteed to return a value of this type if the field - is accessed. To this end, the constructor is used by the `normalize` - and `from_sql` methods and the `default` property. - """ - - @property - def null(self): - """The value to be exposed when the underlying value is None. - """ - return self.model_type() - - def format(self, value): - """Given a value of this type, produce a Unicode string - representing the value. This is used in template evaluation. - """ - if value is None: - value = self.null - # `self.null` might be `None` - if value is None: - value = u'' - if isinstance(value, bytes): - value = value.decode('utf8', 'ignore') - - return unicode(value) - - def parse(self, string): - """Parse a (possibly human-written) string and return the - indicated value of this type. - """ - try: - return self.model_type(string) - except ValueError: - return self.null - - def normalize(self, value): - """Given a value that will be assigned into a field of this - type, normalize the value to have the appropriate type. This - base implementation only reinterprets `None`. - """ - if value is None: - return self.null - else: - # TODO This should eventually be replaced by - # `self.model_type(value)` - return value - - def from_sql(self, sql_value): - """Receives the value stored in the SQL backend and return the - value to be stored in the model. - - For fixed fields the type of `value` is determined by the column - type affinity given in the `sql` property and the SQL to Python - mapping of the database adapter. For more information see: - http://www.sqlite.org/datatype3.html - https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types - - Flexible fields have the type affinity `TEXT`. This means the - `sql_value` is either a `buffer` or a `unicode` object` and the - method must handle these in addition. - """ - if isinstance(sql_value, buffer): - sql_value = bytes(sql_value).decode('utf8', 'ignore') - if isinstance(sql_value, unicode): - return self.parse(sql_value) - else: - return self.normalize(sql_value) - - def to_sql(self, model_value): - """Convert a value as stored in the model object to a value used - by the database adapter. - """ - return model_value - - -# Reusable types. - -class Default(Type): - null = None - - -class Integer(Type): - """A basic integer type. - """ - sql = u'INTEGER' - query = query.NumericQuery - model_type = int - - -class PaddedInt(Integer): - """An integer field that is formatted with a given number of digits, - padded with zeroes. - """ - def __init__(self, digits): - self.digits = digits - - def format(self, value): - return u'{0:0{1}d}'.format(value or 0, self.digits) - - -class ScaledInt(Integer): - """An integer whose formatting operation scales the number by a - constant and adds a suffix. Good for units with large magnitudes. - """ - def __init__(self, unit, suffix=u''): - self.unit = unit - self.suffix = suffix - - def format(self, value): - return u'{0}{1}'.format((value or 0) // self.unit, self.suffix) - - -class Id(Integer): - """An integer used as the row id or a foreign key in a SQLite table. - This type is nullable: None values are not translated to zero. - """ - null = None - - def __init__(self, primary=True): - if primary: - self.sql = u'INTEGER PRIMARY KEY' - - -class Float(Type): - """A basic floating-point type. - """ - sql = u'REAL' - query = query.NumericQuery - model_type = float - - def format(self, value): - return u'{0:.1f}'.format(value or 0.0) - - -class NullFloat(Float): - """Same as `Float`, but does not normalize `None` to `0.0`. - """ - null = None - - -class String(Type): - """A Unicode string type. - """ - sql = u'TEXT' - query = query.SubstringQuery - - -class Boolean(Type): - """A boolean type. - """ - sql = u'INTEGER' - query = query.BooleanQuery - model_type = bool - - def format(self, value): - return unicode(bool(value)) - - def parse(self, string): - return str2bool(string) - - -# Shared instances of common types. -DEFAULT = Default() -INTEGER = Integer() -PRIMARY_ID = Id(True) -FOREIGN_ID = Id(False) -FLOAT = Float() -NULL_FLOAT = NullFloat() -STRING = String() -BOOLEAN = Boolean() diff --git a/libs/beets/ui/__init__.py b/libs/beets/ui/__init__.py deleted file mode 100644 index 797df44d3..000000000 --- a/libs/beets/ui/__init__.py +++ /dev/null @@ -1,1278 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""This module contains all of the core logic for beets' command-line -interface. To invoke the CLI, just call beets.ui.main(). The actual -CLI commands are implemented in the ui.commands module. -""" - -from __future__ import division, absolute_import, print_function - -import locale -import optparse -import textwrap -import sys -from difflib import SequenceMatcher -import sqlite3 -import errno -import re -import struct -import traceback -import os.path - -from beets import logging -from beets import library -from beets import plugins -from beets import util -from beets.util.functemplate import Template -from beets import config -from beets.util import confit -from beets.autotag import mb -from beets.dbcore import query as db_query - -# On Windows platforms, use colorama to support "ANSI" terminal colors. -if sys.platform == 'win32': - try: - import colorama - except ImportError: - pass - else: - colorama.init() - - -log = logging.getLogger('beets') -if not log.handlers: - log.addHandler(logging.StreamHandler()) -log.propagate = False # Don't propagate to root handler. - - -PF_KEY_QUERIES = { - 'comp': u'comp:true', - 'singleton': u'singleton:true', -} - - -class UserError(Exception): - """UI exception. Commands should throw this in order to display - nonrecoverable errors to the user. - """ - - -# Encoding utilities. - - -def _in_encoding(default=u'utf-8'): - """Get the encoding to use for *inputting* strings from the console. - - :param default: the fallback sys.stdin encoding - """ - - return config['terminal_encoding'].get() or getattr(sys.stdin, 'encoding', - default) - - -def _out_encoding(): - """Get the encoding to use for *outputting* strings to the console. - """ - # Configured override? - encoding = config['terminal_encoding'].get() - if encoding: - return encoding - - # For testing: When sys.stdout is a StringIO under the test harness, - # it doesn't have an `encoding` attribute. Just use UTF-8. - if not hasattr(sys.stdout, 'encoding'): - return 'utf8' - - # Python's guessed output stream encoding, or UTF-8 as a fallback - # (e.g., when piped to a file). - return sys.stdout.encoding or 'utf8' - - -def _arg_encoding(): - """Get the encoding for command-line arguments (and other OS - locale-sensitive strings). - """ - try: - return locale.getdefaultlocale()[1] or 'utf8' - except ValueError: - # Invalid locale environment variable setting. To avoid - # failing entirely for no good reason, assume UTF-8. - return 'utf8' - - -def decargs(arglist): - """Given a list of command-line argument bytestrings, attempts to - decode them to Unicode strings. - """ - return [s.decode(_arg_encoding()) for s in arglist] - - -def print_(*strings, **kwargs): - """Like print, but rather than raising an error when a character - is not in the terminal's encoding's character set, just silently - replaces it. - - If the arguments are strings then they're expected to share the same - type: either bytes or unicode. - - The `end` keyword argument behaves similarly to the built-in `print` - (it defaults to a newline). The value should have the same string - type as the arguments. - """ - end = kwargs.get('end') - - if not strings or isinstance(strings[0], unicode): - txt = u' '.join(strings) - txt += u'\n' if end is None else end - else: - txt = b' '.join(strings) - txt += b'\n' if end is None else end - - # Always send bytes to the stdout stream. - if isinstance(txt, unicode): - txt = txt.encode(_out_encoding(), 'replace') - - sys.stdout.write(txt) - - -# Configuration wrappers. - -def _bool_fallback(a, b): - """Given a boolean or None, return the original value or a fallback. - """ - if a is None: - assert isinstance(b, bool) - return b - else: - assert isinstance(a, bool) - return a - - -def should_write(write_opt=None): - """Decide whether a command that updates metadata should also write - tags, using the importer configuration as the default. - """ - return _bool_fallback(write_opt, config['import']['write'].get(bool)) - - -def should_move(move_opt=None): - """Decide whether a command that updates metadata should also move - files when they're inside the library, using the importer - configuration as the default. - - Specifically, commands should move files after metadata updates only - when the importer is configured *either* to move *or* to copy files. - They should avoid moving files when the importer is configured not - to touch any filenames. - """ - return _bool_fallback( - move_opt, - config['import']['move'].get(bool) or - config['import']['copy'].get(bool) - ) - - -# Input prompts. - -def input_(prompt=None): - """Like `raw_input`, but decodes the result to a Unicode string. - Raises a UserError if stdin is not available. The prompt is sent to - stdout rather than stderr. A printed between the prompt and the - input cursor. - """ - # raw_input incorrectly sends prompts to stderr, not stdout, so we - # use print() explicitly to display prompts. - # http://bugs.python.org/issue1927 - if prompt: - print_(prompt, end=' ') - - try: - resp = raw_input() - except EOFError: - raise UserError(u'stdin stream ended while input required') - - return resp.decode(_in_encoding(), 'ignore') - - -def input_options(options, require=False, prompt=None, fallback_prompt=None, - numrange=None, default=None, max_width=72): - """Prompts a user for input. The sequence of `options` defines the - choices the user has. A single-letter shortcut is inferred for each - option; the user's choice is returned as that single, lower-case - letter. The options should be provided as lower-case strings unless - a particular shortcut is desired; in that case, only that letter - should be capitalized. - - By default, the first option is the default. `default` can be provided to - override this. If `require` is provided, then there is no default. The - prompt and fallback prompt are also inferred but can be overridden. - - If numrange is provided, it is a pair of `(high, low)` (both ints) - indicating that, in addition to `options`, the user may enter an - integer in that inclusive range. - - `max_width` specifies the maximum number of columns in the - automatically generated prompt string. - """ - # Assign single letters to each option. Also capitalize the options - # to indicate the letter. - letters = {} - display_letters = [] - capitalized = [] - first = True - for option in options: - # Is a letter already capitalized? - for letter in option: - if letter.isalpha() and letter.upper() == letter: - found_letter = letter - break - else: - # Infer a letter. - for letter in option: - if not letter.isalpha(): - continue # Don't use punctuation. - if letter not in letters: - found_letter = letter - break - else: - raise ValueError(u'no unambiguous lettering found') - - letters[found_letter.lower()] = option - index = option.index(found_letter) - - # Mark the option's shortcut letter for display. - if not require and ( - (default is None and not numrange and first) or - (isinstance(default, basestring) and - found_letter.lower() == default.lower())): - # The first option is the default; mark it. - show_letter = '[%s]' % found_letter.upper() - is_default = True - else: - show_letter = found_letter.upper() - is_default = False - - # Colorize the letter shortcut. - show_letter = colorize('action_default' if is_default else 'action', - show_letter) - - # Insert the highlighted letter back into the word. - capitalized.append( - option[:index] + show_letter + option[index + 1:] - ) - display_letters.append(found_letter.upper()) - - first = False - - # The default is just the first option if unspecified. - if require: - default = None - elif default is None: - if numrange: - default = numrange[0] - else: - default = display_letters[0].lower() - - # Make a prompt if one is not provided. - if not prompt: - prompt_parts = [] - prompt_part_lengths = [] - if numrange: - if isinstance(default, int): - default_name = unicode(default) - default_name = colorize('action_default', default_name) - tmpl = '# selection (default %s)' - prompt_parts.append(tmpl % default_name) - prompt_part_lengths.append(len(tmpl % unicode(default))) - else: - prompt_parts.append('# selection') - prompt_part_lengths.append(len(prompt_parts[-1])) - prompt_parts += capitalized - prompt_part_lengths += [len(s) for s in options] - - # Wrap the query text. - prompt = '' - line_length = 0 - for i, (part, length) in enumerate(zip(prompt_parts, - prompt_part_lengths)): - # Add punctuation. - if i == len(prompt_parts) - 1: - part += '?' - else: - part += ',' - length += 1 - - # Choose either the current line or the beginning of the next. - if line_length + length + 1 > max_width: - prompt += '\n' - line_length = 0 - - if line_length != 0: - # Not the beginning of the line; need a space. - part = ' ' + part - length += 1 - - prompt += part - line_length += length - - # Make a fallback prompt too. This is displayed if the user enters - # something that is not recognized. - if not fallback_prompt: - fallback_prompt = u'Enter one of ' - if numrange: - fallback_prompt += u'%i-%i, ' % numrange - fallback_prompt += ', '.join(display_letters) + ':' - - resp = input_(prompt) - while True: - resp = resp.strip().lower() - - # Try default option. - if default is not None and not resp: - resp = default - - # Try an integer input if available. - if numrange: - try: - resp = int(resp) - except ValueError: - pass - else: - low, high = numrange - if low <= resp <= high: - return resp - else: - resp = None - - # Try a normal letter input. - if resp: - resp = resp[0] - if resp in letters: - return resp - - # Prompt for new input. - resp = input_(fallback_prompt) - - -def input_yn(prompt, require=False): - """Prompts the user for a "yes" or "no" response. The default is - "yes" unless `require` is `True`, in which case there is no default. - """ - sel = input_options( - ('y', 'n'), require, prompt, u'Enter Y or N:' - ) - return sel == u'y' - - -def input_select_objects(prompt, objs, rep): - """Prompt to user to choose all, none, or some of the given objects. - Return the list of selected objects. - - `prompt` is the prompt string to use for each question (it should be - phrased as an imperative verb). `rep` is a function to call on each - object to print it out when confirming objects individually. - """ - choice = input_options( - (u'y', u'n', u's'), False, - u'%s? (Yes/no/select)' % prompt) - print() # Blank line. - - if choice == u'y': # Yes. - return objs - - elif choice == u's': # Select. - out = [] - for obj in objs: - rep(obj) - if input_yn(u'%s? (yes/no)' % prompt, True): - out.append(obj) - print() # go to a new line - return out - - else: # No. - return [] - - -# Human output formatting. - -def human_bytes(size): - """Formats size, a number of bytes, in a human-readable way.""" - powers = [u'', u'K', u'M', u'G', u'T', u'P', u'E', u'Z', u'Y', u'H'] - unit = 'B' - for power in powers: - if size < 1024: - return u"%3.1f %s%s" % (size, power, unit) - size /= 1024.0 - unit = u'iB' - return u"big" - - -def human_seconds(interval): - """Formats interval, a number of seconds, as a human-readable time - interval using English words. - """ - units = [ - (1, u'second'), - (60, u'minute'), - (60, u'hour'), - (24, u'day'), - (7, u'week'), - (52, u'year'), - (10, u'decade'), - ] - for i in range(len(units) - 1): - increment, suffix = units[i] - next_increment, _ = units[i + 1] - interval /= float(increment) - if interval < next_increment: - break - else: - # Last unit. - increment, suffix = units[-1] - interval /= float(increment) - - return u"%3.1f %ss" % (interval, suffix) - - -def human_seconds_short(interval): - """Formats a number of seconds as a short human-readable M:SS - string. - """ - interval = int(interval) - return u'%i:%02i' % (interval // 60, interval % 60) - - -# Colorization. - -# ANSI terminal colorization code heavily inspired by pygments: -# http://dev.pocoo.org/hg/pygments-main/file/b2deea5b5030/pygments/console.py -# (pygments is by Tim Hatch, Armin Ronacher, et al.) -COLOR_ESCAPE = "\x1b[" -DARK_COLORS = { - "black": 0, - "darkred": 1, - "darkgreen": 2, - "brown": 3, - "darkyellow": 3, - "darkblue": 4, - "purple": 5, - "darkmagenta": 5, - "teal": 6, - "darkcyan": 6, - "lightgray": 7 -} -LIGHT_COLORS = { - "darkgray": 0, - "red": 1, - "green": 2, - "yellow": 3, - "blue": 4, - "fuchsia": 5, - "magenta": 5, - "turquoise": 6, - "cyan": 6, - "white": 7 -} -RESET_COLOR = COLOR_ESCAPE + "39;49;00m" - -# These abstract COLOR_NAMES are lazily mapped on to the actual color in COLORS -# as they are defined in the configuration files, see function: colorize -COLOR_NAMES = ['text_success', 'text_warning', 'text_error', 'text_highlight', - 'text_highlight_minor', 'action_default', 'action'] -COLORS = None - - -def _colorize(color, text): - """Returns a string that prints the given text in the given color - in a terminal that is ANSI color-aware. The color must be something - in DARK_COLORS or LIGHT_COLORS. - """ - if color in DARK_COLORS: - escape = COLOR_ESCAPE + "%im" % (DARK_COLORS[color] + 30) - elif color in LIGHT_COLORS: - escape = COLOR_ESCAPE + "%i;01m" % (LIGHT_COLORS[color] + 30) - else: - raise ValueError(u'no such color %s', color) - return escape + text + RESET_COLOR - - -def colorize(color_name, text): - """Colorize text if colored output is enabled. (Like _colorize but - conditional.) - """ - if config['ui']['color']: - global COLORS - if not COLORS: - COLORS = dict((name, config['ui']['colors'][name].get(unicode)) - for name in COLOR_NAMES) - # In case a 3rd party plugin is still passing the actual color ('red') - # instead of the abstract color name ('text_error') - color = COLORS.get(color_name) - if not color: - log.debug(u'Invalid color_name: {0}', color_name) - color = color_name - return _colorize(color, text) - else: - return text - - -def _colordiff(a, b, highlight='text_highlight', - minor_highlight='text_highlight_minor'): - """Given two values, return the same pair of strings except with - their differences highlighted in the specified color. Strings are - highlighted intelligently to show differences; other values are - stringified and highlighted in their entirety. - """ - if not isinstance(a, basestring) or not isinstance(b, basestring): - # Non-strings: use ordinary equality. - a = unicode(a) - b = unicode(b) - if a == b: - return a, b - else: - return colorize(highlight, a), colorize(highlight, b) - - if isinstance(a, bytes) or isinstance(b, bytes): - # A path field. - a = util.displayable_path(a) - b = util.displayable_path(b) - - a_out = [] - b_out = [] - - matcher = SequenceMatcher(lambda x: False, a, b) - for op, a_start, a_end, b_start, b_end in matcher.get_opcodes(): - if op == 'equal': - # In both strings. - a_out.append(a[a_start:a_end]) - b_out.append(b[b_start:b_end]) - elif op == 'insert': - # Right only. - b_out.append(colorize(highlight, b[b_start:b_end])) - elif op == 'delete': - # Left only. - a_out.append(colorize(highlight, a[a_start:a_end])) - elif op == 'replace': - # Right and left differ. Colorise with second highlight if - # it's just a case change. - if a[a_start:a_end].lower() != b[b_start:b_end].lower(): - color = highlight - else: - color = minor_highlight - a_out.append(colorize(color, a[a_start:a_end])) - b_out.append(colorize(color, b[b_start:b_end])) - else: - assert(False) - - return u''.join(a_out), u''.join(b_out) - - -def colordiff(a, b, highlight='text_highlight'): - """Colorize differences between two values if color is enabled. - (Like _colordiff but conditional.) - """ - if config['ui']['color']: - return _colordiff(a, b, highlight) - else: - return unicode(a), unicode(b) - - -def get_path_formats(subview=None): - """Get the configuration's path formats as a list of query/template - pairs. - """ - path_formats = [] - subview = subview or config['paths'] - for query, view in subview.items(): - query = PF_KEY_QUERIES.get(query, query) # Expand common queries. - path_formats.append((query, Template(view.get(unicode)))) - return path_formats - - -def get_replacements(): - """Confit validation function that reads regex/string pairs. - """ - replacements = [] - for pattern, repl in config['replace'].get(dict).items(): - repl = repl or '' - try: - replacements.append((re.compile(pattern), repl)) - except re.error: - raise UserError( - u'malformed regular expression in replace: {0}'.format( - pattern - ) - ) - return replacements - - -def term_width(): - """Get the width (columns) of the terminal.""" - fallback = config['ui']['terminal_width'].get(int) - - # The fcntl and termios modules are not available on non-Unix - # platforms, so we fall back to a constant. - try: - import fcntl - import termios - except ImportError: - return fallback - - try: - buf = fcntl.ioctl(0, termios.TIOCGWINSZ, ' ' * 4) - except IOError: - return fallback - try: - height, width = struct.unpack('hh', buf) - except struct.error: - return fallback - return width - - -FLOAT_EPSILON = 0.01 - - -def _field_diff(field, old, new): - """Given two Model objects, format their values for `field` and - highlight changes among them. Return a human-readable string. If the - value has not changed, return None instead. - """ - oldval = old.get(field) - newval = new.get(field) - - # If no change, abort. - if isinstance(oldval, float) and isinstance(newval, float) and \ - abs(oldval - newval) < FLOAT_EPSILON: - return None - elif oldval == newval: - return None - - # Get formatted values for output. - oldstr = old.formatted().get(field, u'') - newstr = new.formatted().get(field, u'') - - # For strings, highlight changes. For others, colorize the whole - # thing. - if isinstance(oldval, basestring): - oldstr, newstr = colordiff(oldval, newstr) - else: - oldstr = colorize('text_error', oldstr) - newstr = colorize('text_error', newstr) - - return u'{0} -> {1}'.format(oldstr, newstr) - - -def show_model_changes(new, old=None, fields=None, always=False): - """Given a Model object, print a list of changes from its pristine - version stored in the database. Return a boolean indicating whether - any changes were found. - - `old` may be the "original" object to avoid using the pristine - version from the database. `fields` may be a list of fields to - restrict the detection to. `always` indicates whether the object is - always identified, regardless of whether any changes are present. - """ - old = old or new._db._get(type(new), new.id) - - # Build up lines showing changed fields. - changes = [] - for field in old: - # Subset of the fields. Never show mtime. - if field == 'mtime' or (fields and field not in fields): - continue - - # Detect and show difference for this field. - line = _field_diff(field, old, new) - if line: - changes.append(u' {0}: {1}'.format(field, line)) - - # New fields. - for field in set(new) - set(old): - if fields and field not in fields: - continue - - changes.append(u' {0}: {1}'.format( - field, - colorize('text_highlight', new.formatted()[field]) - )) - - # Print changes. - if changes or always: - print_(format(old)) - if changes: - print_(u'\n'.join(changes)) - - return bool(changes) - - -def show_path_changes(path_changes): - """Given a list of tuples (source, destination) that indicate the - path changes, log the changes as INFO-level output to the beets log. - The output is guaranteed to be unicode. - - Every pair is shown on a single line if the terminal width permits it, - else it is split over two lines. E.g., - - Source -> Destination - - vs. - - Source - -> Destination - """ - sources, destinations = zip(*path_changes) - - # Ensure unicode output - sources = list(map(util.displayable_path, sources)) - destinations = list(map(util.displayable_path, destinations)) - - # Calculate widths for terminal split - col_width = (term_width() - len(' -> ')) // 2 - max_width = len(max(sources + destinations, key=len)) - - if max_width > col_width: - # Print every change over two lines - for source, dest in zip(sources, destinations): - log.info(u'{0} \n -> {1}', source, dest) - else: - # Print every change on a single line, and add a header - title_pad = max_width - len('Source ') + len(' -> ') - - log.info(u'Source {0} Destination', ' ' * title_pad) - for source, dest in zip(sources, destinations): - pad = max_width - len(source) - log.info(u'{0} {1} -> {2}', source, ' ' * pad, dest) - - -class CommonOptionsParser(optparse.OptionParser, object): - """Offers a simple way to add common formatting options. - - Options available include: - - matching albums instead of tracks: add_album_option() - - showing paths instead of items/albums: add_path_option() - - changing the format of displayed items/albums: add_format_option() - - The last one can have several behaviors: - - against a special target - - with a certain format - - autodetected target with the album option - - Each method is fully documented in the related method. - """ - def __init__(self, *args, **kwargs): - super(CommonOptionsParser, self).__init__(*args, **kwargs) - self._album_flags = False - # this serves both as an indicator that we offer the feature AND allows - # us to check whether it has been specified on the CLI - bypassing the - # fact that arguments may be in any order - - def add_album_option(self, flags=('-a', '--album')): - """Add a -a/--album option to match albums instead of tracks. - - If used then the format option can auto-detect whether we're setting - the format for items or albums. - Sets the album property on the options extracted from the CLI. - """ - album = optparse.Option(*flags, action='store_true', - help=u'match albums instead of tracks') - self.add_option(album) - self._album_flags = set(flags) - - def _set_format(self, option, opt_str, value, parser, target=None, - fmt=None, store_true=False): - """Internal callback that sets the correct format while parsing CLI - arguments. - """ - if store_true: - setattr(parser.values, option.dest, True) - - value = fmt or value and unicode(value) or '' - parser.values.format = value - if target: - config[target._format_config_key].set(value) - else: - if self._album_flags: - if parser.values.album: - target = library.Album - else: - # the option is either missing either not parsed yet - if self._album_flags & set(parser.rargs): - target = library.Album - else: - target = library.Item - config[target._format_config_key].set(value) - else: - config[library.Item._format_config_key].set(value) - config[library.Album._format_config_key].set(value) - - def add_path_option(self, flags=('-p', '--path')): - """Add a -p/--path option to display the path instead of the default - format. - - By default this affects both items and albums. If add_album_option() - is used then the target will be autodetected. - - Sets the format property to u'$path' on the options extracted from the - CLI. - """ - path = optparse.Option(*flags, nargs=0, action='callback', - callback=self._set_format, - callback_kwargs={'fmt': '$path', - 'store_true': True}, - help=u'print paths for matched items or albums') - self.add_option(path) - - def add_format_option(self, flags=('-f', '--format'), target=None): - """Add -f/--format option to print some LibModel instances with a - custom format. - - `target` is optional and can be one of ``library.Item``, 'item', - ``library.Album`` and 'album'. - - Several behaviors are available: - - if `target` is given then the format is only applied to that - LibModel - - if the album option is used then the target will be autodetected - - otherwise the format is applied to both items and albums. - - Sets the format property on the options extracted from the CLI. - """ - kwargs = {} - if target: - if isinstance(target, basestring): - target = {'item': library.Item, - 'album': library.Album}[target] - kwargs['target'] = target - - opt = optparse.Option(*flags, action='callback', - callback=self._set_format, - callback_kwargs=kwargs, - help=u'print with custom format') - self.add_option(opt) - - def add_all_common_options(self): - """Add album, path and format options. - """ - self.add_album_option() - self.add_path_option() - self.add_format_option() - - -# Subcommand parsing infrastructure. -# -# This is a fairly generic subcommand parser for optparse. It is -# maintained externally here: -# http://gist.github.com/462717 -# There you will also find a better description of the code and a more -# succinct example program. - -class Subcommand(object): - """A subcommand of a root command-line application that may be - invoked by a SubcommandOptionParser. - """ - def __init__(self, name, parser=None, help='', aliases=(), hide=False): - """Creates a new subcommand. name is the primary way to invoke - the subcommand; aliases are alternate names. parser is an - OptionParser responsible for parsing the subcommand's options. - help is a short description of the command. If no parser is - given, it defaults to a new, empty CommonOptionsParser. - """ - self.name = name - self.parser = parser or CommonOptionsParser() - self.aliases = aliases - self.help = help - self.hide = hide - self._root_parser = None - - def print_help(self): - self.parser.print_help() - - def parse_args(self, args): - return self.parser.parse_args(args) - - @property - def root_parser(self): - return self._root_parser - - @root_parser.setter - def root_parser(self, root_parser): - self._root_parser = root_parser - self.parser.prog = '{0} {1}'.format( - root_parser.get_prog_name().decode('utf8'), self.name) - - -class SubcommandsOptionParser(CommonOptionsParser): - """A variant of OptionParser that parses subcommands and their - arguments. - """ - - def __init__(self, *args, **kwargs): - """Create a new subcommand-aware option parser. All of the - options to OptionParser.__init__ are supported in addition - to subcommands, a sequence of Subcommand objects. - """ - # A more helpful default usage. - if 'usage' not in kwargs: - kwargs['usage'] = u""" - %prog COMMAND [ARGS...] - %prog help COMMAND""" - kwargs['add_help_option'] = False - - # Super constructor. - super(SubcommandsOptionParser, self).__init__(*args, **kwargs) - - # Our root parser needs to stop on the first unrecognized argument. - self.disable_interspersed_args() - - self.subcommands = [] - - def add_subcommand(self, *cmds): - """Adds a Subcommand object to the parser's list of commands. - """ - for cmd in cmds: - cmd.root_parser = self - self.subcommands.append(cmd) - - # Add the list of subcommands to the help message. - def format_help(self, formatter=None): - # Get the original help message, to which we will append. - out = super(SubcommandsOptionParser, self).format_help(formatter) - if formatter is None: - formatter = self.formatter - - # Subcommands header. - result = ["\n"] - result.append(formatter.format_heading('Commands')) - formatter.indent() - - # Generate the display names (including aliases). - # Also determine the help position. - disp_names = [] - help_position = 0 - subcommands = [c for c in self.subcommands if not c.hide] - subcommands.sort(key=lambda c: c.name) - for subcommand in subcommands: - name = subcommand.name - if subcommand.aliases: - name += ' (%s)' % ', '.join(subcommand.aliases) - disp_names.append(name) - - # Set the help position based on the max width. - proposed_help_position = len(name) + formatter.current_indent + 2 - if proposed_help_position <= formatter.max_help_position: - help_position = max(help_position, proposed_help_position) - - # Add each subcommand to the output. - for subcommand, name in zip(subcommands, disp_names): - # Lifted directly from optparse.py. - name_width = help_position - formatter.current_indent - 2 - if len(name) > name_width: - name = "%*s%s\n" % (formatter.current_indent, "", name) - indent_first = help_position - else: - name = "%*s%-*s " % (formatter.current_indent, "", - name_width, name) - indent_first = 0 - result.append(name) - help_width = formatter.width - help_position - help_lines = textwrap.wrap(subcommand.help, help_width) - help_line = help_lines[0] if help_lines else '' - result.append("%*s%s\n" % (indent_first, "", help_line)) - result.extend(["%*s%s\n" % (help_position, "", line) - for line in help_lines[1:]]) - formatter.dedent() - - # Concatenate the original help message with the subcommand - # list. - return out + "".join(result) - - def _subcommand_for_name(self, name): - """Return the subcommand in self.subcommands matching the - given name. The name may either be the name of a subcommand or - an alias. If no subcommand matches, returns None. - """ - for subcommand in self.subcommands: - if name == subcommand.name or \ - name in subcommand.aliases: - return subcommand - return None - - def parse_global_options(self, args): - """Parse options up to the subcommand argument. Returns a tuple - of the options object and the remaining arguments. - """ - options, subargs = self.parse_args(args) - - # Force the help command - if options.help: - subargs = ['help'] - elif options.version: - subargs = ['version'] - return options, subargs - - def parse_subcommand(self, args): - """Given the `args` left unused by a `parse_global_options`, - return the invoked subcommand, the subcommand options, and the - subcommand arguments. - """ - # Help is default command - if not args: - args = ['help'] - - cmdname = args.pop(0) - subcommand = self._subcommand_for_name(cmdname) - if not subcommand: - raise UserError(u"unknown command '{0}'".format(cmdname)) - - suboptions, subargs = subcommand.parse_args(args) - return subcommand, suboptions, subargs - - -optparse.Option.ALWAYS_TYPED_ACTIONS += ('callback',) - - -def vararg_callback(option, opt_str, value, parser): - """Callback for an option with variable arguments. - Manually collect arguments right of a callback-action - option (ie. with action="callback"), and add the resulting - list to the destination var. - - Usage: - parser.add_option("-c", "--callback", dest="vararg_attr", - action="callback", callback=vararg_callback) - - Details: - http://docs.python.org/2/library/optparse.html#callback-example-6-variable - -arguments - """ - value = [value] - - def floatable(str): - try: - float(str) - return True - except ValueError: - return False - - for arg in parser.rargs: - # stop on --foo like options - if arg[:2] == "--" and len(arg) > 2: - break - # stop on -a, but not on -3 or -3.0 - if arg[:1] == "-" and len(arg) > 1 and not floatable(arg): - break - value.append(arg) - - del parser.rargs[:len(value) - 1] - setattr(parser.values, option.dest, value) - - -# The main entry point and bootstrapping. - -def _load_plugins(config): - """Load the plugins specified in the configuration. - """ - paths = config['pluginpath'].get(confit.StrSeq(split=False)) - paths = map(util.normpath, paths) - log.debug(u'plugin paths: {0}', util.displayable_path(paths)) - - import beetsplug - beetsplug.__path__ = paths + beetsplug.__path__ - # For backwards compatibility. - sys.path += paths - - plugins.load_plugins(config['plugins'].as_str_seq()) - plugins.send("pluginload") - return plugins - - -def _setup(options, lib=None): - """Prepare and global state and updates it with command line options. - - Returns a list of subcommands, a list of plugins, and a library instance. - """ - # Configure the MusicBrainz API. - mb.configure() - - config = _configure(options) - - plugins = _load_plugins(config) - - # Get the default subcommands. - from beets.ui.commands import default_commands - - subcommands = list(default_commands) - subcommands.extend(plugins.commands()) - - if lib is None: - lib = _open_library(config) - plugins.send("library_opened", lib=lib) - library.Item._types.update(plugins.types(library.Item)) - library.Album._types.update(plugins.types(library.Album)) - - return subcommands, plugins, lib - - -def _configure(options): - """Amend the global configuration object with command line options. - """ - # Add any additional config files specified with --config. This - # special handling lets specified plugins get loaded before we - # finish parsing the command line. - if getattr(options, 'config', None) is not None: - config_path = options.config - del options.config - config.set_file(config_path) - config.set_args(options) - - # Configure the logger. - if config['verbose'].get(int): - log.set_global_level(logging.DEBUG) - else: - log.set_global_level(logging.INFO) - - # Ensure compatibility with old (top-level) color configuration. - # Deprecation msg to motivate user to switch to config['ui']['color]. - if config['color'].exists(): - log.warning(u'Warning: top-level configuration of `color` ' - u'is deprecated. Configure color use under `ui`. ' - u'See documentation for more info.') - config['ui']['color'].set(config['color'].get(bool)) - - # Compatibility from list_format_{item,album} to format_{item,album} - for elem in ('item', 'album'): - old_key = 'list_format_{0}'.format(elem) - if config[old_key].exists(): - new_key = 'format_{0}'.format(elem) - log.warning( - u'Warning: configuration uses "{0}" which is deprecated' - u' in favor of "{1}" now that it affects all commands. ' - u'See changelog & documentation.', - old_key, - new_key, - ) - config[new_key].set(config[old_key]) - - config_path = config.user_config_path() - if os.path.isfile(config_path): - log.debug(u'user configuration: {0}', - util.displayable_path(config_path)) - else: - log.debug(u'no user configuration found at {0}', - util.displayable_path(config_path)) - - log.debug(u'data directory: {0}', - util.displayable_path(config.config_dir())) - return config - - -def _open_library(config): - """Create a new library instance from the configuration. - """ - dbpath = config['library'].as_filename() - try: - lib = library.Library( - dbpath, - config['directory'].as_filename(), - get_path_formats(), - get_replacements(), - ) - lib.get_item(0) # Test database connection. - except (sqlite3.OperationalError, sqlite3.DatabaseError): - log.debug(u'{}', traceback.format_exc()) - raise UserError(u"database file {0} could not be opened".format( - util.displayable_path(dbpath) - )) - log.debug(u'library database: {0}\n' - u'library directory: {1}', - util.displayable_path(lib.path), - util.displayable_path(lib.directory)) - return lib - - -def _raw_main(args, lib=None): - """A helper function for `main` without top-level exception - handling. - """ - parser = SubcommandsOptionParser() - parser.add_format_option(flags=('--format-item',), target=library.Item) - parser.add_format_option(flags=('--format-album',), target=library.Album) - parser.add_option('-l', '--library', dest='library', - help=u'library database file to use') - parser.add_option('-d', '--directory', dest='directory', - help=u"destination music directory") - parser.add_option('-v', '--verbose', dest='verbose', action='count', - help=u'log more details (use twice for even more)') - parser.add_option('-c', '--config', dest='config', - help=u'path to configuration file') - parser.add_option('-h', '--help', dest='help', action='store_true', - help=u'show this help message and exit') - parser.add_option('--version', dest='version', action='store_true', - help=optparse.SUPPRESS_HELP) - - options, subargs = parser.parse_global_options(args) - - # Special case for the `config --edit` command: bypass _setup so - # that an invalid configuration does not prevent the editor from - # starting. - if subargs and subargs[0] == 'config' \ - and ('-e' in subargs or '--edit' in subargs): - from beets.ui.commands import config_edit - return config_edit() - - subcommands, plugins, lib = _setup(options, lib) - parser.add_subcommand(*subcommands) - - subcommand, suboptions, subargs = parser.parse_subcommand(subargs) - subcommand.func(lib, suboptions, subargs) - - plugins.send('cli_exit', lib=lib) - - -def main(args=None): - """Run the main command-line interface for beets. Includes top-level - exception handlers that print friendly error messages. - """ - try: - _raw_main(args) - except UserError as exc: - message = exc.args[0] if exc.args else None - log.error(u'error: {0}', message) - sys.exit(1) - except util.HumanReadableException as exc: - exc.log(log) - sys.exit(1) - except library.FileOperationError as exc: - # These errors have reasonable human-readable descriptions, but - # we still want to log their tracebacks for debugging. - log.debug('{}', traceback.format_exc()) - log.error('{}', exc) - sys.exit(1) - except confit.ConfigError as exc: - log.error(u'configuration error: {0}', exc) - sys.exit(1) - except db_query.InvalidQueryError as exc: - log.error(u'invalid query: {0}', exc) - sys.exit(1) - except IOError as exc: - if exc.errno == errno.EPIPE: - # "Broken pipe". End silently. - pass - else: - raise - except KeyboardInterrupt: - # Silently ignore ^C except in verbose mode. - log.debug(u'{}', traceback.format_exc()) diff --git a/libs/beets/ui/commands.py b/libs/beets/ui/commands.py deleted file mode 100644 index 867a47379..000000000 --- a/libs/beets/ui/commands.py +++ /dev/null @@ -1,1754 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""This module provides the default commands for beets' command-line -interface. -""" - -from __future__ import division, absolute_import, print_function - -import os -import re -from collections import namedtuple, Counter -from itertools import chain - -import beets -from beets import ui -from beets.ui import print_, input_, decargs, show_path_changes -from beets import autotag -from beets.autotag import Recommendation -from beets.autotag import hooks -from beets import plugins -from beets import importer -from beets import util -from beets.util import syspath, normpath, ancestry, displayable_path -from beets import library -from beets import config -from beets import logging -from beets.util.confit import _package_path - -VARIOUS_ARTISTS = u'Various Artists' -PromptChoice = namedtuple('ExtraChoice', ['short', 'long', 'callback']) - -# Global logger. -log = logging.getLogger('beets') - -# The list of default subcommands. This is populated with Subcommand -# objects that can be fed to a SubcommandsOptionParser. -default_commands = [] - - -# Utilities. - -def _do_query(lib, query, album, also_items=True): - """For commands that operate on matched items, performs a query - and returns a list of matching items and a list of matching - albums. (The latter is only nonempty when album is True.) Raises - a UserError if no items match. also_items controls whether, when - fetching albums, the associated items should be fetched also. - """ - if album: - albums = list(lib.albums(query)) - items = [] - if also_items: - for al in albums: - items += al.items() - - else: - albums = [] - items = list(lib.items(query)) - - if album and not albums: - raise ui.UserError(u'No matching albums found.') - elif not album and not items: - raise ui.UserError(u'No matching items found.') - - return items, albums - - -# fields: Shows a list of available fields for queries and format strings. - -def _print_keys(query): - """Given a SQLite query result, print the `key` field of each - returned row, with identation of 2 spaces. - """ - for row in query: - print_(' ' * 2 + row['key']) - - -def fields_func(lib, opts, args): - def _print_rows(names): - names.sort() - print_(" " + "\n ".join(names)) - - print_(u"Item fields:") - _print_rows(library.Item.all_keys()) - - print_(u"Album fields:") - _print_rows(library.Album.all_keys()) - - with lib.transaction() as tx: - # The SQL uses the DISTINCT to get unique values from the query - unique_fields = 'SELECT DISTINCT key FROM (%s)' - - print_(u"Item flexible attributes:") - _print_keys(tx.query(unique_fields % library.Item._flex_table)) - - print_(u"Album flexible attributes:") - _print_keys(tx.query(unique_fields % library.Album._flex_table)) - -fields_cmd = ui.Subcommand( - 'fields', - help=u'show fields available for queries and format strings' -) -fields_cmd.func = fields_func -default_commands.append(fields_cmd) - - -# help: Print help text for commands - -class HelpCommand(ui.Subcommand): - - def __init__(self): - super(HelpCommand, self).__init__( - 'help', aliases=('?',), - help=u'give detailed help on a specific sub-command', - ) - - def func(self, lib, opts, args): - if args: - cmdname = args[0] - helpcommand = self.root_parser._subcommand_for_name(cmdname) - if not helpcommand: - raise ui.UserError(u"unknown command '{0}'".format(cmdname)) - helpcommand.print_help() - else: - self.root_parser.print_help() - - -default_commands.append(HelpCommand()) - - -# import: Autotagger and importer. - -# Importer utilities and support. - -def disambig_string(info): - """Generate a string for an AlbumInfo or TrackInfo object that - provides context that helps disambiguate similar-looking albums and - tracks. - """ - disambig = [] - if info.data_source and info.data_source != 'MusicBrainz': - disambig.append(info.data_source) - - if isinstance(info, hooks.AlbumInfo): - if info.media: - if info.mediums > 1: - disambig.append(u'{0}x{1}'.format( - info.mediums, info.media - )) - else: - disambig.append(info.media) - if info.year: - disambig.append(unicode(info.year)) - if info.country: - disambig.append(info.country) - if info.label: - disambig.append(info.label) - if info.albumdisambig: - disambig.append(info.albumdisambig) - - if disambig: - return u', '.join(disambig) - - -def dist_string(dist): - """Formats a distance (a float) as a colorized similarity percentage - string. - """ - out = u'%.1f%%' % ((1 - dist) * 100) - if dist <= config['match']['strong_rec_thresh'].as_number(): - out = ui.colorize('text_success', out) - elif dist <= config['match']['medium_rec_thresh'].as_number(): - out = ui.colorize('text_warning', out) - else: - out = ui.colorize('text_error', out) - return out - - -def penalty_string(distance, limit=None): - """Returns a colorized string that indicates all the penalties - applied to a distance object. - """ - penalties = [] - for key in distance.keys(): - key = key.replace('album_', '') - key = key.replace('track_', '') - key = key.replace('_', ' ') - penalties.append(key) - if penalties: - if limit and len(penalties) > limit: - penalties = penalties[:limit] + ['...'] - return ui.colorize('text_warning', u'(%s)' % ', '.join(penalties)) - - -def show_change(cur_artist, cur_album, match): - """Print out a representation of the changes that will be made if an - album's tags are changed according to `match`, which must be an AlbumMatch - object. - """ - def show_album(artist, album): - if artist: - album_description = u' %s - %s' % (artist, album) - elif album: - album_description = u' %s' % album - else: - album_description = u' (unknown album)' - print_(album_description) - - def format_index(track_info): - """Return a string representing the track index of the given - TrackInfo or Item object. - """ - if isinstance(track_info, hooks.TrackInfo): - index = track_info.index - medium_index = track_info.medium_index - medium = track_info.medium - mediums = match.info.mediums - else: - index = medium_index = track_info.track - medium = track_info.disc - mediums = track_info.disctotal - if config['per_disc_numbering']: - if mediums > 1: - return u'{0}-{1}'.format(medium, medium_index) - else: - return unicode(medium_index) - else: - return unicode(index) - - # Identify the album in question. - if cur_artist != match.info.artist or \ - (cur_album != match.info.album and - match.info.album != VARIOUS_ARTISTS): - artist_l, artist_r = cur_artist or '', match.info.artist - album_l, album_r = cur_album or '', match.info.album - if artist_r == VARIOUS_ARTISTS: - # Hide artists for VA releases. - artist_l, artist_r = u'', u'' - - artist_l, artist_r = ui.colordiff(artist_l, artist_r) - album_l, album_r = ui.colordiff(album_l, album_r) - - print_(u"Correcting tags from:") - show_album(artist_l, album_l) - print_(u"To:") - show_album(artist_r, album_r) - else: - print_(u"Tagging:\n {0.artist} - {0.album}".format(match.info)) - - # Data URL. - if match.info.data_url: - print_(u'URL:\n %s' % match.info.data_url) - - # Info line. - info = [] - # Similarity. - info.append(u'(Similarity: %s)' % dist_string(match.distance)) - # Penalties. - penalties = penalty_string(match.distance) - if penalties: - info.append(penalties) - # Disambiguation. - disambig = disambig_string(match.info) - if disambig: - info.append(ui.colorize('text_highlight_minor', u'(%s)' % disambig)) - print_(' '.join(info)) - - # Tracks. - pairs = match.mapping.items() - pairs.sort(key=lambda item_and_track_info: item_and_track_info[1].index) - - # Build up LHS and RHS for track difference display. The `lines` list - # contains ``(lhs, rhs, width)`` tuples where `width` is the length (in - # characters) of the uncolorized LHS. - lines = [] - medium = disctitle = None - for item, track_info in pairs: - - # Medium number and title. - if medium != track_info.medium or disctitle != track_info.disctitle: - media = match.info.media or 'Media' - if match.info.mediums > 1 and track_info.disctitle: - lhs = u'%s %s: %s' % (media, track_info.medium, - track_info.disctitle) - elif match.info.mediums > 1: - lhs = u'%s %s' % (media, track_info.medium) - elif track_info.disctitle: - lhs = u'%s: %s' % (media, track_info.disctitle) - else: - lhs = None - if lhs: - lines.append((lhs, u'', 0)) - medium, disctitle = track_info.medium, track_info.disctitle - - # Titles. - new_title = track_info.title - if not item.title.strip(): - # If there's no title, we use the filename. - cur_title = displayable_path(os.path.basename(item.path)) - lhs, rhs = cur_title, new_title - else: - cur_title = item.title.strip() - lhs, rhs = ui.colordiff(cur_title, new_title) - lhs_width = len(cur_title) - - # Track number change. - cur_track, new_track = format_index(item), format_index(track_info) - if cur_track != new_track: - if item.track in (track_info.index, track_info.medium_index): - color = 'text_highlight_minor' - else: - color = 'text_highlight' - templ = ui.colorize(color, u' (#{0})') - lhs += templ.format(cur_track) - rhs += templ.format(new_track) - lhs_width += len(cur_track) + 4 - - # Length change. - if item.length and track_info.length and \ - abs(item.length - track_info.length) > \ - config['ui']['length_diff_thresh'].as_number(): - cur_length = ui.human_seconds_short(item.length) - new_length = ui.human_seconds_short(track_info.length) - templ = ui.colorize('text_highlight', u' ({0})') - lhs += templ.format(cur_length) - rhs += templ.format(new_length) - lhs_width += len(cur_length) + 3 - - # Penalties. - penalties = penalty_string(match.distance.tracks[track_info]) - if penalties: - rhs += ' %s' % penalties - - if lhs != rhs: - lines.append((u' * %s' % lhs, rhs, lhs_width)) - elif config['import']['detail']: - lines.append((u' * %s' % lhs, '', lhs_width)) - - # Print each track in two columns, or across two lines. - col_width = (ui.term_width() - len(''.join([' * ', ' -> ']))) // 2 - if lines: - max_width = max(w for _, _, w in lines) - for lhs, rhs, lhs_width in lines: - if not rhs: - print_(lhs) - elif max_width > col_width: - print_(u'%s ->\n %s' % (lhs, rhs)) - else: - pad = max_width - lhs_width - print_(u'%s%s -> %s' % (lhs, ' ' * pad, rhs)) - - # Missing and unmatched tracks. - if match.extra_tracks: - print_(u'Missing tracks ({0}/{1} - {2:.1%}):'.format( - len(match.extra_tracks), - len(match.info.tracks), - len(match.extra_tracks) / len(match.info.tracks) - )) - for track_info in match.extra_tracks: - line = u' ! %s (#%s)' % (track_info.title, format_index(track_info)) - if track_info.length: - line += u' (%s)' % ui.human_seconds_short(track_info.length) - print_(ui.colorize('text_warning', line)) - if match.extra_items: - print_(u'Unmatched tracks ({0}):'.format(len(match.extra_items))) - for item in match.extra_items: - line = u' ! %s (#%s)' % (item.title, format_index(item)) - if item.length: - line += u' (%s)' % ui.human_seconds_short(item.length) - print_(ui.colorize('text_warning', line)) - - -def show_item_change(item, match): - """Print out the change that would occur by tagging `item` with the - metadata from `match`, a TrackMatch object. - """ - cur_artist, new_artist = item.artist, match.info.artist - cur_title, new_title = item.title, match.info.title - - if cur_artist != new_artist or cur_title != new_title: - cur_artist, new_artist = ui.colordiff(cur_artist, new_artist) - cur_title, new_title = ui.colordiff(cur_title, new_title) - - print_(u"Correcting track tags from:") - print_(u" %s - %s" % (cur_artist, cur_title)) - print_(u"To:") - print_(u" %s - %s" % (new_artist, new_title)) - - else: - print_(u"Tagging track: %s - %s" % (cur_artist, cur_title)) - - # Data URL. - if match.info.data_url: - print_(u'URL:\n %s' % match.info.data_url) - - # Info line. - info = [] - # Similarity. - info.append(u'(Similarity: %s)' % dist_string(match.distance)) - # Penalties. - penalties = penalty_string(match.distance) - if penalties: - info.append(penalties) - # Disambiguation. - disambig = disambig_string(match.info) - if disambig: - info.append(ui.colorize('text_highlight_minor', u'(%s)' % disambig)) - print_(' '.join(info)) - - -def summarize_items(items, singleton): - """Produces a brief summary line describing a set of items. Used for - manually resolving duplicates during import. - - `items` is a list of `Item` objects. `singleton` indicates whether - this is an album or single-item import (if the latter, them `items` - should only have one element). - """ - summary_parts = [] - if not singleton: - summary_parts.append(u"{0} items".format(len(items))) - - format_counts = {} - for item in items: - format_counts[item.format] = format_counts.get(item.format, 0) + 1 - if len(format_counts) == 1: - # A single format. - summary_parts.append(items[0].format) - else: - # Enumerate all the formats by decreasing frequencies: - for fmt, count in sorted( - format_counts.items(), - key=lambda fmt_and_count: (-fmt_and_count[1], fmt_and_count[0]) - ): - summary_parts.append('{0} {1}'.format(fmt, count)) - - if items: - average_bitrate = sum([item.bitrate for item in items]) / len(items) - total_duration = sum([item.length for item in items]) - total_filesize = sum([item.filesize for item in items]) - summary_parts.append(u'{0}kbps'.format(int(average_bitrate / 1000))) - summary_parts.append(ui.human_seconds_short(total_duration)) - summary_parts.append(ui.human_bytes(total_filesize)) - - return u', '.join(summary_parts) - - -def _summary_judgment(rec): - """Determines whether a decision should be made without even asking - the user. This occurs in quiet mode and when an action is chosen for - NONE recommendations. Return an action or None if the user should be - queried. May also print to the console if a summary judgment is - made. - """ - if config['import']['quiet']: - if rec == Recommendation.strong: - return importer.action.APPLY - else: - action = config['import']['quiet_fallback'].as_choice({ - 'skip': importer.action.SKIP, - 'asis': importer.action.ASIS, - }) - - elif rec == Recommendation.none: - action = config['import']['none_rec_action'].as_choice({ - 'skip': importer.action.SKIP, - 'asis': importer.action.ASIS, - 'ask': None, - }) - - else: - return None - - if action == importer.action.SKIP: - print_(u'Skipping.') - elif action == importer.action.ASIS: - print_(u'Importing as-is.') - return action - - -def choose_candidate(candidates, singleton, rec, cur_artist=None, - cur_album=None, item=None, itemcount=None, - extra_choices=[]): - """Given a sorted list of candidates, ask the user for a selection - of which candidate to use. Applies to both full albums and - singletons (tracks). Candidates are either AlbumMatch or TrackMatch - objects depending on `singleton`. for albums, `cur_artist`, - `cur_album`, and `itemcount` must be provided. For singletons, - `item` must be provided. - - `extra_choices` is a list of `PromptChoice`s, containg the choices - appended by the plugins after receiving the `before_choose_candidate` - event. If not empty, the choices are appended to the prompt presented - to the user. - - Returns one of the following: - * the result of the choice, which may be SKIP, ASIS, TRACKS, or MANUAL - * a candidate (an AlbumMatch/TrackMatch object) - * the short letter of a `PromptChoice` (if the user selected one of - the `extra_choices`). - """ - # Sanity check. - if singleton: - assert item is not None - else: - assert cur_artist is not None - assert cur_album is not None - - # Build helper variables for extra choices. - extra_opts = tuple(c.long for c in extra_choices) - extra_actions = tuple(c.short for c in extra_choices) - - # Zero candidates. - if not candidates: - if singleton: - print_(u"No matching recordings found.") - opts = (u'Use as-is', u'Skip', u'Enter search', u'enter Id', - u'aBort') - else: - print_(u"No matching release found for {0} tracks." - .format(itemcount)) - print_(u'For help, see: ' - u'http://beets.readthedocs.org/en/latest/faq.html#nomatch') - opts = (u'Use as-is', u'as Tracks', u'Group albums', u'Skip', - u'Enter search', u'enter Id', u'aBort') - sel = ui.input_options(opts + extra_opts) - if sel == u'u': - return importer.action.ASIS - elif sel == u't': - assert not singleton - return importer.action.TRACKS - elif sel == u'e': - return importer.action.MANUAL - elif sel == u's': - return importer.action.SKIP - elif sel == u'b': - raise importer.ImportAbort() - elif sel == u'i': - return importer.action.MANUAL_ID - elif sel == u'g': - return importer.action.ALBUMS - elif sel in extra_actions: - return sel - else: - assert False - - # Is the change good enough? - bypass_candidates = False - if rec != Recommendation.none: - match = candidates[0] - bypass_candidates = True - - while True: - # Display and choose from candidates. - require = rec <= Recommendation.low - - if not bypass_candidates: - # Display list of candidates. - print_(u'Finding tags for {0} "{1} - {2}".'.format( - u'track' if singleton else u'album', - item.artist if singleton else cur_artist, - item.title if singleton else cur_album, - )) - - print_(u'Candidates:') - for i, match in enumerate(candidates): - # Index, metadata, and distance. - line = [ - u'{0}.'.format(i + 1), - u'{0} - {1}'.format( - match.info.artist, - match.info.title if singleton else match.info.album, - ), - u'({0})'.format(dist_string(match.distance)), - ] - - # Penalties. - penalties = penalty_string(match.distance, 3) - if penalties: - line.append(penalties) - - # Disambiguation - disambig = disambig_string(match.info) - if disambig: - line.append(ui.colorize('text_highlight_minor', - u'(%s)' % disambig)) - - print_(u' '.join(line)) - - # Ask the user for a choice. - if singleton: - opts = (u'Skip', u'Use as-is', u'Enter search', u'enter Id', - u'aBort') - else: - opts = (u'Skip', u'Use as-is', u'as Tracks', u'Group albums', - u'Enter search', u'enter Id', u'aBort') - sel = ui.input_options(opts + extra_opts, - numrange=(1, len(candidates))) - if sel == u's': - return importer.action.SKIP - elif sel == u'u': - return importer.action.ASIS - elif sel == u'm': - pass - elif sel == u'e': - return importer.action.MANUAL - elif sel == u't': - assert not singleton - return importer.action.TRACKS - elif sel == u'b': - raise importer.ImportAbort() - elif sel == u'i': - return importer.action.MANUAL_ID - elif sel == u'g': - return importer.action.ALBUMS - elif sel in extra_actions: - return sel - else: # Numerical selection. - match = candidates[sel - 1] - if sel != 1: - # When choosing anything but the first match, - # disable the default action. - require = True - bypass_candidates = False - - # Show what we're about to do. - if singleton: - show_item_change(item, match) - else: - show_change(cur_artist, cur_album, match) - - # Exact match => tag automatically if we're not in timid mode. - if rec == Recommendation.strong and not config['import']['timid']: - return match - - # Ask for confirmation. - if singleton: - opts = (u'Apply', u'More candidates', u'Skip', u'Use as-is', - u'Enter search', u'enter Id', u'aBort') - else: - opts = (u'Apply', u'More candidates', u'Skip', u'Use as-is', - u'as Tracks', u'Group albums', u'Enter search', - u'enter Id', u'aBort') - default = config['import']['default_action'].as_choice({ - u'apply': u'a', - u'skip': u's', - u'asis': u'u', - u'none': None, - }) - if default is None: - require = True - sel = ui.input_options(opts + extra_opts, require=require, - default=default) - if sel == u'a': - return match - elif sel == u'g': - return importer.action.ALBUMS - elif sel == u's': - return importer.action.SKIP - elif sel == u'u': - return importer.action.ASIS - elif sel == u't': - assert not singleton - return importer.action.TRACKS - elif sel == u'e': - return importer.action.MANUAL - elif sel == u'b': - raise importer.ImportAbort() - elif sel == u'i': - return importer.action.MANUAL_ID - elif sel in extra_actions: - return sel - - -def manual_search(singleton): - """Input either an artist and album (for full albums) or artist and - track name (for singletons) for manual search. - """ - artist = input_(u'Artist:') - name = input_(u'Track:' if singleton else u'Album:') - return artist.strip(), name.strip() - - -def manual_id(singleton): - """Input an ID, either for an album ("release") or a track ("recording"). - """ - prompt = u'Enter {0} ID:'.format(u'recording' if singleton else u'release') - return input_(prompt).strip() - - -class TerminalImportSession(importer.ImportSession): - """An import session that runs in a terminal. - """ - def choose_match(self, task): - """Given an initial autotagging of items, go through an interactive - dance with the user to ask for a choice of metadata. Returns an - AlbumMatch object, ASIS, or SKIP. - """ - # Show what we're tagging. - print_() - print_(displayable_path(task.paths, u'\n') + - u' ({0} items)'.format(len(task.items))) - - # Take immediate action if appropriate. - action = _summary_judgment(task.rec) - if action == importer.action.APPLY: - match = task.candidates[0] - show_change(task.cur_artist, task.cur_album, match) - return match - elif action is not None: - return action - - # Loop until we have a choice. - candidates, rec = task.candidates, task.rec - while True: - # Gather extra choices from plugins. - extra_choices = self._get_plugin_choices(task) - extra_ops = {c.short: c.callback for c in extra_choices} - - # Ask for a choice from the user. - choice = choose_candidate( - candidates, False, rec, task.cur_artist, task.cur_album, - itemcount=len(task.items), extra_choices=extra_choices - ) - - # Choose which tags to use. - if choice in (importer.action.SKIP, importer.action.ASIS, - importer.action.TRACKS, importer.action.ALBUMS): - # Pass selection to main control flow. - return choice - elif choice is importer.action.MANUAL: - # Try again with manual search terms. - search_artist, search_album = manual_search(False) - _, _, candidates, rec = autotag.tag_album( - task.items, search_artist, search_album - ) - elif choice is importer.action.MANUAL_ID: - # Try a manually-entered ID. - search_id = manual_id(False) - if search_id: - _, _, candidates, rec = autotag.tag_album( - task.items, search_ids=search_id.split() - ) - elif choice in extra_ops.keys(): - # Allow extra ops to automatically set the post-choice. - post_choice = extra_ops[choice](self, task) - if isinstance(post_choice, importer.action): - # MANUAL and MANUAL_ID have no effect, even if returned. - return post_choice - else: - # We have a candidate! Finish tagging. Here, choice is an - # AlbumMatch object. - assert isinstance(choice, autotag.AlbumMatch) - return choice - - def choose_item(self, task): - """Ask the user for a choice about tagging a single item. Returns - either an action constant or a TrackMatch object. - """ - print_() - print_(task.item.path) - candidates, rec = task.candidates, task.rec - - # Take immediate action if appropriate. - action = _summary_judgment(task.rec) - if action == importer.action.APPLY: - match = candidates[0] - show_item_change(task.item, match) - return match - elif action is not None: - return action - - while True: - extra_choices = self._get_plugin_choices(task) - extra_ops = {c.short: c.callback for c in extra_choices} - - # Ask for a choice. - choice = choose_candidate(candidates, True, rec, item=task.item, - extra_choices=extra_choices) - - if choice in (importer.action.SKIP, importer.action.ASIS): - return choice - elif choice == importer.action.TRACKS: - assert False # TRACKS is only legal for albums. - elif choice == importer.action.MANUAL: - # Continue in the loop with a new set of candidates. - search_artist, search_title = manual_search(True) - candidates, rec = autotag.tag_item(task.item, search_artist, - search_title) - elif choice == importer.action.MANUAL_ID: - # Ask for a track ID. - search_id = manual_id(True) - if search_id: - candidates, rec = autotag.tag_item( - task.item, search_ids=search_id.split()) - elif choice in extra_ops.keys(): - # Allow extra ops to automatically set the post-choice. - post_choice = extra_ops[choice](self, task) - if isinstance(post_choice, importer.action): - # MANUAL and MANUAL_ID have no effect, even if returned. - return post_choice - else: - # Chose a candidate. - assert isinstance(choice, autotag.TrackMatch) - return choice - - def resolve_duplicate(self, task, found_duplicates): - """Decide what to do when a new album or item seems similar to one - that's already in the library. - """ - log.warn(u"This {0} is already in the library!", - (u"album" if task.is_album else u"item")) - - if config['import']['quiet']: - # In quiet mode, don't prompt -- just skip. - log.info(u'Skipping.') - sel = u's' - else: - # Print some detail about the existing and new items so the - # user can make an informed decision. - for duplicate in found_duplicates: - print_(u"Old: " + summarize_items( - list(duplicate.items()) if task.is_album else [duplicate], - not task.is_album, - )) - - print_(u"New: " + summarize_items( - task.imported_items(), - not task.is_album, - )) - - sel = ui.input_options( - (u'Skip new', u'Keep both', u'Remove old') - ) - - if sel == u's': - # Skip new. - task.set_choice(importer.action.SKIP) - elif sel == u'k': - # Keep both. Do nothing; leave the choice intact. - pass - elif sel == u'r': - # Remove old. - task.should_remove_duplicates = True - else: - assert False - - def should_resume(self, path): - return ui.input_yn(u"Import of the directory:\n{0}\n" - u"was interrupted. Resume (Y/n)?" - .format(displayable_path(path))) - - def _get_plugin_choices(self, task): - """Get the extra choices appended to the plugins to the ui prompt. - - The `before_choose_candidate` event is sent to the plugins, with - session and task as its parameters. Plugins are responsible for - checking the right conditions and returning a list of `PromptChoice`s, - which is flattened and checked for conflicts. - - If two or more choices have the same short letter, a warning is - emitted and all but one choices are discarded, giving preference - to the default importer choices. - - Returns a list of `PromptChoice`s. - """ - # Send the before_choose_candidate event and flatten list. - extra_choices = list(chain(*plugins.send('before_choose_candidate', - session=self, task=task))) - # Take into account default options, for duplicate checking. - all_choices = [PromptChoice(u'a', u'Apply', None), - PromptChoice(u's', u'Skip', None), - PromptChoice(u'u', u'Use as-is', None), - PromptChoice(u't', u'as Tracks', None), - PromptChoice(u'g', u'Group albums', None), - PromptChoice(u'e', u'Enter search', None), - PromptChoice(u'i', u'enter Id', None), - PromptChoice(u'b', u'aBort', None)] +\ - extra_choices - - short_letters = [c.short for c in all_choices] - if len(short_letters) != len(set(short_letters)): - # Duplicate short letter has been found. - duplicates = [i for i, count in Counter(short_letters).items() - if count > 1] - for short in duplicates: - # Keep the first of the choices, removing the rest. - dup_choices = [c for c in all_choices if c.short == short] - for c in dup_choices[1:]: - log.warn(u"Prompt choice '{0}' removed due to conflict " - u"with '{1}' (short letter: '{2}')", - c.long, dup_choices[0].long, c.short) - extra_choices.remove(c) - return extra_choices - - -# The import command. - - -def import_files(lib, paths, query): - """Import the files in the given list of paths or matching the - query. - """ - # Check the user-specified directories. - for path in paths: - if not os.path.exists(syspath(normpath(path))): - raise ui.UserError(u'no such file or directory: {0}'.format( - displayable_path(path))) - - # Check parameter consistency. - if config['import']['quiet'] and config['import']['timid']: - raise ui.UserError(u"can't be both quiet and timid") - - # Open the log. - if config['import']['log'].get() is not None: - logpath = syspath(config['import']['log'].as_filename()) - try: - loghandler = logging.FileHandler(logpath) - except IOError: - raise ui.UserError(u"could not open log file for writing: " - u"{0}".format(displayable_path(logpath))) - else: - loghandler = None - - # Never ask for input in quiet mode. - if config['import']['resume'].get() == 'ask' and \ - config['import']['quiet']: - config['import']['resume'] = False - - session = TerminalImportSession(lib, loghandler, paths, query) - session.run() - - # Emit event. - plugins.send('import', lib=lib, paths=paths) - - -def import_func(lib, opts, args): - config['import'].set_args(opts) - - # Special case: --copy flag suppresses import_move (which would - # otherwise take precedence). - if opts.copy: - config['import']['move'] = False - - if opts.library: - query = decargs(args) - paths = [] - else: - query = None - paths = args - if not paths: - raise ui.UserError(u'no path specified') - - import_files(lib, paths, query) - - -import_cmd = ui.Subcommand( - u'import', help=u'import new music', aliases=(u'imp', u'im') -) -import_cmd.parser.add_option( - u'-c', u'--copy', action='store_true', default=None, - help=u"copy tracks into library directory (default)" -) -import_cmd.parser.add_option( - u'-C', u'--nocopy', action='store_false', dest='copy', - help=u"don't copy tracks (opposite of -c)" -) -import_cmd.parser.add_option( - u'-w', u'--write', action='store_true', default=None, - help=u"write new metadata to files' tags (default)" -) -import_cmd.parser.add_option( - u'-W', u'--nowrite', action='store_false', dest='write', - help=u"don't write metadata (opposite of -w)" -) -import_cmd.parser.add_option( - u'-a', u'--autotag', action='store_true', dest='autotag', - help=u"infer tags for imported files (default)" -) -import_cmd.parser.add_option( - u'-A', u'--noautotag', action='store_false', dest='autotag', - help=u"don't infer tags for imported files (opposite of -a)" -) -import_cmd.parser.add_option( - u'-p', u'--resume', action='store_true', default=None, - help=u"resume importing if interrupted" -) -import_cmd.parser.add_option( - u'-P', u'--noresume', action='store_false', dest='resume', - help=u"do not try to resume importing" -) -import_cmd.parser.add_option( - u'-q', u'--quiet', action='store_true', dest='quiet', - help=u"never prompt for input: skip albums instead" -) -import_cmd.parser.add_option( - u'-l', u'--log', dest='log', - help=u'file to log untaggable albums for later review' -) -import_cmd.parser.add_option( - u'-s', u'--singletons', action='store_true', - help=u'import individual tracks instead of full albums' -) -import_cmd.parser.add_option( - u'-t', u'--timid', dest='timid', action='store_true', - help=u'always confirm all actions' -) -import_cmd.parser.add_option( - u'-L', u'--library', dest='library', action='store_true', - help=u'retag items matching a query' -) -import_cmd.parser.add_option( - u'-i', u'--incremental', dest='incremental', action='store_true', - help=u'skip already-imported directories' -) -import_cmd.parser.add_option( - u'-I', u'--noincremental', dest='incremental', action='store_false', - help=u'do not skip already-imported directories' -) -import_cmd.parser.add_option( - u'--flat', dest='flat', action='store_true', - help=u'import an entire tree as a single album' -) -import_cmd.parser.add_option( - u'-g', u'--group-albums', dest='group_albums', action='store_true', - help=u'group tracks in a folder into separate albums' -) -import_cmd.parser.add_option( - u'--pretend', dest='pretend', action='store_true', - help=u'just print the files to import' -) -import_cmd.parser.add_option( - u'-S', u'--search-id', dest='search_ids', action='append', - metavar='BACKEND_ID', - help=u'restrict matching to a specific metadata backend ID' -) -import_cmd.func = import_func -default_commands.append(import_cmd) - - -# list: Query and show library contents. - -def list_items(lib, query, album, fmt=''): - """Print out items in lib matching query. If album, then search for - albums instead of single items. - """ - if album: - for album in lib.albums(query): - ui.print_(format(album, fmt)) - else: - for item in lib.items(query): - ui.print_(format(item, fmt)) - - -def list_func(lib, opts, args): - list_items(lib, decargs(args), opts.album) - - -list_cmd = ui.Subcommand(u'list', help=u'query the library', aliases=(u'ls',)) -list_cmd.parser.usage += u"\n" \ - u'Example: %prog -f \'$album: $title\' artist:beatles' -list_cmd.parser.add_all_common_options() -list_cmd.func = list_func -default_commands.append(list_cmd) - - -# update: Update library contents according to on-disk tags. - -def update_items(lib, query, album, move, pretend): - """For all the items matched by the query, update the library to - reflect the item's embedded tags. - """ - with lib.transaction(): - items, _ = _do_query(lib, query, album) - - # Walk through the items and pick up their changes. - affected_albums = set() - for item in items: - # Item deleted? - if not os.path.exists(syspath(item.path)): - ui.print_(format(item)) - ui.print_(ui.colorize('text_error', u' deleted')) - if not pretend: - item.remove(True) - affected_albums.add(item.album_id) - continue - - # Did the item change since last checked? - if item.current_mtime() <= item.mtime: - log.debug(u'skipping {0} because mtime is up to date ({1})', - displayable_path(item.path), item.mtime) - continue - - # Read new data. - try: - item.read() - except library.ReadError as exc: - log.error(u'error reading {0}: {1}', - displayable_path(item.path), exc) - continue - - # Special-case album artist when it matches track artist. (Hacky - # but necessary for preserving album-level metadata for non- - # autotagged imports.) - if not item.albumartist: - old_item = lib.get_item(item.id) - if old_item.albumartist == old_item.artist == item.artist: - item.albumartist = old_item.albumartist - item._dirty.discard(u'albumartist') - - # Check for and display changes. - changed = ui.show_model_changes(item, - fields=library.Item._media_fields) - - # Save changes. - if not pretend: - if changed: - # Move the item if it's in the library. - if move and lib.directory in ancestry(item.path): - item.move() - - item.store() - affected_albums.add(item.album_id) - else: - # The file's mtime was different, but there were no - # changes to the metadata. Store the new mtime, - # which is set in the call to read(), so we don't - # check this again in the future. - item.store() - - # Skip album changes while pretending. - if pretend: - return - - # Modify affected albums to reflect changes in their items. - for album_id in affected_albums: - if album_id is None: # Singletons. - continue - album = lib.get_album(album_id) - if not album: # Empty albums have already been removed. - log.debug(u'emptied album {0}', album_id) - continue - first_item = album.items().get() - - # Update album structure to reflect an item in it. - for key in library.Album.item_keys: - album[key] = first_item[key] - album.store() - - # Move album art (and any inconsistent items). - if move and lib.directory in ancestry(first_item.path): - log.debug(u'moving album {0}', album_id) - album.move() - - -def update_func(lib, opts, args): - update_items(lib, decargs(args), opts.album, ui.should_move(opts.move), - opts.pretend) - - -update_cmd = ui.Subcommand( - u'update', help=u'update the library', aliases=(u'upd', u'up',) -) -update_cmd.parser.add_album_option() -update_cmd.parser.add_format_option() -update_cmd.parser.add_option( - u'-m', u'--move', action='store_true', dest='move', - help=u"move files in the library directory" -) -update_cmd.parser.add_option( - u'-M', u'--nomove', action='store_false', dest='move', - help=u"don't move files in library" -) -update_cmd.parser.add_option( - u'-p', u'--pretend', action='store_true', - help=u"show all changes but do nothing" -) -update_cmd.func = update_func -default_commands.append(update_cmd) - - -# remove: Remove items from library, delete files. - -def remove_items(lib, query, album, delete): - """Remove items matching query from lib. If album, then match and - remove whole albums. If delete, also remove files from disk. - """ - # Get the matching items. - items, albums = _do_query(lib, query, album) - - # Prepare confirmation with user. - print_() - if delete: - fmt = u'$path - $title' - prompt = u'Really DELETE %i file%s (y/n)?' % \ - (len(items), 's' if len(items) > 1 else '') - else: - fmt = '' - prompt = u'Really remove %i item%s from the library (y/n)?' % \ - (len(items), 's' if len(items) > 1 else '') - - # Show all the items. - for item in items: - ui.print_(format(item, fmt)) - - # Confirm with user. - if not ui.input_yn(prompt, True): - return - - # Remove (and possibly delete) items. - with lib.transaction(): - for obj in (albums if album else items): - obj.remove(delete) - - -def remove_func(lib, opts, args): - remove_items(lib, decargs(args), opts.album, opts.delete) - - -remove_cmd = ui.Subcommand( - u'remove', help=u'remove matching items from the library', aliases=(u'rm',) -) -remove_cmd.parser.add_option( - u"-d", u"--delete", action="store_true", - help=u"also remove files from disk" -) -remove_cmd.parser.add_album_option() -remove_cmd.func = remove_func -default_commands.append(remove_cmd) - - -# stats: Show library/query statistics. - -def show_stats(lib, query, exact): - """Shows some statistics about the matched items.""" - items = lib.items(query) - - total_size = 0 - total_time = 0.0 - total_items = 0 - artists = set() - albums = set() - album_artists = set() - - for item in items: - if exact: - try: - total_size += os.path.getsize(syspath(item.path)) - except OSError as exc: - log.info(u'could not get size of {}: {}', item.path, exc) - else: - total_size += int(item.length * item.bitrate / 8) - total_time += item.length - total_items += 1 - artists.add(item.artist) - album_artists.add(item.albumartist) - if item.album_id: - albums.add(item.album_id) - - size_str = u'' + ui.human_bytes(total_size) - if exact: - size_str += u' ({0} bytes)'.format(total_size) - - print_(u"""Tracks: {0} -Total time: {1}{2} -{3}: {4} -Artists: {5} -Albums: {6} -Album artists: {7}""".format( - total_items, - ui.human_seconds(total_time), - u' ({0:.2f} seconds)'.format(total_time) if exact else '', - u'Total size' if exact else u'Approximate total size', - size_str, - len(artists), - len(albums), - len(album_artists)), - ) - - -def stats_func(lib, opts, args): - show_stats(lib, decargs(args), opts.exact) - - -stats_cmd = ui.Subcommand( - u'stats', help=u'show statistics about the library or a query' -) -stats_cmd.parser.add_option( - u'-e', u'--exact', action='store_true', - help=u'exact size and time' -) -stats_cmd.func = stats_func -default_commands.append(stats_cmd) - - -# version: Show current beets version. - -def show_version(lib, opts, args): - print_(u'beets version %s' % beets.__version__) - # Show plugins. - names = sorted(p.name for p in plugins.find_plugins()) - if names: - print_(u'plugins:', ', '.join(names)) - else: - print_(u'no plugins loaded') - - -version_cmd = ui.Subcommand( - u'version', help=u'output version information' -) -version_cmd.func = show_version -default_commands.append(version_cmd) - - -# modify: Declaratively change metadata. - -def modify_items(lib, mods, dels, query, write, move, album, confirm): - """Modifies matching items according to user-specified assignments and - deletions. - - `mods` is a dictionary of field and value pairse indicating - assignments. `dels` is a list of fields to be deleted. - """ - # Parse key=value specifications into a dictionary. - model_cls = library.Album if album else library.Item - - for key, value in mods.items(): - mods[key] = model_cls._parse(key, value) - - # Get the items to modify. - items, albums = _do_query(lib, query, album, False) - objs = albums if album else items - - # Apply changes *temporarily*, preview them, and collect modified - # objects. - print_(u'Modifying {0} {1}s.' - .format(len(objs), u'album' if album else u'item')) - changed = set() - for obj in objs: - if print_and_modify(obj, mods, dels): - changed.add(obj) - - # Still something to do? - if not changed: - print_(u'No changes to make.') - return - - # Confirm action. - if confirm: - if write and move: - extra = u', move and write tags' - elif write: - extra = u' and write tags' - elif move: - extra = u' and move' - else: - extra = u'' - - changed = ui.input_select_objects( - u'Really modify%s' % extra, changed, - lambda o: print_and_modify(o, mods, dels) - ) - - # Apply changes to database and files - with lib.transaction(): - for obj in changed: - obj.try_sync(write, move) - - -def print_and_modify(obj, mods, dels): - """Print the modifications to an item and return a bool indicating - whether any changes were made. - - `mods` is a dictionary of fields and values to update on the object; - `dels` is a sequence of fields to delete. - """ - obj.update(mods) - for field in dels: - try: - del obj[field] - except KeyError: - pass - return ui.show_model_changes(obj) - - -def modify_parse_args(args): - """Split the arguments for the modify subcommand into query parts, - assignments (field=value), and deletions (field!). Returns the result as - a three-tuple in that order. - """ - mods = {} - dels = [] - query = [] - for arg in args: - if arg.endswith('!') and '=' not in arg and ':' not in arg: - dels.append(arg[:-1]) # Strip trailing !. - elif '=' in arg and ':' not in arg.split('=', 1)[0]: - key, val = arg.split('=', 1) - mods[key] = val - else: - query.append(arg) - return query, mods, dels - - -def modify_func(lib, opts, args): - query, mods, dels = modify_parse_args(decargs(args)) - if not mods and not dels: - raise ui.UserError(u'no modifications specified') - modify_items(lib, mods, dels, query, ui.should_write(opts.write), - ui.should_move(opts.move), opts.album, not opts.yes) - - -modify_cmd = ui.Subcommand( - u'modify', help=u'change metadata fields', aliases=(u'mod',) -) -modify_cmd.parser.add_option( - u'-m', u'--move', action='store_true', dest='move', - help=u"move files in the library directory" -) -modify_cmd.parser.add_option( - u'-M', u'--nomove', action='store_false', dest='move', - help=u"don't move files in library" -) -modify_cmd.parser.add_option( - u'-w', u'--write', action='store_true', default=None, - help=u"write new metadata to files' tags (default)" -) -modify_cmd.parser.add_option( - u'-W', u'--nowrite', action='store_false', dest='write', - help=u"don't write metadata (opposite of -w)" -) -modify_cmd.parser.add_album_option() -modify_cmd.parser.add_format_option(target='item') -modify_cmd.parser.add_option( - u'-y', u'--yes', action='store_true', - help=u'skip confirmation' -) -modify_cmd.func = modify_func -default_commands.append(modify_cmd) - - -# move: Move/copy files to the library or a new base directory. - -def move_items(lib, dest, query, copy, album, pretend, confirm=False): - """Moves or copies items to a new base directory, given by dest. If - dest is None, then the library's base directory is used, making the - command "consolidate" files. - """ - items, albums = _do_query(lib, query, album, False) - objs = albums if album else items - - # Filter out files that don't need to be moved. - isitemmoved = lambda item: item.path != item.destination(basedir=dest) - isalbummoved = lambda album: any(isitemmoved(i) for i in album.items()) - objs = [o for o in objs if (isalbummoved if album else isitemmoved)(o)] - - action = u'Copying' if copy else u'Moving' - act = u'copy' if copy else u'move' - entity = u'album' if album else u'item' - log.info(u'{0} {1} {2}{3}.', action, len(objs), entity, - u's' if len(objs) != 1 else u'') - if not objs: - return - - if pretend: - if album: - show_path_changes([(item.path, item.destination(basedir=dest)) - for obj in objs for item in obj.items()]) - else: - show_path_changes([(obj.path, obj.destination(basedir=dest)) - for obj in objs]) - else: - if confirm: - objs = ui.input_select_objects( - u'Really %s' % act, objs, - lambda o: show_path_changes( - [(o.path, o.destination(basedir=dest))])) - - for obj in objs: - log.debug(u'moving: {0}', util.displayable_path(obj.path)) - - obj.move(copy, basedir=dest) - obj.store() - - -def move_func(lib, opts, args): - dest = opts.dest - if dest is not None: - dest = normpath(dest) - if not os.path.isdir(dest): - raise ui.UserError(u'no such directory: %s' % dest) - - move_items(lib, dest, decargs(args), opts.copy, opts.album, opts.pretend, - opts.timid) - - -move_cmd = ui.Subcommand( - u'move', help=u'move or copy items', aliases=(u'mv',) -) -move_cmd.parser.add_option( - u'-d', u'--dest', metavar='DIR', dest='dest', - help=u'destination directory' -) -move_cmd.parser.add_option( - u'-c', u'--copy', default=False, action='store_true', - help=u'copy instead of moving' -) -move_cmd.parser.add_option( - u'-p', u'--pretend', default=False, action='store_true', - help=u'show how files would be moved, but don\'t touch anything' -) -move_cmd.parser.add_option( - u'-t', u'--timid', dest='timid', action='store_true', - help=u'always confirm all actions' -) -move_cmd.parser.add_album_option() -move_cmd.func = move_func -default_commands.append(move_cmd) - - -# write: Write tags into files. - -def write_items(lib, query, pretend, force): - """Write tag information from the database to the respective files - in the filesystem. - """ - items, albums = _do_query(lib, query, False, False) - - for item in items: - # Item deleted? - if not os.path.exists(syspath(item.path)): - log.info(u'missing file: {0}', util.displayable_path(item.path)) - continue - - # Get an Item object reflecting the "clean" (on-disk) state. - try: - clean_item = library.Item.from_path(item.path) - except library.ReadError as exc: - log.error(u'error reading {0}: {1}', - displayable_path(item.path), exc) - continue - - # Check for and display changes. - changed = ui.show_model_changes(item, clean_item, - library.Item._media_tag_fields, force) - if (changed or force) and not pretend: - # We use `try_sync` here to keep the mtime up to date in the - # database. - item.try_sync(True, False) - - -def write_func(lib, opts, args): - write_items(lib, decargs(args), opts.pretend, opts.force) - - -write_cmd = ui.Subcommand(u'write', help=u'write tag information to files') -write_cmd.parser.add_option( - u'-p', u'--pretend', action='store_true', - help=u"show all changes but do nothing" -) -write_cmd.parser.add_option( - u'-f', u'--force', action='store_true', - help=u"write tags even if the existing tags match the database" -) -write_cmd.func = write_func -default_commands.append(write_cmd) - - -# config: Show and edit user configuration. - -def config_func(lib, opts, args): - # Make sure lazy configuration is loaded - config.resolve() - - # Print paths. - if opts.paths: - filenames = [] - for source in config.sources: - if not opts.defaults and source.default: - continue - if source.filename: - filenames.append(source.filename) - - # In case the user config file does not exist, prepend it to the - # list. - user_path = config.user_config_path() - if user_path not in filenames: - filenames.insert(0, user_path) - - for filename in filenames: - print_(filename) - - # Open in editor. - elif opts.edit: - config_edit() - - # Dump configuration. - else: - print_(config.dump(full=opts.defaults, redact=opts.redact)) - - -def config_edit(): - """Open a program to edit the user configuration. - An empty config file is created if no existing config file exists. - """ - path = config.user_config_path() - editor = util.editor_command() - try: - if not os.path.isfile(path): - open(path, 'w+').close() - util.interactive_open([path], editor) - except OSError as exc: - message = u"Could not edit configuration: {0}".format(exc) - if not editor: - message += u". Please set the EDITOR environment variable" - raise ui.UserError(message) - -config_cmd = ui.Subcommand(u'config', - help=u'show or edit the user configuration') -config_cmd.parser.add_option( - u'-p', u'--paths', action='store_true', - help=u'show files that configuration was loaded from' -) -config_cmd.parser.add_option( - u'-e', u'--edit', action='store_true', - help=u'edit user configuration with $EDITOR' -) -config_cmd.parser.add_option( - u'-d', u'--defaults', action='store_true', - help=u'include the default configuration' -) -config_cmd.parser.add_option( - u'-c', u'--clear', action='store_false', - dest='redact', default=True, - help=u'do not redact sensitive fields' -) -config_cmd.func = config_func -default_commands.append(config_cmd) - - -# completion: print completion script - -def print_completion(*args): - for line in completion_script(default_commands + plugins.commands()): - print_(line, end='') - if not any(map(os.path.isfile, BASH_COMPLETION_PATHS)): - log.warn(u'Warning: Unable to find the bash-completion package. ' - u'Command line completion might not work.') - -BASH_COMPLETION_PATHS = map(syspath, [ - u'/etc/bash_completion', - u'/usr/share/bash-completion/bash_completion', - u'/usr/share/local/bash-completion/bash_completion', - u'/opt/local/share/bash-completion/bash_completion', # SmartOS - u'/usr/local/etc/bash_completion', # Homebrew -]) - - -def completion_script(commands): - """Yield the full completion shell script as strings. - - ``commands`` is alist of ``ui.Subcommand`` instances to generate - completion data for. - """ - base_script = os.path.join(_package_path('beets.ui'), 'completion_base.sh') - with open(base_script, 'r') as base_script: - yield base_script.read() - - options = {} - aliases = {} - command_names = [] - - # Collect subcommands - for cmd in commands: - name = cmd.name - command_names.append(name) - - for alias in cmd.aliases: - if re.match(r'^\w+$', alias): - aliases[alias] = name - - options[name] = {'flags': [], 'opts': []} - for opts in cmd.parser._get_all_options()[1:]: - if opts.action in ('store_true', 'store_false'): - option_type = 'flags' - else: - option_type = 'opts' - - options[name][option_type].extend( - opts._short_opts + opts._long_opts - ) - - # Add global options - options['_global'] = { - 'flags': [u'-v', u'--verbose'], - 'opts': u'-l --library -c --config -d --directory -h --help'.split( - u' ') - } - - # Add flags common to all commands - options['_common'] = { - 'flags': [u'-h', u'--help'] - } - - # Start generating the script - yield u"_beet() {\n" - - # Command names - yield u" local commands='%s'\n" % ' '.join(command_names) - yield u"\n" - - # Command aliases - yield u" local aliases='%s'\n" % ' '.join(aliases.keys()) - for alias, cmd in aliases.items(): - yield u" local alias__%s=%s\n" % (alias, cmd) - yield u'\n' - - # Fields - yield u" fields='%s'\n" % ' '.join( - set(library.Item._fields.keys() + library.Album._fields.keys()) - ) - - # Command options - for cmd, opts in options.items(): - for option_type, option_list in opts.items(): - if option_list: - option_list = ' '.join(option_list) - yield u" local %s__%s='%s'\n" % ( - option_type, cmd, option_list) - - yield u' _beet_dispatch\n' - yield u'}\n' - - -completion_cmd = ui.Subcommand( - 'completion', - help=u'print shell script that provides command line completion' -) -completion_cmd.func = print_completion -completion_cmd.hide = True -default_commands.append(completion_cmd) diff --git a/libs/beets/util/__init__.py b/libs/beets/util/__init__.py deleted file mode 100644 index 3cc270aee..000000000 --- a/libs/beets/util/__init__.py +++ /dev/null @@ -1,862 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Miscellaneous utility functions.""" - -from __future__ import division, absolute_import, print_function -import os -import sys -import re -import shutil -import fnmatch -from collections import Counter -import traceback -import subprocess -import platform -import shlex -from beets.util import hidden - - -MAX_FILENAME_LENGTH = 200 -WINDOWS_MAGIC_PREFIX = u'\\\\?\\' - - -class HumanReadableException(Exception): - """An Exception that can include a human-readable error message to - be logged without a traceback. Can preserve a traceback for - debugging purposes as well. - - Has at least two fields: `reason`, the underlying exception or a - string describing the problem; and `verb`, the action being - performed during the error. - - If `tb` is provided, it is a string containing a traceback for the - associated exception. (Note that this is not necessary in Python 3.x - and should be removed when we make the transition.) - """ - error_kind = 'Error' # Human-readable description of error type. - - def __init__(self, reason, verb, tb=None): - self.reason = reason - self.verb = verb - self.tb = tb - super(HumanReadableException, self).__init__(self.get_message()) - - def _gerund(self): - """Generate a (likely) gerund form of the English verb. - """ - if u' ' in self.verb: - return self.verb - gerund = self.verb[:-1] if self.verb.endswith(u'e') else self.verb - gerund += u'ing' - return gerund - - def _reasonstr(self): - """Get the reason as a string.""" - if isinstance(self.reason, unicode): - return self.reason - elif isinstance(self.reason, basestring): # Byte string. - return self.reason.decode('utf8', 'ignore') - elif hasattr(self.reason, 'strerror'): # i.e., EnvironmentError - return self.reason.strerror - else: - return u'"{0}"'.format(unicode(self.reason)) - - def get_message(self): - """Create the human-readable description of the error, sans - introduction. - """ - raise NotImplementedError - - def log(self, logger): - """Log to the provided `logger` a human-readable message as an - error and a verbose traceback as a debug message. - """ - if self.tb: - logger.debug(self.tb) - logger.error(u'{0}: {1}', self.error_kind, self.args[0]) - - -class FilesystemError(HumanReadableException): - """An error that occurred while performing a filesystem manipulation - via a function in this module. The `paths` field is a sequence of - pathnames involved in the operation. - """ - def __init__(self, reason, verb, paths, tb=None): - self.paths = paths - super(FilesystemError, self).__init__(reason, verb, tb) - - def get_message(self): - # Use a nicer English phrasing for some specific verbs. - if self.verb in ('move', 'copy', 'rename'): - clause = u'while {0} {1} to {2}'.format( - self._gerund(), - displayable_path(self.paths[0]), - displayable_path(self.paths[1]) - ) - elif self.verb in ('delete', 'write', 'create', 'read'): - clause = u'while {0} {1}'.format( - self._gerund(), - displayable_path(self.paths[0]) - ) - else: - clause = u'during {0} of paths {1}'.format( - self.verb, u', '.join(displayable_path(p) for p in self.paths) - ) - - return u'{0} {1}'.format(self._reasonstr(), clause) - - -def normpath(path): - """Provide the canonical form of the path suitable for storing in - the database. - """ - path = syspath(path, prefix=False) - path = os.path.normpath(os.path.abspath(os.path.expanduser(path))) - return bytestring_path(path) - - -def ancestry(path): - """Return a list consisting of path's parent directory, its - grandparent, and so on. For instance: - - >>> ancestry('/a/b/c') - ['/', '/a', '/a/b'] - - The argument should *not* be the result of a call to `syspath`. - """ - out = [] - last_path = None - while path: - path = os.path.dirname(path) - - if path == last_path: - break - last_path = path - - if path: - # don't yield '' - out.insert(0, path) - return out - - -def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None): - """Like `os.walk`, but yields things in case-insensitive sorted, - breadth-first order. Directory and file names matching any glob - pattern in `ignore` are skipped. If `logger` is provided, then - warning messages are logged there when a directory cannot be listed. - """ - # Make sure the path isn't a Unicode string. - path = bytestring_path(path) - - # Get all the directories and files at this level. - try: - contents = os.listdir(syspath(path)) - except OSError as exc: - if logger: - logger.warn(u'could not list directory {0}: {1}'.format( - displayable_path(path), exc.strerror - )) - return - dirs = [] - files = [] - for base in contents: - base = bytestring_path(base) - - # Skip ignored filenames. - skip = False - for pat in ignore: - if fnmatch.fnmatch(base, pat): - skip = True - break - if skip: - continue - - # Add to output as either a file or a directory. - cur = os.path.join(path, base) - if (ignore_hidden and not hidden.is_hidden(cur)) or not ignore_hidden: - if os.path.isdir(syspath(cur)): - dirs.append(base) - else: - files.append(base) - - # Sort lists (case-insensitive) and yield the current level. - dirs.sort(key=bytes.lower) - files.sort(key=bytes.lower) - yield (path, dirs, files) - - # Recurse into directories. - for base in dirs: - cur = os.path.join(path, base) - # yield from sorted_walk(...) - for res in sorted_walk(cur, ignore, ignore_hidden, logger): - yield res - - -def mkdirall(path): - """Make all the enclosing directories of path (like mkdir -p on the - parent). - """ - for ancestor in ancestry(path): - if not os.path.isdir(syspath(ancestor)): - try: - os.mkdir(syspath(ancestor)) - except (OSError, IOError) as exc: - raise FilesystemError(exc, 'create', (ancestor,), - traceback.format_exc()) - - -def fnmatch_all(names, patterns): - """Determine whether all strings in `names` match at least one of - the `patterns`, which should be shell glob expressions. - """ - for name in names: - matches = False - for pattern in patterns: - matches = fnmatch.fnmatch(name, pattern) - if matches: - break - if not matches: - return False - return True - - -def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')): - """If path is an empty directory, then remove it. Recursively remove - path's ancestry up to root (which is never removed) where there are - empty directories. If path is not contained in root, then nothing is - removed. Glob patterns in clutter are ignored when determining - emptiness. If root is not provided, then only path may be removed - (i.e., no recursive removal). - """ - path = normpath(path) - if root is not None: - root = normpath(root) - - ancestors = ancestry(path) - if root is None: - # Only remove the top directory. - ancestors = [] - elif root in ancestors: - # Only remove directories below the root. - ancestors = ancestors[ancestors.index(root) + 1:] - else: - # Remove nothing. - return - - # Traverse upward from path. - ancestors.append(path) - ancestors.reverse() - for directory in ancestors: - directory = syspath(directory) - if not os.path.exists(directory): - # Directory gone already. - continue - if fnmatch_all(os.listdir(directory), clutter): - # Directory contains only clutter (or nothing). - try: - shutil.rmtree(directory) - except OSError: - break - else: - break - - -def components(path): - """Return a list of the path components in path. For instance: - - >>> components('/a/b/c') - ['a', 'b', 'c'] - - The argument should *not* be the result of a call to `syspath`. - """ - comps = [] - ances = ancestry(path) - for anc in ances: - comp = os.path.basename(anc) - if comp: - comps.append(comp) - else: # root - comps.append(anc) - - last = os.path.basename(path) - if last: - comps.append(last) - - return comps - - -def _fsencoding(): - """Get the system's filesystem encoding. On Windows, this is always - UTF-8 (not MBCS). - """ - encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() - if encoding == 'mbcs': - # On Windows, a broken encoding known to Python as "MBCS" is - # used for the filesystem. However, we only use the Unicode API - # for Windows paths, so the encoding is actually immaterial so - # we can avoid dealing with this nastiness. We arbitrarily - # choose UTF-8. - encoding = 'utf8' - return encoding - - -def bytestring_path(path): - """Given a path, which is either a bytes or a unicode, returns a str - path (ensuring that we never deal with Unicode pathnames). - """ - # Pass through bytestrings. - if isinstance(path, bytes): - return path - - # On Windows, remove the magic prefix added by `syspath`. This makes - # ``bytestring_path(syspath(X)) == X``, i.e., we can safely - # round-trip through `syspath`. - if os.path.__name__ == 'ntpath' and path.startswith(WINDOWS_MAGIC_PREFIX): - path = path[len(WINDOWS_MAGIC_PREFIX):] - - # Try to encode with default encodings, but fall back to UTF8. - try: - return path.encode(_fsencoding()) - except (UnicodeError, LookupError): - return path.encode('utf8') - - -def displayable_path(path, separator=u'; '): - """Attempts to decode a bytestring path to a unicode object for the - purpose of displaying it to the user. If the `path` argument is a - list or a tuple, the elements are joined with `separator`. - """ - if isinstance(path, (list, tuple)): - return separator.join(displayable_path(p) for p in path) - elif isinstance(path, unicode): - return path - elif not isinstance(path, bytes): - # A non-string object: just get its unicode representation. - return unicode(path) - - try: - return path.decode(_fsencoding(), 'ignore') - except (UnicodeError, LookupError): - return path.decode('utf8', 'ignore') - - -def syspath(path, prefix=True): - """Convert a path for use by the operating system. In particular, - paths on Windows must receive a magic prefix and must be converted - to Unicode before they are sent to the OS. To disable the magic - prefix on Windows, set `prefix` to False---but only do this if you - *really* know what you're doing. - """ - # Don't do anything if we're not on windows - if os.path.__name__ != 'ntpath': - return path - - if not isinstance(path, unicode): - # Beets currently represents Windows paths internally with UTF-8 - # arbitrarily. But earlier versions used MBCS because it is - # reported as the FS encoding by Windows. Try both. - try: - path = path.decode('utf8') - except UnicodeError: - # The encoding should always be MBCS, Windows' broken - # Unicode representation. - encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() - path = path.decode(encoding, 'replace') - - # Add the magic prefix if it isn't already there. - # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx - if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX): - if path.startswith(u'\\\\'): - # UNC path. Final path should look like \\?\UNC\... - path = u'UNC' + path[1:] - path = WINDOWS_MAGIC_PREFIX + path - - return path - - -def samefile(p1, p2): - """Safer equality for paths.""" - return shutil._samefile(syspath(p1), syspath(p2)) - - -def remove(path, soft=True): - """Remove the file. If `soft`, then no error will be raised if the - file does not exist. - """ - path = syspath(path) - if soft and not os.path.exists(path): - return - try: - os.remove(path) - except (OSError, IOError) as exc: - raise FilesystemError(exc, 'delete', (path,), traceback.format_exc()) - - -def copy(path, dest, replace=False): - """Copy a plain file. Permissions are not copied. If `dest` already - exists, raises a FilesystemError unless `replace` is True. Has no - effect if `path` is the same as `dest`. Paths are translated to - system paths before the syscall. - """ - if samefile(path, dest): - return - path = syspath(path) - dest = syspath(dest) - if not replace and os.path.exists(dest): - raise FilesystemError(u'file exists', 'copy', (path, dest)) - try: - shutil.copyfile(path, dest) - except (OSError, IOError) as exc: - raise FilesystemError(exc, 'copy', (path, dest), - traceback.format_exc()) - - -def move(path, dest, replace=False): - """Rename a file. `dest` may not be a directory. If `dest` already - exists, raises an OSError unless `replace` is True. Has no effect if - `path` is the same as `dest`. If the paths are on different - filesystems (or the rename otherwise fails), a copy is attempted - instead, in which case metadata will *not* be preserved. Paths are - translated to system paths. - """ - if samefile(path, dest): - return - path = syspath(path) - dest = syspath(dest) - if os.path.exists(dest) and not replace: - raise FilesystemError(u'file exists', 'rename', (path, dest), - traceback.format_exc()) - - # First, try renaming the file. - try: - os.rename(path, dest) - except OSError: - # Otherwise, copy and delete the original. - try: - shutil.copyfile(path, dest) - os.remove(path) - except (OSError, IOError) as exc: - raise FilesystemError(exc, 'move', (path, dest), - traceback.format_exc()) - - -def link(path, dest, replace=False): - """Create a symbolic link from path to `dest`. Raises an OSError if - `dest` already exists, unless `replace` is True. Does nothing if - `path` == `dest`.""" - if (samefile(path, dest)): - return - - path = syspath(path) - dest = syspath(dest) - if os.path.exists(dest) and not replace: - raise FilesystemError(u'file exists', 'rename', (path, dest), - traceback.format_exc()) - try: - os.symlink(path, dest) - except OSError: - raise FilesystemError(u'Operating system does not support symbolic ' - u'links.', 'link', (path, dest), - traceback.format_exc()) - - -def unique_path(path): - """Returns a version of ``path`` that does not exist on the - filesystem. Specifically, if ``path` itself already exists, then - something unique is appended to the path. - """ - if not os.path.exists(syspath(path)): - return path - - base, ext = os.path.splitext(path) - match = re.search(br'\.(\d)+$', base) - if match: - num = int(match.group(1)) - base = base[:match.start()] - else: - num = 0 - while True: - num += 1 - new_path = b'%s.%i%s' % (base, num, ext) - if not os.path.exists(new_path): - return new_path - -# Note: The Windows "reserved characters" are, of course, allowed on -# Unix. They are forbidden here because they cause problems on Samba -# shares, which are sufficiently common as to cause frequent problems. -# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx -CHAR_REPLACE = [ - (re.compile(r'[\\/]'), u'_'), # / and \ -- forbidden everywhere. - (re.compile(r'^\.'), u'_'), # Leading dot (hidden files on Unix). - (re.compile(r'[\x00-\x1f]'), u''), # Control characters. - (re.compile(r'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters". - (re.compile(r'\.$'), u'_'), # Trailing dots. - (re.compile(r'\s+$'), u''), # Trailing whitespace. -] - - -def sanitize_path(path, replacements=None): - """Takes a path (as a Unicode string) and makes sure that it is - legal. Returns a new path. Only works with fragments; won't work - reliably on Windows when a path begins with a drive letter. Path - separators (including altsep!) should already be cleaned from the - path components. If replacements is specified, it is used *instead* - of the default set of replacements; it must be a list of (compiled - regex, replacement string) pairs. - """ - replacements = replacements or CHAR_REPLACE - - comps = components(path) - if not comps: - return '' - for i, comp in enumerate(comps): - for regex, repl in replacements: - comp = regex.sub(repl, comp) - comps[i] = comp - return os.path.join(*comps) - - -def truncate_path(path, length=MAX_FILENAME_LENGTH): - """Given a bytestring path or a Unicode path fragment, truncate the - components to a legal length. In the last component, the extension - is preserved. - """ - comps = components(path) - - out = [c[:length] for c in comps] - base, ext = os.path.splitext(comps[-1]) - if ext: - # Last component has an extension. - base = base[:length - len(ext)] - out[-1] = base + ext - - return os.path.join(*out) - - -def _legalize_stage(path, replacements, length, extension, fragment): - """Perform a single round of path legalization steps - (sanitation/replacement, encoding from Unicode to bytes, - extension-appending, and truncation). Return the path (Unicode if - `fragment` is set, `bytes` otherwise) and whether truncation was - required. - """ - # Perform an initial sanitization including user replacements. - path = sanitize_path(path, replacements) - - # Encode for the filesystem. - if not fragment: - path = bytestring_path(path) - - # Preserve extension. - path += extension.lower() - - # Truncate too-long components. - pre_truncate_path = path - path = truncate_path(path, length) - - return path, path != pre_truncate_path - - -def legalize_path(path, replacements, length, extension, fragment): - """Given a path-like Unicode string, produce a legal path. Return - the path and a flag indicating whether some replacements had to be - ignored (see below). - - The legalization process (see `_legalize_stage`) consists of - applying the sanitation rules in `replacements`, encoding the string - to bytes (unless `fragment` is set), truncating components to - `length`, appending the `extension`. - - This function performs up to three calls to `_legalize_stage` in - case truncation conflicts with replacements (as can happen when - truncation creates whitespace at the end of the string, for - example). The limited number of iterations iterations avoids the - possibility of an infinite loop of sanitation and truncation - operations, which could be caused by replacement rules that make the - string longer. The flag returned from this function indicates that - the path has to be truncated twice (indicating that replacements - made the string longer again after it was truncated); the - application should probably log some sort of warning. - """ - - if fragment: - # Outputting Unicode. - extension = extension.decode('utf8', 'ignore') - - first_stage_path, _ = _legalize_stage( - path, replacements, length, extension, fragment - ) - - # Convert back to Unicode with extension removed. - first_stage_path, _ = os.path.splitext(displayable_path(first_stage_path)) - - # Re-sanitize following truncation (including user replacements). - second_stage_path, retruncated = _legalize_stage( - first_stage_path, replacements, length, extension, fragment - ) - - # If the path was once again truncated, discard user replacements - # and run through one last legalization stage. - if retruncated: - second_stage_path, _ = _legalize_stage( - first_stage_path, None, length, extension, fragment - ) - - return second_stage_path, retruncated - - -def str2bool(value): - """Returns a boolean reflecting a human-entered string.""" - return value.lower() in (u'yes', u'1', u'true', u't', u'y') - - -def as_string(value): - """Convert a value to a Unicode object for matching with a query. - None becomes the empty string. Bytestrings are silently decoded. - """ - if value is None: - return u'' - elif isinstance(value, buffer): - return bytes(value).decode('utf8', 'ignore') - elif isinstance(value, bytes): - return value.decode('utf8', 'ignore') - else: - return unicode(value) - - -def plurality(objs): - """Given a sequence of hashble objects, returns the object that - is most common in the set and the its number of appearance. The - sequence must contain at least one object. - """ - c = Counter(objs) - if not c: - raise ValueError(u'sequence must be non-empty') - return c.most_common(1)[0] - - -def cpu_count(): - """Return the number of hardware thread contexts (cores or SMT - threads) in the system. - """ - # Adapted from the soundconverter project: - # https://github.com/kassoulet/soundconverter - if sys.platform == 'win32': - try: - num = int(os.environ['NUMBER_OF_PROCESSORS']) - except (ValueError, KeyError): - num = 0 - elif sys.platform == 'darwin': - try: - num = int(command_output([b'/usr/sbin/sysctl', b'-n', b'hw.ncpu'])) - except (ValueError, OSError, subprocess.CalledProcessError): - num = 0 - else: - try: - num = os.sysconf('SC_NPROCESSORS_ONLN') - except (ValueError, OSError, AttributeError): - num = 0 - if num >= 1: - return num - else: - return 1 - - -def command_output(cmd, shell=False): - """Runs the command and returns its output after it has exited. - - ``cmd`` is a list of byte string arguments starting with the command names. - If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a - shell to execute. - - If the process exits with a non-zero return code - ``subprocess.CalledProcessError`` is raised. May also raise - ``OSError``. - - This replaces `subprocess.check_output` which can have problems if lots of - output is sent to stderr. - """ - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=platform.system() != 'Windows', - shell=shell - ) - stdout, stderr = proc.communicate() - if proc.returncode: - raise subprocess.CalledProcessError( - returncode=proc.returncode, - cmd=b' '.join(cmd), - output=stdout + stderr, - ) - return stdout - - -def max_filename_length(path, limit=MAX_FILENAME_LENGTH): - """Attempt to determine the maximum filename length for the - filesystem containing `path`. If the value is greater than `limit`, - then `limit` is used instead (to prevent errors when a filesystem - misreports its capacity). If it cannot be determined (e.g., on - Windows), return `limit`. - """ - if hasattr(os, 'statvfs'): - try: - res = os.statvfs(path) - except OSError: - return limit - return min(res[9], limit) - else: - return limit - - -def open_anything(): - """Return the system command that dispatches execution to the correct - program. - """ - sys_name = platform.system() - if sys_name == 'Darwin': - base_cmd = 'open' - elif sys_name == 'Windows': - base_cmd = 'start' - else: # Assume Unix - base_cmd = 'xdg-open' - return base_cmd - - -def editor_command(): - """Get a command for opening a text file. - - Use the `EDITOR` environment variable by default. If it is not - present, fall back to `open_anything()`, the platform-specific tool - for opening files in general. - """ - editor = os.environ.get('EDITOR') - if editor: - return editor - return open_anything() - - -def shlex_split(s): - """Split a Unicode or bytes string according to shell lexing rules. - - Raise `ValueError` if the string is not a well-formed shell string. - This is a workaround for a bug in some versions of Python. - """ - if isinstance(s, bytes): - # Shlex works fine. - return shlex.split(s) - - elif isinstance(s, unicode): - # Work around a Python bug. - # http://bugs.python.org/issue6988 - bs = s.encode('utf8') - return [c.decode('utf8') for c in shlex.split(bs)] - - else: - raise TypeError(u'shlex_split called with non-string') - - -def interactive_open(targets, command): - """Open the files in `targets` by `exec`ing a new `command`, given - as a Unicode string. (The new program takes over, and Python - execution ends: this does not fork a subprocess.) - - Can raise `OSError`. - """ - assert command - - # Split the command string into its arguments. - try: - args = shlex_split(command) - except ValueError: # Malformed shell tokens. - args = [command] - - args.insert(0, args[0]) # for argv[0] - - args += targets - - return os.execlp(*args) - - -def _windows_long_path_name(short_path): - """Use Windows' `GetLongPathNameW` via ctypes to get the canonical, - long path given a short filename. - """ - if not isinstance(short_path, unicode): - short_path = unicode(short_path) - - import ctypes - buf = ctypes.create_unicode_buffer(260) - get_long_path_name_w = ctypes.windll.kernel32.GetLongPathNameW - return_value = get_long_path_name_w(short_path, buf, 260) - - if return_value == 0 or return_value > 260: - # An error occurred - return short_path - else: - long_path = buf.value - # GetLongPathNameW does not change the case of the drive - # letter. - if len(long_path) > 1 and long_path[1] == ':': - long_path = long_path[0].upper() + long_path[1:] - return long_path - - -def case_sensitive(path): - """Check whether the filesystem at the given path is case sensitive. - - To work best, the path should point to a file or a directory. If the path - does not exist, assume a case sensitive file system on every platform - except Windows. - """ - # A fallback in case the path does not exist. - if not os.path.exists(syspath(path)): - # By default, the case sensitivity depends on the platform. - return platform.system() != 'Windows' - - # If an upper-case version of the path exists but a lower-case - # version does not, then the filesystem must be case-sensitive. - # (Otherwise, we have more work to do.) - if not (os.path.exists(syspath(path.lower())) and - os.path.exists(syspath(path.upper()))): - return True - - # Both versions of the path exist on the file system. Check whether - # they refer to different files by their inodes. Alas, - # `os.path.samefile` is only available on Unix systems on Python 2. - if platform.system() != 'Windows': - return not os.path.samefile(syspath(path.lower()), - syspath(path.upper())) - - # On Windows, we check whether the canonical, long filenames for the - # files are the same. - lower = _windows_long_path_name(path.lower()) - upper = _windows_long_path_name(path.upper()) - return lower != upper - - -def raw_seconds_short(string): - """Formats a human-readable M:SS string as a float (number of seconds). - - Raises ValueError if the conversion cannot take place due to `string` not - being in the right format. - """ - match = re.match(r'^(\d+):([0-5]\d)$', string) - if not match: - raise ValueError(u'String not in M:SS format') - minutes, seconds = map(int, match.groups()) - return float(minutes * 60 + seconds) diff --git a/libs/beetsplug/acousticbrainz.py b/libs/beetsplug/acousticbrainz.py deleted file mode 100644 index df790b26f..000000000 --- a/libs/beetsplug/acousticbrainz.py +++ /dev/null @@ -1,165 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2015-2016, Ohm Patel. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Fetch various AcousticBrainz metadata using MBID. -""" -from __future__ import division, absolute_import, print_function - -import requests -import operator - -from beets import plugins, ui -from functools import reduce - -ACOUSTIC_BASE = "https://acousticbrainz.org/" -LEVELS = ["/low-level", "/high-level"] - - -class AcousticPlugin(plugins.BeetsPlugin): - def __init__(self): - super(AcousticPlugin, self).__init__() - - self.config.add({'auto': True}) - if self.config['auto']: - self.register_listener('import_task_files', - self.import_task_files) - - def commands(self): - cmd = ui.Subcommand('acousticbrainz', - help=u"fetch metadata from AcousticBrainz") - - def func(lib, opts, args): - items = lib.items(ui.decargs(args)) - fetch_info(self._log, items, ui.should_write()) - - cmd.func = func - return [cmd] - - def import_task_files(self, session, task): - """Function is called upon beet import. - """ - - items = task.imported_items() - fetch_info(self._log, items, False) - - -def fetch_info(log, items, write): - """Get data from AcousticBrainz for the items. - """ - - def get_value(*map_path): - try: - return reduce(operator.getitem, map_path, data) - except KeyError: - log.debug(u'Invalid Path: {}', map_path) - - for item in items: - if item.mb_trackid: - log.info(u'getting data for: {}', item) - - # Fetch the data from the AB API. - urls = [generate_url(item.mb_trackid, path) for path in LEVELS] - log.debug(u'fetching URLs: {}', ' '.join(urls)) - try: - res = [requests.get(url) for url in urls] - except requests.RequestException as exc: - log.info(u'request error: {}', exc) - continue - - # Check for missing tracks. - if any(r.status_code == 404 for r in res): - log.info(u'recording ID {} not found', item.mb_trackid) - continue - - # Parse the JSON response. - try: - data = res[0].json() - data.update(res[1].json()) - except ValueError: - log.debug(u'Invalid Response: {} & {}', [r.text for r in res]) - - # Get each field and assign it on the item. - item.danceable = get_value( - "highlevel", "danceability", "all", "danceable", - ) - item.gender = get_value( - "highlevel", "gender", "value", - ) - item.genre_rosamerica = get_value( - "highlevel", "genre_rosamerica", "value" - ) - item.mood_acoustic = get_value( - "highlevel", "mood_acoustic", "all", "acoustic" - ) - item.mood_aggressive = get_value( - "highlevel", "mood_aggressive", "all", "aggressive" - ) - item.mood_electronic = get_value( - "highlevel", "mood_electronic", "all", "electronic" - ) - item.mood_happy = get_value( - "highlevel", "mood_happy", "all", "happy" - ) - item.mood_party = get_value( - "highlevel", "mood_party", "all", "party" - ) - item.mood_relaxed = get_value( - "highlevel", "mood_relaxed", "all", "relaxed" - ) - item.mood_sad = get_value( - "highlevel", "mood_sad", "all", "sad" - ) - item.rhythm = get_value( - "highlevel", "ismir04_rhythm", "value" - ) - item.tonal = get_value( - "highlevel", "tonal_atonal", "all", "tonal" - ) - item.voice_instrumental = get_value( - "highlevel", "voice_instrumental", "value" - ) - item.average_loudness = get_value( - "lowlevel", "average_loudness" - ) - item.chords_changes_rate = get_value( - "tonal", "chords_changes_rate" - ) - item.chords_key = get_value( - "tonal", "chords_key" - ) - item.chords_number_rate = get_value( - "tonal", "chords_number_rate" - ) - item.chords_scale = get_value( - "tonal", "chords_scale" - ) - item.initial_key = '{} {}'.format( - get_value("tonal", "key_key"), - get_value("tonal", "key_scale") - ) - item.key_strength = get_value( - "tonal", "key_strength" - ) - - # Store the data. - item.store() - if write: - item.try_write() - - -def generate_url(mbid, level): - """Generates AcousticBrainz end point url for given MBID. - """ - return ACOUSTIC_BASE + mbid + level diff --git a/libs/beetsplug/badfiles.py b/libs/beetsplug/badfiles.py deleted file mode 100644 index f9704d484..000000000 --- a/libs/beetsplug/badfiles.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, François-Xavier Thomas. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Use command-line tools to check for audio file corruption. -""" - -from __future__ import division, absolute_import, print_function - -from beets.plugins import BeetsPlugin -from beets.ui import Subcommand -from beets.util import displayable_path, confit -from beets import ui -from subprocess import check_output, CalledProcessError, list2cmdline, STDOUT -import shlex -import os -import errno -import sys - - -class BadFiles(BeetsPlugin): - def run_command(self, cmd): - self._log.debug(u"running command: {}", - displayable_path(list2cmdline(cmd))) - try: - output = check_output(cmd, stderr=STDOUT) - errors = 0 - status = 0 - except CalledProcessError as e: - output = e.output - errors = 1 - status = e.returncode - except OSError as e: - if e.errno == errno.ENOENT: - ui.print_(u"command not found: {}".format(cmd[0])) - sys.exit(1) - else: - raise - output = output.decode(sys.getfilesystemencoding()) - return status, errors, [line for line in output.split("\n") if line] - - def check_mp3val(self, path): - status, errors, output = self.run_command(["mp3val", path]) - if status == 0: - output = [line for line in output if line.startswith("WARNING:")] - errors = len(output) - return status, errors, output - - def check_flac(self, path): - return self.run_command(["flac", "-wst", path]) - - def check_custom(self, command): - def checker(path): - cmd = shlex.split(command) - cmd.append(path) - return self.run_command(cmd) - return checker - - def get_checker(self, ext): - ext = ext.lower() - try: - command = self.config['commands'].get(dict).get(ext) - except confit.NotFoundError: - command = None - if command: - return self.check_custom(command) - elif ext == "mp3": - return self.check_mp3val - elif ext == "flac": - return self.check_flac - - def check_bad(self, lib, opts, args): - for item in lib.items(ui.decargs(args)): - - # First, check whether the path exists. If not, the user - # should probably run `beet update` to cleanup your library. - dpath = displayable_path(item.path) - self._log.debug(u"checking path: {}", dpath) - if not os.path.exists(item.path): - ui.print_(u"{}: file does not exist".format( - ui.colorize('text_error', dpath))) - - # Run the checker against the file if one is found - ext = os.path.splitext(item.path)[1][1:] - checker = self.get_checker(ext) - if not checker: - continue - path = item.path - if not isinstance(path, unicode): - path = item.path.decode(sys.getfilesystemencoding()) - status, errors, output = checker(path) - if status > 0: - ui.print_(u"{}: checker exited withs status {}" - .format(ui.colorize('text_error', dpath), status)) - for line in output: - ui.print_(" {}".format(displayable_path(line))) - elif errors > 0: - ui.print_(u"{}: checker found {} errors or warnings" - .format(ui.colorize('text_warning', dpath), errors)) - for line in output: - ui.print_(u" {}".format(displayable_path(line))) - else: - ui.print_(u"{}: ok".format(ui.colorize('text_success', dpath))) - - def commands(self): - bad_command = Subcommand('bad', - help=u'check for corrupt or missing files') - bad_command.func = self.check_bad - return [bad_command] diff --git a/libs/beetsplug/bpd/__init__.py b/libs/beetsplug/bpd/__init__.py deleted file mode 100644 index 33deda02c..000000000 --- a/libs/beetsplug/bpd/__init__.py +++ /dev/null @@ -1,1193 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""A clone of the Music Player Daemon (MPD) that plays music from a -Beets library. Attempts to implement a compatible protocol to allow -use of the wide range of MPD clients. -""" - -from __future__ import division, absolute_import, print_function - -import re -from string import Template -import traceback -import random -import time - -import beets -from beets.plugins import BeetsPlugin -import beets.ui -from beets import logging -from beets import vfs -from beets.util import bluelet -from beets.library import Item -from beets import dbcore -from beets.mediafile import MediaFile - -PROTOCOL_VERSION = '0.13.0' -BUFSIZE = 1024 - -HELLO = 'OK MPD %s' % PROTOCOL_VERSION -CLIST_BEGIN = 'command_list_begin' -CLIST_VERBOSE_BEGIN = 'command_list_ok_begin' -CLIST_END = 'command_list_end' -RESP_OK = 'OK' -RESP_CLIST_VERBOSE = 'list_OK' -RESP_ERR = 'ACK' - -NEWLINE = u"\n" - -ERROR_NOT_LIST = 1 -ERROR_ARG = 2 -ERROR_PASSWORD = 3 -ERROR_PERMISSION = 4 -ERROR_UNKNOWN = 5 -ERROR_NO_EXIST = 50 -ERROR_PLAYLIST_MAX = 51 -ERROR_SYSTEM = 52 -ERROR_PLAYLIST_LOAD = 53 -ERROR_UPDATE_ALREADY = 54 -ERROR_PLAYER_SYNC = 55 -ERROR_EXIST = 56 - -VOLUME_MIN = 0 -VOLUME_MAX = 100 - -SAFE_COMMANDS = ( - # Commands that are available when unauthenticated. - u'close', u'commands', u'notcommands', u'password', u'ping', -) - -ITEM_KEYS_WRITABLE = set(MediaFile.fields()).intersection(Item._fields.keys()) - -# Loggers. -log = logging.getLogger('beets.bpd') -global_log = logging.getLogger('beets') - - -# Gstreamer import error. -class NoGstreamerError(Exception): - pass - - -# Error-handling, exceptions, parameter parsing. - -class BPDError(Exception): - """An error that should be exposed to the client to the BPD - server. - """ - def __init__(self, code, message, cmd_name='', index=0): - self.code = code - self.message = message - self.cmd_name = cmd_name - self.index = index - - template = Template(u'$resp [$code@$index] {$cmd_name} $message') - - def response(self): - """Returns a string to be used as the response code for the - erring command. - """ - return self.template.substitute({ - 'resp': RESP_ERR, - 'code': self.code, - 'index': self.index, - 'cmd_name': self.cmd_name, - 'message': self.message, - }) - - -def make_bpd_error(s_code, s_message): - """Create a BPDError subclass for a static code and message. - """ - - class NewBPDError(BPDError): - code = s_code - message = s_message - cmd_name = '' - index = 0 - - def __init__(self): - pass - return NewBPDError - -ArgumentTypeError = make_bpd_error(ERROR_ARG, u'invalid type for argument') -ArgumentIndexError = make_bpd_error(ERROR_ARG, u'argument out of range') -ArgumentNotFoundError = make_bpd_error(ERROR_NO_EXIST, u'argument not found') - - -def cast_arg(t, val): - """Attempts to call t on val, raising a ArgumentTypeError - on ValueError. - - If 't' is the special string 'intbool', attempts to cast first - to an int and then to a bool (i.e., 1=True, 0=False). - """ - if t == 'intbool': - return cast_arg(bool, cast_arg(int, val)) - else: - try: - return t(val) - except ValueError: - raise ArgumentTypeError() - - -class BPDClose(Exception): - """Raised by a command invocation to indicate that the connection - should be closed. - """ - -# Generic server infrastructure, implementing the basic protocol. - - -class BaseServer(object): - """A MPD-compatible music player server. - - The functions with the `cmd_` prefix are invoked in response to - client commands. For instance, if the client says `status`, - `cmd_status` will be invoked. The arguments to the client's commands - are used as function arguments following the connection issuing the - command. The functions may send data on the connection. They may - also raise BPDError exceptions to report errors. - - This is a generic superclass and doesn't support many commands. - """ - - def __init__(self, host, port, password): - """Create a new server bound to address `host` and listening - on port `port`. If `password` is given, it is required to do - anything significant on the server. - """ - self.host, self.port, self.password = host, port, password - - # Default server values. - self.random = False - self.repeat = False - self.volume = VOLUME_MAX - self.crossfade = 0 - self.playlist = [] - self.playlist_version = 0 - self.current_index = -1 - self.paused = False - self.error = None - - # Object for random numbers generation - self.random_obj = random.Random() - - def run(self): - """Block and start listening for connections from clients. An - interrupt (^C) closes the server. - """ - self.startup_time = time.time() - bluelet.run(bluelet.server(self.host, self.port, - Connection.handler(self))) - - def _item_info(self, item): - """An abstract method that should response lines containing a - single song's metadata. - """ - raise NotImplementedError - - def _item_id(self, item): - """An abstract method returning the integer id for an item. - """ - raise NotImplementedError - - def _id_to_index(self, track_id): - """Searches the playlist for a song with the given id and - returns its index in the playlist. - """ - track_id = cast_arg(int, track_id) - for index, track in enumerate(self.playlist): - if self._item_id(track) == track_id: - return index - # Loop finished with no track found. - raise ArgumentNotFoundError() - - def _random_idx(self): - """Returns a random index different from the current one. - If there are no songs in the playlist it returns -1. - If there is only one song in the playlist it returns 0. - """ - if len(self.playlist) < 2: - return len(self.playlist) - 1 - new_index = self.random_obj.randint(0, len(self.playlist) - 1) - while new_index == self.current_index: - new_index = self.random_obj.randint(0, len(self.playlist) - 1) - return new_index - - def _succ_idx(self): - """Returns the index for the next song to play. - It also considers random and repeat flags. - No boundaries are checked. - """ - if self.repeat: - return self.current_index - if self.random: - return self._random_idx() - return self.current_index + 1 - - def _prev_idx(self): - """Returns the index for the previous song to play. - It also considers random and repeat flags. - No boundaries are checked. - """ - if self.repeat: - return self.current_index - if self.random: - return self._random_idx() - return self.current_index - 1 - - def cmd_ping(self, conn): - """Succeeds.""" - pass - - def cmd_kill(self, conn): - """Exits the server process.""" - exit(0) - - def cmd_close(self, conn): - """Closes the connection.""" - raise BPDClose() - - def cmd_password(self, conn, password): - """Attempts password authentication.""" - if password == self.password: - conn.authenticated = True - else: - conn.authenticated = False - raise BPDError(ERROR_PASSWORD, u'incorrect password') - - def cmd_commands(self, conn): - """Lists the commands available to the user.""" - if self.password and not conn.authenticated: - # Not authenticated. Show limited list of commands. - for cmd in SAFE_COMMANDS: - yield u'command: ' + cmd - - else: - # Authenticated. Show all commands. - for func in dir(self): - if func.startswith('cmd_'): - yield u'command: ' + func[4:] - - def cmd_notcommands(self, conn): - """Lists all unavailable commands.""" - if self.password and not conn.authenticated: - # Not authenticated. Show privileged commands. - for func in dir(self): - if func.startswith('cmd_'): - cmd = func[4:] - if cmd not in SAFE_COMMANDS: - yield u'command: ' + cmd - - else: - # Authenticated. No commands are unavailable. - pass - - def cmd_status(self, conn): - """Returns some status information for use with an - implementation of cmd_status. - - Gives a list of response-lines for: volume, repeat, random, - playlist, playlistlength, and xfade. - """ - yield ( - u'volume: ' + unicode(self.volume), - u'repeat: ' + unicode(int(self.repeat)), - u'random: ' + unicode(int(self.random)), - u'playlist: ' + unicode(self.playlist_version), - u'playlistlength: ' + unicode(len(self.playlist)), - u'xfade: ' + unicode(self.crossfade), - ) - - if self.current_index == -1: - state = u'stop' - elif self.paused: - state = u'pause' - else: - state = u'play' - yield u'state: ' + state - - if self.current_index != -1: # i.e., paused or playing - current_id = self._item_id(self.playlist[self.current_index]) - yield u'song: ' + unicode(self.current_index) - yield u'songid: ' + unicode(current_id) - - if self.error: - yield u'error: ' + self.error - - def cmd_clearerror(self, conn): - """Removes the persistent error state of the server. This - error is set when a problem arises not in response to a - command (for instance, when playing a file). - """ - self.error = None - - def cmd_random(self, conn, state): - """Set or unset random (shuffle) mode.""" - self.random = cast_arg('intbool', state) - - def cmd_repeat(self, conn, state): - """Set or unset repeat mode.""" - self.repeat = cast_arg('intbool', state) - - def cmd_setvol(self, conn, vol): - """Set the player's volume level (0-100).""" - vol = cast_arg(int, vol) - if vol < VOLUME_MIN or vol > VOLUME_MAX: - raise BPDError(ERROR_ARG, u'volume out of range') - self.volume = vol - - def cmd_crossfade(self, conn, crossfade): - """Set the number of seconds of crossfading.""" - crossfade = cast_arg(int, crossfade) - if crossfade < 0: - raise BPDError(ERROR_ARG, u'crossfade time must be nonnegative') - - def cmd_clear(self, conn): - """Clear the playlist.""" - self.playlist = [] - self.playlist_version += 1 - self.cmd_stop(conn) - - def cmd_delete(self, conn, index): - """Remove the song at index from the playlist.""" - index = cast_arg(int, index) - try: - del(self.playlist[index]) - except IndexError: - raise ArgumentIndexError() - self.playlist_version += 1 - - if self.current_index == index: # Deleted playing song. - self.cmd_stop(conn) - elif index < self.current_index: # Deleted before playing. - # Shift playing index down. - self.current_index -= 1 - - def cmd_deleteid(self, conn, track_id): - self.cmd_delete(conn, self._id_to_index(track_id)) - - def cmd_move(self, conn, idx_from, idx_to): - """Move a track in the playlist.""" - idx_from = cast_arg(int, idx_from) - idx_to = cast_arg(int, idx_to) - try: - track = self.playlist.pop(idx_from) - self.playlist.insert(idx_to, track) - except IndexError: - raise ArgumentIndexError() - - # Update currently-playing song. - if idx_from == self.current_index: - self.current_index = idx_to - elif idx_from < self.current_index <= idx_to: - self.current_index -= 1 - elif idx_from > self.current_index >= idx_to: - self.current_index += 1 - - self.playlist_version += 1 - - def cmd_moveid(self, conn, idx_from, idx_to): - idx_from = self._id_to_index(idx_from) - return self.cmd_move(conn, idx_from, idx_to) - - def cmd_swap(self, conn, i, j): - """Swaps two tracks in the playlist.""" - i = cast_arg(int, i) - j = cast_arg(int, j) - try: - track_i = self.playlist[i] - track_j = self.playlist[j] - except IndexError: - raise ArgumentIndexError() - - self.playlist[j] = track_i - self.playlist[i] = track_j - - # Update currently-playing song. - if self.current_index == i: - self.current_index = j - elif self.current_index == j: - self.current_index = i - - self.playlist_version += 1 - - def cmd_swapid(self, conn, i_id, j_id): - i = self._id_to_index(i_id) - j = self._id_to_index(j_id) - return self.cmd_swap(conn, i, j) - - def cmd_urlhandlers(self, conn): - """Indicates supported URL schemes. None by default.""" - pass - - def cmd_playlistinfo(self, conn, index=-1): - """Gives metadata information about the entire playlist or a - single track, given by its index. - """ - index = cast_arg(int, index) - if index == -1: - for track in self.playlist: - yield self._item_info(track) - else: - try: - track = self.playlist[index] - except IndexError: - raise ArgumentIndexError() - yield self._item_info(track) - - def cmd_playlistid(self, conn, track_id=-1): - return self.cmd_playlistinfo(conn, self._id_to_index(track_id)) - - def cmd_plchanges(self, conn, version): - """Sends playlist changes since the given version. - - This is a "fake" implementation that ignores the version and - just returns the entire playlist (rather like version=0). This - seems to satisfy many clients. - """ - return self.cmd_playlistinfo(conn) - - def cmd_plchangesposid(self, conn, version): - """Like plchanges, but only sends position and id. - - Also a dummy implementation. - """ - for idx, track in enumerate(self.playlist): - yield u'cpos: ' + unicode(idx) - yield u'Id: ' + unicode(track.id) - - def cmd_currentsong(self, conn): - """Sends information about the currently-playing song. - """ - if self.current_index != -1: # -1 means stopped. - track = self.playlist[self.current_index] - yield self._item_info(track) - - def cmd_next(self, conn): - """Advance to the next song in the playlist.""" - self.current_index = self._succ_idx() - if self.current_index >= len(self.playlist): - # Fallen off the end. Just move to stopped state. - return self.cmd_stop(conn) - else: - return self.cmd_play(conn) - - def cmd_previous(self, conn): - """Step back to the last song.""" - self.current_index = self._prev_idx() - if self.current_index < 0: - return self.cmd_stop(conn) - else: - return self.cmd_play(conn) - - def cmd_pause(self, conn, state=None): - """Set the pause state playback.""" - if state is None: - self.paused = not self.paused # Toggle. - else: - self.paused = cast_arg('intbool', state) - - def cmd_play(self, conn, index=-1): - """Begin playback, possibly at a specified playlist index.""" - index = cast_arg(int, index) - - if index < -1 or index > len(self.playlist): - raise ArgumentIndexError() - - if index == -1: # No index specified: start where we are. - if not self.playlist: # Empty playlist: stop immediately. - return self.cmd_stop(conn) - if self.current_index == -1: # No current song. - self.current_index = 0 # Start at the beginning. - # If we have a current song, just stay there. - - else: # Start with the specified index. - self.current_index = index - - self.paused = False - - def cmd_playid(self, conn, track_id=0): - track_id = cast_arg(int, track_id) - if track_id == -1: - index = -1 - else: - index = self._id_to_index(track_id) - return self.cmd_play(conn, index) - - def cmd_stop(self, conn): - """Stop playback.""" - self.current_index = -1 - self.paused = False - - def cmd_seek(self, conn, index, pos): - """Seek to a specified point in a specified song.""" - index = cast_arg(int, index) - if index < 0 or index >= len(self.playlist): - raise ArgumentIndexError() - self.current_index = index - - def cmd_seekid(self, conn, track_id, pos): - index = self._id_to_index(track_id) - return self.cmd_seek(conn, index, pos) - - def cmd_profile(self, conn): - """Memory profiling for debugging.""" - from guppy import hpy - heap = hpy().heap() - print(heap) - - -class Connection(object): - """A connection between a client and the server. Handles input and - output from and to the client. - """ - def __init__(self, server, sock): - """Create a new connection for the accepted socket `client`. - """ - self.server = server - self.sock = sock - self.authenticated = False - - def send(self, lines): - """Send lines, which which is either a single string or an - iterable consisting of strings, to the client. A newline is - added after every string. Returns a Bluelet event that sends - the data. - """ - if isinstance(lines, basestring): - lines = [lines] - out = NEWLINE.join(lines) + NEWLINE - log.debug('{}', out[:-1]) # Don't log trailing newline. - if isinstance(out, unicode): - out = out.encode('utf8') - return self.sock.sendall(out) - - def do_command(self, command): - """A coroutine that runs the given command and sends an - appropriate response.""" - try: - yield bluelet.call(command.run(self)) - except BPDError as e: - # Send the error. - yield self.send(e.response()) - else: - # Send success code. - yield self.send(RESP_OK) - - def run(self): - """Send a greeting to the client and begin processing commands - as they arrive. - """ - yield self.send(HELLO) - - clist = None # Initially, no command list is being constructed. - while True: - line = yield self.sock.readline() - if not line: - break - line = line.strip() - if not line: - break - log.debug('{}', line) - - if clist is not None: - # Command list already opened. - if line == CLIST_END: - yield bluelet.call(self.do_command(clist)) - clist = None # Clear the command list. - else: - clist.append(Command(line)) - - elif line == CLIST_BEGIN or line == CLIST_VERBOSE_BEGIN: - # Begin a command list. - clist = CommandList([], line == CLIST_VERBOSE_BEGIN) - - else: - # Ordinary command. - try: - yield bluelet.call(self.do_command(Command(line))) - except BPDClose: - # Command indicates that the conn should close. - self.sock.close() - return - - @classmethod - def handler(cls, server): - def _handle(sock): - """Creates a new `Connection` and runs it. - """ - return cls(server, sock).run() - return _handle - - -class Command(object): - """A command issued by the client for processing by the server. - """ - - command_re = re.compile(br'^([^ \t]+)[ \t]*') - arg_re = re.compile(br'"((?:\\"|[^"])+)"|([^ \t"]+)') - - def __init__(self, s): - """Creates a new `Command` from the given string, `s`, parsing - the string for command name and arguments. - """ - command_match = self.command_re.match(s) - self.name = command_match.group(1) - - self.args = [] - arg_matches = self.arg_re.findall(s[command_match.end():]) - for match in arg_matches: - if match[0]: - # Quoted argument. - arg = match[0] - arg = arg.replace(b'\\"', b'"').replace(b'\\\\', b'\\') - else: - # Unquoted argument. - arg = match[1] - arg = arg.decode('utf8') - self.args.append(arg) - - def run(self, conn): - """A coroutine that executes the command on the given - connection. - """ - # Attempt to get correct command function. - func_name = 'cmd_' + self.name - if not hasattr(conn.server, func_name): - raise BPDError(ERROR_UNKNOWN, u'unknown command', self.name) - func = getattr(conn.server, func_name) - - # Ensure we have permission for this command. - if conn.server.password and \ - not conn.authenticated and \ - self.name not in SAFE_COMMANDS: - raise BPDError(ERROR_PERMISSION, u'insufficient privileges') - - try: - args = [conn] + self.args - results = func(*args) - if results: - for data in results: - yield conn.send(data) - - except BPDError as e: - # An exposed error. Set the command name and then let - # the Connection handle it. - e.cmd_name = self.name - raise e - - except BPDClose: - # An indication that the connection should close. Send - # it on the Connection. - raise - - except Exception as e: - # An "unintentional" error. Hide it from the client. - log.error('{}', traceback.format_exc(e)) - raise BPDError(ERROR_SYSTEM, u'server error', self.name) - - -class CommandList(list): - """A list of commands issued by the client for processing by the - server. May be verbose, in which case the response is delimited, or - not. Should be a list of `Command` objects. - """ - - def __init__(self, sequence=None, verbose=False): - """Create a new `CommandList` from the given sequence of - `Command`s. If `verbose`, this is a verbose command list. - """ - if sequence: - for item in sequence: - self.append(item) - self.verbose = verbose - - def run(self, conn): - """Coroutine executing all the commands in this list. - """ - for i, command in enumerate(self): - try: - yield bluelet.call(command.run(conn)) - except BPDError as e: - # If the command failed, stop executing. - e.index = i # Give the error the correct index. - raise e - - # Otherwise, possibly send the output delimeter if we're in a - # verbose ("OK") command list. - if self.verbose: - yield conn.send(RESP_CLIST_VERBOSE) - - -# A subclass of the basic, protocol-handling server that actually plays -# music. - -class Server(BaseServer): - """An MPD-compatible server using GStreamer to play audio and beets - to store its library. - """ - - def __init__(self, library, host, port, password): - try: - from beetsplug.bpd import gstplayer - except ImportError as e: - # This is a little hacky, but it's the best I know for now. - if e.args[0].endswith(' gst'): - raise NoGstreamerError() - else: - raise - super(Server, self).__init__(host, port, password) - self.lib = library - self.player = gstplayer.GstPlayer(self.play_finished) - self.cmd_update(None) - - def run(self): - self.player.run() - super(Server, self).run() - - def play_finished(self): - """A callback invoked every time our player finishes a - track. - """ - self.cmd_next(None) - - # Metadata helper functions. - - def _item_info(self, item): - info_lines = [ - u'file: ' + item.destination(fragment=True), - u'Time: ' + unicode(int(item.length)), - u'Title: ' + item.title, - u'Artist: ' + item.artist, - u'Album: ' + item.album, - u'Genre: ' + item.genre, - ] - - track = unicode(item.track) - if item.tracktotal: - track += u'/' + unicode(item.tracktotal) - info_lines.append(u'Track: ' + track) - - info_lines.append(u'Date: ' + unicode(item.year)) - - try: - pos = self._id_to_index(item.id) - info_lines.append(u'Pos: ' + unicode(pos)) - except ArgumentNotFoundError: - # Don't include position if not in playlist. - pass - - info_lines.append(u'Id: ' + unicode(item.id)) - - return info_lines - - def _item_id(self, item): - return item.id - - # Database updating. - - def cmd_update(self, conn, path=u'/'): - """Updates the catalog to reflect the current database state. - """ - # Path is ignored. Also, the real MPD does this asynchronously; - # this is done inline. - print(u'Building directory tree...') - self.tree = vfs.libtree(self.lib) - print(u'... done.') - self.updated_time = time.time() - - # Path (directory tree) browsing. - - def _resolve_path(self, path): - """Returns a VFS node or an item ID located at the path given. - If the path does not exist, raises a - """ - components = path.split(u'/') - node = self.tree - - for component in components: - if not component: - continue - - if isinstance(node, int): - # We're trying to descend into a file node. - raise ArgumentNotFoundError() - - if component in node.files: - node = node.files[component] - elif component in node.dirs: - node = node.dirs[component] - else: - raise ArgumentNotFoundError() - - return node - - def _path_join(self, p1, p2): - """Smashes together two BPD paths.""" - out = p1 + u'/' + p2 - return out.replace(u'//', u'/').replace(u'//', u'/') - - def cmd_lsinfo(self, conn, path=u"/"): - """Sends info on all the items in the path.""" - node = self._resolve_path(path) - if isinstance(node, int): - # Trying to list a track. - raise BPDError(ERROR_ARG, u'this is not a directory') - else: - for name, itemid in iter(sorted(node.files.items())): - item = self.lib.get_item(itemid) - yield self._item_info(item) - for name, _ in iter(sorted(node.dirs.iteritems())): - dirpath = self._path_join(path, name) - if dirpath.startswith(u"/"): - # Strip leading slash (libmpc rejects this). - dirpath = dirpath[1:] - yield u'directory: %s' % dirpath - - def _listall(self, basepath, node, info=False): - """Helper function for recursive listing. If info, show - tracks' complete info; otherwise, just show items' paths. - """ - if isinstance(node, int): - # List a single file. - if info: - item = self.lib.get_item(node) - yield self._item_info(item) - else: - yield u'file: ' + basepath - else: - # List a directory. Recurse into both directories and files. - for name, itemid in sorted(node.files.iteritems()): - newpath = self._path_join(basepath, name) - # "yield from" - for v in self._listall(newpath, itemid, info): - yield v - for name, subdir in sorted(node.dirs.iteritems()): - newpath = self._path_join(basepath, name) - yield u'directory: ' + newpath - for v in self._listall(newpath, subdir, info): - yield v - - def cmd_listall(self, conn, path=u"/"): - """Send the paths all items in the directory, recursively.""" - return self._listall(path, self._resolve_path(path), False) - - def cmd_listallinfo(self, conn, path=u"/"): - """Send info on all the items in the directory, recursively.""" - return self._listall(path, self._resolve_path(path), True) - - # Playlist manipulation. - - def _all_items(self, node): - """Generator yielding all items under a VFS node. - """ - if isinstance(node, int): - # Could be more efficient if we built up all the IDs and - # then issued a single SELECT. - yield self.lib.get_item(node) - else: - # Recurse into a directory. - for name, itemid in sorted(node.files.iteritems()): - # "yield from" - for v in self._all_items(itemid): - yield v - for name, subdir in sorted(node.dirs.iteritems()): - for v in self._all_items(subdir): - yield v - - def _add(self, path, send_id=False): - """Adds a track or directory to the playlist, specified by the - path. If `send_id`, write each item's id to the client. - """ - for item in self._all_items(self._resolve_path(path)): - self.playlist.append(item) - if send_id: - yield u'Id: ' + unicode(item.id) - self.playlist_version += 1 - - def cmd_add(self, conn, path): - """Adds a track or directory to the playlist, specified by a - path. - """ - return self._add(path, False) - - def cmd_addid(self, conn, path): - """Same as `cmd_add` but sends an id back to the client.""" - return self._add(path, True) - - # Server info. - - def cmd_status(self, conn): - for line in super(Server, self).cmd_status(conn): - yield line - if self.current_index > -1: - item = self.playlist[self.current_index] - - yield u'bitrate: ' + unicode(item.bitrate / 1000) - # Missing 'audio'. - - (pos, total) = self.player.time() - yield u'time: ' + unicode(pos) + u':' + unicode(total) - - # Also missing 'updating_db'. - - def cmd_stats(self, conn): - """Sends some statistics about the library.""" - with self.lib.transaction() as tx: - statement = 'SELECT COUNT(DISTINCT artist), ' \ - 'COUNT(DISTINCT album), ' \ - 'COUNT(id), ' \ - 'SUM(length) ' \ - 'FROM items' - artists, albums, songs, totaltime = tx.query(statement)[0] - - yield ( - u'artists: ' + unicode(artists), - u'albums: ' + unicode(albums), - u'songs: ' + unicode(songs), - u'uptime: ' + unicode(int(time.time() - self.startup_time)), - u'playtime: ' + u'0', # Missing. - u'db_playtime: ' + unicode(int(totaltime)), - u'db_update: ' + unicode(int(self.updated_time)), - ) - - # Searching. - - tagtype_map = { - u'Artist': u'artist', - u'Album': u'album', - u'Title': u'title', - u'Track': u'track', - u'AlbumArtist': u'albumartist', - u'AlbumArtistSort': u'albumartist_sort', - # Name? - u'Genre': u'genre', - u'Date': u'year', - u'Composer': u'composer', - # Performer? - u'Disc': u'disc', - u'filename': u'path', # Suspect. - } - - def cmd_tagtypes(self, conn): - """Returns a list of the metadata (tag) fields available for - searching. - """ - for tag in self.tagtype_map: - yield u'tagtype: ' + tag - - def _tagtype_lookup(self, tag): - """Uses `tagtype_map` to look up the beets column name for an - MPD tagtype (or throw an appropriate exception). Returns both - the canonical name of the MPD tagtype and the beets column - name. - """ - for test_tag, key in self.tagtype_map.items(): - # Match case-insensitively. - if test_tag.lower() == tag.lower(): - return test_tag, key - raise BPDError(ERROR_UNKNOWN, u'no such tagtype') - - def _metadata_query(self, query_type, any_query_type, kv): - """Helper function returns a query object that will find items - according to the library query type provided and the key-value - pairs specified. The any_query_type is used for queries of - type "any"; if None, then an error is thrown. - """ - if kv: # At least one key-value pair. - queries = [] - # Iterate pairwise over the arguments. - it = iter(kv) - for tag, value in zip(it, it): - if tag.lower() == u'any': - if any_query_type: - queries.append(any_query_type(value, - ITEM_KEYS_WRITABLE, - query_type)) - else: - raise BPDError(ERROR_UNKNOWN, u'no such tagtype') - else: - _, key = self._tagtype_lookup(tag) - queries.append(query_type(key, value)) - return dbcore.query.AndQuery(queries) - else: # No key-value pairs. - return dbcore.query.TrueQuery() - - def cmd_search(self, conn, *kv): - """Perform a substring match for items.""" - query = self._metadata_query(dbcore.query.SubstringQuery, - dbcore.query.AnyFieldQuery, - kv) - for item in self.lib.items(query): - yield self._item_info(item) - - def cmd_find(self, conn, *kv): - """Perform an exact match for items.""" - query = self._metadata_query(dbcore.query.MatchQuery, - None, - kv) - for item in self.lib.items(query): - yield self._item_info(item) - - def cmd_list(self, conn, show_tag, *kv): - """List distinct metadata values for show_tag, possibly - filtered by matching match_tag to match_term. - """ - show_tag_canon, show_key = self._tagtype_lookup(show_tag) - query = self._metadata_query(dbcore.query.MatchQuery, None, kv) - - clause, subvals = query.clause() - statement = 'SELECT DISTINCT ' + show_key + \ - ' FROM items WHERE ' + clause + \ - ' ORDER BY ' + show_key - with self.lib.transaction() as tx: - rows = tx.query(statement, subvals) - - for row in rows: - yield show_tag_canon + u': ' + unicode(row[0]) - - def cmd_count(self, conn, tag, value): - """Returns the number and total time of songs matching the - tag/value query. - """ - _, key = self._tagtype_lookup(tag) - songs = 0 - playtime = 0.0 - for item in self.lib.items(dbcore.query.MatchQuery(key, value)): - songs += 1 - playtime += item.length - yield u'songs: ' + unicode(songs) - yield u'playtime: ' + unicode(int(playtime)) - - # "Outputs." Just a dummy implementation because we don't control - # any outputs. - - def cmd_outputs(self, conn): - """List the available outputs.""" - yield ( - u'outputid: 0', - u'outputname: gstreamer', - u'outputenabled: 1', - ) - - def cmd_enableoutput(self, conn, output_id): - output_id = cast_arg(int, output_id) - if output_id != 0: - raise ArgumentIndexError() - - def cmd_disableoutput(self, conn, output_id): - output_id = cast_arg(int, output_id) - if output_id == 0: - raise BPDError(ERROR_ARG, u'cannot disable this output') - else: - raise ArgumentIndexError() - - # Playback control. The functions below hook into the - # half-implementations provided by the base class. Together, they're - # enough to implement all normal playback functionality. - - def cmd_play(self, conn, index=-1): - new_index = index != -1 and index != self.current_index - was_paused = self.paused - super(Server, self).cmd_play(conn, index) - - if self.current_index > -1: # Not stopped. - if was_paused and not new_index: - # Just unpause. - self.player.play() - else: - self.player.play_file(self.playlist[self.current_index].path) - - def cmd_pause(self, conn, state=None): - super(Server, self).cmd_pause(conn, state) - if self.paused: - self.player.pause() - elif self.player.playing: - self.player.play() - - def cmd_stop(self, conn): - super(Server, self).cmd_stop(conn) - self.player.stop() - - def cmd_seek(self, conn, index, pos): - """Seeks to the specified position in the specified song.""" - index = cast_arg(int, index) - pos = cast_arg(int, pos) - super(Server, self).cmd_seek(conn, index, pos) - self.player.seek(pos) - - # Volume control. - - def cmd_setvol(self, conn, vol): - vol = cast_arg(int, vol) - super(Server, self).cmd_setvol(conn, vol) - self.player.volume = float(vol) / 100 - - -# Beets plugin hooks. - -class BPDPlugin(BeetsPlugin): - """Provides the "beet bpd" command for running a music player - server. - """ - def __init__(self): - super(BPDPlugin, self).__init__() - self.config.add({ - 'host': u'', - 'port': 6600, - 'password': u'', - 'volume': VOLUME_MAX, - }) - self.config['password'].redact = True - - def start_bpd(self, lib, host, port, password, volume, debug): - """Starts a BPD server.""" - if debug: # FIXME this should be managed by BeetsPlugin - self._log.setLevel(logging.DEBUG) - else: - self._log.setLevel(logging.WARNING) - try: - server = Server(lib, host, port, password) - server.cmd_setvol(None, volume) - server.run() - except NoGstreamerError: - global_log.error(u'Gstreamer Python bindings not found.') - global_log.error(u'Install "python-gst0.10", "py27-gst-python", ' - u'or similar package to use BPD.') - - def commands(self): - cmd = beets.ui.Subcommand( - 'bpd', help=u'run an MPD-compatible music player server' - ) - cmd.parser.add_option( - '-d', '--debug', action='store_true', - help=u'dump all MPD traffic to stdout' - ) - - def func(lib, opts, args): - host = args.pop(0) if args else self.config['host'].get(unicode) - port = args.pop(0) if args else self.config['port'].get(int) - if args: - raise beets.ui.UserError(u'too many arguments') - password = self.config['password'].get(unicode) - volume = self.config['volume'].get(int) - debug = opts.debug or False - self.start_bpd(lib, host, int(port), password, volume, debug) - - cmd.func = func - return [cmd] diff --git a/libs/beetsplug/discogs.py b/libs/beetsplug/discogs.py deleted file mode 100644 index 62a78a5ff..000000000 --- a/libs/beetsplug/discogs.py +++ /dev/null @@ -1,350 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Adds Discogs album search support to the autotagger. Requires the -discogs-client library. -""" -from __future__ import division, absolute_import, print_function - -import beets.ui -from beets import logging -from beets import config -from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance -from beets.plugins import BeetsPlugin -from beets.util import confit -from discogs_client import Release, Client -from discogs_client.exceptions import DiscogsAPIError -from requests.exceptions import ConnectionError -import beets -import re -import time -import json -import socket -import httplib -import os - - -# Silence spurious INFO log lines generated by urllib3. -urllib3_logger = logging.getLogger('requests.packages.urllib3') -urllib3_logger.setLevel(logging.CRITICAL) - -USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__) - -# Exceptions that discogs_client should really handle but does not. -CONNECTION_ERRORS = (ConnectionError, socket.error, httplib.HTTPException, - ValueError, # JSON decoding raises a ValueError. - DiscogsAPIError) - - -class DiscogsPlugin(BeetsPlugin): - - def __init__(self): - super(DiscogsPlugin, self).__init__() - self.config.add({ - 'apikey': 'rAzVUQYRaoFjeBjyWuWZ', - 'apisecret': 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy', - 'tokenfile': 'discogs_token.json', - 'source_weight': 0.5, - }) - self.config['apikey'].redact = True - self.config['apisecret'].redact = True - self.discogs_client = None - self.register_listener('import_begin', self.setup) - - def setup(self, session=None): - """Create the `discogs_client` field. Authenticate if necessary. - """ - c_key = self.config['apikey'].get(unicode) - c_secret = self.config['apisecret'].get(unicode) - - # Get the OAuth token from a file or log in. - try: - with open(self._tokenfile()) as f: - tokendata = json.load(f) - except IOError: - # No token yet. Generate one. - token, secret = self.authenticate(c_key, c_secret) - else: - token = tokendata['token'] - secret = tokendata['secret'] - - self.discogs_client = Client(USER_AGENT, c_key, c_secret, - token, secret) - - def reset_auth(self): - """Delete toke file & redo the auth steps. - """ - os.remove(self._tokenfile()) - self.setup() - - def _tokenfile(self): - """Get the path to the JSON file for storing the OAuth token. - """ - return self.config['tokenfile'].get(confit.Filename(in_app_dir=True)) - - def authenticate(self, c_key, c_secret): - # Get the link for the OAuth page. - auth_client = Client(USER_AGENT, c_key, c_secret) - try: - _, _, url = auth_client.get_authorize_url() - except CONNECTION_ERRORS as e: - self._log.debug(u'connection error: {0}', e) - raise beets.ui.UserError(u'communication with Discogs failed') - - beets.ui.print_(u"To authenticate with Discogs, visit:") - beets.ui.print_(url) - - # Ask for the code and validate it. - code = beets.ui.input_(u"Enter the code:") - try: - token, secret = auth_client.get_access_token(code) - except DiscogsAPIError: - raise beets.ui.UserError(u'Discogs authorization failed') - except CONNECTION_ERRORS as e: - self._log.debug(u'connection error: {0}', e) - raise beets.ui.UserError(u'Discogs token request failed') - - # Save the token for later use. - self._log.debug(u'Discogs token {0}, secret {1}', token, secret) - with open(self._tokenfile(), 'w') as f: - json.dump({'token': token, 'secret': secret}, f) - - return token, secret - - def album_distance(self, items, album_info, mapping): - """Returns the album distance. - """ - dist = Distance() - if album_info.data_source == 'Discogs': - dist.add('source', self.config['source_weight'].as_number()) - return dist - - def candidates(self, items, artist, album, va_likely): - """Returns a list of AlbumInfo objects for discogs search results - matching an album and artist (if not various). - """ - if not self.discogs_client: - return - - if va_likely: - query = album - else: - query = '%s %s' % (artist, album) - try: - return self.get_albums(query) - except DiscogsAPIError as e: - self._log.debug(u'API Error: {0} (query: {1})', e, query) - if e.status_code == 401: - self.reset_auth() - return self.candidates(items, artist, album, va_likely) - else: - return [] - except CONNECTION_ERRORS: - self._log.debug(u'Connection error in album search', exc_info=True) - return [] - - def album_for_id(self, album_id): - """Fetches an album by its Discogs ID and returns an AlbumInfo object - or None if the album is not found. - """ - if not self.discogs_client: - return - - self._log.debug(u'Searching for release {0}', album_id) - # Discogs-IDs are simple integers. We only look for those at the end - # of an input string as to avoid confusion with other metadata plugins. - # An optional bracket can follow the integer, as this is how discogs - # displays the release ID on its webpage. - match = re.search(r'(^|\[*r|discogs\.com/.+/release/)(\d+)($|\])', - album_id) - if not match: - return None - result = Release(self.discogs_client, {'id': int(match.group(2))}) - # Try to obtain title to verify that we indeed have a valid Release - try: - getattr(result, 'title') - except DiscogsAPIError as e: - if e.status_code != 404: - self._log.debug(u'API Error: {0} (query: {1})', e, result._uri) - if e.status_code == 401: - self.reset_auth() - return self.album_for_id(album_id) - return None - except CONNECTION_ERRORS: - self._log.debug(u'Connection error in album lookup', exc_info=True) - return None - return self.get_album_info(result) - - def get_albums(self, query): - """Returns a list of AlbumInfo objects for a discogs search query. - """ - # Strip non-word characters from query. Things like "!" and "-" can - # cause a query to return no results, even if they match the artist or - # album title. Use `re.UNICODE` flag to avoid stripping non-english - # word characters. - # TEMPORARY: Encode as ASCII to work around a bug: - # https://github.com/beetbox/beets/issues/1051 - # When the library is fixed, we should encode as UTF-8. - query = re.sub(r'(?u)\W+', ' ', query).encode('ascii', "replace") - # Strip medium information from query, Things like "CD1" and "disk 1" - # can also negate an otherwise positive result. - query = re.sub(r'(?i)\b(CD|disc)\s*\d+', '', query) - try: - releases = self.discogs_client.search(query, - type='release').page(1) - except CONNECTION_ERRORS: - self._log.debug(u"Communication error while searching for {0!r}", - query, exc_info=True) - return [] - return [self.get_album_info(release) for release in releases[:5]] - - def get_album_info(self, result): - """Returns an AlbumInfo object for a discogs Release object. - """ - artist, artist_id = self.get_artist([a.data for a in result.artists]) - album = re.sub(r' +', ' ', result.title) - album_id = result.data['id'] - # Use `.data` to access the tracklist directly instead of the - # convenient `.tracklist` property, which will strip out useful artist - # information and leave us with skeleton `Artist` objects that will - # each make an API call just to get the same data back. - tracks = self.get_tracks(result.data['tracklist']) - albumtype = ', '.join( - result.data['formats'][0].get('descriptions', [])) or None - va = result.data['artists'][0]['name'].lower() == 'various' - if va: - artist = config['va_name'].get(unicode) - year = result.data['year'] - label = result.data['labels'][0]['name'] - mediums = len(set(t.medium for t in tracks)) - catalogno = result.data['labels'][0]['catno'] - if catalogno == 'none': - catalogno = None - country = result.data.get('country') - media = result.data['formats'][0]['name'] - data_url = result.data['uri'] - return AlbumInfo(album, album_id, artist, artist_id, tracks, asin=None, - albumtype=albumtype, va=va, year=year, month=None, - day=None, label=label, mediums=mediums, - artist_sort=None, releasegroup_id=None, - catalognum=catalogno, script=None, language=None, - country=country, albumstatus=None, media=media, - albumdisambig=None, artist_credit=None, - original_year=None, original_month=None, - original_day=None, data_source='Discogs', - data_url=data_url) - - def get_artist(self, artists): - """Returns an artist string (all artists) and an artist_id (the main - artist) for a list of discogs album or track artists. - """ - artist_id = None - bits = [] - for i, artist in enumerate(artists): - if not artist_id: - artist_id = artist['id'] - name = artist['name'] - # Strip disambiguation number. - name = re.sub(r' \(\d+\)$', '', name) - # Move articles to the front. - name = re.sub(r'(?i)^(.*?), (a|an|the)$', r'\2 \1', name) - bits.append(name) - if artist['join'] and i < len(artists) - 1: - bits.append(artist['join']) - artist = ' '.join(bits).replace(' ,', ',') or None - return artist, artist_id - - def get_tracks(self, tracklist): - """Returns a list of TrackInfo objects for a discogs tracklist. - """ - tracks = [] - index_tracks = {} - index = 0 - for track in tracklist: - # Only real tracks have `position`. Otherwise, it's an index track. - if track['position']: - index += 1 - tracks.append(self.get_track_info(track, index)) - else: - index_tracks[index + 1] = track['title'] - - # Fix up medium and medium_index for each track. Discogs position is - # unreliable, but tracks are in order. - medium = None - medium_count, index_count = 0, 0 - for track in tracks: - # Handle special case where a different medium does not indicate a - # new disc, when there is no medium_index and the ordinal of medium - # is not sequential. For example, I, II, III, IV, V. Assume these - # are the track index, not the medium. - medium_is_index = track.medium and not track.medium_index and ( - len(track.medium) != 1 or - ord(track.medium) - 64 != medium_count + 1 - ) - - if not medium_is_index and medium != track.medium: - # Increment medium_count and reset index_count when medium - # changes. - medium = track.medium - medium_count += 1 - index_count = 0 - index_count += 1 - track.medium, track.medium_index = medium_count, index_count - - # Get `disctitle` from Discogs index tracks. Assume that an index track - # before the first track of each medium is a disc title. - for track in tracks: - if track.medium_index == 1: - if track.index in index_tracks: - disctitle = index_tracks[track.index] - else: - disctitle = None - track.disctitle = disctitle - - return tracks - - def get_track_info(self, track, index): - """Returns a TrackInfo object for a discogs track. - """ - title = track['title'] - track_id = None - medium, medium_index = self.get_track_index(track['position']) - artist, artist_id = self.get_artist(track.get('artists', [])) - length = self.get_track_length(track['duration']) - return TrackInfo(title, track_id, artist, artist_id, length, index, - medium, medium_index, artist_sort=None, - disctitle=None, artist_credit=None) - - def get_track_index(self, position): - """Returns the medium and medium index for a discogs track position. - """ - # medium_index is a number at the end of position. medium is everything - # else. E.g. (A)(1), (Side A, Track )(1), (A)(), ()(1), etc. - match = re.match(r'^(.*?)(\d*)$', position.upper()) - if match: - medium, index = match.groups() - else: - self._log.debug(u'Invalid position: {0}', position) - medium = index = None - return medium or None, index or None - - def get_track_length(self, duration): - """Returns the track length in seconds for a discogs duration. - """ - try: - length = time.strptime(duration, '%M:%S') - except ValueError: - return None - return length.tm_min * 60 + length.tm_sec diff --git a/libs/beetsplug/embyupdate.py b/libs/beetsplug/embyupdate.py deleted file mode 100644 index 38f8929e5..000000000 --- a/libs/beetsplug/embyupdate.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Updates the Emby Library whenever the beets library is changed. - - emby: - host: localhost - port: 8096 - username: user - password: password -""" -from __future__ import division, absolute_import, print_function - -from beets import config -from beets.plugins import BeetsPlugin -from urllib import urlencode -from urlparse import urljoin, parse_qs, urlsplit, urlunsplit -import hashlib -import requests - - -def api_url(host, port, endpoint): - """Returns a joined url. - """ - joined = urljoin('http://{0}:{1}'.format(host, port), endpoint) - scheme, netloc, path, query_string, fragment = urlsplit(joined) - query_params = parse_qs(query_string) - - query_params['format'] = ['json'] - new_query_string = urlencode(query_params, doseq=True) - - return urlunsplit((scheme, netloc, path, new_query_string, fragment)) - - -def password_data(username, password): - """Returns a dict with username and its encoded password. - """ - return { - 'username': username, - 'password': hashlib.sha1(password).hexdigest(), - 'passwordMd5': hashlib.md5(password).hexdigest() - } - - -def create_headers(user_id, token=None): - """Return header dict that is needed to talk to the Emby API. - """ - headers = { - 'Authorization': 'MediaBrowser', - 'UserId': user_id, - 'Client': 'other', - 'Device': 'empy', - 'DeviceId': 'beets', - 'Version': '0.0.0' - } - - if token: - headers['X-MediaBrowser-Token'] = token - - return headers - - -def get_token(host, port, headers, auth_data): - """Return token for a user. - """ - url = api_url(host, port, '/Users/AuthenticateByName') - r = requests.post(url, headers=headers, data=auth_data) - - return r.json().get('AccessToken') - - -def get_user(host, port, username): - """Return user dict from server or None if there is no user. - """ - url = api_url(host, port, '/Users/Public') - r = requests.get(url) - user = [i for i in r.json() if i['Name'] == username] - - return user - - -class EmbyUpdate(BeetsPlugin): - def __init__(self): - super(EmbyUpdate, self).__init__() - - # Adding defaults. - config['emby'].add({ - u'host': u'localhost', - u'port': 8096 - }) - - self.register_listener('database_change', self.listen_for_db_change) - - def listen_for_db_change(self, lib, model): - """Listens for beets db change and register the update for the end. - """ - self.register_listener('cli_exit', self.update) - - def update(self, lib): - """When the client exists try to send refresh request to Emby. - """ - self._log.info(u'Updating Emby library...') - - host = config['emby']['host'].get() - port = config['emby']['port'].get() - username = config['emby']['username'].get() - password = config['emby']['password'].get() - - # Get user information from the Emby API. - user = get_user(host, port, username) - if not user: - self._log.warning(u'User {0} could not be found.'.format(username)) - return - - # Create Authentication data and headers. - auth_data = password_data(username, password) - headers = create_headers(user[0]['Id']) - - # Get authentication token. - token = get_token(host, port, headers, auth_data) - if not token: - self._log.warning( - u'Could not get token for user {0}', username - ) - return - - # Recreate headers with a token. - headers = create_headers(user[0]['Id'], token=token) - - # Trigger the Update. - url = api_url(host, port, '/Library/Refresh') - r = requests.post(url, headers=headers) - if r.status_code != 204: - self._log.warning(u'Update could not be triggered') - else: - self._log.info(u'Update triggered.') diff --git a/libs/beetsplug/hook.py b/libs/beetsplug/hook.py deleted file mode 100644 index 4f2b8f0e2..000000000 --- a/libs/beetsplug/hook.py +++ /dev/null @@ -1,108 +0,0 @@ -# This file is part of beets. -# Copyright 2015, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Allows custom commands to be run when an event is emitted by beets""" -from __future__ import division, absolute_import, print_function - -import string -import subprocess - -from beets.plugins import BeetsPlugin -from beets.ui import _arg_encoding -from beets.util import shlex_split - - -class CodingFormatter(string.Formatter): - """A custom string formatter that decodes the format string and it's - fields. - """ - - def __init__(self, coding): - """Creates a new coding formatter with the provided coding.""" - self._coding = coding - - def format(self, format_string, *args, **kwargs): - """Formats the provided string using the provided arguments and keyword - arguments. - - This method decodes the format string using the formatter's coding. - - See str.format and string.Formatter.format. - """ - try: - format_string = format_string.decode(self._coding) - except UnicodeEncodeError: - pass - - return super(CodingFormatter, self).format(format_string, *args, - **kwargs) - - def convert_field(self, value, conversion): - """Converts the provided value given a conversion type. - - This method decodes the converted value using the formatter's coding. - - See string.Formatter.convert_field. - """ - converted = super(CodingFormatter, self).convert_field(value, - conversion) - try: - converted = converted.decode(self._coding) - except UnicodeEncodeError: - pass - - return converted - - -class HookPlugin(BeetsPlugin): - """Allows custom commands to be run when an event is emitted by beets""" - def __init__(self): - super(HookPlugin, self).__init__() - - self.config.add({ - 'hooks': [] - }) - - hooks = self.config['hooks'].get(list) - - for hook_index in range(len(hooks)): - hook = self.config['hooks'][hook_index] - - hook_event = hook['event'].get(unicode) - hook_command = hook['command'].get(unicode) - - self.create_and_register_hook(hook_event, hook_command) - - def create_and_register_hook(self, event, command): - def hook_function(**kwargs): - if command is None or len(command) == 0: - self._log.error('invalid command "{0}"', command) - return - - formatter = CodingFormatter(_arg_encoding()) - command_pieces = shlex_split(command) - - for i, piece in enumerate(command_pieces): - command_pieces[i] = formatter.format(piece, event=event, - **kwargs) - - self._log.debug(u'running command "{0}" for event {1}', - u' '.join(command_pieces), event) - - try: - subprocess.Popen(command_pieces).wait() - except OSError as exc: - self._log.error(u'hook for {0} failed: {1}', event, exc) - - self.register_listener(event, hook_function) diff --git a/libs/beetsplug/lastgenre/__init__.py b/libs/beetsplug/lastgenre/__init__.py deleted file mode 100644 index a4b8f062b..000000000 --- a/libs/beetsplug/lastgenre/__init__.py +++ /dev/null @@ -1,425 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -from __future__ import division, absolute_import, print_function - -"""Gets genres for imported music based on Last.fm tags. - -Uses a provided whitelist file to determine which tags are valid genres. -The included (default) genre list was originally produced by scraping Wikipedia -and has been edited to remove some questionable entries. -The scraper script used is available here: -https://gist.github.com/1241307 -""" -import pylast -import os -import yaml -import traceback - -from beets import plugins -from beets import ui -from beets import config -from beets.util import normpath, plurality -from beets import library - - -LASTFM = pylast.LastFMNetwork(api_key=plugins.LASTFM_KEY) - -PYLAST_EXCEPTIONS = ( - pylast.WSError, - pylast.MalformedResponseError, - pylast.NetworkError, -) - -REPLACE = { - u'\u2010': '-', -} - - -def deduplicate(seq): - """Remove duplicates from sequence wile preserving order. - """ - seen = set() - return [x for x in seq if x not in seen and not seen.add(x)] - - -# Canonicalization tree processing. - -def flatten_tree(elem, path, branches): - """Flatten nested lists/dictionaries into lists of strings - (branches). - """ - if not path: - path = [] - - if isinstance(elem, dict): - for (k, v) in elem.items(): - flatten_tree(v, path + [k], branches) - elif isinstance(elem, list): - for sub in elem: - flatten_tree(sub, path, branches) - else: - branches.append(path + [unicode(elem)]) - - -def find_parents(candidate, branches): - """Find parents genre of a given genre, ordered from the closest to - the further parent. - """ - for branch in branches: - try: - idx = branch.index(candidate.lower()) - return list(reversed(branch[:idx + 1])) - except ValueError: - continue - return [candidate] - - -# Main plugin logic. - -WHITELIST = os.path.join(os.path.dirname(__file__), 'genres.txt') -C14N_TREE = os.path.join(os.path.dirname(__file__), 'genres-tree.yaml') - - -class LastGenrePlugin(plugins.BeetsPlugin): - def __init__(self): - super(LastGenrePlugin, self).__init__() - - self.config.add({ - 'whitelist': True, - 'min_weight': 10, - 'count': 1, - 'fallback': None, - 'canonical': False, - 'source': 'album', - 'force': True, - 'auto': True, - 'separator': u', ', - }) - - self.setup() - - def setup(self): - """Setup plugin from config options - """ - if self.config['auto']: - self.import_stages = [self.imported] - - self._genre_cache = {} - - # Read the whitelist file if enabled. - self.whitelist = set() - wl_filename = self.config['whitelist'].get() - if wl_filename in (True, ''): # Indicates the default whitelist. - wl_filename = WHITELIST - if wl_filename: - wl_filename = normpath(wl_filename) - with open(wl_filename, 'r') as f: - for line in f: - line = line.decode('utf8').strip().lower() - if line and not line.startswith(u'#'): - self.whitelist.add(line) - - # Read the genres tree for canonicalization if enabled. - self.c14n_branches = [] - c14n_filename = self.config['canonical'].get() - if c14n_filename in (True, ''): # Default tree. - c14n_filename = C14N_TREE - if c14n_filename: - c14n_filename = normpath(c14n_filename) - genres_tree = yaml.load(open(c14n_filename, 'r')) - flatten_tree(genres_tree, [], self.c14n_branches) - - @property - def sources(self): - """A tuple of allowed genre sources. May contain 'track', - 'album', or 'artist.' - """ - source = self.config['source'].as_choice(('track', 'album', 'artist')) - if source == 'track': - return 'track', 'album', 'artist' - elif source == 'album': - return 'album', 'artist' - elif source == 'artist': - return 'artist', - - def _resolve_genres(self, tags): - """Given a list of strings, return a genre by joining them into a - single string and (optionally) canonicalizing each. - """ - if not tags: - return None - - count = self.config['count'].get(int) - if self.c14n_branches: - # Extend the list to consider tags parents in the c14n tree - tags_all = [] - for tag in tags: - # Add parents that are in the whitelist, or add the oldest - # ancestor if no whitelist - if self.whitelist: - parents = [x for x in find_parents(tag, self.c14n_branches) - if self._is_allowed(x)] - else: - parents = [find_parents(tag, self.c14n_branches)[-1]] - - tags_all += parents - if len(tags_all) >= count: - break - tags = tags_all - - tags = deduplicate(tags) - - # c14n only adds allowed genres but we may have had forbidden genres in - # the original tags list - tags = [x.title() for x in tags if self._is_allowed(x)] - - return self.config['separator'].get(unicode).join( - tags[:self.config['count'].get(int)] - ) - - def fetch_genre(self, lastfm_obj): - """Return the genre for a pylast entity or None if no suitable genre - can be found. Ex. 'Electronic, House, Dance' - """ - min_weight = self.config['min_weight'].get(int) - return self._resolve_genres(self._tags_for(lastfm_obj, min_weight)) - - def _is_allowed(self, genre): - """Determine whether the genre is present in the whitelist, - returning a boolean. - """ - if genre is None: - return False - if not self.whitelist or genre in self.whitelist: - return True - return False - - # Cached entity lookups. - - def _last_lookup(self, entity, method, *args): - """Get a genre based on the named entity using the callable `method` - whose arguments are given in the sequence `args`. The genre lookup - is cached based on the entity name and the arguments. Before the - lookup, each argument is has some Unicode characters replaced with - rough ASCII equivalents in order to return better results from the - Last.fm database. - """ - # Shortcut if we're missing metadata. - if any(not s for s in args): - return None - - key = u'{0}.{1}'.format(entity, u'-'.join(unicode(a) for a in args)) - if key in self._genre_cache: - return self._genre_cache[key] - else: - args_replaced = [] - for arg in args: - for k, v in REPLACE.items(): - arg = arg.replace(k, v) - args_replaced.append(arg) - - genre = self.fetch_genre(method(*args_replaced)) - self._genre_cache[key] = genre - return genre - - def fetch_album_genre(self, obj): - """Return the album genre for this Item or Album. - """ - return self._last_lookup( - u'album', LASTFM.get_album, obj.albumartist, obj.album - ) - - def fetch_album_artist_genre(self, obj): - """Return the album artist genre for this Item or Album. - """ - return self._last_lookup( - u'artist', LASTFM.get_artist, obj.albumartist - ) - - def fetch_artist_genre(self, item): - """Returns the track artist genre for this Item. - """ - return self._last_lookup( - u'artist', LASTFM.get_artist, item.artist - ) - - def fetch_track_genre(self, obj): - """Returns the track genre for this Item. - """ - return self._last_lookup( - u'track', LASTFM.get_track, obj.artist, obj.title - ) - - def _get_genre(self, obj): - """Get the genre string for an Album or Item object based on - self.sources. Return a `(genre, source)` pair. The - prioritization order is: - - track (for Items only) - - album - - artist - - original - - fallback - - None - """ - - # Shortcut to existing genre if not forcing. - if not self.config['force'] and self._is_allowed(obj.genre): - return obj.genre, 'keep' - - # Track genre (for Items only). - if isinstance(obj, library.Item): - if 'track' in self.sources: - result = self.fetch_track_genre(obj) - if result: - return result, 'track' - - # Album genre. - if 'album' in self.sources: - result = self.fetch_album_genre(obj) - if result: - return result, 'album' - - # Artist (or album artist) genre. - if 'artist' in self.sources: - result = None - if isinstance(obj, library.Item): - result = self.fetch_artist_genre(obj) - elif obj.albumartist != config['va_name'].get(unicode): - result = self.fetch_album_artist_genre(obj) - else: - # For "Various Artists", pick the most popular track genre. - item_genres = [] - for item in obj.items(): - item_genre = None - if 'track' in self.sources: - item_genre = self.fetch_track_genre(item) - if not item_genre: - item_genre = self.fetch_artist_genre(item) - if item_genre: - item_genres.append(item_genre) - if item_genres: - result, _ = plurality(item_genres) - - if result: - return result, 'artist' - - # Filter the existing genre. - if obj.genre: - result = self._resolve_genres([obj.genre]) - if result: - return result, 'original' - - # Fallback string. - fallback = self.config['fallback'].get() - if fallback: - return fallback, 'fallback' - - return None, None - - def commands(self): - lastgenre_cmd = ui.Subcommand('lastgenre', help=u'fetch genres') - lastgenre_cmd.parser.add_option( - u'-f', u'--force', dest='force', - action='store_true', default=False, - help=u're-download genre when already present' - ) - lastgenre_cmd.parser.add_option( - u'-s', u'--source', dest='source', type='string', - help=u'genre source: artist, album, or track' - ) - - def lastgenre_func(lib, opts, args): - write = ui.should_write() - self.config.set_args(opts) - - for album in lib.albums(ui.decargs(args)): - album.genre, src = self._get_genre(album) - self._log.info(u'genre for album {0} ({1}): {0.genre}', - album, src) - album.store() - - for item in album.items(): - # If we're using track-level sources, also look up each - # track on the album. - if 'track' in self.sources: - item.genre, src = self._get_genre(item) - item.store() - self._log.info(u'genre for track {0} ({1}): {0.genre}', - item, src) - - if write: - item.try_write() - - lastgenre_cmd.func = lastgenre_func - return [lastgenre_cmd] - - def imported(self, session, task): - """Event hook called when an import task finishes.""" - if task.is_album: - album = task.album - album.genre, src = self._get_genre(album) - self._log.debug(u'added last.fm album genre ({0}): {1}', - src, album.genre) - album.store() - - if 'track' in self.sources: - for item in album.items(): - item.genre, src = self._get_genre(item) - self._log.debug(u'added last.fm item genre ({0}): {1}', - src, item.genre) - item.store() - - else: - item = task.item - item.genre, src = self._get_genre(item) - self._log.debug(u'added last.fm item genre ({0}): {1}', - src, item.genre) - item.store() - - def _tags_for(self, obj, min_weight=None): - """Core genre identification routine. - - Given a pylast entity (album or track), return a list of - tag names for that entity. Return an empty list if the entity is - not found or another error occurs. - - If `min_weight` is specified, tags are filtered by weight. - """ - # Work around an inconsistency in pylast where - # Album.get_top_tags() does not return TopItem instances. - # https://code.google.com/p/pylast/issues/detail?id=85 - if isinstance(obj, pylast.Album): - obj = super(pylast.Album, obj) - - try: - res = obj.get_top_tags() - except PYLAST_EXCEPTIONS as exc: - self._log.debug(u'last.fm error: {0}', exc) - return [] - except Exception as exc: - # Isolate bugs in pylast. - self._log.debug(u'{}', traceback.format_exc()) - self._log.error(u'error in pylast library: {0}', exc) - return [] - - # Filter by weight (optionally). - if min_weight: - res = [el for el in res if (int(el.weight or 0)) >= min_weight] - - # Get strings from tags. - res = [el.item.get_name().lower() for el in res] - - return res diff --git a/libs/beetsplug/lyrics.py b/libs/beetsplug/lyrics.py deleted file mode 100644 index b6936e1be..000000000 --- a/libs/beetsplug/lyrics.py +++ /dev/null @@ -1,760 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Fetches, embeds, and displays lyrics. -""" - -from __future__ import absolute_import, division, print_function - -import difflib -import itertools -import json -import re -import requests -import unicodedata -import urllib -import warnings -from HTMLParser import HTMLParseError - -try: - from bs4 import SoupStrainer, BeautifulSoup - HAS_BEAUTIFUL_SOUP = True -except ImportError: - HAS_BEAUTIFUL_SOUP = False - -try: - import langdetect - HAS_LANGDETECT = True -except ImportError: - HAS_LANGDETECT = False - -from beets import plugins -from beets import ui - - -DIV_RE = re.compile(r'<(/?)div>?', re.I) -COMMENT_RE = re.compile(r'', re.S) -TAG_RE = re.compile(r'<[^>]*>') -BREAK_RE = re.compile(r'\n?\s*]*)*>\s*\n?', re.I) -URL_CHARACTERS = { - u'\u2018': u"'", - u'\u2019': u"'", - u'\u201c': u'"', - u'\u201d': u'"', - u'\u2010': u'-', - u'\u2011': u'-', - u'\u2012': u'-', - u'\u2013': u'-', - u'\u2014': u'-', - u'\u2015': u'-', - u'\u2016': u'-', - u'\u2026': u'...', -} - - -# Utilities. - - -def unescape(text): - """Resolve &#xxx; HTML entities (and some others).""" - if isinstance(text, bytes): - text = text.decode('utf8', 'ignore') - out = text.replace(u' ', u' ') - - def replchar(m): - num = m.group(1) - return unichr(int(num)) - out = re.sub(u"&#(\d+);", replchar, out) - return out - - -def extract_text_between(html, start_marker, end_marker): - try: - _, html = html.split(start_marker, 1) - html, _ = html.split(end_marker, 1) - except ValueError: - return u'' - return html - - -def extract_text_in(html, starttag): - """Extract the text from a
tag in the HTML starting with - ``starttag``. Returns None if parsing fails. - """ - - # Strip off the leading text before opening tag. - try: - _, html = html.split(starttag, 1) - except ValueError: - return - - # Walk through balanced DIV tags. - level = 0 - parts = [] - pos = 0 - for match in DIV_RE.finditer(html): - if match.group(1): # Closing tag. - level -= 1 - if level == 0: - pos = match.end() - else: # Opening tag. - if level == 0: - parts.append(html[pos:match.start()]) - level += 1 - - if level == -1: - parts.append(html[pos:match.start()]) - break - else: - print(u'no closing tag found!') - return - return u''.join(parts) - - -def search_pairs(item): - """Yield a pairs of artists and titles to search for. - - The first item in the pair is the name of the artist, the second - item is a list of song names. - - In addition to the artist and title obtained from the `item` the - method tries to strip extra information like paranthesized suffixes - and featured artists from the strings and add them as candidates. - The method also tries to split multiple titles separated with `/`. - """ - - title, artist = item.title, item.artist - titles = [title] - artists = [artist] - - # Remove any featuring artists from the artists name - pattern = r"(.*?) {0}".format(plugins.feat_tokens()) - match = re.search(pattern, artist, re.IGNORECASE) - if match: - artists.append(match.group(1)) - - # Remove a parenthesized suffix from a title string. Common - # examples include (live), (remix), and (acoustic). - pattern = r"(.+?)\s+[(].*[)]$" - match = re.search(pattern, title, re.IGNORECASE) - if match: - titles.append(match.group(1)) - - # Remove any featuring artists from the title - pattern = r"(.*?) {0}".format(plugins.feat_tokens(for_artist=False)) - for title in titles[:]: - match = re.search(pattern, title, re.IGNORECASE) - if match: - titles.append(match.group(1)) - - # Check for a dual song (e.g. Pink Floyd - Speak to Me / Breathe) - # and each of them. - multi_titles = [] - for title in titles: - multi_titles.append([title]) - if '/' in title: - multi_titles.append([x.strip() for x in title.split('/')]) - - return itertools.product(artists, multi_titles) - - -class Backend(object): - def __init__(self, config, log): - self._log = log - - @staticmethod - def _encode(s): - """Encode the string for inclusion in a URL""" - if isinstance(s, unicode): - for char, repl in URL_CHARACTERS.items(): - s = s.replace(char, repl) - s = s.encode('utf8', 'ignore') - return urllib.quote(s) - - def build_url(self, artist, title): - return self.URL_PATTERN % (self._encode(artist.title()), - self._encode(title.title())) - - def fetch_url(self, url): - """Retrieve the content at a given URL, or return None if the source - is unreachable. - """ - try: - # Disable the InsecureRequestWarning that comes from using - # `verify=false`. - # https://github.com/kennethreitz/requests/issues/2214 - # We're not overly worried about the NSA MITMing our lyrics scraper - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - r = requests.get(url, verify=False) - except requests.RequestException as exc: - self._log.debug(u'lyrics request failed: {0}', exc) - return - if r.status_code == requests.codes.ok: - return r.text - else: - self._log.debug(u'failed to fetch: {0} ({1})', url, r.status_code) - - def fetch(self, artist, title): - raise NotImplementedError() - - -class SymbolsReplaced(Backend): - REPLACEMENTS = { - r'\s+': '_', - '<': 'Less_Than', - '>': 'Greater_Than', - '#': 'Number_', - r'[\[\{]': '(', - r'[\[\{]': ')' - } - - @classmethod - def _encode(cls, s): - for old, new in cls.REPLACEMENTS.iteritems(): - s = re.sub(old, new, s) - - return super(SymbolsReplaced, cls)._encode(s) - - -class MusiXmatch(SymbolsReplaced): - REPLACEMENTS = dict(SymbolsReplaced.REPLACEMENTS, **{ - r'\s+': '-' - }) - - URL_PATTERN = 'https://www.musixmatch.com/lyrics/%s/%s' - - def fetch(self, artist, title): - url = self.build_url(artist, title) - html = self.fetch_url(url) - if not html: - return - lyrics = extract_text_between(html, - '"body":', '"language":') - return lyrics.strip(',"').replace('\\n', '\n') - - -class Genius(Backend): - """Fetch lyrics from Genius via genius-api.""" - def __init__(self, config, log): - super(Genius, self).__init__(config, log) - self.api_key = config['genius_api_key'].get(unicode) - self.headers = {'Authorization': "Bearer %s" % self.api_key} - - def search_genius(self, artist, title): - query = u"%s %s" % (artist, title) - url = u'https://api.genius.com/search?q=%s' \ - % (urllib.quote(query.encode('utf8'))) - - self._log.debug(u'genius: requesting search {}', url) - try: - req = requests.get( - url, - headers=self.headers, - allow_redirects=True - ) - req.raise_for_status() - except requests.RequestException as exc: - self._log.debug(u'genius: request error: {}', exc) - return None - - try: - return req.json() - except ValueError: - self._log.debug(u'genius: invalid response: {}', req.text) - return None - - def get_lyrics(self, link): - url = u'http://genius-api.com/api/lyricsInfo' - - self._log.debug(u'genius: requesting lyrics for link {}', link) - try: - req = requests.post( - url, - data={'link': link}, - headers=self.headers, - allow_redirects=True - ) - req.raise_for_status() - except requests.RequestException as exc: - self._log.debug(u'genius: request error: {}', exc) - return None - - try: - return req.json() - except ValueError: - self._log.debug(u'genius: invalid response: {}', req.text) - return None - - def build_lyric_string(self, lyrics): - if 'lyrics' not in lyrics: - return - sections = lyrics['lyrics']['sections'] - - lyrics_list = [] - for section in sections: - lyrics_list.append(section['name']) - lyrics_list.append('\n') - for verse in section['verses']: - if 'content' in verse: - lyrics_list.append(verse['content']) - - return ''.join(lyrics_list) - - def fetch(self, artist, title): - search_data = self.search_genius(artist, title) - if not search_data: - return - - if not search_data['meta']['status'] == 200: - return - else: - records = search_data['response']['hits'] - if not records: - return - - record_url = records[0]['result']['url'] - lyric_data = self.get_lyrics(record_url) - if not lyric_data: - return - lyrics = self.build_lyric_string(lyric_data) - - return lyrics - - -class LyricsWiki(SymbolsReplaced): - """Fetch lyrics from LyricsWiki.""" - URL_PATTERN = 'http://lyrics.wikia.com/%s:%s' - - def fetch(self, artist, title): - url = self.build_url(artist, title) - html = self.fetch_url(url) - if not html: - return - - # Get the HTML fragment inside the appropriate HTML element and then - # extract the text from it. - html_frag = extract_text_in(html, u"
") - if html_frag: - lyrics = _scrape_strip_cruft(html_frag, True) - - if lyrics and 'Unfortunately, we are not licensed' not in lyrics: - return lyrics - - -class LyricsCom(Backend): - """Fetch lyrics from Lyrics.com.""" - URL_PATTERN = 'http://www.lyrics.com/%s-lyrics-%s.html' - NOT_FOUND = ( - 'Sorry, we do not have the lyric', - 'Submit Lyrics', - ) - - @classmethod - def _encode(cls, s): - s = re.sub(r'[^\w\s-]', '', s) - s = re.sub(r'\s+', '-', s) - return super(LyricsCom, cls)._encode(s).lower() - - def fetch(self, artist, title): - url = self.build_url(artist, title) - html = self.fetch_url(url) - if not html: - return - lyrics = extract_text_between(html, '
', '
') - if not lyrics: - return - for not_found_str in self.NOT_FOUND: - if not_found_str in lyrics: - return - - parts = lyrics.split('\n---\nLyrics powered by', 1) - if parts: - return parts[0] - - -def remove_credits(text): - """Remove first/last line of text if it contains the word 'lyrics' - eg 'Lyrics by songsdatabase.com' - """ - textlines = text.split('\n') - credits = None - for i in (0, -1): - if textlines and 'lyrics' in textlines[i].lower(): - credits = textlines.pop(i) - if credits: - text = '\n'.join(textlines) - return text - - -def _scrape_strip_cruft(html, plain_text_out=False): - """Clean up HTML - """ - html = unescape(html) - - html = html.replace('\r', '\n') # Normalize EOL. - html = re.sub(r' +', ' ', html) # Whitespaces collapse. - html = BREAK_RE.sub('\n', html) #
eats up surrounding '\n'. - html = re.sub(r'<(script).*?(?s)', '', html) # Strip script tags. - - if plain_text_out: # Strip remaining HTML tags - html = COMMENT_RE.sub('', html) - html = TAG_RE.sub('', html) - - html = '\n'.join([x.strip() for x in html.strip().split('\n')]) - html = re.sub(r'\n{3,}', r'\n\n', html) - return html - - -def _scrape_merge_paragraphs(html): - html = re.sub(r'

\s*]*)>', '\n', html) - return re.sub(r'
\s*
', '\n', html) - - -def scrape_lyrics_from_html(html): - """Scrape lyrics from a URL. If no lyrics can be found, return None - instead. - """ - if not HAS_BEAUTIFUL_SOUP: - return None - - if not html: - return None - - def is_text_notcode(text): - length = len(text) - return (length > 20 and - text.count(' ') > length / 25 and - (text.find('{') == -1 or text.find(';') == -1)) - html = _scrape_strip_cruft(html) - html = _scrape_merge_paragraphs(html) - - # extract all long text blocks that are not code - try: - soup = BeautifulSoup(html, "html.parser", - parse_only=SoupStrainer(text=is_text_notcode)) - except HTMLParseError: - return None - - # Get the longest text element (if any). - strings = sorted(soup.stripped_strings, key=len, reverse=True) - if strings: - return strings[0] - else: - return None - - -class Google(Backend): - """Fetch lyrics from Google search results.""" - def __init__(self, config, log): - super(Google, self).__init__(config, log) - self.api_key = config['google_API_key'].get(unicode) - self.engine_id = config['google_engine_ID'].get(unicode) - - def is_lyrics(self, text, artist=None): - """Determine whether the text seems to be valid lyrics. - """ - if not text: - return False - bad_triggers_occ = [] - nb_lines = text.count('\n') - if nb_lines <= 1: - self._log.debug(u"Ignoring too short lyrics '{0}'", text) - return False - elif nb_lines < 5: - bad_triggers_occ.append('too_short') - else: - # Lyrics look legit, remove credits to avoid being penalized - # further down - text = remove_credits(text) - - bad_triggers = ['lyrics', 'copyright', 'property', 'links'] - if artist: - bad_triggers_occ += [artist] - - for item in bad_triggers: - bad_triggers_occ += [item] * len(re.findall(r'\W%s\W' % item, - text, re.I)) - - if bad_triggers_occ: - self._log.debug(u'Bad triggers detected: {0}', bad_triggers_occ) - return len(bad_triggers_occ) < 2 - - def slugify(self, text): - """Normalize a string and remove non-alphanumeric characters. - """ - text = re.sub(r"[-'_\s]", '_', text) - text = re.sub(r"_+", '_', text).strip('_') - pat = "([^,\(]*)\((.*?)\)" # Remove content within parentheses - text = re.sub(pat, '\g<1>', text).strip() - try: - text = unicodedata.normalize('NFKD', text).encode('ascii', - 'ignore') - text = unicode(re.sub('[-\s]+', ' ', text)) - except UnicodeDecodeError: - self._log.exception(u"Failing to normalize '{0}'", text) - return text - - BY_TRANS = ['by', 'par', 'de', 'von'] - LYRICS_TRANS = ['lyrics', 'paroles', 'letras', 'liedtexte'] - - def is_page_candidate(self, url_link, url_title, title, artist): - """Return True if the URL title makes it a good candidate to be a - page that contains lyrics of title by artist. - """ - title = self.slugify(title.lower()) - artist = self.slugify(artist.lower()) - sitename = re.search(u"//([^/]+)/.*", - self.slugify(url_link.lower())).group(1) - url_title = self.slugify(url_title.lower()) - - # Check if URL title contains song title (exact match) - if url_title.find(title) != -1: - return True - - # or try extracting song title from URL title and check if - # they are close enough - tokens = [by + '_' + artist for by in self.BY_TRANS] + \ - [artist, sitename, sitename.replace('www.', '')] + \ - self.LYRICS_TRANS - tokens = [re.escape(t) for t in tokens] - song_title = re.sub(u'(%s)' % u'|'.join(tokens), u'', url_title) - - song_title = song_title.strip('_|') - typo_ratio = .9 - ratio = difflib.SequenceMatcher(None, song_title, title).ratio() - return ratio >= typo_ratio - - def fetch(self, artist, title): - query = u"%s %s" % (artist, title) - url = u'https://www.googleapis.com/customsearch/v1?key=%s&cx=%s&q=%s' \ - % (self.api_key, self.engine_id, - urllib.quote(query.encode('utf8'))) - - data = urllib.urlopen(url) - data = json.load(data) - if 'error' in data: - reason = data['error']['errors'][0]['reason'] - self._log.debug(u'google lyrics backend error: {0}', reason) - return - - if 'items' in data.keys(): - for item in data['items']: - url_link = item['link'] - url_title = item.get('title', u'') - if not self.is_page_candidate(url_link, url_title, - title, artist): - continue - html = self.fetch_url(url_link) - lyrics = scrape_lyrics_from_html(html) - if not lyrics: - continue - - if self.is_lyrics(lyrics, artist): - self._log.debug(u'got lyrics from {0}', - item['displayLink']) - return lyrics - - -class LyricsPlugin(plugins.BeetsPlugin): - SOURCES = ['google', 'lyricwiki', 'lyrics.com', 'musixmatch'] - SOURCE_BACKENDS = { - 'google': Google, - 'lyricwiki': LyricsWiki, - 'lyrics.com': LyricsCom, - 'musixmatch': MusiXmatch, - 'genius': Genius, - } - - def __init__(self): - super(LyricsPlugin, self).__init__() - self.import_stages = [self.imported] - self.config.add({ - 'auto': True, - 'bing_client_secret': None, - 'bing_lang_from': [], - 'bing_lang_to': None, - 'google_API_key': None, - 'google_engine_ID': u'009217259823014548361:lndtuqkycfu', - 'genius_api_key': - "Ryq93pUGm8bM6eUWwD_M3NOFFDAtp2yEE7W" - "76V-uFL5jks5dNvcGCdarqFjDhP9c", - 'fallback': None, - 'force': False, - 'sources': self.SOURCES, - }) - self.config['bing_client_secret'].redact = True - self.config['google_API_key'].redact = True - self.config['google_engine_ID'].redact = True - self.config['genius_api_key'].redact = True - - available_sources = list(self.SOURCES) - sources = plugins.sanitize_choices( - self.config['sources'].as_str_seq(), available_sources) - - if 'google' in sources: - if not self.config['google_API_key'].get(): - self._log.warn(u'To use the google lyrics source, you must ' - u'provide an API key in the configuration. ' - u'See the documentation for further details.') - sources.remove('google') - if not HAS_BEAUTIFUL_SOUP: - self._log.warn(u'To use the google lyrics source, you must ' - u'install the beautifulsoup4 module. See the ' - u'documentation for further details.') - sources.remove('google') - - self.config['bing_lang_from'] = [ - x.lower() for x in self.config['bing_lang_from'].as_str_seq()] - self.bing_auth_token = None - - if not HAS_LANGDETECT and self.config['bing_client_secret'].get(): - self._log.warn(u'To use bing translations, you need to ' - u'install the langdetect module. See the ' - u'documentation for further details.') - - self.backends = [self.SOURCE_BACKENDS[source](self.config, self._log) - for source in sources] - - def get_bing_access_token(self): - params = { - 'client_id': 'beets', - 'client_secret': self.config['bing_client_secret'], - 'scope': 'http://api.microsofttranslator.com', - 'grant_type': 'client_credentials', - } - - oauth_url = 'https://datamarket.accesscontrol.windows.net/v2/OAuth2-13' - oauth_token = json.loads(requests.post( - oauth_url, - data=urllib.urlencode(params)).content) - if 'access_token' in oauth_token: - return "Bearer " + oauth_token['access_token'] - else: - self._log.warning(u'Could not get Bing Translate API access token.' - u' Check your "bing_client_secret" password') - - def commands(self): - cmd = ui.Subcommand('lyrics', help='fetch song lyrics') - cmd.parser.add_option( - u'-p', u'--print', dest='printlyr', - action='store_true', default=False, - help=u'print lyrics to console', - ) - cmd.parser.add_option( - u'-f', u'--force', dest='force_refetch', - action='store_true', default=False, - help=u'always re-download lyrics', - ) - - def func(lib, opts, args): - # The "write to files" option corresponds to the - # import_write config value. - write = ui.should_write() - for item in lib.items(ui.decargs(args)): - self.fetch_item_lyrics( - lib, item, write, - opts.force_refetch or self.config['force'], - ) - if opts.printlyr and item.lyrics: - ui.print_(item.lyrics) - - cmd.func = func - return [cmd] - - def imported(self, session, task): - """Import hook for fetching lyrics automatically. - """ - if self.config['auto']: - for item in task.imported_items(): - self.fetch_item_lyrics(session.lib, item, - False, self.config['force']) - - def fetch_item_lyrics(self, lib, item, write, force): - """Fetch and store lyrics for a single item. If ``write``, then the - lyrics will also be written to the file itself.""" - # Skip if the item already has lyrics. - if not force and item.lyrics: - self._log.info(u'lyrics already present: {0}', item) - return - - lyrics = None - for artist, titles in search_pairs(item): - lyrics = [self.get_lyrics(artist, title) for title in titles] - if any(lyrics): - break - - lyrics = u"\n\n---\n\n".join([l for l in lyrics if l]) - - if lyrics: - self._log.info(u'fetched lyrics: {0}', item) - if HAS_LANGDETECT and self.config['bing_client_secret'].get(): - lang_from = langdetect.detect(lyrics) - if self.config['bing_lang_to'].get() != lang_from and ( - not self.config['bing_lang_from'] or ( - lang_from in self.config[ - 'bing_lang_from'].as_str_seq())): - lyrics = self.append_translation( - lyrics, self.config['bing_lang_to']) - else: - self._log.info(u'lyrics not found: {0}', item) - fallback = self.config['fallback'].get() - if fallback: - lyrics = fallback - else: - return - item.lyrics = lyrics - if write: - item.try_write() - item.store() - - def get_lyrics(self, artist, title): - """Fetch lyrics, trying each source in turn. Return a string or - None if no lyrics were found. - """ - for backend in self.backends: - lyrics = backend.fetch(artist, title) - if lyrics: - self._log.debug(u'got lyrics from backend: {0}', - backend.__class__.__name__) - return _scrape_strip_cruft(lyrics, True) - - def append_translation(self, text, to_lang): - import xml.etree.ElementTree as ET - - if not self.bing_auth_token: - self.bing_auth_token = self.get_bing_access_token() - if self.bing_auth_token: - # Extract unique lines to limit API request size per song - text_lines = set(text.split('\n')) - url = ('http://api.microsofttranslator.com/v2/Http.svc/' - 'Translate?text=%s&to=%s' % ('|'.join(text_lines), to_lang)) - r = requests.get(url, - headers={"Authorization ": self.bing_auth_token}) - if r.status_code != 200: - self._log.debug('translation API error {}: {}', r.status_code, - r.text) - if 'token has expired' in r.text: - self.bing_auth_token = None - return self.append_translation(text, to_lang) - return text - lines_translated = ET.fromstring(r.text.encode('utf8')).text - # Use a translation mapping dict to build resulting lyrics - translations = dict(zip(text_lines, lines_translated.split('|'))) - result = '' - for line in text.split('\n'): - result += '%s / %s\n' % (line, translations[line]) - return result diff --git a/libs/beetsplug/mbcollection.py b/libs/beetsplug/mbcollection.py deleted file mode 100644 index b95ba6fed..000000000 --- a/libs/beetsplug/mbcollection.py +++ /dev/null @@ -1,114 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2011, Jeffrey Aylesworth -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import division, absolute_import, print_function - -from beets.plugins import BeetsPlugin -from beets.ui import Subcommand -from beets import ui -from beets import config -import musicbrainzngs - -import re - -SUBMISSION_CHUNK_SIZE = 200 -UUID_REGEX = r'^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$' - - -def mb_call(func, *args, **kwargs): - """Call a MusicBrainz API function and catch exceptions. - """ - try: - return func(*args, **kwargs) - except musicbrainzngs.AuthenticationError: - raise ui.UserError(u'authentication with MusicBrainz failed') - except (musicbrainzngs.ResponseError, musicbrainzngs.NetworkError) as exc: - raise ui.UserError(u'MusicBrainz API error: {0}'.format(exc)) - except musicbrainzngs.UsageError: - raise ui.UserError(u'MusicBrainz credentials missing') - - -def submit_albums(collection_id, release_ids): - """Add all of the release IDs to the indicated collection. Multiple - requests are made if there are many release IDs to submit. - """ - for i in range(0, len(release_ids), SUBMISSION_CHUNK_SIZE): - chunk = release_ids[i:i + SUBMISSION_CHUNK_SIZE] - mb_call( - musicbrainzngs.add_releases_to_collection, - collection_id, chunk - ) - - -class MusicBrainzCollectionPlugin(BeetsPlugin): - def __init__(self): - super(MusicBrainzCollectionPlugin, self).__init__() - config['musicbrainz']['pass'].redact = True - musicbrainzngs.auth( - config['musicbrainz']['user'].get(unicode), - config['musicbrainz']['pass'].get(unicode), - ) - self.config.add({'auto': False}) - if self.config['auto']: - self.import_stages = [self.imported] - - def commands(self): - mbupdate = Subcommand('mbupdate', - help=u'Update MusicBrainz collection') - mbupdate.func = self.update_collection - return [mbupdate] - - def update_collection(self, lib, opts, args): - self.update_album_list(lib.albums()) - - def imported(self, session, task): - """Add each imported album to the collection. - """ - if task.is_album: - self.update_album_list([task.album]) - - def update_album_list(self, album_list): - """Update the MusicBrainz colleciton from a list of Beets albums - """ - # Get the available collections. - collections = mb_call(musicbrainzngs.get_collections) - if not collections['collection-list']: - raise ui.UserError(u'no collections exist for user') - - # Get the first release collection. MusicBrainz also has event - # collections, so we need to avoid adding to those. - for collection in collections['collection-list']: - if 'release-count' in collection: - collection_id = collection['id'] - break - else: - raise ui.UserError(u'No collection found.') - - # Get a list of all the album IDs. - album_ids = [] - for album in album_list: - aid = album.mb_albumid - if aid: - if re.match(UUID_REGEX, aid): - album_ids.append(aid) - else: - self._log.info(u'skipping invalid MBID: {0}', aid) - - # Submit to MusicBrainz. - self._log.info( - u'Updating MusicBrainz collection {0}...', collection_id - ) - submit_albums(collection_id, album_ids) - self._log.info(u'...MusicBrainz collection updated.') diff --git a/libs/beetsplug/metasync/__init__.py b/libs/beetsplug/metasync/__init__.py deleted file mode 100644 index 3fc0be4cc..000000000 --- a/libs/beetsplug/metasync/__init__.py +++ /dev/null @@ -1,142 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Heinz Wiesinger. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Synchronize information from music player libraries -""" - -from __future__ import division, absolute_import, print_function - -from abc import abstractmethod, ABCMeta -from importlib import import_module - -from beets.util.confit import ConfigValueError -from beets import ui -from beets.plugins import BeetsPlugin - - -METASYNC_MODULE = 'beetsplug.metasync' - -# Dictionary to map the MODULE and the CLASS NAME of meta sources -SOURCES = { - 'amarok': 'Amarok', - 'itunes': 'Itunes', -} - - -class MetaSource(object): - __metaclass__ = ABCMeta - - def __init__(self, config, log): - self.item_types = {} - self.config = config - self._log = log - - @abstractmethod - def sync_from_source(self, item): - pass - - -def load_meta_sources(): - """ Returns a dictionary of all the MetaSources - E.g., {'itunes': Itunes} with isinstance(Itunes, MetaSource) true - """ - meta_sources = {} - - for module_path, class_name in SOURCES.items(): - module = import_module(METASYNC_MODULE + '.' + module_path) - meta_sources[class_name.lower()] = getattr(module, class_name) - - return meta_sources - - -META_SOURCES = load_meta_sources() - - -def load_item_types(): - """ Returns a dictionary containing the item_types of all the MetaSources - """ - item_types = {} - for meta_source in META_SOURCES.values(): - item_types.update(meta_source.item_types) - return item_types - - -class MetaSyncPlugin(BeetsPlugin): - - item_types = load_item_types() - - def __init__(self): - super(MetaSyncPlugin, self).__init__() - - def commands(self): - cmd = ui.Subcommand('metasync', - help='update metadata from music player libraries') - cmd.parser.add_option('-p', '--pretend', action='store_true', - help='show all changes but do nothing') - cmd.parser.add_option('-s', '--source', default=[], - action='append', dest='sources', - help='comma-separated list of sources to sync') - cmd.parser.add_format_option() - cmd.func = self.func - return [cmd] - - def func(self, lib, opts, args): - """Command handler for the metasync function. - """ - pretend = opts.pretend - query = ui.decargs(args) - - sources = [] - for source in opts.sources: - sources.extend(source.split(',')) - - sources = sources or self.config['source'].as_str_seq() - - meta_source_instances = {} - items = lib.items(query) - - # Avoid needlessly instantiating meta sources (can be expensive) - if not items: - self._log.info(u'No items found matching query') - return - - # Instantiate the meta sources - for player in sources: - try: - cls = META_SOURCES[player] - except KeyError: - self._log.error(u'Unknown metadata source \'{0}\''.format( - player)) - - try: - meta_source_instances[player] = cls(self.config, self._log) - except (ImportError, ConfigValueError) as e: - self._log.error(u'Failed to instantiate metadata source ' - u'\'{0}\': {1}'.format(player, e)) - - # Avoid needlessly iterating over items - if not meta_source_instances: - self._log.error(u'No valid metadata sources found') - return - - # Sync the items with all of the meta sources - for item in items: - for meta_source in meta_source_instances.values(): - meta_source.sync_from_source(item) - - changed = ui.show_model_changes(item) - - if changed and not pretend: - item.store() diff --git a/libs/beetsplug/missing.py b/libs/beetsplug/missing.py deleted file mode 100644 index 8fff659fe..000000000 --- a/libs/beetsplug/missing.py +++ /dev/null @@ -1,146 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Pedro Silva. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""List missing tracks. -""" -from __future__ import division, absolute_import, print_function - -from beets.autotag import hooks -from beets.library import Item -from beets.plugins import BeetsPlugin -from beets.ui import decargs, print_, Subcommand -from beets import config - - -def _missing_count(album): - """Return number of missing items in `album`. - """ - return (album.albumtotal or 0) - len(album.items()) - - -def _item(track_info, album_info, album_id): - """Build and return `item` from `track_info` and `album info` - objects. `item` is missing what fields cannot be obtained from - MusicBrainz alone (encoder, rg_track_gain, rg_track_peak, - rg_album_gain, rg_album_peak, original_year, original_month, - original_day, length, bitrate, format, samplerate, bitdepth, - channels, mtime.) - """ - t = track_info - a = album_info - - return Item(**{ - 'album_id': album_id, - 'album': a.album, - 'albumartist': a.artist, - 'albumartist_credit': a.artist_credit, - 'albumartist_sort': a.artist_sort, - 'albumdisambig': a.albumdisambig, - 'albumstatus': a.albumstatus, - 'albumtype': a.albumtype, - 'artist': t.artist, - 'artist_credit': t.artist_credit, - 'artist_sort': t.artist_sort, - 'asin': a.asin, - 'catalognum': a.catalognum, - 'comp': a.va, - 'country': a.country, - 'day': a.day, - 'disc': t.medium, - 'disctitle': t.disctitle, - 'disctotal': a.mediums, - 'label': a.label, - 'language': a.language, - 'length': t.length, - 'mb_albumid': a.album_id, - 'mb_artistid': t.artist_id, - 'mb_releasegroupid': a.releasegroup_id, - 'mb_trackid': t.track_id, - 'media': t.media, - 'month': a.month, - 'script': a.script, - 'title': t.title, - 'track': t.index, - 'tracktotal': len(a.tracks), - 'year': a.year, - }) - - -class MissingPlugin(BeetsPlugin): - """List missing tracks - """ - def __init__(self): - super(MissingPlugin, self).__init__() - - self.config.add({ - 'count': False, - 'total': False, - }) - - self.album_template_fields['missing'] = _missing_count - - self._command = Subcommand('missing', - help=__doc__, - aliases=['miss']) - self._command.parser.add_option( - u'-c', u'--count', dest='count', action='store_true', - help=u'count missing tracks per album') - self._command.parser.add_option( - u'-t', u'--total', dest='total', action='store_true', - help=u'count total of missing tracks') - self._command.parser.add_format_option() - - def commands(self): - def _miss(lib, opts, args): - self.config.set_args(opts) - count = self.config['count'].get() - total = self.config['total'].get() - fmt = config['format_album' if count else 'format_item'].get() - - albums = lib.albums(decargs(args)) - if total: - print(sum([_missing_count(a) for a in albums])) - return - - # Default format string for count mode. - if count: - fmt += ': $missing' - - for album in albums: - if count: - if _missing_count(album): - print_(format(album, fmt)) - - else: - for item in self._missing(album): - print_(format(item, fmt)) - - self._command.func = _miss - return [self._command] - - def _missing(self, album): - """Query MusicBrainz to determine items missing from `album`. - """ - item_mbids = map(lambda x: x.mb_trackid, album.items()) - if len([i for i in album.items()]) < album.albumtotal: - # fetch missing items - # TODO: Implement caching that without breaking other stuff - album_info = hooks.album_for_mbid(album.mb_albumid) - for track_info in getattr(album_info, 'tracks', []): - if track_info.track_id not in item_mbids: - item = _item(track_info, album_info, album.id) - self._log.debug(u'track {0} in album {1}', - track_info.track_id, album_info.album_id) - yield item diff --git a/libs/beetsplug/permissions.py b/libs/beetsplug/permissions.py deleted file mode 100644 index 0de8978c0..000000000 --- a/libs/beetsplug/permissions.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import division, absolute_import, print_function - -"""Fixes file permissions after the file gets written on import. Put something -like the following in your config.yaml to configure: - - permissions: - file: 644 - dir: 755 -""" -import os -from beets import config, util -from beets.plugins import BeetsPlugin -from beets.util import ancestry - - -def convert_perm(perm): - """If the perm is a int it will first convert it to a string and back - to an oct int. Else it just converts it to oct. - """ - if isinstance(perm, int): - return int(bytes(perm), 8) - else: - return int(perm, 8) - - -def check_permissions(path, permission): - """Checks the permissions of a path. - """ - return oct(os.stat(path).st_mode & 0o777) == oct(permission) - - -def dirs_in_library(library, item): - """Creates a list of ancestor directories in the beets library path. - """ - return [ancestor - for ancestor in ancestry(item) - if ancestor.startswith(library)][1:] - - -class Permissions(BeetsPlugin): - def __init__(self): - super(Permissions, self).__init__() - - # Adding defaults. - self.config.add({ - u'file': 644, - u'dir': 755 - }) - - self.register_listener('item_imported', permissions) - self.register_listener('album_imported', permissions) - - -def permissions(lib, item=None, album=None): - """Running the permission fixer. - """ - # Getting the config. - file_perm = config['permissions']['file'].get() - dir_perm = config['permissions']['dir'].get() - - # Converts permissions to oct. - file_perm = convert_perm(file_perm) - dir_perm = convert_perm(dir_perm) - - # Create chmod_queue. - file_chmod_queue = [] - if item: - file_chmod_queue.append(item.path) - elif album: - for album_item in album.items(): - file_chmod_queue.append(album_item.path) - - # A set of directories to change permissions for. - dir_chmod_queue = set() - - for path in file_chmod_queue: - # Changing permissions on the destination file. - os.chmod(util.bytestring_path(path), file_perm) - - # Checks if the destination path has the permissions configured. - if not check_permissions(util.bytestring_path(path), file_perm): - message = u'There was a problem setting permission on {}'.format( - path) - print(message) - - # Adding directories to the directory chmod queue. - dir_chmod_queue.update( - dirs_in_library(lib.directory, - path)) - - # Change permissions for the directories. - for path in dir_chmod_queue: - # Chaning permissions on the destination directory. - os.chmod(util.bytestring_path(path), dir_perm) - - # Checks if the destination path has the permissions configured. - if not check_permissions(util.bytestring_path(path), dir_perm): - message = u'There was a problem setting permission on {}'.format( - path) - print(message) diff --git a/libs/beetsplug/play.py b/libs/beetsplug/play.py deleted file mode 100644 index fa70f2bc0..000000000 --- a/libs/beetsplug/play.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, David Hamp-Gonsalves -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Send the results of a query to the configured music player as a playlist. -""" -from __future__ import division, absolute_import, print_function - -from beets.plugins import BeetsPlugin -from beets.ui import Subcommand -from beets import config -from beets import ui -from beets import util -from os.path import relpath -from tempfile import NamedTemporaryFile - -# Indicate where arguments should be inserted into the command string. -# If this is missing, they're placed at the end. -ARGS_MARKER = '$args' - - -class PlayPlugin(BeetsPlugin): - - def __init__(self): - super(PlayPlugin, self).__init__() - - config['play'].add({ - 'command': None, - 'use_folders': False, - 'relative_to': None, - 'raw': False, - # Backwards compatibility. See #1803 and line 74 - 'warning_threshold': -2, - 'warning_treshold': 100, - }) - - def commands(self): - play_command = Subcommand( - 'play', - help=u'send music to a player as a playlist' - ) - play_command.parser.add_album_option() - play_command.parser.add_option( - u'-A', u'--args', - action='store', - help=u'add additional arguments to the command', - ) - play_command.func = self.play_music - return [play_command] - - def play_music(self, lib, opts, args): - """Execute query, create temporary playlist and execute player - command passing that playlist, at request insert optional arguments. - """ - command_str = config['play']['command'].get() - if not command_str: - command_str = util.open_anything() - use_folders = config['play']['use_folders'].get(bool) - relative_to = config['play']['relative_to'].get() - raw = config['play']['raw'].get(bool) - warning_threshold = config['play']['warning_threshold'].get(int) - # We use -2 as a default value for warning_threshold to detect if it is - # set or not. We can't use a falsey value because it would have an - # actual meaning in the configuration of this plugin, and we do not use - # -1 because some people might use it as a value to obtain no warning, - # which wouldn't be that bad of a practice. - if warning_threshold == -2: - # if warning_threshold has not been set by user, look for - # warning_treshold, to preserve backwards compatibility. See #1803. - # warning_treshold has the correct default value of 100. - warning_threshold = config['play']['warning_treshold'].get(int) - - if relative_to: - relative_to = util.normpath(relative_to) - - # Add optional arguments to the player command. - if opts.args: - if ARGS_MARKER in command_str: - command_str = command_str.replace(ARGS_MARKER, opts.args) - else: - command_str = u"{} {}".format(command_str, opts.args) - - # Perform search by album and add folders rather than tracks to - # playlist. - if opts.album: - selection = lib.albums(ui.decargs(args)) - paths = [] - - sort = lib.get_default_album_sort() - for album in selection: - if use_folders: - paths.append(album.item_dir()) - else: - paths.extend(item.path - for item in sort.sort(album.items())) - item_type = 'album' - - # Perform item query and add tracks to playlist. - else: - selection = lib.items(ui.decargs(args)) - paths = [item.path for item in selection] - if relative_to: - paths = [relpath(path, relative_to) for path in paths] - item_type = 'track' - - item_type += 's' if len(selection) > 1 else '' - - if not selection: - ui.print_(ui.colorize('text_warning', - u'No {0} to play.'.format(item_type))) - return - - # Warn user before playing any huge playlists. - if warning_threshold and len(selection) > warning_threshold: - ui.print_(ui.colorize( - 'text_warning', - u'You are about to queue {0} {1}.'.format( - len(selection), item_type))) - - if ui.input_options(('Continue', 'Abort')) == 'a': - return - - ui.print_(u'Playing {0} {1}.'.format(len(selection), item_type)) - if raw: - open_args = paths - else: - open_args = [self._create_tmp_playlist(paths)] - - self._log.debug(u'executing command: {} {}', command_str, - b' '.join(open_args)) - try: - util.interactive_open(open_args, command_str) - except OSError as exc: - raise ui.UserError( - "Could not play the query: {0}".format(exc)) - - def _create_tmp_playlist(self, paths_list): - """Create a temporary .m3u file. Return the filename. - """ - m3u = NamedTemporaryFile('w', suffix='.m3u', delete=False) - for item in paths_list: - m3u.write(item + b'\n') - m3u.close() - return m3u.name diff --git a/libs/beetsplug/random.py b/libs/beetsplug/random.py deleted file mode 100644 index e1c6fea4a..000000000 --- a/libs/beetsplug/random.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Philippe Mongeau. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Get a random song or album from the library. -""" -from __future__ import division, absolute_import, print_function - -from beets.plugins import BeetsPlugin -from beets.ui import Subcommand, decargs, print_ -import random -from operator import attrgetter -from itertools import groupby - - -def random_item(lib, opts, args): - query = decargs(args) - - if opts.album: - objs = list(lib.albums(query)) - else: - objs = list(lib.items(query)) - - if opts.equal_chance: - # Group the objects by artist so we can sample from them. - key = attrgetter('albumartist') - objs.sort(key=key) - objs_by_artists = {} - for artist, v in groupby(objs, key): - objs_by_artists[artist] = list(v) - - objs = [] - for _ in range(opts.number): - # Terminate early if we're out of objects to select. - if not objs_by_artists: - break - - # Choose an artist and an object for that artist, removing - # this choice from the pool. - artist = random.choice(objs_by_artists.keys()) - objs_from_artist = objs_by_artists[artist] - i = random.randint(0, len(objs_from_artist) - 1) - objs.append(objs_from_artist.pop(i)) - - # Remove the artist if we've used up all of its objects. - if not objs_from_artist: - del objs_by_artists[artist] - - else: - number = min(len(objs), opts.number) - objs = random.sample(objs, number) - - for item in objs: - print_(format(item)) - -random_cmd = Subcommand('random', - help=u'chose a random track or album') -random_cmd.parser.add_option( - u'-n', u'--number', action='store', type="int", - help=u'number of objects to choose', default=1) -random_cmd.parser.add_option( - u'-e', u'--equal-chance', action='store_true', - help=u'each artist has the same chance') -random_cmd.parser.add_all_common_options() -random_cmd.func = random_item - - -class Random(BeetsPlugin): - def commands(self): - return [random_cmd] diff --git a/libs/beetsplug/web/__init__.py b/libs/beetsplug/web/__init__.py deleted file mode 100644 index 67d99db67..000000000 --- a/libs/beetsplug/web/__init__.py +++ /dev/null @@ -1,328 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""A Web interface to beets.""" -from __future__ import division, absolute_import, print_function - -from beets.plugins import BeetsPlugin -from beets import ui -from beets import util -import beets.library -import flask -from flask import g -from werkzeug.routing import BaseConverter, PathConverter -import os -import json - - -# Utilities. - -def _rep(obj, expand=False): - """Get a flat -- i.e., JSON-ish -- representation of a beets Item or - Album object. For Albums, `expand` dictates whether tracks are - included. - """ - out = dict(obj) - - if isinstance(obj, beets.library.Item): - del out['path'] - - # Get the size (in bytes) of the backing file. This is useful - # for the Tomahawk resolver API. - try: - out['size'] = os.path.getsize(util.syspath(obj.path)) - except OSError: - out['size'] = 0 - - return out - - elif isinstance(obj, beets.library.Album): - del out['artpath'] - if expand: - out['items'] = [_rep(item) for item in obj.items()] - return out - - -def json_generator(items, root): - """Generator that dumps list of beets Items or Albums as JSON - - :param root: root key for JSON - :param items: list of :class:`Item` or :class:`Album` to dump - :returns: generator that yields strings - """ - yield '{"%s":[' % root - first = True - for item in items: - if first: - first = False - else: - yield ',' - yield json.dumps(_rep(item)) - yield ']}' - - -def resource(name): - """Decorates a function to handle RESTful HTTP requests for a resource. - """ - def make_responder(retriever): - def responder(ids): - entities = [retriever(id) for id in ids] - entities = [entity for entity in entities if entity] - - if len(entities) == 1: - return flask.jsonify(_rep(entities[0])) - elif entities: - return app.response_class( - json_generator(entities, root=name), - mimetype='application/json' - ) - else: - return flask.abort(404) - responder.__name__ = 'get_{0}'.format(name) - return responder - return make_responder - - -def resource_query(name): - """Decorates a function to handle RESTful HTTP queries for resources. - """ - def make_responder(query_func): - def responder(queries): - return app.response_class( - json_generator(query_func(queries), root='results'), - mimetype='application/json' - ) - responder.__name__ = 'query_{0}'.format(name) - return responder - return make_responder - - -def resource_list(name): - """Decorates a function to handle RESTful HTTP request for a list of - resources. - """ - def make_responder(list_all): - def responder(): - return app.response_class( - json_generator(list_all(), root=name), - mimetype='application/json' - ) - responder.__name__ = 'all_{0}'.format(name) - return responder - return make_responder - - -def _get_unique_table_field_values(model, field, sort_field): - """ retrieve all unique values belonging to a key from a model """ - if field not in model.all_keys() or sort_field not in model.all_keys(): - raise KeyError - with g.lib.transaction() as tx: - rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' - .format(field, model._table, sort_field)) - return [row[0] for row in rows] - - -class IdListConverter(BaseConverter): - """Converts comma separated lists of ids in urls to integer lists. - """ - - def to_python(self, value): - ids = [] - for id in value.split(','): - try: - ids.append(int(id)) - except ValueError: - pass - return ids - - def to_url(self, value): - return ','.join(value) - - -class QueryConverter(PathConverter): - """Converts slash separated lists of queries in the url to string list. - """ - - def to_python(self, value): - return value.split('/') - - def to_url(self, value): - return ','.join(value) - - -# Flask setup. - -app = flask.Flask(__name__) -app.url_map.converters['idlist'] = IdListConverter -app.url_map.converters['query'] = QueryConverter - - -@app.before_request -def before_request(): - g.lib = app.config['lib'] - - -# Items. - -@app.route('/item/') -@resource('items') -def get_item(id): - return g.lib.get_item(id) - - -@app.route('/item/') -@app.route('/item/query/') -@resource_list('items') -def all_items(): - return g.lib.items() - - -@app.route('/item//file') -def item_file(item_id): - item = g.lib.get_item(item_id) - response = flask.send_file(item.path, as_attachment=True, - attachment_filename=os.path.basename(item.path)) - response.headers['Content-Length'] = os.path.getsize(item.path) - return response - - -@app.route('/item/query/') -@resource_query('items') -def item_query(queries): - return g.lib.items(queries) - - -@app.route('/item/values/') -def item_unique_field_values(key): - sort_key = flask.request.args.get('sort_key', key) - try: - values = _get_unique_table_field_values(beets.library.Item, key, - sort_key) - except KeyError: - return flask.abort(404) - return flask.jsonify(values=values) - - -# Albums. - -@app.route('/album/') -@resource('albums') -def get_album(id): - return g.lib.get_album(id) - - -@app.route('/album/') -@app.route('/album/query/') -@resource_list('albums') -def all_albums(): - return g.lib.albums() - - -@app.route('/album/query/') -@resource_query('albums') -def album_query(queries): - return g.lib.albums(queries) - - -@app.route('/album//art') -def album_art(album_id): - album = g.lib.get_album(album_id) - if album.artpath: - return flask.send_file(album.artpath) - else: - return flask.abort(404) - - -@app.route('/album/values/') -def album_unique_field_values(key): - sort_key = flask.request.args.get('sort_key', key) - try: - values = _get_unique_table_field_values(beets.library.Album, key, - sort_key) - except KeyError: - return flask.abort(404) - return flask.jsonify(values=values) - - -# Artists. - -@app.route('/artist/') -def all_artists(): - with g.lib.transaction() as tx: - rows = tx.query("SELECT DISTINCT albumartist FROM albums") - all_artists = [row[0] for row in rows] - return flask.jsonify(artist_names=all_artists) - - -# Library information. - -@app.route('/stats') -def stats(): - with g.lib.transaction() as tx: - item_rows = tx.query("SELECT COUNT(*) FROM items") - album_rows = tx.query("SELECT COUNT(*) FROM albums") - return flask.jsonify({ - 'items': item_rows[0][0], - 'albums': album_rows[0][0], - }) - - -# UI. - -@app.route('/') -def home(): - return flask.render_template('index.html') - - -# Plugin hook. - -class WebPlugin(BeetsPlugin): - def __init__(self): - super(WebPlugin, self).__init__() - self.config.add({ - 'host': u'127.0.0.1', - 'port': 8337, - 'cors': '', - }) - - def commands(self): - cmd = ui.Subcommand('web', help=u'start a Web interface') - cmd.parser.add_option(u'-d', u'--debug', action='store_true', - default=False, help=u'debug mode') - - def func(lib, opts, args): - args = ui.decargs(args) - if args: - self.config['host'] = args.pop(0) - if args: - self.config['port'] = int(args.pop(0)) - - app.config['lib'] = lib - # Enable CORS if required. - if self.config['cors']: - self._log.info(u'Enabling CORS with origin: {0}', - self.config['cors']) - from flask.ext.cors import CORS - app.config['CORS_ALLOW_HEADERS'] = "Content-Type" - app.config['CORS_RESOURCES'] = { - r"/*": {"origins": self.config['cors'].get(str)} - } - CORS(app) - # Start the web application. - app.run(host=self.config['host'].get(unicode), - port=self.config['port'].get(int), - debug=opts.debug, threaded=True) - cmd.func = func - return [cmd] diff --git a/libs/beetsplug/zero.py b/libs/beetsplug/zero.py deleted file mode 100644 index d20f76166..000000000 --- a/libs/beetsplug/zero.py +++ /dev/null @@ -1,140 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Blemjhoo Tezoulbr . -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -""" Clears tag fields in media files.""" - -from __future__ import division, absolute_import, print_function - -import re -from beets.plugins import BeetsPlugin -from beets.mediafile import MediaFile -from beets.importer import action -from beets.util import confit - -__author__ = 'baobab@heresiarch.info' -__version__ = '0.10' - - -class ZeroPlugin(BeetsPlugin): - - _instance = None - - def __init__(self): - super(ZeroPlugin, self).__init__() - - # Listeners. - self.register_listener('write', self.write_event) - self.register_listener('import_task_choice', - self.import_task_choice_event) - - self.config.add({ - 'fields': [], - 'keep_fields': [], - 'update_database': False, - }) - - self.patterns = {} - self.warned = False - - # We'll only handle `fields` or `keep_fields`, but not both. - if self.config['fields'] and self.config['keep_fields']: - self._log.warn(u'cannot blacklist and whitelist at the same time') - - # Blacklist mode. - if self.config['fields']: - self.validate_config('fields') - for field in self.config['fields'].as_str_seq(): - self.set_pattern(field) - - # Whitelist mode. - elif self.config['keep_fields']: - self.validate_config('keep_fields') - - for field in MediaFile.fields(): - if field in self.config['keep_fields'].as_str_seq(): - continue - self.set_pattern(field) - - # These fields should always be preserved. - for key in ('id', 'path', 'album_id'): - if key in self.patterns: - del self.patterns[key] - - def validate_config(self, mode): - """Check whether fields in the configuration are valid. - - `mode` should either be "fields" or "keep_fields", indicating - the section of the configuration to validate. - """ - for field in self.config[mode].as_str_seq(): - if field not in MediaFile.fields(): - self._log.error(u'invalid field: {0}', field) - continue - if mode == 'fields' and field in ('id', 'path', 'album_id'): - self._log.warn(u'field \'{0}\' ignored, zeroing ' - u'it would be dangerous', field) - continue - - def set_pattern(self, field): - """Set a field in `self.patterns` to a string list corresponding to - the configuration, or `True` if the field has no specific - configuration. - """ - try: - self.patterns[field] = self.config[field].as_str_seq() - except confit.NotFoundError: - # Matches everything - self.patterns[field] = True - - def import_task_choice_event(self, session, task): - """Listen for import_task_choice event.""" - if task.choice_flag == action.ASIS and not self.warned: - self._log.warn(u'cannot zero in \"as-is\" mode') - self.warned = True - # TODO request write in as-is mode - - @classmethod - def match_patterns(cls, field, patterns): - """Check if field (as string) is matching any of the patterns in - the list. - """ - if patterns is True: - return True - for p in patterns: - if re.search(p, unicode(field), flags=re.IGNORECASE): - return True - return False - - def write_event(self, item, path, tags): - """Set values in tags to `None` if the key and value are matched - by `self.patterns`. - """ - if not self.patterns: - self._log.warn(u'no fields, nothing to do') - return - - for field, patterns in self.patterns.items(): - if field in tags: - value = tags[field] - match = self.match_patterns(tags[field], patterns) - else: - value = '' - match = patterns is True - - if match: - self._log.debug(u'{0}: {1} -> None', field, value) - tags[field] = None - if self.config['update_database']: - item[field] = None diff --git a/libs/bs4/__init__.py b/libs/bs4/__init__.py deleted file mode 100644 index aa818ae4d..000000000 --- a/libs/bs4/__init__.py +++ /dev/null @@ -1,529 +0,0 @@ -"""Beautiful Soup -Elixir and Tonic -"The Screen-Scraper's Friend" -http://www.crummy.com/software/BeautifulSoup/ - -Beautiful Soup uses a pluggable XML or HTML parser to parse a -(possibly invalid) document into a tree representation. Beautiful Soup -provides methods and Pythonic idioms that make it easy to navigate, -search, and modify the parse tree. - -Beautiful Soup works with Python 2.7 and up. It works better if lxml -and/or html5lib is installed. - -For more than you ever wanted to know about Beautiful Soup, see the -documentation: -http://www.crummy.com/software/BeautifulSoup/bs4/doc/ - -""" - -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -__author__ = "Leonard Richardson (leonardr@segfault.org)" -__version__ = "4.5.1" -__copyright__ = "Copyright (c) 2004-2016 Leonard Richardson" -__license__ = "MIT" - -__all__ = ['BeautifulSoup'] - -import os -import re -import traceback -import warnings - -from .builder import builder_registry, ParserRejectedMarkup -from .dammit import UnicodeDammit -from .element import ( - CData, - Comment, - DEFAULT_OUTPUT_ENCODING, - Declaration, - Doctype, - NavigableString, - PageElement, - ProcessingInstruction, - ResultSet, - SoupStrainer, - Tag, - ) - -# The very first thing we do is give a useful error if someone is -# running this code under Python 3 without converting it. -'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'<>'You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).' - -class BeautifulSoup(Tag): - """ - This class defines the basic interface called by the tree builders. - - These methods will be called by the parser: - reset() - feed(markup) - - The tree builder may call these methods from its feed() implementation: - handle_starttag(name, attrs) # See note about return value - handle_endtag(name) - handle_data(data) # Appends to the current data node - endData(containerClass=NavigableString) # Ends the current data node - - No matter how complicated the underlying parser is, you should be - able to build a tree using 'start tag' events, 'end tag' events, - 'data' events, and "done with data" events. - - If you encounter an empty-element tag (aka a self-closing tag, - like HTML's
tag), call handle_starttag and then - handle_endtag. - """ - ROOT_TAG_NAME = u'[document]' - - # If the end-user gives no indication which tree builder they - # want, look for one with these features. - DEFAULT_BUILDER_FEATURES = ['html', 'fast'] - - ASCII_SPACES = '\x20\x0a\x09\x0c\x0d' - - NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, change code that looks like this:\n\n BeautifulSoup([your markup])\n\nto this:\n\n BeautifulSoup([your markup], \"%(parser)s\")\n" - - def __init__(self, markup="", features=None, builder=None, - parse_only=None, from_encoding=None, exclude_encodings=None, - **kwargs): - """The Soup object is initialized as the 'root tag', and the - provided markup (which can be a string or a file-like object) - is fed into the underlying parser.""" - - if 'convertEntities' in kwargs: - warnings.warn( - "BS4 does not respect the convertEntities argument to the " - "BeautifulSoup constructor. Entities are always converted " - "to Unicode characters.") - - if 'markupMassage' in kwargs: - del kwargs['markupMassage'] - warnings.warn( - "BS4 does not respect the markupMassage argument to the " - "BeautifulSoup constructor. The tree builder is responsible " - "for any necessary markup massage.") - - if 'smartQuotesTo' in kwargs: - del kwargs['smartQuotesTo'] - warnings.warn( - "BS4 does not respect the smartQuotesTo argument to the " - "BeautifulSoup constructor. Smart quotes are always converted " - "to Unicode characters.") - - if 'selfClosingTags' in kwargs: - del kwargs['selfClosingTags'] - warnings.warn( - "BS4 does not respect the selfClosingTags argument to the " - "BeautifulSoup constructor. The tree builder is responsible " - "for understanding self-closing tags.") - - if 'isHTML' in kwargs: - del kwargs['isHTML'] - warnings.warn( - "BS4 does not respect the isHTML argument to the " - "BeautifulSoup constructor. Suggest you use " - "features='lxml' for HTML and features='lxml-xml' for " - "XML.") - - def deprecated_argument(old_name, new_name): - if old_name in kwargs: - warnings.warn( - 'The "%s" argument to the BeautifulSoup constructor ' - 'has been renamed to "%s."' % (old_name, new_name)) - value = kwargs[old_name] - del kwargs[old_name] - return value - return None - - parse_only = parse_only or deprecated_argument( - "parseOnlyThese", "parse_only") - - from_encoding = from_encoding or deprecated_argument( - "fromEncoding", "from_encoding") - - if from_encoding and isinstance(markup, unicode): - warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.") - from_encoding = None - - if len(kwargs) > 0: - arg = kwargs.keys().pop() - raise TypeError( - "__init__() got an unexpected keyword argument '%s'" % arg) - - if builder is None: - original_features = features - if isinstance(features, basestring): - features = [features] - if features is None or len(features) == 0: - features = self.DEFAULT_BUILDER_FEATURES - builder_class = builder_registry.lookup(*features) - if builder_class is None: - raise FeatureNotFound( - "Couldn't find a tree builder with the features you " - "requested: %s. Do you need to install a parser library?" - % ",".join(features)) - builder = builder_class() - if not (original_features == builder.NAME or - original_features in builder.ALTERNATE_NAMES): - if builder.is_xml: - markup_type = "XML" - else: - markup_type = "HTML" - - caller = traceback.extract_stack()[0] - filename = caller[0] - line_number = caller[1] - warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % dict( - filename=filename, - line_number=line_number, - parser=builder.NAME, - markup_type=markup_type)) - - self.builder = builder - self.is_xml = builder.is_xml - self.known_xml = self.is_xml - self.builder.soup = self - - self.parse_only = parse_only - - if hasattr(markup, 'read'): # It's a file-type object. - markup = markup.read() - elif len(markup) <= 256 and ( - (isinstance(markup, bytes) and not b'<' in markup) - or (isinstance(markup, unicode) and not u'<' in markup) - ): - # Print out warnings for a couple beginner problems - # involving passing non-markup to Beautiful Soup. - # Beautiful Soup will still parse the input as markup, - # just in case that's what the user really wants. - if (isinstance(markup, unicode) - and not os.path.supports_unicode_filenames): - possible_filename = markup.encode("utf8") - else: - possible_filename = markup - is_file = False - try: - is_file = os.path.exists(possible_filename) - except Exception, e: - # This is almost certainly a problem involving - # characters not valid in filenames on this - # system. Just let it go. - pass - if is_file: - if isinstance(markup, unicode): - markup = markup.encode("utf8") - warnings.warn( - '"%s" looks like a filename, not markup. You should' - 'probably open this file and pass the filehandle into' - 'Beautiful Soup.' % markup) - self._check_markup_is_url(markup) - - for (self.markup, self.original_encoding, self.declared_html_encoding, - self.contains_replacement_characters) in ( - self.builder.prepare_markup( - markup, from_encoding, exclude_encodings=exclude_encodings)): - self.reset() - try: - self._feed() - break - except ParserRejectedMarkup: - pass - - # Clear out the markup and remove the builder's circular - # reference to this object. - self.markup = None - self.builder.soup = None - - def __copy__(self): - copy = type(self)( - self.encode('utf-8'), builder=self.builder, from_encoding='utf-8' - ) - - # Although we encoded the tree to UTF-8, that may not have - # been the encoding of the original markup. Set the copy's - # .original_encoding to reflect the original object's - # .original_encoding. - copy.original_encoding = self.original_encoding - return copy - - def __getstate__(self): - # Frequently a tree builder can't be pickled. - d = dict(self.__dict__) - if 'builder' in d and not self.builder.picklable: - d['builder'] = None - return d - - @staticmethod - def _check_markup_is_url(markup): - """ - Check if markup looks like it's actually a url and raise a warning - if so. Markup can be unicode or str (py2) / bytes (py3). - """ - if isinstance(markup, bytes): - space = b' ' - cant_start_with = (b"http:", b"https:") - elif isinstance(markup, unicode): - space = u' ' - cant_start_with = (u"http:", u"https:") - else: - return - - if any(markup.startswith(prefix) for prefix in cant_start_with): - if not space in markup: - if isinstance(markup, bytes): - decoded_markup = markup.decode('utf-8', 'replace') - else: - decoded_markup = markup - warnings.warn( - '"%s" looks like a URL. Beautiful Soup is not an' - ' HTTP client. You should probably use an HTTP client like' - ' requests to get the document behind the URL, and feed' - ' that document to Beautiful Soup.' % decoded_markup - ) - - def _feed(self): - # Convert the document to Unicode. - self.builder.reset() - - self.builder.feed(self.markup) - # Close out any unfinished strings and close all the open tags. - self.endData() - while self.currentTag.name != self.ROOT_TAG_NAME: - self.popTag() - - def reset(self): - Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME) - self.hidden = 1 - self.builder.reset() - self.current_data = [] - self.currentTag = None - self.tagStack = [] - self.preserve_whitespace_tag_stack = [] - self.pushTag(self) - - def new_tag(self, name, namespace=None, nsprefix=None, **attrs): - """Create a new tag associated with this soup.""" - return Tag(None, self.builder, name, namespace, nsprefix, attrs) - - def new_string(self, s, subclass=NavigableString): - """Create a new NavigableString associated with this soup.""" - return subclass(s) - - def insert_before(self, successor): - raise NotImplementedError("BeautifulSoup objects don't support insert_before().") - - def insert_after(self, successor): - raise NotImplementedError("BeautifulSoup objects don't support insert_after().") - - def popTag(self): - tag = self.tagStack.pop() - if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]: - self.preserve_whitespace_tag_stack.pop() - #print "Pop", tag.name - if self.tagStack: - self.currentTag = self.tagStack[-1] - return self.currentTag - - def pushTag(self, tag): - #print "Push", tag.name - if self.currentTag: - self.currentTag.contents.append(tag) - self.tagStack.append(tag) - self.currentTag = self.tagStack[-1] - if tag.name in self.builder.preserve_whitespace_tags: - self.preserve_whitespace_tag_stack.append(tag) - - def endData(self, containerClass=NavigableString): - if self.current_data: - current_data = u''.join(self.current_data) - # If whitespace is not preserved, and this string contains - # nothing but ASCII spaces, replace it with a single space - # or newline. - if not self.preserve_whitespace_tag_stack: - strippable = True - for i in current_data: - if i not in self.ASCII_SPACES: - strippable = False - break - if strippable: - if '\n' in current_data: - current_data = '\n' - else: - current_data = ' ' - - # Reset the data collector. - self.current_data = [] - - # Should we add this string to the tree at all? - if self.parse_only and len(self.tagStack) <= 1 and \ - (not self.parse_only.text or \ - not self.parse_only.search(current_data)): - return - - o = containerClass(current_data) - self.object_was_parsed(o) - - def object_was_parsed(self, o, parent=None, most_recent_element=None): - """Add an object to the parse tree.""" - parent = parent or self.currentTag - previous_element = most_recent_element or self._most_recent_element - - next_element = previous_sibling = next_sibling = None - if isinstance(o, Tag): - next_element = o.next_element - next_sibling = o.next_sibling - previous_sibling = o.previous_sibling - if not previous_element: - previous_element = o.previous_element - - o.setup(parent, previous_element, next_element, previous_sibling, next_sibling) - - self._most_recent_element = o - parent.contents.append(o) - - if parent.next_sibling: - # This node is being inserted into an element that has - # already been parsed. Deal with any dangling references. - index = len(parent.contents)-1 - while index >= 0: - if parent.contents[index] is o: - break - index -= 1 - else: - raise ValueError( - "Error building tree: supposedly %r was inserted " - "into %r after the fact, but I don't see it!" % ( - o, parent - ) - ) - if index == 0: - previous_element = parent - previous_sibling = None - else: - previous_element = previous_sibling = parent.contents[index-1] - if index == len(parent.contents)-1: - next_element = parent.next_sibling - next_sibling = None - else: - next_element = next_sibling = parent.contents[index+1] - - o.previous_element = previous_element - if previous_element: - previous_element.next_element = o - o.next_element = next_element - if next_element: - next_element.previous_element = o - o.next_sibling = next_sibling - if next_sibling: - next_sibling.previous_sibling = o - o.previous_sibling = previous_sibling - if previous_sibling: - previous_sibling.next_sibling = o - - def _popToTag(self, name, nsprefix=None, inclusivePop=True): - """Pops the tag stack up to and including the most recent - instance of the given tag. If inclusivePop is false, pops the tag - stack up to but *not* including the most recent instqance of - the given tag.""" - #print "Popping to %s" % name - if name == self.ROOT_TAG_NAME: - # The BeautifulSoup object itself can never be popped. - return - - most_recently_popped = None - - stack_size = len(self.tagStack) - for i in range(stack_size - 1, 0, -1): - t = self.tagStack[i] - if (name == t.name and nsprefix == t.prefix): - if inclusivePop: - most_recently_popped = self.popTag() - break - most_recently_popped = self.popTag() - - return most_recently_popped - - def handle_starttag(self, name, namespace, nsprefix, attrs): - """Push a start tag on to the stack. - - If this method returns None, the tag was rejected by the - SoupStrainer. You should proceed as if the tag had not occurred - in the document. For instance, if this was a self-closing tag, - don't call handle_endtag. - """ - - # print "Start tag %s: %s" % (name, attrs) - self.endData() - - if (self.parse_only and len(self.tagStack) <= 1 - and (self.parse_only.text - or not self.parse_only.search_tag(name, attrs))): - return None - - tag = Tag(self, self.builder, name, namespace, nsprefix, attrs, - self.currentTag, self._most_recent_element) - if tag is None: - return tag - if self._most_recent_element: - self._most_recent_element.next_element = tag - self._most_recent_element = tag - self.pushTag(tag) - return tag - - def handle_endtag(self, name, nsprefix=None): - #print "End tag: " + name - self.endData() - self._popToTag(name, nsprefix) - - def handle_data(self, data): - self.current_data.append(data) - - def decode(self, pretty_print=False, - eventual_encoding=DEFAULT_OUTPUT_ENCODING, - formatter="minimal"): - """Returns a string or Unicode representation of this document. - To get Unicode, pass None for encoding.""" - - if self.is_xml: - # Print the XML declaration - encoding_part = '' - if eventual_encoding != None: - encoding_part = ' encoding="%s"' % eventual_encoding - prefix = u'\n' % encoding_part - else: - prefix = u'' - if not pretty_print: - indent_level = None - else: - indent_level = 0 - return prefix + super(BeautifulSoup, self).decode( - indent_level, eventual_encoding, formatter) - -# Alias to make it easier to type import: 'from bs4 import _soup' -_s = BeautifulSoup -_soup = BeautifulSoup - -class BeautifulStoneSoup(BeautifulSoup): - """Deprecated interface to an XML parser.""" - - def __init__(self, *args, **kwargs): - kwargs['features'] = 'xml' - warnings.warn( - 'The BeautifulStoneSoup class is deprecated. Instead of using ' - 'it, pass features="xml" into the BeautifulSoup constructor.') - super(BeautifulStoneSoup, self).__init__(*args, **kwargs) - - -class StopParsing(Exception): - pass - -class FeatureNotFound(ValueError): - pass - - -#By default, act as an HTML pretty-printer. -if __name__ == '__main__': - import sys - soup = BeautifulSoup(sys.stdin) - print soup.prettify() diff --git a/libs/bs4/builder/__init__.py b/libs/bs4/builder/__init__.py deleted file mode 100644 index 601979bf4..000000000 --- a/libs/bs4/builder/__init__.py +++ /dev/null @@ -1,328 +0,0 @@ -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -from collections import defaultdict -import itertools -import sys -from bs4.element import ( - CharsetMetaAttributeValue, - ContentMetaAttributeValue, - HTMLAwareEntitySubstitution, - whitespace_re - ) - -__all__ = [ - 'HTMLTreeBuilder', - 'SAXTreeBuilder', - 'TreeBuilder', - 'TreeBuilderRegistry', - ] - -# Some useful features for a TreeBuilder to have. -FAST = 'fast' -PERMISSIVE = 'permissive' -STRICT = 'strict' -XML = 'xml' -HTML = 'html' -HTML_5 = 'html5' - - -class TreeBuilderRegistry(object): - - def __init__(self): - self.builders_for_feature = defaultdict(list) - self.builders = [] - - def register(self, treebuilder_class): - """Register a treebuilder based on its advertised features.""" - for feature in treebuilder_class.features: - self.builders_for_feature[feature].insert(0, treebuilder_class) - self.builders.insert(0, treebuilder_class) - - def lookup(self, *features): - if len(self.builders) == 0: - # There are no builders at all. - return None - - if len(features) == 0: - # They didn't ask for any features. Give them the most - # recently registered builder. - return self.builders[0] - - # Go down the list of features in order, and eliminate any builders - # that don't match every feature. - features = list(features) - features.reverse() - candidates = None - candidate_set = None - while len(features) > 0: - feature = features.pop() - we_have_the_feature = self.builders_for_feature.get(feature, []) - if len(we_have_the_feature) > 0: - if candidates is None: - candidates = we_have_the_feature - candidate_set = set(candidates) - else: - # Eliminate any candidates that don't have this feature. - candidate_set = candidate_set.intersection( - set(we_have_the_feature)) - - # The only valid candidates are the ones in candidate_set. - # Go through the original list of candidates and pick the first one - # that's in candidate_set. - if candidate_set is None: - return None - for candidate in candidates: - if candidate in candidate_set: - return candidate - return None - -# The BeautifulSoup class will take feature lists from developers and use them -# to look up builders in this registry. -builder_registry = TreeBuilderRegistry() - -class TreeBuilder(object): - """Turn a document into a Beautiful Soup object tree.""" - - NAME = "[Unknown tree builder]" - ALTERNATE_NAMES = [] - features = [] - - is_xml = False - picklable = False - preserve_whitespace_tags = set() - empty_element_tags = None # A tag will be considered an empty-element - # tag when and only when it has no contents. - - # A value for these tag/attribute combinations is a space- or - # comma-separated list of CDATA, rather than a single CDATA. - cdata_list_attributes = {} - - - def __init__(self): - self.soup = None - - def reset(self): - pass - - def can_be_empty_element(self, tag_name): - """Might a tag with this name be an empty-element tag? - - The final markup may or may not actually present this tag as - self-closing. - - For instance: an HTMLBuilder does not consider a

tag to be - an empty-element tag (it's not in - HTMLBuilder.empty_element_tags). This means an empty

tag - will be presented as "

", not "

". - - The default implementation has no opinion about which tags are - empty-element tags, so a tag will be presented as an - empty-element tag if and only if it has no contents. - "" will become "", and "bar" will - be left alone. - """ - if self.empty_element_tags is None: - return True - return tag_name in self.empty_element_tags - - def feed(self, markup): - raise NotImplementedError() - - def prepare_markup(self, markup, user_specified_encoding=None, - document_declared_encoding=None): - return markup, None, None, False - - def test_fragment_to_document(self, fragment): - """Wrap an HTML fragment to make it look like a document. - - Different parsers do this differently. For instance, lxml - introduces an empty tag, and html5lib - doesn't. Abstracting this away lets us write simple tests - which run HTML fragments through the parser and compare the - results against other HTML fragments. - - This method should not be used outside of tests. - """ - return fragment - - def set_up_substitutions(self, tag): - return False - - def _replace_cdata_list_attribute_values(self, tag_name, attrs): - """Replaces class="foo bar" with class=["foo", "bar"] - - Modifies its input in place. - """ - if not attrs: - return attrs - if self.cdata_list_attributes: - universal = self.cdata_list_attributes.get('*', []) - tag_specific = self.cdata_list_attributes.get( - tag_name.lower(), None) - for attr in attrs.keys(): - if attr in universal or (tag_specific and attr in tag_specific): - # We have a "class"-type attribute whose string - # value is a whitespace-separated list of - # values. Split it into a list. - value = attrs[attr] - if isinstance(value, basestring): - values = whitespace_re.split(value) - else: - # html5lib sometimes calls setAttributes twice - # for the same tag when rearranging the parse - # tree. On the second call the attribute value - # here is already a list. If this happens, - # leave the value alone rather than trying to - # split it again. - values = value - attrs[attr] = values - return attrs - -class SAXTreeBuilder(TreeBuilder): - """A Beautiful Soup treebuilder that listens for SAX events.""" - - def feed(self, markup): - raise NotImplementedError() - - def close(self): - pass - - def startElement(self, name, attrs): - attrs = dict((key[1], value) for key, value in list(attrs.items())) - #print "Start %s, %r" % (name, attrs) - self.soup.handle_starttag(name, attrs) - - def endElement(self, name): - #print "End %s" % name - self.soup.handle_endtag(name) - - def startElementNS(self, nsTuple, nodeName, attrs): - # Throw away (ns, nodeName) for now. - self.startElement(nodeName, attrs) - - def endElementNS(self, nsTuple, nodeName): - # Throw away (ns, nodeName) for now. - self.endElement(nodeName) - #handler.endElementNS((ns, node.nodeName), node.nodeName) - - def startPrefixMapping(self, prefix, nodeValue): - # Ignore the prefix for now. - pass - - def endPrefixMapping(self, prefix): - # Ignore the prefix for now. - # handler.endPrefixMapping(prefix) - pass - - def characters(self, content): - self.soup.handle_data(content) - - def startDocument(self): - pass - - def endDocument(self): - pass - - -class HTMLTreeBuilder(TreeBuilder): - """This TreeBuilder knows facts about HTML. - - Such as which tags are empty-element tags. - """ - - preserve_whitespace_tags = HTMLAwareEntitySubstitution.preserve_whitespace_tags - empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base']) - - # The HTML standard defines these attributes as containing a - # space-separated list of values, not a single value. That is, - # class="foo bar" means that the 'class' attribute has two values, - # 'foo' and 'bar', not the single value 'foo bar'. When we - # encounter one of these attributes, we will parse its value into - # a list of values if possible. Upon output, the list will be - # converted back into a string. - cdata_list_attributes = { - "*" : ['class', 'accesskey', 'dropzone'], - "a" : ['rel', 'rev'], - "link" : ['rel', 'rev'], - "td" : ["headers"], - "th" : ["headers"], - "td" : ["headers"], - "form" : ["accept-charset"], - "object" : ["archive"], - - # These are HTML5 specific, as are *.accesskey and *.dropzone above. - "area" : ["rel"], - "icon" : ["sizes"], - "iframe" : ["sandbox"], - "output" : ["for"], - } - - def set_up_substitutions(self, tag): - # We are only interested in tags - if tag.name != 'meta': - return False - - http_equiv = tag.get('http-equiv') - content = tag.get('content') - charset = tag.get('charset') - - # We are interested in tags that say what encoding the - # document was originally in. This means HTML 5-style - # tags that provide the "charset" attribute. It also means - # HTML 4-style tags that provide the "content" - # attribute and have "http-equiv" set to "content-type". - # - # In both cases we will replace the value of the appropriate - # attribute with a standin object that can take on any - # encoding. - meta_encoding = None - if charset is not None: - # HTML 5 style: - # - meta_encoding = charset - tag['charset'] = CharsetMetaAttributeValue(charset) - - elif (content is not None and http_equiv is not None - and http_equiv.lower() == 'content-type'): - # HTML 4 style: - # - tag['content'] = ContentMetaAttributeValue(content) - - return (meta_encoding is not None) - -def register_treebuilders_from(module): - """Copy TreeBuilders from the given module into this module.""" - # I'm fairly sure this is not the best way to do this. - this_module = sys.modules['bs4.builder'] - for name in module.__all__: - obj = getattr(module, name) - - if issubclass(obj, TreeBuilder): - setattr(this_module, name, obj) - this_module.__all__.append(name) - # Register the builder while we're at it. - this_module.builder_registry.register(obj) - -class ParserRejectedMarkup(Exception): - pass - -# Builders are registered in reverse order of priority, so that custom -# builder registrations will take precedence. In general, we want lxml -# to take precedence over html5lib, because it's faster. And we only -# want to use HTMLParser as a last result. -from . import _htmlparser -register_treebuilders_from(_htmlparser) -try: - from . import _html5lib - register_treebuilders_from(_html5lib) -except ImportError: - # They don't have html5lib installed. - pass -try: - from . import _lxml - register_treebuilders_from(_lxml) -except ImportError: - # They don't have lxml installed. - pass diff --git a/libs/bs4/builder/_html5lib.py b/libs/bs4/builder/_html5lib.py deleted file mode 100644 index c46f88232..000000000 --- a/libs/bs4/builder/_html5lib.py +++ /dev/null @@ -1,356 +0,0 @@ -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -__all__ = [ - 'HTML5TreeBuilder', - ] - -import warnings -from bs4.builder import ( - PERMISSIVE, - HTML, - HTML_5, - HTMLTreeBuilder, - ) -from bs4.element import ( - NamespacedAttribute, - whitespace_re, -) -import html5lib -from html5lib.constants import namespaces -from bs4.element import ( - Comment, - Doctype, - NavigableString, - Tag, - ) - -try: - # Pre-0.99999999 - from html5lib.treebuilders import _base as treebuilder_base - new_html5lib = False -except ImportError, e: - # 0.99999999 and up - from html5lib.treebuilders import base as treebuilder_base - new_html5lib = True - -class HTML5TreeBuilder(HTMLTreeBuilder): - """Use html5lib to build a tree.""" - - NAME = "html5lib" - - features = [NAME, PERMISSIVE, HTML_5, HTML] - - def prepare_markup(self, markup, user_specified_encoding, - document_declared_encoding=None, exclude_encodings=None): - # Store the user-specified encoding for use later on. - self.user_specified_encoding = user_specified_encoding - - # document_declared_encoding and exclude_encodings aren't used - # ATM because the html5lib TreeBuilder doesn't use - # UnicodeDammit. - if exclude_encodings: - warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.") - yield (markup, None, None, False) - - # These methods are defined by Beautiful Soup. - def feed(self, markup): - if self.soup.parse_only is not None: - warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.") - parser = html5lib.HTMLParser(tree=self.create_treebuilder) - - extra_kwargs = dict() - if not isinstance(markup, unicode): - if new_html5lib: - extra_kwargs['override_encoding'] = self.user_specified_encoding - else: - extra_kwargs['encoding'] = self.user_specified_encoding - doc = parser.parse(markup, **extra_kwargs) - - # Set the character encoding detected by the tokenizer. - if isinstance(markup, unicode): - # We need to special-case this because html5lib sets - # charEncoding to UTF-8 if it gets Unicode input. - doc.original_encoding = None - else: - original_encoding = parser.tokenizer.stream.charEncoding[0] - if not isinstance(original_encoding, basestring): - # In 0.99999999 and up, the encoding is an html5lib - # Encoding object. We want to use a string for compatibility - # with other tree builders. - original_encoding = original_encoding.name - doc.original_encoding = original_encoding - - def create_treebuilder(self, namespaceHTMLElements): - self.underlying_builder = TreeBuilderForHtml5lib( - self.soup, namespaceHTMLElements) - return self.underlying_builder - - def test_fragment_to_document(self, fragment): - """See `TreeBuilder`.""" - return u'%s' % fragment - - -class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder): - - def __init__(self, soup, namespaceHTMLElements): - self.soup = soup - super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements) - - def documentClass(self): - self.soup.reset() - return Element(self.soup, self.soup, None) - - def insertDoctype(self, token): - name = token["name"] - publicId = token["publicId"] - systemId = token["systemId"] - - doctype = Doctype.for_name_and_ids(name, publicId, systemId) - self.soup.object_was_parsed(doctype) - - def elementClass(self, name, namespace): - tag = self.soup.new_tag(name, namespace) - return Element(tag, self.soup, namespace) - - def commentClass(self, data): - return TextNode(Comment(data), self.soup) - - def fragmentClass(self): - self.soup = BeautifulSoup("") - self.soup.name = "[document_fragment]" - return Element(self.soup, self.soup, None) - - def appendChild(self, node): - # XXX This code is not covered by the BS4 tests. - self.soup.append(node.element) - - def getDocument(self): - return self.soup - - def getFragment(self): - return treebuilder_base.TreeBuilder.getFragment(self).element - -class AttrList(object): - def __init__(self, element): - self.element = element - self.attrs = dict(self.element.attrs) - def __iter__(self): - return list(self.attrs.items()).__iter__() - def __setitem__(self, name, value): - # If this attribute is a multi-valued attribute for this element, - # turn its value into a list. - list_attr = HTML5TreeBuilder.cdata_list_attributes - if (name in list_attr['*'] - or (self.element.name in list_attr - and name in list_attr[self.element.name])): - # A node that is being cloned may have already undergone - # this procedure. - if not isinstance(value, list): - value = whitespace_re.split(value) - self.element[name] = value - def items(self): - return list(self.attrs.items()) - def keys(self): - return list(self.attrs.keys()) - def __len__(self): - return len(self.attrs) - def __getitem__(self, name): - return self.attrs[name] - def __contains__(self, name): - return name in list(self.attrs.keys()) - - -class Element(treebuilder_base.Node): - def __init__(self, element, soup, namespace): - treebuilder_base.Node.__init__(self, element.name) - self.element = element - self.soup = soup - self.namespace = namespace - - def appendChild(self, node): - string_child = child = None - if isinstance(node, basestring): - # Some other piece of code decided to pass in a string - # instead of creating a TextElement object to contain the - # string. - string_child = child = node - elif isinstance(node, Tag): - # Some other piece of code decided to pass in a Tag - # instead of creating an Element object to contain the - # Tag. - child = node - elif node.element.__class__ == NavigableString: - string_child = child = node.element - else: - child = node.element - - if not isinstance(child, basestring) and child.parent is not None: - node.element.extract() - - if (string_child and self.element.contents - and self.element.contents[-1].__class__ == NavigableString): - # We are appending a string onto another string. - # TODO This has O(n^2) performance, for input like - # "aaa..." - old_element = self.element.contents[-1] - new_element = self.soup.new_string(old_element + string_child) - old_element.replace_with(new_element) - self.soup._most_recent_element = new_element - else: - if isinstance(node, basestring): - # Create a brand new NavigableString from this string. - child = self.soup.new_string(node) - - # Tell Beautiful Soup to act as if it parsed this element - # immediately after the parent's last descendant. (Or - # immediately after the parent, if it has no children.) - if self.element.contents: - most_recent_element = self.element._last_descendant(False) - elif self.element.next_element is not None: - # Something from further ahead in the parse tree is - # being inserted into this earlier element. This is - # very annoying because it means an expensive search - # for the last element in the tree. - most_recent_element = self.soup._last_descendant() - else: - most_recent_element = self.element - - self.soup.object_was_parsed( - child, parent=self.element, - most_recent_element=most_recent_element) - - def getAttributes(self): - return AttrList(self.element) - - def setAttributes(self, attributes): - - if attributes is not None and len(attributes) > 0: - - converted_attributes = [] - for name, value in list(attributes.items()): - if isinstance(name, tuple): - new_name = NamespacedAttribute(*name) - del attributes[name] - attributes[new_name] = value - - self.soup.builder._replace_cdata_list_attribute_values( - self.name, attributes) - for name, value in attributes.items(): - self.element[name] = value - - # The attributes may contain variables that need substitution. - # Call set_up_substitutions manually. - # - # The Tag constructor called this method when the Tag was created, - # but we just set/changed the attributes, so call it again. - self.soup.builder.set_up_substitutions(self.element) - attributes = property(getAttributes, setAttributes) - - def insertText(self, data, insertBefore=None): - if insertBefore: - text = TextNode(self.soup.new_string(data), self.soup) - self.insertBefore(data, insertBefore) - else: - self.appendChild(data) - - def insertBefore(self, node, refNode): - index = self.element.index(refNode.element) - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[index-1].__class__ == NavigableString): - # (See comments in appendChild) - old_node = self.element.contents[index-1] - new_str = self.soup.new_string(old_node + node.element) - old_node.replace_with(new_str) - else: - self.element.insert(index, node.element) - node.parent = self - - def removeChild(self, node): - node.element.extract() - - def reparentChildren(self, new_parent): - """Move all of this tag's children into another tag.""" - # print "MOVE", self.element.contents - # print "FROM", self.element - # print "TO", new_parent.element - element = self.element - new_parent_element = new_parent.element - # Determine what this tag's next_element will be once all the children - # are removed. - final_next_element = element.next_sibling - - new_parents_last_descendant = new_parent_element._last_descendant(False, False) - if len(new_parent_element.contents) > 0: - # The new parent already contains children. We will be - # appending this tag's children to the end. - new_parents_last_child = new_parent_element.contents[-1] - new_parents_last_descendant_next_element = new_parents_last_descendant.next_element - else: - # The new parent contains no children. - new_parents_last_child = None - new_parents_last_descendant_next_element = new_parent_element.next_element - - to_append = element.contents - append_after = new_parent_element.contents - if len(to_append) > 0: - # Set the first child's previous_element and previous_sibling - # to elements within the new parent - first_child = to_append[0] - if new_parents_last_descendant: - first_child.previous_element = new_parents_last_descendant - else: - first_child.previous_element = new_parent_element - first_child.previous_sibling = new_parents_last_child - if new_parents_last_descendant: - new_parents_last_descendant.next_element = first_child - else: - new_parent_element.next_element = first_child - if new_parents_last_child: - new_parents_last_child.next_sibling = first_child - - # Fix the last child's next_element and next_sibling - last_child = to_append[-1] - last_child.next_element = new_parents_last_descendant_next_element - if new_parents_last_descendant_next_element: - new_parents_last_descendant_next_element.previous_element = last_child - last_child.next_sibling = None - - for child in to_append: - child.parent = new_parent_element - new_parent_element.contents.append(child) - - # Now that this element has no children, change its .next_element. - element.contents = [] - element.next_element = final_next_element - - # print "DONE WITH MOVE" - # print "FROM", self.element - # print "TO", new_parent_element - - def cloneNode(self): - tag = self.soup.new_tag(self.element.name, self.namespace) - node = Element(tag, self.soup, self.namespace) - for key,value in self.attributes: - node.attributes[key] = value - return node - - def hasContent(self): - return self.element.contents - - def getNameTuple(self): - if self.namespace == None: - return namespaces["html"], self.name - else: - return self.namespace, self.name - - nameTuple = property(getNameTuple) - -class TextNode(Element): - def __init__(self, element, soup): - treebuilder_base.Node.__init__(self, None) - self.element = element - self.soup = soup - - def cloneNode(self): - raise NotImplementedError diff --git a/libs/bs4/builder/_htmlparser.py b/libs/bs4/builder/_htmlparser.py deleted file mode 100644 index 823ca15aa..000000000 --- a/libs/bs4/builder/_htmlparser.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Use the HTMLParser library to parse HTML files that aren't too bad.""" - -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -__all__ = [ - 'HTMLParserTreeBuilder', - ] - -from HTMLParser import HTMLParser - -try: - from HTMLParser import HTMLParseError -except ImportError, e: - # HTMLParseError is removed in Python 3.5. Since it can never be - # thrown in 3.5, we can just define our own class as a placeholder. - class HTMLParseError(Exception): - pass - -import sys -import warnings - -# Starting in Python 3.2, the HTMLParser constructor takes a 'strict' -# argument, which we'd like to set to False. Unfortunately, -# http://bugs.python.org/issue13273 makes strict=True a better bet -# before Python 3.2.3. -# -# At the end of this file, we monkeypatch HTMLParser so that -# strict=True works well on Python 3.2.2. -major, minor, release = sys.version_info[:3] -CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3 -CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3 -CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4 - - -from bs4.element import ( - CData, - Comment, - Declaration, - Doctype, - ProcessingInstruction, - ) -from bs4.dammit import EntitySubstitution, UnicodeDammit - -from bs4.builder import ( - HTML, - HTMLTreeBuilder, - STRICT, - ) - - -HTMLPARSER = 'html.parser' - -class BeautifulSoupHTMLParser(HTMLParser): - def handle_starttag(self, name, attrs): - # XXX namespace - attr_dict = {} - for key, value in attrs: - # Change None attribute values to the empty string - # for consistency with the other tree builders. - if value is None: - value = '' - attr_dict[key] = value - attrvalue = '""' - self.soup.handle_starttag(name, None, None, attr_dict) - - def handle_endtag(self, name): - self.soup.handle_endtag(name) - - def handle_data(self, data): - self.soup.handle_data(data) - - def handle_charref(self, name): - # XXX workaround for a bug in HTMLParser. Remove this once - # it's fixed in all supported versions. - # http://bugs.python.org/issue13633 - if name.startswith('x'): - real_name = int(name.lstrip('x'), 16) - elif name.startswith('X'): - real_name = int(name.lstrip('X'), 16) - else: - real_name = int(name) - - try: - data = unichr(real_name) - except (ValueError, OverflowError), e: - data = u"\N{REPLACEMENT CHARACTER}" - - self.handle_data(data) - - def handle_entityref(self, name): - character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name) - if character is not None: - data = character - else: - data = "&%s;" % name - self.handle_data(data) - - def handle_comment(self, data): - self.soup.endData() - self.soup.handle_data(data) - self.soup.endData(Comment) - - def handle_decl(self, data): - self.soup.endData() - if data.startswith("DOCTYPE "): - data = data[len("DOCTYPE "):] - elif data == 'DOCTYPE': - # i.e. "" - data = '' - self.soup.handle_data(data) - self.soup.endData(Doctype) - - def unknown_decl(self, data): - if data.upper().startswith('CDATA['): - cls = CData - data = data[len('CDATA['):] - else: - cls = Declaration - self.soup.endData() - self.soup.handle_data(data) - self.soup.endData(cls) - - def handle_pi(self, data): - self.soup.endData() - self.soup.handle_data(data) - self.soup.endData(ProcessingInstruction) - - -class HTMLParserTreeBuilder(HTMLTreeBuilder): - - is_xml = False - picklable = True - NAME = HTMLPARSER - features = [NAME, HTML, STRICT] - - def __init__(self, *args, **kwargs): - if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED: - kwargs['strict'] = False - if CONSTRUCTOR_TAKES_CONVERT_CHARREFS: - kwargs['convert_charrefs'] = False - self.parser_args = (args, kwargs) - - def prepare_markup(self, markup, user_specified_encoding=None, - document_declared_encoding=None, exclude_encodings=None): - """ - :return: A 4-tuple (markup, original encoding, encoding - declared within markup, whether any characters had to be - replaced with REPLACEMENT CHARACTER). - """ - if isinstance(markup, unicode): - yield (markup, None, None, False) - return - - try_encodings = [user_specified_encoding, document_declared_encoding] - dammit = UnicodeDammit(markup, try_encodings, is_html=True, - exclude_encodings=exclude_encodings) - yield (dammit.markup, dammit.original_encoding, - dammit.declared_html_encoding, - dammit.contains_replacement_characters) - - def feed(self, markup): - args, kwargs = self.parser_args - parser = BeautifulSoupHTMLParser(*args, **kwargs) - parser.soup = self.soup - try: - parser.feed(markup) - except HTMLParseError, e: - warnings.warn(RuntimeWarning( - "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help.")) - raise e - -# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some -# 3.2.3 code. This ensures they don't treat markup like

as a -# string. -# -# XXX This code can be removed once most Python 3 users are on 3.2.3. -if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT: - import re - attrfind_tolerant = re.compile( - r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*' - r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?') - HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant - - locatestarttagend = re.compile(r""" - <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name - (?:\s+ # whitespace before attribute name - (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name - (?:\s*=\s* # value indicator - (?:'[^']*' # LITA-enclosed value - |\"[^\"]*\" # LIT-enclosed value - |[^'\">\s]+ # bare value - ) - )? - ) - )* - \s* # trailing whitespace -""", re.VERBOSE) - BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend - - from html.parser import tagfind, attrfind - - def parse_starttag(self, i): - self.__starttag_text = None - endpos = self.check_for_whole_start_tag(i) - if endpos < 0: - return endpos - rawdata = self.rawdata - self.__starttag_text = rawdata[i:endpos] - - # Now parse the data between i+1 and j into a tag and attrs - attrs = [] - match = tagfind.match(rawdata, i+1) - assert match, 'unexpected call to parse_starttag()' - k = match.end() - self.lasttag = tag = rawdata[i+1:k].lower() - while k < endpos: - if self.strict: - m = attrfind.match(rawdata, k) - else: - m = attrfind_tolerant.match(rawdata, k) - if not m: - break - attrname, rest, attrvalue = m.group(1, 2, 3) - if not rest: - attrvalue = None - elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ - attrvalue[:1] == '"' == attrvalue[-1:]: - attrvalue = attrvalue[1:-1] - if attrvalue: - attrvalue = self.unescape(attrvalue) - attrs.append((attrname.lower(), attrvalue)) - k = m.end() - - end = rawdata[k:endpos].strip() - if end not in (">", "/>"): - lineno, offset = self.getpos() - if "\n" in self.__starttag_text: - lineno = lineno + self.__starttag_text.count("\n") - offset = len(self.__starttag_text) \ - - self.__starttag_text.rfind("\n") - else: - offset = offset + len(self.__starttag_text) - if self.strict: - self.error("junk characters in start tag: %r" - % (rawdata[k:endpos][:20],)) - self.handle_data(rawdata[i:endpos]) - return endpos - if end.endswith('/>'): - # XHTML-style empty tag: - self.handle_startendtag(tag, attrs) - else: - self.handle_starttag(tag, attrs) - if tag in self.CDATA_CONTENT_ELEMENTS: - self.set_cdata_mode(tag) - return endpos - - def set_cdata_mode(self, elem): - self.cdata_elem = elem.lower() - self.interesting = re.compile(r'' % self.cdata_elem, re.I) - - BeautifulSoupHTMLParser.parse_starttag = parse_starttag - BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode - - CONSTRUCTOR_TAKES_STRICT = True diff --git a/libs/bs4/builder/_lxml.py b/libs/bs4/builder/_lxml.py deleted file mode 100644 index d2ca2872d..000000000 --- a/libs/bs4/builder/_lxml.py +++ /dev/null @@ -1,258 +0,0 @@ -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -__all__ = [ - 'LXMLTreeBuilderForXML', - 'LXMLTreeBuilder', - ] - -from io import BytesIO -from StringIO import StringIO -import collections -from lxml import etree -from bs4.element import ( - Comment, - Doctype, - NamespacedAttribute, - ProcessingInstruction, - XMLProcessingInstruction, -) -from bs4.builder import ( - FAST, - HTML, - HTMLTreeBuilder, - PERMISSIVE, - ParserRejectedMarkup, - TreeBuilder, - XML) -from bs4.dammit import EncodingDetector - -LXML = 'lxml' - -class LXMLTreeBuilderForXML(TreeBuilder): - DEFAULT_PARSER_CLASS = etree.XMLParser - - is_xml = True - processing_instruction_class = XMLProcessingInstruction - - NAME = "lxml-xml" - ALTERNATE_NAMES = ["xml"] - - # Well, it's permissive by XML parser standards. - features = [NAME, LXML, XML, FAST, PERMISSIVE] - - CHUNK_SIZE = 512 - - # This namespace mapping is specified in the XML Namespace - # standard. - DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"} - - def default_parser(self, encoding): - # This can either return a parser object or a class, which - # will be instantiated with default arguments. - if self._default_parser is not None: - return self._default_parser - return etree.XMLParser( - target=self, strip_cdata=False, recover=True, encoding=encoding) - - def parser_for(self, encoding): - # Use the default parser. - parser = self.default_parser(encoding) - - if isinstance(parser, collections.Callable): - # Instantiate the parser with default arguments - parser = parser(target=self, strip_cdata=False, encoding=encoding) - return parser - - def __init__(self, parser=None, empty_element_tags=None): - # TODO: Issue a warning if parser is present but not a - # callable, since that means there's no way to create new - # parsers for different encodings. - self._default_parser = parser - if empty_element_tags is not None: - self.empty_element_tags = set(empty_element_tags) - self.soup = None - self.nsmaps = [self.DEFAULT_NSMAPS] - - def _getNsTag(self, tag): - # Split the namespace URL out of a fully-qualified lxml tag - # name. Copied from lxml's src/lxml/sax.py. - if tag[0] == '{': - return tuple(tag[1:].split('}', 1)) - else: - return (None, tag) - - def prepare_markup(self, markup, user_specified_encoding=None, - exclude_encodings=None, - document_declared_encoding=None): - """ - :yield: A series of 4-tuples. - (markup, encoding, declared encoding, - has undergone character replacement) - - Each 4-tuple represents a strategy for parsing the document. - """ - # Instead of using UnicodeDammit to convert the bytestring to - # Unicode using different encodings, use EncodingDetector to - # iterate over the encodings, and tell lxml to try to parse - # the document as each one in turn. - is_html = not self.is_xml - if is_html: - self.processing_instruction_class = ProcessingInstruction - else: - self.processing_instruction_class = XMLProcessingInstruction - - if isinstance(markup, unicode): - # We were given Unicode. Maybe lxml can parse Unicode on - # this system? - yield markup, None, document_declared_encoding, False - - if isinstance(markup, unicode): - # No, apparently not. Convert the Unicode to UTF-8 and - # tell lxml to parse it as UTF-8. - yield (markup.encode("utf8"), "utf8", - document_declared_encoding, False) - - try_encodings = [user_specified_encoding, document_declared_encoding] - detector = EncodingDetector( - markup, try_encodings, is_html, exclude_encodings) - for encoding in detector.encodings: - yield (detector.markup, encoding, document_declared_encoding, False) - - def feed(self, markup): - if isinstance(markup, bytes): - markup = BytesIO(markup) - elif isinstance(markup, unicode): - markup = StringIO(markup) - - # Call feed() at least once, even if the markup is empty, - # or the parser won't be initialized. - data = markup.read(self.CHUNK_SIZE) - try: - self.parser = self.parser_for(self.soup.original_encoding) - self.parser.feed(data) - while len(data) != 0: - # Now call feed() on the rest of the data, chunk by chunk. - data = markup.read(self.CHUNK_SIZE) - if len(data) != 0: - self.parser.feed(data) - self.parser.close() - except (UnicodeDecodeError, LookupError, etree.ParserError), e: - raise ParserRejectedMarkup(str(e)) - - def close(self): - self.nsmaps = [self.DEFAULT_NSMAPS] - - def start(self, name, attrs, nsmap={}): - # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy. - attrs = dict(attrs) - nsprefix = None - # Invert each namespace map as it comes in. - if len(self.nsmaps) > 1: - # There are no new namespaces for this tag, but - # non-default namespaces are in play, so we need a - # separate tag stack to know when they end. - self.nsmaps.append(None) - elif len(nsmap) > 0: - # A new namespace mapping has come into play. - inverted_nsmap = dict((value, key) for key, value in nsmap.items()) - self.nsmaps.append(inverted_nsmap) - # Also treat the namespace mapping as a set of attributes on the - # tag, so we can recreate it later. - attrs = attrs.copy() - for prefix, namespace in nsmap.items(): - attribute = NamespacedAttribute( - "xmlns", prefix, "http://www.w3.org/2000/xmlns/") - attrs[attribute] = namespace - - # Namespaces are in play. Find any attributes that came in - # from lxml with namespaces attached to their names, and - # turn then into NamespacedAttribute objects. - new_attrs = {} - for attr, value in attrs.items(): - namespace, attr = self._getNsTag(attr) - if namespace is None: - new_attrs[attr] = value - else: - nsprefix = self._prefix_for_namespace(namespace) - attr = NamespacedAttribute(nsprefix, attr, namespace) - new_attrs[attr] = value - attrs = new_attrs - - namespace, name = self._getNsTag(name) - nsprefix = self._prefix_for_namespace(namespace) - self.soup.handle_starttag(name, namespace, nsprefix, attrs) - - def _prefix_for_namespace(self, namespace): - """Find the currently active prefix for the given namespace.""" - if namespace is None: - return None - for inverted_nsmap in reversed(self.nsmaps): - if inverted_nsmap is not None and namespace in inverted_nsmap: - return inverted_nsmap[namespace] - return None - - def end(self, name): - self.soup.endData() - completed_tag = self.soup.tagStack[-1] - namespace, name = self._getNsTag(name) - nsprefix = None - if namespace is not None: - for inverted_nsmap in reversed(self.nsmaps): - if inverted_nsmap is not None and namespace in inverted_nsmap: - nsprefix = inverted_nsmap[namespace] - break - self.soup.handle_endtag(name, nsprefix) - if len(self.nsmaps) > 1: - # This tag, or one of its parents, introduced a namespace - # mapping, so pop it off the stack. - self.nsmaps.pop() - - def pi(self, target, data): - self.soup.endData() - self.soup.handle_data(target + ' ' + data) - self.soup.endData(self.processing_instruction_class) - - def data(self, content): - self.soup.handle_data(content) - - def doctype(self, name, pubid, system): - self.soup.endData() - doctype = Doctype.for_name_and_ids(name, pubid, system) - self.soup.object_was_parsed(doctype) - - def comment(self, content): - "Handle comments as Comment objects." - self.soup.endData() - self.soup.handle_data(content) - self.soup.endData(Comment) - - def test_fragment_to_document(self, fragment): - """See `TreeBuilder`.""" - return u'\n%s' % fragment - - -class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML): - - NAME = LXML - ALTERNATE_NAMES = ["lxml-html"] - - features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE] - is_xml = False - processing_instruction_class = ProcessingInstruction - - def default_parser(self, encoding): - return etree.HTMLParser - - def feed(self, markup): - encoding = self.soup.original_encoding - try: - self.parser = self.parser_for(encoding) - self.parser.feed(markup) - self.parser.close() - except (UnicodeDecodeError, LookupError, etree.ParserError), e: - raise ParserRejectedMarkup(str(e)) - - - def test_fragment_to_document(self, fragment): - """See `TreeBuilder`.""" - return u'%s' % fragment diff --git a/libs/bs4/dammit.py b/libs/bs4/dammit.py deleted file mode 100644 index 2bf67f7f3..000000000 --- a/libs/bs4/dammit.py +++ /dev/null @@ -1,842 +0,0 @@ -# -*- coding: utf-8 -*- -"""Beautiful Soup bonus library: Unicode, Dammit - -This library converts a bytestream to Unicode through any means -necessary. It is heavily based on code from Mark Pilgrim's Universal -Feed Parser. It works best on XML and HTML, but it does not rewrite the -XML or HTML to reflect a new encoding; that's the tree builder's job. -""" -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -__license__ = "MIT" - -import codecs -from htmlentitydefs import codepoint2name -import re -import logging -import string - -# Import a library to autodetect character encodings. -chardet_type = None -try: - # First try the fast C implementation. - # PyPI package: cchardet - import cchardet - def chardet_dammit(s): - return cchardet.detect(s)['encoding'] -except ImportError: - try: - # Fall back to the pure Python implementation - # Debian package: python-chardet - # PyPI package: chardet - import chardet - def chardet_dammit(s): - return chardet.detect(s)['encoding'] - #import chardet.constants - #chardet.constants._debug = 1 - except ImportError: - # No chardet available. - def chardet_dammit(s): - return None - -# Available from http://cjkpython.i18n.org/. -try: - import iconv_codec -except ImportError: - pass - -xml_encoding_re = re.compile( - '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I) -html_meta_re = re.compile( - '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I) - -class EntitySubstitution(object): - - """Substitute XML or HTML entities for the corresponding characters.""" - - def _populate_class_variables(): - lookup = {} - reverse_lookup = {} - characters_for_re = [] - for codepoint, name in list(codepoint2name.items()): - character = unichr(codepoint) - if codepoint != 34: - # There's no point in turning the quotation mark into - # ", unless it happens within an attribute value, which - # is handled elsewhere. - characters_for_re.append(character) - lookup[character] = name - # But we do want to turn " into the quotation mark. - reverse_lookup[name] = character - re_definition = "[%s]" % "".join(characters_for_re) - return lookup, reverse_lookup, re.compile(re_definition) - (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER, - CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables() - - CHARACTER_TO_XML_ENTITY = { - "'": "apos", - '"': "quot", - "&": "amp", - "<": "lt", - ">": "gt", - } - - BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" - "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" - ")") - - AMPERSAND_OR_BRACKET = re.compile("([<>&])") - - @classmethod - def _substitute_html_entity(cls, matchobj): - entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0)) - return "&%s;" % entity - - @classmethod - def _substitute_xml_entity(cls, matchobj): - """Used with a regular expression to substitute the - appropriate XML entity for an XML special character.""" - entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)] - return "&%s;" % entity - - @classmethod - def quoted_attribute_value(self, value): - """Make a value into a quoted XML attribute, possibly escaping it. - - Most strings will be quoted using double quotes. - - Bob's Bar -> "Bob's Bar" - - If a string contains double quotes, it will be quoted using - single quotes. - - Welcome to "my bar" -> 'Welcome to "my bar"' - - If a string contains both single and double quotes, the - double quotes will be escaped, and the string will be quoted - using double quotes. - - Welcome to "Bob's Bar" -> "Welcome to "Bob's bar" - """ - quote_with = '"' - if '"' in value: - if "'" in value: - # The string contains both single and double - # quotes. Turn the double quotes into - # entities. We quote the double quotes rather than - # the single quotes because the entity name is - # """ whether this is HTML or XML. If we - # quoted the single quotes, we'd have to decide - # between ' and &squot;. - replace_with = """ - value = value.replace('"', replace_with) - else: - # There are double quotes but no single quotes. - # We can use single quotes to quote the attribute. - quote_with = "'" - return quote_with + value + quote_with - - @classmethod - def substitute_xml(cls, value, make_quoted_attribute=False): - """Substitute XML entities for special XML characters. - - :param value: A string to be substituted. The less-than sign - will become <, the greater-than sign will become >, - and any ampersands will become &. If you want ampersands - that appear to be part of an entity definition to be left - alone, use substitute_xml_containing_entities() instead. - - :param make_quoted_attribute: If True, then the string will be - quoted, as befits an attribute value. - """ - # Escape angle brackets and ampersands. - value = cls.AMPERSAND_OR_BRACKET.sub( - cls._substitute_xml_entity, value) - - if make_quoted_attribute: - value = cls.quoted_attribute_value(value) - return value - - @classmethod - def substitute_xml_containing_entities( - cls, value, make_quoted_attribute=False): - """Substitute XML entities for special XML characters. - - :param value: A string to be substituted. The less-than sign will - become <, the greater-than sign will become >, and any - ampersands that are not part of an entity defition will - become &. - - :param make_quoted_attribute: If True, then the string will be - quoted, as befits an attribute value. - """ - # Escape angle brackets, and ampersands that aren't part of - # entities. - value = cls.BARE_AMPERSAND_OR_BRACKET.sub( - cls._substitute_xml_entity, value) - - if make_quoted_attribute: - value = cls.quoted_attribute_value(value) - return value - - @classmethod - def substitute_html(cls, s): - """Replace certain Unicode characters with named HTML entities. - - This differs from data.encode(encoding, 'xmlcharrefreplace') - in that the goal is to make the result more readable (to those - with ASCII displays) rather than to recover from - errors. There's absolutely nothing wrong with a UTF-8 string - containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that - character with "é" will make it more readable to some - people. - """ - return cls.CHARACTER_TO_HTML_ENTITY_RE.sub( - cls._substitute_html_entity, s) - - -class EncodingDetector: - """Suggests a number of possible encodings for a bytestring. - - Order of precedence: - - 1. Encodings you specifically tell EncodingDetector to try first - (the override_encodings argument to the constructor). - - 2. An encoding declared within the bytestring itself, either in an - XML declaration (if the bytestring is to be interpreted as an XML - document), or in a tag (if the bytestring is to be - interpreted as an HTML document.) - - 3. An encoding detected through textual analysis by chardet, - cchardet, or a similar external library. - - 4. UTF-8. - - 5. Windows-1252. - """ - def __init__(self, markup, override_encodings=None, is_html=False, - exclude_encodings=None): - self.override_encodings = override_encodings or [] - exclude_encodings = exclude_encodings or [] - self.exclude_encodings = set([x.lower() for x in exclude_encodings]) - self.chardet_encoding = None - self.is_html = is_html - self.declared_encoding = None - - # First order of business: strip a byte-order mark. - self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup) - - def _usable(self, encoding, tried): - if encoding is not None: - encoding = encoding.lower() - if encoding in self.exclude_encodings: - return False - if encoding not in tried: - tried.add(encoding) - return True - return False - - @property - def encodings(self): - """Yield a number of encodings that might work for this markup.""" - tried = set() - for e in self.override_encodings: - if self._usable(e, tried): - yield e - - # Did the document originally start with a byte-order mark - # that indicated its encoding? - if self._usable(self.sniffed_encoding, tried): - yield self.sniffed_encoding - - # Look within the document for an XML or HTML encoding - # declaration. - if self.declared_encoding is None: - self.declared_encoding = self.find_declared_encoding( - self.markup, self.is_html) - if self._usable(self.declared_encoding, tried): - yield self.declared_encoding - - # Use third-party character set detection to guess at the - # encoding. - if self.chardet_encoding is None: - self.chardet_encoding = chardet_dammit(self.markup) - if self._usable(self.chardet_encoding, tried): - yield self.chardet_encoding - - # As a last-ditch effort, try utf-8 and windows-1252. - for e in ('utf-8', 'windows-1252'): - if self._usable(e, tried): - yield e - - @classmethod - def strip_byte_order_mark(cls, data): - """If a byte-order mark is present, strip it and return the encoding it implies.""" - encoding = None - if isinstance(data, unicode): - # Unicode data cannot have a byte-order mark. - return data, encoding - if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \ - and (data[2:4] != '\x00\x00'): - encoding = 'utf-16be' - data = data[2:] - elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \ - and (data[2:4] != '\x00\x00'): - encoding = 'utf-16le' - data = data[2:] - elif data[:3] == b'\xef\xbb\xbf': - encoding = 'utf-8' - data = data[3:] - elif data[:4] == b'\x00\x00\xfe\xff': - encoding = 'utf-32be' - data = data[4:] - elif data[:4] == b'\xff\xfe\x00\x00': - encoding = 'utf-32le' - data = data[4:] - return data, encoding - - @classmethod - def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False): - """Given a document, tries to find its declared encoding. - - An XML encoding is declared at the beginning of the document. - - An HTML encoding is declared in a tag, hopefully near the - beginning of the document. - """ - if search_entire_document: - xml_endpos = html_endpos = len(markup) - else: - xml_endpos = 1024 - html_endpos = max(2048, int(len(markup) * 0.05)) - - declared_encoding = None - declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos) - if not declared_encoding_match and is_html: - declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos) - if declared_encoding_match is not None: - declared_encoding = declared_encoding_match.groups()[0].decode( - 'ascii', 'replace') - if declared_encoding: - return declared_encoding.lower() - return None - -class UnicodeDammit: - """A class for detecting the encoding of a *ML document and - converting it to a Unicode string. If the source encoding is - windows-1252, can replace MS smart quotes with their HTML or XML - equivalents.""" - - # This dictionary maps commonly seen values for "charset" in HTML - # meta tags to the corresponding Python codec names. It only covers - # values that aren't in Python's aliases and can't be determined - # by the heuristics in find_codec. - CHARSET_ALIASES = {"macintosh": "mac-roman", - "x-sjis": "shift-jis"} - - ENCODINGS_WITH_SMART_QUOTES = [ - "windows-1252", - "iso-8859-1", - "iso-8859-2", - ] - - def __init__(self, markup, override_encodings=[], - smart_quotes_to=None, is_html=False, exclude_encodings=[]): - self.smart_quotes_to = smart_quotes_to - self.tried_encodings = [] - self.contains_replacement_characters = False - self.is_html = is_html - self.log = logging.getLogger(__name__) - self.detector = EncodingDetector( - markup, override_encodings, is_html, exclude_encodings) - - # Short-circuit if the data is in Unicode to begin with. - if isinstance(markup, unicode) or markup == '': - self.markup = markup - self.unicode_markup = unicode(markup) - self.original_encoding = None - return - - # The encoding detector may have stripped a byte-order mark. - # Use the stripped markup from this point on. - self.markup = self.detector.markup - - u = None - for encoding in self.detector.encodings: - markup = self.detector.markup - u = self._convert_from(encoding) - if u is not None: - break - - if not u: - # None of the encodings worked. As an absolute last resort, - # try them again with character replacement. - - for encoding in self.detector.encodings: - if encoding != "ascii": - u = self._convert_from(encoding, "replace") - if u is not None: - self.log.warning( - "Some characters could not be decoded, and were " - "replaced with REPLACEMENT CHARACTER." - ) - self.contains_replacement_characters = True - break - - # If none of that worked, we could at this point force it to - # ASCII, but that would destroy so much data that I think - # giving up is better. - self.unicode_markup = u - if not u: - self.original_encoding = None - - def _sub_ms_char(self, match): - """Changes a MS smart quote character to an XML or HTML - entity, or an ASCII character.""" - orig = match.group(1) - if self.smart_quotes_to == 'ascii': - sub = self.MS_CHARS_TO_ASCII.get(orig).encode() - else: - sub = self.MS_CHARS.get(orig) - if type(sub) == tuple: - if self.smart_quotes_to == 'xml': - sub = '&#x'.encode() + sub[1].encode() + ';'.encode() - else: - sub = '&'.encode() + sub[0].encode() + ';'.encode() - else: - sub = sub.encode() - return sub - - def _convert_from(self, proposed, errors="strict"): - proposed = self.find_codec(proposed) - if not proposed or (proposed, errors) in self.tried_encodings: - return None - self.tried_encodings.append((proposed, errors)) - markup = self.markup - # Convert smart quotes to HTML if coming from an encoding - # that might have them. - if (self.smart_quotes_to is not None - and proposed in self.ENCODINGS_WITH_SMART_QUOTES): - smart_quotes_re = b"([\x80-\x9f])" - smart_quotes_compiled = re.compile(smart_quotes_re) - markup = smart_quotes_compiled.sub(self._sub_ms_char, markup) - - try: - #print "Trying to convert document to %s (errors=%s)" % ( - # proposed, errors) - u = self._to_unicode(markup, proposed, errors) - self.markup = u - self.original_encoding = proposed - except Exception as e: - #print "That didn't work!" - #print e - return None - #print "Correct encoding: %s" % proposed - return self.markup - - def _to_unicode(self, data, encoding, errors="strict"): - '''Given a string and its encoding, decodes the string into Unicode. - %encoding is a string recognized by encodings.aliases''' - return unicode(data, encoding, errors) - - @property - def declared_html_encoding(self): - if not self.is_html: - return None - return self.detector.declared_encoding - - def find_codec(self, charset): - value = (self._codec(self.CHARSET_ALIASES.get(charset, charset)) - or (charset and self._codec(charset.replace("-", ""))) - or (charset and self._codec(charset.replace("-", "_"))) - or (charset and charset.lower()) - or charset - ) - if value: - return value.lower() - return None - - def _codec(self, charset): - if not charset: - return charset - codec = None - try: - codecs.lookup(charset) - codec = charset - except (LookupError, ValueError): - pass - return codec - - - # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities. - MS_CHARS = {b'\x80': ('euro', '20AC'), - b'\x81': ' ', - b'\x82': ('sbquo', '201A'), - b'\x83': ('fnof', '192'), - b'\x84': ('bdquo', '201E'), - b'\x85': ('hellip', '2026'), - b'\x86': ('dagger', '2020'), - b'\x87': ('Dagger', '2021'), - b'\x88': ('circ', '2C6'), - b'\x89': ('permil', '2030'), - b'\x8A': ('Scaron', '160'), - b'\x8B': ('lsaquo', '2039'), - b'\x8C': ('OElig', '152'), - b'\x8D': '?', - b'\x8E': ('#x17D', '17D'), - b'\x8F': '?', - b'\x90': '?', - b'\x91': ('lsquo', '2018'), - b'\x92': ('rsquo', '2019'), - b'\x93': ('ldquo', '201C'), - b'\x94': ('rdquo', '201D'), - b'\x95': ('bull', '2022'), - b'\x96': ('ndash', '2013'), - b'\x97': ('mdash', '2014'), - b'\x98': ('tilde', '2DC'), - b'\x99': ('trade', '2122'), - b'\x9a': ('scaron', '161'), - b'\x9b': ('rsaquo', '203A'), - b'\x9c': ('oelig', '153'), - b'\x9d': '?', - b'\x9e': ('#x17E', '17E'), - b'\x9f': ('Yuml', ''),} - - # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains - # horrors like stripping diacritical marks to turn á into a, but also - # contains non-horrors like turning “ into ". - MS_CHARS_TO_ASCII = { - b'\x80' : 'EUR', - b'\x81' : ' ', - b'\x82' : ',', - b'\x83' : 'f', - b'\x84' : ',,', - b'\x85' : '...', - b'\x86' : '+', - b'\x87' : '++', - b'\x88' : '^', - b'\x89' : '%', - b'\x8a' : 'S', - b'\x8b' : '<', - b'\x8c' : 'OE', - b'\x8d' : '?', - b'\x8e' : 'Z', - b'\x8f' : '?', - b'\x90' : '?', - b'\x91' : "'", - b'\x92' : "'", - b'\x93' : '"', - b'\x94' : '"', - b'\x95' : '*', - b'\x96' : '-', - b'\x97' : '--', - b'\x98' : '~', - b'\x99' : '(TM)', - b'\x9a' : 's', - b'\x9b' : '>', - b'\x9c' : 'oe', - b'\x9d' : '?', - b'\x9e' : 'z', - b'\x9f' : 'Y', - b'\xa0' : ' ', - b'\xa1' : '!', - b'\xa2' : 'c', - b'\xa3' : 'GBP', - b'\xa4' : '$', #This approximation is especially parochial--this is the - #generic currency symbol. - b'\xa5' : 'YEN', - b'\xa6' : '|', - b'\xa7' : 'S', - b'\xa8' : '..', - b'\xa9' : '', - b'\xaa' : '(th)', - b'\xab' : '<<', - b'\xac' : '!', - b'\xad' : ' ', - b'\xae' : '(R)', - b'\xaf' : '-', - b'\xb0' : 'o', - b'\xb1' : '+-', - b'\xb2' : '2', - b'\xb3' : '3', - b'\xb4' : ("'", 'acute'), - b'\xb5' : 'u', - b'\xb6' : 'P', - b'\xb7' : '*', - b'\xb8' : ',', - b'\xb9' : '1', - b'\xba' : '(th)', - b'\xbb' : '>>', - b'\xbc' : '1/4', - b'\xbd' : '1/2', - b'\xbe' : '3/4', - b'\xbf' : '?', - b'\xc0' : 'A', - b'\xc1' : 'A', - b'\xc2' : 'A', - b'\xc3' : 'A', - b'\xc4' : 'A', - b'\xc5' : 'A', - b'\xc6' : 'AE', - b'\xc7' : 'C', - b'\xc8' : 'E', - b'\xc9' : 'E', - b'\xca' : 'E', - b'\xcb' : 'E', - b'\xcc' : 'I', - b'\xcd' : 'I', - b'\xce' : 'I', - b'\xcf' : 'I', - b'\xd0' : 'D', - b'\xd1' : 'N', - b'\xd2' : 'O', - b'\xd3' : 'O', - b'\xd4' : 'O', - b'\xd5' : 'O', - b'\xd6' : 'O', - b'\xd7' : '*', - b'\xd8' : 'O', - b'\xd9' : 'U', - b'\xda' : 'U', - b'\xdb' : 'U', - b'\xdc' : 'U', - b'\xdd' : 'Y', - b'\xde' : 'b', - b'\xdf' : 'B', - b'\xe0' : 'a', - b'\xe1' : 'a', - b'\xe2' : 'a', - b'\xe3' : 'a', - b'\xe4' : 'a', - b'\xe5' : 'a', - b'\xe6' : 'ae', - b'\xe7' : 'c', - b'\xe8' : 'e', - b'\xe9' : 'e', - b'\xea' : 'e', - b'\xeb' : 'e', - b'\xec' : 'i', - b'\xed' : 'i', - b'\xee' : 'i', - b'\xef' : 'i', - b'\xf0' : 'o', - b'\xf1' : 'n', - b'\xf2' : 'o', - b'\xf3' : 'o', - b'\xf4' : 'o', - b'\xf5' : 'o', - b'\xf6' : 'o', - b'\xf7' : '/', - b'\xf8' : 'o', - b'\xf9' : 'u', - b'\xfa' : 'u', - b'\xfb' : 'u', - b'\xfc' : 'u', - b'\xfd' : 'y', - b'\xfe' : 'b', - b'\xff' : 'y', - } - - # A map used when removing rogue Windows-1252/ISO-8859-1 - # characters in otherwise UTF-8 documents. - # - # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in - # Windows-1252. - WINDOWS_1252_TO_UTF8 = { - 0x80 : b'\xe2\x82\xac', # € - 0x82 : b'\xe2\x80\x9a', # ‚ - 0x83 : b'\xc6\x92', # Æ’ - 0x84 : b'\xe2\x80\x9e', # „ - 0x85 : b'\xe2\x80\xa6', # … - 0x86 : b'\xe2\x80\xa0', # † - 0x87 : b'\xe2\x80\xa1', # ‡ - 0x88 : b'\xcb\x86', # ˆ - 0x89 : b'\xe2\x80\xb0', # ‰ - 0x8a : b'\xc5\xa0', # Å  - 0x8b : b'\xe2\x80\xb9', # ‹ - 0x8c : b'\xc5\x92', # Å’ - 0x8e : b'\xc5\xbd', # Ž - 0x91 : b'\xe2\x80\x98', # ‘ - 0x92 : b'\xe2\x80\x99', # ’ - 0x93 : b'\xe2\x80\x9c', # “ - 0x94 : b'\xe2\x80\x9d', # †- 0x95 : b'\xe2\x80\xa2', # • - 0x96 : b'\xe2\x80\x93', # – - 0x97 : b'\xe2\x80\x94', # — - 0x98 : b'\xcb\x9c', # Ëœ - 0x99 : b'\xe2\x84\xa2', # â„¢ - 0x9a : b'\xc5\xa1', # Å¡ - 0x9b : b'\xe2\x80\xba', # › - 0x9c : b'\xc5\x93', # Å“ - 0x9e : b'\xc5\xbe', # ž - 0x9f : b'\xc5\xb8', # Ÿ - 0xa0 : b'\xc2\xa0', #   - 0xa1 : b'\xc2\xa1', # ¡ - 0xa2 : b'\xc2\xa2', # ¢ - 0xa3 : b'\xc2\xa3', # £ - 0xa4 : b'\xc2\xa4', # ¤ - 0xa5 : b'\xc2\xa5', # Â¥ - 0xa6 : b'\xc2\xa6', # ¦ - 0xa7 : b'\xc2\xa7', # § - 0xa8 : b'\xc2\xa8', # ¨ - 0xa9 : b'\xc2\xa9', # © - 0xaa : b'\xc2\xaa', # ª - 0xab : b'\xc2\xab', # « - 0xac : b'\xc2\xac', # ¬ - 0xad : b'\xc2\xad', # ­ - 0xae : b'\xc2\xae', # ® - 0xaf : b'\xc2\xaf', # ¯ - 0xb0 : b'\xc2\xb0', # ° - 0xb1 : b'\xc2\xb1', # ± - 0xb2 : b'\xc2\xb2', # ² - 0xb3 : b'\xc2\xb3', # ³ - 0xb4 : b'\xc2\xb4', # ´ - 0xb5 : b'\xc2\xb5', # µ - 0xb6 : b'\xc2\xb6', # ¶ - 0xb7 : b'\xc2\xb7', # · - 0xb8 : b'\xc2\xb8', # ¸ - 0xb9 : b'\xc2\xb9', # ¹ - 0xba : b'\xc2\xba', # º - 0xbb : b'\xc2\xbb', # » - 0xbc : b'\xc2\xbc', # ¼ - 0xbd : b'\xc2\xbd', # ½ - 0xbe : b'\xc2\xbe', # ¾ - 0xbf : b'\xc2\xbf', # ¿ - 0xc0 : b'\xc3\x80', # À - 0xc1 : b'\xc3\x81', # à - 0xc2 : b'\xc3\x82', #  - 0xc3 : b'\xc3\x83', # à - 0xc4 : b'\xc3\x84', # Ä - 0xc5 : b'\xc3\x85', # Ã… - 0xc6 : b'\xc3\x86', # Æ - 0xc7 : b'\xc3\x87', # Ç - 0xc8 : b'\xc3\x88', # È - 0xc9 : b'\xc3\x89', # É - 0xca : b'\xc3\x8a', # Ê - 0xcb : b'\xc3\x8b', # Ë - 0xcc : b'\xc3\x8c', # ÃŒ - 0xcd : b'\xc3\x8d', # à - 0xce : b'\xc3\x8e', # ÃŽ - 0xcf : b'\xc3\x8f', # à - 0xd0 : b'\xc3\x90', # à - 0xd1 : b'\xc3\x91', # Ñ - 0xd2 : b'\xc3\x92', # Ã’ - 0xd3 : b'\xc3\x93', # Ó - 0xd4 : b'\xc3\x94', # Ô - 0xd5 : b'\xc3\x95', # Õ - 0xd6 : b'\xc3\x96', # Ö - 0xd7 : b'\xc3\x97', # × - 0xd8 : b'\xc3\x98', # Ø - 0xd9 : b'\xc3\x99', # Ù - 0xda : b'\xc3\x9a', # Ú - 0xdb : b'\xc3\x9b', # Û - 0xdc : b'\xc3\x9c', # Ãœ - 0xdd : b'\xc3\x9d', # à - 0xde : b'\xc3\x9e', # Þ - 0xdf : b'\xc3\x9f', # ß - 0xe0 : b'\xc3\xa0', # à - 0xe1 : b'\xa1', # á - 0xe2 : b'\xc3\xa2', # â - 0xe3 : b'\xc3\xa3', # ã - 0xe4 : b'\xc3\xa4', # ä - 0xe5 : b'\xc3\xa5', # Ã¥ - 0xe6 : b'\xc3\xa6', # æ - 0xe7 : b'\xc3\xa7', # ç - 0xe8 : b'\xc3\xa8', # è - 0xe9 : b'\xc3\xa9', # é - 0xea : b'\xc3\xaa', # ê - 0xeb : b'\xc3\xab', # ë - 0xec : b'\xc3\xac', # ì - 0xed : b'\xc3\xad', # í - 0xee : b'\xc3\xae', # î - 0xef : b'\xc3\xaf', # ï - 0xf0 : b'\xc3\xb0', # ð - 0xf1 : b'\xc3\xb1', # ñ - 0xf2 : b'\xc3\xb2', # ò - 0xf3 : b'\xc3\xb3', # ó - 0xf4 : b'\xc3\xb4', # ô - 0xf5 : b'\xc3\xb5', # õ - 0xf6 : b'\xc3\xb6', # ö - 0xf7 : b'\xc3\xb7', # ÷ - 0xf8 : b'\xc3\xb8', # ø - 0xf9 : b'\xc3\xb9', # ù - 0xfa : b'\xc3\xba', # ú - 0xfb : b'\xc3\xbb', # û - 0xfc : b'\xc3\xbc', # ü - 0xfd : b'\xc3\xbd', # ý - 0xfe : b'\xc3\xbe', # þ - } - - MULTIBYTE_MARKERS_AND_SIZES = [ - (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF - (0xe0, 0xef, 3), # 3-byte characters start with E0-EF - (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4 - ] - - FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0] - LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1] - - @classmethod - def detwingle(cls, in_bytes, main_encoding="utf8", - embedded_encoding="windows-1252"): - """Fix characters from one encoding embedded in some other encoding. - - Currently the only situation supported is Windows-1252 (or its - subset ISO-8859-1), embedded in UTF-8. - - The input must be a bytestring. If you've already converted - the document to Unicode, you're too late. - - The output is a bytestring in which `embedded_encoding` - characters have been converted to their `main_encoding` - equivalents. - """ - if embedded_encoding.replace('_', '-').lower() not in ( - 'windows-1252', 'windows_1252'): - raise NotImplementedError( - "Windows-1252 and ISO-8859-1 are the only currently supported " - "embedded encodings.") - - if main_encoding.lower() not in ('utf8', 'utf-8'): - raise NotImplementedError( - "UTF-8 is the only currently supported main encoding.") - - byte_chunks = [] - - chunk_start = 0 - pos = 0 - while pos < len(in_bytes): - byte = in_bytes[pos] - if not isinstance(byte, int): - # Python 2.x - byte = ord(byte) - if (byte >= cls.FIRST_MULTIBYTE_MARKER - and byte <= cls.LAST_MULTIBYTE_MARKER): - # This is the start of a UTF-8 multibyte character. Skip - # to the end. - for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES: - if byte >= start and byte <= end: - pos += size - break - elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8: - # We found a Windows-1252 character! - # Save the string up to this point as a chunk. - byte_chunks.append(in_bytes[chunk_start:pos]) - - # Now translate the Windows-1252 character into UTF-8 - # and add it as another, one-byte chunk. - byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte]) - pos += 1 - chunk_start = pos - else: - # Go on to the next character. - pos += 1 - if chunk_start == 0: - # The string is unchanged. - return in_bytes - else: - # Store the final chunk. - byte_chunks.append(in_bytes[chunk_start:]) - return b''.join(byte_chunks) - diff --git a/libs/bs4/diagnose.py b/libs/bs4/diagnose.py deleted file mode 100644 index 8768332f5..000000000 --- a/libs/bs4/diagnose.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Diagnostic functions, mainly for use when doing tech support.""" - -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -__license__ = "MIT" - -import cProfile -from StringIO import StringIO -from HTMLParser import HTMLParser -import bs4 -from bs4 import BeautifulSoup, __version__ -from bs4.builder import builder_registry - -import os -import pstats -import random -import tempfile -import time -import traceback -import sys -import cProfile - -def diagnose(data): - """Diagnostic suite for isolating common problems.""" - print "Diagnostic running on Beautiful Soup %s" % __version__ - print "Python version %s" % sys.version - - basic_parsers = ["html.parser", "html5lib", "lxml"] - for name in basic_parsers: - for builder in builder_registry.builders: - if name in builder.features: - break - else: - basic_parsers.remove(name) - print ( - "I noticed that %s is not installed. Installing it may help." % - name) - - if 'lxml' in basic_parsers: - basic_parsers.append(["lxml", "xml"]) - try: - from lxml import etree - print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)) - except ImportError, e: - print ( - "lxml is not installed or couldn't be imported.") - - - if 'html5lib' in basic_parsers: - try: - import html5lib - print "Found html5lib version %s" % html5lib.__version__ - except ImportError, e: - print ( - "html5lib is not installed or couldn't be imported.") - - if hasattr(data, 'read'): - data = data.read() - elif os.path.exists(data): - print '"%s" looks like a filename. Reading data from the file.' % data - with open(data) as fp: - data = fp.read() - elif data.startswith("http:") or data.startswith("https:"): - print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data - print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup." - return - print - - for parser in basic_parsers: - print "Trying to parse your markup with %s" % parser - success = False - try: - soup = BeautifulSoup(data, parser) - success = True - except Exception, e: - print "%s could not parse the markup." % parser - traceback.print_exc() - if success: - print "Here's what %s did with the markup:" % parser - print soup.prettify() - - print "-" * 80 - -def lxml_trace(data, html=True, **kwargs): - """Print out the lxml events that occur during parsing. - - This lets you see how lxml parses a document when no Beautiful - Soup code is running. - """ - from lxml import etree - for event, element in etree.iterparse(StringIO(data), html=html, **kwargs): - print("%s, %4s, %s" % (event, element.tag, element.text)) - -class AnnouncingParser(HTMLParser): - """Announces HTMLParser parse events, without doing anything else.""" - - def _p(self, s): - print(s) - - def handle_starttag(self, name, attrs): - self._p("%s START" % name) - - def handle_endtag(self, name): - self._p("%s END" % name) - - def handle_data(self, data): - self._p("%s DATA" % data) - - def handle_charref(self, name): - self._p("%s CHARREF" % name) - - def handle_entityref(self, name): - self._p("%s ENTITYREF" % name) - - def handle_comment(self, data): - self._p("%s COMMENT" % data) - - def handle_decl(self, data): - self._p("%s DECL" % data) - - def unknown_decl(self, data): - self._p("%s UNKNOWN-DECL" % data) - - def handle_pi(self, data): - self._p("%s PI" % data) - -def htmlparser_trace(data): - """Print out the HTMLParser events that occur during parsing. - - This lets you see how HTMLParser parses a document when no - Beautiful Soup code is running. - """ - parser = AnnouncingParser() - parser.feed(data) - -_vowels = "aeiou" -_consonants = "bcdfghjklmnpqrstvwxyz" - -def rword(length=5): - "Generate a random word-like string." - s = '' - for i in range(length): - if i % 2 == 0: - t = _consonants - else: - t = _vowels - s += random.choice(t) - return s - -def rsentence(length=4): - "Generate a random sentence-like string." - return " ".join(rword(random.randint(4,9)) for i in range(length)) - -def rdoc(num_elements=1000): - """Randomly generate an invalid HTML document.""" - tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table'] - elements = [] - for i in range(num_elements): - choice = random.randint(0,3) - if choice == 0: - # New tag. - tag_name = random.choice(tag_names) - elements.append("<%s>" % tag_name) - elif choice == 1: - elements.append(rsentence(random.randint(1,4))) - elif choice == 2: - # Close a tag. - tag_name = random.choice(tag_names) - elements.append("" % tag_name) - return "" + "\n".join(elements) + "" - -def benchmark_parsers(num_elements=100000): - """Very basic head-to-head performance benchmark.""" - print "Comparative parser benchmark on Beautiful Soup %s" % __version__ - data = rdoc(num_elements) - print "Generated a large invalid HTML document (%d bytes)." % len(data) - - for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]: - success = False - try: - a = time.time() - soup = BeautifulSoup(data, parser) - b = time.time() - success = True - except Exception, e: - print "%s could not parse the markup." % parser - traceback.print_exc() - if success: - print "BS4+%s parsed the markup in %.2fs." % (parser, b-a) - - from lxml import etree - a = time.time() - etree.HTML(data) - b = time.time() - print "Raw lxml parsed the markup in %.2fs." % (b-a) - - import html5lib - parser = html5lib.HTMLParser() - a = time.time() - parser.parse(data) - b = time.time() - print "Raw html5lib parsed the markup in %.2fs." % (b-a) - -def profile(num_elements=100000, parser="lxml"): - - filehandle = tempfile.NamedTemporaryFile() - filename = filehandle.name - - data = rdoc(num_elements) - vars = dict(bs4=bs4, data=data, parser=parser) - cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename) - - stats = pstats.Stats(filename) - # stats.strip_dirs() - stats.sort_stats("cumulative") - stats.print_stats('_html5lib|bs4', 50) - -if __name__ == '__main__': - diagnose(sys.stdin.read()) diff --git a/libs/bs4/element.py b/libs/bs4/element.py deleted file mode 100644 index b100d18bb..000000000 --- a/libs/bs4/element.py +++ /dev/null @@ -1,1755 +0,0 @@ -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -__license__ = "MIT" - -import collections -import re -import shlex -import sys -import warnings -from bs4.dammit import EntitySubstitution - -DEFAULT_OUTPUT_ENCODING = "utf-8" -PY3K = (sys.version_info[0] > 2) - -whitespace_re = re.compile("\s+") - -def _alias(attr): - """Alias one attribute name to another for backward compatibility""" - @property - def alias(self): - return getattr(self, attr) - - @alias.setter - def alias(self): - return setattr(self, attr) - return alias - - -class NamespacedAttribute(unicode): - - def __new__(cls, prefix, name, namespace=None): - if name is None: - obj = unicode.__new__(cls, prefix) - elif prefix is None: - # Not really namespaced. - obj = unicode.__new__(cls, name) - else: - obj = unicode.__new__(cls, prefix + ":" + name) - obj.prefix = prefix - obj.name = name - obj.namespace = namespace - return obj - -class AttributeValueWithCharsetSubstitution(unicode): - """A stand-in object for a character encoding specified in HTML.""" - -class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution): - """A generic stand-in for the value of a meta tag's 'charset' attribute. - - When Beautiful Soup parses the markup '', the - value of the 'charset' attribute will be one of these objects. - """ - - def __new__(cls, original_value): - obj = unicode.__new__(cls, original_value) - obj.original_value = original_value - return obj - - def encode(self, encoding): - return encoding - - -class ContentMetaAttributeValue(AttributeValueWithCharsetSubstitution): - """A generic stand-in for the value of a meta tag's 'content' attribute. - - When Beautiful Soup parses the markup: - - - The value of the 'content' attribute will be one of these objects. - """ - - CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M) - - def __new__(cls, original_value): - match = cls.CHARSET_RE.search(original_value) - if match is None: - # No substitution necessary. - return unicode.__new__(unicode, original_value) - - obj = unicode.__new__(cls, original_value) - obj.original_value = original_value - return obj - - def encode(self, encoding): - def rewrite(match): - return match.group(1) + encoding - return self.CHARSET_RE.sub(rewrite, self.original_value) - -class HTMLAwareEntitySubstitution(EntitySubstitution): - - """Entity substitution rules that are aware of some HTML quirks. - - Specifically, the contents of - -Hello, world! - - -''' - soup = self.soup(html) - self.assertEqual("text/javascript", soup.find('script')['type']) - - def test_comment(self): - # Comments are represented as Comment objects. - markup = "

foobaz

" - self.assertSoupEquals(markup) - - soup = self.soup(markup) - comment = soup.find(text="foobar") - self.assertEqual(comment.__class__, Comment) - - # The comment is properly integrated into the tree. - foo = soup.find(text="foo") - self.assertEqual(comment, foo.next_element) - baz = soup.find(text="baz") - self.assertEqual(comment, baz.previous_element) - - def test_preserved_whitespace_in_pre_and_textarea(self): - """Whitespace must be preserved in 
 and "
-        self.assertSoupEquals(pre_markup)
-        self.assertSoupEquals(textarea_markup)
-
-        soup = self.soup(pre_markup)
-        self.assertEqual(soup.pre.prettify(), pre_markup)
-
-        soup = self.soup(textarea_markup)
-        self.assertEqual(soup.textarea.prettify(), textarea_markup)
-
-        soup = self.soup("")
-        self.assertEqual(soup.textarea.prettify(), "")
-
-    def test_nested_inline_elements(self):
-        """Inline elements can be nested indefinitely."""
-        b_tag = "Inside a B tag"
-        self.assertSoupEquals(b_tag)
-
-        nested_b_tag = "
A nested tag
"
-        self.assertSoupEquals(nested_b_tag)
-
-        double_nested_b_tag = "
A doubly nested tag
"
-        self.assertSoupEquals(nested_b_tag)
-
-    def test_nested_block_level_elements(self):
-        """Block elements can be nested."""
-        soup = self.soup('
Foo
')
-        blockquote = soup.blockquote
-        self.assertEqual(blockquote.p.b.string, 'Foo')
-        self.assertEqual(blockquote.b.string, 'Foo')
-
-    def test_correctly_nested_tables(self):
-        """One table can go inside another one."""
-        markup = (''
-                  ''
-                  "')
-
-        self.assertSoupEquals(
-            markup,
-            '
Here's another table:"
-                  ''
-                  ''
-                  '
foo
Here\'s another table:'
-            '
foo
'
-            '
')
-
-        self.assertSoupEquals(
-            ""
-            ""
-            "
Foo
Bar
Baz
")
-
-    def test_deeply_nested_multivalued_attribute(self):
-        # html5lib can set the attributes of the same tag many times
-        # as it rearranges the tree. This has caused problems with
-        # multivalued attributes.
-        markup = '
'
-        soup = self.soup(markup)
-        self.assertEqual(["css"], soup.div.div['class'])
-
-    def test_multivalued_attribute_on_html(self):
-        # html5lib uses a different API to set the attributes ot the
-        #  tag. This has caused problems with multivalued
-        # attributes.
-        markup = ''
-        soup = self.soup(markup)
-        self.assertEqual(["a", "b"], soup.html['class'])
-
-    def test_angle_brackets_in_attribute_values_are_escaped(self):
-        self.assertSoupEquals('', '')
-
-    def test_entities_in_attributes_converted_to_unicode(self):
-        expect = u'
'
-        self.assertSoupEquals('
', expect)
-        self.assertSoupEquals('
', expect)
-        self.assertSoupEquals('
', expect)
-        self.assertSoupEquals('
', expect)
-
-    def test_entities_in_text_converted_to_unicode(self):
-        expect = u'
pi\N{LATIN SMALL LETTER N WITH TILDE}ata
'
-        self.assertSoupEquals("
piñata
", expect)
-        self.assertSoupEquals("
piñata
", expect)
-        self.assertSoupEquals("
piñata
", expect)
-        self.assertSoupEquals("
piñata
", expect)
-
-    def test_quot_entity_converted_to_quotation_mark(self):
-        self.assertSoupEquals("
I said "good day!"
",
-                              '
I said "good day!"
')
-
-    def test_out_of_range_entity(self):
-        expect = u"\N{REPLACEMENT CHARACTER}"
-        self.assertSoupEquals("�", expect)
-        self.assertSoupEquals("�", expect)
-        self.assertSoupEquals("�", expect)
-
-    def test_multipart_strings(self):
-        "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
-        soup = self.soup("
\nfoo
")
-        self.assertEqual("p", soup.h2.string.next_element.name)
-        self.assertEqual("p", soup.p.name)
-        self.assertConnectedness(soup)
-
-    def test_head_tag_between_head_and_body(self):
-        "Prevent recurrence of a bug in the html5lib treebuilder."
-        content = """
-  
-  foo
-
-"""
-        soup = self.soup(content)
-        self.assertNotEqual(None, soup.html.body)
-        self.assertConnectedness(soup)
-
-    def test_multiple_copies_of_a_tag(self):
-        "Prevent recurrence of a bug in the html5lib treebuilder."
-        content = """
-
- 
-   
-  
-
-"""
-        soup = self.soup(content)
-        self.assertConnectedness(soup.article)
-
-    def test_basic_namespaces(self):
-        """Parsers don't need to *understand* namespaces, but at the
-        very least they should not choke on namespaces or lose
-        data."""
-
-        markup = b'4'
-        soup = self.soup(markup)
-        self.assertEqual(markup, soup.encode())
-        html = soup.html
-        self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
-        self.assertEqual(
-            'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
-        self.assertEqual(
-            'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
-
-    def test_multivalued_attribute_value_becomes_list(self):
-        markup = b''
-        soup = self.soup(markup)
-        self.assertEqual(['foo', 'bar'], soup.a['class'])
-
-    #
-    # Generally speaking, tests below this point are more tests of
-    # Beautiful Soup than tests of the tree builders. But parsers are
-    # weird, so we run these tests separately for every tree builder
-    # to detect any differences between them.
-    #
-
-    def test_can_parse_unicode_document(self):
-        # A seemingly innocuous document... but it's in Unicode! And
-        # it contains characters that can't be represented in the
-        # encoding found in the  declaration! The horror!
-        markup = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
-        soup = self.soup(markup)
-        self.assertEqual(u'Sacr\xe9 bleu!', soup.body.string)
-
-    def test_soupstrainer(self):
-        """Parsers should be able to work with SoupStrainers."""
-        strainer = SoupStrainer("b")
-        soup = self.soup("A bold  statement",
-                         parse_only=strainer)
-        self.assertEqual(soup.decode(), "bold")
-
-    def test_single_quote_attribute_values_become_double_quotes(self):
-        self.assertSoupEquals("",
-                              '')
-
-    def test_attribute_values_with_nested_quotes_are_left_alone(self):
-        text = """a"""
-        self.assertSoupEquals(text)
-
-    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
-        text = """a"""
-        soup = self.soup(text)
-        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
-        self.assertSoupEquals(
-            soup.foo.decode(),
-            """a""")
-
-    def test_ampersand_in_attribute_value_gets_escaped(self):
-        self.assertSoupEquals('',
-                              '')
-
-        self.assertSoupEquals(
-            'foo',
-            'foo')
-
-    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
-        self.assertSoupEquals('')
-
-    def test_entities_in_strings_converted_during_parsing(self):
-        # Both XML and HTML entities are converted to Unicode characters
-        # during parsing.
-        text = "
<<sacré bleu!>>
"
-        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
"
-        self.assertSoupEquals(text, expected)
-
-    def test_smart_quotes_converted_on_the_way_in(self):
-        # Microsoft smart quotes are converted to Unicode characters during
-        # parsing.
-        quote = b"
\x91Foo\x92
"
-        soup = self.soup(quote)
-        self.assertEqual(
-            soup.p.string,
-            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
-
-    def test_non_breaking_spaces_converted_on_the_way_in(self):
-        soup = self.soup("  ")
-        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
-
-    def test_entities_converted_on_the_way_out(self):
-        text = "
<<sacré bleu!>>
"
-        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
".encode("utf-8")
-        soup = self.soup(text)
-        self.assertEqual(soup.p.encode("utf-8"), expected)
-
-    def test_real_iso_latin_document(self):
-        # Smoke test of interrelated functionality, using an
-        # easy-to-understand document.
-
-        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
-        unicode_html = u'
Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!
'
-
-        # That's because we're going to encode it into ISO-Latin-1, and use
-        # that to test.
-        iso_latin_html = unicode_html.encode("iso-8859-1")
-
-        # Parse the ISO-Latin-1 HTML.
-        soup = self.soup(iso_latin_html)
-        # Encode it to UTF-8.
-        result = soup.encode("utf-8")
-
-        # What do we expect the result to look like? Well, it would
-        # look like unicode_html, except that the META tag would say
-        # UTF-8 instead of ISO-Latin-1.
-        expected = unicode_html.replace("ISO-Latin-1", "utf-8")
-
-        # And, of course, it would be in UTF-8, not Unicode.
-        expected = expected.encode("utf-8")
-
-        # Ta-da!
-        self.assertEqual(result, expected)
-
-    def test_real_shift_jis_document(self):
-        # Smoke test to make sure the parser can handle a document in
-        # Shift-JIS encoding, without choking.
-        shift_jis_html = (
-            b'
'
-            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
-            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
-            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
-            b'
')
-        unicode_html = shift_jis_html.decode("shift-jis")
-        soup = self.soup(unicode_html)
-
-        # Make sure the parse tree is correctly encoded to various
-        # encodings.
-        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
-        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
-
-    def test_real_hebrew_document(self):
-        # A real-world test to make sure we can convert ISO-8859-9 (a
-        # Hebrew encoding) to UTF-8.
-        hebrew_document = b'Hebrew (ISO 8859-8) in Visual Directionality
Hebrew (ISO 8859-8) in Visual Directionality
\xed\xe5\xec\xf9'
-        soup = self.soup(
-            hebrew_document, from_encoding="iso8859-8")
-        # Some tree builders call it iso8859-8, others call it iso-8859-9.
-        # That's not a difference we really care about.
-        assert soup.original_encoding in ('iso8859-8', 'iso-8859-8')
-        self.assertEqual(
-            soup.encode('utf-8'),
-            hebrew_document.decode("iso8859-8").encode("utf-8"))
-
-    def test_meta_tag_reflects_current_encoding(self):
-        # Here's the  tag saying that a document is
-        # encoded in Shift-JIS.
-        meta_tag = ('')
-
-        # Here's a document incorporating that meta tag.
-        shift_jis_html = (
-            '\n%s\n'
-            ''
-            'Shift-JIS markup goes here.') % meta_tag
-        soup = self.soup(shift_jis_html)
-
-        # Parse the document, and the charset is seemingly unaffected.
-        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
-        content = parsed_meta['content']
-        self.assertEqual('text/html; charset=x-sjis', content)
-
-        # But that value is actually a ContentMetaAttributeValue object.
-        self.assertTrue(isinstance(content, ContentMetaAttributeValue))
-
-        # And it will take on a value that reflects its current
-        # encoding.
-        self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
-
-        # For the rest of the story, see TestSubstitutions in
-        # test_tree.py.
-
-    def test_html5_style_meta_tag_reflects_current_encoding(self):
-        # Here's the  tag saying that a document is
-        # encoded in Shift-JIS.
-        meta_tag = ('')
-
-        # Here's a document incorporating that meta tag.
-        shift_jis_html = (
-            '\n%s\n'
-            ''
-            'Shift-JIS markup goes here.') % meta_tag
-        soup = self.soup(shift_jis_html)
-
-        # Parse the document, and the charset is seemingly unaffected.
-        parsed_meta = soup.find('meta', id="encoding")
-        charset = parsed_meta['charset']
-        self.assertEqual('x-sjis', charset)
-
-        # But that value is actually a CharsetMetaAttributeValue object.
-        self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
-
-        # And it will take on a value that reflects its current
-        # encoding.
-        self.assertEqual('utf8', charset.encode("utf8"))
-
-    def test_tag_with_no_attributes_can_have_attributes_added(self):
-        data = self.soup("text")
-        data.a['foo'] = 'bar'
-        self.assertEqual('text', data.a.decode())
-
-class XMLTreeBuilderSmokeTest(object):
-
-    def test_pickle_and_unpickle_identity(self):
-        # Pickling a tree, then unpickling it, yields a tree identical
-        # to the original.
-        tree = self.soup("foo")
-        dumped = pickle.dumps(tree, 2)
-        loaded = pickle.loads(dumped)
-        self.assertEqual(loaded.__class__, BeautifulSoup)
-        self.assertEqual(loaded.decode(), tree.decode())
-
-    def test_docstring_generated(self):
-        soup = self.soup("")
-        self.assertEqual(
-            soup.encode(), b'\n')
-
-    def test_xml_declaration(self):
-        markup = b"""\n"""
-        soup = self.soup(markup)
-        self.assertEqual(markup, soup.encode("utf8"))
-
-    def test_processing_instruction(self):
-        markup = b"""\n"""
-        soup = self.soup(markup)
-        self.assertEqual(markup, soup.encode("utf8"))
-
-    def test_real_xhtml_document(self):
-        """A real XHTML document should come out *exactly* the same as it went in."""
-        markup = b"""
-
-
-Hello.
-Goodbye.
-"""
-        soup = self.soup(markup)
-        self.assertEqual(
-            soup.encode("utf-8"), markup)
-
-    def test_formatter_processes_script_tag_for_xml_documents(self):
-        doc = """
-  
-"""
-        soup = BeautifulSoup(doc, "lxml-xml")
-        # lxml would have stripped this while parsing, but we can add
-        # it later.
-        soup.script.string = 'console.log("< < hey > > ");'
-        encoded = soup.encode()
-        self.assertTrue(b"< < hey > >" in encoded)
-
-    def test_can_parse_unicode_document(self):
-        markup = u'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
-        soup = self.soup(markup)
-        self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string)
-
-    def test_popping_namespaced_tag(self):
-        markup = 'b2012-07-02T20:33:42Zcd'
-        soup = self.soup(markup)
-        self.assertEqual(
-            unicode(soup.rss), markup)
-
-    def test_docstring_includes_correct_encoding(self):
-        soup = self.soup("")
-        self.assertEqual(
-            soup.encode("latin1"),
-            b'\n')
-
-    def test_large_xml_document(self):
-        """A large XML document should come out the same as it went in."""
-        markup = (b'\n'
-                  + b'0' * (2**12)
-                  + b'')
-        soup = self.soup(markup)
-        self.assertEqual(soup.encode("utf-8"), markup)
-
-
-    def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
-        self.assertSoupEquals("
", "
")
-        self.assertSoupEquals("
foo
")
-
-    def test_namespaces_are_preserved(self):
-        markup = 'This tag is in the a namespaceThis tag is in the b namespace'
-        soup = self.soup(markup)
-        root = soup.root
-        self.assertEqual("http://example.com/", root['xmlns:a'])
-        self.assertEqual("http://example.net/", root['xmlns:b'])
-
-    def test_closing_namespaced_tag(self):
-        markup = '
20010504
'
-        soup = self.soup(markup)
-        self.assertEqual(unicode(soup.p), markup)
-
-    def test_namespaced_attributes(self):
-        markup = ''
-        soup = self.soup(markup)
-        self.assertEqual(unicode(soup.foo), markup)
-
-    def test_namespaced_attributes_xml_namespace(self):
-        markup = 'bar'
-        soup = self.soup(markup)
-        self.assertEqual(unicode(soup.foo), markup)
-
-class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
-    """Smoke test for a tree builder that supports HTML5."""
-
-    def test_real_xhtml_document(self):
-        # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
-        # XHTML documents in any particular way.
-        pass
-
-    def test_html_tags_have_namespace(self):
-        markup = ""
-        soup = self.soup(markup)
-        self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
-
-    def test_svg_tags_have_namespace(self):
-        markup = ''
-        soup = self.soup(markup)
-        namespace = "http://www.w3.org/2000/svg"
-        self.assertEqual(namespace, soup.svg.namespace)
-        self.assertEqual(namespace, soup.circle.namespace)
-
-
-    def test_mathml_tags_have_namespace(self):
-        markup = '5'
-        soup = self.soup(markup)
-        namespace = 'http://www.w3.org/1998/Math/MathML'
-        self.assertEqual(namespace, soup.math.namespace)
-        self.assertEqual(namespace, soup.msqrt.namespace)
-
-    def test_xml_declaration_becomes_comment(self):
-        markup = ''
-        soup = self.soup(markup)
-        self.assertTrue(isinstance(soup.contents[0], Comment))
-        self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?')
-        self.assertEqual("html", soup.contents[0].next_element.name)
-
-def skipIf(condition, reason):
-   def nothing(test, *args, **kwargs):
-       return None
-
-   def decorator(test_item):
-       if condition:
-           return nothing
-       else:
-           return test_item
-
-   return decorator
diff --git a/libs/bs4/tests/test_html5lib.py b/libs/bs4/tests/test_html5lib.py
deleted file mode 100644
index 8e3cba689..000000000
--- a/libs/bs4/tests/test_html5lib.py
+++ /dev/null
@@ -1,109 +0,0 @@
-"""Tests to ensure that the html5lib tree builder generates good trees."""
-
-import warnings
-
-try:
-    from bs4.builder import HTML5TreeBuilder
-    HTML5LIB_PRESENT = True
-except ImportError, e:
-    HTML5LIB_PRESENT = False
-from bs4.element import SoupStrainer
-from bs4.testing import (
-    HTML5TreeBuilderSmokeTest,
-    SoupTest,
-    skipIf,
-)
-
-@skipIf(
-    not HTML5LIB_PRESENT,
-    "html5lib seems not to be present, not testing its tree builder.")
-class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest):
-    """See ``HTML5TreeBuilderSmokeTest``."""
-
-    @property
-    def default_builder(self):
-        return HTML5TreeBuilder()
-
-    def test_soupstrainer(self):
-        # The html5lib tree builder does not support SoupStrainers.
-        strainer = SoupStrainer("b")
-        markup = "
A bold statement.
"
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup(markup, parse_only=strainer)
-        self.assertEqual(
-            soup.decode(), self.document_for(markup))
-
-        self.assertTrue(
-            "the html5lib tree builder doesn't support parse_only" in
-            str(w[0].message))
-
-    def test_correctly_nested_tables(self):
-        """html5lib inserts  tags where other parsers don't."""
-        markup = (''
-                  ''
-                  "')
-
-        self.assertSoupEquals(
-            markup,
-            '
Here's another table:"
-                  ''
-                  ''
-                  '
foo
Here\'s another table:'
-            '
foo
'
-            '
')
-
-        self.assertSoupEquals(
-            ""
-            ""
-            "
Foo
Bar
Baz
")
-
-    def test_xml_declaration_followed_by_doctype(self):
-        markup = '''
-
-
-  
-  
-  
-   
foo

-  
-'''
-        soup = self.soup(markup)
-        # Verify that we can reach the 
 tag; this means the tree is connected.
-        self.assertEqual(b"
foo
", soup.p.encode())
-
-    def test_reparented_markup(self):
-        markup = '
foo
\n
bar
'
-        soup = self.soup(markup)
-        self.assertEqual(u"
foo
\n
bar
", soup.body.decode())
-        self.assertEqual(2, len(soup.find_all('p')))
-
-
-    def test_reparented_markup_ends_with_whitespace(self):
-        markup = '
foo
\n
bar
\n'
-        soup = self.soup(markup)
-        self.assertEqual(u"
foo
\n
bar
\n", soup.body.decode())
-        self.assertEqual(2, len(soup.find_all('p')))
-
-    def test_reparented_markup_containing_identical_whitespace_nodes(self):
-        """Verify that we keep the two whitespace nodes in this
-        document distinct when reparenting the adjacent  tags.
-        """
-        markup = '
  
'
-        soup = self.soup(markup)
-        space1, space2 = soup.find_all(string=' ')
-        tbody1, tbody2 = soup.find_all('tbody')
-        assert space1.next_element is tbody1
-        assert tbody2.next_element is space2
-
-    def test_processing_instruction(self):
-        """Processing instructions become comments."""
-        markup = b""""""
-        soup = self.soup(markup)
-        assert str(soup).startswith("")
-
-    def test_cloned_multivalue_node(self):
-        markup = b"""
"""
-        soup = self.soup(markup)
-        a1, a2 = soup.find_all('a')
-        self.assertEqual(a1, a2)
-        assert a1 is not a2
diff --git a/libs/bs4/tests/test_htmlparser.py b/libs/bs4/tests/test_htmlparser.py
deleted file mode 100644
index b45e35f99..000000000
--- a/libs/bs4/tests/test_htmlparser.py
+++ /dev/null
@@ -1,32 +0,0 @@
-"""Tests to ensure that the html.parser tree builder generates good
-trees."""
-
-from pdb import set_trace
-import pickle
-from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest
-from bs4.builder import HTMLParserTreeBuilder
-
-class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
-
-    @property
-    def default_builder(self):
-        return HTMLParserTreeBuilder()
-
-    def test_namespaced_system_doctype(self):
-        # html.parser can't handle namespaced doctypes, so skip this one.
-        pass
-
-    def test_namespaced_public_doctype(self):
-        # html.parser can't handle namespaced doctypes, so skip this one.
-        pass
-
-    def test_builder_is_pickled(self):
-        """Unlike most tree builders, HTMLParserTreeBuilder and will
-        be restored after pickling.
-        """
-        tree = self.soup("foo")
-        dumped = pickle.dumps(tree, 2)
-        loaded = pickle.loads(dumped)
-        self.assertTrue(isinstance(loaded.builder, type(tree.builder)))
-
-
diff --git a/libs/bs4/tests/test_lxml.py b/libs/bs4/tests/test_lxml.py
deleted file mode 100644
index a05870b91..000000000
--- a/libs/bs4/tests/test_lxml.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""Tests to ensure that the lxml tree builder generates good trees."""
-
-import re
-import warnings
-
-try:
-    import lxml.etree
-    LXML_PRESENT = True
-    LXML_VERSION = lxml.etree.LXML_VERSION
-except ImportError, e:
-    LXML_PRESENT = False
-    LXML_VERSION = (0,)
-
-if LXML_PRESENT:
-    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
-
-from bs4 import (
-    BeautifulSoup,
-    BeautifulStoneSoup,
-    )
-from bs4.element import Comment, Doctype, SoupStrainer
-from bs4.testing import skipIf
-from bs4.tests import test_htmlparser
-from bs4.testing import (
-    HTMLTreeBuilderSmokeTest,
-    XMLTreeBuilderSmokeTest,
-    SoupTest,
-    skipIf,
-)
-
-@skipIf(
-    not LXML_PRESENT,
-    "lxml seems not to be present, not testing its tree builder.")
-class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
-    """See ``HTMLTreeBuilderSmokeTest``."""
-
-    @property
-    def default_builder(self):
-        return LXMLTreeBuilder()
-
-    def test_out_of_range_entity(self):
-        self.assertSoupEquals(
-            "
foo�bar
", "
foobar
")
-        self.assertSoupEquals(
-            "
foo�bar
", "
foobar
")
-        self.assertSoupEquals(
-            "
foo�bar
", "
foobar
")
-
-    # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this
-    # test if an old version of lxml is installed.
-
-    @skipIf(
-        not LXML_PRESENT or LXML_VERSION < (2,3,5,0),
-        "Skipping doctype test for old version of lxml to avoid segfault.")
-    def test_empty_doctype(self):
-        soup = self.soup("")
-        doctype = soup.contents[0]
-        self.assertEqual("", doctype.strip())
-
-    def test_beautifulstonesoup_is_xml_parser(self):
-        # Make sure that the deprecated BSS class uses an xml builder
-        # if one is installed.
-        with warnings.catch_warnings(record=True) as w:
-            soup = BeautifulStoneSoup("")
-        self.assertEqual(u"", unicode(soup.b))
-        self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message))
-
-@skipIf(
-    not LXML_PRESENT,
-    "lxml seems not to be present, not testing its XML tree builder.")
-class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest):
-    """See ``HTMLTreeBuilderSmokeTest``."""
-
-    @property
-    def default_builder(self):
-        return LXMLTreeBuilderForXML()
diff --git a/libs/bs4/tests/test_tree.py b/libs/bs4/tests/test_tree.py
deleted file mode 100644
index a4fe0b166..000000000
--- a/libs/bs4/tests/test_tree.py
+++ /dev/null
@@ -1,2044 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Tests for Beautiful Soup's tree traversal methods.
-
-The tree traversal methods are the main advantage of using Beautiful
-Soup over just using a parser.
-
-Different parsers will build different Beautiful Soup trees given the
-same markup, but all Beautiful Soup trees can be traversed with the
-methods tested here.
-"""
-
-from pdb import set_trace
-import copy
-import pickle
-import re
-import warnings
-from bs4 import BeautifulSoup
-from bs4.builder import (
-    builder_registry,
-    HTMLParserTreeBuilder,
-)
-from bs4.element import (
-    PY3K,
-    CData,
-    Comment,
-    Declaration,
-    Doctype,
-    NavigableString,
-    SoupStrainer,
-    Tag,
-)
-from bs4.testing import (
-    SoupTest,
-    skipIf,
-)
-
-XML_BUILDER_PRESENT = (builder_registry.lookup("xml") is not None)
-LXML_PRESENT = (builder_registry.lookup("lxml") is not None)
-
-class TreeTest(SoupTest):
-
-    def assertSelects(self, tags, should_match):
-        """Make sure that the given tags have the correct text.
-
-        This is used in tests that define a bunch of tags, each
-        containing a single string, and then select certain strings by
-        some mechanism.
-        """
-        self.assertEqual([tag.string for tag in tags], should_match)
-
-    def assertSelectsIDs(self, tags, should_match):
-        """Make sure that the given tags have the correct IDs.
-
-        This is used in tests that define a bunch of tags, each
-        containing a single string, and then select certain strings by
-        some mechanism.
-        """
-        self.assertEqual([tag['id'] for tag in tags], should_match)
-
-
-class TestFind(TreeTest):
-    """Basic tests of the find() method.
-
-    find() just calls find_all() with limit=1, so it's not tested all
-    that thouroughly here.
-    """
-
-    def test_find_tag(self):
-        soup = self.soup("1234")
-        self.assertEqual(soup.find("b").string, "2")
-
-    def test_unicode_text_find(self):
-        soup = self.soup(u'
Räksmörgås
')
-        self.assertEqual(soup.find(string=u'Räksmörgås'), u'Räksmörgås')
-
-    def test_unicode_attribute_find(self):
-        soup = self.soup(u'
here it is
')
-        str(soup)
-        self.assertEqual("here it is", soup.find(id=u'Räksmörgås').text)
-
-
-    def test_find_everything(self):
-        """Test an optimization that finds all tags."""
-        soup = self.soup("foobar")
-        self.assertEqual(2, len(soup.find_all()))
-
-    def test_find_everything_with_name(self):
-        """Test an optimization that finds all tags with a given name."""
-        soup = self.soup("foobarbaz")
-        self.assertEqual(2, len(soup.find_all('a')))
-
-class TestFindAll(TreeTest):
-    """Basic tests of the find_all() method."""
-
-    def test_find_all_text_nodes(self):
-        """You can search the tree for text nodes."""
-        soup = self.soup("Foobar\xbb")
-        # Exact match.
-        self.assertEqual(soup.find_all(string="bar"), [u"bar"])
-        self.assertEqual(soup.find_all(text="bar"), [u"bar"])
-        # Match any of a number of strings.
-        self.assertEqual(
-            soup.find_all(text=["Foo", "bar"]), [u"Foo", u"bar"])
-        # Match a regular expression.
-        self.assertEqual(soup.find_all(text=re.compile('.*')),
-                         [u"Foo", u"bar", u'\xbb'])
-        # Match anything.
-        self.assertEqual(soup.find_all(text=True),
-                         [u"Foo", u"bar", u'\xbb'])
-
-    def test_find_all_limit(self):
-        """You can limit the number of items returned by find_all."""
-        soup = self.soup("12345")
-        self.assertSelects(soup.find_all('a', limit=3), ["1", "2", "3"])
-        self.assertSelects(soup.find_all('a', limit=1), ["1"])
-        self.assertSelects(
-            soup.find_all('a', limit=10), ["1", "2", "3", "4", "5"])
-
-        # A limit of 0 means no limit.
-        self.assertSelects(
-            soup.find_all('a', limit=0), ["1", "2", "3", "4", "5"])
-
-    def test_calling_a_tag_is_calling_findall(self):
-        soup = self.soup("123")
-        self.assertSelects(soup('a', limit=1), ["1"])
-        self.assertSelects(soup.b(id="foo"), ["3"])
-
-    def test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion(self):
-        soup = self.soup("")
-        # Create a self-referential list.
-        l = []
-        l.append(l)
-
-        # Without special code in _normalize_search_value, this would cause infinite
-        # recursion.
-        self.assertEqual([], soup.find_all(l))
-
-    def test_find_all_resultset(self):
-        """All find_all calls return a ResultSet"""
-        soup = self.soup("")
-        result = soup.find_all("a")
-        self.assertTrue(hasattr(result, "source"))
-
-        result = soup.find_all(True)
-        self.assertTrue(hasattr(result, "source"))
-
-        result = soup.find_all(text="foo")
-        self.assertTrue(hasattr(result, "source"))
-
-
-class TestFindAllBasicNamespaces(TreeTest):
-
-    def test_find_by_namespaced_name(self):
-        soup = self.soup('4')
-        self.assertEqual("4", soup.find("mathml:msqrt").string)
-        self.assertEqual("a", soup.find(attrs= { "svg:fill" : "red" }).name)
-
-
-class TestFindAllByName(TreeTest):
-    """Test ways of finding tags by tag name."""
-
-    def setUp(self):
-        super(TreeTest, self).setUp()
-        self.tree =  self.soup("""First tag.
-                                  Second tag.
-                                  Third Nested tag. tag.""")
-
-    def test_find_all_by_tag_name(self):
-        # Find all the  tags.
-        self.assertSelects(
-            self.tree.find_all('a'), ['First tag.', 'Nested tag.'])
-
-    def test_find_all_by_name_and_text(self):
-        self.assertSelects(
-            self.tree.find_all('a', text='First tag.'), ['First tag.'])
-
-        self.assertSelects(
-            self.tree.find_all('a', text=True), ['First tag.', 'Nested tag.'])
-
-        self.assertSelects(
-            self.tree.find_all('a', text=re.compile("tag")),
-            ['First tag.', 'Nested tag.'])
-
-
-    def test_find_all_on_non_root_element(self):
-        # You can call find_all on any node, not just the root.
-        self.assertSelects(self.tree.c.find_all('a'), ['Nested tag.'])
-
-    def test_calling_element_invokes_find_all(self):
-        self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.'])
-
-    def test_find_all_by_tag_strainer(self):
-        self.assertSelects(
-            self.tree.find_all(SoupStrainer('a')),
-            ['First tag.', 'Nested tag.'])
-
-    def test_find_all_by_tag_names(self):
-        self.assertSelects(
-            self.tree.find_all(['a', 'b']),
-            ['First tag.', 'Second tag.', 'Nested tag.'])
-
-    def test_find_all_by_tag_dict(self):
-        self.assertSelects(
-            self.tree.find_all({'a' : True, 'b' : True}),
-            ['First tag.', 'Second tag.', 'Nested tag.'])
-
-    def test_find_all_by_tag_re(self):
-        self.assertSelects(
-            self.tree.find_all(re.compile('^[ab]$')),
-            ['First tag.', 'Second tag.', 'Nested tag.'])
-
-    def test_find_all_with_tags_matching_method(self):
-        # You can define an oracle method that determines whether
-        # a tag matches the search.
-        def id_matches_name(tag):
-            return tag.name == tag.get('id')
-
-        tree = self.soup("""Match 1.
-                            Does not match.
-                            Match 2.""")
-
-        self.assertSelects(
-            tree.find_all(id_matches_name), ["Match 1.", "Match 2."])
-
-    def test_find_with_multi_valued_attribute(self):
-        soup = self.soup(
-            "
1
2
3
"
-        )
-        r1 = soup.find('div', 'a d');
-        r2 = soup.find('div', re.compile(r'a d'));
-        r3, r4 = soup.find_all('div', ['a b', 'a d']);
-        self.assertEqual('3', r1.string)
-        self.assertEqual('3', r2.string)
-        self.assertEqual('1', r3.string)
-        self.assertEqual('3', r4.string)
-
-class TestFindAllByAttribute(TreeTest):
-
-    def test_find_all_by_attribute_name(self):
-        # You can pass in keyword arguments to find_all to search by
-        # attribute.
-        tree = self.soup("""
-                         Matching a.
-                         
-                          Non-matching Matching b.a.
-                         """)
-        self.assertSelects(tree.find_all(id='first'),
-                           ["Matching a.", "Matching b."])
-
-    def test_find_all_by_utf8_attribute_value(self):
-        peace = u"×ולש".encode("utf8")
-        data = u''.encode("utf8")
-        soup = self.soup(data)
-        self.assertEqual([soup.a], soup.find_all(title=peace))
-        self.assertEqual([soup.a], soup.find_all(title=peace.decode("utf8")))
-        self.assertEqual([soup.a], soup.find_all(title=[peace, "something else"]))
-
-    def test_find_all_by_attribute_dict(self):
-        # You can pass in a dictionary as the argument 'attrs'. This
-        # lets you search for attributes like 'name' (a fixed argument
-        # to find_all) and 'class' (a reserved word in Python.)
-        tree = self.soup("""
-                         Name match.
-                         Class match.
-                         Non-match.
-                         A tag called 'name1'.
-                         """)
-
-        # This doesn't do what you want.
-        self.assertSelects(tree.find_all(name='name1'),
-                           ["A tag called 'name1'."])
-        # This does what you want.
-        self.assertSelects(tree.find_all(attrs={'name' : 'name1'}),
-                           ["Name match."])
-
-        self.assertSelects(tree.find_all(attrs={'class' : 'class2'}),
-                           ["Class match."])
-
-    def test_find_all_by_class(self):
-        tree = self.soup("""
-                         Class 1.
-                         Class 2.
-                         Class 1.
-                         Class 3 and 4.
-                         """)
-
-        # Passing in the class_ keyword argument will search against
-        # the 'class' attribute.
-        self.assertSelects(tree.find_all('a', class_='1'), ['Class 1.'])
-        self.assertSelects(tree.find_all('c', class_='3'), ['Class 3 and 4.'])
-        self.assertSelects(tree.find_all('c', class_='4'), ['Class 3 and 4.'])
-
-        # Passing in a string to 'attrs' will also search the CSS class.
-        self.assertSelects(tree.find_all('a', '1'), ['Class 1.'])
-        self.assertSelects(tree.find_all(attrs='1'), ['Class 1.', 'Class 1.'])
-        self.assertSelects(tree.find_all('c', '3'), ['Class 3 and 4.'])
-        self.assertSelects(tree.find_all('c', '4'), ['Class 3 and 4.'])
-
-    def test_find_by_class_when_multiple_classes_present(self):
-        tree = self.soup("Found it")
-
-        f = tree.find_all("gar", class_=re.compile("o"))
-        self.assertSelects(f, ["Found it"])
-
-        f = tree.find_all("gar", class_=re.compile("a"))
-        self.assertSelects(f, ["Found it"])
-
-        # If the search fails to match the individual strings "foo" and "bar",
-        # it will be tried against the combined string "foo bar".
-        f = tree.find_all("gar", class_=re.compile("o b"))
-        self.assertSelects(f, ["Found it"])
-
-    def test_find_all_with_non_dictionary_for_attrs_finds_by_class(self):
-        soup = self.soup("Found it")
-
-        self.assertSelects(soup.find_all("a", re.compile("ba")), ["Found it"])
-
-        def big_attribute_value(value):
-            return len(value) > 3
-
-        self.assertSelects(soup.find_all("a", big_attribute_value), [])
-
-        def small_attribute_value(value):
-            return len(value) <= 3
-
-        self.assertSelects(
-            soup.find_all("a", small_attribute_value), ["Found it"])
-
-    def test_find_all_with_string_for_attrs_finds_multiple_classes(self):
-        soup = self.soup('')
-        a, a2 = soup.find_all("a")
-        self.assertEqual([a, a2], soup.find_all("a", "foo"))
-        self.assertEqual([a], soup.find_all("a", "bar"))
-
-        # If you specify the class as a string that contains a
-        # space, only that specific value will be found.
-        self.assertEqual([a], soup.find_all("a", class_="foo bar"))
-        self.assertEqual([a], soup.find_all("a", "foo bar"))
-        self.assertEqual([], soup.find_all("a", "bar foo"))
-
-    def test_find_all_by_attribute_soupstrainer(self):
-        tree = self.soup("""
-                         Match.
-                         Non-match.""")
-
-        strainer = SoupStrainer(attrs={'id' : 'first'})
-        self.assertSelects(tree.find_all(strainer), ['Match.'])
-
-    def test_find_all_with_missing_attribute(self):
-        # You can pass in None as the value of an attribute to find_all.
-        # This will match tags that do not have that attribute set.
-        tree = self.soup("""ID present.
-                            No ID present.
-                            ID is empty.""")
-        self.assertSelects(tree.find_all('a', id=None), ["No ID present."])
-
-    def test_find_all_with_defined_attribute(self):
-        # You can pass in None as the value of an attribute to find_all.
-        # This will match tags that have that attribute set to any value.
-        tree = self.soup("""ID present.
-                            No ID present.
-                            ID is empty.""")
-        self.assertSelects(
-            tree.find_all(id=True), ["ID present.", "ID is empty."])
-
-    def test_find_all_with_numeric_attribute(self):
-        # If you search for a number, it's treated as a string.
-        tree = self.soup("""Unquoted attribute.
-                            Quoted attribute.""")
-
-        expected = ["Unquoted attribute.", "Quoted attribute."]
-        self.assertSelects(tree.find_all(id=1), expected)
-        self.assertSelects(tree.find_all(id="1"), expected)
-
-    def test_find_all_with_list_attribute_values(self):
-        # You can pass a list of attribute values instead of just one,
-        # and you'll get tags that match any of the values.
-        tree = self.soup("""1
-                            2
-                            3
-                            No ID.""")
-        self.assertSelects(tree.find_all(id=["1", "3", "4"]),
-                           ["1", "3"])
-
-    def test_find_all_with_regular_expression_attribute_value(self):
-        # You can pass a regular expression as an attribute value, and
-        # you'll get tags whose values for that attribute match the
-        # regular expression.
-        tree = self.soup("""One a.
-                            Two as.
-                            Mixed as and bs.
-                            One b.
-                            No ID.""")
-
-        self.assertSelects(tree.find_all(id=re.compile("^a+$")),
-                           ["One a.", "Two as."])
-
-    def test_find_by_name_and_containing_string(self):
-        soup = self.soup("foobarfoo")
-        a = soup.a
-
-        self.assertEqual([a], soup.find_all("a", text="foo"))
-        self.assertEqual([], soup.find_all("a", text="bar"))
-        self.assertEqual([], soup.find_all("a", text="bar"))
-
-    def test_find_by_name_and_containing_string_when_string_is_buried(self):
-        soup = self.soup("foofoo")
-        self.assertEqual(soup.find_all("a"), soup.find_all("a", text="foo"))
-
-    def test_find_by_attribute_and_containing_string(self):
-        soup = self.soup('foofoo')
-        a = soup.a
-
-        self.assertEqual([a], soup.find_all(id=2, text="foo"))
-        self.assertEqual([], soup.find_all(id=1, text="bar"))
-
-
-
-
-class TestIndex(TreeTest):
-    """Test Tag.index"""
-    def test_index(self):
-        tree = self.soup("""

-                            Identical
-                            Not identical
-                            Identical
-
-                            Identical with child
-                            Also not identical
-                            Identical with child
-                            
""")
-        div = tree.div
-        for i, element in enumerate(div.contents):
-            self.assertEqual(i, div.index(element))
-        self.assertRaises(ValueError, tree.index, 1)
-
-
-class TestParentOperations(TreeTest):
-    """Test navigation and searching through an element's parents."""
-
-    def setUp(self):
-        super(TestParentOperations, self).setUp()
-        self.tree = self.soup('''
    
-                                 
      
-                                  
        
-                                   
          
-                                    Start here
-                                   
        
-                                  
      ''')
-        self.start = self.tree.b
-
-
-    def test_parent(self):
-        self.assertEqual(self.start.parent['id'], 'bottom')
-        self.assertEqual(self.start.parent.parent['id'], 'middle')
-        self.assertEqual(self.start.parent.parent.parent['id'], 'top')
-
-    def test_parent_of_top_tag_is_soup_object(self):
-        top_tag = self.tree.contents[0]
-        self.assertEqual(top_tag.parent, self.tree)
-
-    def test_soup_object_has_no_parent(self):
-        self.assertEqual(None, self.tree.parent)
-
-    def test_find_parents(self):
-        self.assertSelectsIDs(
-            self.start.find_parents('ul'), ['bottom', 'middle', 'top'])
-        self.assertSelectsIDs(
-            self.start.find_parents('ul', id="middle"), ['middle'])
-
-    def test_find_parent(self):
-        self.assertEqual(self.start.find_parent('ul')['id'], 'bottom')
-        self.assertEqual(self.start.find_parent('ul', id='top')['id'], 'top')
-
-    def test_parent_of_text_element(self):
-        text = self.tree.find(text="Start here")
-        self.assertEqual(text.parent.name, 'b')
-
-    def test_text_element_find_parent(self):
-        text = self.tree.find(text="Start here")
-        self.assertEqual(text.find_parent('ul')['id'], 'bottom')
-
-    def test_parent_generator(self):
-        parents = [parent['id'] for parent in self.start.parents
-                   if parent is not None and 'id' in parent.attrs]
-        self.assertEqual(parents, ['bottom', 'middle', 'top'])
-
-
-class ProximityTest(TreeTest):
-
-    def setUp(self):
-        super(TreeTest, self).setUp()
-        self.tree = self.soup(
-            'OneTwoThree')
-
-
-class TestNextOperations(ProximityTest):
-
-    def setUp(self):
-        super(TestNextOperations, self).setUp()
-        self.start = self.tree.b
-
-    def test_next(self):
-        self.assertEqual(self.start.next_element, "One")
-        self.assertEqual(self.start.next_element.next_element['id'], "2")
-
-    def test_next_of_last_item_is_none(self):
-        last = self.tree.find(text="Three")
-        self.assertEqual(last.next_element, None)
-
-    def test_next_of_root_is_none(self):
-        # The document root is outside the next/previous chain.
-        self.assertEqual(self.tree.next_element, None)
-
-    def test_find_all_next(self):
-        self.assertSelects(self.start.find_all_next('b'), ["Two", "Three"])
-        self.start.find_all_next(id=3)
-        self.assertSelects(self.start.find_all_next(id=3), ["Three"])
-
-    def test_find_next(self):
-        self.assertEqual(self.start.find_next('b')['id'], '2')
-        self.assertEqual(self.start.find_next(text="Three"), "Three")
-
-    def test_find_next_for_text_element(self):
-        text = self.tree.find(text="One")
-        self.assertEqual(text.find_next("b").string, "Two")
-        self.assertSelects(text.find_all_next("b"), ["Two", "Three"])
-
-    def test_next_generator(self):
-        start = self.tree.find(text="Two")
-        successors = [node for node in start.next_elements]
-        # There are two successors: the final  tag and its text contents.
-        tag, contents = successors
-        self.assertEqual(tag['id'], '3')
-        self.assertEqual(contents, "Three")
-
-class TestPreviousOperations(ProximityTest):
-
-    def setUp(self):
-        super(TestPreviousOperations, self).setUp()
-        self.end = self.tree.find(text="Three")
-
-    def test_previous(self):
-        self.assertEqual(self.end.previous_element['id'], "3")
-        self.assertEqual(self.end.previous_element.previous_element, "Two")
-
-    def test_previous_of_first_item_is_none(self):
-        first = self.tree.find('html')
-        self.assertEqual(first.previous_element, None)
-
-    def test_previous_of_root_is_none(self):
-        # The document root is outside the next/previous chain.
-        # XXX This is broken!
-        #self.assertEqual(self.tree.previous_element, None)
-        pass
-
-    def test_find_all_previous(self):
-        # The  tag containing the "Three" node is the predecessor
-        # of the "Three" node itself, which is why "Three" shows up
-        # here.
-        self.assertSelects(
-            self.end.find_all_previous('b'), ["Three", "Two", "One"])
-        self.assertSelects(self.end.find_all_previous(id=1), ["One"])
-
-    def test_find_previous(self):
-        self.assertEqual(self.end.find_previous('b')['id'], '3')
-        self.assertEqual(self.end.find_previous(text="One"), "One")
-
-    def test_find_previous_for_text_element(self):
-        text = self.tree.find(text="Three")
-        self.assertEqual(text.find_previous("b").string, "Three")
-        self.assertSelects(
-            text.find_all_previous("b"), ["Three", "Two", "One"])
-
-    def test_previous_generator(self):
-        start = self.tree.find(text="One")
-        predecessors = [node for node in start.previous_elements]
-
-        # There are four predecessors: the  tag containing "One"
-        # the  tag, the  tag, and the  tag.
-        b, body, head, html = predecessors
-        self.assertEqual(b['id'], '1')
-        self.assertEqual(body.name, "body")
-        self.assertEqual(head.name, "head")
-        self.assertEqual(html.name, "html")
-
-
-class SiblingTest(TreeTest):
-
-    def setUp(self):
-        super(SiblingTest, self).setUp()
-        markup = '''
-                    
-                     
-                    
-                    
-                     
-                    
-                    
-                     
-                    
-                    
-                    '''
-        # All that whitespace looks good but makes the tests more
-        # difficult. Get rid of it.
-        markup = re.compile("\n\s*").sub("", markup)
-        self.tree = self.soup(markup)
-
-
-class TestNextSibling(SiblingTest):
-
-    def setUp(self):
-        super(TestNextSibling, self).setUp()
-        self.start = self.tree.find(id="1")
-
-    def test_next_sibling_of_root_is_none(self):
-        self.assertEqual(self.tree.next_sibling, None)
-
-    def test_next_sibling(self):
-        self.assertEqual(self.start.next_sibling['id'], '2')
-        self.assertEqual(self.start.next_sibling.next_sibling['id'], '3')
-
-        # Note the difference between next_sibling and next_element.
-        self.assertEqual(self.start.next_element['id'], '1.1')
-
-    def test_next_sibling_may_not_exist(self):
-        self.assertEqual(self.tree.html.next_sibling, None)
-
-        nested_span = self.tree.find(id="1.1")
-        self.assertEqual(nested_span.next_sibling, None)
-
-        last_span = self.tree.find(id="4")
-        self.assertEqual(last_span.next_sibling, None)
-
-    def test_find_next_sibling(self):
-        self.assertEqual(self.start.find_next_sibling('span')['id'], '2')
-
-    def test_next_siblings(self):
-        self.assertSelectsIDs(self.start.find_next_siblings("span"),
-                              ['2', '3', '4'])
-
-        self.assertSelectsIDs(self.start.find_next_siblings(id='3'), ['3'])
-
-    def test_next_sibling_for_text_element(self):
-        soup = self.soup("Foobarbaz")
-        start = soup.find(text="Foo")
-        self.assertEqual(start.next_sibling.name, 'b')
-        self.assertEqual(start.next_sibling.next_sibling, 'baz')
-
-        self.assertSelects(start.find_next_siblings('b'), ['bar'])
-        self.assertEqual(start.find_next_sibling(text="baz"), "baz")
-        self.assertEqual(start.find_next_sibling(text="nonesuch"), None)
-
-
-class TestPreviousSibling(SiblingTest):
-
-    def setUp(self):
-        super(TestPreviousSibling, self).setUp()
-        self.end = self.tree.find(id="4")
-
-    def test_previous_sibling_of_root_is_none(self):
-        self.assertEqual(self.tree.previous_sibling, None)
-
-    def test_previous_sibling(self):
-        self.assertEqual(self.end.previous_sibling['id'], '3')
-        self.assertEqual(self.end.previous_sibling.previous_sibling['id'], '2')
-
-        # Note the difference between previous_sibling and previous_element.
-        self.assertEqual(self.end.previous_element['id'], '3.1')
-
-    def test_previous_sibling_may_not_exist(self):
-        self.assertEqual(self.tree.html.previous_sibling, None)
-
-        nested_span = self.tree.find(id="1.1")
-        self.assertEqual(nested_span.previous_sibling, None)
-
-        first_span = self.tree.find(id="1")
-        self.assertEqual(first_span.previous_sibling, None)
-
-    def test_find_previous_sibling(self):
-        self.assertEqual(self.end.find_previous_sibling('span')['id'], '3')
-
-    def test_previous_siblings(self):
-        self.assertSelectsIDs(self.end.find_previous_siblings("span"),
-                              ['3', '2', '1'])
-
-        self.assertSelectsIDs(self.end.find_previous_siblings(id='1'), ['1'])
-
-    def test_previous_sibling_for_text_element(self):
-        soup = self.soup("Foobarbaz")
-        start = soup.find(text="baz")
-        self.assertEqual(start.previous_sibling.name, 'b')
-        self.assertEqual(start.previous_sibling.previous_sibling, 'Foo')
-
-        self.assertSelects(start.find_previous_siblings('b'), ['bar'])
-        self.assertEqual(start.find_previous_sibling(text="Foo"), "Foo")
-        self.assertEqual(start.find_previous_sibling(text="nonesuch"), None)
-
-
-class TestTagCreation(SoupTest):
-    """Test the ability to create new tags."""
-    def test_new_tag(self):
-        soup = self.soup("")
-        new_tag = soup.new_tag("foo", bar="baz")
-        self.assertTrue(isinstance(new_tag, Tag))
-        self.assertEqual("foo", new_tag.name)
-        self.assertEqual(dict(bar="baz"), new_tag.attrs)
-        self.assertEqual(None, new_tag.parent)
-
-    def test_tag_inherits_self_closing_rules_from_builder(self):
-        if XML_BUILDER_PRESENT:
-            xml_soup = BeautifulSoup("", "lxml-xml")
-            xml_br = xml_soup.new_tag("br")
-            xml_p = xml_soup.new_tag("p")
-
-            # Both the 
       and 
       tag are empty-element, just because
-            # they have no contents.
-            self.assertEqual(b"
      ", xml_br.encode())
-            self.assertEqual(b"
      ", xml_p.encode())
-
-        html_soup = BeautifulSoup("", "html.parser")
-        html_br = html_soup.new_tag("br")
-        html_p = html_soup.new_tag("p")
-
-        # The HTML builder users HTML's rules about which tags are
-        # empty-element tags, and the new tags reflect these rules.
-        self.assertEqual(b"
      ", html_br.encode())
-        self.assertEqual(b"
      ", html_p.encode())
-
-    def test_new_string_creates_navigablestring(self):
-        soup = self.soup("")
-        s = soup.new_string("foo")
-        self.assertEqual("foo", s)
-        self.assertTrue(isinstance(s, NavigableString))
-
-    def test_new_string_can_create_navigablestring_subclass(self):
-        soup = self.soup("")
-        s = soup.new_string("foo", Comment)
-        self.assertEqual("foo", s)
-        self.assertTrue(isinstance(s, Comment))
-
-class TestTreeModification(SoupTest):
-
-    def test_attribute_modification(self):
-        soup = self.soup('')
-        soup.a['id'] = 2
-        self.assertEqual(soup.decode(), self.document_for(''))
-        del(soup.a['id'])
-        self.assertEqual(soup.decode(), self.document_for(''))
-        soup.a['id2'] = 'foo'
-        self.assertEqual(soup.decode(), self.document_for(''))
-
-    def test_new_tag_creation(self):
-        builder = builder_registry.lookup('html')()
-        soup = self.soup("", builder=builder)
-        a = Tag(soup, builder, 'a')
-        ol = Tag(soup, builder, 'ol')
-        a['href'] = 'http://foo.com/'
-        soup.body.insert(0, a)
-        soup.body.insert(1, ol)
-        self.assertEqual(
-            soup.body.encode(),
-            b'
        ')
-
-    def test_append_to_contents_moves_tag(self):
-        doc = """
        Don't leave me here.
        
-                
        Don\'t leave!
        """
-        soup = self.soup(doc)
-        second_para = soup.find(id='2')
-        bold = soup.b
-
-        # Move the  tag to the end of the second paragraph.
-        soup.find(id='2').append(soup.b)
-
-        # The  tag is now a child of the second paragraph.
-        self.assertEqual(bold.parent, second_para)
-
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                '
        Don\'t leave me .
        \n'
-                '
        Don\'t leave!here
        '))
-
-    def test_replace_with_returns_thing_that_was_replaced(self):
-        text = ""
-        soup = self.soup(text)
-        a = soup.a
-        new_a = a.replace_with(soup.c)
-        self.assertEqual(a, new_a)
-
-    def test_unwrap_returns_thing_that_was_replaced(self):
-        text = ""
-        soup = self.soup(text)
-        a = soup.a
-        new_a = a.unwrap()
-        self.assertEqual(a, new_a)
-
-    def test_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parent(self):
-        soup = self.soup("FooBar")
-        a = soup.a
-        a.extract()
-        self.assertEqual(None, a.parent)
-        self.assertRaises(ValueError, a.unwrap)
-        self.assertRaises(ValueError, a.replace_with, soup.c)
-
-    def test_replace_tag_with_itself(self):
-        text = "Foo"
-        soup = self.soup(text)
-        c = soup.c
-        soup.c.replace_with(c)
-        self.assertEqual(soup.decode(), self.document_for(text))
-
-    def test_replace_tag_with_its_parent_raises_exception(self):
-        text = ""
-        soup = self.soup(text)
-        self.assertRaises(ValueError, soup.b.replace_with, soup.a)
-
-    def test_insert_tag_into_itself_raises_exception(self):
-        text = ""
-        soup = self.soup(text)
-        self.assertRaises(ValueError, soup.a.insert, 0, soup.a)
-
-    def test_replace_with_maintains_next_element_throughout(self):
-        soup = self.soup('
        onethree
        ')
-        a = soup.a
-        b = a.contents[0]
-        # Make it so the  tag has two text children.
-        a.insert(1, "two")
-
-        # Now replace each one with the empty string.
-        left, right = a.contents
-        left.replaceWith('')
-        right.replaceWith('')
-
-        # The  tag is still connected to the tree.
-        self.assertEqual("three", soup.b.string)
-
-    def test_replace_final_node(self):
-        soup = self.soup("Argh!")
-        soup.find(text="Argh!").replace_with("Hooray!")
-        new_text = soup.find(text="Hooray!")
-        b = soup.b
-        self.assertEqual(new_text.previous_element, b)
-        self.assertEqual(new_text.parent, b)
-        self.assertEqual(new_text.previous_element.next_element, new_text)
-        self.assertEqual(new_text.next_element, None)
-
-    def test_consecutive_text_nodes(self):
-        # A builder should never create two consecutive text nodes,
-        # but if you insert one next to another, Beautiful Soup will
-        # handle it correctly.
-        soup = self.soup("Argh!")
-        soup.b.insert(1, "Hooray!")
-
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                "Argh!Hooray!"))
-
-        new_text = soup.find(text="Hooray!")
-        self.assertEqual(new_text.previous_element, "Argh!")
-        self.assertEqual(new_text.previous_element.next_element, new_text)
-
-        self.assertEqual(new_text.previous_sibling, "Argh!")
-        self.assertEqual(new_text.previous_sibling.next_sibling, new_text)
-
-        self.assertEqual(new_text.next_sibling, None)
-        self.assertEqual(new_text.next_element, soup.c)
-
-    def test_insert_string(self):
-        soup = self.soup("")
-        soup.a.insert(0, "bar")
-        soup.a.insert(0, "foo")
-        # The string were added to the tag.
-        self.assertEqual(["foo", "bar"], soup.a.contents)
-        # And they were converted to NavigableStrings.
-        self.assertEqual(soup.a.contents[0].next_element, "bar")
-
-    def test_insert_tag(self):
-        builder = self.default_builder
-        soup = self.soup(
-            "Findlady!", builder=builder)
-        magic_tag = Tag(soup, builder, 'magictag')
-        magic_tag.insert(0, "the")
-        soup.a.insert(1, magic_tag)
-
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                "Findthelady!"))
-
-        # Make sure all the relationships are hooked up correctly.
-        b_tag = soup.b
-        self.assertEqual(b_tag.next_sibling, magic_tag)
-        self.assertEqual(magic_tag.previous_sibling, b_tag)
-
-        find = b_tag.find(text="Find")
-        self.assertEqual(find.next_element, magic_tag)
-        self.assertEqual(magic_tag.previous_element, find)
-
-        c_tag = soup.c
-        self.assertEqual(magic_tag.next_sibling, c_tag)
-        self.assertEqual(c_tag.previous_sibling, magic_tag)
-
-        the = magic_tag.find(text="the")
-        self.assertEqual(the.parent, magic_tag)
-        self.assertEqual(the.next_element, c_tag)
-        self.assertEqual(c_tag.previous_element, the)
-
-    def test_append_child_thats_already_at_the_end(self):
-        data = ""
-        soup = self.soup(data)
-        soup.a.append(soup.b)
-        self.assertEqual(data, soup.decode())
-
-    def test_move_tag_to_beginning_of_parent(self):
-        data = ""
-        soup = self.soup(data)
-        soup.a.insert(0, soup.d)
-        self.assertEqual("", soup.decode())
-
-    def test_insert_works_on_empty_element_tag(self):
-        # This is a little strange, since most HTML parsers don't allow
-        # markup like this to come through. But in general, we don't
-        # know what the parser would or wouldn't have allowed, so
-        # I'm letting this succeed for now.
-        soup = self.soup("
        ")
-        soup.br.insert(1, "Contents")
-        self.assertEqual(str(soup.br), "
        Contents
        ")
-
-    def test_insert_before(self):
-        soup = self.soup("foobar")
-        soup.b.insert_before("BAZ")
-        soup.a.insert_before("QUUX")
-        self.assertEqual(
-            soup.decode(), self.document_for("QUUXfooBAZbar"))
-
-        soup.a.insert_before(soup.b)
-        self.assertEqual(
-            soup.decode(), self.document_for("QUUXbarfooBAZ"))
-
-    def test_insert_after(self):
-        soup = self.soup("foobar")
-        soup.b.insert_after("BAZ")
-        soup.a.insert_after("QUUX")
-        self.assertEqual(
-            soup.decode(), self.document_for("fooQUUXbarBAZ"))
-        soup.b.insert_after(soup.a)
-        self.assertEqual(
-            soup.decode(), self.document_for("QUUXbarfooBAZ"))
-
-    def test_insert_after_raises_exception_if_after_has_no_meaning(self):
-        soup = self.soup("")
-        tag = soup.new_tag("a")
-        string = soup.new_string("")
-        self.assertRaises(ValueError, string.insert_after, tag)
-        self.assertRaises(NotImplementedError, soup.insert_after, tag)
-        self.assertRaises(ValueError, tag.insert_after, tag)
-
-    def test_insert_before_raises_notimplementederror_if_before_has_no_meaning(self):
-        soup = self.soup("")
-        tag = soup.new_tag("a")
-        string = soup.new_string("")
-        self.assertRaises(ValueError, string.insert_before, tag)
-        self.assertRaises(NotImplementedError, soup.insert_before, tag)
-        self.assertRaises(ValueError, tag.insert_before, tag)
-
-    def test_replace_with(self):
-        soup = self.soup(
-                "
        There's no business like show business
        ")
-        no, show = soup.find_all('b')
-        show.replace_with(no)
-        self.assertEqual(
-            soup.decode(),
-            self.document_for(
-                "
        There's  business like no business
        "))
-
-        self.assertEqual(show.parent, None)
-        self.assertEqual(no.parent, soup.p)
-        self.assertEqual(no.next_element, "no")
-        self.assertEqual(no.next_sibling, " business")
-
-    def test_replace_first_child(self):
-        data = ""
-        soup = self.soup(data)
-        soup.b.replace_with(soup.c)
-        self.assertEqual("", soup.decode())
-
-    def test_replace_last_child(self):
-        data = ""
-        soup = self.soup(data)
-        soup.c.replace_with(soup.b)
-        self.assertEqual("", soup.decode())
-
-    def test_nested_tag_replace_with(self):
-        soup = self.soup(
-            """Wereservetherighttorefuseservice""")
-
-        # Replace the entire  tag and its contents ("reserve the
-        # right") with the  tag ("refuse").
-        remove_tag = soup.b
-        move_tag = soup.f
-        remove_tag.replace_with(move_tag)
-
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                "Werefusetoservice"))
-
-        # The  tag is now an orphan.
-        self.assertEqual(remove_tag.parent, None)
-        self.assertEqual(remove_tag.find(text="right").next_element, None)
-        self.assertEqual(remove_tag.previous_element, None)
-        self.assertEqual(remove_tag.next_sibling, None)
-        self.assertEqual(remove_tag.previous_sibling, None)
-
-        # The  tag is now connected to the  tag.
-        self.assertEqual(move_tag.parent, soup.a)
-        self.assertEqual(move_tag.previous_element, "We")
-        self.assertEqual(move_tag.next_element.next_element, soup.e)
-        self.assertEqual(move_tag.next_sibling, None)
-
-        # The gap where the  tag used to be has been mended, and
-        # the word "to" is now connected to the  tag.
-        to_text = soup.find(text="to")
-        g_tag = soup.g
-        self.assertEqual(to_text.next_element, g_tag)
-        self.assertEqual(to_text.next_sibling, g_tag)
-        self.assertEqual(g_tag.previous_element, to_text)
-        self.assertEqual(g_tag.previous_sibling, to_text)
-
-    def test_unwrap(self):
-        tree = self.soup("""
-            
        Unneeded formatting is unneeded
        
-            """)
-        tree.em.unwrap()
-        self.assertEqual(tree.em, None)
-        self.assertEqual(tree.p.text, "Unneeded formatting is unneeded")
-
-    def test_wrap(self):
-        soup = self.soup("I wish I was bold.")
-        value = soup.string.wrap(soup.new_tag("b"))
-        self.assertEqual(value.decode(), "I wish I was bold.")
-        self.assertEqual(
-            soup.decode(), self.document_for("I wish I was bold."))
-
-    def test_wrap_extracts_tag_from_elsewhere(self):
-        soup = self.soup("I wish I was bold.")
-        soup.b.next_sibling.wrap(soup.b)
-        self.assertEqual(
-            soup.decode(), self.document_for("I wish I was bold."))
-
-    def test_wrap_puts_new_contents_at_the_end(self):
-        soup = self.soup("I like being bold.I wish I was bold.")
-        soup.b.next_sibling.wrap(soup.b)
-        self.assertEqual(2, len(soup.b.contents))
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                "I like being bold.I wish I was bold."))
-
-    def test_extract(self):
-        soup = self.soup(
-            'Some content.  More content.')
-
-        self.assertEqual(len(soup.body.contents), 3)
-        extracted = soup.find(id="nav").extract()
-
-        self.assertEqual(
-            soup.decode(), "Some content.  More content.")
-        self.assertEqual(extracted.decode(), '')
-
-        # The extracted tag is now an orphan.
-        self.assertEqual(len(soup.body.contents), 2)
-        self.assertEqual(extracted.parent, None)
-        self.assertEqual(extracted.previous_element, None)
-        self.assertEqual(extracted.next_element.next_element, None)
-
-        # The gap where the extracted tag used to be has been mended.
-        content_1 = soup.find(text="Some content. ")
-        content_2 = soup.find(text=" More content.")
-        self.assertEqual(content_1.next_element, content_2)
-        self.assertEqual(content_1.next_sibling, content_2)
-        self.assertEqual(content_2.previous_element, content_1)
-        self.assertEqual(content_2.previous_sibling, content_1)
-
-    def test_extract_distinguishes_between_identical_strings(self):
-        soup = self.soup("        foobar")
-        foo_1 = soup.a.string
-        bar_1 = soup.b.string
-        foo_2 = soup.new_string("foo")
-        bar_2 = soup.new_string("bar")
-        soup.a.append(foo_2)
-        soup.b.append(bar_2)
-
-        # Now there are two identical strings in the  tag, and two
-        # in the  tag. Let's remove the first "foo" and the second
-        # "bar".
-        foo_1.extract()
-        bar_2.extract()
-        self.assertEqual(foo_2, soup.a.string)
-        self.assertEqual(bar_2, soup.b.string)
-
-    def test_extract_multiples_of_same_tag(self):
-        soup = self.soup("""
-
-
-
-
-
- 
- 
-
-
-""")
-        [soup.script.extract() for i in soup.find_all("script")]
-        self.assertEqual("\n\n\n", unicode(soup.body))
-
-
-    def test_extract_works_when_element_is_surrounded_by_identical_strings(self):
-        soup = self.soup(
- '\n'
- 'hi\n'
- '')
-        soup.find('body').extract()
-        self.assertEqual(None, soup.find('body'))
-
-
-    def test_clear(self):
-        """Tag.clear()"""
-        soup = self.soup("
        String Italicized and another
        ")
-        # clear using extract()
-        a = soup.a
-        soup.p.clear()
-        self.assertEqual(len(soup.p.contents), 0)
-        self.assertTrue(hasattr(a, "contents"))
-
-        # clear using decompose()
-        em = a.em
-        a.clear(decompose=True)
-        self.assertEqual(0, len(em.contents))
-
-    def test_string_set(self):
-        """Tag.string = 'string'"""
-        soup = self.soup(" ")
-        soup.a.string = "foo"
-        self.assertEqual(soup.a.contents, ["foo"])
-        soup.b.string = "bar"
-        self.assertEqual(soup.b.contents, ["bar"])
-
-    def test_string_set_does_not_affect_original_string(self):
-        soup = self.soup("foobar")
-        soup.b.string = soup.c.string
-        self.assertEqual(soup.a.encode(), b"barbar")
-
-    def test_set_string_preserves_class_of_string(self):
-        soup = self.soup("")
-        cdata = CData("foo")
-        soup.a.string = cdata
-        self.assertTrue(isinstance(soup.a.string, CData))
-
-class TestElementObjects(SoupTest):
-    """Test various features of element objects."""
-
-    def test_len(self):
-        """The length of an element is its number of children."""
-        soup = self.soup("123")
-
-        # The BeautifulSoup object itself contains one element: the
-        #  tag.
-        self.assertEqual(len(soup.contents), 1)
-        self.assertEqual(len(soup), 1)
-
-        # The  tag contains three elements: the text node "1", the
-        #  tag, and the text node "3".
-        self.assertEqual(len(soup.top), 3)
-        self.assertEqual(len(soup.top.contents), 3)
-
-    def test_member_access_invokes_find(self):
-        """Accessing a Python member .foo invokes find('foo')"""
-        soup = self.soup('')
-        self.assertEqual(soup.b, soup.find('b'))
-        self.assertEqual(soup.b.i, soup.find('b').find('i'))
-        self.assertEqual(soup.a, None)
-
-    def test_deprecated_member_access(self):
-        soup = self.soup('')
-        with warnings.catch_warnings(record=True) as w:
-            tag = soup.bTag
-        self.assertEqual(soup.b, tag)
-        self.assertEqual(
-            '.bTag is deprecated, use .find("b") instead.',
-            str(w[0].message))
-
-    def test_has_attr(self):
-        """has_attr() checks for the presence of an attribute.
-
-        Please note note: has_attr() is different from
-        __in__. has_attr() checks the tag's attributes and __in__
-        checks the tag's chidlren.
-        """
-        soup = self.soup("")
-        self.assertTrue(soup.foo.has_attr('attr'))
-        self.assertFalse(soup.foo.has_attr('attr2'))
-
-
-    def test_attributes_come_out_in_alphabetical_order(self):
-        markup = ''
-        self.assertSoupEquals(markup, '')
-
-    def test_string(self):
-        # A tag that contains only a text node makes that node
-        # available as .string.
-        soup = self.soup("foo")
-        self.assertEqual(soup.b.string, 'foo')
-
-    def test_empty_tag_has_no_string(self):
-        # A tag with no children has no .stirng.
-        soup = self.soup("")
-        self.assertEqual(soup.b.string, None)
-
-    def test_tag_with_multiple_children_has_no_string(self):
-        # A tag with no children has no .string.
-        soup = self.soup("foo")
-        self.assertEqual(soup.b.string, None)
-
-        soup = self.soup("foobar        ")
-        self.assertEqual(soup.b.string, None)
-
-        # Even if all the children are strings, due to trickery,
-        # it won't work--but this would be a good optimization.
-        soup = self.soup("foo        ")
-        soup.a.insert(1, "bar")
-        self.assertEqual(soup.a.string, None)
-
-    def test_tag_with_recursive_string_has_string(self):
-        # A tag with a single child which has a .string inherits that
-        # .string.
-        soup = self.soup("foo")
-        self.assertEqual(soup.a.string, "foo")
-        self.assertEqual(soup.string, "foo")
-
-    def test_lack_of_string(self):
-        """Only a tag containing a single text node has a .string."""
-        soup = self.soup("feo")
-        self.assertFalse(soup.b.string)
-
-        soup = self.soup("")
-        self.assertFalse(soup.b.string)
-
-    def test_all_text(self):
-        """Tag.text and Tag.get_text(sep=u"") -> all child text, concatenated"""
-        soup = self.soup("ar    t ")
-        self.assertEqual(soup.a.text, "ar  t ")
-        self.assertEqual(soup.a.get_text(strip=True), "art")
-        self.assertEqual(soup.a.get_text(","), "a,r, , t ")
-        self.assertEqual(soup.a.get_text(",", strip=True), "a,r,t")
-
-    def test_get_text_ignores_comments(self):
-        soup = self.soup("foobar")
-        self.assertEqual(soup.get_text(), "foobar")
-
-        self.assertEqual(
-            soup.get_text(types=(NavigableString, Comment)), "fooIGNOREbar")
-        self.assertEqual(
-            soup.get_text(types=None), "fooIGNOREbar")
-
-    def test_all_strings_ignores_comments(self):
-        soup = self.soup("foobar")
-        self.assertEqual(['foo', 'bar'], list(soup.strings))
-
-class TestCDAtaListAttributes(SoupTest):
-
-    """Testing cdata-list attributes like 'class'.
-    """
-    def test_single_value_becomes_list(self):
-        soup = self.soup("")
-        self.assertEqual(["foo"],soup.a['class'])
-
-    def test_multiple_values_becomes_list(self):
-        soup = self.soup("")
-        self.assertEqual(["foo", "bar"], soup.a['class'])
-
-    def test_multiple_values_separated_by_weird_whitespace(self):
-        soup = self.soup("")
-        self.assertEqual(["foo", "bar", "baz"],soup.a['class'])
-
-    def test_attributes_joined_into_string_on_output(self):
-        soup = self.soup("")
-        self.assertEqual(b'', soup.a.encode())
-
-    def test_accept_charset(self):
-        soup = self.soup('
        ')
-        self.assertEqual(['ISO-8859-1', 'UTF-8'], soup.form['accept-charset'])
-
-    def test_cdata_attribute_applying_only_to_one_tag(self):
-        data = ''
-        soup = self.soup(data)
-        # We saw in another test that accept-charset is a cdata-list
-        # attribute for the  tag. But it's not a cdata-list
-        # attribute for any other tag.
-        self.assertEqual('ISO-8859-1 UTF-8', soup.a['accept-charset'])
-
-    def test_string_has_immutable_name_property(self):
-        string = self.soup("s").string
-        self.assertEqual(None, string.name)
-        def t():
-            string.name = 'foo'
-        self.assertRaises(AttributeError, t)
-
-class TestPersistence(SoupTest):
-    "Testing features like pickle and deepcopy."
-
-    def setUp(self):
-        super(TestPersistence, self).setUp()
-        self.page = """
-
-
-
-Beautiful Soup: We called him Tortoise because he taught us.
-
-
-
-
-
-
-foo
-bar
-
-"""
-        self.tree = self.soup(self.page)
-
-    def test_pickle_and_unpickle_identity(self):
-        # Pickling a tree, then unpickling it, yields a tree identical
-        # to the original.
-        dumped = pickle.dumps(self.tree, 2)
-        loaded = pickle.loads(dumped)
-        self.assertEqual(loaded.__class__, BeautifulSoup)
-        self.assertEqual(loaded.decode(), self.tree.decode())
-
-    def test_deepcopy_identity(self):
-        # Making a deepcopy of a tree yields an identical tree.
-        copied = copy.deepcopy(self.tree)
-        self.assertEqual(copied.decode(), self.tree.decode())
-
-    def test_copy_preserves_encoding(self):
-        soup = BeautifulSoup(b'
         
        ', 'html.parser')
-        encoding = soup.original_encoding
-        copy = soup.__copy__()
-        self.assertEqual(u"
         
        ", unicode(copy))
-        self.assertEqual(encoding, copy.original_encoding)
-
-    def test_unicode_pickle(self):
-        # A tree containing Unicode characters can be pickled.
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL)
-        loaded = pickle.loads(dumped)
-        self.assertEqual(loaded.decode(), soup.decode())
-
-    def test_copy_navigablestring_is_not_attached_to_tree(self):
-        html = u"FooBar"
-        soup = self.soup(html)
-        s1 = soup.find(string="Foo")
-        s2 = copy.copy(s1)
-        self.assertEqual(s1, s2)
-        self.assertEqual(None, s2.parent)
-        self.assertEqual(None, s2.next_element)
-        self.assertNotEqual(None, s1.next_sibling)
-        self.assertEqual(None, s2.next_sibling)
-        self.assertEqual(None, s2.previous_element)
-
-    def test_copy_navigablestring_subclass_has_same_type(self):
-        html = u""
-        soup = self.soup(html)
-        s1 = soup.string
-        s2 = copy.copy(s1)
-        self.assertEqual(s1, s2)
-        self.assertTrue(isinstance(s2, Comment))
-
-    def test_copy_entire_soup(self):
-        html = u"
        FooBar
        end"
-        soup = self.soup(html)
-        soup_copy = copy.copy(soup)
-        self.assertEqual(soup, soup_copy)
-
-    def test_copy_tag_copies_contents(self):
-        html = u"
        FooBar
        end"
-        soup = self.soup(html)
-        div = soup.div
-        div_copy = copy.copy(div)
-
-        # The two tags look the same, and evaluate to equal.
-        self.assertEqual(unicode(div), unicode(div_copy))
-        self.assertEqual(div, div_copy)
-
-        # But they're not the same object.
-        self.assertFalse(div is div_copy)
-
-        # And they don't have the same relation to the parse tree. The
-        # copy is not associated with a parse tree at all.
-        self.assertEqual(None, div_copy.parent)
-        self.assertEqual(None, div_copy.previous_element)
-        self.assertEqual(None, div_copy.find(string='Bar').next_element)
-        self.assertNotEqual(None, div.find(string='Bar').next_element)
-
-class TestSubstitutions(SoupTest):
-
-    def test_default_formatter_is_minimal(self):
-        markup = u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter="minimal")
-        # The < is converted back into < but the e-with-acute is left alone.
-        self.assertEqual(
-            decoded,
-            self.document_for(
-                u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"))
-
-    def test_formatter_html(self):
-        markup = u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter="html")
-        self.assertEqual(
-            decoded,
-            self.document_for("<<Sacré bleu!>>"))
-
-    def test_formatter_minimal(self):
-        markup = u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter="minimal")
-        # The < is converted back into < but the e-with-acute is left alone.
-        self.assertEqual(
-            decoded,
-            self.document_for(
-                u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"))
-
-    def test_formatter_null(self):
-        markup = u"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter=None)
-        # Neither the angle brackets nor the e-with-acute are converted.
-        # This is not valid HTML, but it's what the user wanted.
-        self.assertEqual(decoded,
-                          self.document_for(u"<>"))
-
-    def test_formatter_custom(self):
-        markup = u"<foo>bar"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter = lambda x: x.upper())
-        # Instead of normal entity conversion code, the custom
-        # callable is called on every string.
-        self.assertEqual(
-            decoded,
-            self.document_for(u"BAR"))
-
-    def test_formatter_is_run_on_attribute_values(self):
-        markup = u'e'
-        soup = self.soup(markup)
-        a = soup.a
-
-        expect_minimal = u'e'
-
-        self.assertEqual(expect_minimal, a.decode())
-        self.assertEqual(expect_minimal, a.decode(formatter="minimal"))
-
-        expect_html = u'e'
-        self.assertEqual(expect_html, a.decode(formatter="html"))
-
-        self.assertEqual(markup, a.decode(formatter=None))
-        expect_upper = u'E'
-        self.assertEqual(expect_upper, a.decode(formatter=lambda x: x.upper()))
-
-    def test_formatter_skips_script_tag_for_html_documents(self):
-        doc = """
-  
-"""
-        encoded = BeautifulSoup(doc, 'html.parser').encode()
-        self.assertTrue(b"< < hey > >" in encoded)
-
-    def test_formatter_skips_style_tag_for_html_documents(self):
-        doc = """
-  
-"""
-        encoded = BeautifulSoup(doc, 'html.parser').encode()
-        self.assertTrue(b"< < hey > >" in encoded)
-
-    def test_prettify_leaves_preformatted_text_alone(self):
-        soup = self.soup("
          foo  
          \tbar\n  \n  
          baz  ")
-        # Everything outside the 
         tag is reformatted, but everything
-        # inside is left alone.
-        self.assertEqual(
-            u'
        \n foo\n 
          \tbar\n  \n  
        \n baz\n
        ',
-            soup.div.prettify())
-
-    def test_prettify_accepts_formatter(self):
-        soup = BeautifulSoup("foo", 'html.parser')
-        pretty = soup.prettify(formatter = lambda x: x.upper())
-        self.assertTrue("FOO" in pretty)
-
-    def test_prettify_outputs_unicode_by_default(self):
-        soup = self.soup("")
-        self.assertEqual(unicode, type(soup.prettify()))
-
-    def test_prettify_can_encode_data(self):
-        soup = self.soup("")
-        self.assertEqual(bytes, type(soup.prettify("utf-8")))
-
-    def test_html_entity_substitution_off_by_default(self):
-        markup = u"Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!"
-        soup = self.soup(markup)
-        encoded = soup.b.encode("utf-8")
-        self.assertEqual(encoded, markup.encode('utf-8'))
-
-    def test_encoding_substitution(self):
-        # Here's the  tag saying that a document is
-        # encoded in Shift-JIS.
-        meta_tag = ('')
-        soup = self.soup(meta_tag)
-
-        # Parse the document, and the charset apprears unchanged.
-        self.assertEqual(soup.meta['content'], 'text/html; charset=x-sjis')
-
-        # Encode the document into some encoding, and the encoding is
-        # substituted into the meta tag.
-        utf_8 = soup.encode("utf-8")
-        self.assertTrue(b"charset=utf-8" in utf_8)
-
-        euc_jp = soup.encode("euc_jp")
-        self.assertTrue(b"charset=euc_jp" in euc_jp)
-
-        shift_jis = soup.encode("shift-jis")
-        self.assertTrue(b"charset=shift-jis" in shift_jis)
-
-        utf_16_u = soup.encode("utf-16").decode("utf-16")
-        self.assertTrue("charset=utf-16" in utf_16_u)
-
-    def test_encoding_substitution_doesnt_happen_if_tag_is_strained(self):
-        markup = ('
        foo
        ')
-
-        # Beautiful Soup used to try to rewrite the meta tag even if the
-        # meta tag got filtered out by the strainer. This test makes
-        # sure that doesn't happen.
-        strainer = SoupStrainer('pre')
-        soup = self.soup(markup, parse_only=strainer)
-        self.assertEqual(soup.contents[0].name, 'pre')
-
-class TestEncoding(SoupTest):
-    """Test the ability to encode objects into strings."""
-
-    def test_unicode_string_can_be_encoded(self):
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        self.assertEqual(soup.b.string.encode("utf-8"),
-                          u"\N{SNOWMAN}".encode("utf-8"))
-
-    def test_tag_containing_unicode_string_can_be_encoded(self):
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        self.assertEqual(
-            soup.b.encode("utf-8"), html.encode("utf-8"))
-
-    def test_encoding_substitutes_unrecognized_characters_by_default(self):
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        self.assertEqual(soup.b.encode("ascii"), b"☃")
-
-    def test_encoding_can_be_made_strict(self):
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        self.assertRaises(
-            UnicodeEncodeError, soup.encode, "ascii", errors="strict")
-
-    def test_decode_contents(self):
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        self.assertEqual(u"\N{SNOWMAN}", soup.b.decode_contents())
-
-    def test_encode_contents(self):
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        self.assertEqual(
-            u"\N{SNOWMAN}".encode("utf8"), soup.b.encode_contents(
-                encoding="utf8"))
-
-    def test_deprecated_renderContents(self):
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        self.assertEqual(
-            u"\N{SNOWMAN}".encode("utf8"), soup.b.renderContents())
-
-    def test_repr(self):
-        html = u"\N{SNOWMAN}"
-        soup = self.soup(html)
-        if PY3K:
-            self.assertEqual(html, repr(soup))
-        else:
-            self.assertEqual(b'\\u2603', repr(soup))
-
-class TestNavigableStringSubclasses(SoupTest):
-
-    def test_cdata(self):
-        # None of the current builders turn CDATA sections into CData
-        # objects, but you can create them manually.
-        soup = self.soup("")
-        cdata = CData("foo")
-        soup.insert(1, cdata)
-        self.assertEqual(str(soup), "")
-        self.assertEqual(soup.find(text="foo"), "foo")
-        self.assertEqual(soup.contents[0], "foo")
-
-    def test_cdata_is_never_formatted(self):
-        """Text inside a CData object is passed into the formatter.
-
-        But the return value is ignored.
-        """
-
-        self.count = 0
-        def increment(*args):
-            self.count += 1
-            return "BITTER FAILURE"
-
-        soup = self.soup("")
-        cdata = CData("<><><>")
-        soup.insert(1, cdata)
-        self.assertEqual(
-            b"<><>]]>", soup.encode(formatter=increment))
-        self.assertEqual(1, self.count)
-
-    def test_doctype_ends_in_newline(self):
-        # Unlike other NavigableString subclasses, a DOCTYPE always ends
-        # in a newline.
-        doctype = Doctype("foo")
-        soup = self.soup("")
-        soup.insert(1, doctype)
-        self.assertEqual(soup.encode(), b"\n")
-
-    def test_declaration(self):
-        d = Declaration("foo")
-        self.assertEqual("", d.output_ready())
-
-class TestSoupSelector(TreeTest):
-
-    HTML = """
-
-
-
-The title
-
-
-
-Hello there.
-
        
-
        
-
        An H1
        
-
        Some text
        
-
        Some more text
        
-
        An H2
        
-
        Another
        
-Bob
-
        Another H2
        
-me
-
-span1a1
-span1a2 test
-
-span2a1
-
-
-
-
        
-
-
        
-
-
-
-
-
-
-
-
-
        English
        
-
        English UK
        
-
        English US
        
-
        French
        
-
        
-
-
-"""
-
-    def setUp(self):
-        self.soup = BeautifulSoup(self.HTML, 'html.parser')
-
-    def assertSelects(self, selector, expected_ids, **kwargs):
-        el_ids = [el['id'] for el in self.soup.select(selector, **kwargs)]
-        el_ids.sort()
-        expected_ids.sort()
-        self.assertEqual(expected_ids, el_ids,
-            "Selector %s, expected [%s], got [%s]" % (
-                selector, ', '.join(expected_ids), ', '.join(el_ids)
-            )
-        )
-
-    assertSelect = assertSelects
-
-    def assertSelectMultiple(self, *tests):
-        for selector, expected_ids in tests:
-            self.assertSelect(selector, expected_ids)
-
-    def test_one_tag_one(self):
-        els = self.soup.select('title')
-        self.assertEqual(len(els), 1)
-        self.assertEqual(els[0].name, 'title')
-        self.assertEqual(els[0].contents, [u'The title'])
-
-    def test_one_tag_many(self):
-        els = self.soup.select('div')
-        self.assertEqual(len(els), 4)
-        for div in els:
-            self.assertEqual(div.name, 'div')
-
-        el = self.soup.select_one('div')
-        self.assertEqual('main', el['id'])
-
-    def test_select_one_returns_none_if_no_match(self):
-        match = self.soup.select_one('nonexistenttag')
-        self.assertEqual(None, match)
-
-
-    def test_tag_in_tag_one(self):
-        els = self.soup.select('div div')
-        self.assertSelects('div div', ['inner', 'data1'])
-
-    def test_tag_in_tag_many(self):
-        for selector in ('html div', 'html body div', 'body div'):
-            self.assertSelects(selector, ['data1', 'main', 'inner', 'footer'])
-
-
-    def test_limit(self):
-        self.assertSelects('html div', ['main'], limit=1)
-        self.assertSelects('html body div', ['inner', 'main'], limit=2)
-        self.assertSelects('body div', ['data1', 'main', 'inner', 'footer'],
-                           limit=10)
-
-    def test_tag_no_match(self):
-        self.assertEqual(len(self.soup.select('del')), 0)
-
-    def test_invalid_tag(self):
-        self.assertRaises(ValueError, self.soup.select, 'tag%t')
-
-    def test_select_dashed_tag_ids(self):
-        self.assertSelects('custom-dashed-tag', ['dash1', 'dash2'])
-
-    def test_select_dashed_by_id(self):
-        dashed = self.soup.select('custom-dashed-tag[id=\"dash2\"]')
-        self.assertEqual(dashed[0].name, 'custom-dashed-tag')
-        self.assertEqual(dashed[0]['id'], 'dash2')
-
-    def test_dashed_tag_text(self):
-        self.assertEqual(self.soup.select('body > custom-dashed-tag')[0].text, u'Hello there.')
-
-    def test_select_dashed_matches_find_all(self):
-        self.assertEqual(self.soup.select('custom-dashed-tag'), self.soup.find_all('custom-dashed-tag'))
-
-    def test_header_tags(self):
-        self.assertSelectMultiple(
-            ('h1', ['header1']),
-            ('h2', ['header2', 'header3']),
-        )
-
-    def test_class_one(self):
-        for selector in ('.onep', 'p.onep', 'html p.onep'):
-            els = self.soup.select(selector)
-            self.assertEqual(len(els), 1)
-            self.assertEqual(els[0].name, 'p')
-            self.assertEqual(els[0]['class'], ['onep'])
-
-    def test_class_mismatched_tag(self):
-        els = self.soup.select('div.onep')
-        self.assertEqual(len(els), 0)
-
-    def test_one_id(self):
-        for selector in ('div#inner', '#inner', 'div div#inner'):
-            self.assertSelects(selector, ['inner'])
-
-    def test_bad_id(self):
-        els = self.soup.select('#doesnotexist')
-        self.assertEqual(len(els), 0)
-
-    def test_items_in_id(self):
-        els = self.soup.select('div#inner p')
-        self.assertEqual(len(els), 3)
-        for el in els:
-            self.assertEqual(el.name, 'p')
-        self.assertEqual(els[1]['class'], ['onep'])
-        self.assertFalse(els[0].has_attr('class'))
-
-    def test_a_bunch_of_emptys(self):
-        for selector in ('div#main del', 'div#main div.oops', 'div div#main'):
-            self.assertEqual(len(self.soup.select(selector)), 0)
-
-    def test_multi_class_support(self):
-        for selector in ('.class1', 'p.class1', '.class2', 'p.class2',
-            '.class3', 'p.class3', 'html p.class2', 'div#inner .class2'):
-            self.assertSelects(selector, ['pmulti'])
-
-    def test_multi_class_selection(self):
-        for selector in ('.class1.class3', '.class3.class2',
-                         '.class1.class2.class3'):
-            self.assertSelects(selector, ['pmulti'])
-
-    def test_child_selector(self):
-        self.assertSelects('.s1 > a', ['s1a1', 's1a2'])
-        self.assertSelects('.s1 > a span', ['s1a2s1'])
-
-    def test_child_selector_id(self):
-        self.assertSelects('.s1 > a#s1a2 span', ['s1a2s1'])
-
-    def test_attribute_equals(self):
-        self.assertSelectMultiple(
-            ('p[class="onep"]', ['p1']),
-            ('p[id="p1"]', ['p1']),
-            ('[class="onep"]', ['p1']),
-            ('[id="p1"]', ['p1']),
-            ('link[rel="stylesheet"]', ['l1']),
-            ('link[type="text/css"]', ['l1']),
-            ('link[href="blah.css"]', ['l1']),
-            ('link[href="no-blah.css"]', []),
-            ('[rel="stylesheet"]', ['l1']),
-            ('[type="text/css"]', ['l1']),
-            ('[href="blah.css"]', ['l1']),
-            ('[href="no-blah.css"]', []),
-            ('p[href="no-blah.css"]', []),
-            ('[href="no-blah.css"]', []),
-        )
-
-    def test_attribute_tilde(self):
-        self.assertSelectMultiple(
-            ('p[class~="class1"]', ['pmulti']),
-            ('p[class~="class2"]', ['pmulti']),
-            ('p[class~="class3"]', ['pmulti']),
-            ('[class~="class1"]', ['pmulti']),
-            ('[class~="class2"]', ['pmulti']),
-            ('[class~="class3"]', ['pmulti']),
-            ('a[rel~="friend"]', ['bob']),
-            ('a[rel~="met"]', ['bob']),
-            ('[rel~="friend"]', ['bob']),
-            ('[rel~="met"]', ['bob']),
-        )
-
-    def test_attribute_startswith(self):
-        self.assertSelectMultiple(
-            ('[rel^="style"]', ['l1']),
-            ('link[rel^="style"]', ['l1']),
-            ('notlink[rel^="notstyle"]', []),
-            ('[rel^="notstyle"]', []),
-            ('link[rel^="notstyle"]', []),
-            ('link[href^="bla"]', ['l1']),
-            ('a[href^="http://"]', ['bob', 'me']),
-            ('[href^="http://"]', ['bob', 'me']),
-            ('[id^="p"]', ['pmulti', 'p1']),
-            ('[id^="m"]', ['me', 'main']),
-            ('div[id^="m"]', ['main']),
-            ('a[id^="m"]', ['me']),
-            ('div[data-tag^="dashed"]', ['data1'])
-        )
-
-    def test_attribute_endswith(self):
-        self.assertSelectMultiple(
-            ('[href$=".css"]', ['l1']),
-            ('link[href$=".css"]', ['l1']),
-            ('link[id$="1"]', ['l1']),
-            ('[id$="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's2a1', 's1a2s1', 'dash1']),
-            ('div[id$="1"]', ['data1']),
-            ('[id$="noending"]', []),
-        )
-
-    def test_attribute_contains(self):
-        self.assertSelectMultiple(
-            # From test_attribute_startswith
-            ('[rel*="style"]', ['l1']),
-            ('link[rel*="style"]', ['l1']),
-            ('notlink[rel*="notstyle"]', []),
-            ('[rel*="notstyle"]', []),
-            ('link[rel*="notstyle"]', []),
-            ('link[href*="bla"]', ['l1']),
-            ('[href*="http://"]', ['bob', 'me']),
-            ('[id*="p"]', ['pmulti', 'p1']),
-            ('div[id*="m"]', ['main']),
-            ('a[id*="m"]', ['me']),
-            # From test_attribute_endswith
-            ('[href*=".css"]', ['l1']),
-            ('link[href*=".css"]', ['l1']),
-            ('link[id*="1"]', ['l1']),
-            ('[id*="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's1a2', 's2a1', 's1a2s1', 'dash1']),
-            ('div[id*="1"]', ['data1']),
-            ('[id*="noending"]', []),
-            # New for this test
-            ('[href*="."]', ['bob', 'me', 'l1']),
-            ('a[href*="."]', ['bob', 'me']),
-            ('link[href*="."]', ['l1']),
-            ('div[id*="n"]', ['main', 'inner']),
-            ('div[id*="nn"]', ['inner']),
-            ('div[data-tag*="edval"]', ['data1'])
-        )
-
-    def test_attribute_exact_or_hypen(self):
-        self.assertSelectMultiple(
-            ('p[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
-            ('[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
-            ('p[lang|="fr"]', ['lang-fr']),
-            ('p[lang|="gb"]', []),
-        )
-
-    def test_attribute_exists(self):
-        self.assertSelectMultiple(
-            ('[rel]', ['l1', 'bob', 'me']),
-            ('link[rel]', ['l1']),
-            ('a[rel]', ['bob', 'me']),
-            ('[lang]', ['lang-en', 'lang-en-gb', 'lang-en-us', 'lang-fr']),
-            ('p[class]', ['p1', 'pmulti']),
-            ('[blah]', []),
-            ('p[blah]', []),
-            ('div[data-tag]', ['data1'])
-        )
-
-    def test_quoted_space_in_selector_name(self):
-        html = """
        nope
        
-        
        yes
        
-        """
-        soup = BeautifulSoup(html, 'html.parser')
-        [chosen] = soup.select('div[style="display: right"]')
-        self.assertEqual("yes", chosen.string)
-
-    def test_unsupported_pseudoclass(self):
-        self.assertRaises(
-            NotImplementedError, self.soup.select, "a:no-such-pseudoclass")
-
-        self.assertRaises(
-            NotImplementedError, self.soup.select, "a:nth-of-type(a)")
-
-
-    def test_nth_of_type(self):
-        # Try to select first paragraph
-        els = self.soup.select('div#inner p:nth-of-type(1)')
-        self.assertEqual(len(els), 1)
-        self.assertEqual(els[0].string, u'Some text')
-
-        # Try to select third paragraph
-        els = self.soup.select('div#inner p:nth-of-type(3)')
-        self.assertEqual(len(els), 1)
-        self.assertEqual(els[0].string, u'Another')
-
-        # Try to select (non-existent!) fourth paragraph
-        els = self.soup.select('div#inner p:nth-of-type(4)')
-        self.assertEqual(len(els), 0)
-
-        # Pass in an invalid value.
-        self.assertRaises(
-            ValueError, self.soup.select, 'div p:nth-of-type(0)')
-
-    def test_nth_of_type_direct_descendant(self):
-        els = self.soup.select('div#inner > p:nth-of-type(1)')
-        self.assertEqual(len(els), 1)
-        self.assertEqual(els[0].string, u'Some text')
-
-    def test_id_child_selector_nth_of_type(self):
-        self.assertSelects('#inner > p:nth-of-type(2)', ['p1'])
-
-    def test_select_on_element(self):
-        # Other tests operate on the tree; this operates on an element
-        # within the tree.
-        inner = self.soup.find("div", id="main")
-        selected = inner.select("div")
-        # The 
         tag was selected. The