diff options
| -rw-r--r-- | bazarr/api.py | 135 | ||||
| -rw-r--r-- | bazarr/config.py | 14 | ||||
| -rw-r--r-- | bazarr/database.py | 34 | ||||
| -rw-r--r-- | bazarr/get_episodes.py | 29 | ||||
| -rw-r--r-- | bazarr/get_movies.py | 18 | ||||
| -rw-r--r-- | bazarr/get_providers.py | 9 | ||||
| -rw-r--r-- | bazarr/get_subtitle.py | 46 | ||||
| -rw-r--r-- | bazarr/helper.py | 4 | ||||
| -rw-r--r-- | bazarr/main.py | 2 | ||||
| -rw-r--r-- | bazarr/subsyncer.py | 39 | ||||
| -rw-r--r-- | libs/ffsubsync/_version.py | 4 | ||||
| -rw-r--r-- | libs/ffsubsync/ffsubsync.py | 30 | ||||
| -rw-r--r-- | libs/ffsubsync/speech_transformers.py | 2 | ||||
| -rw-r--r-- | libs/subliminal_patch/providers/subdivx.py | 3 | ||||
| -rw-r--r-- | libs/subliminal_patch/providers/subssabbz.py | 30 | ||||
| -rw-r--r-- | libs/subliminal_patch/providers/subsunacs.py | 41 | ||||
| -rw-r--r-- | libs/subliminal_patch/providers/yavkanet.py | 30 | ||||
| -rw-r--r-- | libs/subliminal_patch/providers/yifysubtitles.py | 22 | ||||
| -rw-r--r-- | views/_main.html | 9 | ||||
| -rw-r--r-- | views/episodes.html | 10 | ||||
| -rw-r--r-- | views/movieseditor.html | 4 | ||||
| -rw-r--r-- | views/series.html | 4 | ||||
| -rw-r--r-- | views/serieseditor.html | 6 | ||||
| -rw-r--r-- | views/settingssubtitles.html | 17 |
24 files changed, 351 insertions, 191 deletions
diff --git a/bazarr/api.py b/bazarr/api.py index c61da14bf..c28178474 100644 --- a/bazarr/api.py +++ b/bazarr/api.py @@ -18,7 +18,7 @@ from config import settings, base_url, save_settings from init import * import logging -from database import database, filter_exclusions +from database import database, get_exclusion_clause from helper import path_mappings from get_languages import language_from_alpha3, language_from_alpha2, alpha2_from_alpha3, alpha2_from_language, \ alpha3_from_language, alpha3_from_alpha2 @@ -54,8 +54,11 @@ def authenticate(actual_method): apikey_settings = settings.auth.apikey apikey_get = request.args.get('apikey') apikey_post = request.form.get('apikey') + apikey_header = None + if 'X-Api-Key' in request.headers: + apikey_header = request.headers['X-Api-Key'] - if apikey_settings in [apikey_get, apikey_post]: + if apikey_settings in [apikey_get, apikey_post, apikey_header]: return actual_method(*args, **kwargs) return abort(401, message="Unauthorized") @@ -83,8 +86,7 @@ class BadgesSeries(Resource): missing_episodes = database.execute("SELECT table_shows.tags, table_episodes.monitored, table_shows.seriesType " "FROM table_episodes INNER JOIN table_shows on table_shows.sonarrSeriesId =" " table_episodes.sonarrSeriesId WHERE missing_subtitles is not null AND " - "missing_subtitles != '[]'") - missing_episodes = filter_exclusions(missing_episodes, 'series') + "missing_subtitles != '[]'" + get_exclusion_clause('series')) missing_episodes = len(missing_episodes) result = { @@ -97,8 +99,7 @@ class BadgesMovies(Resource): @authenticate def get(self): missing_movies = database.execute("SELECT tags, monitored FROM table_movies WHERE missing_subtitles is not " - "null AND missing_subtitles != '[]'") - missing_movies = filter_exclusions(missing_movies, 'movie') + "null AND missing_subtitles != '[]'" + get_exclusion_clause('movie')) missing_movies = len(missing_movies) result = { @@ -327,8 +328,8 @@ class Series(Resource): "table_shows.seriesType FROM table_episodes INNER JOIN table_shows " "on table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId " "WHERE table_episodes.sonarrSeriesId=? AND missing_subtitles is not " - "null AND missing_subtitles != '[]'", (item['sonarrSeriesId'],)) - episodeMissingCount = filter_exclusions(episodeMissingCount, 'series') + "null AND missing_subtitles != '[]'" + + get_exclusion_clause('series'), (item['sonarrSeriesId'],)) episodeMissingCount = len(episodeMissingCount) item.update({"episodeMissingCount": episodeMissingCount}) @@ -336,8 +337,8 @@ class Series(Resource): episodeFileCount = database.execute("SELECT table_shows.tags, table_episodes.monitored, " "table_shows.seriesType FROM table_episodes INNER JOIN table_shows on " "table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE " - "table_episodes.sonarrSeriesId=?", (item['sonarrSeriesId'],)) - episodeFileCount = filter_exclusions(episodeFileCount, 'series') + "table_episodes.sonarrSeriesId=?" + get_exclusion_clause('series'), + (item['sonarrSeriesId'],)) episodeFileCount = len(episodeFileCount) item.update({"episodeFileCount": episodeFileCount}) @@ -386,6 +387,36 @@ class Series(Resource): return '', 204 +class SeriesEditor(Resource): + @authenticate + def get(self, **kwargs): + draw = request.args.get('draw') + + result = database.execute("SELECT sonarrSeriesId, title, languages, hearing_impaired, forced, audio_language " + "FROM table_shows ORDER BY sortTitle") + + row_count = len(result) + + for item in result: + # Add Datatables rowId + item.update({"DT_RowId": 'row_' + str(item['sonarrSeriesId'])}) + + # Parse audio language + item.update({"audio_language": {"name": item['audio_language'], + "code2": alpha2_from_language(item['audio_language']) or None, + "code3": alpha3_from_language(item['audio_language']) or None}}) + + # Parse desired languages + if item['languages'] and item['languages'] != 'None': + item.update({"languages": ast.literal_eval(item['languages'])}) + for i, subs in enumerate(item['languages']): + item['languages'][i] = {"name": language_from_alpha2(subs), + "code2": subs, + "code3": alpha3_from_alpha2(subs)} + + return jsonify(draw=draw, recordsTotal=row_count, recordsFiltered=row_count, data=result) + + class SeriesEditSave(Resource): @authenticate def post(self): @@ -427,7 +458,7 @@ class SeriesEditSave(Resource): list_missing_subtitles(no=seriesId, send_event=False) event_stream(type='series_editor', action='update') - event_stream(type='badges') + event_stream(type='badges_series') return '', 204 @@ -462,6 +493,11 @@ class Episodes(Resource): # Add Datatables rowId item.update({"DT_RowId": 'row_' + str(item['sonarrEpisodeId'])}) + # Parse audio language + item.update({"audio_language": {"name": item['audio_language'], + "code2": alpha2_from_language(item['audio_language']) or None, + "code3": alpha3_from_language(item['audio_language']) or None}}) + # Parse subtitles if item['subtitles']: item.update({"subtitles": ast.literal_eval(item['subtitles'])}) @@ -540,8 +576,8 @@ class EpisodesSubtitlesDownload(Resource): title = request.form.get('title') providers_list = get_providers() providers_auth = get_providers_auth() - audio_language = database.execute("SELECT audio_language FROM table_shows WHERE sonarrSeriesId=?", - (sonarrSeriesId,), only_one=True)['audio_language'] + audio_language = database.execute("SELECT audio_language FROM table_episodes WHERE sonarrEpisodeId=?", + (sonarrEpisodeId,), only_one=True)['audio_language'] try: result = download_subtitle(episodePath, language, audio_language, hi, forced, providers_list, providers_auth, sceneName, @@ -609,8 +645,8 @@ class EpisodesSubtitlesManualDownload(Resource): sonarrEpisodeId = request.form.get('sonarrEpisodeId') title = request.form.get('title') providers_auth = get_providers_auth() - audio_language = database.execute("SELECT audio_language FROM table_shows WHERE sonarrSeriesId=?", - (sonarrSeriesId,), only_one=True)['audio_language'] + audio_language = database.execute("SELECT audio_language FROM table_episodes WHERE sonarrEpisodeId=?", + (sonarrEpisodeId,), only_one=True)['audio_language'] try: result = manual_download_subtitle(episodePath, language, audio_language, hi, forced, subtitle, @@ -709,6 +745,7 @@ class EpisodesHistory(Resource): "sonarrEpisodeId, subs_id, video_path, subtitles_path FROM table_history " "WHERE sonarrEpisodeId=? ORDER BY timestamp DESC", (episodeid,)) for item in episode_history: + item['raw_timestamp'] = item['timestamp'] item['timestamp'] = "<div title='" + \ time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(item['timestamp'])) + \ "' data-toggle='tooltip' data-placement='left'>" + \ @@ -916,6 +953,36 @@ class Movies(Resource): return '', 204 +class MoviesEditor(Resource): + @authenticate + def get(self): + draw = request.args.get('draw') + + result = database.execute("SELECT radarrId, title, languages, hearing_impaired, forced, audio_language " + "FROM table_movies ORDER BY sortTitle") + + row_count = len(result) + + for item in result: + # Add Datatables rowId + item.update({"DT_RowId": 'row_' + str(item['radarrId'])}) + + # Parse audio language + item.update({"audio_language": {"name": item['audio_language'], + "code2": alpha2_from_language(item['audio_language']) or None, + "code3": alpha3_from_language(item['audio_language']) or None}}) + + # Parse desired languages + if item['languages'] and item['languages'] != 'None': + item.update({"languages": ast.literal_eval(item['languages'])}) + for i, subs in enumerate(item['languages']): + item['languages'][i] = {"name": language_from_alpha2(subs), + "code2": subs, + "code3": alpha3_from_alpha2(subs)} + + return jsonify(draw=draw, recordsTotal=row_count, recordsFiltered=row_count, data=result) + + class MoviesEditSave(Resource): @authenticate def post(self): @@ -956,7 +1023,7 @@ class MoviesEditSave(Resource): list_missing_subtitles_movies(no=radarrId, send_event=False) event_stream(type='movies_editor', action='update') - event_stream(type='badges') + event_stream(type='badges_movies') return '', 204 @@ -1163,6 +1230,7 @@ class MovieHistory(Resource): "video_path, subtitles_path FROM table_history_movie WHERE radarrId=? ORDER " "BY timestamp DESC", (radarrid,)) for item in movie_history: + item['raw_timestamp'] = item['timestamp'] item['timestamp'] = "<div title='" + \ time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(item['timestamp'])) + \ "' data-toggle='tooltip' data-placement='left'>" + \ @@ -1261,9 +1329,9 @@ class HistorySeries(Resource): "table_shows.seriesType FROM table_history INNER JOIN table_episodes on " "table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId INNER JOIN table_shows on " "table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE action IN (" + - ','.join(map(str, query_actions)) + ") AND timestamp > ? AND score is not null GROUP BY " - "table_history.video_path, table_history.language", (minimum_timestamp,)) - upgradable_episodes = filter_exclusions(upgradable_episodes, 'series') + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND score is not null" + + get_exclusion_clause('series') + " GROUP BY table_history.video_path, table_history.language", + (minimum_timestamp,)) for upgradable_episode in upgradable_episodes: if upgradable_episode['timestamp'] > minimum_timestamp: @@ -1307,6 +1375,7 @@ class HistorySeries(Resource): # Make timestamp pretty if item['timestamp']: + item['raw_timestamp'] = item['timestamp'] item['timestamp'] = pretty.date(int(item['timestamp'])) if item['path']: @@ -1364,9 +1433,8 @@ class HistoryMovies(Resource): upgradable_movies = database.execute( "SELECT video_path, MAX(timestamp) as timestamp, score, tags, monitored FROM table_history_movie " "INNER JOIN table_movies on table_movies.radarrId=table_history_movie.radarrId WHERE action IN (" + - ','.join(map(str, query_actions)) + ") AND timestamp > ? AND score is not NULL GROUP BY video_path, " - "language", (minimum_timestamp,)) - upgradable_movies = filter_exclusions(upgradable_movies, 'movie') + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND score is not NULL" + + get_exclusion_clause('movie') + " GROUP BY video_path, language", (minimum_timestamp,)) for upgradable_movie in upgradable_movies: if upgradable_movie['timestamp'] > minimum_timestamp: @@ -1407,6 +1475,7 @@ class HistoryMovies(Resource): # Make timestamp pretty if item['timestamp']: + item['raw_timestamp'] = item['timestamp'] item['timestamp'] = pretty.date(int(item['timestamp'])) if item['video_path']: @@ -1503,8 +1572,8 @@ class WantedSeries(Resource): data_count = database.execute("SELECT table_episodes.monitored, table_shows.tags, table_shows.seriesType FROM " "table_episodes INNER JOIN table_shows on table_shows.sonarrSeriesId = " - "table_episodes.sonarrSeriesId WHERE table_episodes.missing_subtitles != '[]'") - data_count = filter_exclusions(data_count, 'series') + "table_episodes.sonarrSeriesId WHERE table_episodes.missing_subtitles != '[]'" + + get_exclusion_clause('series')) row_count = len(data_count) data = database.execute("SELECT table_shows.title as seriesTitle, table_episodes.monitored, " "table_episodes.season || 'x' || table_episodes.episode as episode_number, " @@ -1513,8 +1582,8 @@ class WantedSeries(Resource): "table_episodes.sonarrEpisodeId, table_episodes.scene_name, table_shows.tags, " "table_episodes.failedAttempts, table_shows.seriesType FROM table_episodes INNER JOIN " "table_shows on table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE " - "table_episodes.missing_subtitles != '[]' ORDER BY table_episodes._rowid_ DESC") - data = filter_exclusions(data, 'series')[int(start):int(start)+int(length)] + "table_episodes.missing_subtitles != '[]'" + get_exclusion_clause('series') + + " ORDER BY table_episodes._rowid_ DESC LIMIT " + length + " OFFSET " + start) for item in data: # Parse missing subtitles @@ -1546,13 +1615,13 @@ class WantedMovies(Resource): length = request.args.get('length') or -1 draw = request.args.get('draw') - data_count = database.execute("SELECT tags, monitored FROM table_movies WHERE missing_subtitles != '[]'") - data_count = filter_exclusions(data_count, 'movie') + data_count = database.execute("SELECT tags, monitored FROM table_movies WHERE missing_subtitles != '[]'" + + get_exclusion_clause('movie')) row_count = len(data_count) data = database.execute("SELECT title, missing_subtitles, radarrId, path, hearing_impaired, sceneName, " - "failedAttempts, tags, monitored FROM table_movies WHERE missing_subtitles != '[]' " - "ORDER BY _rowid_ DESC") - data = filter_exclusions(data, 'movie')[int(start):int(start)+int(length)] + "failedAttempts, tags, monitored FROM table_movies WHERE missing_subtitles != '[]'" + + get_exclusion_clause('movie') + " ORDER BY _rowid_ DESC LIMIT " + length + " OFFSET " + + start) for item in data: # Parse missing subtitles @@ -1609,6 +1678,7 @@ class BlacklistSeries(Resource): "OFFSET ?", (length, start)) for item in data: + item['raw_timestamp'] = item['timestamp'] # Make timestamp pretty item.update({'timestamp': pretty.date(datetime.datetime.fromtimestamp(item['timestamp']))}) @@ -1686,6 +1756,7 @@ class BlacklistMovies(Resource): "OFFSET ?", (length, start)) for item in data: + item['raw_timestamp'] = item['timestamp'] # Make timestamp pretty item.update({'timestamp': pretty.date(datetime.datetime.fromtimestamp(item['timestamp']))}) @@ -1835,6 +1906,7 @@ api.add_resource(SystemStatus, '/systemstatus') api.add_resource(SystemReleases, '/systemreleases') api.add_resource(Series, '/series') +api.add_resource(SeriesEditor, '/series_editor') api.add_resource(SeriesEditSave, '/series_edit_save') api.add_resource(Episodes, '/episodes') api.add_resource(EpisodesSubtitlesDelete, '/episodes_subtitles_delete') @@ -1848,6 +1920,7 @@ api.add_resource(EpisodesHistory, '/episodes_history') api.add_resource(EpisodesTools, '/episodes_tools') api.add_resource(Movies, '/movies') +api.add_resource(MoviesEditor, '/movies_editor') api.add_resource(MoviesEditSave, '/movies_edit_save') api.add_resource(MovieSubtitlesDelete, '/movie_subtitles_delete') api.add_resource(MovieSubtitlesDownload, '/movie_subtitles_download') diff --git a/bazarr/config.py b/bazarr/config.py index b266280a2..3adddc3a4 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -162,7 +162,8 @@ defaults = { 'use_subsync_threshold': 'False', 'subsync_threshold': '90', 'use_subsync_movie_threshold': 'False', - 'subsync_movie_threshold': '70' + 'subsync_movie_threshold': '70', + 'debug': 'False' } } @@ -181,6 +182,7 @@ def save_settings(settings_items): update_schedule = False update_path_map = False configure_proxy = False + exclusion_updated = False for key, value in settings_items: # Intercept database stored settings @@ -227,6 +229,11 @@ def save_settings(settings_items): 'settings-proxy-password']: configure_proxy = True + if key in ['settings-sonarr-excluded_tags', 'settings-sonarr-only_monitored', + 'settings-sonarr-excluded_series_types', 'settings.radarr.excluded_tags', + 'settings-radarr-only_monitored']: + exclusion_updated = True + if settings_keys[0] == 'settings': settings[settings_keys[1]][settings_keys[2]] = str(value) @@ -252,6 +259,11 @@ def save_settings(settings_items): if configure_proxy: configure_proxy_func() + if exclusion_updated: + from event_handler import event_stream + event_stream(type='badges_series') + event_stream(type='badges_movies') + def url_sonarr(): if settings.sonarr.getboolean('ssl'): diff --git a/bazarr/database.py b/bazarr/database.py index 15e9c2d60..005d96345 100644 --- a/bazarr/database.py +++ b/bazarr/database.py @@ -99,6 +99,7 @@ def db_upgrade(): ['table_episodes', 'video_codec', 'text'], ['table_episodes', 'audio_codec', 'text'], ['table_episodes', 'episode_file_id', 'integer'], + ['table_episodes', 'audio_language', 'text'], ['table_movies', 'sortTitle', 'text'], ['table_movies', 'year', 'text'], ['table_movies', 'alternativeTitles', 'text'], @@ -148,28 +149,29 @@ def db_upgrade(): "provider text, subs_id text, language text)") -def filter_exclusions(dicts_list, type): +def get_exclusion_clause(type): + where_clause = '' if type == 'series': tagsList = ast.literal_eval(settings.sonarr.excluded_tags) - monitoredOnly = settings.sonarr.getboolean('only_monitored') + for tag in tagsList: + where_clause += ' AND table_shows.tags NOT LIKE "%\'' + tag + '\'%"' else: tagsList = ast.literal_eval(settings.radarr.excluded_tags) - monitoredOnly = settings.radarr.getboolean('only_monitored') + for tag in tagsList: + where_clause += ' AND table_movies.tags NOT LIKE "%\'' + tag + '\'%"' - # Filter tags - dictsList_tags_filtered = [item for item in dicts_list if set(tagsList).isdisjoint(ast.literal_eval(item['tags']))] - - # Filter monitored - if monitoredOnly: - dictsList_monitored_filtered = [item for item in dictsList_tags_filtered if item['monitored'] == 'True'] + if type == 'series': + monitoredOnly = settings.sonarr.getboolean('only_monitored') + if monitoredOnly: + where_clause += ' AND table_episodes.monitored = "True"' else: - dictsList_monitored_filtered = dictsList_tags_filtered + monitoredOnly = settings.radarr.getboolean('only_monitored') + if monitoredOnly: + where_clause += ' AND table_movies.monitored = "True"' - # Filter series type if type == 'series': - dictsList_types_filtered = [item for item in dictsList_monitored_filtered if item['seriesType'] not in - ast.literal_eval(settings.sonarr.excluded_series_types)] - else: - dictsList_types_filtered = dictsList_monitored_filtered + typesList = ast.literal_eval(settings.sonarr.excluded_series_types) + for type in typesList: + where_clause += ' AND table_shows.seriesType != "' + type + '"' - return dictsList_types_filtered + return where_clause diff --git a/bazarr/get_episodes.py b/bazarr/get_episodes.py index d9e485422..8e8bee97c 100644 --- a/bazarr/get_episodes.py +++ b/bazarr/get_episodes.py @@ -1,7 +1,7 @@ # coding=utf-8 import requests import logging -from database import database, dict_converter +from database import database, dict_converter, get_exclusion_clause from config import settings, url_sonarr from helper import path_mappings @@ -85,6 +85,13 @@ def sync_episodes(): videoCodec = None audioCodec = None + audio_language = None + if 'language' in episode['episodeFile'] and len(episode['episodeFile']['language']): + item = episode['episodeFile']['language'] + if isinstance(item, dict): + if 'name' in item: + audio_language = item['name'] + # Add episodes in sonarr to current episode list current_episodes_sonarr.append(episode['id']) @@ -101,7 +108,8 @@ def sync_episodes(): 'resolution': resolution, 'video_codec': videoCodec, 'audio_codec': audioCodec, - 'episode_file_id': episode['episodeFile']['id']}) + 'episode_file_id': episode['episodeFile']['id'], + 'audio_language': audio_language}) else: episodes_to_add.append({'sonarrSeriesId': episode['seriesId'], 'sonarrEpisodeId': episode['id'], @@ -115,7 +123,8 @@ def sync_episodes(): 'resolution': resolution, 'video_codec': videoCodec, 'audio_codec': audioCodec, - 'episode_file_id': episode['episodeFile']['id']}) + 'episode_file_id': episode['episodeFile']['id'], + 'audio_language': audio_language}) # Remove old episodes from DB removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr)) @@ -131,7 +140,7 @@ def sync_episodes(): episode_in_db_list = [] episodes_in_db = database.execute("SELECT sonarrSeriesId, sonarrEpisodeId, title, path, season, episode, " "scene_name, monitored, format, resolution, video_codec, audio_codec, " - "episode_file_id FROM table_episodes") + "episode_file_id, audio_language FROM table_episodes") for item in episodes_in_db: episode_in_db_list.append(item) @@ -171,11 +180,13 @@ def sync_episodes(): if len(altered_episodes) <= 5: logging.debug("BAZARR No more than 5 episodes were added during this sync then we'll search for subtitles.") for altered_episode in altered_episodes: - if settings.sonarr.getboolean('only_monitored'): - if altered_episode[2] == 'True': - episode_download_subtitles(altered_episode[0]) - else: - episode_download_subtitles(altered_episode[0]) + data = database.execute("SELECT table_episodes.sonarrEpisodeId, table_episodes.monitored, table_shows.tags," + " table_shows.seriesType FROM table_episodes LEFT JOIN table_shows on " + "table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId WHERE " + "sonarrEpisodeId = ?" + get_exclusion_clause('series'), (altered_episode[0],), + only_one=True) + + episode_download_subtitles(data['sonarrEpisodeId']) else: logging.debug("BAZARR More than 5 episodes were added during this sync then we wont search for subtitles right now.") diff --git a/bazarr/get_movies.py b/bazarr/get_movies.py index 39f24ce62..3fcc8a849 100644 --- a/bazarr/get_movies.py +++ b/bazarr/get_movies.py @@ -10,7 +10,7 @@ from utils import get_radarr_version from list_subtitles import store_subtitles_movie, list_missing_subtitles_movies, movies_full_scan_subtitles from get_subtitle import movies_download_subtitles -from database import database, dict_converter +from database import database, dict_converter, get_exclusion_clause def update_all_movies(): @@ -150,8 +150,12 @@ def update_movies(): if radarr_version.startswith('0'): audio_language = profile_id_to_language(movie['qualityProfileId'], audio_profiles) else: - if len(movie['movieFile']['languages']): - audio_language = movie['movieFile']['languages'][0]['name'] + if 'languages' in movie['movieFile'] and len(movie['movieFile']['languages']): + for item in movie['movieFile']['languages']: + if isinstance(item, dict): + if 'name' in item: + audio_language = item['name'] + break tags = [d['label'] for d in tagsDict if d['id'] in movie['tags']] @@ -261,11 +265,9 @@ def update_movies(): if len(altered_movies) <= 5: logging.debug("BAZARR No more than 5 movies were added during this sync then we'll search for subtitles.") for altered_movie in altered_movies: - if settings.radarr.getboolean('only_monitored'): - if altered_movie[3] == 'True': - movies_download_subtitles(altered_movie[2]) - else: - movies_download_subtitles(altered_movie[2]) + data = database.execute("SELECT * FROM table_movies WHERE radarrId = ?" + + get_exclusion_clause('movie'), (altered_movie[2],), only_one=True) + movies_download_subtitles(data['radarrId']) else: logging.debug("BAZARR More than 5 movies were added during this sync then we wont search for subtitles.") diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index a29abac35..5e65d4eea 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -30,8 +30,8 @@ hours_until_end_of_day = time_until_end_of_day().seconds // 3600 + 1 VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled, ParseResponseError, IPAddressBlocked) -VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled', requests.Timeout, - requests.ReadTimeout, socket.timeout) +VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled', requests.exceptions.Timeout, + requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, socket.timeout) PROVIDER_THROTTLE_MAP = { "default": { @@ -40,9 +40,10 @@ PROVIDER_THROTTLE_MAP = { ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"), APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"), ParseResponseError: (datetime.timedelta(hours=6), "6 hours"), - requests.Timeout: (datetime.timedelta(hours=1), "1 hour"), + requests.exceptions.Timeout: (datetime.timedelta(hours=1), "1 hour"), socket.timeout: (datetime.timedelta(hours=1), "1 hour"), - requests.ReadTimeout: (datetime.timedelta(hours=1), "1 hour"), + requests.exceptions.ConnectTimeout: (datetime.timedelta(hours=1), "1 hour"), + requests.exceptions.ReadTimeout: (datetime.timedelta(hours=1), "1 hour"), }, "opensubtitles": { TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index b30d5a911..7ef841614 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -29,11 +29,10 @@ from notifier import send_notifications, send_notifications_movie from get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool from knowit import api from subsyncer import subsync -from database import database, dict_mapper, filter_exclusions +from database import database, dict_mapper, get_exclusion_clause from analytics import track_event from locale import getpreferredencoding -import chardet def get_video(path, title, sceneName, providers=None, media_type="movie"): @@ -641,16 +640,16 @@ def manual_upload_subtitle(path, language, forced, title, scene_name, media_type def series_download_subtitles(no): episodes_details = database.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, monitored, " "table_episodes.sonarrEpisodeId, table_episodes.scene_name, table_shows.tags, " - "table_shows.seriesType FROM table_episodes INNER JOIN table_shows on " - "table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE " - "table_episodes.sonarrSeriesId=? and missing_subtitles!='[]'", (no,)) - episodes_details = filter_exclusions(episodes_details, 'series') + "table_shows.seriesType, table_episodes.audio_language FROM table_episodes " + "INNER JOIN table_shows on table_shows.sonarrSeriesId = " + "table_episodes.sonarrSeriesId WHERE table_episodes.sonarrSeriesId=? and " + "missing_subtitles!='[]'" + get_exclusion_clause('series'), (no,)) if not episodes_details: logging.debug("BAZARR no episode for that sonarrSeriesId can be found in database:", str(no)) return series_details = database.execute( - "SELECT hearing_impaired, audio_language, title, forced FROM table_shows WHERE sonarrSeriesId=?", + "SELECT hearing_impaired, title, forced FROM table_shows WHERE sonarrSeriesId=?", (no,), only_one=True) if not series_details: logging.debug("BAZARR no series with that sonarrSeriesId can be found in database:", str(no)) @@ -667,7 +666,7 @@ def series_download_subtitles(no): if language is not None: result = download_subtitle(path_mappings.path_replace(episode['path']), str(alpha3_from_alpha2(language.split(':')[0])), - series_details['audio_language'], + episode['audio_language'], series_details['hearing_impaired'], "True" if len(language.split(':')) > 1 else "False", providers_list, @@ -697,10 +696,10 @@ def episode_download_subtitles(no): episodes_details = database.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, monitored, " "table_episodes.sonarrEpisodeId, table_episodes.scene_name, table_shows.tags, " "table_shows.hearing_impaired, table_shows.title, table_shows.sonarrSeriesId, " - "table_shows.forced, table_shows.audio_language, table_shows.seriesType FROM " + "table_shows.forced, table_episodes.audio_language, table_shows.seriesType FROM " "table_episodes LEFT JOIN table_shows on table_episodes.sonarrSeriesId = " - "table_shows.sonarrSeriesId WHERE sonarrEpisodeId=?", (no,)) - episodes_details = filter_exclusions(episodes_details, 'series') + "table_shows.sonarrSeriesId WHERE sonarrEpisodeId=?" + + get_exclusion_clause('series'), (no,)) if not episodes_details: logging.debug("BAZARR no episode with that sonarrEpisodeId can be found in database:", str(no)) return @@ -743,8 +742,7 @@ def episode_download_subtitles(no): def movies_download_subtitles(no): movies = database.execute( "SELECT path, missing_subtitles, audio_language, radarrId, sceneName, hearing_impaired, title, forced, tags, " - "monitored FROM table_movies WHERE radarrId=?", (no,)) - movies = filter_exclusions(movies, 'movie') + "monitored FROM table_movies WHERE radarrId=?" + get_exclusion_clause('movie'), (no,)) if not len(movies): logging.debug("BAZARR no movie with that radarrId can be found in database:", str(no)) return @@ -792,7 +790,7 @@ def movies_download_subtitles(no): def wanted_download_subtitles(path, l, count_episodes): episodes_details = database.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, " "table_episodes.sonarrEpisodeId, table_episodes.sonarrSeriesId, " - "table_shows.hearing_impaired, table_shows.audio_language, table_episodes.scene_name," + "table_shows.hearing_impaired, table_episodes.audio_language, table_episodes.scene_name," "table_episodes.failedAttempts, table_shows.title, table_shows.forced " "FROM table_episodes LEFT JOIN table_shows on " "table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId " @@ -911,8 +909,7 @@ def wanted_search_missing_subtitles_series(): episodes = database.execute("SELECT table_episodes.path, table_shows.tags, table_episodes.monitored, " "table_shows.seriesType FROM table_episodes INNER JOIN table_shows on " "table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE missing_subtitles != " - "'[]'") - episodes = filter_exclusions(episodes, 'series') + "'[]'" + get_exclusion_clause('series')) # path_replace dict_mapper.path_replace(episodes) @@ -929,8 +926,8 @@ def wanted_search_missing_subtitles_series(): def wanted_search_missing_subtitles_movies(): - movies = database.execute("SELECT path, tags, monitored FROM table_movies WHERE missing_subtitles != '[]'") - movies = filter_exclusions(movies, 'movie') + movies = database.execute("SELECT path, tags, monitored FROM table_movies WHERE missing_subtitles != '[]'" + + get_exclusion_clause('movie')) # path_replace dict_mapper.path_replace_movie(movies) @@ -1071,7 +1068,7 @@ def upgrade_subtitles(): if settings.general.getboolean('use_sonarr'): upgradable_episodes = database.execute("SELECT table_history.video_path, table_history.language, " "table_history.score, table_shows.hearing_impaired, " - "table_shows.audio_language, table_episodes.scene_name, table_episodes.title," + "table_episodes.audio_language, table_episodes.scene_name, table_episodes.title," "table_episodes.sonarrSeriesId, table_episodes.sonarrEpisodeId," "MAX(table_history.timestamp) as timestamp, table_episodes.monitored, " "table_shows.languages, table_shows.forced, table_shows.tags, " @@ -1079,11 +1076,10 @@ def upgrade_subtitles(): "table_shows.sonarrSeriesId = table_history.sonarrSeriesId INNER JOIN " "table_episodes on table_episodes.sonarrEpisodeId = " "table_history.sonarrEpisodeId WHERE action IN " - "(" + ','.join(map(str, query_actions)) + - ") AND timestamp > ? AND score is not null GROUP BY " + "(" + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND score" + " is not null" + get_exclusion_clause('series') + " GROUP BY " "table_history.video_path, table_history.language", (minimum_timestamp,)) - upgradable_episodes = filter_exclusions(upgradable_episodes, 'series') upgradable_episodes_not_perfect = [] for upgradable_episode in upgradable_episodes: if upgradable_episode['timestamp'] > minimum_timestamp: @@ -1111,9 +1107,9 @@ def upgrade_subtitles(): "table_movies.monitored FROM table_history_movie INNER JOIN table_movies " "on table_movies.radarrId = table_history_movie.radarrId WHERE action IN " "(" + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND score " - "is not null GROUP BY table_history_movie.video_path, " - "table_history_movie.language", (minimum_timestamp,)) - upgradable_movies = filter_exclusions(upgradable_movies, 'movie') + "is not null" + get_exclusion_clause('movie') + " GROUP BY " + "table_history_movie.video_path, table_history_movie.language", + (minimum_timestamp,)) upgradable_movies_not_perfect = [] for upgradable_movie in upgradable_movies: if upgradable_movie['timestamp'] > minimum_timestamp: diff --git a/bazarr/helper.py b/bazarr/helper.py index 4d7f1b432..4307affde 100644 --- a/bazarr/helper.py +++ b/bazarr/helper.py @@ -16,8 +16,8 @@ class PathMappings: self.path_mapping_movies = [] def update(self): - self.path_mapping_series = ast.literal_eval(settings.general.path_mappings) - self.path_mapping_movies = ast.literal_eval(settings.general.path_mappings_movie) + self.path_mapping_series = [x for x in ast.literal_eval(settings.general.path_mappings) if x[0] != x[1]] + self.path_mapping_movies = [x for x in ast.literal_eval(settings.general.path_mappings_movie) if x[0] != x[1]] def path_replace(self, path): if path is None: diff --git a/bazarr/main.py b/bazarr/main.py index 4446c725a..8a2bad4bf 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1,6 +1,6 @@ # coding=utf-8 -bazarr_version = '0.9.0.2' +bazarr_version = '0.9.0.3' import os os.environ["BAZARR_VERSION"] = bazarr_version diff --git a/bazarr/subsyncer.py b/bazarr/subsyncer.py index b0cfa018c..220f492ae 100644 --- a/bazarr/subsyncer.py +++ b/bazarr/subsyncer.py @@ -5,6 +5,8 @@ from utils import get_binary from utils import history_log, history_log_movie from get_languages import alpha2_from_alpha3, language_from_alpha3 from helper import path_mappings +from config import settings +from get_args import args class SubSyncer: @@ -15,12 +17,13 @@ class SubSyncer: self.ffmpeg_path = None self.args = None self.vad = 'subs_then_auditok' + self.log_dir_path = os.path.join(args.config_dir, 'log') def sync(self, video_path, srt_path, srt_lang, media_type, sonarr_series_id=None, sonarr_episode_id=None, radarr_id=None): self.reference = video_path self.srtin = srt_path - self.srtout = None + self.srtout = '{}.synced.srt'.format(os.path.splitext(self.srtin)[0]) self.args = None ffprobe_exe = get_binary('ffprobe') @@ -39,8 +42,10 @@ class SubSyncer: self.ffmpeg_path = os.path.dirname(ffmpeg_exe) try: - unparsed_args = [self.reference, '-i', self.srtin, '--overwrite-input', '--ffmpegpath', self.ffmpeg_path, - '--vad', self.vad] + unparsed_args = [self.reference, '-i', self.srtin, '-o', self.srtout, '--ffmpegpath', self.ffmpeg_path, + '--vad', self.vad, '--log-dir-path', self.log_dir_path] + if settings.subsync.getboolean('debug'): + unparsed_args.append('--make-test-case') parser = make_parser() self.args = parser.parse_args(args=unparsed_args) result = run(self.args) @@ -49,18 +54,24 @@ class SubSyncer: '{0}'.format(self.srtin)) else: if result['sync_was_successful']: - message = "{0} subtitles synchronization ended with an offset of {1} seconds and a framerate scale " \ - "factor of {2}.".format(language_from_alpha3(srt_lang), result['offset_seconds'], - "{:.2f}".format(result['framerate_scale_factor'])) + if not settings.subsync.getboolean('debug'): + os.remove(self.srtin) + os.rename(self.srtout, self.srtin) + + offset_seconds = result['offset_seconds'] or 0 + framerate_scale_factor = result['framerate_scale_factor'] or 0 + message = "{0} subtitles synchronization ended with an offset of {1} seconds and a framerate " \ + "scale factor of {2}.".format(language_from_alpha3(srt_lang), offset_seconds, + "{:.2f}".format(framerate_scale_factor)) - if media_type == 'series': - history_log(action=5, sonarr_series_id=sonarr_series_id, sonarr_episode_id=sonarr_episode_id, - description=message, video_path=path_mappings.path_replace_reverse(self.reference), - language=alpha2_from_alpha3(srt_lang), subtitles_path=srt_path) - else: - history_log_movie(action=5, radarr_id=radarr_id, description=message, - video_path=path_mappings.path_replace_reverse_movie(self.reference), - language=alpha2_from_alpha3(srt_lang), subtitles_path=srt_path) + if media_type == 'series': + history_log(action=5, sonarr_series_id=sonarr_series_id, sonarr_episode_id=sonarr_episode_id, + description=message, video_path=path_mappings.path_replace_reverse(self.reference), + language=alpha2_from_alpha3(srt_lang), subtitles_path=srt_path) + else: + history_log_movie(action=5, radarr_id=radarr_id, description=message, + video_path=path_mappings.path_replace_reverse_movie(self.reference), + language=alpha2_from_alpha3(srt_lang), subtitles_path=srt_path) else: logging.error('BAZARR unable to sync subtitles: {0}'.format(self.srtin)) diff --git a/libs/ffsubsync/_version.py b/libs/ffsubsync/_version.py index 83abafbaf..fac1f364c 100644 --- a/libs/ffsubsync/_version.py +++ b/libs/ffsubsync/_version.py @@ -24,8 +24,8 @@ def get_keywords(): # each be defined on a line of their own. _version.py will just call # get_keywords(). git_refnames = " (HEAD -> master)" - git_full = "997749de8aac74ec19137a2e641b97ef1bba81ea" - git_date = "2020-08-04 20:06:18 -0700" + git_full = "ce46d91fa2d325a13c2830f8030a316ed49b6cc9" + git_date = "2020-09-05 11:15:34 -0700" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords diff --git a/libs/ffsubsync/ffsubsync.py b/libs/ffsubsync/ffsubsync.py index 18d772c12..e3b08430b 100644 --- a/libs/ffsubsync/ffsubsync.py +++ b/libs/ffsubsync/ffsubsync.py @@ -41,12 +41,15 @@ def make_test_case(args, npy_savename, sync_was_successful): raise ValueError('need non-null npy_savename') tar_dir = '{}.{}'.format( args.reference, - datetime.now().strftime('%Y-%m-%d-%H:%M:%S') + datetime.now().strftime('%Y-%m-%d-%H-%M-%S') ) logger.info('creating test archive {}.tar.gz...'.format(tar_dir)) os.mkdir(tar_dir) try: - shutil.move('ffsubsync.log', tar_dir) + log_path = 'ffsubsync.log' + if args.log_dir_path and os.path.isdir(args.log_dir_path): + log_path = os.path.join(args.log_dir_path, log_path) + shutil.copy(log_path, tar_dir) shutil.copy(args.srtin, tar_dir) if sync_was_successful: shutil.move(args.srtout, tar_dir) @@ -60,7 +63,7 @@ def make_test_case(args, npy_savename, sync_was_successful): preferred_formats = ['gztar', 'bztar', 'xztar', 'zip', 'tar'] for archive_format in preferred_formats: if archive_format in supported_formats: - shutil.make_archive(tar_dir, 'gztar', os.curdir, tar_dir) + shutil.make_archive(tar_dir, archive_format, os.curdir, tar_dir) break else: logger.error('failed to create test archive; no formats supported ' @@ -264,9 +267,14 @@ def run(args): 'when reference composed of subtitles') result['retval'] = 1 return result + log_handler = None + log_path = None if args.make_test_case: - handler = logging.FileHandler('ffsubsync.log') - logger.addHandler(handler) + log_path = 'ffsubsync.log' + if args.log_dir_path and os.path.isdir(args.log_dir_path): + log_path = os.path.join(args.log_dir_path, log_path) + log_handler = logging.FileHandler(log_path) + logger.addHandler(log_handler) if args.extract_subs_from_stream is not None: result['retval'] = extract_subtitles_from_reference(args) return result @@ -285,8 +293,14 @@ def run(args): return result srt_pipes = make_srt_pipes(args) sync_was_successful = try_sync(args, reference_pipe, srt_pipes, result) - if args.make_test_case: - result['retval'] += make_test_case(args, npy_savename, sync_was_successful) + if log_handler is not None and log_path is not None: + assert args.make_test_case + log_handler.close() + logger.removeHandler(log_handler) + try: + result['retval'] += make_test_case(args, npy_savename, sync_was_successful) + finally: + os.remove(log_path) return result @@ -354,6 +368,8 @@ def add_cli_only_args(parser): '--ffmpeg-path', '--ffmpegpath', default=None, help='Where to look for ffmpeg and ffprobe. Uses the system PATH by default.' ) + parser.add_argument('--log-dir-path', default=None, help='Where to save ffsubsync.log file (must be an existing ' + 'directory).') parser.add_argument('--vlc-mode', action='store_true', help=argparse.SUPPRESS) parser.add_argument('--gui-mode', action='store_true', help=argparse.SUPPRESS) diff --git a/libs/ffsubsync/speech_transformers.py b/libs/ffsubsync/speech_transformers.py index 02a7f332b..8290f82f9 100644 --- a/libs/ffsubsync/speech_transformers.py +++ b/libs/ffsubsync/speech_transformers.py @@ -233,7 +233,7 @@ class VideoSpeechTransformer(TransformerMixin): if not in_bytes: break newstuff = len(in_bytes) / float(bytes_per_frame) / self.frame_rate - if simple_progress + newstuff > total_duration: + if total_duration is not None and simple_progress + newstuff > total_duration: newstuff = total_duration - simple_progress simple_progress += newstuff pbar.update(newstuff) diff --git a/libs/subliminal_patch/providers/subdivx.py b/libs/subliminal_patch/providers/subdivx.py index 998460ff0..f144a4cdc 100644 --- a/libs/subliminal_patch/providers/subdivx.py +++ b/libs/subliminal_patch/providers/subdivx.py @@ -168,7 +168,8 @@ class SubdivxSubtitlesProvider(Provider): download_link = self._get_download_link(subtitle) # download zip / rar file with the subtitle - response = self.session.get(download_link, headers={'Referer': subtitle.page_link}, timeout=30) + response = self.session.get(self.server_url + download_link, headers={'Referer': subtitle.page_link}, + timeout=30) self._check_response(response) # open the compressed archive diff --git a/libs/subliminal_patch/providers/subssabbz.py b/libs/subliminal_patch/providers/subssabbz.py index 386e3c19f..960d36429 100644 --- a/libs/subliminal_patch/providers/subssabbz.py +++ b/libs/subliminal_patch/providers/subssabbz.py @@ -55,7 +55,12 @@ class SubsSabBzSubtitle(Subtitle): self.video = video self.fps = fps self.num_cds = num_cds - self.release_info = os.path.splitext(filename)[0] + self.release_info = filename + if fps: + if video.fps and float(video.fps) == fps: + self.release_info += " <b>[{:.3f}]</b>".format(fps) + else: + self.release_info += " [{:.3f}]".format(fps) @property def id(self): @@ -168,7 +173,7 @@ class SubsSabBzProvider(Provider): element = a_element_wrapper.find('a') if element: link = element.get('href') - notes = element.get('onmouseover') + notes = re.sub(r'ddrivetip\(\'<div.*/></div>(.*)\',\'#[0-9]+\'\)', r'\1', element.get('onmouseover')) title = element.get_text() try: @@ -248,12 +253,15 @@ class SubsSabBzProvider(Provider): else: logger.info('Cache file: %s', codecs.encode(cache_key, 'hex_codec').decode('utf-8')) - archive_stream = io.BytesIO(request.content) - if is_rarfile(archive_stream): - return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, num_cds) - elif is_zipfile(archive_stream): - return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, num_cds) - else: - logger.error('Ignore unsupported archive %r', request.headers) - region.delete(cache_key) - return [] + try: + archive_stream = io.BytesIO(request.content) + if is_rarfile(archive_stream): + return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, num_cds) + elif is_zipfile(archive_stream): + return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, num_cds) + except: + pass + + logger.error('Ignore unsupported archive %r', request.headers) + region.delete(cache_key) + return [] diff --git a/libs/subliminal_patch/providers/subsunacs.py b/libs/subliminal_patch/providers/subsunacs.py index 19c5eff86..137d2f7bf 100644 --- a/libs/subliminal_patch/providers/subsunacs.py +++ b/libs/subliminal_patch/providers/subsunacs.py @@ -55,7 +55,12 @@ class SubsUnacsSubtitle(Subtitle): self.video = video self.fps = fps self.num_cds = num_cds - self.release_info = os.path.splitext(filename)[0] + self.release_info = filename + if fps: + if video.fps and float(video.fps) == fps: + self.release_info += " <b>[{:.3f}]</b>".format(fps) + else: + self.release_info += " [{:.3f}]".format(fps) @property def id(self): @@ -168,7 +173,7 @@ class SubsUnacsProvider(Provider): element = a_element_wrapper.find('a', {'class': 'tooltip'}) if element: link = element.get('href') - notes = element.get('title') + notes = re.sub(r'(<img.*)(src=")(/)(.*.jpg">)', r"", element.get('title')) title = element.get_text() try: @@ -230,11 +235,8 @@ class SubsUnacsProvider(Provider): is_7zip = isinstance(archiveStream, SevenZipFile) if is_7zip: - try: - file_content = archiveStream.readall() - file_list = sorted(file_content) - except: - return [] + file_content = archiveStream.readall() + file_list = sorted(file_content) else: file_list = sorted(archiveStream.namelist()) @@ -268,14 +270,17 @@ class SubsUnacsProvider(Provider): else: logger.info('Cache file: %s', codecs.encode(cache_key, 'hex_codec').decode('utf-8')) - archive_stream = io.BytesIO(request.content) - if is_rarfile(archive_stream): - return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, num_cds) - elif is_zipfile(archive_stream): - return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, num_cds) - elif archive_stream.seek(0) == 0 and is_7zfile(archive_stream): - return self.process_archive_subtitle_files(SevenZipFile(archive_stream), language, video, link, fps, num_cds) - else: - logger.error('Ignore unsupported archive %r', request.headers) - region.delete(cache_key) - return [] + try: + archive_stream = io.BytesIO(request.content) + if is_rarfile(archive_stream): + return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, num_cds) + elif is_zipfile(archive_stream): + return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, num_cds) + elif archive_stream.seek(0) == 0 and is_7zfile(archive_stream): + return self.process_archive_subtitle_files(SevenZipFile(archive_stream), language, video, link, fps, num_cds) + except: + pass + + logger.error('Ignore unsupported archive %r', request.headers) + region.delete(cache_key) + return [] diff --git a/libs/subliminal_patch/providers/yavkanet.py b/libs/subliminal_patch/providers/yavkanet.py index 6de60ef35..4555fd177 100644 --- a/libs/subliminal_patch/providers/yavkanet.py +++ b/libs/subliminal_patch/providers/yavkanet.py @@ -37,7 +37,12 @@ class YavkaNetSubtitle(Subtitle): self.type = type self.video = video self.fps = fps - self.release_info = os.path.splitext(filename)[0] + self.release_info = filename + if fps: + if video.fps and float(video.fps) == fps: + self.release_info += " <b>[{:.3f}]</b>".format(fps) + else: + self.release_info += " [{:.3f}]".format(fps) @property def id(self): @@ -141,7 +146,7 @@ class YavkaNetProvider(Provider): element = row.find('a', {'class': 'selector'}) if element: link = element.get('href') - notes = element.get('content') + notes = re.sub(r'(?s)<p.*><img [A-z0-9=\'/\. :;#]*>(.*)</p>', r"\1", element.get('content')) title = element.get_text() try: @@ -205,12 +210,15 @@ class YavkaNetProvider(Provider): else: logger.info('Cache file: %s', codecs.encode(cache_key, 'hex_codec').decode('utf-8')) - archive_stream = io.BytesIO(request.content) - if is_rarfile(archive_stream): - return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps) - elif is_zipfile(archive_stream): - return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps) - else: - logger.error('Ignore unsupported archive %r', request.headers) - region.delete(cache_key) - return [] + try: + archive_stream = io.BytesIO(request.content) + if is_rarfile(archive_stream): + return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps) + elif is_zipfile(archive_stream): + return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps) + except: + pass + + logger.error('Ignore unsupported archive %r', request.headers) + region.delete(cache_key) + return [] diff --git a/libs/subliminal_patch/providers/yifysubtitles.py b/libs/subliminal_patch/providers/yifysubtitles.py index 59d683577..e41dc6c69 100644 --- a/libs/subliminal_patch/providers/yifysubtitles.py +++ b/libs/subliminal_patch/providers/yifysubtitles.py @@ -95,7 +95,7 @@ class YifySubtitlesProvider(Provider): ] languages = {Language(l, c) for (_, l, c) in YifyLanguages} - server_url = 'https://www.yifysubtitles.com' + server_urls = ['https://yifysubtitles.org', 'https://www.yifysubtitles.com'] video_types = (Movie,) def initialize(self): @@ -112,20 +112,20 @@ class YifySubtitlesProvider(Provider): def terminate(self): self.session.close() - def _parse_row(self, row, languages): + def _parse_row(self, row, languages, server_url): td = row.findAll('td') rating = int(td[0].text) sub_lang = td[1].text release = re.sub(r'^subtitle ', '', td[2].text) sub_link = td[2].find('a').get('href') - sub_link = re.sub(r'^/subtitles/', self.server_url + '/subtitle/', sub_link) + '.zip' + page_link = server_url + sub_link + sub_link = re.sub(r'^/subtitles/', server_url + '/subtitle/', sub_link) + '.zip' hi = True if td[3].find('span', {'class': 'hi-subtitle'}) else False uploader = td[4].text - page_link = self.server_url + td[5].find('a').get('href') _, l, c = next(x for x in self.YifyLanguages if x[0] == sub_lang) lang = Language(l, c) - if languages & set([lang]): + if languages & {lang}: return [YifySubtitle(lang, page_link, release, uploader, sub_link, rating, hi)] return [] @@ -134,9 +134,13 @@ class YifySubtitlesProvider(Provider): subtitles = [] logger.info('Searching subtitle %r', imdb_id) - response = self.session.get(self.server_url + '/movie-imdb/' + imdb_id, - allow_redirects=False, timeout=10, - headers={'Referer': self.server_url}) + for server_url in self.server_urls: + response = self.session.get(server_url + '/movie-imdb/' + imdb_id, + allow_redirects=False, timeout=10, + headers={'Referer': server_url}) + if response.status_code == 200: + break + response.raise_for_status() if response.status_code != 200: @@ -150,7 +154,7 @@ class YifySubtitlesProvider(Provider): for row in rows: try: - subtitles = subtitles + self._parse_row(row, languages) + subtitles = subtitles + self._parse_row(row, languages, server_url) except Exception as e: pass diff --git a/views/_main.html b/views/_main.html index 014a4dfa4..adb4eec35 100644 --- a/views/_main.html +++ b/views/_main.html @@ -495,14 +495,7 @@ // Add apikey to all AJAX requests. $.ajaxSetup({ - data: { - apikey: "{{ settings.auth.apikey }}" - } - }); - $.ajaxPrefilter(function (options, originalOptions, jqXHR) { - if (originalOptions.data instanceof FormData) { - originalOptions.data.append("apikey", "{{ settings.auth.apikey }}"); - } + headers: { 'X-Api-Key': "{{ settings.auth.apikey }}" } }); $(window).on('beforeunload', function () { diff --git a/views/episodes.html b/views/episodes.html index 9d6995ae1..0fa3509e9 100644 --- a/views/episodes.html +++ b/views/episodes.html @@ -113,6 +113,7 @@ <th></th> <th>Episode</th> <th>Title</th> + <th>Audio Language</th> <th>Existing Subtitles</th> <th>Missing Subtitles</th> <th>Manual Search</th> @@ -232,7 +233,7 @@ <div class="container-fluid"> <div class="row"> <div class="col-sm-3 text-right"> - Audio Language + Audio Profile </div> <div class="form-group col-sm-8 pl-sm-0"> <span id="edit_audio_language_span"></span> @@ -580,6 +581,9 @@ } }, { + data: 'audio_language.name' + }, + { data: null, render: function (data) { if (data.subtitles !== 'None') { @@ -629,7 +633,7 @@ data: null, render: function (data) { if (data.desired_languages !== '[]') { - return '<a href="" class="upload_subtitle badge badge-secondary" data-episodePath="' + data.mapped_path + '" data-sceneName"' + data.scene_name + '" data-sonarrSeriesId="' + seriesDetails['sonarrSeriesId'] + '" data-sonarrEpisodeId="' + data.sonarrEpisodeId + '" data-season="' + data.season + '" data-episode="' + data.episode + '" data-episode_title="' + data.title + '"><i class="fas fa-cloud-upload-alt"></i></a>'; + return '<a href="" class="upload_subtitle badge badge-secondary" data-episodePath="' + data.mapped_path + '" data-sceneName"' + data.scene_name + '" data-sonarrSeriesId="' + seriesDetails['sonarrSeriesId'] + '" data-sonarrEpisodeId="' + data.sonarrEpisodeId + '" data-season="' + data.season + '" data-episode="' + data.episode + '" data-episode_title="' + data.title + '" data-audio_language="' + data.audio_language.name + '"><i class="fas fa-cloud-upload-alt"></i></a>'; } else { return '' } @@ -876,7 +880,7 @@ $('#upload_sonarrSeriesId').val($(this).data("sonarrseriesid")); $('#upload_sonarrEpisodeId').val($(this).data("sonarrepisodeid")); $('#upload_title').val($(this).data("episode_title")); - $('#upload_audioLanguage').val(seriesDetails['audio_language']['name']); + $('#upload_audioLanguage').val($(this).data("audio_language")); $('#manual_language_select').empty(); $.each(enabledLanguages, function (i, item) { diff --git a/views/movieseditor.html b/views/movieseditor.html index 97bd75a60..f87c9ab0e 100644 --- a/views/movieseditor.html +++ b/views/movieseditor.html @@ -67,7 +67,7 @@ var event_json = JSON.parse(event); if (event_json.type === 'movies_editor' && event_json.action === 'update') { $.ajax({ - url: "{{ url_for('api.movies') }}", + url: "{{ url_for('api.movieseditor') }}", success: function (data) { if (data.data.length) { $('#movies').DataTable().ajax.reload(resetPaging = false); @@ -96,7 +96,7 @@ lengthChange: true, responsive: true, paging: false, - ajax: "{{ url_for('api.movies') }}", + ajax: "{{ url_for('api.movieseditor') }}", columnDefs: [{ orderable: false, className: 'select-checkbox', diff --git a/views/series.html b/views/series.html index ad3afab83..74171b032 100644 --- a/views/series.html +++ b/views/series.html @@ -22,7 +22,7 @@ <tr> <th>Name</th> <th>Path Exist</th> - <th>Audio Language</th> + <th>Audio Profile</th> <th>Subtitles Languages</th> <th>Hearing-Impaired</th> <th>Forced</th> @@ -46,7 +46,7 @@ <div class="container-fluid"> <div class="row"> <div class="col-sm-3 text-right"> - Audio Language + Audio Profile </div> <div class="form-group col-sm-8 pl-sm-0"> <span id="edit_audio_language_span"></span> diff --git a/views/serieseditor.html b/views/serieseditor.html index 81a13d438..058422338 100644 --- a/views/serieseditor.html +++ b/views/serieseditor.html @@ -16,7 +16,7 @@ <tr> <th></th> <th>Name</th> - <th>Audio Language</th> + <th>Audio Profile</th> <th>Subtitles Languages</th> <th>Hearing-Impaired</th> <th>Forced</th> @@ -67,7 +67,7 @@ var event_json = JSON.parse(event); if (event_json.type === 'series_editor' && event_json.action === 'update') { $.ajax({ - url: "{{ url_for('api.series') }}", + url: "{{ url_for('api.serieseditor') }}", success: function (data) { if (data.data.length) { $('#series').DataTable().ajax.reload(resetPaging = false); @@ -96,7 +96,7 @@ lengthChange: true, responsive: true, paging: false, - ajax: "{{ url_for('api.series') }}", + ajax: "{{ url_for('api.serieseditor') }}", columnDefs: [{ orderable: false, className: 'select-checkbox', diff --git a/views/settingssubtitles.html b/views/settingssubtitles.html index c04197aad..e01012f93 100644 --- a/views/settingssubtitles.html +++ b/views/settingssubtitles.html @@ -412,14 +412,26 @@ {% endif %} <div class="row"> <div class="col-sm-3 text-right"> - <b>Subtitles synchronization</b> + <b>Automatic subtitles synchronization</b> </div> <div class="form-group col-sm-8"> <label class="custom-control custom-checkbox"> <input type="checkbox" class="custom-control-input" id="settings-subsync-use_subsync" name="settings-subsync-use_subsync"> <span class="custom-control-label" for="settings-subsync-use_subsync"></span> </label> - <label>Enable the subtitles synchronization after downloading a subtitles.</label> + <label>Enable the automatic subtitles synchronization after downloading a subtitles.</label> + </div> + </div> + <div class="row"> + <div class="col-sm-4 text-right"> + <b>Subtitles synchronization debugging</b> + </div> + <div class="form-group col-sm-8"> + <label class="custom-control custom-checkbox"> + <input type="checkbox" class="custom-control-input" id="settings-subsync-debug" name="settings-subsync-debug"> + <span class="custom-control-label" for="settings-subsync-debug"></span> + </label> + <label>Do not actually sync the subtitles but generate a .tar.gz file to be able to open an issue for ffsubsync. This file will reside alongside the media file.</label> </div> </div> <div id="subsync_div"> @@ -680,6 +692,7 @@ $('#settings-general-chmod_enabled').prop('checked', {{'true' if settings.general.getboolean('chmod_enabled') else 'false'}}).trigger('change'); $('#settings-subsync-use_subsync').prop('checked', {{'true' if settings.subsync.getboolean('use_subsync') else 'false'}}).trigger('change'); $('#settings-subsync-use_subsync_threshold').prop('checked', {{'true' if settings.subsync.getboolean('use_subsync_threshold') else 'false'}}).trigger('change'); + $('#settings-subsync-debug').prop('checked', {{'true' if settings.subsync.getboolean('debug') else 'false'}}).trigger('change'); $('#settings-subsync-use_subsync_movie_threshold').prop('checked', {{'true' if settings.subsync.getboolean('use_subsync_movie_threshold') else 'false'}}).trigger('change'); $('#settings-general-use_postprocessing').prop('checked', {{'true' if settings.general.getboolean('use_postprocessing') else 'false'}}).trigger('change'); $('#settings-general-use_postprocessing_threshold').prop('checked', {{'true' if settings.general.getboolean('use_postprocessing_threshold') else 'false'}}).trigger('change'); |