Added Sync Optimization for Movies, Series and Episodes to ignore unmonitored itemsv1.4.1-beta.18

9 files changed, 222 insertions, 46 deletions

diff --git a/bazarr/app/config.py b/bazarr/app/config.py
index d490a6a4e..d1967cc38 100644
--- a/bazarr/app/config.py
+++ b/bazarr/app/config.py
@@ -161,6 +161,8 @@ validators = [
     Validator('sonarr.use_ffprobe_cache', must_exist=True, default=True, is_type_of=bool),
     Validator('sonarr.exclude_season_zero', must_exist=True, default=False, is_type_of=bool),
     Validator('sonarr.defer_search_signalr', must_exist=True, default=False, is_type_of=bool),
+    Validator('sonarr.sync_only_monitored_series', must_exist=True, default=False, is_type_of=bool),
+    Validator('sonarr.sync_only_monitored_episodes', must_exist=True, default=False, is_type_of=bool),
 
     # radarr section
     Validator('radarr.ip', must_exist=True, default='127.0.0.1', is_type_of=str),
@@ -180,6 +182,7 @@ validators = [
     Validator('radarr.excluded_tags', must_exist=True, default=[], is_type_of=list),
     Validator('radarr.use_ffprobe_cache', must_exist=True, default=True, is_type_of=bool),
     Validator('radarr.defer_search_signalr', must_exist=True, default=False, is_type_of=bool),
+    Validator('radarr.sync_only_monitored_movies', must_exist=True, default=False, is_type_of=bool),
 
     # proxy section
     Validator('proxy.type', must_exist=True, default=None, is_type_of=(NoneType, str),
diff --git a/bazarr/constants.py b/bazarr/constants.py
index a746e7628..4f8af9614 100644
--- a/bazarr/constants.py
+++ b/bazarr/constants.py
@@ -8,3 +8,6 @@ headers = {"User-Agent": os.environ["SZ_USER_AGENT"]}
 
 # hearing-impaired detection regex
 hi_regex = re.compile(r'[*¶♫♪].{3,}[*¶♫♪]|[\[\(\{].{3,}[\]\)\}](?<!{\\an\d})')
+
+# minimum file size for Bazarr to consider it a video
+MINIMUM_VIDEO_SIZE = 20480
diff --git a/bazarr/radarr/sync/movies.py b/bazarr/radarr/sync/movies.py
index 909b7a5a4..6227a478c 100644
--- a/bazarr/radarr/sync/movies.py
+++ b/bazarr/radarr/sync/movies.py
@@ -2,6 +2,7 @@
 
 import os
 import logging
+from constants import MINIMUM_VIDEO_SIZE
 
 from sqlalchemy.exc import IntegrityError
 
@@ -16,6 +17,13 @@ from app.event_handler import event_stream, show_progress, hide_progress
 from .utils import get_profile_list, get_tags, get_movies_from_radarr_api
 from .parser import movieParser
 
+# map between booleans and strings in DB
+bool_map = {"True": True, "False": False}
+
+FEATURE_PREFIX = "SYNC_MOVIES "
+def trace(message):
+    if settings.general.debug:
+        logging.debug(FEATURE_PREFIX + message)
 
 def update_all_movies():
     movies_full_scan_subtitles()
@@ -45,6 +53,16 @@ def update_movie(updated_movie, send_event):
             event_stream(type='movie', action='update', payload=updated_movie['radarrId'])
 
 
+def get_movie_monitored_status(movie_id):
+    existing_movie_monitored = database.execute(
+        select(TableMovies.monitored)
+        .where(TableMovies.tmdbId == movie_id))\
+        .first()
+    if existing_movie_monitored is None:
+        return True
+    else:
+        return bool_map[existing_movie_monitored[0]]
+
 # Insert new movies in DB
 def add_movie(added_movie, send_event):
     try:
@@ -104,12 +122,12 @@ def update_movies(send_event=True):
 
             current_movies_radarr = [str(movie['tmdbId']) for movie in movies if movie['hasFile'] and
                                      'movieFile' in movie and
-                                     (movie['movieFile']['size'] > 20480 or
-                                      get_movie_file_size_from_db(movie['movieFile']['path']) > 20480)]
+                                     (movie['movieFile']['size'] > MINIMUM_VIDEO_SIZE or
+                                      get_movie_file_size_from_db(movie['movieFile']['path']) > MINIMUM_VIDEO_SIZE)]
 
-            # Remove old movies from DB
+            # Remove movies from DB that either no longer exist in Radarr or exist and Radarr says do not have a movie file
             movies_to_delete = list(set(current_movies_id_db) - set(current_movies_radarr))
-
+            movies_deleted = []
             if len(movies_to_delete):
                 try:
                     database.execute(delete(TableMovies).where(TableMovies.tmdbId.in_(movies_to_delete)))
@@ -117,11 +135,19 @@ def update_movies(send_event=True):
                     logging.error(f"BAZARR cannot delete movies because of {e}")
                 else:
                     for removed_movie in movies_to_delete:
+                        movies_deleted.append(removed_movie['title'])
                         if send_event:
                             event_stream(type='movie', action='delete', payload=removed_movie)
 
-            # Build new and updated movies
+            # Add new movies and update movies that Radarr says have media files
+            # Any new movies added to Radarr that don't have media files yet will not be added to DB
             movies_count = len(movies)
+            sync_monitored = settings.radarr.sync_only_monitored_movies
+            if sync_monitored:
+                skipped_count = 0
+            files_missing = 0
+            movies_added = []
+            movies_updated = []
             for i, movie in enumerate(movies):
                 if send_event:
                     show_progress(id='movies_progress',
@@ -129,12 +155,22 @@ def update_movies(send_event=True):
                                   name=movie['title'],
                                   value=i,
                                   count=movies_count)
-
+                # Only movies that Radarr says have files downloaded will be kept up to date in the DB
                 if movie['hasFile'] is True:
                     if 'movieFile' in movie:
-                        if (movie['movieFile']['size'] > 20480 or
-                                get_movie_file_size_from_db(movie['movieFile']['path']) > 20480):
-                            # Add movies in radarr to current movies list
+                        if sync_monitored:   
+                            if get_movie_monitored_status(movie['tmdbId']) != movie['monitored']:
+                                # monitored status is not the same as our DB
+                                trace(f"{i}: (Monitor Status Mismatch) {movie['title']}")
+                            elif not movie['monitored']:
+                                trace(f"{i}: (Skipped Unmonitored) {movie['title']}")
+                                skipped_count += 1
+                                continue
+
+                        if (movie['movieFile']['size'] > MINIMUM_VIDEO_SIZE or
+                                get_movie_file_size_from_db(movie['movieFile']['path']) > MINIMUM_VIDEO_SIZE):
+                            # Add/update movies from Radarr that have a movie file to current movies list
+                            trace(f"{i}: (Processing) {movie['title']}")
                             if str(movie['tmdbId']) in current_movies_id_db:
                                 parsed_movie = movieParser(movie, action='update',
                                                            tags_dict=tagsDict,
@@ -142,16 +178,29 @@ def update_movies(send_event=True):
                                                            audio_profiles=audio_profiles)
                                 if not any([parsed_movie.items() <= x for x in current_movies_db_kv]):
                                     update_movie(parsed_movie, send_event)
+                                    movies_updated.append(parsed_movie['title'])
                             else:
                                 parsed_movie = movieParser(movie, action='insert',
                                                            tags_dict=tagsDict,
                                                            movie_default_profile=movie_default_profile,
                                                            audio_profiles=audio_profiles)
                                 add_movie(parsed_movie, send_event)
+                                movies_added.append(parsed_movie['title'])
+                else:
+                     trace(f"{i}: (Skipped File Missing) {movie['title']}")
+                     files_missing += 1
 
             if send_event:
                 hide_progress(id='movies_progress')
 
+            trace(f"Skipped {files_missing} file missing movies out of {i}")
+            if sync_monitored:
+                trace(f"Skipped {skipped_count} unmonitored movies out of {i}")
+                trace(f"Processed {i - files_missing - skipped_count} movies out of {i} " +
+                              f"with {len(movies_added)} added, {len(movies_updated)} updated and {len(movies_deleted)} deleted")
+            else:
+                trace(f"Processed {i - files_missing} movies out of {i} with {len(movies_added)} added and {len(movies_updated)} updated")
+
             logging.debug('BAZARR All movies synced from Radarr into database.')
 
 
diff --git a/bazarr/radarr/sync/parser.py b/bazarr/radarr/sync/parser.py
index 0d7e915ee..598d824c5 100644
--- a/bazarr/radarr/sync/parser.py
+++ b/bazarr/radarr/sync/parser.py
@@ -13,12 +13,6 @@ from .converter import RadarrFormatAudioCodec, RadarrFormatVideoCodec
 
 def movieParser(movie, action, tags_dict, movie_default_profile, audio_profiles):
     if 'movieFile' in movie:
-        # Detect file separator
-        if movie['path'][0] == "/":
-            separator = "/"
-        else:
-            separator = "\\"
-
         try:
             overview = str(movie['overview'])
         except Exception:
@@ -120,10 +114,9 @@ def movieParser(movie, action, tags_dict, movie_default_profile, audio_profiles)
 
         tags = [d['label'] for d in tags_dict if d['id'] in movie['tags']]
 
-        if action == 'update':
-            return {'radarrId': int(movie["id"]),
+        parsed_movie = {'radarrId': int(movie["id"]),
                     'title': movie["title"],
-                    'path': movie["path"] + separator + movie['movieFile']['relativePath'],
+                    'path': os.path.join(movie["path"], movie['movieFile']['relativePath']),                    
                     'tmdbId': str(movie["tmdbId"]),
                     'poster': poster,
                     'fanart': fanart,
@@ -142,30 +135,12 @@ def movieParser(movie, action, tags_dict, movie_default_profile, audio_profiles)
                     'movie_file_id': int(movie['movieFile']['id']),
                     'tags': str(tags),
                     'file_size': movie['movieFile']['size']}
-        else:
-            return {'radarrId': int(movie["id"]),
-                    'title': movie["title"],
-                    'path': movie["path"] + separator + movie['movieFile']['relativePath'],
-                    'tmdbId': str(movie["tmdbId"]),
-                    'subtitles': '[]',
-                    'overview': overview,
-                    'poster': poster,
-                    'fanart': fanart,
-                    'audio_language': str(audio_language),
-                    'sceneName': sceneName,
-                    'monitored': str(bool(movie['monitored'])),
-                    'sortTitle': movie['sortTitle'],
-                    'year': str(movie['year']),
-                    'alternativeTitles': alternativeTitles,
-                    'format': format,
-                    'resolution': resolution,
-                    'video_codec': videoCodec,
-                    'audio_codec': audioCodec,
-                    'imdbId': imdbId,
-                    'movie_file_id': int(movie['movieFile']['id']),
-                    'tags': str(tags),
-                    'profileId': movie_default_profile,
-                    'file_size': movie['movieFile']['size']}
+        
+        if action == 'insert':
+            parsed_movie['subtitles'] = '[]'
+            parsed_movie['profileId'] = movie_default_profile
+
+        return parsed_movie
 
 
 def profile_id_to_language(id, profiles):
diff --git a/bazarr/sonarr/sync/episodes.py b/bazarr/sonarr/sync/episodes.py
index 7894d9061..fc4b568ee 100644
--- a/bazarr/sonarr/sync/episodes.py
+++ b/bazarr/sonarr/sync/episodes.py
@@ -2,10 +2,11 @@
 
 import os
 import logging
+from constants import MINIMUM_VIDEO_SIZE
 
 from sqlalchemy.exc import IntegrityError
 
-from app.database import database, TableEpisodes, delete, update, insert, select
+from app.database import database, TableShows, TableEpisodes, delete, update, insert, select
 from app.config import settings
 from utilities.path_mappings import path_mappings
 from subtitles.indexer.series import store_subtitles, series_full_scan_subtitles
@@ -16,14 +17,29 @@ from sonarr.info import get_sonarr_info, url_sonarr
 from .parser import episodeParser
 from .utils import get_episodes_from_sonarr_api, get_episodesFiles_from_sonarr_api
 
+# map between booleans and strings in DB
+bool_map = {"True": True, "False": False}
 
+FEATURE_PREFIX = "SYNC_EPISODES "
+def trace(message):
+    if settings.general.debug:
+        logging.debug(FEATURE_PREFIX + message)
+
+def get_episodes_monitored_table(series_id):
+    episodes_monitored = database.execute(
+        select(TableEpisodes.episode_file_id, TableEpisodes.monitored)
+        .where(TableEpisodes.sonarrSeriesId == series_id))\
+        .all()
+    episode_dict = dict((x, y) for x, y in episodes_monitored)
+    return episode_dict
+  
 def update_all_episodes():
     series_full_scan_subtitles()
     logging.info('BAZARR All existing episode subtitles indexed from disk.')
 
 
 def sync_episodes(series_id, send_event=True):
-    logging.debug('BAZARR Starting episodes sync from Sonarr.')
+    logging.debug(f'BAZARR Starting episodes sync from Sonarr for series ID {series_id}.')
     apikey_sonarr = settings.sonarr.apikey
 
     # Get current episodes id in DB
@@ -58,16 +74,42 @@ def sync_episodes(series_id, send_event=True):
                     if item:
                         episode['episodeFile'] = item[0]
 
+
+        sync_monitored = settings.sonarr.sync_only_monitored_series and settings.sonarr.sync_only_monitored_episodes
+        if sync_monitored:
+            episodes_monitored = get_episodes_monitored_table(series_id)
+            skipped_count = 0
+
         for episode in episodes:
             if 'hasFile' in episode:
                 if episode['hasFile'] is True:
                     if 'episodeFile' in episode:
+                        # monitored_status_db = get_episodes_monitored_status(episode['episodeFileId'])
+                        if sync_monitored:
+                            try:
+                                monitored_status_db = bool_map[episodes_monitored[episode['episodeFileId']]]
+                            except KeyError:
+                                monitored_status_db = None
+
+                            if monitored_status_db is None:
+                                # not in db, might need to add, if we have a file on disk
+                                pass
+                            elif monitored_status_db != episode['monitored']:
+                                # monitored status changed and we don't know about it until now
+                                trace(f"(Monitor Status Mismatch) {episode['title']}")
+                                # pass
+                            elif not episode['monitored']:
+                                # Add unmonitored episode in sonarr to current episode list, otherwise it will be deleted from db
+                                current_episodes_sonarr.append(episode['id'])
+                                skipped_count += 1
+                                continue
+
                         try:
                             bazarr_file_size = \
                                 os.path.getsize(path_mappings.path_replace(episode['episodeFile']['path']))
                         except OSError:
                             bazarr_file_size = 0
-                        if episode['episodeFile']['size'] > 20480 or bazarr_file_size > 20480:
+                        if episode['episodeFile']['size'] > MINIMUM_VIDEO_SIZE or bazarr_file_size > MINIMUM_VIDEO_SIZE:
                             # Add episodes in sonarr to current episode list
                             current_episodes_sonarr.append(episode['id'])
 
@@ -80,6 +122,12 @@ def sync_episodes(series_id, send_event=True):
                                 episodes_to_add.append(episodeParser(episode))
     else:
         return
+    
+    if sync_monitored:
+        # try to avoid unnecessary database calls
+        if settings.general.debug:
+            series_title = database.execute(select(TableShows.title).where(TableShows.sonarrSeriesId == series_id)).first()[0]
+            trace(f"Skipped {skipped_count} unmonitored episodes out of {len(episodes)} for {series_title}")
 
     # Remove old episodes from DB
     episodes_to_delete = list(set(current_episodes_id_db_list) - set(current_episodes_sonarr))
diff --git a/bazarr/sonarr/sync/parser.py b/bazarr/sonarr/sync/parser.py
index ad3fae852..d8fce1697 100644
--- a/bazarr/sonarr/sync/parser.py
+++ b/bazarr/sonarr/sync/parser.py
@@ -4,6 +4,7 @@ import os
 
 from app.config import settings
 from app.database import TableShows, database, select
+from constants import MINIMUM_VIDEO_SIZE
 from utilities.path_mappings import path_mappings
 from utilities.video_analyzer import embedded_audio_reader
 from sonarr.info import get_sonarr_info
@@ -92,7 +93,7 @@ def episodeParser(episode):
                     bazarr_file_size = os.path.getsize(path_mappings.path_replace(episode['episodeFile']['path']))
                 except OSError:
                     bazarr_file_size = 0
-                if episode['episodeFile']['size'] > 20480 or bazarr_file_size > 20480:
+                if episode['episodeFile']['size'] > MINIMUM_VIDEO_SIZE or bazarr_file_size > MINIMUM_VIDEO_SIZE:
                     if 'sceneName' in episode['episodeFile']:
                         sceneName = episode['episodeFile']['sceneName']
                     else:
diff --git a/bazarr/sonarr/sync/series.py b/bazarr/sonarr/sync/series.py
index 41eb4ee35..47bf4d59d 100644
--- a/bazarr/sonarr/sync/series.py
+++ b/bazarr/sonarr/sync/series.py
@@ -16,6 +16,20 @@ from .episodes import sync_episodes
 from .parser import seriesParser
 from .utils import get_profile_list, get_tags, get_series_from_sonarr_api
 
+# map between booleans and strings in DB
+bool_map = {"True": True, "False": False}
+
+FEATURE_PREFIX = "SYNC_SERIES "
+def trace(message):
+    if settings.general.debug:
+        logging.debug(FEATURE_PREFIX + message)
+
+def get_series_monitored_table():
+    series_monitored = database.execute(
+        select(TableShows.tvdbId, TableShows.monitored))\
+        .all()
+    series_dict = dict((x, y) for x, y in series_monitored)
+    return series_dict
 
 def update_series(send_event=True):
     check_sonarr_rootfolder()
@@ -55,6 +69,12 @@ def update_series(send_event=True):
         current_shows_sonarr = []
 
         series_count = len(series)
+        sync_monitored = settings.sonarr.sync_only_monitored_series
+        if sync_monitored:
+            series_monitored = get_series_monitored_table()
+            skipped_count = 0
+        trace(f"Starting sync for {series_count} shows")
+        
         for i, show in enumerate(series):
             if send_event:
                 show_progress(id='series_progress',
@@ -63,6 +83,26 @@ def update_series(send_event=True):
                               value=i,
                               count=series_count)
 
+            if sync_monitored:
+                try:
+                    monitored_status_db = bool_map[series_monitored[show['tvdbId']]]
+                except KeyError:
+                    monitored_status_db = None
+                if monitored_status_db is None:
+                    # not in db, need to add
+                    pass
+                elif monitored_status_db != show['monitored']:
+                    # monitored status changed and we don't know about it until now
+                    trace(f"{i}: (Monitor Status Mismatch) {show['title']}")
+                    # pass
+                elif not show['monitored']:
+                    # Add unmonitored series in sonarr to current series list, otherwise it will be deleted from db
+                    trace(f"{i}: (Skipped Unmonitored) {show['title']}")
+                    current_shows_sonarr.append(show['id'])
+                    skipped_count += 1
+                    continue
+
+            trace(f"{i}: (Processing) {show['title']}")
             # Add shows in Sonarr to current shows list
             current_shows_sonarr.append(show['id'])
 
@@ -76,6 +116,7 @@ def update_series(send_event=True):
                         .filter_by(**updated_series))\
                         .first():
                     try:
+                        trace(f"Updating {show['title']}")
                         database.execute(
                             update(TableShows)
                             .values(updated_series)
@@ -92,6 +133,7 @@ def update_series(send_event=True):
                                             audio_profiles=audio_profiles)
 
                 try:
+                    trace(f"Inserting {show['title']}")
                     database.execute(
                         insert(TableShows)
                         .values(added_series))
@@ -110,6 +152,10 @@ def update_series(send_event=True):
         removed_series = list(set(current_shows_db) - set(current_shows_sonarr))
 
         for series in removed_series:
+             # try to avoid unnecessary database calls
+            if settings.general.debug:
+                series_title = database.execute(select(TableShows.title).where(TableShows.sonarrSeriesId == series)).first()[0]
+                trace(f"Deleting {series_title}")
             database.execute(
                 delete(TableShows)
                 .where(TableShows.sonarrSeriesId == series))
@@ -120,6 +166,8 @@ def update_series(send_event=True):
         if send_event:
             hide_progress(id='series_progress')
 
+        if sync_monitored:
+            trace(f"skipped {skipped_count} unmonitored series out of {i}")
         logging.debug('BAZARR All series synced from Sonarr into database.')
 
 
diff --git a/bazarr/utilities/video_analyzer.py b/bazarr/utilities/video_analyzer.py
index 1aad9b859..bd4cac011 100644
--- a/bazarr/utilities/video_analyzer.py
+++ b/bazarr/utilities/video_analyzer.py
@@ -266,6 +266,11 @@ def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=No
     elif embedded_subs_parser == 'mediainfo':
         mediainfo_path = get_binary("mediainfo")
 
+    # see if file exists (perhaps offline)
+    if not os.path.exists(file):
+        logging.error(f'Video file "{file}" cannot be found for analysis')
+        return None
+    
     # if we have ffprobe available
     if ffprobe_path:
         try:
diff --git a/frontend/src/pages/Settings/Scheduler/index.tsx b/frontend/src/pages/Settings/Scheduler/index.tsx
index 3bd6da91a..a6cd2ca74 100644
--- a/frontend/src/pages/Settings/Scheduler/index.tsx
+++ b/frontend/src/pages/Settings/Scheduler/index.tsx
@@ -35,11 +35,55 @@ const SettingsSchedulerView: FunctionComponent = () => {
           options={seriesSyncOptions}
           settingKey="settings-sonarr-series_sync"
         ></Selector>
+        <Check
+          label="Sync Only Monitored Series"
+          settingKey={"settings-sonarr-sync_only_monitored_series"}
+        ></Check>
+        <CollapseBox settingKey={"settings-sonarr-sync_only_monitored_series"}>
+          <Message>
+            If enabled, only series with a monitored status in Sonarr will be
+            synced. If you make changes to a specific unmonitored Sonarr series
+            and you want Bazarr to know about those changes, simply toggle the
+            monitored status back on in Sonarr and Bazarr will sync any changes.
+          </Message>
+        </CollapseBox>
+        <CollapseBox settingKey={"settings-sonarr-sync_only_monitored_series"}>
+          <Check
+            label="Sync Only Monitored Episodes"
+            settingKey={"settings-sonarr-sync_only_monitored_episodes"}
+          ></Check>
+          <CollapseBox
+            settingKey={"settings-sonarr-sync_only_monitored_episodes"}
+          >
+            <Message>
+              If enabled, only episodes with a monitored status in Sonarr will
+              be synced. If you make changes to a specific unmonitored Sonarr
+              episode (or season) and you want Bazarr to know about those
+              changes, simply toggle the monitored status back on in Sonarr and
+              Bazarr will sync any changes. This setting is especially helpful
+              for long running TV series with many seasons and many episodes,
+              but that are still actively producing new episodes (e.g. Saturday
+              Night Live).
+            </Message>
+          </CollapseBox>
+        </CollapseBox>
         <Selector
           label="Sync with Radarr"
           options={moviesSyncOptions}
           settingKey="settings-radarr-movies_sync"
         ></Selector>
+        <Check
+          label="Sync Only Monitored Movies"
+          settingKey={"settings-radarr-sync_only_monitored_movies"}
+        ></Check>
+        <CollapseBox settingKey={"settings-radarr-sync_only_monitored_movies"}>
+          <Message>
+            If enabled, only movies with a monitored status in Radarr will be
+            synced. If you make changes to a specific unmonitored Radarr movie
+            and you want Bazarr to know about those changes, simply toggle the
+            monitored status back on in Radarr and Bazarr will sync any changes.
+          </Message>
+        </CollapseBox>
       </Section>
       <Section header="Disk Indexing">
         <Selector