summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--bazarr/api/system/languages.py4
-rw-r--r--bazarr/app/config.py6
-rw-r--r--bazarr/app/get_providers.py44
-rw-r--r--bazarr/subtitles/pool.py18
-rw-r--r--frontend/src/components/bazarr/LanguageSelector.tsx34
-rw-r--r--frontend/src/pages/Settings/Languages/equals.test.ts196
-rw-r--r--frontend/src/pages/Settings/Languages/equals.tsx365
-rw-r--r--frontend/src/pages/Settings/Languages/index.tsx9
-rw-r--r--frontend/src/pages/Settings/keys.ts2
-rw-r--r--frontend/src/types/api.d.ts1
-rw-r--r--frontend/src/types/settings.d.ts1
-rw-r--r--frontend/src/utilities/languages.ts9
-rw-r--r--libs/subliminal_patch/core.py56
-rw-r--r--tests/bazarr/app/test_get_providers.py45
-rw-r--r--tests/bazarr/conftest.py1
-rw-r--r--tests/bazarr/test_app_get_providers.py115
-rw-r--r--tests/bazarr/test_subtitles_pool.py10
-rw-r--r--tests/subliminal_patch/test_core.py96
18 files changed, 956 insertions, 56 deletions
diff --git a/bazarr/api/system/languages.py b/bazarr/api/system/languages.py
index 757e84a32..75a680f36 100644
--- a/bazarr/api/system/languages.py
+++ b/bazarr/api/system/languages.py
@@ -4,7 +4,7 @@ from flask_restx import Resource, Namespace, reqparse
from operator import itemgetter
from app.database import TableHistory, TableHistoryMovie, TableSettingsLanguages
-from languages.get_languages import alpha2_from_alpha3, language_from_alpha2
+from languages.get_languages import alpha2_from_alpha3, language_from_alpha2, alpha3_from_alpha2
from ..utils import authenticate, False_Keys
@@ -46,6 +46,7 @@ class Languages(Resource):
try:
languages_dicts.append({
'code2': code2,
+ 'code3': alpha3_from_alpha2(code2),
'name': language_from_alpha2(code2),
# Compatibility: Use false temporarily
'enabled': False
@@ -55,6 +56,7 @@ class Languages(Resource):
else:
languages_dicts = TableSettingsLanguages.select(TableSettingsLanguages.name,
TableSettingsLanguages.code2,
+ TableSettingsLanguages.code3,
TableSettingsLanguages.enabled)\
.order_by(TableSettingsLanguages.name).dicts()
languages_dicts = list(languages_dicts)
diff --git a/bazarr/app/config.py b/bazarr/app/config.py
index 306c46539..b3840e0c4 100644
--- a/bazarr/app/config.py
+++ b/bazarr/app/config.py
@@ -83,7 +83,8 @@ defaults = {
'default_und_audio_lang': '',
'default_und_embedded_subtitles_lang': '',
'parse_embedded_audio_track': 'False',
- 'skip_hashing': 'False'
+ 'skip_hashing': 'False',
+ 'language_equals': '[]',
},
'auth': {
'type': 'None',
@@ -300,7 +301,8 @@ array_keys = ['excluded_tags',
'excluded_series_types',
'enabled_providers',
'path_mappings',
- 'path_mappings_movie']
+ 'path_mappings_movie',
+ 'language_equals']
str_keys = ['chmod']
diff --git a/bazarr/app/get_providers.py b/bazarr/app/get_providers.py
index 8b51a4e2d..04593ea15 100644
--- a/bazarr/app/get_providers.py
+++ b/bazarr/app/get_providers.py
@@ -20,6 +20,7 @@ from subliminal_patch.extensions import provider_registry
from app.get_args import args
from app.config import settings, get_array_from
+from languages.get_languages import CustomLanguage
from app.event_handler import event_stream
from utilities.binaries import get_binary
from radarr.blacklist import blacklist_log_movie
@@ -115,6 +116,49 @@ def provider_pool():
return subliminal_patch.core.SZProviderPool
+def _lang_from_str(content: str):
+ " Formats: es-MX en@hi es-MX@forced "
+ extra_info = content.split("@")
+ if len(extra_info) > 1:
+ kwargs = {extra_info[-1]: True}
+ else:
+ kwargs = {}
+
+ content = extra_info[0]
+
+ try:
+ code, country = content.split("-")
+ except ValueError:
+ lang = CustomLanguage.from_value(content)
+ if lang is not None:
+ lang = lang.subzero_language()
+ return lang.rebuild(lang, **kwargs)
+
+ code, country = content, None
+
+ return subliminal_patch.core.Language(code, country, **kwargs)
+
+
+def get_language_equals(settings_=None):
+ settings_ = settings_ or settings
+
+ equals = get_array_from(settings_.general.language_equals)
+ if not equals:
+ return []
+
+ items = []
+ for equal in equals:
+ try:
+ from_, to_ = equal.split(":")
+ from_, to_ = _lang_from_str(from_), _lang_from_str(to_)
+ except Exception as error:
+ logging.info("Invalid equal value: '%s' [%s]", equal, error)
+ else:
+ items.append((from_, to_))
+
+ return items
+
+
def get_providers():
providers_list = []
existing_providers = provider_registry.names()
diff --git a/bazarr/subtitles/pool.py b/bazarr/subtitles/pool.py
index c1c771807..c70e8f98c 100644
--- a/bazarr/subtitles/pool.py
+++ b/bazarr/subtitles/pool.py
@@ -8,7 +8,7 @@ from inspect import getfullargspec
from radarr.blacklist import get_blacklist_movie
from sonarr.blacklist import get_blacklist
-from app.get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool
+from app.get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool, get_language_equals
from .utils import get_ban_list
@@ -19,10 +19,11 @@ def _init_pool(media_type, profile_id=None, providers=None):
return pool(
providers=providers or get_providers(),
provider_configs=get_providers_auth(),
- blacklist=get_blacklist() if media_type == 'series' else get_blacklist_movie(),
+ blacklist=get_blacklist() if media_type == "series" else get_blacklist_movie(),
throttle_callback=provider_throttle,
ban_list=get_ban_list(profile_id),
language_hook=None,
+ language_equals=get_language_equals(),
)
@@ -54,8 +55,19 @@ def _update_pool(media_type, profile_id=None):
return pool.update(
get_providers(),
get_providers_auth(),
- get_blacklist() if media_type == 'series' else get_blacklist_movie(),
+ get_blacklist() if media_type == "series" else get_blacklist_movie(),
get_ban_list(profile_id),
+ get_language_equals(),
+ )
+
+
+def _pool_update(pool, media_type, profile_id=None):
+ return pool.update(
+ get_providers(),
+ get_providers_auth(),
+ get_blacklist() if media_type == "series" else get_blacklist_movie(),
+ get_ban_list(profile_id),
+ get_language_equals(),
)
diff --git a/frontend/src/components/bazarr/LanguageSelector.tsx b/frontend/src/components/bazarr/LanguageSelector.tsx
new file mode 100644
index 000000000..84ce363d5
--- /dev/null
+++ b/frontend/src/components/bazarr/LanguageSelector.tsx
@@ -0,0 +1,34 @@
+import { useLanguages } from "@/apis/hooks";
+import { Selector, SelectorProps } from "@/components/inputs";
+import { useSelectorOptions } from "@/utilities";
+import { FunctionComponent, useMemo } from "react";
+
+interface LanguageSelectorProps
+ extends Omit<SelectorProps<Language.Server>, "options" | "getkey"> {
+ enabled?: boolean;
+}
+
+const LanguageSelector: FunctionComponent<LanguageSelectorProps> = ({
+ enabled = false,
+ ...selector
+}) => {
+ const { data } = useLanguages();
+
+ const filteredData = useMemo(() => {
+ if (enabled) {
+ return data?.filter((value) => value.enabled);
+ } else {
+ return data;
+ }
+ }, [data, enabled]);
+
+ const options = useSelectorOptions(
+ filteredData ?? [],
+ (value) => value.name,
+ (value) => value.code3
+ );
+
+ return <Selector {...options} searchable {...selector}></Selector>;
+};
+
+export default LanguageSelector;
diff --git a/frontend/src/pages/Settings/Languages/equals.test.ts b/frontend/src/pages/Settings/Languages/equals.test.ts
new file mode 100644
index 000000000..19c641fcb
--- /dev/null
+++ b/frontend/src/pages/Settings/Languages/equals.test.ts
@@ -0,0 +1,196 @@
+import {
+ decodeEqualData,
+ encodeEqualData,
+ LanguageEqualData,
+ LanguageEqualImmediateData,
+} from "@/pages/Settings/Languages/equals";
+import { describe, expect, it } from "vitest";
+
+describe("Equals Parser", () => {
+ it("should parse from string correctly", () => {
+ interface TestData {
+ text: string;
+ expected: LanguageEqualImmediateData;
+ }
+
+ function testParsedResult(
+ text: string,
+ expected: LanguageEqualImmediateData
+ ) {
+ const result = decodeEqualData(text);
+
+ if (result === undefined) {
+ expect(false, `Cannot parse '${text}' as language equal data`);
+ return;
+ }
+
+ expect(
+ result,
+ `${text} does not match with the expected equal data`
+ ).toStrictEqual(expected);
+ }
+
+ const testValues: TestData[] = [
+ {
+ text: "spa-MX:spa",
+ expected: {
+ source: {
+ content: "spa-MX",
+ hi: false,
+ forced: false,
+ },
+ target: {
+ content: "spa",
+ hi: false,
+ forced: false,
+ },
+ },
+ },
+ {
+ text: "zho@hi:zht",
+ expected: {
+ source: {
+ content: "zho",
+ hi: true,
+ forced: false,
+ },
+ target: {
+ content: "zht",
+ hi: false,
+ forced: false,
+ },
+ },
+ },
+ {
+ text: "es-MX@forced:es-MX",
+ expected: {
+ source: {
+ content: "es-MX",
+ hi: false,
+ forced: true,
+ },
+ target: {
+ content: "es-MX",
+ hi: false,
+ forced: false,
+ },
+ },
+ },
+ {
+ text: "en:en@hi",
+ expected: {
+ source: {
+ content: "en",
+ hi: false,
+ forced: false,
+ },
+ target: {
+ content: "en",
+ hi: true,
+ forced: false,
+ },
+ },
+ },
+ ];
+
+ testValues.forEach((data) => {
+ testParsedResult(data.text, data.expected);
+ });
+ });
+
+ it("should encode to string correctly", () => {
+ interface TestData {
+ source: LanguageEqualData;
+ expected: string;
+ }
+
+ const testValues: TestData[] = [
+ {
+ source: {
+ source: {
+ content: {
+ name: "Abkhazian",
+ code2: "ab",
+ code3: "abk",
+ enabled: false,
+ },
+ hi: false,
+ forced: false,
+ },
+ target: {
+ content: {
+ name: "Aragonese",
+ code2: "an",
+ code3: "arg",
+ enabled: false,
+ },
+ hi: false,
+ forced: false,
+ },
+ },
+ expected: "abk:arg",
+ },
+ {
+ source: {
+ source: {
+ content: {
+ name: "Abkhazian",
+ code2: "ab",
+ code3: "abk",
+ enabled: false,
+ },
+ hi: true,
+ forced: false,
+ },
+ target: {
+ content: {
+ name: "Aragonese",
+ code2: "an",
+ code3: "arg",
+ enabled: false,
+ },
+ hi: false,
+ forced: false,
+ },
+ },
+ expected: "abk@hi:arg",
+ },
+ {
+ source: {
+ source: {
+ content: {
+ name: "Abkhazian",
+ code2: "ab",
+ code3: "abk",
+ enabled: false,
+ },
+ hi: false,
+ forced: true,
+ },
+ target: {
+ content: {
+ name: "Aragonese",
+ code2: "an",
+ code3: "arg",
+ enabled: false,
+ },
+ hi: false,
+ forced: false,
+ },
+ },
+ expected: "abk@forced:arg",
+ },
+ ];
+
+ function testEncodeResult({ source, expected }: TestData) {
+ const encoded = encodeEqualData(source);
+
+ expect(
+ encoded,
+ `Encoded result '${encoded}' is not matched to '${expected}'`
+ ).toEqual(expected);
+ }
+
+ testValues.forEach(testEncodeResult);
+ });
+});
diff --git a/frontend/src/pages/Settings/Languages/equals.tsx b/frontend/src/pages/Settings/Languages/equals.tsx
new file mode 100644
index 000000000..a958237df
--- /dev/null
+++ b/frontend/src/pages/Settings/Languages/equals.tsx
@@ -0,0 +1,365 @@
+import { useLanguages } from "@/apis/hooks";
+import { Action, SimpleTable } from "@/components";
+import LanguageSelector from "@/components/bazarr/LanguageSelector";
+import { languageEqualsKey } from "@/pages/Settings/keys";
+import { useFormActions } from "@/pages/Settings/utilities/FormValues";
+import { useSettingValue } from "@/pages/Settings/utilities/hooks";
+import { LOG } from "@/utilities/console";
+import { faEquals, faTrash } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { Button, Checkbox } from "@mantine/core";
+import { FunctionComponent, useCallback, useMemo } from "react";
+import { Column } from "react-table";
+
+interface GenericEqualTarget<T> {
+ content: T;
+ hi: boolean;
+ forced: boolean;
+}
+
+interface LanguageEqualGenericData<T> {
+ source: GenericEqualTarget<T>;
+ target: GenericEqualTarget<T>;
+}
+
+export type LanguageEqualImmediateData =
+ LanguageEqualGenericData<Language.CodeType>;
+
+export type LanguageEqualData = LanguageEqualGenericData<Language.Server>;
+
+function decodeEqualTarget(
+ text: string
+): GenericEqualTarget<Language.CodeType> | undefined {
+ const [code, decoration] = text.split("@");
+
+ if (code.length === 0) {
+ return undefined;
+ }
+
+ const forced = decoration === "forced";
+ const hi = decoration === "hi";
+
+ return {
+ content: code,
+ forced,
+ hi,
+ };
+}
+
+export function decodeEqualData(
+ text: string
+): LanguageEqualImmediateData | undefined {
+ const [first, second] = text.split(":");
+
+ const source = decodeEqualTarget(first);
+ const target = decodeEqualTarget(second);
+
+ if (source === undefined || target === undefined) {
+ return undefined;
+ }
+
+ return {
+ source,
+ target,
+ };
+}
+
+function encodeEqualTarget(data: GenericEqualTarget<Language.Server>): string {
+ let text = data.content.code3;
+ if (data.hi) {
+ text += "@hi";
+ } else if (data.forced) {
+ text += "@forced";
+ }
+
+ return text;
+}
+
+export function encodeEqualData(data: LanguageEqualData): string {
+ const source = encodeEqualTarget(data.source);
+ const target = encodeEqualTarget(data.target);
+
+ return `${source}:${target}`;
+}
+
+export function useLatestLanguageEquals(): LanguageEqualData[] {
+ const { data } = useLanguages();
+
+ const latest = useSettingValue<string[]>(languageEqualsKey);
+
+ return useMemo(
+ () =>
+ latest
+ ?.map(decodeEqualData)
+ .map((parsed) => {
+ if (parsed === undefined) {
+ return undefined;
+ }
+
+ const source = data?.find(
+ (value) => value.code3 === parsed.source.content
+ );
+ const target = data?.find(
+ (value) => value.code3 === parsed.target.content
+ );
+
+ if (source === undefined || target === undefined) {
+ return undefined;
+ }
+
+ return {
+ source: { ...parsed.source, content: source },
+ target: { ...parsed.target, content: target },
+ };
+ })
+ .filter((v): v is LanguageEqualData => v !== undefined) ?? [],
+ [data, latest]
+ );
+}
+
+interface EqualsTableProps {}
+
+const EqualsTable: FunctionComponent<EqualsTableProps> = () => {
+ const { data: languages } = useLanguages();
+ const canAdd = languages !== undefined;
+
+ const equals = useLatestLanguageEquals();
+
+ const { setValue } = useFormActions();
+
+ const setEquals = useCallback(
+ (values: LanguageEqualData[]) => {
+ const encodedValues = values.map(encodeEqualData);
+
+ LOG("info", "updating language equals data", values);
+ setValue(encodedValues, languageEqualsKey);
+ },
+ [setValue]
+ );
+
+ const add = useCallback(() => {
+ if (languages === undefined) {
+ return;
+ }
+
+ const enabled = languages.find((value) => value.enabled);
+
+ if (enabled === undefined) {
+ return;
+ }
+
+ const newValue: LanguageEqualData[] = [
+ ...equals,
+ {
+ source: {
+ content: enabled,
+ hi: false,
+ forced: false,
+ },
+ target: {
+ content: enabled,
+ hi: false,
+ forced: false,
+ },
+ },
+ ];
+
+ setEquals(newValue);
+ }, [equals, languages, setEquals]);
+
+ const update = useCallback(
+ (index: number, value: LanguageEqualData) => {
+ if (index < 0 || index >= equals.length) {
+ return;
+ }
+
+ const newValue: LanguageEqualData[] = [...equals];
+
+ newValue[index] = { ...value };
+ setEquals(newValue);
+ },
+ [equals, setEquals]
+ );
+
+ const remove = useCallback(
+ (index: number) => {
+ if (index < 0 || index >= equals.length) {
+ return;
+ }
+
+ const newValue: LanguageEqualData[] = [...equals];
+
+ newValue.splice(index, 1);
+
+ setEquals(newValue);
+ },
+ [equals, setEquals]
+ );
+
+ const columns = useMemo<Column<LanguageEqualData>[]>(
+ () => [
+ {
+ Header: "Source",
+ id: "source-lang",
+ accessor: "source",
+ Cell: ({ value: { content }, row }) => {
+ return (
+ <LanguageSelector
+ enabled
+ value={content}
+ onChange={(result) => {
+ if (result !== null) {
+ update(row.index, {
+ ...row.original,
+ source: { ...row.original.source, content: result },
+ });
+ }
+ }}
+ ></LanguageSelector>
+ );
+ },
+ },
+ {
+ id: "source-hi",
+ accessor: "source",
+ Cell: ({ value: { hi }, row }) => {
+ return (
+ <Checkbox
+ label="HI"
+ checked={hi}
+ onChange={({ currentTarget: { checked } }) => {
+ update(row.index, {
+ ...row.original,
+ source: {
+ ...row.original.source,
+ hi: checked,
+ forced: checked ? false : row.original.source.forced,
+ },
+ });
+ }}
+ ></Checkbox>
+ );
+ },
+ },
+ {
+ id: "source-forced",
+ accessor: "source",
+ Cell: ({ value: { forced }, row }) => {
+ return (
+ <Checkbox
+ label="Forced"
+ checked={forced}
+ onChange={({ currentTarget: { checked } }) => {
+ update(row.index, {
+ ...row.original,
+ source: {
+ ...row.original.source,
+ forced: checked,
+ hi: checked ? false : row.original.source.hi,
+ },
+ });
+ }}
+ ></Checkbox>
+ );
+ },
+ },
+ {
+ id: "equal-icon",
+ Cell: () => {
+ return <FontAwesomeIcon icon={faEquals} />;
+ },
+ },
+ {
+ Header: "Target",
+ id: "target-lang",
+ accessor: "target",
+ Cell: ({ value: { content }, row }) => {
+ return (
+ <LanguageSelector
+ enabled
+ value={content}
+ onChange={(result) => {
+ if (result !== null) {
+ update(row.index, {
+ ...row.original,
+ target: { ...row.original.target, content: result },
+ });
+ }
+ }}
+ ></LanguageSelector>
+ );
+ },
+ },
+ {
+ id: "target-hi",
+ accessor: "target",
+ Cell: ({ value: { hi }, row }) => {
+ return (
+ <Checkbox
+ label="HI"
+ checked={hi}
+ onChange={({ currentTarget: { checked } }) => {
+ update(row.index, {
+ ...row.original,
+ target: {
+ ...row.original.target,
+ hi: checked,
+ forced: checked ? false : row.original.target.forced,
+ },
+ });
+ }}
+ ></Checkbox>
+ );
+ },
+ },
+ {
+ id: "target-forced",
+ accessor: "target",
+ Cell: ({ value: { forced }, row }) => {
+ return (
+ <Checkbox
+ label="Forced"
+ checked={forced}
+ onChange={({ currentTarget: { checked } }) => {
+ update(row.index, {
+ ...row.original,
+ target: {
+ ...row.original.target,
+ forced: checked,
+ hi: checked ? false : row.original.target.hi,
+ },
+ });
+ }}
+ ></Checkbox>
+ );
+ },
+ },
+ {
+ id: "action",
+ accessor: "target",
+ Cell: ({ row }) => {
+ return (
+ <Action
+ label="Remove"
+ icon={faTrash}
+ color="red"
+ onClick={() => remove(row.index)}
+ ></Action>
+ );
+ },
+ },
+ ],
+ [remove, update]
+ );
+
+ return (
+ <>
+ <SimpleTable data={equals} columns={columns}></SimpleTable>
+ <Button fullWidth disabled={!canAdd} color="light" onClick={add}>
+ {canAdd ? "Add Equal" : "No Enabled Languages"}
+ </Button>
+ </>
+ );
+};
+
+export default EqualsTable;
diff --git a/frontend/src/pages/Settings/Languages/index.tsx b/frontend/src/pages/Settings/Languages/index.tsx
index 993820478..762174133 100644
--- a/frontend/src/pages/Settings/Languages/index.tsx
+++ b/frontend/src/pages/Settings/Languages/index.tsx
@@ -17,6 +17,7 @@ import {
} from "../keys";
import { useSettingValue } from "../utilities/hooks";
import { LanguageSelector, ProfileSelector } from "./components";
+import EqualsTable from "./equals";
import Table from "./table";
export function useLatestEnabledLanguages() {
@@ -69,6 +70,13 @@ const SettingsLanguagesView: FunctionComponent = () => {
></LanguageSelector>
</Section>
+ <Section header="Language Equals">
+ <Message>
+ Treat the following languages as equal across all providers.
+ </Message>
+ <EqualsTable></EqualsTable>
+ </Section>
+
<Section header="Embedded Tracks Language">
<Check
label="Deep analyze media file to get audio tracks language."
@@ -91,7 +99,6 @@ const SettingsLanguagesView: FunctionComponent = () => {
}}
></Selector>
</CollapseBox>
-
<Selector
clearable
settingKey={defaultUndEmbeddedSubtitlesLang}
diff --git a/frontend/src/pages/Settings/keys.ts b/frontend/src/pages/Settings/keys.ts
index 40b6a252d..3d6444882 100644
--- a/frontend/src/pages/Settings/keys.ts
+++ b/frontend/src/pages/Settings/keys.ts
@@ -5,6 +5,8 @@ export const defaultUndEmbeddedSubtitlesLang =
export const languageProfileKey = "languages-profiles";
export const notificationsKey = "notifications-providers";
+export const languageEqualsKey = "settings-general-language_equals";
+
export const pathMappingsKey = "settings-general-path_mappings";
export const pathMappingsMovieKey = "settings-general-path_mappings_movie";
diff --git a/frontend/src/types/api.d.ts b/frontend/src/types/api.d.ts
index b19c682c0..75f7e4ebd 100644
--- a/frontend/src/types/api.d.ts
+++ b/frontend/src/types/api.d.ts
@@ -12,6 +12,7 @@ declare namespace Language {
type CodeType = string;
interface Server {
code2: CodeType;
+ code3: CodeType;
name: string;
enabled: boolean;
}
diff --git a/frontend/src/types/settings.d.ts b/frontend/src/types/settings.d.ts
index 26da89bd9..8d94794d3 100644
--- a/frontend/src/types/settings.d.ts
+++ b/frontend/src/types/settings.d.ts
@@ -25,6 +25,7 @@ interface Settings {
titlovi: Settings.Titlovi;
ktuvit: Settings.Ktuvit;
notifications: Settings.Notifications;
+ language_equals: string[][];
}
declare namespace Settings {
diff --git a/frontend/src/utilities/languages.ts b/frontend/src/utilities/languages.ts
index f80383797..282db22a9 100644
--- a/frontend/src/utilities/languages.ts
+++ b/frontend/src/utilities/languages.ts
@@ -42,3 +42,12 @@ export function useProfileItemsToLanguages(profile?: Language.Profile) {
[data, profile?.items]
);
}
+
+export function useLanguageFromCode3(code3: string) {
+ const { data } = useLanguages();
+
+ return useMemo(
+ () => data?.find((value) => value.code3 === code3),
+ [data, code3]
+ );
+}
diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py
index c31d5ecd0..b649e4288 100644
--- a/libs/subliminal_patch/core.py
+++ b/libs/subliminal_patch/core.py
@@ -153,9 +153,52 @@ class _Blacklist(list):
return not blacklisted
+class _LanguageEquals(list):
+ """ An optional config field for the pool. It will treat a couple of languages as equal for
+ list-subtitles operations. It's optional; its methods won't do anything if an empy list
+ is set.
+
+ Example usage: [(language_instance, language_instance), ...]"""
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ for item in self:
+ if len(item) != 2 or not any(isinstance(i, Language) for i in item):
+ raise ValueError(f"Not a valid equal tuple: {item}")
+
+ def check_set(self, items: set):
+ """ Check a set of languages. For example, if the set is {Language('es')} and one of the
+ equals of the instance is (Language('es'), Language('es', 'MX')), the set will now have
+ to {Language('es'), Language('es', 'MX')}.
+
+ It will return a copy of the original set to avoid messing up outside its scope.
+
+ Note that hearing_impaired and forced language attributes are not yet tested.
+ """
+ to_add = []
+ for equals in self:
+ from_, to_ = equals
+ if from_ in items:
+ logger.debug("Adding %s to %s", to_, items)
+ to_add.append(to_)
+
+ new_items = items.copy()
+ new_items.update(to_add)
+ logger.debug("New set: %s", new_items)
+ return new_items
+
+ def update_subtitle(self, subtitle):
+ for equals in self:
+ from_, to_ = equals
+ if from_ == subtitle.language:
+ logger.debug("Updating language for %s (to %s)", subtitle, to_)
+ subtitle.language = to_
+ break
+
+
class SZProviderPool(ProviderPool):
def __init__(self, providers=None, provider_configs=None, blacklist=None, ban_list=None, throttle_callback=None,
- pre_download_hook=None, post_download_hook=None, language_hook=None):
+ pre_download_hook=None, post_download_hook=None, language_hook=None, language_equals=None):
#: Name of providers to use
self.providers = set(providers or [])
@@ -170,6 +213,8 @@ class SZProviderPool(ProviderPool):
#: Should be a dict of 2 lists of strings
self.ban_list = _Banlist(**(ban_list or {'must_contain': [], 'must_not_contain': []}))
+ self.lang_equals = _LanguageEquals(language_equals or [])
+
self.throttle_callback = throttle_callback
self.pre_download_hook = pre_download_hook
@@ -185,7 +230,7 @@ class SZProviderPool(ProviderPool):
self.provider_configs = _ProviderConfigs(self)
self.provider_configs.update(provider_configs or {})
- def update(self, providers, provider_configs, blacklist, ban_list):
+ def update(self, providers, provider_configs, blacklist, ban_list, language_equals=None):
# Check if the pool was initialized enough hours ago
self._check_lifetime()
@@ -222,6 +267,7 @@ class SZProviderPool(ProviderPool):
self.blacklist = _Blacklist(blacklist or [])
self.ban_list = _Banlist(**ban_list or {'must_contain': [], 'must_not_contain': []})
+ self.lang_equals = _LanguageEquals(language_equals or [])
return updated
@@ -299,7 +345,7 @@ class SZProviderPool(ProviderPool):
return []
# check supported languages
- provider_languages = provider_registry[provider].languages & use_languages
+ provider_languages = self.lang_equals.check_set(set(provider_registry[provider].languages)) & use_languages
if not provider_languages:
logger.info('Skipping provider %r: no language to search for', provider)
return []
@@ -312,6 +358,8 @@ class SZProviderPool(ProviderPool):
seen = []
out = []
for s in results:
+ self.lang_equals.update_subtitle(s)
+
if not self.blacklist.is_valid(provider, s):
continue
@@ -569,7 +617,7 @@ class SZProviderPool(ProviderPool):
continue
# add the languages for this provider
- languages.append({'provider': name, 'languages': provider_languages})
+ languages.append({'provider': name, 'languages': self.lang_equals.check_set(set(provider_languages))})
return languages
diff --git a/tests/bazarr/app/test_get_providers.py b/tests/bazarr/app/test_get_providers.py
deleted file mode 100644
index caafa17d9..000000000
--- a/tests/bazarr/app/test_get_providers.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import pytest
-
-import inspect
-
-from bazarr.app import get_providers
-
-
-def test_get_providers_auth():
- for val in get_providers.get_providers_auth().values():
- assert isinstance(val, dict)
-
-
-def test_get_providers_auth_with_provider_registry():
- """Make sure all providers will be properly initialized with bazarr
- configs"""
- from subliminal_patch.extensions import provider_registry
-
- auths = get_providers.get_providers_auth()
- for key, val in auths.items():
- provider = provider_registry[key]
- sign = inspect.signature(provider.__init__)
- for sub_key in val.keys():
- if sub_key not in sign.parameters:
- raise ValueError(f"'{sub_key}' parameter not present in {provider}")
-
- assert sign.parameters[sub_key] is not None
-
-
-def test_get_providers_auth_embeddedsubtitles():
- item = get_providers.get_providers_auth()["embeddedsubtitles"]
- assert isinstance(item["included_codecs"], list)
- assert isinstance(item["hi_fallback"], bool)
- assert isinstance(item["cache_dir"], str)
- assert isinstance(item["ffprobe_path"], str)
- assert isinstance(item["ffmpeg_path"], str)
- assert isinstance(item["timeout"], str)
- assert isinstance(item["unknown_as_english"], bool)
-
-
-def test_get_providers_auth_karagarga():
- item = get_providers.get_providers_auth()["karagarga"]
- assert item["username"] is not None
- assert item["password"] is not None
- assert item["f_username"] is not None
- assert item["f_password"] is not None
diff --git a/tests/bazarr/conftest.py b/tests/bazarr/conftest.py
index 865b92767..d4bc23974 100644
--- a/tests/bazarr/conftest.py
+++ b/tests/bazarr/conftest.py
@@ -3,5 +3,6 @@ import logging
os.environ["NO_CLI"] = "true"
os.environ["SZ_USER_AGENT"] = "test"
+os.environ["BAZARR_VERSION"] = "test" # fixme
logging.getLogger("rebulk").setLevel(logging.WARNING)
diff --git a/tests/bazarr/test_app_get_providers.py b/tests/bazarr/test_app_get_providers.py
new file mode 100644
index 000000000..e6a31a39f
--- /dev/null
+++ b/tests/bazarr/test_app_get_providers.py
@@ -0,0 +1,115 @@
+import inspect
+
+import pytest
+from subliminal_patch.core import Language
+
+from bazarr.app import get_providers
+
+
+def test_get_providers_auth():
+ for val in get_providers.get_providers_auth().values():
+ assert isinstance(val, dict)
+
+
+def test_get_providers_auth_with_provider_registry():
+ """Make sure all providers will be properly initialized with bazarr
+ configs"""
+ from subliminal_patch.extensions import provider_registry
+
+ auths = get_providers.get_providers_auth()
+ for key, val in auths.items():
+ provider = provider_registry[key]
+ sign = inspect.signature(provider.__init__)
+ for sub_key in val.keys():
+ if sub_key not in sign.parameters:
+ raise ValueError(f"'{sub_key}' parameter not present in {provider}")
+
+ assert sign.parameters[sub_key] is not None
+
+
+def test_get_providers_auth_embeddedsubtitles():
+ item = get_providers.get_providers_auth()["embeddedsubtitles"]
+ assert isinstance(item["included_codecs"], list)
+ assert isinstance(item["hi_fallback"], bool)
+ assert isinstance(item["cache_dir"], str)
+ assert isinstance(item["ffprobe_path"], str)
+ assert isinstance(item["ffmpeg_path"], str)
+ assert isinstance(item["timeout"], str)
+ assert isinstance(item["unknown_as_english"], bool)
+
+
+def test_get_providers_auth_karagarga():
+ item = get_providers.get_providers_auth()["karagarga"]
+ assert item["username"] is not None
+ assert item["password"] is not None
+ assert item["f_username"] is not None
+ assert item["f_password"] is not None
+
+
+def test_get_language_equals_default_settings():
+ assert isinstance(get_providers.get_language_equals(), list)
+
+
+def test_get_language_equals_injected_settings_invalid():
+ config = get_providers.settings
+ config.set("general", "language_equals", '["invalid"]')
+ assert not get_providers.get_language_equals(config)
+
+
+def test_get_language_equals_injected_settings_valid():
+ config = get_providers.settings
+ config.set("general", "language_equals", '["spa:spa-MX"]')
+
+ result = get_providers.get_language_equals(config)
+ assert result == [(Language("spa"), Language("spa", "MX"))]
+
+
+ "config_value,expected",
+ [
+ ('["spa:spl"]', (Language("spa"), Language("spa", "MX"))),
+ ('["por:pob"]', (Language("por"), Language("por", "BR"))),
+ ('["zho:zht"]', (Language("zho"), Language("zho", "TW"))),
+ ],
+)
+def test_get_language_equals_injected_settings_custom_lang_alpha3(
+ config_value, expected
+):
+ config = get_providers.settings
+
+ config.set("general", "language_equals", config_value)
+
+ result = get_providers.get_language_equals(config)
+ assert result == [expected]
+
+
+def test_get_language_equals_injected_settings_multiple():
+ config = get_providers.settings
+
+ config.set(
+ "general",
+ "language_equals",
+ "['eng@hi:eng', 'spa:spl', 'spa@hi:spl', 'spl@hi:spl']",
+ )
+
+ result = get_providers.get_language_equals(config)
+ assert len(result) == 4
+
+
+def test_get_language_equals_injected_settings_valid_multiple():
+ config = get_providers.settings
+ config.set("general", "language_equals", '["spa:spa-MX", "spa-MX:spa"]')
+
+ result = get_providers.get_language_equals(config)
+ assert result == [
+ (Language("spa"), Language("spa", "MX")),
+ (Language("spa", "MX"), Language("spa")),
+ ]
+
+
+def test_get_language_equals_injected_settings_hi():
+ config = get_providers.settings
+ config.set("general", "language_equals", '["eng@hi:eng"]')
+
+ result = get_providers.get_language_equals(config)
+ assert result == [(Language("eng", hi=True), Language("eng"))]
diff --git a/tests/bazarr/test_subtitles_pool.py b/tests/bazarr/test_subtitles_pool.py
new file mode 100644
index 000000000..862c6d493
--- /dev/null
+++ b/tests/bazarr/test_subtitles_pool.py
@@ -0,0 +1,10 @@
+from bazarr.subtitles import pool
+
+
+def test_init_pool():
+ assert pool._init_pool("movie")
+
+
+def test_pool_update():
+ pool_ = pool._init_pool("movie")
+ assert pool._pool_update(pool_, "movie")
diff --git a/tests/subliminal_patch/test_core.py b/tests/subliminal_patch/test_core.py
index d6481ee16..fadf4e493 100644
--- a/tests/subliminal_patch/test_core.py
+++ b/tests/subliminal_patch/test_core.py
@@ -70,3 +70,99 @@ def test_pool_update_discarded_providers_2(pool_instance):
# Provider should not disappear from discarded providers
assert pool_instance.discarded_providers == {"argenteam"}
+
+
+def test_language_equals_init():
+ assert core._LanguageEquals([(core.Language("spa"), core.Language("spa", "MX"))])
+
+
+def test_language_equals_init_invalid():
+ with pytest.raises(ValueError):
+ assert core._LanguageEquals([(core.Language("spa", "MX"),)])
+
+
+def test_language_equals_init_empty_list_gracefully():
+ assert core._LanguageEquals([]) == []
+
+
+ "langs",
+ [
+ [(core.Language("spa"), core.Language("spa", "MX"))],
+ [(core.Language("por"), core.Language("por", "BR"))],
+ [(core.Language("zho"), core.Language("zho", "TW"))],
+ ],
+)
+def test_language_equals_check_set(langs):
+ equals = core._LanguageEquals(langs)
+ lang_set = {langs[0]}
+ assert equals.check_set(lang_set) == set(langs)
+
+
+def test_language_equals_check_set_do_nothing():
+ equals = core._LanguageEquals([(core.Language("eng"), core.Language("spa"))])
+ lang_set = {core.Language("spa")}
+ assert equals.check_set(lang_set) == {core.Language("spa")}
+
+
+def test_language_equals_check_set_do_nothing_w_forced():
+ equals = core._LanguageEquals(
+ [(core.Language("spa", forced=True), core.Language("spa", "MX"))]
+ )
+ lang_set = {core.Language("spa")}
+ assert equals.check_set(lang_set) == {core.Language("spa")}
+
+
+def language_equals_pool_intance():
+ equals = [(core.Language("spa"), core.Language("spa", "MX"))]
+ yield core.SZProviderPool({"subdivx"}, language_equals=equals)
+
+
+def test_language_equals_pool_intance_list_subtitles(
+ language_equals_pool_intance, movies
+):
+ subs = language_equals_pool_intance.list_subtitles(
+ movies["dune"], {core.Language("spa")}
+ )
+ assert subs
+ assert all(sub.language == core.Language("spa", "MX") for sub in subs)
+
+
+def test_language_equals_pool_intance_list_subtitles_reversed(movies):
+ equals = [(core.Language("spa", "MX"), core.Language("spa"))]
+ language_equals_pool_intance = core.SZProviderPool(
+ {"subdivx"}, language_equals=equals
+ )
+ subs = language_equals_pool_intance.list_subtitles(
+ movies["dune"], {core.Language("spa")}
+ )
+ assert subs
+ assert all(sub.language == core.Language("spa") for sub in subs)
+
+
+def test_language_equals_pool_intance_list_subtitles_empty_lang_equals(movies):
+ language_equals_pool_intance = core.SZProviderPool(
+ {"subdivx"}, language_equals=None
+ )
+ subs = language_equals_pool_intance.list_subtitles(
+ movies["dune"], {core.Language("spa")}
+ )
+ assert subs
+ assert not all(sub.language == core.Language("spa", "MX") for sub in subs)
+
+
+def test_language_equals_pool_intance_list_subtitles_return_nothing(movies):
+ equals = [
+ (core.Language("spa", "MX"), core.Language("eng")),
+ (core.Language("spa"), core.Language("eng")),
+ ]
+ language_equals_pool_intance = core.SZProviderPool(
+ {"subdivx"}, language_equals=equals
+ )
+ subs = language_equals_pool_intance.list_subtitles(
+ movies["dune"], {core.Language("spa")}
+ )
+ assert not language_equals_pool_intance.download_best_subtitles(
+ subs, movies["dune"], {core.Language("spa")}
+ )