Added feature to treat couples of languages as equal when searching for subtitlesv1.2.2-beta.9

* Add 'Language-equals' support

This feature will treat couples of languages as equal for list-subtitles
operations. It's optional; its methods won't do anything if an empy list
is set. See more info at docstrings from 'subliminal_patch.core'.

For example, let's say I only want to have "Spanish (es.srt)" subtitles
and I don't care about the differences between Spain and LATAM spanish.
This feature will allow me to always get European Spanish even from
LATAM Spanish providers like Argenteam and Subdivx.

Example for config.ini:

language_equals = ['spa-MX:spa']

(Which means all Latam Spanish subtitles from every provider will be
converted to European Spanish)

* Add PT and ZH language tests

* Add HI and Forced parsing for language pairs

Format example: ["en@HI:en", "es-MX@forced:es-MX"]

* Update languages.py

* Update API definition to reflect the previous change

* Add language equals table to the UI (test only)

* Add global language selector and get language from code3 utilities

* Add unit tests for language equal feature

* Add encode function to language equal feature

* Add CRUD methods to the language equals panel

* Add equals description

* Add parsing support for alpha3 custom languages

* no log: add more tests

* Add forced and hi support to the language equal target

---------

Co-authored-by: morpheus65535 <[email protected]>
Co-authored-by: LASER-Yi <[email protected]>

8 files changed, 616 insertions, 1 deletions

diff --git a/frontend/src/components/bazarr/LanguageSelector.tsx b/frontend/src/components/bazarr/LanguageSelector.tsx
new file mode 100644
index 000000000..84ce363d5
--- /dev/null
+++ b/frontend/src/components/bazarr/LanguageSelector.tsx
@@ -0,0 +1,34 @@
+import { useLanguages } from "@/apis/hooks";
+import { Selector, SelectorProps } from "@/components/inputs";
+import { useSelectorOptions } from "@/utilities";
+import { FunctionComponent, useMemo } from "react";
+
+interface LanguageSelectorProps
+  extends Omit<SelectorProps<Language.Server>, "options" | "getkey"> {
+  enabled?: boolean;
+}
+
+const LanguageSelector: FunctionComponent<LanguageSelectorProps> = ({
+  enabled = false,
+  ...selector
+}) => {
+  const { data } = useLanguages();
+
+  const filteredData = useMemo(() => {
+    if (enabled) {
+      return data?.filter((value) => value.enabled);
+    } else {
+      return data;
+    }
+  }, [data, enabled]);
+
+  const options = useSelectorOptions(
+    filteredData ?? [],
+    (value) => value.name,
+    (value) => value.code3
+  );
+
+  return <Selector {...options} searchable {...selector}></Selector>;
+};
+
+export default LanguageSelector;
diff --git a/frontend/src/pages/Settings/Languages/equals.test.ts b/frontend/src/pages/Settings/Languages/equals.test.ts
new file mode 100644
index 000000000..19c641fcb
--- /dev/null
+++ b/frontend/src/pages/Settings/Languages/equals.test.ts
@@ -0,0 +1,196 @@
+import {
+  decodeEqualData,
+  encodeEqualData,
+  LanguageEqualData,
+  LanguageEqualImmediateData,
+} from "@/pages/Settings/Languages/equals";
+import { describe, expect, it } from "vitest";
+
+describe("Equals Parser", () => {
+  it("should parse from string correctly", () => {
+    interface TestData {
+      text: string;
+      expected: LanguageEqualImmediateData;
+    }
+
+    function testParsedResult(
+      text: string,
+      expected: LanguageEqualImmediateData
+    ) {
+      const result = decodeEqualData(text);
+
+      if (result === undefined) {
+        expect(false, `Cannot parse '${text}' as language equal data`);
+        return;
+      }
+
+      expect(
+        result,
+        `${text} does not match with the expected equal data`
+      ).toStrictEqual(expected);
+    }
+
+    const testValues: TestData[] = [
+      {
+        text: "spa-MX:spa",
+        expected: {
+          source: {
+            content: "spa-MX",
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: "spa",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "zho@hi:zht",
+        expected: {
+          source: {
+            content: "zho",
+            hi: true,
+            forced: false,
+          },
+          target: {
+            content: "zht",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "es-MX@forced:es-MX",
+        expected: {
+          source: {
+            content: "es-MX",
+            hi: false,
+            forced: true,
+          },
+          target: {
+            content: "es-MX",
+            hi: false,
+            forced: false,
+          },
+        },
+      },
+      {
+        text: "en:en@hi",
+        expected: {
+          source: {
+            content: "en",
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: "en",
+            hi: true,
+            forced: false,
+          },
+        },
+      },
+    ];
+
+    testValues.forEach((data) => {
+      testParsedResult(data.text, data.expected);
+    });
+  });
+
+  it("should encode to string correctly", () => {
+    interface TestData {
+      source: LanguageEqualData;
+      expected: string;
+    }
+
+    const testValues: TestData[] = [
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk:arg",
+      },
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: true,
+            forced: false,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk@hi:arg",
+      },
+      {
+        source: {
+          source: {
+            content: {
+              name: "Abkhazian",
+              code2: "ab",
+              code3: "abk",
+              enabled: false,
+            },
+            hi: false,
+            forced: true,
+          },
+          target: {
+            content: {
+              name: "Aragonese",
+              code2: "an",
+              code3: "arg",
+              enabled: false,
+            },
+            hi: false,
+            forced: false,
+          },
+        },
+        expected: "abk@forced:arg",
+      },
+    ];
+
+    function testEncodeResult({ source, expected }: TestData) {
+      const encoded = encodeEqualData(source);
+
+      expect(
+        encoded,
+        `Encoded result '${encoded}' is not matched to '${expected}'`
+      ).toEqual(expected);
+    }
+
+    testValues.forEach(testEncodeResult);
+  });
+});
diff --git a/frontend/src/pages/Settings/Languages/equals.tsx b/frontend/src/pages/Settings/Languages/equals.tsx
new file mode 100644
index 000000000..a958237df
--- /dev/null
+++ b/frontend/src/pages/Settings/Languages/equals.tsx
@@ -0,0 +1,365 @@
+import { useLanguages } from "@/apis/hooks";
+import { Action, SimpleTable } from "@/components";
+import LanguageSelector from "@/components/bazarr/LanguageSelector";
+import { languageEqualsKey } from "@/pages/Settings/keys";
+import { useFormActions } from "@/pages/Settings/utilities/FormValues";
+import { useSettingValue } from "@/pages/Settings/utilities/hooks";
+import { LOG } from "@/utilities/console";
+import { faEquals, faTrash } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { Button, Checkbox } from "@mantine/core";
+import { FunctionComponent, useCallback, useMemo } from "react";
+import { Column } from "react-table";
+
+interface GenericEqualTarget<T> {
+  content: T;
+  hi: boolean;
+  forced: boolean;
+}
+
+interface LanguageEqualGenericData<T> {
+  source: GenericEqualTarget<T>;
+  target: GenericEqualTarget<T>;
+}
+
+export type LanguageEqualImmediateData =
+  LanguageEqualGenericData<Language.CodeType>;
+
+export type LanguageEqualData = LanguageEqualGenericData<Language.Server>;
+
+function decodeEqualTarget(
+  text: string
+): GenericEqualTarget<Language.CodeType> | undefined {
+  const [code, decoration] = text.split("@");
+
+  if (code.length === 0) {
+    return undefined;
+  }
+
+  const forced = decoration === "forced";
+  const hi = decoration === "hi";
+
+  return {
+    content: code,
+    forced,
+    hi,
+  };
+}
+
+export function decodeEqualData(
+  text: string
+): LanguageEqualImmediateData | undefined {
+  const [first, second] = text.split(":");
+
+  const source = decodeEqualTarget(first);
+  const target = decodeEqualTarget(second);
+
+  if (source === undefined || target === undefined) {
+    return undefined;
+  }
+
+  return {
+    source,
+    target,
+  };
+}
+
+function encodeEqualTarget(data: GenericEqualTarget<Language.Server>): string {
+  let text = data.content.code3;
+  if (data.hi) {
+    text += "@hi";
+  } else if (data.forced) {
+    text += "@forced";
+  }
+
+  return text;
+}
+
+export function encodeEqualData(data: LanguageEqualData): string {
+  const source = encodeEqualTarget(data.source);
+  const target = encodeEqualTarget(data.target);
+
+  return `${source}:${target}`;
+}
+
+export function useLatestLanguageEquals(): LanguageEqualData[] {
+  const { data } = useLanguages();
+
+  const latest = useSettingValue<string[]>(languageEqualsKey);
+
+  return useMemo(
+    () =>
+      latest
+        ?.map(decodeEqualData)
+        .map((parsed) => {
+          if (parsed === undefined) {
+            return undefined;
+          }
+
+          const source = data?.find(
+            (value) => value.code3 === parsed.source.content
+          );
+          const target = data?.find(
+            (value) => value.code3 === parsed.target.content
+          );
+
+          if (source === undefined || target === undefined) {
+            return undefined;
+          }
+
+          return {
+            source: { ...parsed.source, content: source },
+            target: { ...parsed.target, content: target },
+          };
+        })
+        .filter((v): v is LanguageEqualData => v !== undefined) ?? [],
+    [data, latest]
+  );
+}
+
+interface EqualsTableProps {}
+
+const EqualsTable: FunctionComponent<EqualsTableProps> = () => {
+  const { data: languages } = useLanguages();
+  const canAdd = languages !== undefined;
+
+  const equals = useLatestLanguageEquals();
+
+  const { setValue } = useFormActions();
+
+  const setEquals = useCallback(
+    (values: LanguageEqualData[]) => {
+      const encodedValues = values.map(encodeEqualData);
+
+      LOG("info", "updating language equals data", values);
+      setValue(encodedValues, languageEqualsKey);
+    },
+    [setValue]
+  );
+
+  const add = useCallback(() => {
+    if (languages === undefined) {
+      return;
+    }
+
+    const enabled = languages.find((value) => value.enabled);
+
+    if (enabled === undefined) {
+      return;
+    }
+
+    const newValue: LanguageEqualData[] = [
+      ...equals,
+      {
+        source: {
+          content: enabled,
+          hi: false,
+          forced: false,
+        },
+        target: {
+          content: enabled,
+          hi: false,
+          forced: false,
+        },
+      },
+    ];
+
+    setEquals(newValue);
+  }, [equals, languages, setEquals]);
+
+  const update = useCallback(
+    (index: number, value: LanguageEqualData) => {
+      if (index < 0 || index >= equals.length) {
+        return;
+      }
+
+      const newValue: LanguageEqualData[] = [...equals];
+
+      newValue[index] = { ...value };
+      setEquals(newValue);
+    },
+    [equals, setEquals]
+  );
+
+  const remove = useCallback(
+    (index: number) => {
+      if (index < 0 || index >= equals.length) {
+        return;
+      }
+
+      const newValue: LanguageEqualData[] = [...equals];
+
+      newValue.splice(index, 1);
+
+      setEquals(newValue);
+    },
+    [equals, setEquals]
+  );
+
+  const columns = useMemo<Column<LanguageEqualData>[]>(
+    () => [
+      {
+        Header: "Source",
+        id: "source-lang",
+        accessor: "source",
+        Cell: ({ value: { content }, row }) => {
+          return (
+            <LanguageSelector
+              enabled
+              value={content}
+              onChange={(result) => {
+                if (result !== null) {
+                  update(row.index, {
+                    ...row.original,
+                    source: { ...row.original.source, content: result },
+                  });
+                }
+              }}
+            ></LanguageSelector>
+          );
+        },
+      },
+      {
+        id: "source-hi",
+        accessor: "source",
+        Cell: ({ value: { hi }, row }) => {
+          return (
+            <Checkbox
+              label="HI"
+              checked={hi}
+              onChange={({ currentTarget: { checked } }) => {
+                update(row.index, {
+                  ...row.original,
+                  source: {
+                    ...row.original.source,
+                    hi: checked,
+                    forced: checked ? false : row.original.source.forced,
+                  },
+                });
+              }}
+            ></Checkbox>
+          );
+        },
+      },
+      {
+        id: "source-forced",
+        accessor: "source",
+        Cell: ({ value: { forced }, row }) => {
+          return (
+            <Checkbox
+              label="Forced"
+              checked={forced}
+              onChange={({ currentTarget: { checked } }) => {
+                update(row.index, {
+                  ...row.original,
+                  source: {
+                    ...row.original.source,
+                    forced: checked,
+                    hi: checked ? false : row.original.source.hi,
+                  },
+                });
+              }}
+            ></Checkbox>
+          );
+        },
+      },
+      {
+        id: "equal-icon",
+        Cell: () => {
+          return <FontAwesomeIcon icon={faEquals} />;
+        },
+      },
+      {
+        Header: "Target",
+        id: "target-lang",
+        accessor: "target",
+        Cell: ({ value: { content }, row }) => {
+          return (
+            <LanguageSelector
+              enabled
+              value={content}
+              onChange={(result) => {
+                if (result !== null) {
+                  update(row.index, {
+                    ...row.original,
+                    target: { ...row.original.target, content: result },
+                  });
+                }
+              }}
+            ></LanguageSelector>
+          );
+        },
+      },
+      {
+        id: "target-hi",
+        accessor: "target",
+        Cell: ({ value: { hi }, row }) => {
+          return (
+            <Checkbox
+              label="HI"
+              checked={hi}
+              onChange={({ currentTarget: { checked } }) => {
+                update(row.index, {
+                  ...row.original,
+                  target: {
+                    ...row.original.target,
+                    hi: checked,
+                    forced: checked ? false : row.original.target.forced,
+                  },
+                });
+              }}
+            ></Checkbox>
+          );
+        },
+      },
+      {
+        id: "target-forced",
+        accessor: "target",
+        Cell: ({ value: { forced }, row }) => {
+          return (
+            <Checkbox
+              label="Forced"
+              checked={forced}
+              onChange={({ currentTarget: { checked } }) => {
+                update(row.index, {
+                  ...row.original,
+                  target: {
+                    ...row.original.target,
+                    forced: checked,
+                    hi: checked ? false : row.original.target.hi,
+                  },
+                });
+              }}
+            ></Checkbox>
+          );
+        },
+      },
+      {
+        id: "action",
+        accessor: "target",
+        Cell: ({ row }) => {
+          return (
+            <Action
+              label="Remove"
+              icon={faTrash}
+              color="red"
+              onClick={() => remove(row.index)}
+            ></Action>
+          );
+        },
+      },
+    ],
+    [remove, update]
+  );
+
+  return (
+    <>
+      <SimpleTable data={equals} columns={columns}></SimpleTable>
+      <Button fullWidth disabled={!canAdd} color="light" onClick={add}>
+        {canAdd ? "Add Equal" : "No Enabled Languages"}
+      </Button>
+    </>
+  );
+};
+
+export default EqualsTable;
diff --git a/frontend/src/pages/Settings/Languages/index.tsx b/frontend/src/pages/Settings/Languages/index.tsx
index 993820478..762174133 100644
--- a/frontend/src/pages/Settings/Languages/index.tsx
+++ b/frontend/src/pages/Settings/Languages/index.tsx
@@ -17,6 +17,7 @@ import {
 } from "../keys";
 import { useSettingValue } from "../utilities/hooks";
 import { LanguageSelector, ProfileSelector } from "./components";
+import EqualsTable from "./equals";
 import Table from "./table";
 
 export function useLatestEnabledLanguages() {
@@ -69,6 +70,13 @@ const SettingsLanguagesView: FunctionComponent = () => {
         ></LanguageSelector>
       </Section>
 
+      <Section header="Language Equals">
+        <Message>
+          Treat the following languages as equal across all providers.
+        </Message>
+        <EqualsTable></EqualsTable>
+      </Section>
+
       <Section header="Embedded Tracks Language">
         <Check
           label="Deep analyze media file to get audio tracks language."
@@ -91,7 +99,6 @@ const SettingsLanguagesView: FunctionComponent = () => {
             }}
           ></Selector>
         </CollapseBox>
-
         <Selector
           clearable
           settingKey={defaultUndEmbeddedSubtitlesLang}
diff --git a/frontend/src/pages/Settings/keys.ts b/frontend/src/pages/Settings/keys.ts
index 40b6a252d..3d6444882 100644
--- a/frontend/src/pages/Settings/keys.ts
+++ b/frontend/src/pages/Settings/keys.ts
@@ -5,6 +5,8 @@ export const defaultUndEmbeddedSubtitlesLang =
 export const languageProfileKey = "languages-profiles";
 export const notificationsKey = "notifications-providers";
 
+export const languageEqualsKey = "settings-general-language_equals";
+
 export const pathMappingsKey = "settings-general-path_mappings";
 export const pathMappingsMovieKey = "settings-general-path_mappings_movie";
 
diff --git a/frontend/src/types/api.d.ts b/frontend/src/types/api.d.ts
index b19c682c0..75f7e4ebd 100644
--- a/frontend/src/types/api.d.ts
+++ b/frontend/src/types/api.d.ts
@@ -12,6 +12,7 @@ declare namespace Language {
   type CodeType = string;
   interface Server {
     code2: CodeType;
+    code3: CodeType;
     name: string;
     enabled: boolean;
   }
diff --git a/frontend/src/types/settings.d.ts b/frontend/src/types/settings.d.ts
index 26da89bd9..8d94794d3 100644
--- a/frontend/src/types/settings.d.ts
+++ b/frontend/src/types/settings.d.ts
@@ -25,6 +25,7 @@ interface Settings {
   titlovi: Settings.Titlovi;
   ktuvit: Settings.Ktuvit;
   notifications: Settings.Notifications;
+  language_equals: string[][];
 }
 
 declare namespace Settings {
diff --git a/frontend/src/utilities/languages.ts b/frontend/src/utilities/languages.ts
index f80383797..282db22a9 100644
--- a/frontend/src/utilities/languages.ts
+++ b/frontend/src/utilities/languages.ts
@@ -42,3 +42,12 @@ export function useProfileItemsToLanguages(profile?: Language.Profile) {
     [data, profile?.items]
   );
 }
+
+export function useLanguageFromCode3(code3: string) {
+  const { data } = useLanguages();
+
+  return useMemo(
+    () => data?.find((value) => value.code3 === code3),
+    [data, code3]
+  );
+}