diff options
| -rw-r--r-- | .github/workflows/updateInvidous.yml | 7 | ||||
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | ci/generateList.ts | 63 | ||||
| -rw-r--r-- | ci/invidiousCI.ts | 36 | ||||
| -rw-r--r-- | ci/pipedCI.ts | 92 | ||||
| -rw-r--r-- | package.json | 2 |
6 files changed, 168 insertions, 35 deletions
diff --git a/.github/workflows/updateInvidous.yml b/.github/workflows/updateInvidous.yml index 29b42af8..0dcf556c 100644 --- a/.github/workflows/updateInvidous.yml +++ b/.github/workflows/updateInvidous.yml @@ -11,9 +11,10 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive - - name: Download instance list + - name: Download instance lists run: | - wget https://api.invidious.io/instances.json -O ci/data.json + wget https://api.invidious.io/instances.json -O ci/invidious_instances.json + wget https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json -O ci/piped_instances.json - name: Install dependencies run: npm ci - name: "Run CI" @@ -24,7 +25,7 @@ jobs: # v4.2.3 with: commit-message: Update Invidious List - author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + author: github-actions[bot] <github-actions[bot]@users.noreply.github.com> branch: ci/update_invidious_list title: Update Invidious List body: Automated Invidious list update
\ No newline at end of file @@ -7,5 +7,6 @@ web-ext-artifacts dist/ tmp/ .DS_Store -ci/data.json +ci/invidious_instances.json +ci/piped_instances.json test-results
\ No newline at end of file diff --git a/ci/generateList.ts b/ci/generateList.ts new file mode 100644 index 00000000..7c0f8882 --- /dev/null +++ b/ci/generateList.ts @@ -0,0 +1,63 @@ +/* +This file is only ran by GitHub Actions in order to populate the Invidious instances list + +This file should not be shipped with the extension +*/ + +/* +Criteria for inclusion: +Invidious +- 30d uptime >= 90% +- available for at least 80/90 days +- must have been up for at least 90 days +- HTTPS only +- url includes name (this is to avoid redirects) + +Piped +- 30d uptime >= 90% +- available for at least 80/90 days +- must have been up for at least 90 days +- must not be a wildcard redirect to piped.video +- must be currently up +- must have a functioning frontend +- must have a functioning API +*/ + +import { writeFile, existsSync } from "fs" +import { join } from "path" +import { getInvidiousList } from "./invidiousCI"; +// import { getPipedList } from "./pipedCI"; + +const checkPath = (path: string) => existsSync(path); +const fixArray = (arr: string[]) => [...new Set(arr)].sort() + +async function generateList() { + // import file from https://api.invidious.io/instances.json + const invidiousPath = join(__dirname, "invidious_instances.json"); + // import file from https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json + const pipedPath = join(__dirname, "piped_instances.json"); + + // check if files exist + if (!checkPath(invidiousPath) || !checkPath(pipedPath)) { + console.log("Missing files") + process.exit(1); + } + + // static non-invidious instances + const staticInstances = ["www.youtubekids.com"]; + // invidious instances + const invidiousList = fixArray(getInvidiousList()) + // piped instnaces + // const pipedList = fixArray(await getPipedList()) + + console.log([...staticInstances, ...invidiousList]) + + writeFile( + join(__dirname, "./invidiouslist.json"), + JSON.stringify([...staticInstances, ...invidiousList]), + (err) => { + if (err) return console.log(err); + } + ); +} +generateList() diff --git a/ci/invidiousCI.ts b/ci/invidiousCI.ts index 1e2e8348..d27a3a4e 100644 --- a/ci/invidiousCI.ts +++ b/ci/invidiousCI.ts @@ -1,23 +1,6 @@ -/* -This file is only ran by GitHub Actions in order to populate the Invidious instances list - -This file should not be shipped with the extension -*/ - -import { writeFile, existsSync } from "fs" -import { join } from "path" import { InvidiousInstance, instanceMap } from "./invidiousType" -// import file from https://api.invidious.io/instances.json -if (!existsSync(join(__dirname, "data.json"))) { - process.exit(1); -} -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -import * as data from "../ci/data.json"; - -// static non-invidious instances -const staticInstances = ["www.youtubekids.com"]; +import * as data from "../ci/invidious_instances.json"; // only https servers const mapped: instanceMap = data @@ -33,7 +16,7 @@ const mapped: instanceMap = data // reliability and sanity checks const reliableCheck = mapped - .filter((instance) => { + .filter(instance => { // 30d uptime >= 90% const thirtyDayUptime = Number(instance.thirtyDayUptime) >= 90; // available for at least 80/90 days @@ -41,15 +24,8 @@ const reliableCheck = mapped return thirtyDayUptime && dailyRatioCheck.length >= 80; }) // url includes name - .filter((instance) => instance.url.includes(instance.name)); - -// finally map to array -const result: string[] = reliableCheck.map((instance) => instance.name).sort(); + .filter(instance => instance.url.includes(instance.name)); -writeFile( - join(__dirname, "./invidiouslist.json"), - JSON.stringify([...staticInstances, ...result]), - (err) => { - if (err) return console.log(err); - } -); +export function getInvidiousList(): string[] { + return reliableCheck.map(instance => instance.name).sort() +}
\ No newline at end of file diff --git a/ci/pipedCI.ts b/ci/pipedCI.ts new file mode 100644 index 00000000..80424295 --- /dev/null +++ b/ci/pipedCI.ts @@ -0,0 +1,92 @@ +import * as data from "../ci/piped_instances.json"; + +type percent = string +type dailyMinutesDown = Record<string, number> + +type PipedInstance = { + name: string; + url: string; + icon: string; + slug: string; + status: string; + uptime: percent; + uptimeDay: percent; + uptimeWeek: percent; + uptimeMonth: percent; + uptimeYear: percent; + time: number; + timeDay: number; + timeWeek: number; + timeMonth: number; + timeYear: number; + dailyMinutesDown: dailyMinutesDown +} + +const percentNumber = (percent: percent) => Number(percent.replace("%", "")) +const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000) + +function dailyMinuteFilter (dailyMinutesDown: dailyMinutesDown) { + let daysDown = 0 + for (const [date, minsDown] of Object.entries(dailyMinutesDown)) { + if (new Date(date) >= ninetyDaysAgo && minsDown > 1000) { // if within 90 days and down for more than 1000 minutes + daysDown++ + } + } + // return true f less than 10 days down + return daysDown < 10 +} + +const getHost = (url: string) => new URL(url).host + +const getWatchPage = async (instance: PipedInstance) => + fetch(`https://${getHost(instance.url)}`, { redirect: "manual" }) + .then(res => res.headers.get("Location")) + .catch(e => { console.log (e); return null }) + +const siteOK = async (instance) => { + // check if entire site is redirect + const notRedirect = await fetch(instance.url, { redirect: "manual" }) + .then(res => res.status == 200) + // only allow kavin to return piped.video + // if (instance.url.startsWith("https://piped.video") && instance.slug !== "kavin-rocks-official") return false + // check if frontend is OK + const watchPageStatus = await fetch(instance.frontendUrl) + .then(res => res.ok) + // test API - stream returns ok result + const streamStatus = await fetch(`${instance.apiUrl}/streams/BaW_jenozKc`) + .then(res => res.ok) + // get startTime of monitor + const age = await fetch(instance.historyUrl) + .then(res => res.text()) + .then(text => { // startTime greater than 90 days ago + const date = text.match(/startTime: (.+)/)[1] + return Date.parse(date) < ninetyDaysAgo.valueOf() + }) + // console.log(notRedirect, watchPageStatus, streamStatus, age, instance.frontendUrl, instance.apiUrl) + return notRedirect && watchPageStatus && streamStatus && age +} + +const staticFilters = (data as PipedInstance[]) + .filter(instance => { + const isup = instance.status === "up" + const monthCheck = percentNumber(instance.uptimeMonth) >= 90 + const dailyMinuteCheck = dailyMinuteFilter(instance.dailyMinutesDown) + return isup && monthCheck && dailyMinuteCheck + }) + .map(async instance => { + // get frontend url + const frontendUrl = await getWatchPage(instance) + if (!frontendUrl) return null // return false if frontend doesn't resolve + // get api base + const apiUrl = instance.url.replace("/healthcheck", "") + const historyUrl = `https://raw.githubusercontent.com/TeamPiped/piped-uptime/master/history/${instance.slug}.yml` + const pass = await siteOK({ apiUrl, historyUrl, frontendUrl, url: instance.url }) + const frontendHost = getHost(frontendUrl) + return pass ? frontendHost : null + }) + +export async function getPipedList(): Promise<string[]> { + const instances = await Promise.all(staticFilters) + .then(arr => arr.filter(i => i !== null)) + return instances +} diff --git a/package.json b/package.json index 06439d0c..345cac80 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "build:watch": "npm run build:watch:chrome", "build:watch:chrome": "webpack --env browser=chrome --config webpack/webpack.dev.js --watch", "build:watch:firefox": "webpack --env browser=firefox --config webpack/webpack.dev.js --watch", - "ci:invidious": "ts-node ci/invidiousCI.ts", + "ci:invidious": "ts-node ci/generateList.ts", "dev": "npm run build:dev && concurrently \"npm run web-run\" \"npm run build:watch\"", "dev:firefox": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox\" \"npm run build:watch:firefox\"", "dev:firefox-android": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox-android\" \"npm run build:watch:firefox\"", |