aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authordirkf <[email protected]>2022-09-01 13:28:30 +0100
committerdirkf <[email protected]>2022-09-01 13:28:30 +0100
commit218c423bc042674a8834ffc09520a94fbbe7b138 (patch)
treec3d3cedf62212ff69399c712b5edd8ac503e9959
parent55c823634db890a328ffc23588fcd6f35d9b3ddf (diff)
downloadyoutube-dl-218c423bc042674a8834ffc09520a94fbbe7b138.tar.gz
youtube-dl-218c423bc042674a8834ffc09520a94fbbe7b138.zip
[cache] Add cache validation by program version, based on yt-dlp
-rw-r--r--test/test_cache.py16
-rw-r--r--youtube_dl/cache.py28
2 files changed, 37 insertions, 7 deletions
diff --git a/test/test_cache.py b/test/test_cache.py
index a16160142..931074aa1 100644
--- a/test/test_cache.py
+++ b/test/test_cache.py
@@ -3,17 +3,18 @@
from __future__ import unicode_literals
-import shutil
-
# Allow direct execution
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import shutil
from test.helper import FakeYDL
from youtube_dl.cache import Cache
+from youtube_dl.utils import version_tuple
+from youtube_dl.version import __version__
def _is_empty(d):
@@ -54,6 +55,17 @@ class TestCache(unittest.TestCase):
self.assertFalse(os.path.exists(self.test_dir))
self.assertEqual(c.load('test_cache', 'k.'), None)
+ def test_cache_validation(self):
+ ydl = FakeYDL({
+ 'cachedir': self.test_dir,
+ })
+ c = Cache(ydl)
+ obj = {'x': 1, 'y': ['รค', '\\a', True]}
+ c.store('test_cache', 'k.', obj)
+ self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj)
+ new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
+ self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
+
if __name__ == '__main__':
unittest.main()
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 7bdade1bd..4822439d0 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -10,12 +10,21 @@ import traceback
from .compat import compat_getenv
from .utils import (
+ error_to_compat_str,
expand_path,
+ is_outdated_version,
+ try_get,
write_json_file,
)
+from .version import __version__
class Cache(object):
+
+ _YTDL_DIR = 'youtube-dl'
+ _VERSION_KEY = _YTDL_DIR + '_version'
+ _DEFAULT_VERSION = '2021.12.17'
+
def __init__(self, ydl):
self._ydl = ydl
@@ -23,7 +32,7 @@ class Cache(object):
res = self._ydl.params.get('cachedir')
if res is None:
cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
- res = os.path.join(cache_root, 'youtube-dl')
+ res = os.path.join(cache_root, self._YTDL_DIR)
return expand_path(res)
def _get_cache_fn(self, section, key, dtype):
@@ -50,13 +59,22 @@ class Cache(object):
except OSError as ose:
if ose.errno != errno.EEXIST:
raise
- write_json_file(data, fn)
+ write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn)
except Exception:
tb = traceback.format_exc()
self._ydl.report_warning(
'Writing cache to %r failed: %s' % (fn, tb))
- def load(self, section, key, dtype='json', default=None):
+ def _validate(self, data, min_ver):
+ version = try_get(data, lambda x: x[self._VERSION_KEY])
+ if not version: # Backward compatibility
+ data, version = {'data': data}, self._DEFAULT_VERSION
+ if not is_outdated_version(version, min_ver or '0', assume_new=False):
+ return data['data']
+ self._ydl.to_screen(
+ 'Discarding old cache from version {version} (needs {min_ver})'.format(**locals()))
+
+ def load(self, section, key, dtype='json', default=None, min_ver=None):
assert dtype in ('json',)
if not self.enabled:
@@ -66,12 +84,12 @@ class Cache(object):
try:
try:
with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
- return json.load(cachef)
+ return self._validate(json.load(cachef), min_ver)
except ValueError:
try:
file_size = os.path.getsize(cache_fn)
except (OSError, IOError) as oe:
- file_size = str(oe)
+ file_size = error_to_compat_str(oe)
self._ydl.report_warning(
'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
except IOError: