summaryrefslogtreecommitdiffhomepage
path: root/devscripts/check-porn.py
diff options
context:
space:
mode:
authorFilippo Valsorda <[email protected]>2014-01-14 16:01:00 -0500
committerFilippo Valsorda <[email protected]>2014-01-14 16:01:00 -0500
commita45ea170421170c5787ec4c367a550760a295264 (patch)
treebfb9d6851270f5f49a17324c3221313a5501bded /devscripts/check-porn.py
parent4950f30890495d76d0738b0fae252d4b509196e9 (diff)
downloadyoutube-dl-a45ea170421170c5787ec4c367a550760a295264.tar.gz
youtube-dl-a45ea170421170c5787ec4c367a550760a295264.zip
Implement a different adult sites checking algorithm
Diffstat (limited to 'devscripts/check-porn.py')
-rw-r--r--devscripts/check-porn.py44
1 files changed, 32 insertions, 12 deletions
diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 63401fe18..86aa37b5f 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -3,6 +3,9 @@
"""
This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
if we are not 'age_limit' tagging some porn site
+
+A second approach implemented relies on a list of porn domains, to activate it
+pass the list filename as the only argument
"""
# Allow direct execution
@@ -11,25 +14,42 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from test.helper import get_testcases
+from youtube_dl.utils import compat_urllib_parse_urlparse
from youtube_dl.utils import compat_urllib_request
+if len(sys.argv) > 1:
+ METHOD = 'LIST'
+ LIST = open(sys.argv[1]).read().decode('utf8').strip()
+else:
+ METHOD = 'EURISTIC'
+
for test in get_testcases():
- try:
- webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
- except:
- print('\nFail: {0}'.format(test['name']))
- continue
+ if METHOD == 'EURISTIC':
+ try:
+ webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
+ except:
+ print('\nFail: {0}'.format(test['name']))
+ continue
+
+ webpage = webpage.decode('utf8', 'replace')
+
+ RESULT = 'porn' in webpage.lower()
+
+ elif METHOD == 'LIST':
+ domain = compat_urllib_parse_urlparse(test['url']).netloc
+ if not domain:
+ print('\nFail: {0}'.format(test['name']))
+ continue
+ domain = '.'.join(domain.split('.')[-2:])
- webpage = webpage.decode('utf8', 'replace')
+ RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)
- if 'porn' in webpage.lower() and ('info_dict' not in test
- or 'age_limit' not in test['info_dict']
- or test['info_dict']['age_limit'] != 18):
+ if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
+ or test['info_dict']['age_limit'] != 18):
print('\nPotential missing age_limit check: {0}'.format(test['name']))
- elif 'porn' not in webpage.lower() and ('info_dict' in test and
- 'age_limit' in test['info_dict'] and
- test['info_dict']['age_limit'] == 18):
+ elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
+ and test['info_dict']['age_limit'] == 18):
print('\nPotential false negative: {0}'.format(test['name']))
else: