From 532d8fc11e9153571a8bb9ca6a8085c99fb8d3df Mon Sep 17 00:00:00 2001 From: supergonkas Date: Fri, 18 Nov 2016 10:46:55 +0000 Subject: [PATCH] Remove dead providers (#1580) * Remove dead providers * Goodbye btdigg * RIP KAT * Remove from init * Remove from init * Remove from pytest.ini --- medusa/providers/__init__.py | 12 +- medusa/providers/torrent/__init__.py | 9 +- .../providers/torrent/html/ilovetorrents.py | 202 ------------------ medusa/providers/torrent/json/btdigg.py | 179 ---------------- medusa/providers/torrent/xml/kat.py | 188 ---------------- pytest.ini | 3 - 6 files changed, 9 insertions(+), 584 deletions(-) delete mode 100644 medusa/providers/torrent/html/ilovetorrents.py delete mode 100644 medusa/providers/torrent/json/btdigg.py delete mode 100644 medusa/providers/torrent/xml/kat.py diff --git a/medusa/providers/__init__.py b/medusa/providers/__init__.py index 57bfb467e9..39f0ce2055 100644 --- a/medusa/providers/__init__.py +++ b/medusa/providers/__init__.py @@ -25,24 +25,24 @@ ) from .nzb.newznab import NewznabProvider from .torrent import ( - abnormal, alpharatio, animebytes, bitcannon, bithdtv, bitsnoop, bluetigers, btdigg, btn, cpasbien, danishbits, - elitetorrent, extratorrent, freshontv, gftracker, hd4free, hdbits, hdspace, hdtorrents, hounddawgs, ilovetorrents, - iptorrents, kat, limetorrents, morethantv, newpct, norbits, nyaatorrents, pretome, rarbg, scc, scenetime, shazbat, + abnormal, alpharatio, animebytes, bitcannon, bithdtv, bitsnoop, bluetigers, btn, cpasbien, danishbits, + elitetorrent, extratorrent, freshontv, gftracker, hd4free, hdbits, hdspace, hdtorrents, hounddawgs, + iptorrents, limetorrents, morethantv, newpct, norbits, nyaatorrents, pretome, rarbg, scc, scenetime, shazbat, speedcd, t411, thepiratebay, tntvillage, tokyotoshokan, torrentbytes, torrentday, torrentleech, torrentproject, torrentshack, torrentz2, transmitthenet, tvchaosuk, xthor, zooqle ) from .torrent.rss.rsstorrent import TorrentRssProvider __all__ = [ - 'womble', 'btn', 'thepiratebay', 'kat', 'torrentleech', 'scc', 'hdtorrents', + 'womble', 'btn', 'thepiratebay', 'torrentleech', 'scc', 'hdtorrents', 'torrentday', 'hdbits', 'hounddawgs', 'iptorrents', 'omgwtfnzbs', 'speedcd', 'nyaatorrents', 'torrentbytes', 'freshontv', 'cpasbien', 'morethantv', 't411', 'tokyotoshokan', 'alpharatio', 'shazbat', 'rarbg', 'tntvillage', 'binsearch', 'bluetigers', - 'xthor', 'abnormal', 'scenetime', 'btdigg', 'transmitthenet', 'tvchaosuk', + 'xthor', 'abnormal', 'scenetime', 'transmitthenet', 'tvchaosuk', 'torrentproject', 'extratorrent', 'bitcannon', 'torrentz2', 'pretome', 'gftracker', 'hdspace', 'newpct', 'elitetorrent', 'bitsnoop', 'danishbits', 'hd4free', 'limetorrents', - 'norbits', 'ilovetorrents', 'anizb', 'bithdtv', 'zooqle', 'animebytes', 'torrentshack' + 'norbits', 'anizb', 'bithdtv', 'zooqle', 'animebytes', 'torrentshack' ] diff --git a/medusa/providers/torrent/__init__.py b/medusa/providers/torrent/__init__.py index 05cbd9a870..7df836d968 100644 --- a/medusa/providers/torrent/__init__.py +++ b/medusa/providers/torrent/__init__.py @@ -15,7 +15,6 @@ hdspace, hdtorrents, hounddawgs, - ilovetorrents, iptorrents, limetorrents, morethantv, @@ -39,7 +38,6 @@ from .json import ( bitcannon, - btdigg, btn, hd4free, hdbits, @@ -57,19 +55,18 @@ from .xml import ( bitsnoop, - kat, torrentz2, ) __all__ = [ 'abnormal', 'alpharatio', 'animebytes', 'bithdtv', 'bluetigers', 'cpasbien', 'danishbits', 'elitetorrent', 'extratorrent', 'freshontv', - 'gftracker', 'hdspace', 'hdtorrents', 'hounddawgs', 'ilovetorrents', + 'gftracker', 'hdspace', 'hdtorrents', 'hounddawgs', 'iptorrents', 'limetorrents', 'morethantv', 'newpct', 'pretome', 'scc', 'scenetime', 'speedcd', 'thepiratebay', 'tntvillage', 'tokyotoshokan', 'torrentbytes', 'torrentleech', 'torrentshack', 'transmitthenet', 'tvchaosuk', 'xthor', 'zooqle', 'bitcannon', - 'btdigg', 'btn', 'hd4free', 'hdbits', 'norbits', 'rarbg', 't411', + 'btn', 'hd4free', 'hdbits', 'norbits', 'rarbg', 't411', 'torrentday', 'torrentproject', 'nyaatorrents', 'rsstorrent', - 'shazbat', 'bitsnoop', 'kat', 'torrentz2' + 'shazbat', 'bitsnoop', 'torrentz2' ] diff --git a/medusa/providers/torrent/html/ilovetorrents.py b/medusa/providers/torrent/html/ilovetorrents.py deleted file mode 100644 index 607496cb16..0000000000 --- a/medusa/providers/torrent/html/ilovetorrents.py +++ /dev/null @@ -1,202 +0,0 @@ -# coding=utf-8 -# Author: Gonçalo M. (aka duramato/supergonkas) -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import re -import traceback - -from requests.compat import urljoin -from requests.utils import dict_from_cookiejar -from ..torrent_provider import TorrentProvider -from .... import logger, tv_cache -from ....bs4_parser import BS4Parser -from ....helper.common import convert_size - - -class ILoveTorrentsProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """ILoveTorrents Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'ILoveTorrents') - - # Credentials - self.username = None - self.password = None - - # URLs - self.url = 'https://www.ilovetorrents.me/' - self.urls = { - 'login': urljoin(self.url, 'takelogin.php'), - 'search': urljoin(self.url, 'browse.php'), - 'download': urljoin(self.url, '{link}'), - } - - # Proper Strings - self.proper_strings = ['PROPER', 'REPACK', 'REAL'] - - # Miscellaneous Options - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tv_cache.TVCache(self) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - Search a provider and parse the results - - :param search_strings: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - if not self.login(): - return results - - # Search Params - search_params = { - 'cat': 0 - } - - for mode in search_strings: - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - - search_params['search'] = search_string - response = self.get_url(self.urls['search'], params=search_params, returns='response') - if not response or not response.text: - logger.log('No data returned from provider', logger.DEBUG) - continue - - results += self.parse(response.text, mode) - - return results - - def parse(self, data, mode): - """ - Parse search results for items. - - :param data: The raw response from a search - :param mode: The current mode used to search, e.g. RSS - - :return: A list of items found - """ - - items = [] - - with BS4Parser(data, 'html.parser') as html: - torrent_table = html.find('table', class_='koptekst') - torrent_rows = torrent_table('tr') if torrent_table else [] - - # Continue only if at least one release is found - if len(torrent_rows) < 2: - logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) - return items - - # Skip column headers - for row in torrent_rows[1:]: - cells = row('td') - if len(cells) < 11: - continue - - try: - link = cells[1].find('a') - title = link.getText() - download_url = self.urls['download'].format(link=cells[2].find('a')['href']) - if not all([title, download_url]): - continue - - seeders = int(cells[10].getText().replace(',', '')) - leechers = int(cells[11].getText().replace(',', '')) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - # Use same failsafe as Bitsoup - if seeders >= 32768 or leechers >= 32768: - continue - - torrent_size = cells[8].getText() - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'torrent_hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - - return items - - def login(self): - """Login method used for logging in before doing search and torrent downloads.""" - if any(dict_from_cookiejar(self.session.cookies).values()): - return True - - login_params = { - 'username': self.username, - 'password': self.password, - 'logout': 'false', - 'submit': 'Welcome to ILT' - } - - response = self.get_url(self.urls['login'], post_data=login_params, returns='response') - if not response or not response.text: - logger.log('Unable to connect to provider', logger.WARNING) - return False - - if re.search('Username or password incorrect', response.text): - logger.log('Invalid username or password. Check your settings', logger.WARNING) - return False - - return True - - def _check_auth(self): - if not self.username or not self.password: - logger.log(u'Invalid username or password. Check your settings', logger.WARNING) - - return True - - -provider = ILoveTorrentsProvider() diff --git a/medusa/providers/torrent/json/btdigg.py b/medusa/providers/torrent/json/btdigg.py deleted file mode 100644 index 4d2585dedf..0000000000 --- a/medusa/providers/torrent/json/btdigg.py +++ /dev/null @@ -1,179 +0,0 @@ -# coding=utf-8 -# Author: Jodi Jones -# Rewrite: Gonçalo M. (aka duramato/supergonkas) -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import traceback - -from requests.compat import urljoin - -from ..torrent_provider import TorrentProvider -from .... import logger, tv_cache -from ....helper.common import convert_size - - -class BTDiggProvider(TorrentProvider): - """BTDigg Torrent provider""" - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'BTDigg') - - # Credentials - self.public = True - - # URLs - self.url = 'https://btdigg.org' - self.urls = { - 'api': 'https://api.btdigg.org/api/private-341ada3245790954/s02', - } - self.custom_url = None - - # Proper Strings - self.proper_strings = ['PROPER', 'REPACK'] - - # Miscellaneous Options - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - # Use this hacky way for RSS search since most results will use this codecs - cache_params = {'RSS': ['x264', 'x264.HDTV', '720.HDTV.x264']} - # Only poll BTDigg every 30 minutes max, since BTDigg takes some time to crawl - self.cache = tv_cache.TVCache(self, min_time=30, search_params=cache_params) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - Search a provider and parse the results - - :param search_strings: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Not used - :returns: A list of search results (structure) - """ - results = [] - - if self.custom_url: - # if not validators.url(self.custom_url): - # logger.log('Invalid custom url set, please check your settings', logger.WARNING) - # return results - search_url = urljoin(self.custom_url, 'api/private-341ada3245790954/s02') - else: - search_url = self.urls['api'] - - # Search Params - search_params = { - 'p': 0, - 'order': 2, - } - - for mode in search_strings: - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - search_params['q'] = search_string - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - search_params['order'] = 0 - - response = self.get_url(search_url, params=search_params, returns='response') - if not response or not response.content: - logger.log('No data returned from provider', logger.DEBUG) - continue - - try: - jdata = response.json() - except ValueError: # also catches JSONDecodeError if simplejson is installed - logger.log('No data returned from provider', logger.DEBUG) - continue - - results += self.parse(jdata, mode) - - return results - - def parse(self, data, mode): - """ - Parse search results for items. - - :param data: The raw response from a search - :param mode: The current mode used to search, e.g. RSS - - :return: A list of items found - """ - - items = [] - - torrent_rows = data - - for row in torrent_rows: - try: - title = row.pop('name', '') - download_url = row.pop('magnet') + self._custom_trackers if row['magnet'] else None - if not all([title, download_url]): - continue - - if float(row.pop('ff')): - logger.log("Ignoring result for {0} since it's been" - ' reported as fake (level = {1})'.format - (title, row['ff']), logger.DEBUG) - continue - - if not int(row.pop('files')): - logger.log('Ignoring result for {0} because it has no files'.format - (title), logger.DEBUG) - continue - - leechers = row.pop('leechers', 0) - seeders = row.pop('seeders', 1) - - # Filter unseeded torrent - if seeders < min(self.minseed, 1): - if mode != 'RSS': - logger.log("Discarding torrent because it doesn't meet the " - "minimum seeders: {0}. Seeders: {1}".format - (title, seeders), logger.DEBUG) - continue - - torrent_size = row.pop('size') - size = convert_size(torrent_size) or -1 - - item = { - 'title': title, - 'link': download_url, - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'pubdate': None, - 'torrent_hash': None, - } - if mode != 'RSS': - logger.log('Found result: {0} with {1} seeders and {2} leechers'.format - (title, seeders, leechers), logger.DEBUG) - - items.append(item) - except (AttributeError, TypeError, KeyError, ValueError, IndexError): - logger.log('Failed parsing provider. Traceback: {0!r}'.format - (traceback.format_exc()), logger.ERROR) - - return items - -provider = BTDiggProvider() diff --git a/medusa/providers/torrent/xml/kat.py b/medusa/providers/torrent/xml/kat.py deleted file mode 100644 index 0c95e202a6..0000000000 --- a/medusa/providers/torrent/xml/kat.py +++ /dev/null @@ -1,188 +0,0 @@ -# coding=utf-8 -# Author: Dustyn Gibson -# -# This file is part of Medusa. -# -# Medusa is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Medusa is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Medusa. If not, see . - -from __future__ import unicode_literals - -import traceback - -import medusa as app -from requests.compat import urljoin -import validators -from ..torrent_provider import TorrentProvider -from .... import logger, tv_cache -from ....bs4_parser import BS4Parser -from ....helper.common import convert_size, try_int - - -class KatProvider(TorrentProvider): # pylint: disable=too-many-instance-attributes - """KAT Torrent provider""" - - def __init__(self): - - # Provider Init - TorrentProvider.__init__(self, 'KickAssTorrents') - - # Credentials - self.public = True - - # URLs - self.url = 'https://kat.cr' - self.urls = { - 'base': self.url, - 'search': urljoin(self.url, '%s/'), - } - self.custom_url = None - - # Proper Strings - - # Miscellaneous Options - self.confirmed = True - - # Torrent Stats - self.minseed = None - self.minleech = None - - # Cache - self.cache = tv_cache.TVCache(self, search_params={'RSS': ['tv', 'anime']}) - - def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches - """ - Search a provider and parse the results - - :param search_strings: A dict with mode (key) and the search value (value) - :param age: Not used - :param ep_obj: Used to check if show is anime - :returns: A list of search results (structure) - """ - results = [] - - anime = (self.show and self.show.anime) or (ep_obj and ep_obj.show and ep_obj.show.anime) or False - - # Search Params - search_params = { - 'q': '', - 'field': 'time_add', - 'sorder': 'desc', - 'rss': 1, - 'category': 'anime' if anime else 'tv' - } - - for mode in search_strings: - logger.log('Search mode: {0}'.format(mode), logger.DEBUG) - - for search_string in search_strings[mode]: - if mode != 'RSS': - logger.log('Search string: {search}'.format - (search=search_string), logger.DEBUG) - search_params['field'] = 'seeders' - search_params['q'] = search_string - - search_url = self.urls['search'] % ('usearch' if mode != 'RSS' else search_string) - if self.custom_url: - if not validators.url(self.custom_url): - logger.log('Invalid custom url: {0}'.format(self.custom_url), logger.WARNING) - return results - search_url = urljoin(self.custom_url, search_url.split(self.url)[1]) - - response = self.get_url(search_url, params=search_params, returns='response') - if not response or not response.text: - logger.log('No data returned from provider, maybe try a custom url, or a different one', - logger.DEBUG) - continue - elif not response or not response.text.startswith('