From 60272567366a61367a7d65a343a98c145dd90537 Mon Sep 17 00:00:00 2001 From: Edward Peek Date: Wed, 11 May 2016 16:38:32 +1200 Subject: [PATCH 1/4] Changed loading of service config to be done dynamically --- src/config.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/config.py b/src/config.py index 19e4528..5375c2d 100644 --- a/src/config.py +++ b/src/config.py @@ -47,19 +47,17 @@ def from_file(file_path): config.r_password = sec.get("password", None) config.r_oauth_key = sec.get("oauth_key", None) config.r_oauth_secret = sec.get("oauth_secret", None) - - #TODO: make dynamic - if "service.mal" in parsed: - sec = parsed["service.mal"] - config.services["mal"] = {"username": sec.get("username", None), "password": sec.get("password", None)} - - if "service.anidb" in parsed: - sec = parsed["service.anidb"] - config.services["anidb"] = {"client": sec.get("client", None)} - - if "service.nyaa" in parsed: - sec = parsed["service.nyaa"] - config.services["nyaa"] = {"domain": sec.get("domain", None)} + + # Dynamically load service configuration + service_prefix = "service." + for section_name in [sec for sec in parsed.sections() if sec.startswith(service_prefix)]: + sec = parsed[section_name] + + service_key = section_name[len(service_prefix):] + service_config = config.services[service_key] = {} + + for key in sec.keys(): + service_config[key] = sec.get(key) if "options" in parsed: sec = parsed["options"] From a55120dab895e91e57564aa6ee149ad482981c95 Mon Sep 17 00:00:00 2001 From: Edward Peek Date: Wed, 11 May 2016 16:39:14 +1200 Subject: [PATCH 2/4] Streams now matched with show IDs according to seasonal config --- src/module_edit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module_edit.py b/src/module_edit.py index d0c28fe..e16d84e 100644 --- a/src/module_edit.py +++ b/src/module_edit.py @@ -101,7 +101,7 @@ def _edit_with_file(db, edit_file): else: service = db.get_service(key=service_key) s = db.get_stream(service_tuple=(service, show_key)) - db.update_stream(s, show_key=show_key, remote_offset=remote_offset, commit=False) + db.update_stream(s, show=show_id, show_key=show_key, remote_offset=remote_offset, commit=False) else: error(" Stream handler not installed") From fd67ae961d57230d0a6644834d3d3e8db1c67cf7 Mon Sep 17 00:00:00 2001 From: Edward Peek Date: Wed, 11 May 2016 17:17:01 +1200 Subject: [PATCH 3/4] Added support for AnimeLab streaming links - Added "animelab" ServiceHandler for streaming links to www.animelab.com - Added proxy and debug hints to example config - Added links for current seasonal shows --- config-example.ini | 4 + season_configs/spring_2016.yaml | 9 +++ src/services/stream/animelab.py | 128 ++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 src/services/stream/animelab.py diff --git a/config-example.ini b/config-example.ini index 5cc1ced..0da6958 100644 --- a/config-example.ini +++ b/config-example.ini @@ -21,9 +21,13 @@ client = [service.nyaa] domain = nyaa.se +[service.animelab] +proxy = + [options] # Valid options, separated with a space: tv, movie, ova new_show_types = tv +debug = false [post] title = [Spoilers] {show_name} - Episode {episode} discussion diff --git a/season_configs/spring_2016.yaml b/season_configs/spring_2016.yaml index 51ce577..5de1dca 100644 --- a/season_configs/spring_2016.yaml +++ b/season_configs/spring_2016.yaml @@ -52,6 +52,7 @@ info: streams: crunchyroll: '' funimation: http://www.funimation.com/shows/my-hero-academia/home + animelab: http://www.animelab.com/shows/my-hero-academia --- title: Bungou Stray Dogs type: tv @@ -74,6 +75,7 @@ info: streams: crunchyroll: '' funimation: http://www.funimation.com/shows/concrete-revolutio + animelab: http://www.animelab.com/shows/concrete-revolutio|13 --- title: Endride type: tv @@ -107,6 +109,7 @@ info: streams: crunchyroll: http://www.crunchyroll.com/the-asterisk-war|12 funimation: http://www.funimation.com/shows/the-asterisk-war|12 + animelab: http://www.animelab.com/shows/the-asterisk-war|12 --- title: 'Gyakuten Saiban: Sono "Shinjitsu", Igi Ari! (Ace Attorney)' type: tv @@ -195,6 +198,7 @@ info: streams: crunchyroll: '' funimation: http://www.funimation.com/shows/kumamiko/home + animelab: http://www.animelab.com/shows/kuma-miko-girl-meets-bear --- title: Kuromukuro type: tv @@ -217,6 +221,7 @@ info: streams: crunchyroll: http://www.crunchyroll.com/rin-ne|25 funimation: '' + animelab: https://www.animelab.com/shows/rinne|25 --- title: Mayoiga type: tv @@ -283,6 +288,7 @@ info: streams: crunchyroll: '' funimation: http://www.funimation.com/shows/three-leaves-three-colors/home + animelab: http://www.animelab.com/shows/three-leaves-three-colors-sansha-sanyo --- title: 'Seisen Cerberus: Ryuukoku no Fatalités' type: tv @@ -327,6 +333,7 @@ info: streams: crunchyroll: http://www.crunchyroll.com/tanaka-kun-is-always-listless funimation: '' + animelab: http://www.animelab.com/shows/tanaka-kun-is-always-listless --- title: Terra Formars Revenge type: tv @@ -360,6 +367,7 @@ info: streams: crunchyroll: http://www.crunchyroll.com/ushio-and-tora|26 funimation: '' + animelab: http://www.animelab.com/shows/ushio-and-tora|26 --- title: Wagamama High Spec type: tv @@ -427,6 +435,7 @@ info: streams: crunchyroll: '' funimation: http://www.funimation.com/shows/assassination-classroom/home|22 + animelab: http://www.animelab.com/shows/assassination-classroom|22 --- title: Super Lovers type: tv diff --git a/src/services/stream/animelab.py b/src/services/stream/animelab.py new file mode 100644 index 0000000..e10a29d --- /dev/null +++ b/src/services/stream/animelab.py @@ -0,0 +1,128 @@ +import calendar +import locale +import re +from datetime import datetime +from logging import debug, warning, error + +from data.models import Episode, UnprocessedStream +from .. import AbstractServiceHandler + + +class ServiceHandler(AbstractServiceHandler): + _show_link_url = "http://www.animelab.com/shows/{key}" + _show_link_url_id_re = re.compile("animelab.com/shows/([\w-]+)", re.I) + + _episode_link_url = "http://www.animelab.com/player/{episode}" + + _list_seasonal_shows_url = "http://www.animelab.com/api/simulcasts?limit=100&page={page}" + _list_show_episodes_url = "http://www.animelab.com/api/videoentries/show/{id}?limit=30&page={page}" + _list_aired_episodes_url = "http://www.animelab.com/api/simulcasts/episodes/latest?limit=10&page={page}" + + def __init__(self): + super().__init__("animelab", "AnimeLab", False) + + # Modify all requests to check for and apply proxy + def request(self, url, proxy=None, **kwargs): + if ":" in self.config.get("proxy", ""): + proxy = tuple(self.config["proxy"].split(":")) + + lang = locale.getdefaultlocale()[0] + if proxy is None and "AU" not in lang and "NZ" not in lang: + warning("AnimeLab requires an AU/NZ IP, but no proxy was supplied") + + return super().request(url, proxy=proxy, **kwargs) + + # Episodes are usually delayed at least several hours, so the utility of this will be limited + def get_latest_episode(self, stream, **kwargs): + page_index = 0 + page_url = self._list_aired_episodes_url.format(page=page_index) + + page_json = self.request(page_url, json=True, **kwargs) + if page_json is None: + error("Failed to get recently aired shows list") + return None + + for episode_json in page_json["list"]: + episode_show_key = episode_json["showSlug"] + if stream.show_key == episode_show_key: + return self._episode_from_json(episode_json) + + # TODO Use show-specific API if not found in recently aired list + warning("Skipped checking show-specific episode list") + + return None + + def _episode_from_json(self, episode_json): + episode_number = int(episode_json["episodeNumber"]) + episode_name = episode_json["name"] + episode_slug = episode_json["slug"] + episode_link_url = self._episode_link_url.format(episode=episode_slug) + # TODO Given release date is unreliable so ignore it for now + return Episode(episode_number, episode_name, episode_link_url, datetime.utcnow()) + + def get_stream_info(self, stream, **kwargs): + # TODO Figure out what info might be missing. + return None # or the updated stream + + def get_seasonal_streams(self, year=None, season=None, **kwargs): + found_streams = list() + + # Perform API request for show list + page_index = 0 + page_url = self._list_seasonal_shows_url.format(page=page_index) + + page_json = self.request(page_url, json=True, **kwargs) + if page_json is None: + error("Failed to get seasonal shows list") + return found_streams + + # Extract streams from response JSON + for show_json in page_json["list"]: + if not self._is_airing_during_season(show_json, year, season): + continue + + stream = self._stream_from_json(show_json) + found_streams += [stream] + + # TODO Handle multiple pages of results. + remaining_page_count = page_json["totalPageCount"] - 1 + if remaining_page_count > 0: + warning("Skipped {} pages of results".format(remaining_page_count)) + + return found_streams + + def _stream_from_json(self, show_json): + show_key = show_json["slug"] + show_name = show_json["name"] # Could also use "originalName" + + debug("Found show {}: \"{}\"".format(show_key, show_name)) + + return UnprocessedStream(self.key, show_key, None, show_name, 0, 0) + + @staticmethod + def _is_airing_during_season(show_json, year, season): + if year is None or season is None: + return True + + stream_start_timestamp = show_json["simulcastStartDate"] / 1000 + stream_end_timestamp = show_json["simulcastEndDate"] / 1000 + + # TODO Confirm the appropriate values for the season parameter + midseason_dates = { + 'winter': datetime(year, 2, 15), + 'spring': datetime(year, 5, 15), + 'summer': datetime(year, 8, 15), + 'autumn': datetime(year, 11, 15), + 'fall': datetime(year, 11, 15) + } + + midseason_timestamp = calendar.timegm(midseason_dates[season].utctimetuple()) + + return stream_start_timestamp < midseason_timestamp < stream_end_timestamp + + def get_stream_link(self, stream): + return self._show_link_url.format(key=stream.show_key) + + def extract_show_key(self, url): + match = self._show_link_url_id_re.search(url) + return match.group(1) if match else None From 572185aa3f423e61024cdf09f49919021fa54b2b Mon Sep 17 00:00:00 2001 From: Edward Peek Date: Fri, 13 May 2016 10:16:41 +1200 Subject: [PATCH 4/4] Switched from PEP to project formatting - Spaces to tabs - Single-line spacing --- src/services/stream/animelab.py | 175 ++++++++++++++++---------------- 1 file changed, 87 insertions(+), 88 deletions(-) diff --git a/src/services/stream/animelab.py b/src/services/stream/animelab.py index e10a29d..28e2ff0 100644 --- a/src/services/stream/animelab.py +++ b/src/services/stream/animelab.py @@ -7,122 +7,121 @@ from data.models import Episode, UnprocessedStream from .. import AbstractServiceHandler - class ServiceHandler(AbstractServiceHandler): - _show_link_url = "http://www.animelab.com/shows/{key}" - _show_link_url_id_re = re.compile("animelab.com/shows/([\w-]+)", re.I) + _show_link_url = "http://www.animelab.com/shows/{key}" + _show_link_url_id_re = re.compile("animelab.com/shows/([\w-]+)", re.I) - _episode_link_url = "http://www.animelab.com/player/{episode}" + _episode_link_url = "http://www.animelab.com/player/{episode}" - _list_seasonal_shows_url = "http://www.animelab.com/api/simulcasts?limit=100&page={page}" - _list_show_episodes_url = "http://www.animelab.com/api/videoentries/show/{id}?limit=30&page={page}" - _list_aired_episodes_url = "http://www.animelab.com/api/simulcasts/episodes/latest?limit=10&page={page}" + _list_seasonal_shows_url = "http://www.animelab.com/api/simulcasts?limit=100&page={page}" + _list_show_episodes_url = "http://www.animelab.com/api/videoentries/show/{id}?limit=30&page={page}" + _list_aired_episodes_url = "http://www.animelab.com/api/simulcasts/episodes/latest?limit=10&page={page}" - def __init__(self): - super().__init__("animelab", "AnimeLab", False) + def __init__(self): + super().__init__("animelab", "AnimeLab", False) - # Modify all requests to check for and apply proxy - def request(self, url, proxy=None, **kwargs): - if ":" in self.config.get("proxy", ""): - proxy = tuple(self.config["proxy"].split(":")) + # Modify all requests to check for and apply proxy + def request(self, url, proxy=None, **kwargs): + if ":" in self.config.get("proxy", ""): + proxy = tuple(self.config["proxy"].split(":")) - lang = locale.getdefaultlocale()[0] - if proxy is None and "AU" not in lang and "NZ" not in lang: - warning("AnimeLab requires an AU/NZ IP, but no proxy was supplied") + lang = locale.getdefaultlocale()[0] + if proxy is None and "AU" not in lang and "NZ" not in lang: + warning("AnimeLab requires an AU/NZ IP, but no proxy was supplied") - return super().request(url, proxy=proxy, **kwargs) + return super().request(url, proxy=proxy, **kwargs) - # Episodes are usually delayed at least several hours, so the utility of this will be limited - def get_latest_episode(self, stream, **kwargs): - page_index = 0 - page_url = self._list_aired_episodes_url.format(page=page_index) + # Episodes are usually delayed at least several hours, so the utility of this will be limited + def get_latest_episode(self, stream, **kwargs): + page_index = 0 + page_url = self._list_aired_episodes_url.format(page=page_index) - page_json = self.request(page_url, json=True, **kwargs) - if page_json is None: - error("Failed to get recently aired shows list") - return None + page_json = self.request(page_url, json=True, **kwargs) + if page_json is None: + error("Failed to get recently aired shows list") + return None - for episode_json in page_json["list"]: - episode_show_key = episode_json["showSlug"] - if stream.show_key == episode_show_key: - return self._episode_from_json(episode_json) + for episode_json in page_json["list"]: + episode_show_key = episode_json["showSlug"] + if stream.show_key == episode_show_key: + return self._episode_from_json(episode_json) - # TODO Use show-specific API if not found in recently aired list - warning("Skipped checking show-specific episode list") + # TODO Use show-specific API if not found in recently aired list + warning("Skipped checking show-specific episode list") - return None + return None - def _episode_from_json(self, episode_json): - episode_number = int(episode_json["episodeNumber"]) - episode_name = episode_json["name"] - episode_slug = episode_json["slug"] - episode_link_url = self._episode_link_url.format(episode=episode_slug) - # TODO Given release date is unreliable so ignore it for now - return Episode(episode_number, episode_name, episode_link_url, datetime.utcnow()) + def _episode_from_json(self, episode_json): + episode_number = int(episode_json["episodeNumber"]) + episode_name = episode_json["name"] + episode_slug = episode_json["slug"] + episode_link_url = self._episode_link_url.format(episode=episode_slug) + # TODO Given release date is unreliable so ignore it for now + return Episode(episode_number, episode_name, episode_link_url, datetime.utcnow()) - def get_stream_info(self, stream, **kwargs): - # TODO Figure out what info might be missing. - return None # or the updated stream + def get_stream_info(self, stream, **kwargs): + # TODO Figure out what info might be missing. + return None # or the updated stream - def get_seasonal_streams(self, year=None, season=None, **kwargs): - found_streams = list() + def get_seasonal_streams(self, year=None, season=None, **kwargs): + found_streams = list() - # Perform API request for show list - page_index = 0 - page_url = self._list_seasonal_shows_url.format(page=page_index) + # Perform API request for show list + page_index = 0 + page_url = self._list_seasonal_shows_url.format(page=page_index) - page_json = self.request(page_url, json=True, **kwargs) - if page_json is None: - error("Failed to get seasonal shows list") - return found_streams + page_json = self.request(page_url, json=True, **kwargs) + if page_json is None: + error("Failed to get seasonal shows list") + return found_streams - # Extract streams from response JSON - for show_json in page_json["list"]: - if not self._is_airing_during_season(show_json, year, season): - continue + # Extract streams from response JSON + for show_json in page_json["list"]: + if not self._is_airing_during_season(show_json, year, season): + continue - stream = self._stream_from_json(show_json) - found_streams += [stream] + stream = self._stream_from_json(show_json) + found_streams += [stream] - # TODO Handle multiple pages of results. - remaining_page_count = page_json["totalPageCount"] - 1 - if remaining_page_count > 0: - warning("Skipped {} pages of results".format(remaining_page_count)) + # TODO Handle multiple pages of results. + remaining_page_count = page_json["totalPageCount"] - 1 + if remaining_page_count > 0: + warning("Skipped {} pages of results".format(remaining_page_count)) - return found_streams + return found_streams - def _stream_from_json(self, show_json): - show_key = show_json["slug"] - show_name = show_json["name"] # Could also use "originalName" + def _stream_from_json(self, show_json): + show_key = show_json["slug"] + show_name = show_json["name"] # Could also use "originalName" - debug("Found show {}: \"{}\"".format(show_key, show_name)) + debug("Found show {}: \"{}\"".format(show_key, show_name)) - return UnprocessedStream(self.key, show_key, None, show_name, 0, 0) + return UnprocessedStream(self.key, show_key, None, show_name, 0, 0) - @staticmethod - def _is_airing_during_season(show_json, year, season): - if year is None or season is None: - return True + @staticmethod + def _is_airing_during_season(show_json, year, season): + if year is None or season is None: + return True - stream_start_timestamp = show_json["simulcastStartDate"] / 1000 - stream_end_timestamp = show_json["simulcastEndDate"] / 1000 + stream_start_timestamp = show_json["simulcastStartDate"] / 1000 + stream_end_timestamp = show_json["simulcastEndDate"] / 1000 - # TODO Confirm the appropriate values for the season parameter - midseason_dates = { - 'winter': datetime(year, 2, 15), - 'spring': datetime(year, 5, 15), - 'summer': datetime(year, 8, 15), - 'autumn': datetime(year, 11, 15), - 'fall': datetime(year, 11, 15) - } + # TODO Confirm the appropriate values for the season parameter + midseason_dates = { + 'winter': datetime(year, 2, 15), + 'spring': datetime(year, 5, 15), + 'summer': datetime(year, 8, 15), + 'autumn': datetime(year, 11, 15), + 'fall': datetime(year, 11, 15) + } - midseason_timestamp = calendar.timegm(midseason_dates[season].utctimetuple()) + midseason_timestamp = calendar.timegm(midseason_dates[season].utctimetuple()) - return stream_start_timestamp < midseason_timestamp < stream_end_timestamp + return stream_start_timestamp < midseason_timestamp < stream_end_timestamp - def get_stream_link(self, stream): - return self._show_link_url.format(key=stream.show_key) + def get_stream_link(self, stream): + return self._show_link_url.format(key=stream.show_key) - def extract_show_key(self, url): - match = self._show_link_url_id_re.search(url) - return match.group(1) if match else None + def extract_show_key(self, url): + match = self._show_link_url_id_re.search(url) + return match.group(1) if match else None