From 463c8283d2aa817a7f612a3303222fef7f6fcb7c Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 9 Apr 2015 10:19:38 -0700 Subject: [PATCH] Move dehydration logic from the riak backend into the base backend --- sunspear/backends/base.py | 208 +++++++++++++++++++++++++++++++++++++- sunspear/backends/riak.py | 174 ------------------------------- 2 files changed, 206 insertions(+), 176 deletions(-) diff --git a/sunspear/backends/base.py b/sunspear/backends/base.py index 3e415f9..14c3f0d 100644 --- a/sunspear/backends/base.py +++ b/sunspear/backends/base.py @@ -1,5 +1,7 @@ -from sunspear.activitystreams.models import Activity, ReplyActivity, LikeActivity -from sunspear.exceptions import (SunspearDuplicateEntryException, SunspearInvalidActivityException, +from sunspear.activitystreams.models import ( + Activity, ReplyActivity, LikeActivity, Model) +from sunspear.exceptions import ( + SunspearDuplicateEntryException, SunspearInvalidActivityException, SunspearInvalidObjectException) import uuid @@ -402,3 +404,205 @@ def get_new_id(self): :return: a new id """ return uuid.uuid1().hex + + def _get_many_activities(self, activity_ids=None, raw_filter="", filters=None, + include_public=False, audience_targeting=None): + """ + Given a list of activity ids, returns a list of activities from riak. + + :param list activity_ids: The list of activities you want to retrieve + :param raw_filter: allows you to specify a javascript function as a + string. The function should return ``true`` if the activity should + be included in the result set or ``false`` it shouldn't. If you + specify a raw filter, the filters specified in ``filters`` will not + run. However, the results will still be filtered based on the + ``audience_targeting`` parameter. + :param dict filters: filters list of activities by key, value pair. + For example, ``{'verb': 'comment'}`` would only return activities + where the ``verb`` was ``comment``. + Filters do not work for nested dictionaries. + :param bool include_public: If ``True``, and the ``audience_targeting`` + dictionary is defined, activities that are not targeted towards + anyone are included in the results + :param dict audience_targeting: Filters the list of activities targeted + towards a particular audience. The key for the dictionary is one of + ``to``, ``cc``, ``bto``, or ``bcc``. + + :return list: the activities retrieved from riak + """ + raise NotImplementedError() + + def dehydrate_activities(self, activities): + """ + Takes a raw list of activities returned from riak and replace keys with + contain ids for riak objects with actual riak object + """ + activities = self._extract_sub_activities(activities) + + #collect a list of unique object ids. We only iterate through the fields that we know + #for sure are objects. User is responsible for hydrating all other fields. + object_ids = set() + for activity in activities: + object_ids.update(self._extract_object_keys(activity)) + + #Get the objects for the ids we have collected + objects = self.get_obj(object_ids) + objects_dict = dict(((obj["id"], obj,) for obj in objects)) + + #We also need to extract any activities that were diguised as objects. IE activities with + #objectType=activity + activities_in_objects_ids = set() + + #replace the object ids with the hydrated objects + for activity in activities: + activity = self._dehydrate_object_keys(activity, objects_dict) + #Extract keys of any activities that were objects + activities_in_objects_ids.update(self._extract_activity_keys(activity, skip_sub_activities=True)) + + #If we did have activities that were objects, we need to hydrate those activities and + #the objects for those activities + if activities_in_objects_ids: + sub_activities = self._get_many_activities(activities_in_objects_ids) + activities_in_objects_dict = dict(((sub_activity["id"], sub_activity,) for sub_activity in sub_activities)) + for activity in activities: + activity = self._dehydrate_sub_activity(activity, activities_in_objects_dict, skip_sub_activities=True) + + #we have to do one more round of object dehydration for our new sub-activities + object_ids.update(self._extract_object_keys(activity)) + + #now get all the objects we don't already have and for sub-activities and and hydrate them into + #our list of activities + object_ids -= set(objects_dict.keys()) + objects = self.get_obj(object_ids) + for obj in objects: + objects_dict[obj["id"]] = obj + + for activity in activities: + activity = self._dehydrate_object_keys(activity, objects_dict) + + return activities + + def _extract_sub_activities(self, activities): + """ + Extract all objects that have an objectType of activity as an activity + """ + #We might also have to get sub activities for things like replies and likes + activity_ids = set() + activities_dict = dict(((activity["id"], activity,) for activity in activities)) + + for activity in activities: + activity_ids.update(self._extract_activity_keys(activity)) + + if activity_ids: + #don't bother fetching the activities we already have + activity_ids -= set(activities_dict.keys()) + if activity_ids: + sub_activities = self._get_many_activities(activity_ids) + for sub_activity in sub_activities: + activities_dict[sub_activity["id"]] = sub_activity + + #Dehydrate out any subactivities we may have + for activity in activities: + activity = self._dehydrate_sub_activity(activity, activities_dict) + + return activities + + def _extract_activity_keys(self, activity, skip_sub_activities=False): + keys = [] + for activity_key in Model._object_fields + ['inReplyTo']: + if activity_key not in activity: + continue + obj = activity.get(activity_key) + if isinstance(obj, dict): + if obj.get('objectType', None) == 'activity': + keys.append(obj['id']) + if obj.get('inReplyTo', None): + [keys.append(in_reply_to_obj['id']) for in_reply_to_obj in obj['inReplyTo']] + + if not skip_sub_activities: + for collection in Activity._response_fields: + if collection in activity and activity[collection]['items']: + for item in activity[collection]['items']: + keys.extend(self._extract_activity_keys(item)) + return keys + + def _dehydrate_sub_activity(self, sub_activity, obj_list, skip_sub_activities=False): + for activity_key in Model._object_fields: + if activity_key not in sub_activity: + continue + if isinstance(sub_activity[activity_key], dict): + if sub_activity[activity_key].get('objectType', None) == 'activity': + sub_activity[activity_key].update(obj_list[sub_activity[activity_key]['id']]) + if sub_activity[activity_key].get('inReplyTo', None): + for i, in_reply_to_obj in enumerate(sub_activity[activity_key]['inReplyTo']): + sub_activity[activity_key]['inReplyTo'][i]\ + .update(obj_list[sub_activity[activity_key]['inReplyTo'][i]['id']]) + + if not skip_sub_activities: + for collection in Activity._response_fields: + if collection in sub_activity and sub_activity[collection]['items']: + dehydrated_sub_items = [] + for i, item in enumerate(sub_activity[collection]['items']): + try: + dehydrated_sub_items.append(self._dehydrate_sub_activity(item, obj_list)) + except KeyError, e: + pass + sub_activity[collection]['items'] = dehydrated_sub_items + sub_activity[collection]['totalItems'] = len(dehydrated_sub_items) + + return sub_activity + + def _extract_object_keys(self, activity, skip_sub_activities=False): + keys = [] + for object_key in Model._object_fields + Activity._direct_audience_targeting_fields \ + + Activity._indirect_audience_targeting_fields: + if object_key not in activity: + continue + objects = activity.get(object_key) + if isinstance(objects, dict): + if objects.get('objectType', None) == 'activity': + keys = keys + self._extract_object_keys(objects) + if objects.get('inReplyTo', None): + [keys.extend(self._extract_object_keys(in_reply_to_obj, skip_sub_activities=skip_sub_activities)) \ + for in_reply_to_obj in objects['inReplyTo']] + if isinstance(objects, list): + for item in objects: + if isinstance(item, basestring): + keys.append(item) + if isinstance(objects, basestring): + keys.append(objects) + + if not skip_sub_activities: + for collection in Activity._response_fields: + if collection in activity and activity[collection]['items']: + for item in activity[collection]['items']: + keys.extend(self._extract_object_keys(item)) + return keys + + def _dehydrate_object_keys(self, activity, objects_dict, skip_sub_activities=False): + for object_key in Model._object_fields + Activity._direct_audience_targeting_fields \ + + Activity._indirect_audience_targeting_fields: + if object_key not in activity: + continue + activity_objects = activity.get(object_key) + if isinstance(activity_objects, dict): + if activity_objects.get('objectType', None) == 'activity': + activity[object_key] = self._dehydrate_object_keys(activity_objects, objects_dict, skip_sub_activities=skip_sub_activities) + if activity_objects.get('inReplyTo', None): + for i, in_reply_to_obj in enumerate(activity_objects['inReplyTo']): + activity_objects['inReplyTo'][i] = \ + self._dehydrate_object_keys(activity_objects['inReplyTo'][i], \ + objects_dict, skip_sub_activities=skip_sub_activities) + if isinstance(activity_objects, list): + for i, obj_id in enumerate(activity_objects): + if isinstance(activity[object_key][i], basestring): + activity[object_key][i] = objects_dict.get(obj_id, {}) + if isinstance(activity_objects, basestring): + activity[object_key] = objects_dict.get(activity_objects, {}) + + if not skip_sub_activities: + for collection in Activity._response_fields: + if collection in activity and activity[collection]['items']: + for i, item in enumerate(activity[collection]['items']): + activity[collection]['items'][i] = self._dehydrate_object_keys(item, objects_dict) + return activity diff --git a/sunspear/backends/riak.py b/sunspear/backends/riak.py index a77d179..2312a4a 100644 --- a/sunspear/backends/riak.py +++ b/sunspear/backends/riak.py @@ -440,180 +440,6 @@ def set_sub_item_indexes(self, riak_object, **kwargs): return riak_object - def dehydrate_activities(self, activities): - """ - Takes a raw list of activities returned from riak and replace keys with contain ids for riak objects with actual riak object - """ - activities = self._extract_sub_activities(activities) - - #collect a list of unique object ids. We only iterate through the fields that we know - #for sure are objects. User is responsible for hydrating all other fields. - object_ids = set() - for activity in activities: - object_ids.update(self._extract_object_keys(activity)) - - #Get the objects for the ids we have collected - objects = self.get_obj(object_ids) - objects_dict = dict(((obj["id"], obj,) for obj in objects)) - - #We also need to extract any activities that were diguised as objects. IE activities with - #objectType=activity - activities_in_objects_ids = set() - - #replace the object ids with the hydrated objects - for activity in activities: - activity = self._dehydrate_object_keys(activity, objects_dict) - #Extract keys of any activities that were objects - activities_in_objects_ids.update(self._extract_activity_keys(activity, skip_sub_activities=True)) - - #If we did have activities that were objects, we need to hydrate those activities and - #the objects for those activities - if activities_in_objects_ids: - sub_activities = self._get_many_activities(activities_in_objects_ids) - activities_in_objects_dict = dict(((sub_activity["id"], sub_activity,) for sub_activity in sub_activities)) - for activity in activities: - activity = self._dehydrate_sub_activity(activity, activities_in_objects_dict, skip_sub_activities=True) - - #we have to do one more round of object dehydration for our new sub-activities - object_ids.update(self._extract_object_keys(activity)) - - #now get all the objects we don't already have and for sub-activities and and hydrate them into - #our list of activities - object_ids -= set(objects_dict.keys()) - objects = self.get_obj(object_ids) - for obj in objects: - objects_dict[obj["id"]] = obj - - for activity in activities: - activity = self._dehydrate_object_keys(activity, objects_dict) - - return activities - - def _extract_sub_activities(self, activities): - """ - Extract all objects that have an objectType of activity as an activity - """ - #We might also have to get sub activities for things like replies and likes - activity_ids = set() - activities_dict = dict(((activity["id"], activity,) for activity in activities)) - - for activity in activities: - activity_ids.update(self._extract_activity_keys(activity)) - - if activity_ids: - #don't bother fetching the activities we already have - activity_ids -= set(activities_dict.keys()) - if activity_ids: - sub_activities = self._get_many_activities(activity_ids) - for sub_activity in sub_activities: - activities_dict[sub_activity["id"]] = sub_activity - - #Dehydrate out any subactivities we may have - for activity in activities: - activity = self._dehydrate_sub_activity(activity, activities_dict) - - return activities - - def _extract_activity_keys(self, activity, skip_sub_activities=False): - keys = [] - for activity_key in Model._object_fields + ['inReplyTo']: - if activity_key not in activity: - continue - obj = activity.get(activity_key) - if isinstance(obj, dict): - if obj.get('objectType', None) == 'activity': - keys.append(obj['id']) - if obj.get('inReplyTo', None): - [keys.append(in_reply_to_obj['id']) for in_reply_to_obj in obj['inReplyTo']] - - if not skip_sub_activities: - for collection in Activity._response_fields: - if collection in activity and activity[collection]['items']: - for item in activity[collection]['items']: - keys.extend(self._extract_activity_keys(item)) - return keys - - def _dehydrate_sub_activity(self, sub_activity, obj_list, skip_sub_activities=False): - for activity_key in Model._object_fields: - if activity_key not in sub_activity: - continue - if isinstance(sub_activity[activity_key], dict): - if sub_activity[activity_key].get('objectType', None) == 'activity': - sub_activity[activity_key].update(obj_list[sub_activity[activity_key]['id']]) - if sub_activity[activity_key].get('inReplyTo', None): - for i, in_reply_to_obj in enumerate(sub_activity[activity_key]['inReplyTo']): - sub_activity[activity_key]['inReplyTo'][i]\ - .update(obj_list[sub_activity[activity_key]['inReplyTo'][i]['id']]) - - if not skip_sub_activities: - for collection in Activity._response_fields: - if collection in sub_activity and sub_activity[collection]['items']: - dehydrated_sub_items = [] - for i, item in enumerate(sub_activity[collection]['items']): - try: - dehydrated_sub_items.append(self._dehydrate_sub_activity(item, obj_list)) - except KeyError, e: - pass - sub_activity[collection]['items'] = dehydrated_sub_items - sub_activity[collection]['totalItems'] = len(dehydrated_sub_items) - - return sub_activity - - def _extract_object_keys(self, activity, skip_sub_activities=False): - keys = [] - for object_key in Model._object_fields + Activity._direct_audience_targeting_fields \ - + Activity._indirect_audience_targeting_fields: - if object_key not in activity: - continue - objects = activity.get(object_key) - if isinstance(objects, dict): - if objects.get('objectType', None) == 'activity': - keys = keys + self._extract_object_keys(objects) - if objects.get('inReplyTo', None): - [keys.extend(self._extract_object_keys(in_reply_to_obj, skip_sub_activities=skip_sub_activities)) \ - for in_reply_to_obj in objects['inReplyTo']] - if isinstance(objects, list): - for item in objects: - if isinstance(item, basestring): - keys.append(item) - if isinstance(objects, basestring): - keys.append(objects) - - if not skip_sub_activities: - for collection in Activity._response_fields: - if collection in activity and activity[collection]['items']: - for item in activity[collection]['items']: - keys.extend(self._extract_object_keys(item)) - return keys - - def _dehydrate_object_keys(self, activity, objects_dict, skip_sub_activities=False): - for object_key in Model._object_fields + Activity._direct_audience_targeting_fields \ - + Activity._indirect_audience_targeting_fields: - if object_key not in activity: - continue - activity_objects = activity.get(object_key) - if isinstance(activity_objects, dict): - if activity_objects.get('objectType', None) == 'activity': - activity[object_key] = self._dehydrate_object_keys(activity_objects, objects_dict, skip_sub_activities=skip_sub_activities) - if activity_objects.get('inReplyTo', None): - for i, in_reply_to_obj in enumerate(activity_objects['inReplyTo']): - activity_objects['inReplyTo'][i] = \ - self._dehydrate_object_keys(activity_objects['inReplyTo'][i], \ - objects_dict, skip_sub_activities=skip_sub_activities) - if isinstance(activity_objects, list): - for i, obj_id in enumerate(activity_objects): - if isinstance(activity[object_key][i], basestring): - activity[object_key][i] = objects_dict.get(obj_id, {}) - if isinstance(activity_objects, basestring): - activity[object_key] = objects_dict.get(activity_objects, {}) - - if not skip_sub_activities: - for collection in Activity._response_fields: - if collection in activity and activity[collection]['items']: - for i, item in enumerate(activity[collection]['items']): - activity[collection]['items'][i] = self._dehydrate_object_keys(item, objects_dict) - return activity - def _get_many_activities(self, activity_ids=[], raw_filter="", filters={}, include_public=False, audience_targeting={}): """ Given a list of activity ids, returns a list of activities from riak.