Skip to content

Move dehydration logic from the riak backend into the base backend #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kdazzle wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Move dehydration logic from the riak backend into the base backend #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
208 changes: 206 additions & 2 deletions sunspear/backends/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from sunspear.activitystreams.models import Activity, ReplyActivity, LikeActivity
from sunspear.exceptions import (SunspearDuplicateEntryException, SunspearInvalidActivityException,
from sunspear.activitystreams.models import (
Activity, ReplyActivity, LikeActivity, Model)
from sunspear.exceptions import (
SunspearDuplicateEntryException, SunspearInvalidActivityException,
SunspearInvalidObjectException)

import uuid
Expand Down Expand Up @@ -402,3 +404,205 @@ def get_new_id(self):
:return: a new id
"""
return uuid.uuid1().hex

def _get_many_activities(self, activity_ids=None, raw_filter="", filters=None,
include_public=False, audience_targeting=None):
"""
Given a list of activity ids, returns a list of activities from riak.

:param list activity_ids: The list of activities you want to retrieve
:param raw_filter: allows you to specify a javascript function as a
string. The function should return ``true`` if the activity should
be included in the result set or ``false`` it shouldn't. If you
specify a raw filter, the filters specified in ``filters`` will not
run. However, the results will still be filtered based on the
``audience_targeting`` parameter.
:param dict filters: filters list of activities by key, value pair.
For example, ``{'verb': 'comment'}`` would only return activities
where the ``verb`` was ``comment``.
Filters do not work for nested dictionaries.
:param bool include_public: If ``True``, and the ``audience_targeting``
dictionary is defined, activities that are not targeted towards
anyone are included in the results
:param dict audience_targeting: Filters the list of activities targeted
towards a particular audience. The key for the dictionary is one of
``to``, ``cc``, ``bto``, or ``bcc``.

:return list: the activities retrieved from riak
"""
raise NotImplementedError()

def dehydrate_activities(self, activities):
"""
Takes a raw list of activities returned from riak and replace keys with
contain ids for riak objects with actual riak object
"""
activities = self._extract_sub_activities(activities)

#collect a list of unique object ids. We only iterate through the fields that we know
#for sure are objects. User is responsible for hydrating all other fields.
object_ids = set()
for activity in activities:
object_ids.update(self._extract_object_keys(activity))

#Get the objects for the ids we have collected
objects = self.get_obj(object_ids)
objects_dict = dict(((obj["id"], obj,) for obj in objects))

#We also need to extract any activities that were diguised as objects. IE activities with
#objectType=activity
activities_in_objects_ids = set()

#replace the object ids with the hydrated objects
for activity in activities:
activity = self._dehydrate_object_keys(activity, objects_dict)
#Extract keys of any activities that were objects
activities_in_objects_ids.update(self._extract_activity_keys(activity, skip_sub_activities=True))

#If we did have activities that were objects, we need to hydrate those activities and
#the objects for those activities
if activities_in_objects_ids:
sub_activities = self._get_many_activities(activities_in_objects_ids)
activities_in_objects_dict = dict(((sub_activity["id"], sub_activity,) for sub_activity in sub_activities))
for activity in activities:
activity = self._dehydrate_sub_activity(activity, activities_in_objects_dict, skip_sub_activities=True)

#we have to do one more round of object dehydration for our new sub-activities
object_ids.update(self._extract_object_keys(activity))

#now get all the objects we don't already have and for sub-activities and and hydrate them into
#our list of activities
object_ids -= set(objects_dict.keys())
objects = self.get_obj(object_ids)
for obj in objects:
objects_dict[obj["id"]] = obj

for activity in activities:
activity = self._dehydrate_object_keys(activity, objects_dict)

return activities

def _extract_sub_activities(self, activities):
"""
Extract all objects that have an objectType of activity as an activity
"""
#We might also have to get sub activities for things like replies and likes
activity_ids = set()
activities_dict = dict(((activity["id"], activity,) for activity in activities))

for activity in activities:
activity_ids.update(self._extract_activity_keys(activity))

if activity_ids:
#don't bother fetching the activities we already have
activity_ids -= set(activities_dict.keys())
if activity_ids:
sub_activities = self._get_many_activities(activity_ids)
for sub_activity in sub_activities:
activities_dict[sub_activity["id"]] = sub_activity

#Dehydrate out any subactivities we may have
for activity in activities:
activity = self._dehydrate_sub_activity(activity, activities_dict)

return activities

def _extract_activity_keys(self, activity, skip_sub_activities=False):
keys = []
for activity_key in Model._object_fields + ['inReplyTo']:
if activity_key not in activity:
continue
obj = activity.get(activity_key)
if isinstance(obj, dict):
if obj.get('objectType', None) == 'activity':
keys.append(obj['id'])
if obj.get('inReplyTo', None):
[keys.append(in_reply_to_obj['id']) for in_reply_to_obj in obj['inReplyTo']]

if not skip_sub_activities:
for collection in Activity._response_fields:
if collection in activity and activity[collection]['items']:
for item in activity[collection]['items']:
keys.extend(self._extract_activity_keys(item))
return keys

def _dehydrate_sub_activity(self, sub_activity, obj_list, skip_sub_activities=False):
for activity_key in Model._object_fields:
if activity_key not in sub_activity:
continue
if isinstance(sub_activity[activity_key], dict):
if sub_activity[activity_key].get('objectType', None) == 'activity':
sub_activity[activity_key].update(obj_list[sub_activity[activity_key]['id']])
if sub_activity[activity_key].get('inReplyTo', None):
for i, in_reply_to_obj in enumerate(sub_activity[activity_key]['inReplyTo']):
sub_activity[activity_key]['inReplyTo'][i]\
.update(obj_list[sub_activity[activity_key]['inReplyTo'][i]['id']])

if not skip_sub_activities:
for collection in Activity._response_fields:
if collection in sub_activity and sub_activity[collection]['items']:
dehydrated_sub_items = []
for i, item in enumerate(sub_activity[collection]['items']):
try:
dehydrated_sub_items.append(self._dehydrate_sub_activity(item, obj_list))
except KeyError, e:
pass
sub_activity[collection]['items'] = dehydrated_sub_items
sub_activity[collection]['totalItems'] = len(dehydrated_sub_items)

return sub_activity

def _extract_object_keys(self, activity, skip_sub_activities=False):
keys = []
for object_key in Model._object_fields + Activity._direct_audience_targeting_fields \
+ Activity._indirect_audience_targeting_fields:
if object_key not in activity:
continue
objects = activity.get(object_key)
if isinstance(objects, dict):
if objects.get('objectType', None) == 'activity':
keys = keys + self._extract_object_keys(objects)
if objects.get('inReplyTo', None):
[keys.extend(self._extract_object_keys(in_reply_to_obj, skip_sub_activities=skip_sub_activities)) \
for in_reply_to_obj in objects['inReplyTo']]
if isinstance(objects, list):
for item in objects:
if isinstance(item, basestring):
keys.append(item)
if isinstance(objects, basestring):
keys.append(objects)

if not skip_sub_activities:
for collection in Activity._response_fields:
if collection in activity and activity[collection]['items']:
for item in activity[collection]['items']:
keys.extend(self._extract_object_keys(item))
return keys

def _dehydrate_object_keys(self, activity, objects_dict, skip_sub_activities=False):
for object_key in Model._object_fields + Activity._direct_audience_targeting_fields \
+ Activity._indirect_audience_targeting_fields:
if object_key not in activity:
continue
activity_objects = activity.get(object_key)
if isinstance(activity_objects, dict):
if activity_objects.get('objectType', None) == 'activity':
activity[object_key] = self._dehydrate_object_keys(activity_objects, objects_dict, skip_sub_activities=skip_sub_activities)
if activity_objects.get('inReplyTo', None):
for i, in_reply_to_obj in enumerate(activity_objects['inReplyTo']):
activity_objects['inReplyTo'][i] = \
self._dehydrate_object_keys(activity_objects['inReplyTo'][i], \
objects_dict, skip_sub_activities=skip_sub_activities)
if isinstance(activity_objects, list):
for i, obj_id in enumerate(activity_objects):
if isinstance(activity[object_key][i], basestring):
activity[object_key][i] = objects_dict.get(obj_id, {})
if isinstance(activity_objects, basestring):
activity[object_key] = objects_dict.get(activity_objects, {})

if not skip_sub_activities:
for collection in Activity._response_fields:
if collection in activity and activity[collection]['items']:
for i, item in enumerate(activity[collection]['items']):
activity[collection]['items'][i] = self._dehydrate_object_keys(item, objects_dict)
return activity
Loading