diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index 670917387eda91..ecf2c2b69deb05 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -34,6 +34,20 @@ config INFINIBAND_USER_ACCESS libibverbs, libibcm and a hardware driver library from . +config INFINIBAND_EXP_USER_ACCESS + bool "Allow experimental support for Infiniband ABI" + depends on INFINIBAND_USER_ACCESS + ---help--- + IOCTL based ABI support for Infiniband. This allows userspace + to invoke the experimental IOCTL based ABI. + +config INFINIBAND_USE_IOCTL_BACKWARD_COMP + bool "Use IOCTL parsing for write commands" + depends on INFINIBAND_USER_ACCESS + ---help--- + Transform supported write commands to IOCTL commands and + execute them. Serialize the response back to the write schema. + config INFINIBAND_USER_MEM bool depends on INFINIBAND_USER_ACCESS != n diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index edaae9f9853c73..e3ca835bbf8e28 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -28,4 +28,5 @@ ib_umad-y := user_mad.o ib_ucm-y := ucm.o -ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o +ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ + rdma_core.o uverbs_std_types.o uverbs_ioctl.o diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index d29372624f3a05..a3b63f1c5bccfe 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -143,4 +143,18 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb, int ib_nl_handle_ip_res_resp(struct sk_buff *skb, struct netlink_callback *cb); +/* Remove ignored fields set in the attribute mask */ +static inline int modify_qp_mask(enum ib_qp_type qp_type, int mask) +{ + switch (qp_type) { + case IB_QPT_XRC_INI: + return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); + case IB_QPT_XRC_TGT: + return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | + IB_QP_RNR_RETRY); + default: + return mask; + } +} + #endif /* _CORE_PRIV_H */ diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c new file mode 100644 index 00000000000000..e67a61f840e6a0 --- /dev/null +++ b/drivers/infiniband/core/rdma_core.c @@ -0,0 +1,747 @@ +/* + * Copyright (c) 2016, Mellanox Technologies inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include "uverbs.h" +#include "rdma_core.h" + +int uverbs_group_idx(u16 *id, unsigned int ngroups) +{ + int ret = (*id & UVERBS_ID_RESERVED_MASK) >> UVERBS_ID_RESERVED_SHIFT; + + if (ret >= ngroups) + return -EINVAL; + + *id &= ~UVERBS_ID_RESERVED_MASK; + return ret; +} + +const struct uverbs_type *uverbs_get_type(const struct ib_device *ibdev, + uint16_t type) +{ + const struct uverbs_root *groups = ibdev->specs_root; + const struct uverbs_type_group *types; + int ret = uverbs_group_idx(&type, groups->num_groups); + + if (ret < 0) + return NULL; + + types = groups->type_groups[ret]; + + if (type >= types->num_types) + return NULL; + + return types->types[type]; +} + +const struct uverbs_action *uverbs_get_action(const struct uverbs_type *type, + uint16_t action) +{ + const struct uverbs_action_group *action_group; + int ret = uverbs_group_idx(&action, type->num_groups); + + if (ret < 0) + return NULL; + + action_group = type->action_groups[ret]; + if (action >= action_group->num_actions) + return NULL; + + return action_group->actions[action]; +} + +void uverbs_uobject_get(struct ib_uobject *uobject) +{ + kref_get(&uobject->ref); +} + +static void uverbs_uobject_put_ref(struct kref *ref) +{ + struct ib_uobject *uobj = + container_of(ref, struct ib_uobject, ref); + + if (uobj->type->type_class->needs_rcu) + kfree_rcu(uobj, rcu); + else + kfree(uobj); +} + +void uverbs_uobject_put(struct ib_uobject *uobject) +{ + kref_put(&uobject->ref, uverbs_uobject_put_ref); +} + +static int uverbs_try_lock_object(struct ib_uobject *uobj, bool write) +{ + /* + * When a read is required, we use a positive counter. Each read + * request checks that the value != -1 and increment it. Write + * requires an exclusive access, thus we check that the counter is + * zero (nobody claimed this object) and we set it to -1. + * Releasing a read lock is done by simply decreasing the counter. + * As for writes, since only a single write is permitted, setting + * it to zero is enough for releasing it. + */ + if (!write) + return __atomic_add_unless(&uobj->usecnt, 1, -1) == -1 ? + -EBUSY : 0; + + /* lock is either WRITE or DESTROY - should be exclusive */ + return atomic_cmpxchg(&uobj->usecnt, 0, -1) == 0 ? 0 : -EBUSY; +} + +static struct ib_uobject *alloc_uobj(struct ib_ucontext *context, + const struct uverbs_obj_type *type) +{ + struct ib_uobject *uobj = kmalloc(type->obj_size, GFP_KERNEL); + + if (!uobj) + return ERR_PTR(-ENOMEM); + /* + * user_handle should be filled by the handler, + * The object is added to the list in the commit stage. + */ + uobj->context = context; + uobj->type = type; + atomic_set(&uobj->usecnt, 0); + kref_init(&uobj->ref); + + return uobj; +} + +static int idr_add_uobj(struct ib_uobject *uobj) +{ + int ret; + + idr_preload(GFP_KERNEL); + spin_lock(&uobj->context->ufile->idr_lock); + + /* + * We start with allocating an idr pointing to NULL. This represents an + * object which isn't initialized yet. We'll replace it later on with + * the real object once we commit. + */ + ret = idr_alloc(&uobj->context->ufile->idr, NULL, 0, + min_t(unsigned long, U32_MAX - 1, INT_MAX), GFP_NOWAIT); + if (ret >= 0) + uobj->id = ret; + + spin_unlock(&uobj->context->ufile->idr_lock); + idr_preload_end(); + + return ret < 0 ? ret : 0; +} + +/* + * It only removes it from the uobjects list, uverbs_uobject_put() is still + * required. + */ +static void uverbs_idr_remove_uobj(struct ib_uobject *uobj) +{ + spin_lock(&uobj->context->ufile->idr_lock); + idr_remove(&uobj->context->ufile->idr, uobj->id); + spin_unlock(&uobj->context->ufile->idr_lock); +} + +/* Returns the ib_uobject or an error. The caller should check for IS_ERR. */ +static struct ib_uobject *lookup_get_idr_uobject(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext, + int id, bool write) +{ + struct ib_uobject *uobj; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "ib_uverbs: lookup_get_idr_uobject is called without proper synchronization"); + /* object won't be released as we're protected in rcu */ + uobj = idr_find(&ucontext->ufile->idr, id); + if (!uobj) + return ERR_PTR(-ENOENT); + + if (uobj->type != type) + return ERR_PTR(-EINVAL); + + return uobj; +} + +static struct ib_uobject *lookup_get_fd_uobject(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext, + int id, bool write) +{ + struct file *f; + struct ib_uobject *uobject; + const struct uverbs_obj_fd_type *fd_type = + container_of(type, struct uverbs_obj_fd_type, type); + + if (write) + return ERR_PTR(-EOPNOTSUPP); + + f = fget(id); + if (!f) + return ERR_PTR(-EBADF); + + uobject = f->private_data; + /* + * fget(id) ensures we are not currently running uverbs_close_fd, + * and the caller is expected to ensure that uverbs_close_fd is never + * done while a call top lookup is possible. + */ + if (f->f_op != fd_type->fops) { + fput(f); + return ERR_PTR(-EBADF); + } + + return uobject; +} + +struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext, + int id, bool write) +{ + struct ib_uobject *uobj; + int ret; + + if (type->type_class->needs_rcu) + rcu_read_lock(); + + uobj = type->type_class->lookup_get(type, ucontext, id, write); + if (IS_ERR(uobj)) + goto free; + + ret = uverbs_try_lock_object(uobj, write); + if (ret) { + WARN(ucontext->cleanup_reason, + "ib_uverbs: Trying to lookup_get while cleanup context\n"); + uobj = ERR_PTR(ret); + goto free; + } + + uverbs_uobject_get(uobj); +free: + if (type->type_class->needs_rcu) + rcu_read_unlock(); + return uobj; +} + +static struct ib_uobject *alloc_begin_idr_uobject(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext) +{ + int ret; + struct ib_uobject *uobj; + + uobj = alloc_uobj(ucontext, type); + if (IS_ERR(uobj)) + return uobj; + + ret = idr_add_uobj(uobj); + if (ret) { + uverbs_uobject_put(uobj); + return ERR_PTR(ret); + } + + return uobj; +} + +static struct ib_uobject *alloc_begin_fd_uobject(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext) +{ + const struct uverbs_obj_fd_type *fd_type = + container_of(type, struct uverbs_obj_fd_type, type); + int new_fd; + struct ib_uobject *uobj; + struct ib_uobject_file *uobj_file; + struct file *filp; + + new_fd = get_unused_fd_flags(O_CLOEXEC); + if (new_fd < 0) + return ERR_PTR(new_fd); + + uobj = alloc_uobj(ucontext, type); + if (IS_ERR(uobj)) { + put_unused_fd(new_fd); + return uobj; + } + + uobj_file = container_of(uobj, struct ib_uobject_file, uobj); + filp = anon_inode_getfile(fd_type->name, + fd_type->fops, + uobj_file, + fd_type->flags); + if (IS_ERR(filp)) { + put_unused_fd(new_fd); + uverbs_uobject_put(uobj); + return (void *)filp; + } + + uobj_file->uobj.id = new_fd; + uobj_file->uobj.object = filp; + uobj_file->ufile = ucontext->ufile; + INIT_LIST_HEAD(&uobj->list); + kref_get(&uobj_file->ufile->ref); + + return uobj; +} + +struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext) +{ + return type->type_class->alloc_begin(type, ucontext); +} + +static void uverbs_uobject_add(struct ib_uobject *uobject) +{ + mutex_lock(&uobject->context->uobjects_lock); + list_add(&uobject->list, &uobject->context->uobjects); + mutex_unlock(&uobject->context->uobjects_lock); +} + +static int __must_check remove_commit_idr_uobject(struct ib_uobject *uobj, + enum rdma_remove_reason why) +{ + const struct uverbs_obj_idr_type *idr_type = + container_of(uobj->type, struct uverbs_obj_idr_type, + type); + int ret = idr_type->destroy_object(uobj, why); + + /* + * We can only fail gracefully if the user requested to destroy the + * object. In the rest of the cases, just remove whatever you can. + */ + if (why == RDMA_REMOVE_DESTROY && ret) + return ret; + + uverbs_idr_remove_uobj(uobj); + + return ret; +} + +static void alloc_abort_fd_uobject(struct ib_uobject *uobj) +{ + struct ib_uobject_file *uobj_file = + container_of(uobj, struct ib_uobject_file, uobj); + struct file *filp = uobj->object; + int id = uobj_file->uobj.id; + + /* Unsuccessful NEW */ + fput(filp); + put_unused_fd(id); +} + +static int __must_check remove_commit_fd_uobject(struct ib_uobject *uobj, + enum rdma_remove_reason why) +{ + const struct uverbs_obj_fd_type *fd_type = + container_of(uobj->type, struct uverbs_obj_fd_type, type); + struct ib_uobject_file *uobj_file = + container_of(uobj, struct ib_uobject_file, uobj); + int ret = fd_type->context_closed(uobj_file, why); + + if (why == RDMA_REMOVE_DESTROY && ret) + return ret; + + if (why == RDMA_REMOVE_DURING_CLEANUP) { + alloc_abort_fd_uobject(uobj); + return ret; + } + + uobj_file->uobj.context = NULL; + return ret; +} + +static void lockdep_check(struct ib_uobject *uobj, bool write) +{ +#ifdef CONFIG_LOCKDEP + if (write) + WARN_ON(atomic_read(&uobj->usecnt) > 0); + else + WARN_ON(atomic_read(&uobj->usecnt) == -1); +#endif +} + +static int __must_check _rdma_remove_commit_uobject(struct ib_uobject *uobj, + enum rdma_remove_reason why, + bool lock) +{ + int ret; + struct ib_ucontext *ucontext = uobj->context; + + ret = uobj->type->type_class->remove_commit(uobj, why); + if (ret && why == RDMA_REMOVE_DESTROY) { + /* We couldn't remove the object, so just unlock the uobject */ + atomic_set(&uobj->usecnt, 0); + uobj->type->type_class->lookup_put(uobj, true); + } else { + if (lock) + mutex_lock(&ucontext->uobjects_lock); + list_del(&uobj->list); + if (lock) + mutex_unlock(&ucontext->uobjects_lock); + /* put the ref we took when we created the object */ + uverbs_uobject_put(uobj); + } + + return ret; +} + +/* This is called only for user requested DESTROY reasons */ +int __must_check rdma_remove_commit_uobject(struct ib_uobject *uobj) +{ + int ret; + struct ib_ucontext *ucontext = uobj->context; + + /* put the ref count we took at lookup_get */ + uverbs_uobject_put(uobj); + /* Cleanup is running. Calling this should have been impossible */ + if (!down_read_trylock(&ucontext->cleanup_rwsem)) { + WARN(true, "ib_uverbs: Cleanup is running while removing an uobject\n"); + return 0; + } + lockdep_check(uobj, true); + ret = _rdma_remove_commit_uobject(uobj, RDMA_REMOVE_DESTROY, true); + + up_read(&ucontext->cleanup_rwsem); + return ret; +} + +static void alloc_commit_idr_uobject(struct ib_uobject *uobj) +{ + uverbs_uobject_add(uobj); + spin_lock(&uobj->context->ufile->idr_lock); + /* + * We already allocated this IDR with a NULL object, so + * this shouldn't fail. + */ + WARN_ON(idr_replace(&uobj->context->ufile->idr, + uobj, uobj->id)); + spin_unlock(&uobj->context->ufile->idr_lock); +} + +static void alloc_commit_fd_uobject(struct ib_uobject *uobj) +{ + struct ib_uobject_file *uobj_file = + container_of(uobj, struct ib_uobject_file, uobj); + + uverbs_uobject_add(&uobj_file->uobj); + fd_install(uobj_file->uobj.id, uobj->object); + /* This shouldn't be used anymore. Use the file object instead */ + uobj_file->uobj.id = 0; + /* Get another reference as we export this to the fops */ + uverbs_uobject_get(&uobj_file->uobj); +} + +int rdma_alloc_commit_uobject(struct ib_uobject *uobj) +{ + /* Cleanup is running. Calling this should have been impossible */ + if (!down_read_trylock(&uobj->context->cleanup_rwsem)) { + int ret; + + WARN(true, "ib_uverbs: Cleanup is running while allocating an uobject\n"); + ret = uobj->type->type_class->remove_commit(uobj, + RDMA_REMOVE_DURING_CLEANUP); + if (ret) + pr_warn("ib_uverbs: cleanup of idr object %d failed\n", + uobj->id); + return ret; + } + + uobj->type->type_class->alloc_commit(uobj); + up_read(&uobj->context->cleanup_rwsem); + + return 0; +} + +static void alloc_abort_idr_uobject(struct ib_uobject *uobj) +{ + uverbs_idr_remove_uobj(uobj); + uverbs_uobject_put(uobj); +} + +void rdma_alloc_abort_uobject(struct ib_uobject *uobj) +{ + uobj->type->type_class->alloc_abort(uobj); +} + +static void lookup_put_idr_uobject(struct ib_uobject *uobj, bool write) +{ +} + +static void lookup_put_fd_uobject(struct ib_uobject *uobj, bool write) +{ + struct file *filp = uobj->object; + + WARN_ON(write); + /* This indirectly calls uverbs_close_fd and free the object */ + fput(filp); +} + +void rdma_lookup_put_uobject(struct ib_uobject *uobj, bool write) +{ + lockdep_check(uobj, write); + uobj->type->type_class->lookup_put(uobj, write); + /* + * In order to unlock an object, either decrease its usecnt for + * read access or zero it in case of write access. See + * uverbs_try_lock_object for locking schema information. + */ + if (!write) + atomic_dec(&uobj->usecnt); + else + atomic_set(&uobj->usecnt, 0); + + uverbs_uobject_put(uobj); +} + +const struct uverbs_obj_type_class uverbs_idr_class = { + .alloc_begin = alloc_begin_idr_uobject, + .lookup_get = lookup_get_idr_uobject, + .alloc_commit = alloc_commit_idr_uobject, + .alloc_abort = alloc_abort_idr_uobject, + .lookup_put = lookup_put_idr_uobject, + .remove_commit = remove_commit_idr_uobject, + /* + * When we destroy an object, we first just lock it for WRITE and + * actually DESTROY it in the finalize stage. So, the problematic + * scenario is when we just started the finalize stage of the + * destruction (nothing was executed yet). Now, the other thread + * fetched the object for READ access, but it didn't lock it yet. + * The DESTROY thread continues and starts destroying the object. + * When the other thread continue - without the RCU, it would + * access freed memory. However, the rcu_read_lock delays the free + * until the rcu_read_lock of the READ operation quits. Since the + * write lock of the object is still taken by the DESTROY flow, the + * READ operation will get -EBUSY and it'll just bail out. + */ + .needs_rcu = true, +}; + +static void _uverbs_close_fd(struct ib_uobject_file *uobj_file) +{ + struct ib_ucontext *ucontext; + struct ib_uverbs_file *ufile = uobj_file->ufile; + int ret; + + mutex_lock(&uobj_file->ufile->cleanup_mutex); + + /* uobject was either already cleaned up or is cleaned up right now anyway */ + if (!uobj_file->uobj.context || + !down_read_trylock(&uobj_file->uobj.context->cleanup_rwsem)) + goto unlock; + + ucontext = uobj_file->uobj.context; + ret = _rdma_remove_commit_uobject(&uobj_file->uobj, RDMA_REMOVE_CLOSE, + true); + up_read(&ucontext->cleanup_rwsem); + if (ret) + pr_warn("uverbs: unable to clean up uobject file in uverbs_close_fd.\n"); +unlock: + mutex_unlock(&ufile->cleanup_mutex); +} + +void uverbs_close_fd(struct file *f) +{ + struct ib_uobject_file *uobj_file = f->private_data; + struct kref *uverbs_file_ref = &uobj_file->ufile->ref; + + _uverbs_close_fd(uobj_file); + uverbs_uobject_put(&uobj_file->uobj); + kref_put(uverbs_file_ref, ib_uverbs_release_file); +} + +void uverbs_cleanup_ucontext(struct ib_ucontext *ucontext, bool device_removed) +{ + enum rdma_remove_reason reason = device_removed ? + RDMA_REMOVE_DRIVER_REMOVE : RDMA_REMOVE_CLOSE; + unsigned int cur_order = 0; + + ucontext->cleanup_reason = reason; + /* + * Waits for all remove_commit and alloc_commit to finish. Logically, We + * want to hold this forever as the context is going to be destroyed, + * but we'll release it since it causes a "held lock freed" BUG message. + */ + down_write(&ucontext->cleanup_rwsem); + + while (!list_empty(&ucontext->uobjects)) { + struct ib_uobject *obj, *next_obj; + unsigned int next_order = UINT_MAX; + + /* + * This shouldn't run while executing other commands on this + * context. Thus, the only thing we should take care of is + * releasing a FD while traversing this list. The FD could be + * closed and released from the _release fop of this FD. + * In order to mitigate this, we add a lock. + * We take and release the lock per order traversal in order + * to let other threads (which might still use the FDs) chance + * to run. + */ + mutex_lock(&ucontext->uobjects_lock); + list_for_each_entry_safe(obj, next_obj, &ucontext->uobjects, + list) + if (obj->type->destroy_order == cur_order) { + int ret; + + /* + * if we hit this WARN_ON, that means we are + * racing with a lookup_get. + */ + WARN_ON(uverbs_try_lock_object(obj, true)); + ret = _rdma_remove_commit_uobject(obj, reason, + false); + if (ret) + pr_warn("ib_uverbs: failed to remove uobject id %d order %u\n", + obj->id, cur_order); + } else { + next_order = min(next_order, + obj->type->destroy_order); + } + mutex_unlock(&ucontext->uobjects_lock); + cur_order = next_order; + } + up_write(&ucontext->cleanup_rwsem); +} + +void uverbs_initialize_ucontext(struct ib_ucontext *ucontext) +{ + ucontext->cleanup_reason = 0; + mutex_init(&ucontext->uobjects_lock); + INIT_LIST_HEAD(&ucontext->uobjects); + init_rwsem(&ucontext->cleanup_rwsem); +} + +const struct uverbs_obj_type_class uverbs_fd_class = { + .alloc_begin = alloc_begin_fd_uobject, + .lookup_get = lookup_get_fd_uobject, + .alloc_commit = alloc_commit_fd_uobject, + .alloc_abort = alloc_abort_fd_uobject, + .lookup_put = lookup_put_fd_uobject, + .remove_commit = remove_commit_fd_uobject, + .needs_rcu = false, +}; + +struct ib_uobject *uverbs_get_uobject_from_context(const struct uverbs_obj_type *type_attrs, + struct ib_ucontext *ucontext, + enum uverbs_idr_access access, + int id) +{ + switch (access) { + case UVERBS_ACCESS_READ: + return rdma_lookup_get_uobject(type_attrs, ucontext, id, false); + case UVERBS_ACCESS_DESTROY: + case UVERBS_ACCESS_WRITE: + return rdma_lookup_get_uobject(type_attrs, ucontext, id, true); + case UVERBS_ACCESS_NEW: + return rdma_alloc_begin_uobject(type_attrs, ucontext); + default: + WARN_ON(true); + return ERR_PTR(-EOPNOTSUPP); + } +} + +int uverbs_finalize_object(struct ib_uobject *uobj, + enum uverbs_idr_access access, + bool commit) +{ + int ret = 0; + + switch (access) { + case UVERBS_ACCESS_READ: + rdma_lookup_put_uobject(uobj, false); + break; + case UVERBS_ACCESS_WRITE: + rdma_lookup_put_uobject(uobj, true); + break; + case UVERBS_ACCESS_DESTROY: + if (commit) + ret = rdma_remove_commit_uobject(uobj); + else + rdma_lookup_put_uobject(uobj, true); + break; + case UVERBS_ACCESS_NEW: + if (commit) + ret = rdma_alloc_commit_uobject(uobj); + else + rdma_alloc_abort_uobject(uobj); + break; + default: + WARN_ON(true); + } + + return ret; +} + +int uverbs_finalize_objects(struct uverbs_attr_array *attr_array, + size_t num, + const struct uverbs_action *action, + bool commit) +{ + unsigned int i; + int ret = 0; + + for (i = 0; i < num; i++) { + struct uverbs_attr_array *attr_spec_array = &attr_array[i]; + const struct uverbs_attr_spec_group *attr_spec_group = + action->attr_groups[i]; + unsigned int j; + + for (j = 0; j < attr_spec_array->num_attrs; j++) { + struct uverbs_attr *attr = &attr_spec_array->attrs[j]; + struct uverbs_attr_spec *spec = &attr_spec_group->attrs[j]; + + if (!uverbs_is_valid(attr_spec_array, j)) + continue; + + if (spec->type == UVERBS_ATTR_TYPE_IDR || + spec->type == UVERBS_ATTR_TYPE_FD) { + int current_ret; + + /* + * refcounts should be handled at the object + * level and not at the uobject level. Refcounts + * of the objects themselves are done in + * handlers. + */ + current_ret = uverbs_finalize_object(attr->obj_attr.uobject, + spec->obj.access, + commit); + if (!ret) + ret = current_ret; + } + } + } + return ret; +} + diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h new file mode 100644 index 00000000000000..3c1738422d5b2a --- /dev/null +++ b/drivers/infiniband/core/rdma_core.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. + * Copyright (c) 2005-2017 Mellanox Technologies. All rights reserved. + * Copyright (c) 2005 Voltaire, Inc. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef RDMA_CORE_H +#define RDMA_CORE_H + +#include +#include +#include +#include +#include + +int uverbs_group_idx(u16 *id, unsigned int ngroups); +const struct uverbs_type *uverbs_get_type(const struct ib_device *ibdev, + uint16_t type); +const struct uverbs_action *uverbs_get_action(const struct uverbs_type *type, + uint16_t action); +/* + * These functions initialize the context and cleanups its uobjects. + * The context has a list of objects which is protected by a mutex + * on the context. initialize_ucontext should be called when we create + * a context. + * cleanup_ucontext removes all uobjects from the context and puts them. + */ +void uverbs_cleanup_ucontext(struct ib_ucontext *ucontext, bool device_removed); +void uverbs_initialize_ucontext(struct ib_ucontext *ucontext); + +/* + * uverbs_uobject_get is called in order to increase the reference count on + * an uobject. This is useful when a handler wants to keep the uobject's memory + * alive, regardless if this uobject is still alive in the context's objects + * repository. Objects are put via uverbs_uobject_put. + */ +void uverbs_uobject_get(struct ib_uobject *uobject); + +/* + * In order to indicate we no longer needs this uobject, uverbs_uobject_put + * is called. When the reference count is decreased, the uobject is freed. + * For example, this is used when attaching a completion channel to a CQ. + */ +void uverbs_uobject_put(struct ib_uobject *uobject); + +/* Indicate this fd is no longer used by this consumer, but its memory isn't + * necessarily released yet. When the last reference is put, we release the + * memory. After this call is executed, calling uverbs_uobject_get isn't + * allowed. + * This must be called from the release file_operations of the file! + */ +void uverbs_close_fd(struct file *f); + +/* + * Get an ib_uobject that corresponds to the given id from ucontext, assuming + * the object is from the given type. Lock it to the required access. + * This function could create (access == NEW) or destroy (access == DESTROY) + * objects if required. The action will be finalized only when + * uverbs_finalize_object or uverbs_finalize_objects is called. + */ +struct ib_uobject *uverbs_get_uobject_from_context(const struct uverbs_obj_type *type_attrs, + struct ib_ucontext *ucontext, + enum uverbs_idr_access access, + int id); +int uverbs_finalize_object(struct ib_uobject *uobj, + enum uverbs_idr_access access, + bool commit); +/* + * Note that certain finalize stages could return a status: + * (a) alloc_commit could return a failure if the object is committed at the + * same time when the context is destroyed. + * (b) remove_commit could fail if the object wasn't destroyed successfully. + * Since multiple objects could be finalized in one transaction, it is very NOT + * recommanded to have several finalize actions which have side affects. + * For example, it's NOT recommanded to have a certain action which has both + * a commit action and a destroy action or two destroy objects in the same + * action. The rule of thumb is to have one destroy or commit action with + * multiple lookups. + * The first non zero return value of finalize_object is returned from this + * function. + */ +int uverbs_finalize_objects(struct uverbs_attr_array *attr_array, + size_t num, + const struct uverbs_action *action, + bool success); + +#endif /* RDMA_CORE_H */ diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index 455034ac994e68..a5642cc5f2d193 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -46,6 +46,7 @@ #include #include #include +#include #define INIT_UDATA(udata, ibuf, obuf, ilen, olen) \ do { \ @@ -102,17 +103,25 @@ struct ib_uverbs_device { }; struct ib_uverbs_event_file { - struct kref ref; - int is_async; - struct ib_uverbs_file *uverbs_file; spinlock_t lock; int is_closed; wait_queue_head_t poll_wait; struct fasync_struct *async_queue; struct list_head event_list; +}; + +struct ib_uverbs_async_event_file { + struct ib_uverbs_event_file ev_file; + struct ib_uverbs_file *uverbs_file; + struct kref ref; struct list_head list; }; +struct ib_uverbs_completion_event_file { + struct ib_uobject_file uobj_file; + struct ib_uverbs_event_file ev_file; +}; + struct ib_uverbs_file { struct kref ref; struct mutex mutex; @@ -120,9 +129,13 @@ struct ib_uverbs_file { struct ib_uverbs_device *device; struct ib_ucontext *ucontext; struct ib_event_handler event_handler; - struct ib_uverbs_event_file *async_file; + struct ib_uverbs_async_event_file *async_file; struct list_head list; int is_closed; + + struct idr idr; + /* spinlock protects write access to idr */ + spinlock_t idr_lock; }; struct ib_uverbs_event { @@ -159,6 +172,8 @@ struct ib_usrq_object { struct ib_uqp_object { struct ib_uevent_object uevent; + /* lock for mcast list */ + struct mutex mcast_lock; struct list_head mcast_list; struct ib_uxrcd_object *uxrcd; }; @@ -176,32 +191,21 @@ struct ib_ucq_object { u32 async_events_reported; }; -extern spinlock_t ib_uverbs_idr_lock; -extern struct idr ib_uverbs_pd_idr; -extern struct idr ib_uverbs_mr_idr; -extern struct idr ib_uverbs_mw_idr; -extern struct idr ib_uverbs_ah_idr; -extern struct idr ib_uverbs_cq_idr; -extern struct idr ib_uverbs_qp_idr; -extern struct idr ib_uverbs_srq_idr; -extern struct idr ib_uverbs_xrcd_idr; -extern struct idr ib_uverbs_rule_idr; -extern struct idr ib_uverbs_wq_idr; -extern struct idr ib_uverbs_rwq_ind_tbl_idr; - -void idr_remove_uobj(struct idr *idp, struct ib_uobject *uobj); - -struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file, - struct ib_device *ib_dev, - int is_async); +struct uverbs_type_group; + +extern const struct file_operations uverbs_event_fops; +void uverbs_initialize_type_group(const struct uverbs_type_group *type_group); +void ib_uverbs_init_event_file(struct ib_uverbs_event_file *ev_file); +struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file, + struct ib_device *ib_dev); void ib_uverbs_free_async_event_file(struct ib_uverbs_file *uverbs_file); -struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd); void ib_uverbs_release_ucq(struct ib_uverbs_file *file, - struct ib_uverbs_event_file *ev_file, + struct ib_uverbs_completion_event_file *ev_file, struct ib_ucq_object *uobj); void ib_uverbs_release_uevent(struct ib_uverbs_file *file, struct ib_uevent_object *uobj); +void ib_uverbs_release_file(struct kref *ref); void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context); void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr); @@ -210,9 +214,22 @@ void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr); void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr); void ib_uverbs_event_handler(struct ib_event_handler *handler, struct ib_event *event); -void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, struct ib_xrcd *xrcd); +int ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, struct ib_xrcd *xrcd, + enum rdma_remove_reason why); int uverbs_dealloc_mw(struct ib_mw *mw); +void uverbs_copy_query_dev_fields(struct ib_device *ib_dev, + struct ib_uverbs_query_device_resp *resp, + struct ib_device_attr *attr); + +void ib_uverbs_detach_umcast(struct ib_qp *qp, + struct ib_uqp_object *uobj); + +long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +long ib_uverbs_cmd_verbs(struct ib_device *ib_dev, struct ib_uverbs_file *file, + struct ib_uverbs_ioctl_hdr *hdr, void __user *buf, + bool w_legacy); + struct ib_uverbs_flow_spec { union { diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 70078220348383..bd8f38b4d64930 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -37,275 +37,131 @@ #include #include #include +#include #include +#include +#include +#include "rdma_core.h" + #include "uverbs.h" #include "core_priv.h" -struct uverbs_lock_class { - struct lock_class_key key; - char name[16]; -}; - -static struct uverbs_lock_class pd_lock_class = { .name = "PD-uobj" }; -static struct uverbs_lock_class mr_lock_class = { .name = "MR-uobj" }; -static struct uverbs_lock_class mw_lock_class = { .name = "MW-uobj" }; -static struct uverbs_lock_class cq_lock_class = { .name = "CQ-uobj" }; -static struct uverbs_lock_class qp_lock_class = { .name = "QP-uobj" }; -static struct uverbs_lock_class ah_lock_class = { .name = "AH-uobj" }; -static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" }; -static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" }; -static struct uverbs_lock_class rule_lock_class = { .name = "RULE-uobj" }; -static struct uverbs_lock_class wq_lock_class = { .name = "WQ-uobj" }; -static struct uverbs_lock_class rwq_ind_table_lock_class = { .name = "IND_TBL-uobj" }; - -/* - * The ib_uobject locking scheme is as follows: - * - * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it - * needs to be held during all idr write operations. When an object is - * looked up, a reference must be taken on the object's kref before - * dropping this lock. For read operations, the rcu_read_lock() - * and rcu_write_lock() but similarly the kref reference is grabbed - * before the rcu_read_unlock(). - * - * - Each object also has an rwsem. This rwsem must be held for - * reading while an operation that uses the object is performed. - * For example, while registering an MR, the associated PD's - * uobject.mutex must be held for reading. The rwsem must be held - * for writing while initializing or destroying an object. - * - * - In addition, each object has a "live" flag. If this flag is not - * set, then lookups of the object will fail even if it is found in - * the idr. This handles a reader that blocks and does not acquire - * the rwsem until after the object is destroyed. The destroy - * operation will set the live flag to 0 and then drop the rwsem; - * this will allow the reader to acquire the rwsem, see that the - * live flag is 0, and then drop the rwsem and its reference to - * object. The underlying storage will not be freed until the last - * reference to the object is dropped. - */ - -static void init_uobj(struct ib_uobject *uobj, u64 user_handle, - struct ib_ucontext *context, struct uverbs_lock_class *c) -{ - uobj->user_handle = user_handle; - uobj->context = context; - kref_init(&uobj->ref); - init_rwsem(&uobj->mutex); - lockdep_set_class_and_name(&uobj->mutex, &c->key, c->name); - uobj->live = 0; -} - -static void release_uobj(struct kref *kref) -{ - kfree_rcu(container_of(kref, struct ib_uobject, ref), rcu); -} - -static void put_uobj(struct ib_uobject *uobj) -{ - kref_put(&uobj->ref, release_uobj); -} - -static void put_uobj_read(struct ib_uobject *uobj) -{ - up_read(&uobj->mutex); - put_uobj(uobj); -} - -static void put_uobj_write(struct ib_uobject *uobj) -{ - up_write(&uobj->mutex); - put_uobj(uobj); -} - -static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) -{ - int ret; - - idr_preload(GFP_KERNEL); - spin_lock(&ib_uverbs_idr_lock); - - ret = idr_alloc(idr, uobj, 0, 0, GFP_NOWAIT); - if (ret >= 0) - uobj->id = ret; - - spin_unlock(&ib_uverbs_idr_lock); - idr_preload_end(); - - return ret < 0 ? ret : 0; -} - -void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) +static struct ib_uverbs_completion_event_file * +ib_uverbs_lookup_comp_file(int fd, struct ib_ucontext *context) { - spin_lock(&ib_uverbs_idr_lock); - idr_remove(idr, uobj->id); - spin_unlock(&ib_uverbs_idr_lock); -} + struct ib_uobject *uobj = uobj_get_read(uobj_get_type(comp_channel), + fd, context); + struct ib_uobject_file *uobj_file; -static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, - struct ib_ucontext *context) -{ - struct ib_uobject *uobj; + if (IS_ERR(uobj)) + return (void *)uobj; - rcu_read_lock(); - uobj = idr_find(idr, id); - if (uobj) { - if (uobj->context == context) - kref_get(&uobj->ref); - else - uobj = NULL; - } - rcu_read_unlock(); - - return uobj; -} + uobj_file = container_of(uobj, struct ib_uobject_file, uobj); -static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, - struct ib_ucontext *context, int nested) -{ - struct ib_uobject *uobj; - - uobj = __idr_get_uobj(idr, id, context); - if (!uobj) - return NULL; + uverbs_uobject_get(&uobj_file->uobj); + uobj_put_read(uobj); - if (nested) - down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); - else - down_read(&uobj->mutex); - if (!uobj->live) { - put_uobj_read(uobj); - return NULL; - } - - return uobj; -} - -static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, - struct ib_ucontext *context) -{ - struct ib_uobject *uobj; - - uobj = __idr_get_uobj(idr, id, context); - if (!uobj) - return NULL; - - down_write(&uobj->mutex); - if (!uobj->live) { - put_uobj_write(uobj); - return NULL; - } - - return uobj; + return container_of(uobj_file, struct ib_uverbs_completion_event_file, + uobj_file); } -static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, - int nested) -{ - struct ib_uobject *uobj; - - uobj = idr_read_uobj(idr, id, context, nested); - return uobj ? uobj->object : NULL; -} - -static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) -{ - return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); -} - -static void put_pd_read(struct ib_pd *pd) -{ - put_uobj_read(pd->uobject); -} - -static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) -{ - return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); -} - -static void put_cq_read(struct ib_cq *cq) -{ - put_uobj_read(cq->uobject); -} - -static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) -{ - return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); -} - -static void put_ah_read(struct ib_ah *ah) -{ - put_uobj_read(ah->uobject); -} - -static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) +#if IS_ENABLED(CONFIG_INFINIBAND_USE_IOCTL_BACKWARD_COMP) +static int get_vendor_num_attrs(size_t cmd, size_t resp, int in_len, + int out_len) { - return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); + return !!(cmd != in_len) + !!(resp != out_len); } -static struct ib_wq *idr_read_wq(int wq_handle, struct ib_ucontext *context) +static void init_ioctl_hdr(struct ib_uverbs_ioctl_hdr *hdr, + struct ib_device *ib_dev, + size_t num_attrs, + u16 object_type, + u16 action) { - return idr_read_obj(&ib_uverbs_wq_idr, wq_handle, context, 0); + hdr->length = sizeof(*hdr) + num_attrs * sizeof(hdr->attrs[0]); + hdr->flags = 0; + hdr->reserved = 0; + hdr->object_type = object_type; + hdr->action = action; + hdr->num_attrs = num_attrs; } -static void put_wq_read(struct ib_wq *wq) +static void fill_attr_ptr(struct ib_uverbs_attr *attr, u16 attr_id, u16 len, + const void * __user source) { - put_uobj_read(wq->uobject); + attr->attr_id = attr_id; + attr->len = len; + attr->reserved = 0; + attr->data = (__u64)source; } -static struct ib_rwq_ind_table *idr_read_rwq_indirection_table(int ind_table_handle, - struct ib_ucontext *context) +static void fill_hw_attrs(struct ib_uverbs_attr *hw_attrs, + const void __user *in_buf, + const void __user *out_buf, + size_t cmd_size, size_t resp_size, + int in_len, int out_len) { - return idr_read_obj(&ib_uverbs_rwq_ind_tbl_idr, ind_table_handle, context, 0); -} + if (in_len > cmd_size) + fill_attr_ptr(&hw_attrs[UVERBS_UHW_IN], + UVERBS_UHW_IN | UVERBS_UDATA_DRIVER_DATA_FLAG, + in_len - cmd_size, + in_buf + cmd_size); -static void put_rwq_indirection_table_read(struct ib_rwq_ind_table *ind_table) -{ - put_uobj_read(ind_table->uobject); + if (out_len > resp_size) + fill_attr_ptr(&hw_attrs[UVERBS_UHW_OUT], + UVERBS_UHW_OUT | UVERBS_UDATA_DRIVER_DATA_FLAG, + out_len - resp_size, + out_buf + resp_size); } -static struct ib_qp *idr_write_qp(int qp_handle, struct ib_ucontext *context) +ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, + struct ib_device *ib_dev, + const char __user *buf, + int in_len, int out_len) { - struct ib_uobject *uobj; + struct ib_uverbs_get_context cmd; + struct ib_uverbs_get_context_resp resp; + struct { + struct ib_uverbs_ioctl_hdr hdr; + struct ib_uverbs_attr cmd_attrs[GET_CONTEXT_RESP + 1]; + struct ib_uverbs_attr hw_attrs[UVERBS_UHW_OUT + 1]; + } ioctl_cmd; + long err; - uobj = idr_write_uobj(&ib_uverbs_qp_idr, qp_handle, context); - return uobj ? uobj->object : NULL; -} + if (out_len < sizeof(resp)) + return -ENOSPC; -static void put_qp_read(struct ib_qp *qp) -{ - put_uobj_read(qp->uobject); -} + if (copy_from_user(&cmd, buf, sizeof(cmd))) + return -EFAULT; -static void put_qp_write(struct ib_qp *qp) -{ - put_uobj_write(qp->uobject); -} + init_ioctl_hdr(&ioctl_cmd.hdr, ib_dev, ARRAY_SIZE(ioctl_cmd.cmd_attrs) + + get_vendor_num_attrs(sizeof(cmd), sizeof(resp), in_len, + out_len), + UVERBS_TYPE_DEVICE, UVERBS_DEVICE_ALLOC_CONTEXT); -static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) -{ - return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); -} + /* + * We have to have a direct mapping between the new format and the old + * format. It's easily achievable with new attributes. + */ + fill_attr_ptr(&ioctl_cmd.cmd_attrs[GET_CONTEXT_RESP], + GET_CONTEXT_RESP, sizeof(resp), + (const void * __user)cmd.response); + fill_hw_attrs(ioctl_cmd.hw_attrs, buf, + (const void * __user)cmd.response, sizeof(cmd), + sizeof(resp), in_len, out_len); -static void put_srq_read(struct ib_srq *srq) -{ - put_uobj_read(srq->uobject); -} + err = ib_uverbs_cmd_verbs(ib_dev, file, &ioctl_cmd.hdr, + ioctl_cmd.cmd_attrs, true); -static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, struct ib_ucontext *context, - struct ib_uobject **uobj) -{ - *uobj = idr_read_uobj(&ib_uverbs_xrcd_idr, xrcd_handle, context, 0); - return *uobj ? (*uobj)->object : NULL; -} + if (err < 0) + goto err; -static void put_xrcd_read(struct ib_uobject *uobj) -{ - put_uobj_read(uobj); +err: + return err == 0 ? in_len : err; } - +#else ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, struct ib_device *ib_dev, const char __user *buf, @@ -342,17 +198,10 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, } ucontext->device = ib_dev; - INIT_LIST_HEAD(&ucontext->pd_list); - INIT_LIST_HEAD(&ucontext->mr_list); - INIT_LIST_HEAD(&ucontext->mw_list); - INIT_LIST_HEAD(&ucontext->cq_list); - INIT_LIST_HEAD(&ucontext->qp_list); - INIT_LIST_HEAD(&ucontext->srq_list); - INIT_LIST_HEAD(&ucontext->ah_list); - INIT_LIST_HEAD(&ucontext->wq_list); - INIT_LIST_HEAD(&ucontext->rwq_ind_tbl_list); - INIT_LIST_HEAD(&ucontext->xrcd_list); - INIT_LIST_HEAD(&ucontext->rule_list); + /* ufile is required when some objects are released */ + ucontext->ufile = file; + uverbs_initialize_ucontext(ucontext); + rcu_read_lock(); ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); rcu_read_unlock(); @@ -376,7 +225,7 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, goto err_free; resp.async_fd = ret; - filp = ib_uverbs_alloc_event_file(file, ib_dev, 1); + filp = ib_uverbs_alloc_async_event_file(file, ib_dev); if (IS_ERR(filp)) { ret = PTR_ERR(filp); goto err_fd; @@ -411,9 +260,9 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, mutex_unlock(&file->mutex); return ret; } +#endif -static void copy_query_dev_fields(struct ib_uverbs_file *file, - struct ib_device *ib_dev, +void uverbs_copy_query_dev_fields(struct ib_device *ib_dev, struct ib_uverbs_query_device_resp *resp, struct ib_device_attr *attr) { @@ -474,7 +323,7 @@ ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, return -EFAULT; memset(&resp, 0, sizeof resp); - copy_query_dev_fields(file, ib_dev, &resp, &ib_dev->attrs); + uverbs_copy_query_dev_fields(ib_dev, &resp, &ib_dev->attrs); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) @@ -556,12 +405,9 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); - uobj = kmalloc(sizeof *uobj, GFP_KERNEL); - if (!uobj) - return -ENOMEM; - - init_uobj(uobj, 0, file->ucontext, &pd_lock_class); - down_write(&uobj->mutex); + uobj = uobj_alloc(uobj_get_type(pd), file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); pd = ib_dev->alloc_pd(ib_dev, file->ucontext, &udata); if (IS_ERR(pd)) { @@ -575,10 +421,6 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, atomic_set(&pd->usecnt, 0); uobj->object = pd; - ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); - if (ret) - goto err_idr; - memset(&resp, 0, sizeof resp); resp.pd_handle = uobj->id; @@ -588,24 +430,15 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, goto err_copy; } - mutex_lock(&file->mutex); - list_add_tail(&uobj->list, &file->ucontext->pd_list); - mutex_unlock(&file->mutex); - - uobj->live = 1; - - up_write(&uobj->mutex); + uobj_alloc_commit(uobj); return in_len; err_copy: - idr_remove_uobj(&ib_uverbs_pd_idr, uobj); - -err_idr: ib_dealloc_pd(pd); err: - put_uobj_write(uobj); + uobj_alloc_abort(uobj); return ret; } @@ -616,43 +449,19 @@ ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, { struct ib_uverbs_dealloc_pd cmd; struct ib_uobject *uobj; - struct ib_pd *pd; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); - if (!uobj) - return -EINVAL; - pd = uobj->object; + uobj = uobj_get_write(uobj_get_type(pd), cmd.pd_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - if (atomic_read(&pd->usecnt)) { - ret = -EBUSY; - goto err_put; - } - - ret = pd->device->dealloc_pd(uobj->object); - WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd"); - if (ret) - goto err_put; + ret = uobj_remove_commit(uobj); - uobj->live = 0; - put_uobj_write(uobj); - - idr_remove_uobj(&ib_uverbs_pd_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - put_uobj(uobj); - - return in_len; - -err_put: - put_uobj_write(uobj); - return ret; + return ret ?: in_len; } struct xrcd_table_entry { @@ -789,16 +598,13 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, } } - obj = kmalloc(sizeof *obj, GFP_KERNEL); - if (!obj) { - ret = -ENOMEM; + obj = (struct ib_uxrcd_object *)uobj_alloc(uobj_get_type(xrcd), + file->ucontext); + if (IS_ERR(obj)) { + ret = PTR_ERR(obj); goto err_tree_mutex_unlock; } - init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_class); - - down_write(&obj->uobject.mutex); - if (!xrcd) { xrcd = ib_dev->alloc_xrcd(ib_dev, file->ucontext, &udata); if (IS_ERR(xrcd)) { @@ -816,10 +622,6 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, atomic_set(&obj->refcnt, 0); obj->uobject.object = xrcd; - ret = idr_add_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); - if (ret) - goto err_idr; - memset(&resp, 0, sizeof resp); resp.xrcd_handle = obj->uobject.id; @@ -828,7 +630,7 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, /* create new inode/xrcd table entry */ ret = xrcd_table_insert(file->device, inode, xrcd); if (ret) - goto err_insert_xrcd; + goto err_dealloc_xrcd; } atomic_inc(&xrcd->usecnt); } @@ -842,12 +644,7 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, if (f.file) fdput(f); - mutex_lock(&file->mutex); - list_add_tail(&obj->uobject.list, &file->ucontext->xrcd_list); - mutex_unlock(&file->mutex); - - obj->uobject.live = 1; - up_write(&obj->uobject.mutex); + uobj_alloc_commit(&obj->uobject); mutex_unlock(&file->device->xrcd_tree_mutex); return in_len; @@ -859,14 +656,11 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, atomic_dec(&xrcd->usecnt); } -err_insert_xrcd: - idr_remove_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); - -err_idr: +err_dealloc_xrcd: ib_dealloc_xrcd(xrcd); err: - put_uobj_write(&obj->uobject); + uobj_alloc_abort(&obj->uobject); err_tree_mutex_unlock: if (f.file) @@ -884,75 +678,41 @@ ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file, { struct ib_uverbs_close_xrcd cmd; struct ib_uobject *uobj; - struct ib_xrcd *xrcd = NULL; - struct inode *inode = NULL; - struct ib_uxrcd_object *obj; - int live; int ret = 0; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - mutex_lock(&file->device->xrcd_tree_mutex); - uobj = idr_write_uobj(&ib_uverbs_xrcd_idr, cmd.xrcd_handle, file->ucontext); - if (!uobj) { - ret = -EINVAL; - goto out; + uobj = uobj_get_write(uobj_get_type(xrcd), cmd.xrcd_handle, + file->ucontext); + if (IS_ERR(uobj)) { + mutex_unlock(&file->device->xrcd_tree_mutex); + return PTR_ERR(uobj); } - xrcd = uobj->object; - inode = xrcd->inode; - obj = container_of(uobj, struct ib_uxrcd_object, uobject); - if (atomic_read(&obj->refcnt)) { - put_uobj_write(uobj); - ret = -EBUSY; - goto out; - } - - if (!inode || atomic_dec_and_test(&xrcd->usecnt)) { - ret = ib_dealloc_xrcd(uobj->object); - if (!ret) - uobj->live = 0; - } - - live = uobj->live; - if (inode && ret) - atomic_inc(&xrcd->usecnt); - - put_uobj_write(uobj); - - if (ret) - goto out; - - if (inode && !live) - xrcd_table_delete(file->device, inode); - - idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj); - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - put_uobj(uobj); - ret = in_len; - -out: - mutex_unlock(&file->device->xrcd_tree_mutex); - return ret; + ret = uobj_remove_commit(uobj); + return ret ?: in_len; } -void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, - struct ib_xrcd *xrcd) +int ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, + struct ib_xrcd *xrcd, + enum rdma_remove_reason why) { struct inode *inode; + int ret; inode = xrcd->inode; if (inode && !atomic_dec_and_test(&xrcd->usecnt)) - return; + return 0; - ib_dealloc_xrcd(xrcd); + ret = ib_dealloc_xrcd(xrcd); - if (inode) + if (why == RDMA_REMOVE_DESTROY && ret) + atomic_inc(&xrcd->usecnt); + else if (inode) xrcd_table_delete(dev, inode); + + return ret; } ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, @@ -985,14 +745,11 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, if (ret) return ret; - uobj = kmalloc(sizeof *uobj, GFP_KERNEL); - if (!uobj) - return -ENOMEM; - - init_uobj(uobj, 0, file->ucontext, &mr_lock_class); - down_write(&uobj->mutex); + uobj = uobj_alloc(uobj_get_type(mr), file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - pd = idr_read_pd(cmd.pd_handle, file->ucontext); + pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto err_free; @@ -1020,9 +777,6 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, atomic_inc(&pd->usecnt); uobj->object = mr; - ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); - if (ret) - goto err_unreg; memset(&resp, 0, sizeof resp); resp.lkey = mr->lkey; @@ -1035,29 +789,20 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, goto err_copy; } - put_pd_read(pd); + uobj_put_obj_read(pd); - mutex_lock(&file->mutex); - list_add_tail(&uobj->list, &file->ucontext->mr_list); - mutex_unlock(&file->mutex); - - uobj->live = 1; - - up_write(&uobj->mutex); + uobj_alloc_commit(uobj); return in_len; err_copy: - idr_remove_uobj(&ib_uverbs_mr_idr, uobj); - -err_unreg: ib_dereg_mr(mr); err_put: - put_pd_read(pd); + uobj_put_obj_read(pd); err_free: - put_uobj_write(uobj); + uobj_alloc_abort(uobj); return ret; } @@ -1093,11 +838,10 @@ ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) return -EINVAL; - uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, - file->ucontext); - - if (!uobj) - return -EINVAL; + uobj = uobj_get_write(uobj_get_type(mr), cmd.mr_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); mr = uobj->object; @@ -1108,7 +852,7 @@ ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, } if (cmd.flags & IB_MR_REREG_PD) { - pd = idr_read_pd(cmd.pd_handle, file->ucontext); + pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto put_uobjs; @@ -1141,11 +885,10 @@ ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, put_uobj_pd: if (cmd.flags & IB_MR_REREG_PD) - put_pd_read(pd); + uobj_put_obj_read(pd); put_uobjs: - - put_uobj_write(mr->uobject); + uobj_put_write(uobj); return ret; } @@ -1156,37 +899,20 @@ ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, int out_len) { struct ib_uverbs_dereg_mr cmd; - struct ib_mr *mr; struct ib_uobject *uobj; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); - if (!uobj) - return -EINVAL; - - mr = uobj->object; - - ret = ib_dereg_mr(mr); - if (!ret) - uobj->live = 0; - - put_uobj_write(uobj); - - if (ret) - return ret; + uobj = uobj_get_write(uobj_get_type(mr), cmd.mr_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - idr_remove_uobj(&ib_uverbs_mr_idr, uobj); + ret = uobj_remove_commit(uobj); - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - put_uobj(uobj); - - return in_len; + return ret ?: in_len; } ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, @@ -1208,14 +934,11 @@ ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof(cmd))) return -EFAULT; - uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); - if (!uobj) - return -ENOMEM; - - init_uobj(uobj, 0, file->ucontext, &mw_lock_class); - down_write(&uobj->mutex); + uobj = uobj_alloc(uobj_get_type(mw), file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - pd = idr_read_pd(cmd.pd_handle, file->ucontext); + pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto err_free; @@ -1238,9 +961,6 @@ ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, atomic_inc(&pd->usecnt); uobj->object = mw; - ret = idr_add_uobj(&ib_uverbs_mw_idr, uobj); - if (ret) - goto err_unalloc; memset(&resp, 0, sizeof(resp)); resp.rkey = mw->rkey; @@ -1252,29 +972,17 @@ ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, goto err_copy; } - put_pd_read(pd); - - mutex_lock(&file->mutex); - list_add_tail(&uobj->list, &file->ucontext->mw_list); - mutex_unlock(&file->mutex); - - uobj->live = 1; - - up_write(&uobj->mutex); + uobj_put_obj_read(pd); + uobj_alloc_commit(uobj); return in_len; err_copy: - idr_remove_uobj(&ib_uverbs_mw_idr, uobj); - -err_unalloc: uverbs_dealloc_mw(mw); - err_put: - put_pd_read(pd); - + uobj_put_obj_read(pd); err_free: - put_uobj_write(uobj); + uobj_alloc_abort(uobj); return ret; } @@ -1284,37 +992,19 @@ ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file, int out_len) { struct ib_uverbs_dealloc_mw cmd; - struct ib_mw *mw; struct ib_uobject *uobj; int ret = -EINVAL; if (copy_from_user(&cmd, buf, sizeof(cmd))) return -EFAULT; - uobj = idr_write_uobj(&ib_uverbs_mw_idr, cmd.mw_handle, file->ucontext); - if (!uobj) - return -EINVAL; - - mw = uobj->object; - - ret = uverbs_dealloc_mw(mw); - if (!ret) - uobj->live = 0; - - put_uobj_write(uobj); - - if (ret) - return ret; - - idr_remove_uobj(&ib_uverbs_mw_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); + uobj = uobj_get_write(uobj_get_type(mw), cmd.mw_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - put_uobj(uobj); - - return in_len; + ret = uobj_remove_commit(uobj); + return ret ?: in_len; } ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, @@ -1324,8 +1014,8 @@ ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, { struct ib_uverbs_create_comp_channel cmd; struct ib_uverbs_create_comp_channel_resp resp; - struct file *filp; - int ret; + struct ib_uobject *uobj; + struct ib_uverbs_completion_event_file *ev_file; if (out_len < sizeof resp) return -ENOSPC; @@ -1333,25 +1023,23 @@ ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - ret = get_unused_fd_flags(O_CLOEXEC); - if (ret < 0) - return ret; - resp.fd = ret; + uobj = uobj_alloc(uobj_get_type(comp_channel), file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - filp = ib_uverbs_alloc_event_file(file, ib_dev, 0); - if (IS_ERR(filp)) { - put_unused_fd(resp.fd); - return PTR_ERR(filp); - } + resp.fd = uobj->id; + + ev_file = container_of(uobj, struct ib_uverbs_completion_event_file, + uobj_file.uobj); + ib_uverbs_init_event_file(&ev_file->ev_file); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { - put_unused_fd(resp.fd); - fput(filp); + uobj_alloc_abort(uobj); return -EFAULT; } - fd_install(resp.fd, filp); + uobj_alloc_commit(uobj); return in_len; } @@ -1369,7 +1057,7 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, void *context) { struct ib_ucq_object *obj; - struct ib_uverbs_event_file *ev_file = NULL; + struct ib_uverbs_completion_event_file *ev_file = NULL; struct ib_cq *cq; int ret; struct ib_uverbs_ex_create_cq_resp resp; @@ -1378,21 +1066,21 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, if (cmd->comp_vector >= file->device->num_comp_vectors) return ERR_PTR(-EINVAL); - obj = kmalloc(sizeof *obj, GFP_KERNEL); - if (!obj) - return ERR_PTR(-ENOMEM); - - init_uobj(&obj->uobject, cmd->user_handle, file->ucontext, &cq_lock_class); - down_write(&obj->uobject.mutex); + obj = (struct ib_ucq_object *)uobj_alloc(uobj_get_type(cq), + file->ucontext); + if (IS_ERR(obj)) + return obj; if (cmd->comp_channel >= 0) { - ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel); - if (!ev_file) { - ret = -EINVAL; + ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, + file->ucontext); + if (IS_ERR(ev_file)) { + ret = PTR_ERR(ev_file); goto err; } } + obj->uobject.user_handle = cmd->user_handle; obj->uverbs_file = file; obj->comp_events_reported = 0; obj->async_events_reported = 0; @@ -1405,8 +1093,7 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags)) attr.flags = cmd->flags; - cq = ib_dev->create_cq(ib_dev, &attr, - file->ucontext, uhw); + cq = ib_dev->create_cq(ib_dev, &attr, file->ucontext, uhw); if (IS_ERR(cq)) { ret = PTR_ERR(cq); goto err_file; @@ -1416,14 +1103,10 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, cq->uobject = &obj->uobject; cq->comp_handler = ib_uverbs_comp_handler; cq->event_handler = ib_uverbs_cq_event_handler; - cq->cq_context = ev_file; + cq->cq_context = &ev_file->ev_file; atomic_set(&cq->usecnt, 0); obj->uobject.object = cq; - ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); - if (ret) - goto err_free; - memset(&resp, 0, sizeof resp); resp.base.cq_handle = obj->uobject.id; resp.base.cqe = cq->cqe; @@ -1435,20 +1118,11 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, if (ret) goto err_cb; - mutex_lock(&file->mutex); - list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); - mutex_unlock(&file->mutex); - - obj->uobject.live = 1; - - up_write(&obj->uobject.mutex); + uobj_alloc_commit(&obj->uobject); return obj; err_cb: - idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); - -err_free: ib_destroy_cq(cq); err_file: @@ -1456,7 +1130,7 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, ib_uverbs_release_ucq(file, ev_file, obj); err: - put_uobj_write(&obj->uobject); + uobj_alloc_abort(&obj->uobject); return ERR_PTR(ret); } @@ -1579,7 +1253,7 @@ ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); - cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); + cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext); if (!cq) return -EINVAL; @@ -1594,7 +1268,7 @@ ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, ret = -EFAULT; out: - put_cq_read(cq); + uobj_put_obj_read(cq); return ret ? ret : in_len; } @@ -1641,7 +1315,7 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); + cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext); if (!cq) return -EINVAL; @@ -1673,7 +1347,7 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, ret = in_len; out_put: - put_cq_read(cq); + uobj_put_obj_read(cq); return ret; } @@ -1688,14 +1362,14 @@ ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); + cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext); if (!cq) return -EINVAL; ib_req_notify_cq(cq, cmd.solicited_only ? IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); - put_cq_read(cq); + uobj_put_obj_read(cq); return in_len; } @@ -1716,36 +1390,32 @@ ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); - if (!uobj) - return -EINVAL; + uobj = uobj_get_write(uobj_get_type(cq), cmd.cq_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); + + /* + * Make sure we don't free the memory in remove_commit as we still + * needs the uobject memory to create the response. + */ + uverbs_uobject_get(uobj); cq = uobj->object; ev_file = cq->cq_context; obj = container_of(cq->uobject, struct ib_ucq_object, uobject); - ret = ib_destroy_cq(cq); - if (!ret) - uobj->live = 0; - - put_uobj_write(uobj); + memset(&resp, 0, sizeof(resp)); - if (ret) + ret = uobj_remove_commit(uobj); + if (ret) { + uverbs_uobject_put(uobj); return ret; + } - idr_remove_uobj(&ib_uverbs_cq_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - ib_uverbs_release_ucq(file, ev_file, obj); - - memset(&resp, 0, sizeof resp); resp.comp_events_reported = obj->comp_events_reported; resp.async_events_reported = obj->async_events_reported; - put_uobj(uobj); - + uverbs_uobject_put(uobj); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) return -EFAULT; @@ -1767,7 +1437,7 @@ static int create_qp(struct ib_uverbs_file *file, struct ib_device *device; struct ib_pd *pd = NULL; struct ib_xrcd *xrcd = NULL; - struct ib_uobject *uninitialized_var(xrcd_uobj); + struct ib_uobject *xrcd_uobj = ERR_PTR(-ENOENT); struct ib_cq *scq = NULL, *rcq = NULL; struct ib_srq *srq = NULL; struct ib_qp *qp; @@ -1781,18 +1451,20 @@ static int create_qp(struct ib_uverbs_file *file, if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) return -EPERM; - obj = kzalloc(sizeof *obj, GFP_KERNEL); - if (!obj) - return -ENOMEM; + obj = (struct ib_uqp_object *)uobj_alloc(uobj_get_type(qp), + file->ucontext); + if (IS_ERR(obj)) + return PTR_ERR(obj); + obj->uxrcd = NULL; + obj->uevent.uobject.user_handle = cmd->user_handle; + mutex_init(&obj->mcast_lock); - init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, - &qp_lock_class); - down_write(&obj->uevent.uobject.mutex); if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) + sizeof(cmd->rwq_ind_tbl_handle) && (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) { - ind_tbl = idr_read_rwq_indirection_table(cmd->rwq_ind_tbl_handle, - file->ucontext); + ind_tbl = uobj_get_obj_read(rwq_ind_table, + cmd->rwq_ind_tbl_handle, + file->ucontext); if (!ind_tbl) { ret = -EINVAL; goto err_put; @@ -1816,8 +1488,15 @@ static int create_qp(struct ib_uverbs_file *file, has_sq = false; if (cmd->qp_type == IB_QPT_XRC_TGT) { - xrcd = idr_read_xrcd(cmd->pd_handle, file->ucontext, - &xrcd_uobj); + xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd->pd_handle, + file->ucontext); + + if (IS_ERR(xrcd_uobj)) { + ret = -EINVAL; + goto err_put; + } + + xrcd = (struct ib_xrcd *)xrcd_uobj->object; if (!xrcd) { ret = -EINVAL; goto err_put; @@ -1829,8 +1508,8 @@ static int create_qp(struct ib_uverbs_file *file, cmd->max_recv_sge = 0; } else { if (cmd->is_srq) { - srq = idr_read_srq(cmd->srq_handle, - file->ucontext); + srq = uobj_get_obj_read(srq, cmd->srq_handle, + file->ucontext); if (!srq || srq->srq_type != IB_SRQT_BASIC) { ret = -EINVAL; goto err_put; @@ -1839,8 +1518,8 @@ static int create_qp(struct ib_uverbs_file *file, if (!ind_tbl) { if (cmd->recv_cq_handle != cmd->send_cq_handle) { - rcq = idr_read_cq(cmd->recv_cq_handle, - file->ucontext, 0); + rcq = uobj_get_obj_read(cq, cmd->recv_cq_handle, + file->ucontext); if (!rcq) { ret = -EINVAL; goto err_put; @@ -1850,10 +1529,11 @@ static int create_qp(struct ib_uverbs_file *file, } if (has_sq) - scq = idr_read_cq(cmd->send_cq_handle, file->ucontext, !!rcq); + scq = uobj_get_obj_read(cq, cmd->send_cq_handle, + file->ucontext); if (!ind_tbl) rcq = rcq ?: scq; - pd = idr_read_pd(cmd->pd_handle, file->ucontext); + pd = uobj_get_obj_read(pd, cmd->pd_handle, file->ucontext); if (!pd || (!scq && has_sq)) { ret = -EINVAL; goto err_put; @@ -1939,9 +1619,6 @@ static int create_qp(struct ib_uverbs_file *file, qp->uobject = &obj->uevent.uobject; obj->uevent.uobject.object = qp; - ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); - if (ret) - goto err_destroy; memset(&resp, 0, sizeof resp); resp.base.qpn = qp->qp_num; @@ -1963,50 +1640,41 @@ static int create_qp(struct ib_uverbs_file *file, obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); atomic_inc(&obj->uxrcd->refcnt); - put_xrcd_read(xrcd_uobj); + uobj_put_read(xrcd_uobj); } if (pd) - put_pd_read(pd); + uobj_put_obj_read(pd); if (scq) - put_cq_read(scq); + uobj_put_obj_read(scq); if (rcq && rcq != scq) - put_cq_read(rcq); + uobj_put_obj_read(rcq); if (srq) - put_srq_read(srq); + uobj_put_obj_read(srq); if (ind_tbl) - put_rwq_indirection_table_read(ind_tbl); + uobj_put_obj_read(ind_tbl); - mutex_lock(&file->mutex); - list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); - mutex_unlock(&file->mutex); - - obj->uevent.uobject.live = 1; - - up_write(&obj->uevent.uobject.mutex); + uobj_alloc_commit(&obj->uevent.uobject); return 0; err_cb: - idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); - -err_destroy: ib_destroy_qp(qp); err_put: - if (xrcd) - put_xrcd_read(xrcd_uobj); + if (!IS_ERR(xrcd_uobj)) + uobj_put_read(xrcd_uobj); if (pd) - put_pd_read(pd); + uobj_put_obj_read(pd); if (scq) - put_cq_read(scq); + uobj_put_obj_read(scq); if (rcq && rcq != scq) - put_cq_read(rcq); + uobj_put_obj_read(rcq); if (srq) - put_srq_read(srq); + uobj_put_obj_read(srq); if (ind_tbl) - put_rwq_indirection_table_read(ind_tbl); + uobj_put_obj_read(ind_tbl); - put_uobj_write(&obj->uevent.uobject); + uobj_alloc_abort(&obj->uevent.uobject); return ret; } @@ -2142,17 +1810,22 @@ ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, (unsigned long) cmd.response + sizeof resp, in_len - sizeof cmd, out_len - sizeof resp); - obj = kmalloc(sizeof *obj, GFP_KERNEL); - if (!obj) - return -ENOMEM; + obj = (struct ib_uqp_object *)uobj_alloc(uobj_get_type(qp), + file->ucontext); + if (IS_ERR(obj)) + return PTR_ERR(obj); - init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class); - down_write(&obj->uevent.uobject.mutex); + xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd.pd_handle, + file->ucontext); + if (IS_ERR(xrcd_uobj)) { + ret = -EINVAL; + goto err_put; + } - xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj); + xrcd = (struct ib_xrcd *)xrcd_uobj->object; if (!xrcd) { ret = -EINVAL; - goto err_put; + goto err_xrcd; } attr.event_handler = ib_uverbs_qp_event_handler; @@ -2167,15 +1840,11 @@ ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, qp = ib_open_qp(xrcd, &attr); if (IS_ERR(qp)) { ret = PTR_ERR(qp); - goto err_put; + goto err_xrcd; } - qp->uobject = &obj->uevent.uobject; - obj->uevent.uobject.object = qp; - ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); - if (ret) - goto err_destroy; + obj->uevent.uobject.user_handle = cmd.user_handle; memset(&resp, 0, sizeof resp); resp.qpn = qp->qp_num; @@ -2184,32 +1853,25 @@ ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) { ret = -EFAULT; - goto err_remove; + goto err_destroy; } obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); atomic_inc(&obj->uxrcd->refcnt); - put_xrcd_read(xrcd_uobj); - - mutex_lock(&file->mutex); - list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); - mutex_unlock(&file->mutex); + qp->uobject = &obj->uevent.uobject; + uobj_put_read(xrcd_uobj); - obj->uevent.uobject.live = 1; - up_write(&obj->uevent.uobject.mutex); + uobj_alloc_commit(&obj->uevent.uobject); return in_len; -err_remove: - idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); - err_destroy: ib_destroy_qp(qp); - +err_xrcd: + uobj_put_read(xrcd_uobj); err_put: - put_xrcd_read(xrcd_uobj); - put_uobj_write(&obj->uevent.uobject); + uobj_alloc_abort(&obj->uevent.uobject); return ret; } @@ -2235,7 +1897,7 @@ ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, goto out; } - qp = idr_read_qp(cmd.qp_handle, file->ucontext); + qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext); if (!qp) { ret = -EINVAL; goto out; @@ -2243,7 +1905,7 @@ ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); - put_qp_read(qp); + uobj_put_obj_read(qp); if (ret) goto out; @@ -2314,20 +1976,6 @@ ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, return ret ? ret : in_len; } -/* Remove ignored fields set in the attribute mask */ -static int modify_qp_mask(enum ib_qp_type qp_type, int mask) -{ - switch (qp_type) { - case IB_QPT_XRC_INI: - return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); - case IB_QPT_XRC_TGT: - return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | - IB_QP_RNR_RETRY); - default: - return mask; - } -} - static int modify_qp(struct ib_uverbs_file *file, struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata) { @@ -2339,7 +1987,7 @@ static int modify_qp(struct ib_uverbs_file *file, if (!attr) return -ENOMEM; - qp = idr_read_qp(cmd->base.qp_handle, file->ucontext); + qp = uobj_get_obj_read(qp, cmd->base.qp_handle, file->ucontext); if (!qp) { ret = -EINVAL; goto out; @@ -2411,7 +2059,7 @@ static int modify_qp(struct ib_uverbs_file *file, } release_qp: - put_qp_read(qp); + uobj_put_obj_read(qp); out: kfree(attr); @@ -2498,40 +2146,27 @@ ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, memset(&resp, 0, sizeof resp); - uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); - if (!uobj) - return -EINVAL; + uobj = uobj_get_write(uobj_get_type(qp), cmd.qp_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); + qp = uobj->object; obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); + /* + * Make sure we don't free the memory in remove_commit as we still + * needs the uobject memory to create the response. + */ + uverbs_uobject_get(uobj); - if (!list_empty(&obj->mcast_list)) { - put_uobj_write(uobj); - return -EBUSY; - } - - ret = ib_destroy_qp(qp); - if (!ret) - uobj->live = 0; - - put_uobj_write(uobj); - - if (ret) + ret = uobj_remove_commit(uobj); + if (ret) { + uverbs_uobject_put(uobj); return ret; - - if (obj->uxrcd) - atomic_dec(&obj->uxrcd->refcnt); - - idr_remove_uobj(&ib_uverbs_qp_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - ib_uverbs_release_uevent(file, &obj->uevent); + } resp.events_reported = obj->uevent.events_reported; - - put_uobj(uobj); + uverbs_uobject_put(uobj); if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) @@ -2575,7 +2210,7 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, if (!user_wr) return -ENOMEM; - qp = idr_read_qp(cmd.qp_handle, file->ucontext); + qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext); if (!qp) goto out; @@ -2611,7 +2246,8 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, goto out_put; } - ud->ah = idr_read_ah(user_wr->wr.ud.ah, file->ucontext); + ud->ah = uobj_get_obj_read(ah, user_wr->wr.ud.ah, + file->ucontext); if (!ud->ah) { kfree(ud); ret = -EINVAL; @@ -2718,11 +2354,11 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, ret = -EFAULT; out_put: - put_qp_read(qp); + uobj_put_obj_read(qp); while (wr) { if (is_ud && ud_wr(wr)->ah) - put_ah_read(ud_wr(wr)->ah); + uobj_put_obj_read(ud_wr(wr)->ah); next = wr->next; kfree(wr); wr = next; @@ -2839,21 +2475,21 @@ ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, if (IS_ERR(wr)) return PTR_ERR(wr); - qp = idr_read_qp(cmd.qp_handle, file->ucontext); + qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext); if (!qp) goto out; resp.bad_wr = 0; ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr); - put_qp_read(qp); - - if (ret) + uobj_put_obj_read(qp); + if (ret) { for (next = wr; next; next = next->next) { ++resp.bad_wr; if (next == bad_wr) break; } + } if (copy_to_user((void __user *) (unsigned long) cmd.response, &resp, sizeof resp)) @@ -2889,14 +2525,14 @@ ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, if (IS_ERR(wr)) return PTR_ERR(wr); - srq = idr_read_srq(cmd.srq_handle, file->ucontext); + srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext); if (!srq) goto out; resp.bad_wr = 0; ret = srq->device->post_srq_recv(srq, wr, &bad_wr); - put_srq_read(srq); + uobj_put_obj_read(srq); if (ret) for (next = wr; next; next = next->next) { @@ -2943,14 +2579,11 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, (unsigned long)cmd.response + sizeof(resp), in_len - sizeof(cmd), out_len - sizeof(resp)); - uobj = kmalloc(sizeof *uobj, GFP_KERNEL); - if (!uobj) - return -ENOMEM; + uobj = uobj_alloc(uobj_get_type(ah), file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_class); - down_write(&uobj->mutex); - - pd = idr_read_pd(cmd.pd_handle, file->ucontext); + pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto err; @@ -2980,12 +2613,9 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, ah->pd = pd; atomic_inc(&pd->usecnt); ah->uobject = uobj; + uobj->user_handle = cmd.user_handle; uobj->object = ah; - ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); - if (ret) - goto err_destroy; - resp.ah_handle = uobj->id; if (copy_to_user((void __user *) (unsigned long) cmd.response, @@ -2994,29 +2624,19 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, goto err_copy; } - put_pd_read(pd); - - mutex_lock(&file->mutex); - list_add_tail(&uobj->list, &file->ucontext->ah_list); - mutex_unlock(&file->mutex); - - uobj->live = 1; - - up_write(&uobj->mutex); + uobj_put_obj_read(pd); + uobj_alloc_commit(uobj); return in_len; err_copy: - idr_remove_uobj(&ib_uverbs_ah_idr, uobj); - -err_destroy: ib_destroy_ah(ah); err_put: - put_pd_read(pd); + uobj_put_obj_read(pd); err: - put_uobj_write(uobj); + uobj_alloc_abort(uobj); return ret; } @@ -3025,36 +2645,19 @@ ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, const char __user *buf, int in_len, int out_len) { struct ib_uverbs_destroy_ah cmd; - struct ib_ah *ah; struct ib_uobject *uobj; int ret; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); - if (!uobj) - return -EINVAL; - ah = uobj->object; - - ret = ib_destroy_ah(ah); - if (!ret) - uobj->live = 0; - - put_uobj_write(uobj); - - if (ret) - return ret; - - idr_remove_uobj(&ib_uverbs_ah_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - put_uobj(uobj); + uobj = uobj_get_write(uobj_get_type(ah), cmd.ah_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - return in_len; + ret = uobj_remove_commit(uobj); + return ret ?: in_len; } ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, @@ -3071,12 +2674,13 @@ ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - qp = idr_write_qp(cmd.qp_handle, file->ucontext); + qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext); if (!qp) return -EINVAL; obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + mutex_lock(&obj->mcast_lock); list_for_each_entry(mcast, &obj->mcast_list, list) if (cmd.mlid == mcast->lid && !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { @@ -3100,7 +2704,8 @@ ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, kfree(mcast); out_put: - put_qp_write(qp); + mutex_unlock(&obj->mcast_lock); + uobj_put_obj_read(qp); return ret ? ret : in_len; } @@ -3119,16 +2724,17 @@ ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - qp = idr_write_qp(cmd.qp_handle, file->ucontext); + qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext); if (!qp) return -EINVAL; + obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); + mutex_lock(&obj->mcast_lock); + ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); if (ret) goto out_put; - obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); - list_for_each_entry(mcast, &obj->mcast_list, list) if (cmd.mlid == mcast->lid && !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { @@ -3138,8 +2744,8 @@ ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, } out_put: - put_qp_write(qp); - + mutex_unlock(&obj->mcast_lock); + uobj_put_obj_read(qp); return ret ? ret : in_len; } @@ -3300,20 +2906,18 @@ int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file, if (cmd.comp_mask) return -EOPNOTSUPP; - obj = kmalloc(sizeof(*obj), GFP_KERNEL); - if (!obj) - return -ENOMEM; + obj = (struct ib_uwq_object *)uobj_alloc(uobj_get_type(wq), + file->ucontext); + if (IS_ERR(obj)) + return PTR_ERR(obj); - init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, - &wq_lock_class); - down_write(&obj->uevent.uobject.mutex); - pd = idr_read_pd(cmd.pd_handle, file->ucontext); + pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext); if (!pd) { err = -EINVAL; goto err_uobj; } - cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); + cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext); if (!cq) { err = -EINVAL; goto err_put_pd; @@ -3345,9 +2949,6 @@ int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file, atomic_inc(&cq->usecnt); wq->uobject = &obj->uevent.uobject; obj->uevent.uobject.object = wq; - err = idr_add_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject); - if (err) - goto destroy_wq; memset(&resp, 0, sizeof(resp)); resp.wq_handle = obj->uevent.uobject.id; @@ -3360,27 +2961,19 @@ int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file, if (err) goto err_copy; - put_pd_read(pd); - put_cq_read(cq); - - mutex_lock(&file->mutex); - list_add_tail(&obj->uevent.uobject.list, &file->ucontext->wq_list); - mutex_unlock(&file->mutex); - - obj->uevent.uobject.live = 1; - up_write(&obj->uevent.uobject.mutex); + uobj_put_obj_read(pd); + uobj_put_obj_read(cq); + uobj_alloc_commit(&obj->uevent.uobject); return 0; err_copy: - idr_remove_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject); -destroy_wq: ib_destroy_wq(wq); err_put_cq: - put_cq_read(cq); + uobj_put_obj_read(cq); err_put_pd: - put_pd_read(pd); + uobj_put_obj_read(pd); err_uobj: - put_uobj_write(&obj->uevent.uobject); + uobj_alloc_abort(&obj->uevent.uobject); return err; } @@ -3421,31 +3014,27 @@ int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file, return -EOPNOTSUPP; resp.response_length = required_resp_len; - uobj = idr_write_uobj(&ib_uverbs_wq_idr, cmd.wq_handle, - file->ucontext); - if (!uobj) - return -EINVAL; + uobj = uobj_get_write(uobj_get_type(wq), cmd.wq_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); wq = uobj->object; obj = container_of(uobj, struct ib_uwq_object, uevent.uobject); - ret = ib_destroy_wq(wq); - if (!ret) - uobj->live = 0; + /* + * Make sure we don't free the memory in remove_commit as we still + * needs the uobject memory to create the response. + */ + uverbs_uobject_get(uobj); - put_uobj_write(uobj); - if (ret) + ret = uobj_remove_commit(uobj); + if (ret) { + uverbs_uobject_put(uobj); return ret; + } - idr_remove_uobj(&ib_uverbs_wq_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - ib_uverbs_release_uevent(file, &obj->uevent); resp.events_reported = obj->uevent.events_reported; - put_uobj(uobj); - + uverbs_uobject_put(uobj); ret = ib_copy_to_udata(ucore, &resp, resp.response_length); if (ret) return ret; @@ -3483,14 +3072,14 @@ int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file, if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE)) return -EINVAL; - wq = idr_read_wq(cmd.wq_handle, file->ucontext); + wq = uobj_get_obj_read(wq, cmd.wq_handle, file->ucontext); if (!wq) return -EINVAL; wq_attr.curr_wq_state = cmd.curr_wq_state; wq_attr.wq_state = cmd.wq_state; ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw); - put_wq_read(wq); + uobj_put_obj_read(wq); return ret; } @@ -3568,7 +3157,8 @@ int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, for (num_read_wqs = 0; num_read_wqs < num_wq_handles; num_read_wqs++) { - wq = idr_read_wq(wqs_handles[num_read_wqs], file->ucontext); + wq = uobj_get_obj_read(wq, wqs_handles[num_read_wqs], + file->ucontext); if (!wq) { err = -EINVAL; goto put_wqs; @@ -3577,14 +3167,12 @@ int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, wqs[num_read_wqs] = wq; } - uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); - if (!uobj) { - err = -ENOMEM; + uobj = uobj_alloc(uobj_get_type(rwq_ind_table), file->ucontext); + if (IS_ERR(uobj)) { + err = PTR_ERR(uobj); goto put_wqs; } - init_uobj(uobj, 0, file->ucontext, &rwq_ind_table_lock_class); - down_write(&uobj->mutex); init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; init_attr.ind_tbl = wqs; rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw); @@ -3604,10 +3192,6 @@ int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, for (i = 0; i < num_wq_handles; i++) atomic_inc(&wqs[i]->usecnt); - err = idr_add_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); - if (err) - goto destroy_ind_tbl; - resp.ind_tbl_handle = uobj->id; resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; resp.response_length = required_resp_len; @@ -3620,26 +3204,18 @@ int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, kfree(wqs_handles); for (j = 0; j < num_read_wqs; j++) - put_wq_read(wqs[j]); - - mutex_lock(&file->mutex); - list_add_tail(&uobj->list, &file->ucontext->rwq_ind_tbl_list); - mutex_unlock(&file->mutex); + uobj_put_obj_read(wqs[j]); - uobj->live = 1; - - up_write(&uobj->mutex); + uobj_alloc_commit(uobj); return 0; err_copy: - idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); -destroy_ind_tbl: ib_destroy_rwq_ind_table(rwq_ind_tbl); err_uobj: - put_uobj_write(uobj); + uobj_alloc_abort(uobj); put_wqs: for (j = 0; j < num_read_wqs; j++) - put_wq_read(wqs[j]); + uobj_put_obj_read(wqs[j]); err_free: kfree(wqs_handles); kfree(wqs); @@ -3652,10 +3228,8 @@ int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file, struct ib_udata *uhw) { struct ib_uverbs_ex_destroy_rwq_ind_table cmd = {}; - struct ib_rwq_ind_table *rwq_ind_tbl; struct ib_uobject *uobj; int ret; - struct ib_wq **ind_tbl; size_t required_cmd_sz; required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle); @@ -3675,31 +3249,12 @@ int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file, if (cmd.comp_mask) return -EOPNOTSUPP; - uobj = idr_write_uobj(&ib_uverbs_rwq_ind_tbl_idr, cmd.ind_tbl_handle, - file->ucontext); - if (!uobj) - return -EINVAL; - rwq_ind_tbl = uobj->object; - ind_tbl = rwq_ind_tbl->ind_tbl; + uobj = uobj_get_write(uobj_get_type(rwq_ind_table), cmd.ind_tbl_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - ret = ib_destroy_rwq_ind_table(rwq_ind_tbl); - if (!ret) - uobj->live = 0; - - put_uobj_write(uobj); - - if (ret) - return ret; - - idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - put_uobj(uobj); - kfree(ind_tbl); - return ret; + return uobj_remove_commit(uobj); } int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, @@ -3773,15 +3328,13 @@ int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, kern_flow_attr = &cmd.flow_attr; } - uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); - if (!uobj) { - err = -ENOMEM; + uobj = uobj_alloc(uobj_get_type(flow), file->ucontext); + if (IS_ERR(uobj)) { + err = PTR_ERR(uobj); goto err_free_attr; } - init_uobj(uobj, 0, file->ucontext, &rule_lock_class); - down_write(&uobj->mutex); - qp = idr_read_qp(cmd.qp_handle, file->ucontext); + qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext); if (!qp) { err = -EINVAL; goto err_uobj; @@ -3830,10 +3383,6 @@ int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, flow_id->uobject = uobj; uobj->object = flow_id; - err = idr_add_uobj(&ib_uverbs_rule_idr, uobj); - if (err) - goto destroy_flow; - memset(&resp, 0, sizeof(resp)); resp.flow_handle = uobj->id; @@ -3842,28 +3391,20 @@ int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, if (err) goto err_copy; - put_qp_read(qp); - mutex_lock(&file->mutex); - list_add_tail(&uobj->list, &file->ucontext->rule_list); - mutex_unlock(&file->mutex); - - uobj->live = 1; - - up_write(&uobj->mutex); + uobj_put_obj_read(qp); + uobj_alloc_commit(uobj); kfree(flow_attr); if (cmd.flow_attr.num_of_specs) kfree(kern_flow_attr); return 0; err_copy: - idr_remove_uobj(&ib_uverbs_rule_idr, uobj); -destroy_flow: ib_destroy_flow(flow_id); err_free: kfree(flow_attr); err_put: - put_qp_read(qp); + uobj_put_obj_read(qp); err_uobj: - put_uobj_write(uobj); + uobj_alloc_abort(uobj); err_free_attr: if (cmd.flow_attr.num_of_specs) kfree(kern_flow_attr); @@ -3890,27 +3431,14 @@ int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file, if (cmd.comp_mask) return -EINVAL; - uobj = idr_write_uobj(&ib_uverbs_rule_idr, cmd.flow_handle, - file->ucontext); - if (!uobj) - return -EINVAL; - flow_id = uobj->object; - - ret = ib_destroy_flow(flow_id); - if (!ret) - uobj->live = 0; + uobj = uobj_get_write(uobj_get_type(flow), cmd.flow_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); - put_uobj_write(uobj); - - idr_remove_uobj(&ib_uverbs_rule_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - put_uobj(uobj); + flow_id = uobj->object; - return ret; + return uobj_remove_commit(uobj); } static int __uverbs_create_xsrq(struct ib_uverbs_file *file, @@ -3926,31 +3454,37 @@ static int __uverbs_create_xsrq(struct ib_uverbs_file *file, struct ib_srq_init_attr attr; int ret; - obj = kmalloc(sizeof *obj, GFP_KERNEL); - if (!obj) - return -ENOMEM; - - init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_class); - down_write(&obj->uevent.uobject.mutex); + obj = (struct ib_usrq_object *)uobj_alloc(uobj_get_type(srq), + file->ucontext); + if (IS_ERR(obj)) + return PTR_ERR(obj); if (cmd->srq_type == IB_SRQT_XRC) { - attr.ext.xrc.xrcd = idr_read_xrcd(cmd->xrcd_handle, file->ucontext, &xrcd_uobj); - if (!attr.ext.xrc.xrcd) { + xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd->xrcd_handle, + file->ucontext); + if (IS_ERR(xrcd_uobj)) { ret = -EINVAL; goto err; } + attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object; + if (!attr.ext.xrc.xrcd) { + ret = -EINVAL; + goto err_put_xrcd; + } + obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); atomic_inc(&obj->uxrcd->refcnt); - attr.ext.xrc.cq = idr_read_cq(cmd->cq_handle, file->ucontext, 0); + attr.ext.xrc.cq = uobj_get_obj_read(cq, cmd->cq_handle, + file->ucontext); if (!attr.ext.xrc.cq) { ret = -EINVAL; goto err_put_xrcd; } } - pd = idr_read_pd(cmd->pd_handle, file->ucontext); + pd = uobj_get_obj_read(pd, cmd->pd_handle, file->ucontext); if (!pd) { ret = -EINVAL; goto err_put_cq; @@ -3990,9 +3524,7 @@ static int __uverbs_create_xsrq(struct ib_uverbs_file *file, atomic_set(&srq->usecnt, 0); obj->uevent.uobject.object = srq; - ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); - if (ret) - goto err_destroy; + obj->uevent.uobject.user_handle = cmd->user_handle; memset(&resp, 0, sizeof resp); resp.srq_handle = obj->uevent.uobject.id; @@ -4008,42 +3540,32 @@ static int __uverbs_create_xsrq(struct ib_uverbs_file *file, } if (cmd->srq_type == IB_SRQT_XRC) { - put_uobj_read(xrcd_uobj); - put_cq_read(attr.ext.xrc.cq); + uobj_put_read(xrcd_uobj); + uobj_put_obj_read(attr.ext.xrc.cq); } - put_pd_read(pd); - - mutex_lock(&file->mutex); - list_add_tail(&obj->uevent.uobject.list, &file->ucontext->srq_list); - mutex_unlock(&file->mutex); - - obj->uevent.uobject.live = 1; - - up_write(&obj->uevent.uobject.mutex); + uobj_put_obj_read(pd); + uobj_alloc_commit(&obj->uevent.uobject); return 0; err_copy: - idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); - -err_destroy: ib_destroy_srq(srq); err_put: - put_pd_read(pd); + uobj_put_obj_read(pd); err_put_cq: if (cmd->srq_type == IB_SRQT_XRC) - put_cq_read(attr.ext.xrc.cq); + uobj_put_obj_read(attr.ext.xrc.cq); err_put_xrcd: if (cmd->srq_type == IB_SRQT_XRC) { atomic_dec(&obj->uxrcd->refcnt); - put_uobj_read(xrcd_uobj); + uobj_put_read(xrcd_uobj); } err: - put_uobj_write(&obj->uevent.uobject); + uobj_alloc_abort(&obj->uevent.uobject); return ret; } @@ -4128,7 +3650,7 @@ ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, out_len); - srq = idr_read_srq(cmd.srq_handle, file->ucontext); + srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext); if (!srq) return -EINVAL; @@ -4137,7 +3659,7 @@ ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); - put_srq_read(srq); + uobj_put_obj_read(srq); return ret ? ret : in_len; } @@ -4159,13 +3681,13 @@ ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - srq = idr_read_srq(cmd.srq_handle, file->ucontext); + srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext); if (!srq) return -EINVAL; ret = ib_query_srq(srq, &attr); - put_srq_read(srq); + uobj_put_obj_read(srq); if (ret) return ret; @@ -4194,51 +3716,39 @@ ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, struct ib_srq *srq; struct ib_uevent_object *obj; int ret = -EINVAL; - struct ib_usrq_object *us; enum ib_srq_type srq_type; if (copy_from_user(&cmd, buf, sizeof cmd)) return -EFAULT; - uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); - if (!uobj) - return -EINVAL; + uobj = uobj_get_write(uobj_get_type(srq), cmd.srq_handle, + file->ucontext); + if (IS_ERR(uobj)) + return PTR_ERR(uobj); + srq = uobj->object; obj = container_of(uobj, struct ib_uevent_object, uobject); srq_type = srq->srq_type; + /* + * Make sure we don't free the memory in remove_commit as we still + * needs the uobject memory to create the response. + */ + uverbs_uobject_get(uobj); - ret = ib_destroy_srq(srq); - if (!ret) - uobj->live = 0; - - put_uobj_write(uobj); + memset(&resp, 0, sizeof(resp)); - if (ret) + ret = uobj_remove_commit(uobj); + if (ret) { + uverbs_uobject_put(uobj); return ret; - - if (srq_type == IB_SRQT_XRC) { - us = container_of(obj, struct ib_usrq_object, uevent); - atomic_dec(&us->uxrcd->refcnt); } - - idr_remove_uobj(&ib_uverbs_srq_idr, uobj); - - mutex_lock(&file->mutex); - list_del(&uobj->list); - mutex_unlock(&file->mutex); - - ib_uverbs_release_uevent(file, obj); - - memset(&resp, 0, sizeof resp); resp.events_reported = obj->events_reported; + uverbs_uobject_put(uobj); + if (copy_to_user((void __user *)(unsigned long)cmd.response, + &resp, sizeof(resp))) + return -EFAULT; - put_uobj(uobj); - - if (copy_to_user((void __user *) (unsigned long) cmd.response, - &resp, sizeof resp)) - ret = -EFAULT; - - return ret ? ret : in_len; + return in_len; } int ib_uverbs_ex_query_device(struct ib_uverbs_file *file, @@ -4273,7 +3783,7 @@ int ib_uverbs_ex_query_device(struct ib_uverbs_file *file, if (err) return err; - copy_query_dev_fields(file, ib_dev, &resp.base, &attr); + uverbs_copy_query_dev_fields(ib_dev, &resp.base, &attr); if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps)) goto end; diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c new file mode 100644 index 00000000000000..46ca98d3043126 --- /dev/null +++ b/drivers/infiniband/core/uverbs_ioctl.c @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include "rdma_core.h" +#include "uverbs.h" + +static int uverbs_process_attr(struct ib_device *ibdev, + struct ib_ucontext *ucontext, + const struct ib_uverbs_attr *uattr, + u16 attr_id, + const struct uverbs_attr_spec_group *attr_spec_group, + struct uverbs_attr_array *attr_array, + struct ib_uverbs_attr __user *uattr_ptr, + bool w_legacy) +{ + const struct uverbs_attr_spec *spec; + struct uverbs_attr *e; + const struct uverbs_type *type; + struct uverbs_obj_attr *o_attr; + struct uverbs_attr *elements = attr_array->attrs; + + if (uattr->reserved) + return -EINVAL; + + if (attr_id >= attr_spec_group->num_attrs) { + if (uattr->flags & UVERBS_ATTR_F_MANDATORY) + return -EINVAL; + else + return 0; + } + + spec = &attr_spec_group->attrs[attr_id]; + e = &elements[attr_id]; + + switch (spec->type) { + case UVERBS_ATTR_TYPE_PTR_IN: + case UVERBS_ATTR_TYPE_PTR_OUT: + if (uattr->len < spec->len || + (!(spec->flags & UVERBS_ATTR_SPEC_F_MIN_SZ) && + uattr->len > spec->len)) + return -EINVAL; + + e->ptr_attr.ptr = (void * __user)uattr->data; + e->ptr_attr.len = uattr->len; + break; + + case UVERBS_ATTR_TYPE_FLAG: + e->flag_attr.flags = uattr->data; + if (uattr->len) + return -EINVAL; + if (uattr->flags & UVERBS_ATTR_F_MANDATORY && + e->flag_attr.flags & ~spec->flag.mask) + return -EINVAL; + break; + + case UVERBS_ATTR_TYPE_IDR: + if (uattr->data >> 32) + return -EINVAL; + /* fall through */ + case UVERBS_ATTR_TYPE_FD: + if (uattr->len != 0 || !ucontext || uattr->data > INT_MAX) + return -EINVAL; + + o_attr = &e->obj_attr; + type = uverbs_get_type(ibdev, spec->obj.obj_type); + if (!type) + return -EINVAL; + o_attr->type = type->type_attrs; + o_attr->uattr = uattr_ptr; + + o_attr->id = (int)uattr->data; + o_attr->uobject = uverbs_get_uobject_from_context( + o_attr->type, + ucontext, + spec->obj.access, + o_attr->id); + + if (IS_ERR(o_attr->uobject)) + return -EINVAL; + + if (spec->obj.access == UVERBS_ACCESS_NEW) { + u64 id = o_attr->uobject->id; + + if (!w_legacy) { + if (put_user(id, &o_attr->uattr->data)) { + uverbs_finalize_object(o_attr->uobject, + UVERBS_ACCESS_NEW, + false); + return -EFAULT; + } + } else { + o_attr->uattr->data = id; + } + } + + break; + default: + return -EOPNOTSUPP; + }; + + set_bit(attr_id, attr_array->valid_bitmap); + return 0; +} + +static int uverbs_uattrs_process(struct ib_device *ibdev, + struct ib_ucontext *ucontext, + const struct ib_uverbs_attr *uattrs, + size_t num_uattrs, + const struct uverbs_action *action, + struct uverbs_attr_array *attr_array, + struct ib_uverbs_attr __user *uattr_ptr, + bool w_legacy) +{ + size_t i; + int ret = 0; + int num_given_groups = 0; + + for (i = 0; i < num_uattrs; i++) { + const struct ib_uverbs_attr *uattr = &uattrs[i]; + u16 attr_id = uattr->attr_id; + const struct uverbs_attr_spec_group *attr_spec_group; + + ret = uverbs_group_idx(&attr_id, action->num_groups); + if (ret < 0) { + if (uattr->flags & UVERBS_ATTR_F_MANDATORY) + return ret; + + continue; + } + + if (ret >= num_given_groups) + num_given_groups = ret + 1; + + attr_spec_group = action->attr_groups[ret]; + ret = uverbs_process_attr(ibdev, ucontext, uattr, attr_id, + attr_spec_group, &attr_array[ret], + uattr_ptr++, w_legacy); + if (ret) { + uverbs_finalize_objects(attr_array, + num_given_groups, + action, false); + return ret; + } + } + + return ret ?: num_given_groups; +} + +static int uverbs_validate_kernel_mandatory(const struct uverbs_action *action, + struct uverbs_attr_array *attr_array, + unsigned int num_given_groups) +{ + unsigned int i; + + for (i = 0; i < num_given_groups; i++) { + const struct uverbs_attr_spec_group *attr_spec_group = + action->attr_groups[i]; + + if (!bitmap_subset(attr_spec_group->mandatory_attrs_bitmask, + attr_array[i].valid_bitmap, + attr_spec_group->num_attrs)) + return -EINVAL; + } + + return 0; +} + +static int uverbs_handle_action(struct ib_uverbs_attr __user *uattr_ptr, + const struct ib_uverbs_attr *uattrs, + size_t num_uattrs, + struct ib_device *ibdev, + struct ib_uverbs_file *ufile, + const struct uverbs_action *action, + struct uverbs_attr_array *attr_array, + bool w_legacy) +{ + int ret; + int finalize_ret; + int num_given_groups; + + num_given_groups = uverbs_uattrs_process(ibdev, ufile->ucontext, uattrs, + num_uattrs, action, attr_array, + uattr_ptr, w_legacy); + if (num_given_groups <= 0) + return -EINVAL; + + ret = uverbs_validate_kernel_mandatory(action, attr_array, + num_given_groups); + if (ret) + goto cleanup; + + ret = action->handler(ibdev, ufile, attr_array, num_given_groups); +cleanup: + finalize_ret = uverbs_finalize_objects(attr_array, num_given_groups, + action, !ret); + + return ret ? ret : finalize_ret; +} + +#define UVERBS_OPTIMIZE_USING_STACK_SZ 256 +long ib_uverbs_cmd_verbs(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct ib_uverbs_ioctl_hdr *hdr, + void __user *buf, + bool w_legacy) +{ + const struct uverbs_type *type; + const struct uverbs_action *action; + long err = 0; + unsigned int i; + struct { + struct ib_uverbs_attr *uattrs; + struct uverbs_attr_array *uverbs_attr_array; + } *ctx = NULL; + struct uverbs_attr *curr_attr; + unsigned long *curr_bitmap; + size_t ctx_size; +#ifdef UVERBS_OPTIMIZE_USING_STACK_SZ + uintptr_t data[UVERBS_OPTIMIZE_USING_STACK_SZ / sizeof(uintptr_t)]; +#endif + + if (hdr->reserved) + return -EINVAL; + + type = uverbs_get_type(ib_dev, hdr->object_type); + if (!type) + return -EOPNOTSUPP; + + action = uverbs_get_action(type, hdr->action); + if (!action) + return -EOPNOTSUPP; + + if ((action->flags & UVERBS_ACTION_FLAG_CREATE_ROOT) ^ !file->ucontext) + return -EINVAL; + + ctx_size = sizeof(*ctx) + + sizeof(struct uverbs_attr_array) * action->num_groups + + sizeof(*ctx->uattrs) * hdr->num_attrs + + sizeof(*ctx->uverbs_attr_array->attrs) * action->num_child_attrs + + sizeof(*ctx->uverbs_attr_array->valid_bitmap) * + (action->num_child_attrs / BITS_PER_LONG + + action->num_groups); + +#ifdef UVERBS_OPTIMIZE_USING_STACK_SZ + if (ctx_size <= UVERBS_OPTIMIZE_USING_STACK_SZ) + ctx = (void *)data; + + if (!ctx) +#endif + ctx = kmalloc(ctx_size, GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->uverbs_attr_array = (void *)ctx + sizeof(*ctx); + ctx->uattrs = (void *)(ctx->uverbs_attr_array + + action->num_groups); + curr_attr = (void *)(ctx->uattrs + hdr->num_attrs); + curr_bitmap = (void *)(curr_attr + action->num_child_attrs); + + /* + * We just fill the pointers and num_attrs here. The data itself will be + * filled at a later stage (uverbs_process_attr) + */ + for (i = 0; i < action->num_groups; i++) { + unsigned int curr_num_attrs = action->attr_groups[i]->num_attrs; + + ctx->uverbs_attr_array[i].attrs = curr_attr; + curr_attr += curr_num_attrs; + ctx->uverbs_attr_array[i].num_attrs = curr_num_attrs; + ctx->uverbs_attr_array[i].valid_bitmap = curr_bitmap; + bitmap_zero(curr_bitmap, curr_num_attrs); + curr_bitmap += BITS_TO_LONGS(curr_num_attrs); + } + + if (w_legacy) { + memcpy(ctx->uattrs, buf, + sizeof(*ctx->uattrs) * hdr->num_attrs); + } else { + err = copy_from_user(ctx->uattrs, buf, + sizeof(*ctx->uattrs) * hdr->num_attrs); + if (err) { + err = -EFAULT; + goto out; + } + } + + err = uverbs_handle_action(buf, ctx->uattrs, hdr->num_attrs, ib_dev, + file, action, ctx->uverbs_attr_array, + w_legacy); +out: +#ifdef UVERBS_OPTIMIZE_USING_STACK_SZ + if (ctx_size > UVERBS_OPTIMIZE_USING_STACK_SZ) +#endif + kfree(ctx); + return err; +} + +#define IB_UVERBS_MAX_CMD_SZ 4096 + +long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct ib_uverbs_file *file = filp->private_data; + struct ib_uverbs_ioctl_hdr __user *user_hdr = + (struct ib_uverbs_ioctl_hdr __user *)arg; + struct ib_uverbs_ioctl_hdr hdr; + struct ib_device *ib_dev; + int srcu_key; + long err; + + srcu_key = srcu_read_lock(&file->device->disassociate_srcu); + ib_dev = srcu_dereference(file->device->ib_dev, + &file->device->disassociate_srcu); + if (!ib_dev) { + err = -EIO; + goto out; + } + + if (cmd == RDMA_VERBS_IOCTL) { + err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); + + if (err || hdr.length > IB_UVERBS_MAX_CMD_SZ || + hdr.length != sizeof(hdr) + hdr.num_attrs * sizeof(struct ib_uverbs_attr)) { + err = -EINVAL; + goto out; + } + + /* currently there are no flags supported */ + if (hdr.flags) { + err = -EOPNOTSUPP; + goto out; + } + + err = ib_uverbs_cmd_verbs(ib_dev, file, &hdr, + (__user void *)arg + sizeof(hdr), + false); + } else { + err = -ENOIOCTLCMD; + } +out: + srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); + + return err; +} + +static void uverbs_initialize_action(struct uverbs_action *action) +{ + size_t attr_group_idx; + + for (attr_group_idx = 0; attr_group_idx < action->num_groups; + attr_group_idx++) { + struct uverbs_attr_spec_group *attr_group = + action->attr_groups[attr_group_idx]; + size_t attr_idx; + + if (!attr_group) + continue; + action->num_child_attrs += attr_group->num_attrs; + for (attr_idx = 0; attr_idx < attr_group->num_attrs; + attr_idx++) { + struct uverbs_attr_spec *attr = + &attr_group->attrs[attr_idx]; + + if (attr->flags & UVERBS_ATTR_SPEC_F_MANDATORY) + set_bit(attr_idx, + attr_group->mandatory_attrs_bitmask); + } + } +} + +void uverbs_initialize_type_group(const struct uverbs_type_group *type_group) +{ + size_t type_idx; + + for (type_idx = 0; type_idx < type_group->num_types; type_idx++) { + const struct uverbs_type *type = type_group->types[type_idx]; + size_t action_group_idx; + + if (!type) + continue; + for (action_group_idx = 0; + action_group_idx < type->num_groups; + action_group_idx++) { + const struct uverbs_action_group *action_group = + type->action_groups[action_group_idx]; + size_t action_idx; + + if (!action_group) + continue; + for (action_idx = 0; + action_idx < action_group->num_actions; + action_idx++) { + struct uverbs_action *action = + action_group->actions[action_idx]; + + if (!action) + continue; + uverbs_initialize_action(action); + } + } + } +} diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index b3f95d453fba73..3851d79c8b69a0 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -45,12 +45,14 @@ #include #include #include +#include #include #include #include "uverbs.h" +#include "rdma_core.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand userspace verbs access"); @@ -66,19 +68,6 @@ enum { static struct class *uverbs_class; -DEFINE_SPINLOCK(ib_uverbs_idr_lock); -DEFINE_IDR(ib_uverbs_pd_idr); -DEFINE_IDR(ib_uverbs_mr_idr); -DEFINE_IDR(ib_uverbs_mw_idr); -DEFINE_IDR(ib_uverbs_ah_idr); -DEFINE_IDR(ib_uverbs_cq_idr); -DEFINE_IDR(ib_uverbs_qp_idr); -DEFINE_IDR(ib_uverbs_srq_idr); -DEFINE_IDR(ib_uverbs_xrcd_idr); -DEFINE_IDR(ib_uverbs_rule_idr); -DEFINE_IDR(ib_uverbs_wq_idr); -DEFINE_IDR(ib_uverbs_rwq_ind_tbl_idr); - static DEFINE_SPINLOCK(map_lock); static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES); @@ -167,37 +156,37 @@ static struct kobj_type ib_uverbs_dev_ktype = { .release = ib_uverbs_release_dev, }; -static void ib_uverbs_release_event_file(struct kref *ref) +static void ib_uverbs_release_async_event_file(struct kref *ref) { - struct ib_uverbs_event_file *file = - container_of(ref, struct ib_uverbs_event_file, ref); + struct ib_uverbs_async_event_file *file = + container_of(ref, struct ib_uverbs_async_event_file, ref); kfree(file); } void ib_uverbs_release_ucq(struct ib_uverbs_file *file, - struct ib_uverbs_event_file *ev_file, + struct ib_uverbs_completion_event_file *ev_file, struct ib_ucq_object *uobj) { struct ib_uverbs_event *evt, *tmp; if (ev_file) { - spin_lock_irq(&ev_file->lock); + spin_lock_irq(&ev_file->ev_file.lock); list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) { list_del(&evt->list); kfree(evt); } - spin_unlock_irq(&ev_file->lock); + spin_unlock_irq(&ev_file->ev_file.lock); - kref_put(&ev_file->ref, ib_uverbs_release_event_file); + uverbs_uobject_put(&ev_file->uobj_file.uobj); } - spin_lock_irq(&file->async_file->lock); + spin_lock_irq(&file->async_file->ev_file.lock); list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) { list_del(&evt->list); kfree(evt); } - spin_unlock_irq(&file->async_file->lock); + spin_unlock_irq(&file->async_file->ev_file.lock); } void ib_uverbs_release_uevent(struct ib_uverbs_file *file, @@ -205,16 +194,16 @@ void ib_uverbs_release_uevent(struct ib_uverbs_file *file, { struct ib_uverbs_event *evt, *tmp; - spin_lock_irq(&file->async_file->lock); + spin_lock_irq(&file->async_file->ev_file.lock); list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) { list_del(&evt->list); kfree(evt); } - spin_unlock_irq(&file->async_file->lock); + spin_unlock_irq(&file->async_file->ev_file.lock); } -static void ib_uverbs_detach_umcast(struct ib_qp *qp, - struct ib_uqp_object *uobj) +void ib_uverbs_detach_umcast(struct ib_qp *qp, + struct ib_uqp_object *uobj) { struct ib_uverbs_mcast_entry *mcast, *tmp; @@ -226,122 +215,11 @@ static void ib_uverbs_detach_umcast(struct ib_qp *qp, } static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file, - struct ib_ucontext *context) + struct ib_ucontext *context, + bool device_removed) { - struct ib_uobject *uobj, *tmp; - context->closing = 1; - - list_for_each_entry_safe(uobj, tmp, &context->ah_list, list) { - struct ib_ah *ah = uobj->object; - - idr_remove_uobj(&ib_uverbs_ah_idr, uobj); - ib_destroy_ah(ah); - kfree(uobj); - } - - /* Remove MWs before QPs, in order to support type 2A MWs. */ - list_for_each_entry_safe(uobj, tmp, &context->mw_list, list) { - struct ib_mw *mw = uobj->object; - - idr_remove_uobj(&ib_uverbs_mw_idr, uobj); - uverbs_dealloc_mw(mw); - kfree(uobj); - } - - list_for_each_entry_safe(uobj, tmp, &context->rule_list, list) { - struct ib_flow *flow_id = uobj->object; - - idr_remove_uobj(&ib_uverbs_rule_idr, uobj); - ib_destroy_flow(flow_id); - kfree(uobj); - } - - list_for_each_entry_safe(uobj, tmp, &context->qp_list, list) { - struct ib_qp *qp = uobj->object; - struct ib_uqp_object *uqp = - container_of(uobj, struct ib_uqp_object, uevent.uobject); - - idr_remove_uobj(&ib_uverbs_qp_idr, uobj); - if (qp == qp->real_qp) - ib_uverbs_detach_umcast(qp, uqp); - ib_destroy_qp(qp); - ib_uverbs_release_uevent(file, &uqp->uevent); - kfree(uqp); - } - - list_for_each_entry_safe(uobj, tmp, &context->rwq_ind_tbl_list, list) { - struct ib_rwq_ind_table *rwq_ind_tbl = uobj->object; - struct ib_wq **ind_tbl = rwq_ind_tbl->ind_tbl; - - idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); - ib_destroy_rwq_ind_table(rwq_ind_tbl); - kfree(ind_tbl); - kfree(uobj); - } - - list_for_each_entry_safe(uobj, tmp, &context->wq_list, list) { - struct ib_wq *wq = uobj->object; - struct ib_uwq_object *uwq = - container_of(uobj, struct ib_uwq_object, uevent.uobject); - - idr_remove_uobj(&ib_uverbs_wq_idr, uobj); - ib_destroy_wq(wq); - ib_uverbs_release_uevent(file, &uwq->uevent); - kfree(uwq); - } - - list_for_each_entry_safe(uobj, tmp, &context->srq_list, list) { - struct ib_srq *srq = uobj->object; - struct ib_uevent_object *uevent = - container_of(uobj, struct ib_uevent_object, uobject); - - idr_remove_uobj(&ib_uverbs_srq_idr, uobj); - ib_destroy_srq(srq); - ib_uverbs_release_uevent(file, uevent); - kfree(uevent); - } - - list_for_each_entry_safe(uobj, tmp, &context->cq_list, list) { - struct ib_cq *cq = uobj->object; - struct ib_uverbs_event_file *ev_file = cq->cq_context; - struct ib_ucq_object *ucq = - container_of(uobj, struct ib_ucq_object, uobject); - - idr_remove_uobj(&ib_uverbs_cq_idr, uobj); - ib_destroy_cq(cq); - ib_uverbs_release_ucq(file, ev_file, ucq); - kfree(ucq); - } - - list_for_each_entry_safe(uobj, tmp, &context->mr_list, list) { - struct ib_mr *mr = uobj->object; - - idr_remove_uobj(&ib_uverbs_mr_idr, uobj); - ib_dereg_mr(mr); - kfree(uobj); - } - - mutex_lock(&file->device->xrcd_tree_mutex); - list_for_each_entry_safe(uobj, tmp, &context->xrcd_list, list) { - struct ib_xrcd *xrcd = uobj->object; - struct ib_uxrcd_object *uxrcd = - container_of(uobj, struct ib_uxrcd_object, uobject); - - idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj); - ib_uverbs_dealloc_xrcd(file->device, xrcd); - kfree(uxrcd); - } - mutex_unlock(&file->device->xrcd_tree_mutex); - - list_for_each_entry_safe(uobj, tmp, &context->pd_list, list) { - struct ib_pd *pd = uobj->object; - - idr_remove_uobj(&ib_uverbs_pd_idr, uobj); - ib_dealloc_pd(pd); - kfree(uobj); - } - + uverbs_cleanup_ucontext(context, device_removed); put_pid(context->tgid); return context->device->dealloc_ucontext(context); @@ -352,7 +230,7 @@ static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev) complete(&dev->comp); } -static void ib_uverbs_release_file(struct kref *ref) +void ib_uverbs_release_file(struct kref *ref) { struct ib_uverbs_file *file = container_of(ref, struct ib_uverbs_file, ref); @@ -372,10 +250,12 @@ static void ib_uverbs_release_file(struct kref *ref) kfree(file); } -static ssize_t ib_uverbs_event_read(struct file *filp, char __user *buf, - size_t count, loff_t *pos) +static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_file *file, + struct ib_uverbs_file *uverbs_file, + struct file *filp, char __user *buf, + size_t count, loff_t *pos, + bool is_async) { - struct ib_uverbs_event_file *file = filp->private_data; struct ib_uverbs_event *event; int eventsz; int ret = 0; @@ -394,12 +274,12 @@ static ssize_t ib_uverbs_event_read(struct file *filp, char __user *buf, * and wake_up() guarentee this will see the null set * without using RCU */ - !file->uverbs_file->device->ib_dev))) + !uverbs_file->device->ib_dev))) return -ERESTARTSYS; /* If device was disassociated and no event exists set an error */ if (list_empty(&file->event_list) && - !file->uverbs_file->device->ib_dev) + !uverbs_file->device->ib_dev) return -EIO; spin_lock_irq(&file->lock); @@ -407,7 +287,7 @@ static ssize_t ib_uverbs_event_read(struct file *filp, char __user *buf, event = list_entry(file->event_list.next, struct ib_uverbs_event, list); - if (file->is_async) + if (is_async) eventsz = sizeof (struct ib_uverbs_async_event_desc); else eventsz = sizeof (struct ib_uverbs_comp_event_desc); @@ -437,11 +317,31 @@ static ssize_t ib_uverbs_event_read(struct file *filp, char __user *buf, return ret; } -static unsigned int ib_uverbs_event_poll(struct file *filp, +static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct ib_uverbs_async_event_file *file = filp->private_data; + + return ib_uverbs_event_read(&file->ev_file, file->uverbs_file, filp, + buf, count, pos, true); +} + +static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct ib_uverbs_completion_event_file *comp_ev_file = + filp->private_data; + + return ib_uverbs_event_read(&comp_ev_file->ev_file, + comp_ev_file->uobj_file.ufile, filp, + buf, count, pos, false); +} + +static unsigned int ib_uverbs_event_poll(struct ib_uverbs_event_file *file, + struct file *filp, struct poll_table_struct *wait) { unsigned int pollflags = 0; - struct ib_uverbs_event_file *file = filp->private_data; poll_wait(filp, &file->poll_wait, wait); @@ -453,49 +353,98 @@ static unsigned int ib_uverbs_event_poll(struct file *filp, return pollflags; } -static int ib_uverbs_event_fasync(int fd, struct file *filp, int on) +static unsigned int ib_uverbs_async_event_poll(struct file *filp, + struct poll_table_struct *wait) +{ + return ib_uverbs_event_poll(filp->private_data, filp, wait); +} + +static unsigned int ib_uverbs_comp_event_poll(struct file *filp, + struct poll_table_struct *wait) +{ + struct ib_uverbs_completion_event_file *comp_ev_file = + filp->private_data; + + return ib_uverbs_event_poll(&comp_ev_file->ev_file, filp, wait); +} + +static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on) { struct ib_uverbs_event_file *file = filp->private_data; return fasync_helper(fd, filp, on, &file->async_queue); } -static int ib_uverbs_event_close(struct inode *inode, struct file *filp) +static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on) { - struct ib_uverbs_event_file *file = filp->private_data; + struct ib_uverbs_completion_event_file *comp_ev_file = + filp->private_data; + + return fasync_helper(fd, filp, on, &comp_ev_file->ev_file.async_queue); +} + +static int ib_uverbs_async_event_close(struct inode *inode, struct file *filp) +{ + struct ib_uverbs_async_event_file *file = filp->private_data; + struct ib_uverbs_file *uverbs_file = file->uverbs_file; struct ib_uverbs_event *entry, *tmp; int closed_already = 0; - mutex_lock(&file->uverbs_file->device->lists_mutex); - spin_lock_irq(&file->lock); - closed_already = file->is_closed; - file->is_closed = 1; - list_for_each_entry_safe(entry, tmp, &file->event_list, list) { + mutex_lock(&uverbs_file->device->lists_mutex); + spin_lock_irq(&file->ev_file.lock); + closed_already = file->ev_file.is_closed; + file->ev_file.is_closed = 1; + list_for_each_entry_safe(entry, tmp, &file->ev_file.event_list, list) { if (entry->counter) list_del(&entry->obj_list); kfree(entry); } - spin_unlock_irq(&file->lock); + spin_unlock_irq(&file->ev_file.lock); if (!closed_already) { list_del(&file->list); - if (file->is_async) - ib_unregister_event_handler(&file->uverbs_file-> - event_handler); + ib_unregister_event_handler(&uverbs_file->event_handler); + } + mutex_unlock(&uverbs_file->device->lists_mutex); + + kref_put(&uverbs_file->ref, ib_uverbs_release_file); + kref_put(&file->ref, ib_uverbs_release_async_event_file); + + return 0; +} + +static int ib_uverbs_comp_event_close(struct inode *inode, struct file *filp) +{ + struct ib_uverbs_completion_event_file *file = filp->private_data; + struct ib_uverbs_event *entry, *tmp; + + spin_lock_irq(&file->ev_file.lock); + list_for_each_entry_safe(entry, tmp, &file->ev_file.event_list, list) { + if (entry->counter) + list_del(&entry->obj_list); + kfree(entry); } - mutex_unlock(&file->uverbs_file->device->lists_mutex); + spin_unlock_irq(&file->ev_file.lock); - kref_put(&file->uverbs_file->ref, ib_uverbs_release_file); - kref_put(&file->ref, ib_uverbs_release_event_file); + uverbs_close_fd(filp); return 0; } -static const struct file_operations uverbs_event_fops = { +const struct file_operations uverbs_event_fops = { + .owner = THIS_MODULE, + .read = ib_uverbs_comp_event_read, + .poll = ib_uverbs_comp_event_poll, + .release = ib_uverbs_comp_event_close, + .fasync = ib_uverbs_comp_event_fasync, + .llseek = no_llseek, +}; + +static const struct file_operations uverbs_async_event_fops = { .owner = THIS_MODULE, - .read = ib_uverbs_event_read, - .poll = ib_uverbs_event_poll, - .release = ib_uverbs_event_close, - .fasync = ib_uverbs_event_fasync, + .read = ib_uverbs_async_event_read, + .poll = ib_uverbs_async_event_poll, + .release = ib_uverbs_async_event_close, + .fasync = ib_uverbs_async_event_fasync, .llseek = no_llseek, }; @@ -542,15 +491,15 @@ static void ib_uverbs_async_handler(struct ib_uverbs_file *file, struct ib_uverbs_event *entry; unsigned long flags; - spin_lock_irqsave(&file->async_file->lock, flags); - if (file->async_file->is_closed) { - spin_unlock_irqrestore(&file->async_file->lock, flags); + spin_lock_irqsave(&file->async_file->ev_file.lock, flags); + if (file->async_file->ev_file.is_closed) { + spin_unlock_irqrestore(&file->async_file->ev_file.lock, flags); return; } entry = kmalloc(sizeof *entry, GFP_ATOMIC); if (!entry) { - spin_unlock_irqrestore(&file->async_file->lock, flags); + spin_unlock_irqrestore(&file->async_file->ev_file.lock, flags); return; } @@ -559,13 +508,13 @@ static void ib_uverbs_async_handler(struct ib_uverbs_file *file, entry->desc.async.reserved = 0; entry->counter = counter; - list_add_tail(&entry->list, &file->async_file->event_list); + list_add_tail(&entry->list, &file->async_file->ev_file.event_list); if (obj_list) list_add_tail(&entry->obj_list, obj_list); - spin_unlock_irqrestore(&file->async_file->lock, flags); + spin_unlock_irqrestore(&file->async_file->ev_file.lock, flags); - wake_up_interruptible(&file->async_file->poll_wait); - kill_fasync(&file->async_file->async_queue, SIGIO, POLL_IN); + wake_up_interruptible(&file->async_file->ev_file.poll_wait); + kill_fasync(&file->async_file->ev_file.async_queue, SIGIO, POLL_IN); } void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr) @@ -583,7 +532,7 @@ void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr) struct ib_uevent_object *uobj; /* for XRC target qp's, check that qp is live */ - if (!event->element.qp->uobject || !event->element.qp->uobject->live) + if (!event->element.qp->uobject) return; uobj = container_of(event->element.qp->uobject, @@ -628,15 +577,23 @@ void ib_uverbs_event_handler(struct ib_event_handler *handler, void ib_uverbs_free_async_event_file(struct ib_uverbs_file *file) { - kref_put(&file->async_file->ref, ib_uverbs_release_event_file); + kref_put(&file->async_file->ref, ib_uverbs_release_async_event_file); file->async_file = NULL; } -struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file, - struct ib_device *ib_dev, - int is_async) +void ib_uverbs_init_event_file(struct ib_uverbs_event_file *ev_file) { - struct ib_uverbs_event_file *ev_file; + spin_lock_init(&ev_file->lock); + INIT_LIST_HEAD(&ev_file->event_list); + init_waitqueue_head(&ev_file->poll_wait); + ev_file->is_closed = 0; + ev_file->async_queue = NULL; +} + +struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file, + struct ib_device *ib_dev) +{ + struct ib_uverbs_async_event_file *ev_file; struct file *filp; int ret; @@ -644,16 +601,11 @@ struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file, if (!ev_file) return ERR_PTR(-ENOMEM); - kref_init(&ev_file->ref); - spin_lock_init(&ev_file->lock); - INIT_LIST_HEAD(&ev_file->event_list); - init_waitqueue_head(&ev_file->poll_wait); + ib_uverbs_init_event_file(&ev_file->ev_file); ev_file->uverbs_file = uverbs_file; kref_get(&ev_file->uverbs_file->ref); - ev_file->async_queue = NULL; - ev_file->is_closed = 0; - - filp = anon_inode_getfile("[infinibandevent]", &uverbs_event_fops, + kref_init(&ev_file->ref); + filp = anon_inode_getfile("[infinibandevent]", &uverbs_async_event_fops, ev_file, O_RDONLY); if (IS_ERR(filp)) goto err_put_refs; @@ -663,64 +615,33 @@ struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file, &uverbs_file->device->uverbs_events_file_list); mutex_unlock(&uverbs_file->device->lists_mutex); - if (is_async) { - WARN_ON(uverbs_file->async_file); - uverbs_file->async_file = ev_file; - kref_get(&uverbs_file->async_file->ref); - INIT_IB_EVENT_HANDLER(&uverbs_file->event_handler, - ib_dev, - ib_uverbs_event_handler); - ret = ib_register_event_handler(&uverbs_file->event_handler); - if (ret) - goto err_put_file; - - /* At that point async file stuff was fully set */ - ev_file->is_async = 1; - } + WARN_ON(uverbs_file->async_file); + uverbs_file->async_file = ev_file; + kref_get(&uverbs_file->async_file->ref); + INIT_IB_EVENT_HANDLER(&uverbs_file->event_handler, + ib_dev, + ib_uverbs_event_handler); + ret = ib_register_event_handler(&uverbs_file->event_handler); + if (ret) + goto err_put_file; + + /* At that point async file stuff was fully set */ return filp; err_put_file: fput(filp); - kref_put(&uverbs_file->async_file->ref, ib_uverbs_release_event_file); + kref_put(&uverbs_file->async_file->ref, + ib_uverbs_release_async_event_file); uverbs_file->async_file = NULL; return ERR_PTR(ret); err_put_refs: kref_put(&ev_file->uverbs_file->ref, ib_uverbs_release_file); - kref_put(&ev_file->ref, ib_uverbs_release_event_file); + kref_put(&ev_file->ref, ib_uverbs_release_async_event_file); return filp; } -/* - * Look up a completion event file by FD. If lookup is successful, - * takes a ref to the event file struct that it returns; if - * unsuccessful, returns NULL. - */ -struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd) -{ - struct ib_uverbs_event_file *ev_file = NULL; - struct fd f = fdget(fd); - - if (!f.file) - return NULL; - - if (f.file->f_op != &uverbs_event_fops) - goto out; - - ev_file = f.file->private_data; - if (ev_file->is_async) { - ev_file = NULL; - goto out; - } - - kref_get(&ev_file->ref); - -out: - fdput(f); - return ev_file; -} - static int verify_command_mask(struct ib_device *ib_dev, __u32 command) { u64 mask; @@ -966,6 +887,8 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp) } file->device = dev; + spin_lock_init(&file->idr_lock); + idr_init(&file->idr); file->ucontext = NULL; file->async_file = NULL; kref_init(&file->ref); @@ -999,10 +922,11 @@ static int ib_uverbs_close(struct inode *inode, struct file *filp) mutex_lock(&file->cleanup_mutex); if (file->ucontext) { - ib_uverbs_cleanup_ucontext(file, file->ucontext); + ib_uverbs_cleanup_ucontext(file, file->ucontext, false); file->ucontext = NULL; } mutex_unlock(&file->cleanup_mutex); + idr_destroy(&file->idr); mutex_lock(&file->device->lists_mutex); if (!file->is_closed) { @@ -1012,7 +936,8 @@ static int ib_uverbs_close(struct inode *inode, struct file *filp) mutex_unlock(&file->device->lists_mutex); if (file->async_file) - kref_put(&file->async_file->ref, ib_uverbs_release_event_file); + kref_put(&file->async_file->ref, + ib_uverbs_release_async_event_file); kref_put(&file->ref, ib_uverbs_release_file); kobject_put(&dev->kobj); @@ -1026,6 +951,9 @@ static const struct file_operations uverbs_fops = { .open = ib_uverbs_open, .release = ib_uverbs_close, .llseek = no_llseek, +#if IS_ENABLED(CONFIG_INFINIBAND_EXP_USER_ACCESS) + .unlocked_ioctl = ib_uverbs_ioctl, +#endif }; static const struct file_operations uverbs_mmap_fops = { @@ -1035,6 +963,9 @@ static const struct file_operations uverbs_mmap_fops = { .open = ib_uverbs_open, .release = ib_uverbs_close, .llseek = no_llseek, +#if IS_ENABLED(CONFIG_INFINIBAND_EXP_USER_ACCESS) + .unlocked_ioctl = ib_uverbs_ioctl, +#endif }; static struct ib_client uverbs_client = { @@ -1211,7 +1142,7 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, struct ib_device *ib_dev) { struct ib_uverbs_file *file; - struct ib_uverbs_event_file *event_file; + struct ib_uverbs_async_event_file *event_file; struct ib_event event; /* Pending running commands to terminate */ @@ -1248,7 +1179,9 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, * (e.g mmput). */ ib_dev->disassociate_ucontext(ucontext); - ib_uverbs_cleanup_ucontext(file, ucontext); + mutex_lock(&file->cleanup_mutex); + ib_uverbs_cleanup_ucontext(file, ucontext, true); + mutex_unlock(&file->cleanup_mutex); } mutex_lock(&uverbs_dev->lists_mutex); @@ -1258,21 +1191,20 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev, while (!list_empty(&uverbs_dev->uverbs_events_file_list)) { event_file = list_first_entry(&uverbs_dev-> uverbs_events_file_list, - struct ib_uverbs_event_file, + struct ib_uverbs_async_event_file, list); - spin_lock_irq(&event_file->lock); - event_file->is_closed = 1; - spin_unlock_irq(&event_file->lock); + spin_lock_irq(&event_file->ev_file.lock); + event_file->ev_file.is_closed = 1; + spin_unlock_irq(&event_file->ev_file.lock); list_del(&event_file->list); - if (event_file->is_async) { - ib_unregister_event_handler(&event_file->uverbs_file-> - event_handler); - event_file->uverbs_file->event_handler.device = NULL; - } + ib_unregister_event_handler( + &event_file->uverbs_file->event_handler); + event_file->uverbs_file->event_handler.device = + NULL; - wake_up_interruptible(&event_file->poll_wait); - kill_fasync(&event_file->async_queue, SIGIO, POLL_IN); + wake_up_interruptible(&event_file->ev_file.poll_wait); + kill_fasync(&event_file->ev_file.async_queue, SIGIO, POLL_IN); } mutex_unlock(&uverbs_dev->lists_mutex); } @@ -1329,6 +1261,8 @@ static int __init ib_uverbs_init(void) { int ret; + uverbs_initialize_type_group(&uverbs_common_types); + ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES, "infiniband_verbs"); if (ret) { @@ -1376,13 +1310,6 @@ static void __exit ib_uverbs_cleanup(void) unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES); if (overflow_maj) unregister_chrdev_region(overflow_maj, IB_UVERBS_MAX_DEVICES); - idr_destroy(&ib_uverbs_pd_idr); - idr_destroy(&ib_uverbs_mr_idr); - idr_destroy(&ib_uverbs_mw_idr); - idr_destroy(&ib_uverbs_ah_idr); - idr_destroy(&ib_uverbs_cq_idr); - idr_destroy(&ib_uverbs_qp_idr); - idr_destroy(&ib_uverbs_srq_idr); } module_init(ib_uverbs_init); diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c new file mode 100644 index 00000000000000..c6d92ef9d9f224 --- /dev/null +++ b/drivers/infiniband/core/uverbs_std_types.c @@ -0,0 +1,1413 @@ +/* + * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include "rdma_core.h" +#include "uverbs.h" +#include "core_priv.h" + +int uverbs_free_ah(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + return ib_destroy_ah((struct ib_ah *)uobject->object); +} + +int uverbs_free_flow(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + return ib_destroy_flow((struct ib_flow *)uobject->object); +} + +int uverbs_free_mw(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + return uverbs_dealloc_mw((struct ib_mw *)uobject->object); +} + +int uverbs_free_qp(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + struct ib_qp *qp = uobject->object; + struct ib_uqp_object *uqp = + container_of(uobject, struct ib_uqp_object, uevent.uobject); + int ret; + + if (why == RDMA_REMOVE_DESTROY) { + if (!list_empty(&uqp->mcast_list)) + return -EBUSY; + } else if (qp == qp->real_qp) { + ib_uverbs_detach_umcast(qp, uqp); + } + + ret = ib_destroy_qp(qp); + if (ret && why == RDMA_REMOVE_DESTROY) + return ret; + + if (uqp->uxrcd) + atomic_dec(&uqp->uxrcd->refcnt); + + ib_uverbs_release_uevent(uobject->context->ufile, &uqp->uevent); + return ret; +} + +int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + struct ib_rwq_ind_table *rwq_ind_tbl = uobject->object; + struct ib_wq **ind_tbl = rwq_ind_tbl->ind_tbl; + int ret; + + ret = ib_destroy_rwq_ind_table(rwq_ind_tbl); + if (!ret || why != RDMA_REMOVE_DESTROY) + kfree(ind_tbl); + return ret; +} + +int uverbs_free_wq(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + struct ib_wq *wq = uobject->object; + struct ib_uwq_object *uwq = + container_of(uobject, struct ib_uwq_object, uevent.uobject); + int ret; + + ret = ib_destroy_wq(wq); + if (!ret || why != RDMA_REMOVE_DESTROY) + ib_uverbs_release_uevent(uobject->context->ufile, &uwq->uevent); + return ret; +} + +int uverbs_free_srq(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + struct ib_srq *srq = uobject->object; + struct ib_uevent_object *uevent = + container_of(uobject, struct ib_uevent_object, uobject); + enum ib_srq_type srq_type = srq->srq_type; + int ret; + + ret = ib_destroy_srq(srq); + + if (ret && why == RDMA_REMOVE_DESTROY) + return ret; + + if (srq_type == IB_SRQT_XRC) { + struct ib_usrq_object *us = + container_of(uevent, struct ib_usrq_object, uevent); + + atomic_dec(&us->uxrcd->refcnt); + } + + ib_uverbs_release_uevent(uobject->context->ufile, uevent); + return ret; +} + +int uverbs_free_cq(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + struct ib_cq *cq = uobject->object; + struct ib_uverbs_event_file *ev_file = cq->cq_context; + struct ib_ucq_object *ucq = + container_of(uobject, struct ib_ucq_object, uobject); + int ret; + + ret = ib_destroy_cq(cq); + if (!ret || why != RDMA_REMOVE_DESTROY) + ib_uverbs_release_ucq(uobject->context->ufile, ev_file ? + container_of(ev_file, + struct ib_uverbs_completion_event_file, + ev_file) : NULL, + ucq); + return ret; +} + +int uverbs_free_mr(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + return ib_dereg_mr((struct ib_mr *)uobject->object); +} + +int uverbs_free_xrcd(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + struct ib_xrcd *xrcd = uobject->object; + struct ib_uxrcd_object *uxrcd = + container_of(uobject, struct ib_uxrcd_object, uobject); + int ret; + + mutex_lock(&uobject->context->ufile->device->xrcd_tree_mutex); + if (why == RDMA_REMOVE_DESTROY && atomic_read(&uxrcd->refcnt)) + ret = -EBUSY; + else + ret = ib_uverbs_dealloc_xrcd(uobject->context->ufile->device, + xrcd, why); + mutex_unlock(&uobject->context->ufile->device->xrcd_tree_mutex); + + return ret; +} + +int uverbs_free_pd(struct ib_uobject *uobject, + enum rdma_remove_reason why) +{ + struct ib_pd *pd = uobject->object; + + if (why == RDMA_REMOVE_DESTROY && atomic_read(&pd->usecnt)) + return -EBUSY; + + ib_dealloc_pd((struct ib_pd *)uobject->object); + return 0; +} + +int uverbs_hot_unplug_completion_event_file(struct ib_uobject_file *uobj_file, + enum rdma_remove_reason why) +{ + struct ib_uverbs_completion_event_file *comp_event_file = + container_of(uobj_file, struct ib_uverbs_completion_event_file, + uobj_file); + struct ib_uverbs_event_file *event_file = &comp_event_file->ev_file; + + spin_lock_irq(&event_file->lock); + event_file->is_closed = 1; + spin_unlock_irq(&event_file->lock); + + if (why == RDMA_REMOVE_DRIVER_REMOVE) { + wake_up_interruptible(&event_file->poll_wait); + kill_fasync(&event_file->async_queue, SIGIO, POLL_IN); + } + return 0; +}; + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_uhw_compat_spec, + UVERBS_ATTR_PTR_IN_SZ(UVERBS_UHW_IN, 0, UA_FLAGS(UVERBS_ATTR_SPEC_F_MIN_SZ)), + UVERBS_ATTR_PTR_OUT_SZ(UVERBS_UHW_OUT, 0, UA_FLAGS(UVERBS_ATTR_SPEC_F_MIN_SZ))); + +static void create_udata(struct uverbs_attr_array *ctx, size_t num, + struct ib_udata *udata) +{ + /* + * This is for ease of conversion. The purpose is to convert all drivers + * to use uverbs_attr_array instead of ib_udata. + * Assume attr == 0 is input and attr == 1 is output. + */ + void * __user inbuf; + size_t inbuf_len = 0; + void * __user outbuf; + size_t outbuf_len = 0; + + if (num >= UVERBS_UHW_NUM) { + struct uverbs_attr_array *driver = &ctx[UVERBS_UDATA_DRIVER_DATA_GROUP]; + + if (uverbs_is_valid(driver, UVERBS_UHW_IN)) { + inbuf = driver->attrs[UVERBS_UHW_IN].ptr_attr.ptr; + inbuf_len = driver->attrs[UVERBS_UHW_IN].ptr_attr.len; + } + + if (driver->num_attrs >= UVERBS_UHW_OUT && + uverbs_is_valid(driver, UVERBS_UHW_OUT)) { + outbuf = driver->attrs[UVERBS_UHW_OUT].ptr_attr.ptr; + outbuf_len = driver->attrs[UVERBS_UHW_OUT].ptr_attr.len; + } + } + INIT_UDATA_BUF_OR_NULL(udata, inbuf, outbuf, inbuf_len, outbuf_len); +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_get_context_spec, + UVERBS_ATTR_PTR_OUT(GET_CONTEXT_RESP, + struct ib_uverbs_get_context_resp)); + +int uverbs_get_context(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_udata uhw; + struct ib_uverbs_get_context_resp resp; + struct ib_ucontext *ucontext; + struct file *filp; + int ret; + + if (!uverbs_is_valid(common, GET_CONTEXT_RESP)) + return -EINVAL; + + /* Temporary, only until drivers get the new uverbs_attr_array */ + create_udata(ctx, num, &uhw); + + mutex_lock(&file->mutex); + + if (file->ucontext) { + ret = -EINVAL; + goto err; + } + + ucontext = ib_dev->alloc_ucontext(ib_dev, &uhw); + if (IS_ERR(ucontext)) { + ret = PTR_ERR(ucontext); + goto err; + } + + ucontext->device = ib_dev; + /* ufile is required when some objects are released */ + ucontext->ufile = file; + uverbs_initialize_ucontext(ucontext); + + rcu_read_lock(); + ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); + rcu_read_unlock(); + ucontext->closing = 0; + +#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING + ucontext->umem_tree = RB_ROOT; + init_rwsem(&ucontext->umem_rwsem); + ucontext->odp_mrs_count = 0; + INIT_LIST_HEAD(&ucontext->no_private_counters); + + if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING)) + ucontext->invalidate_range = NULL; + +#endif + + resp.num_comp_vectors = file->device->num_comp_vectors; + + ret = get_unused_fd_flags(O_CLOEXEC); + if (ret < 0) + goto err_free; + resp.async_fd = ret; + + filp = ib_uverbs_alloc_async_event_file(file, ib_dev); + if (IS_ERR(filp)) { + ret = PTR_ERR(filp); + goto err_fd; + } + + ret = uverbs_copy_to(common, GET_CONTEXT_RESP, &resp); + if (ret) + goto err_file; + + file->ucontext = ucontext; + ucontext->ufile = file; + + fd_install(resp.async_fd, filp); + + mutex_unlock(&file->mutex); + + return 0; + +err_file: + ib_uverbs_free_async_event_file(file); + fput(filp); + +err_fd: + put_unused_fd(resp.async_fd); + +err_free: + put_pid(ucontext->tgid); + ib_dev->dealloc_ucontext(ucontext); +err: + mutex_unlock(&file->mutex); + return ret; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_query_device_spec, + UVERBS_ATTR_PTR_OUT(QUERY_DEVICE_RESP, struct ib_uverbs_query_device_resp), + UVERBS_ATTR_PTR_OUT(QUERY_DEVICE_ODP, struct ib_uverbs_odp_caps), + UVERBS_ATTR_PTR_OUT(QUERY_DEVICE_TIMESTAMP_MASK, u64), + UVERBS_ATTR_PTR_OUT(QUERY_DEVICE_HCA_CORE_CLOCK, u64), + UVERBS_ATTR_PTR_OUT(QUERY_DEVICE_CAP_FLAGS, u64)); + +int uverbs_query_device_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_device_attr attr = {}; + struct ib_udata uhw; + int err; + + /* Temporary, only until drivers get the new uverbs_attr_array */ + create_udata(ctx, num, &uhw); + + err = ib_dev->query_device(ib_dev, &attr, &uhw); + if (err) + return err; + + if (uverbs_is_valid(common, QUERY_DEVICE_RESP)) { + struct ib_uverbs_query_device_resp resp = {}; + + uverbs_copy_query_dev_fields(ib_dev, &resp, &attr); + if (uverbs_copy_to(common, QUERY_DEVICE_RESP, &resp)) + return -EFAULT; + } + +#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING + if (uverbs_is_valid(common, QUERY_DEVICE_ODP)) { + struct ib_uverbs_odp_caps odp_caps; + + odp_caps.general_caps = attr.odp_caps.general_caps; + odp_caps.per_transport_caps.rc_odp_caps = + attr.odp_caps.per_transport_caps.rc_odp_caps; + odp_caps.per_transport_caps.uc_odp_caps = + attr.odp_caps.per_transport_caps.uc_odp_caps; + odp_caps.per_transport_caps.ud_odp_caps = + attr.odp_caps.per_transport_caps.ud_odp_caps; + + if (uverbs_copy_to(common, QUERY_DEVICE_ODP, &odp_caps)) + return -EFAULT; + } +#endif + if (uverbs_copy_to(common, QUERY_DEVICE_TIMESTAMP_MASK, + &attr.timestamp_mask) == -EFAULT) + return -EFAULT; + + if (uverbs_copy_to(common, QUERY_DEVICE_HCA_CORE_CLOCK, + &attr.hca_core_clock) == -EFAULT) + return -EFAULT; + + if (uverbs_copy_to(common, QUERY_DEVICE_CAP_FLAGS, + &attr.device_cap_flags) == -EFAULT) + return -EFAULT; + + return 0; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_query_port_spec, + UVERBS_ATTR_PTR_IN(QUERY_PORT_PORT_NUM, __u8), + UVERBS_ATTR_PTR_OUT(QUERY_PORT_RESP, struct ib_uverbs_query_port_resp, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_query_port_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_uverbs_query_port_resp resp = {}; + struct ib_port_attr attr; + u8 port_num; + int ret; + + ret = uverbs_copy_from(&port_num, common, QUERY_PORT_PORT_NUM); + if (ret) + return ret; + + ret = ib_query_port(ib_dev, port_num, &attr); + if (ret) + return ret; + + resp.state = attr.state; + resp.max_mtu = attr.max_mtu; + resp.active_mtu = attr.active_mtu; + resp.gid_tbl_len = attr.gid_tbl_len; + resp.port_cap_flags = attr.port_cap_flags; + resp.max_msg_sz = attr.max_msg_sz; + resp.bad_pkey_cntr = attr.bad_pkey_cntr; + resp.qkey_viol_cntr = attr.qkey_viol_cntr; + resp.pkey_tbl_len = attr.pkey_tbl_len; + resp.lid = attr.lid; + resp.sm_lid = attr.sm_lid; + resp.lmc = attr.lmc; + resp.max_vl_num = attr.max_vl_num; + resp.sm_sl = attr.sm_sl; + resp.subnet_timeout = attr.subnet_timeout; + resp.init_type_reply = attr.init_type_reply; + resp.active_width = attr.active_width; + resp.active_speed = attr.active_speed; + resp.phys_state = attr.phys_state; + resp.link_layer = rdma_port_get_link_layer(ib_dev, port_num); + + return uverbs_copy_to(common, QUERY_PORT_RESP, &resp); +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_alloc_pd_spec, + UVERBS_ATTR_IDR(ALLOC_PD_HANDLE, UVERBS_TYPE_PD, + UVERBS_ACCESS_NEW, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_alloc_pd_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_ucontext *ucontext = file->ucontext; + struct ib_udata uhw; + struct ib_uobject *uobject; + struct ib_pd *pd; + + /* Temporary, only until drivers get the new uverbs_attr_array */ + create_udata(ctx, num, &uhw); + + pd = ib_dev->alloc_pd(ib_dev, ucontext, &uhw); + if (IS_ERR(pd)) + return PTR_ERR(pd); + + uobject = common->attrs[ALLOC_PD_HANDLE].obj_attr.uobject; + pd->device = ib_dev; + pd->uobject = uobject; + pd->__internal_mr = NULL; + uobject->object = pd; + atomic_set(&pd->usecnt, 0); + + return 0; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_dealloc_pd_spec, + UVERBS_ATTR_IDR(DEALLOC_PD_HANDLE, UVERBS_TYPE_PD, + UVERBS_ACCESS_DESTROY, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_dealloc_pd_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + return 0; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_reg_mr_spec, + UVERBS_ATTR_IDR(REG_MR_HANDLE, UVERBS_TYPE_MR, UVERBS_ACCESS_NEW, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_IDR(REG_MR_PD_HANDLE, UVERBS_TYPE_PD, UVERBS_ACCESS_READ, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_IN(REG_MR_CMD, struct ib_uverbs_ioctl_reg_mr, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_OUT(REG_MR_RESP, struct ib_uverbs_ioctl_reg_mr_resp, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_reg_mr_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_uverbs_ioctl_reg_mr cmd; + struct ib_uverbs_ioctl_reg_mr_resp resp; + struct ib_udata uhw; + struct ib_uobject *uobject; + struct ib_pd *pd; + struct ib_mr *mr; + int ret; + + if (uverbs_copy_from(&cmd, common, REG_MR_CMD)) + return -EFAULT; + + if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) + return -EINVAL; + + ret = ib_check_mr_access(cmd.access_flags); + if (ret) + return ret; + + /* Temporary, only until drivers get the new uverbs_attr_array */ + create_udata(ctx, num, &uhw); + + uobject = common->attrs[REG_MR_HANDLE].obj_attr.uobject; + pd = common->attrs[REG_MR_PD_HANDLE].obj_attr.uobject->object; + + if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { + if (!(pd->device->attrs.device_cap_flags & + IB_DEVICE_ON_DEMAND_PAGING)) { + pr_debug("ODP support not available\n"); + return -EINVAL; + } + } + + mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, + cmd.access_flags, &uhw); + if (IS_ERR(mr)) + return PTR_ERR(mr); + + mr->device = pd->device; + mr->pd = pd; + mr->uobject = uobject; + atomic_inc(&pd->usecnt); + uobject->object = mr; + + resp.lkey = mr->lkey; + resp.rkey = mr->rkey; + + if (uverbs_copy_to(common, REG_MR_RESP, &resp)) { + ret = -EFAULT; + goto err; + } + + return 0; + +err: + ib_dereg_mr(mr); + return ret; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_rereg_mr_spec, + UVERBS_ATTR_IDR(REREG_MR_HANDLE, UVERBS_TYPE_MR, UVERBS_ACCESS_READ, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_IDR(REREG_MR_PD_HANDLE, UVERBS_TYPE_PD, UVERBS_ACCESS_READ, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_IN(REREG_MR_CMD, struct ib_uverbs_rereg_mr, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_OUT(REG_MR_RESP, struct ib_uverbs_rereg_mr_resp, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_rereg_mr_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + + struct ib_udata udata; + struct ib_uverbs_rereg_mr cmd; + struct ib_uverbs_rereg_mr_resp resp; + struct ib_mr *mr; + struct ib_pd *pd = NULL; + struct ib_pd *old_pd; + int ret; + + if (uverbs_copy_from(&cmd, common, REREG_MR_CMD)) + return -EFAULT; + + if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) + return -EINVAL; + + if ((cmd.flags & IB_MR_REREG_TRANS) && + (!cmd.start || !cmd.hca_va || 0 >= cmd.length || + (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) + return -EINVAL; + + INIT_UDATA(&udata, &cmd, &resp, + sizeof(struct ib_uverbs_rereg_mr), + sizeof(struct ib_uverbs_rereg_mr_resp)); + + mr = common->attrs[REREG_MR_HANDLE].obj_attr.uobject->object; + pd = common->attrs[REREG_MR_PD_HANDLE].obj_attr.uobject->object; + + if (cmd.flags & IB_MR_REREG_ACCESS) { + ret = ib_check_mr_access(cmd.access_flags); + if (ret) + return -EFAULT; + } + + old_pd = mr->pd; + ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start, + cmd.length, cmd.hca_va, + cmd.access_flags, pd, &udata); + if (!ret) { + if (cmd.flags & IB_MR_REREG_PD) { + atomic_inc(&pd->usecnt); + mr->pd = pd; + atomic_dec(&old_pd->usecnt); + } + } else + return -EFAULT; + + memset(&resp, 0, sizeof(resp)); + resp.lkey = mr->lkey; + resp.rkey = mr->rkey; + + if (uverbs_copy_to(common, REREG_MR_RESP, &resp)) + return -EFAULT; + + return 0; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_dereg_mr_spec, + UVERBS_ATTR_IDR(DEREG_MR_HANDLE, UVERBS_TYPE_MR, UVERBS_ACCESS_DESTROY, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_dereg_mr_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + return 0; +}; + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_create_comp_channel_spec, + UVERBS_ATTR_FD(CREATE_COMP_CHANNEL_FD, UVERBS_TYPE_COMP_CHANNEL, + UVERBS_ACCESS_NEW, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_create_comp_channel_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_uverbs_completion_event_file *ev_file; + struct ib_uobject *uobj = + common->attrs[CREATE_COMP_CHANNEL_FD].obj_attr.uobject; + + kref_get(&uobj->ref); + ev_file = container_of(uobj, + struct ib_uverbs_completion_event_file, + uobj_file.uobj); + ib_uverbs_init_event_file(&ev_file->ev_file); + + return 0; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_create_cq_spec, + UVERBS_ATTR_IDR(CREATE_CQ_HANDLE, UVERBS_TYPE_CQ, UVERBS_ACCESS_NEW, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_IN(CREATE_CQ_CQE, u32, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_IN(CREATE_CQ_USER_HANDLE, u64), + UVERBS_ATTR_FD(CREATE_CQ_COMP_CHANNEL, UVERBS_TYPE_COMP_CHANNEL, UVERBS_ACCESS_READ), + /* + * Currently, COMP_VECTOR is mandatory, but that could be lifted in the + * future. + */ + UVERBS_ATTR_PTR_IN(CREATE_CQ_COMP_VECTOR, u32, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_IN(CREATE_CQ_FLAGS, u32), + UVERBS_ATTR_PTR_OUT(CREATE_CQ_RESP_CQE, u32, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_create_cq_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_ucontext *ucontext = file->ucontext; + struct ib_ucq_object *obj; + struct ib_udata uhw; + int ret; + u64 user_handle = 0; + struct ib_cq_init_attr attr = {}; + struct ib_cq *cq; + struct ib_uverbs_completion_event_file *ev_file = NULL; + + ret = uverbs_copy_from(&attr.comp_vector, common, CREATE_CQ_COMP_VECTOR); + if (!ret) + ret = uverbs_copy_from(&attr.cqe, common, CREATE_CQ_CQE); + if (ret) + return ret; + + /* Optional params, if they don't exist, we get -ENOENT and skip them */ + if (uverbs_copy_from(&attr.flags, common, CREATE_CQ_FLAGS) == -EFAULT || + uverbs_copy_from(&user_handle, common, CREATE_CQ_USER_HANDLE) == -EFAULT) + return -EFAULT; + + if (uverbs_is_valid(common, CREATE_CQ_COMP_CHANNEL)) { + struct ib_uobject *ev_file_uobj = + common->attrs[CREATE_CQ_COMP_CHANNEL].obj_attr.uobject; + + ev_file = container_of(ev_file_uobj, + struct ib_uverbs_completion_event_file, + uobj_file.uobj); + kref_get(&ev_file_uobj->ref); + } + + if (attr.comp_vector >= ucontext->ufile->device->num_comp_vectors) + return -EINVAL; + + obj = container_of(common->attrs[CREATE_CQ_HANDLE].obj_attr.uobject, + typeof(*obj), uobject); + obj->uverbs_file = ucontext->ufile; + obj->comp_events_reported = 0; + obj->async_events_reported = 0; + INIT_LIST_HEAD(&obj->comp_list); + INIT_LIST_HEAD(&obj->async_list); + + /* Temporary, only until drivers get the new uverbs_attr_array */ + create_udata(ctx, num, &uhw); + + cq = ib_dev->create_cq(ib_dev, &attr, ucontext, &uhw); + if (IS_ERR(cq)) + return PTR_ERR(cq); + + cq->device = ib_dev; + cq->uobject = &obj->uobject; + cq->comp_handler = ib_uverbs_comp_handler; + cq->event_handler = ib_uverbs_cq_event_handler; + cq->cq_context = &ev_file->ev_file; + obj->uobject.object = cq; + obj->uobject.user_handle = user_handle; + atomic_set(&cq->usecnt, 0); + + ret = uverbs_copy_to(common, CREATE_CQ_RESP_CQE, &cq->cqe); + if (ret) + goto err; + + return 0; +err: + ib_destroy_cq(cq); + return ret; +}; + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_destroy_cq_spec, + UVERBS_ATTR_IDR(DESTROY_CQ_HANDLE, UVERBS_TYPE_CQ, + UVERBS_ACCESS_DESTROY, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_OUT(DESTROY_CQ_RESP, struct ib_uverbs_destroy_cq_resp, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_destroy_cq_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_uverbs_destroy_cq_resp resp; + struct ib_uobject *uobj = + common->attrs[DESTROY_CQ_HANDLE].obj_attr.uobject; + struct ib_ucq_object *obj = container_of(uobj, struct ib_ucq_object, + uobject); + + resp.comp_events_reported = obj->comp_events_reported; + resp.async_events_reported = obj->async_events_reported; + + WARN_ON(uverbs_copy_to(common, DESTROY_CQ_RESP, &resp)); + return 0; +} + +static int qp_fill_attrs(struct ib_qp_init_attr *attr, struct ib_ucontext *ctx, + const struct ib_uverbs_ioctl_create_qp *cmd, + u32 create_flags) +{ + if (create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | + IB_QP_CREATE_CROSS_CHANNEL | + IB_QP_CREATE_MANAGED_SEND | + IB_QP_CREATE_MANAGED_RECV | + IB_QP_CREATE_SCATTER_FCS)) + return -EINVAL; + + attr->create_flags = create_flags; + attr->event_handler = ib_uverbs_qp_event_handler; + attr->qp_context = ctx->ufile; + attr->sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : + IB_SIGNAL_REQ_WR; + attr->qp_type = cmd->qp_type; + + attr->cap.max_send_wr = cmd->max_send_wr; + attr->cap.max_recv_wr = cmd->max_recv_wr; + attr->cap.max_send_sge = cmd->max_send_sge; + attr->cap.max_recv_sge = cmd->max_recv_sge; + attr->cap.max_inline_data = cmd->max_inline_data; + + return 0; +} + +static void qp_init_uqp(struct ib_uqp_object *obj) +{ + obj->uevent.events_reported = 0; + INIT_LIST_HEAD(&obj->uevent.event_list); + INIT_LIST_HEAD(&obj->mcast_list); +} + +static int qp_write_resp(const struct ib_qp_init_attr *attr, + const struct ib_qp *qp, + struct uverbs_attr_array *common) +{ + struct ib_uverbs_ioctl_create_qp_resp resp = { + .qpn = qp->qp_num, + .max_recv_sge = attr->cap.max_recv_sge, + .max_send_sge = attr->cap.max_send_sge, + .max_recv_wr = attr->cap.max_recv_wr, + .max_send_wr = attr->cap.max_send_wr, + .max_inline_data = attr->cap.max_inline_data}; + + return uverbs_copy_to(common, CREATE_QP_RESP, &resp); +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_create_qp_spec, + UVERBS_ATTR_IDR(CREATE_QP_HANDLE, UVERBS_TYPE_QP, UVERBS_ACCESS_NEW, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_IDR(CREATE_QP_PD_HANDLE, UVERBS_TYPE_PD, UVERBS_ACCESS_READ, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_IDR(CREATE_QP_SEND_CQ, UVERBS_TYPE_CQ, UVERBS_ACCESS_READ, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_IDR(CREATE_QP_RECV_CQ, UVERBS_TYPE_CQ, UVERBS_ACCESS_READ, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_IDR(CREATE_QP_SRQ, UVERBS_TYPE_SRQ, UVERBS_ACCESS_READ), + UVERBS_ATTR_PTR_IN(CREATE_QP_USER_HANDLE, u64), + UVERBS_ATTR_PTR_IN(CREATE_QP_CMD, struct ib_uverbs_ioctl_create_qp), + UVERBS_ATTR_PTR_IN(CREATE_QP_CMD_FLAGS, u32), + UVERBS_ATTR_PTR_OUT(CREATE_QP_RESP, struct ib_uverbs_ioctl_create_qp_resp, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_create_qp_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_ucontext *ucontext = file->ucontext; + struct ib_uqp_object *obj; + struct ib_udata uhw; + int ret; + u64 user_handle = 0; + u32 create_flags = 0; + struct ib_uverbs_ioctl_create_qp cmd; + struct ib_qp_init_attr attr = {}; + struct ib_qp *qp; + struct ib_pd *pd; + + ret = uverbs_copy_from(&cmd, common, CREATE_QP_CMD); + if (ret) + return ret; + + /* Optional params */ + if (uverbs_copy_from(&create_flags, common, CREATE_QP_CMD_FLAGS) == -EFAULT || + uverbs_copy_from(&user_handle, common, CREATE_QP_USER_HANDLE) == -EFAULT) + return -EFAULT; + + if (cmd.qp_type == IB_QPT_XRC_INI) { + cmd.max_recv_wr = 0; + cmd.max_recv_sge = 0; + } + + ret = qp_fill_attrs(&attr, ucontext, &cmd, create_flags); + if (ret) + return ret; + + pd = common->attrs[CREATE_QP_PD_HANDLE].obj_attr.uobject->object; + attr.send_cq = common->attrs[CREATE_QP_SEND_CQ].obj_attr.uobject->object; + attr.recv_cq = common->attrs[CREATE_QP_RECV_CQ].obj_attr.uobject->object; + if (uverbs_is_valid(common, CREATE_QP_SRQ)) + attr.srq = common->attrs[CREATE_QP_SRQ].obj_attr.uobject->object; + obj = (struct ib_uqp_object *)common->attrs[CREATE_QP_HANDLE].obj_attr.uobject; + + obj->uxrcd = NULL; + if (attr.srq && attr.srq->srq_type != IB_SRQT_BASIC) + return -EINVAL; + + qp_init_uqp(obj); + create_udata(ctx, num, &uhw); + qp = pd->device->create_qp(pd, &attr, &uhw); + if (IS_ERR(qp)) + return PTR_ERR(qp); + qp->real_qp = qp; + qp->device = pd->device; + qp->pd = pd; + qp->send_cq = attr.send_cq; + qp->recv_cq = attr.recv_cq; + qp->srq = attr.srq; + qp->event_handler = attr.event_handler; + qp->qp_context = attr.qp_context; + qp->qp_type = attr.qp_type; + atomic_set(&qp->usecnt, 0); + atomic_inc(&pd->usecnt); + atomic_inc(&attr.send_cq->usecnt); + if (attr.recv_cq) + atomic_inc(&attr.recv_cq->usecnt); + if (attr.srq) + atomic_inc(&attr.srq->usecnt); + qp->uobject = &obj->uevent.uobject; + obj->uevent.uobject.object = qp; + obj->uevent.uobject.user_handle = user_handle; + + ret = qp_write_resp(&attr, qp, common); + if (ret) { + ib_destroy_qp(qp); + return ret; + } + + return 0; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_create_qp_xrc_tgt_spec, + UVERBS_ATTR_IDR(CREATE_QP_XRC_TGT_HANDLE, UVERBS_TYPE_QP, UVERBS_ACCESS_NEW, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_IDR(CREATE_QP_XRC_TGT_XRCD, UVERBS_TYPE_XRCD, UVERBS_ACCESS_READ, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_IN(CREATE_QP_XRC_TGT_USER_HANDLE, u64), + UVERBS_ATTR_PTR_IN(CREATE_QP_XRC_TGT_CMD, struct ib_uverbs_ioctl_create_qp), + UVERBS_ATTR_PTR_IN(CREATE_QP_XRC_TGT_CMD_FLAGS, u32), + UVERBS_ATTR_PTR_OUT(CREATE_QP_XRC_TGT_RESP, struct ib_uverbs_ioctl_create_qp_resp, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_create_qp_xrc_tgt_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_ucontext *ucontext = file->ucontext; + struct ib_uqp_object *obj; + int ret; + u64 user_handle = 0; + u32 create_flags = 0; + struct ib_uverbs_ioctl_create_qp cmd; + struct ib_qp_init_attr attr = {}; + struct ib_qp *qp; + + ret = uverbs_copy_from(&cmd, common, CREATE_QP_XRC_TGT_CMD); + if (ret) + return ret; + + /* Optional params */ + if (uverbs_copy_from(&create_flags, common, CREATE_QP_CMD_FLAGS) == -EFAULT || + uverbs_copy_from(&user_handle, common, CREATE_QP_USER_HANDLE) == -EFAULT) + return -EFAULT; + + ret = qp_fill_attrs(&attr, ucontext, &cmd, create_flags); + if (ret) + return ret; + + obj = (struct ib_uqp_object *)common->attrs[CREATE_QP_HANDLE].obj_attr.uobject; + obj->uxrcd = container_of(common->attrs[CREATE_QP_XRC_TGT_XRCD].obj_attr.uobject, + struct ib_uxrcd_object, uobject); + attr.xrcd = obj->uxrcd->uobject.object; + + qp_init_uqp(obj); + qp = ib_create_qp(NULL, &attr); + if (IS_ERR(qp)) + return PTR_ERR(qp); + qp->uobject = &obj->uevent.uobject; + obj->uevent.uobject.object = qp; + obj->uevent.uobject.user_handle = user_handle; + atomic_inc(&obj->uxrcd->refcnt); + + ret = qp_write_resp(&attr, qp, common); + if (ret) { + ib_destroy_qp(qp); + return ret; + } + + return 0; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_destroy_qp_spec, + UVERBS_ATTR_IDR(DESTROY_QP_HANDLE, UVERBS_TYPE_QP, + UVERBS_ACCESS_DESTROY, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_OUT(DESTROY_QP_EVENTS_REPORTED, + __u32, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_destroy_qp_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_uobject *uobj = + common->attrs[DESTROY_QP_HANDLE].obj_attr.uobject; + struct ib_uqp_object *obj = container_of(uobj, struct ib_uqp_object, + uevent.uobject); + + WARN_ON(uverbs_copy_to(common, DESTROY_QP_EVENTS_REPORTED, + &obj->uevent.events_reported)); + return 0; +} + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_modify_qp_spec, + UVERBS_ATTR_IDR(MODIFY_QP_HANDLE, UVERBS_TYPE_QP, UVERBS_ACCESS_WRITE, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_IN(MODIFY_QP_STATE, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_CUR_STATE, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_EN_SQD_ASYNC_NOTIFY, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_ACCESS_FLAGS, u32), + UVERBS_ATTR_PTR_IN(MODIFY_QP_PKEY_INDEX, u16), + UVERBS_ATTR_PTR_IN(MODIFY_QP_PORT, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_QKEY, u32), + UVERBS_ATTR_PTR_IN(MODIFY_QP_AV, struct ib_uverbs_qp_dest), + UVERBS_ATTR_PTR_IN(MODIFY_QP_PATH_MTU, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_TIMEOUT, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_RETRY_CNT, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_RNR_RETRY, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_RQ_PSN, u32), + UVERBS_ATTR_PTR_IN(MODIFY_QP_MAX_RD_ATOMIC, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_ALT_PATH, struct ib_uverbs_qp_alt_path), + UVERBS_ATTR_PTR_IN(MODIFY_QP_MIN_RNR_TIMER, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_SQ_PSN, u32), + UVERBS_ATTR_PTR_IN(MODIFY_QP_MAX_DEST_RD_ATOMIC, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_PATH_MIG_STATE, u8), + UVERBS_ATTR_PTR_IN(MODIFY_QP_DEST_QPN, u32), + UVERBS_ATTR_PTR_IN(MODIFY_QP_RATE_LIMIT, u32)); + +int uverbs_modify_qp_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_udata uhw; + struct ib_qp *qp; + struct ib_qp_attr *attr; + struct ib_uverbs_qp_dest av; + struct ib_uverbs_qp_alt_path alt_path; + u32 attr_mask = 0; + int ret = 0; + + qp = common->attrs[MODIFY_QP_HANDLE].obj_attr.uobject->object; + attr = kzalloc(sizeof(*attr), GFP_KERNEL); + if (!attr) + return -ENOMEM; + +#define MODIFY_QP_CPY(_param, _fld, _attr) \ + ({ \ + int ret = uverbs_copy_from(_fld, common, _param); \ + if (!ret) \ + attr_mask |= _attr; \ + ret == -EFAULT ? ret : 0; \ + }) + + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_STATE, &attr->qp_state, + IB_QP_STATE); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_CUR_STATE, &attr->cur_qp_state, + IB_QP_CUR_STATE); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_EN_SQD_ASYNC_NOTIFY, + &attr->en_sqd_async_notify, + IB_QP_EN_SQD_ASYNC_NOTIFY); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_ACCESS_FLAGS, + &attr->qp_access_flags, IB_QP_ACCESS_FLAGS); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_PKEY_INDEX, &attr->pkey_index, + IB_QP_PKEY_INDEX); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_PORT, &attr->port_num, IB_QP_PORT); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_QKEY, &attr->qkey, IB_QP_QKEY); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_PATH_MTU, &attr->path_mtu, + IB_QP_PATH_MTU); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_TIMEOUT, &attr->timeout, + IB_QP_TIMEOUT); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_RETRY_CNT, &attr->retry_cnt, + IB_QP_RETRY_CNT); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_RNR_RETRY, &attr->rnr_retry, + IB_QP_RNR_RETRY); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_RQ_PSN, &attr->rq_psn, + IB_QP_RQ_PSN); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_MAX_RD_ATOMIC, + &attr->max_rd_atomic, + IB_QP_MAX_QP_RD_ATOMIC); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_MIN_RNR_TIMER, + &attr->min_rnr_timer, IB_QP_MIN_RNR_TIMER); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_SQ_PSN, &attr->sq_psn, + IB_QP_SQ_PSN); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_MAX_DEST_RD_ATOMIC, + &attr->max_dest_rd_atomic, + IB_QP_MAX_DEST_RD_ATOMIC); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_PATH_MIG_STATE, + &attr->path_mig_state, IB_QP_PATH_MIG_STATE); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_DEST_QPN, &attr->dest_qp_num, + IB_QP_DEST_QPN); + ret = ret ?: MODIFY_QP_CPY(MODIFY_QP_RATE_LIMIT, &attr->rate_limit, + IB_QP_RATE_LIMIT); + + if (ret) + goto err; + + ret = uverbs_copy_from(&av, common, MODIFY_QP_AV); + if (!ret) { + attr_mask |= IB_QP_AV; + memcpy(attr->ah_attr.grh.dgid.raw, av.dgid, 16); + attr->ah_attr.grh.flow_label = av.flow_label; + attr->ah_attr.grh.sgid_index = av.sgid_index; + attr->ah_attr.grh.hop_limit = av.hop_limit; + attr->ah_attr.grh.traffic_class = av.traffic_class; + attr->ah_attr.dlid = av.dlid; + attr->ah_attr.sl = av.sl; + attr->ah_attr.src_path_bits = av.src_path_bits; + attr->ah_attr.static_rate = av.static_rate; + attr->ah_attr.ah_flags = av.is_global ? IB_AH_GRH : 0; + attr->ah_attr.port_num = av.port_num; + } else if (ret == -EFAULT) { + goto err; + } + + ret = uverbs_copy_from(&alt_path, common, MODIFY_QP_ALT_PATH); + if (!ret) { + attr_mask |= IB_QP_ALT_PATH; + memcpy(attr->alt_ah_attr.grh.dgid.raw, alt_path.dest.dgid, 16); + attr->alt_ah_attr.grh.flow_label = alt_path.dest.flow_label; + attr->alt_ah_attr.grh.sgid_index = alt_path.dest.sgid_index; + attr->alt_ah_attr.grh.hop_limit = alt_path.dest.hop_limit; + attr->alt_ah_attr.grh.traffic_class = alt_path.dest.traffic_class; + attr->alt_ah_attr.dlid = alt_path.dest.dlid; + attr->alt_ah_attr.sl = alt_path.dest.sl; + attr->alt_ah_attr.src_path_bits = alt_path.dest.src_path_bits; + attr->alt_ah_attr.static_rate = alt_path.dest.static_rate; + attr->alt_ah_attr.ah_flags = alt_path.dest.is_global ? IB_AH_GRH : 0; + attr->alt_ah_attr.port_num = alt_path.dest.port_num; + attr->alt_pkey_index = alt_path.pkey_index; + attr->alt_port_num = alt_path.port_num; + attr->alt_timeout = alt_path.timeout; + } else if (ret == -EFAULT) { + goto err; + } + + create_udata(ctx, num, &uhw); + + if (qp->real_qp == qp) { + if (attr_mask & IB_QP_AV) { + ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr); + if (ret) + goto err; + } + ret = qp->device->modify_qp(qp, attr, + modify_qp_mask(qp->qp_type, + attr_mask), + &uhw); + } else { + ret = ib_modify_qp(qp, attr, modify_qp_mask(qp->qp_type, + attr_mask)); + } + + if (ret) + goto err; + + return 0; +err: + kfree(attr); + return ret; +} + + +DECLARE_UVERBS_ATTR_SPEC( + uverbs_query_qp_spec, + UVERBS_ATTR_IDR(QUERY_QP_HANDLE, + UVERBS_TYPE_QP, + UVERBS_ACCESS_READ, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)), + UVERBS_ATTR_PTR_IN(QUERY_QP_ATTR_MASK, u32), + UVERBS_ATTR_PTR_OUT(QUERY_QP_RESP, + struct ib_uverbs_query_qp_resp, + UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY))); + +int uverbs_query_qp_handler(struct ib_device *ib_dev, + struct ib_uverbs_file *file, + struct uverbs_attr_array *ctx, size_t num) +{ + struct uverbs_attr_array *common = &ctx[0]; + struct ib_qp *qp; + u32 attr_mask = 0; + int ret; + + struct ib_qp_attr *attr; + struct ib_qp_init_attr *init_attr; + struct ib_uverbs_query_qp_resp resp; + + printk("--->%s():%d\n", __func__, __LINE__); + + attr = kmalloc(sizeof *attr, GFP_KERNEL); + init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); + if (!attr || !init_attr) + return -ENOMEM; + + uverbs_copy_from(&attr_mask, common, QUERY_QP_ATTR_MASK); + qp = common->attrs[QUERY_QP_HANDLE].obj_attr.uobject->object; + + ret = ib_query_qp(qp, attr, attr_mask, init_attr); + memset(&resp, 0, sizeof resp); + + resp.qp_state = attr->qp_state; + resp.cur_qp_state = attr->cur_qp_state; + resp.path_mtu = attr->path_mtu; + resp.path_mig_state = attr->path_mig_state; + resp.qkey = attr->qkey; + resp.rq_psn = attr->rq_psn; + resp.sq_psn = attr->sq_psn; + resp.dest_qp_num = attr->dest_qp_num; + resp.qp_access_flags = attr->qp_access_flags; + resp.pkey_index = attr->pkey_index; + resp.alt_pkey_index = attr->alt_pkey_index; + resp.sq_draining = attr->sq_draining; + resp.max_rd_atomic = attr->max_rd_atomic; + resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; + resp.min_rnr_timer = attr->min_rnr_timer; + resp.port_num = attr->port_num; + resp.timeout = attr->timeout; + resp.retry_cnt = attr->retry_cnt; + resp.rnr_retry = attr->rnr_retry; + resp.alt_port_num = attr->alt_port_num; + resp.alt_timeout = attr->alt_timeout; + + memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); + resp.dest.flow_label = attr->ah_attr.grh.flow_label; + resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; + resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; + resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; + resp.dest.dlid = attr->ah_attr.dlid; + resp.dest.sl = attr->ah_attr.sl; + resp.dest.src_path_bits = attr->ah_attr.src_path_bits; + resp.dest.static_rate = attr->ah_attr.static_rate; + resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); + resp.dest.port_num = attr->ah_attr.port_num; + + memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); + resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; + resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; + resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; + resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; + resp.alt_dest.dlid = attr->alt_ah_attr.dlid; + resp.alt_dest.sl = attr->alt_ah_attr.sl; + resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; + resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; + resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); + resp.alt_dest.port_num = attr->alt_ah_attr.port_num; + + resp.max_send_wr = init_attr->cap.max_send_wr; + resp.max_recv_wr = init_attr->cap.max_recv_wr; + resp.max_send_sge = init_attr->cap.max_send_sge; + resp.max_recv_sge = init_attr->cap.max_recv_sge; + resp.max_inline_data = init_attr->cap.max_inline_data; + resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; + + + if (uverbs_copy_to(common, QUERY_QP_RESP, &resp)) + return -EFAULT; + + return 0; +} + +DECLARE_UVERBS_TYPE(uverbs_type_comp_channel, + &UVERBS_TYPE_ALLOC_FD(0, sizeof(struct ib_uverbs_completion_event_file), + uverbs_hot_unplug_completion_event_file, + &uverbs_event_fops, + "[infinibandevent]", O_RDONLY), + &UVERBS_ACTIONS( + ADD_UVERBS_ACTION(UVERBS_COMP_CHANNEL_CREATE, + uverbs_create_comp_channel_handler, + &uverbs_create_comp_channel_spec))); + +DECLARE_UVERBS_TYPE(uverbs_type_cq, + &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), 0, + uverbs_free_cq), + &UVERBS_ACTIONS( + ADD_UVERBS_ACTION(UVERBS_CQ_CREATE, + uverbs_create_cq_handler, + &uverbs_create_cq_spec, + &uverbs_uhw_compat_spec), + ADD_UVERBS_ACTION(UVERBS_CQ_DESTROY, + uverbs_destroy_cq_handler, + &uverbs_destroy_cq_spec))); + +DECLARE_UVERBS_TYPE(uverbs_type_qp, + &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), 0, + uverbs_free_qp), + &UVERBS_ACTIONS( + ADD_UVERBS_ACTION(UVERBS_QP_CREATE, + uverbs_create_qp_handler, + &uverbs_create_qp_spec, + &uverbs_uhw_compat_spec), + ADD_UVERBS_ACTION(UVERBS_QP_CREATE_XRC_TGT, + uverbs_create_qp_xrc_tgt_handler, + &uverbs_create_qp_xrc_tgt_spec), + ADD_UVERBS_ACTION(UVERBS_QP_MODIFY, + uverbs_modify_qp_handler, + &uverbs_modify_qp_spec, + &uverbs_uhw_compat_spec), + ADD_UVERBS_ACTION(UVERBS_QP_DESTROY, + uverbs_destroy_qp_handler, + &uverbs_destroy_qp_spec), + ADD_UVERBS_ACTION(UVERBS_QP_QUERY, + uverbs_query_qp_handler, + &uverbs_query_qp_spec))); + +DECLARE_UVERBS_TYPE(uverbs_type_mw, + &UVERBS_TYPE_ALLOC_IDR(0, uverbs_free_mw)); + +DECLARE_UVERBS_TYPE(uverbs_type_mr, + /* 1 is used in order to free the MR after all the MWs */ + &UVERBS_TYPE_ALLOC_IDR(1, uverbs_free_mr), + &UVERBS_ACTIONS( + ADD_UVERBS_ACTION(UVERBS_MR_REG, uverbs_reg_mr_handler, + &uverbs_reg_mr_spec, + &uverbs_uhw_compat_spec), + ADD_UVERBS_ACTION(UVERBS_MR_DEREG, + uverbs_dereg_mr_handler, + &uverbs_dereg_mr_spec), + ADD_UVERBS_ACTION(UVERBS_MR_REREG, + uverbs_rereg_mr_handler, + &uverbs_rereg_mr_spec))); + +DECLARE_UVERBS_TYPE(uverbs_type_srq, + &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object), 0, + uverbs_free_srq)); + +DECLARE_UVERBS_TYPE(uverbs_type_ah, + &UVERBS_TYPE_ALLOC_IDR(0, uverbs_free_ah)); + +DECLARE_UVERBS_TYPE(uverbs_type_flow, + &UVERBS_TYPE_ALLOC_IDR(0, uverbs_free_flow)); + +DECLARE_UVERBS_TYPE(uverbs_type_wq, + &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), 0, + uverbs_free_wq)); + +DECLARE_UVERBS_TYPE(uverbs_type_rwq_ind_table, + &UVERBS_TYPE_ALLOC_IDR(0, uverbs_free_rwq_ind_tbl)); + +DECLARE_UVERBS_TYPE(uverbs_type_xrcd, + &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uxrcd_object), 0, + uverbs_free_xrcd)); + +DECLARE_UVERBS_TYPE(uverbs_type_pd, + /* 2 is used in order to free the PD after MRs */ + &UVERBS_TYPE_ALLOC_IDR(2, uverbs_free_pd), + &UVERBS_ACTIONS( + ADD_UVERBS_ACTION(UVERBS_PD_ALLOC, + uverbs_alloc_pd_handler, + &uverbs_alloc_pd_spec, + &uverbs_uhw_compat_spec), + ADD_UVERBS_ACTION(UVERBS_PD_DEALLOC, + uverbs_dealloc_pd_handler, + &uverbs_dealloc_pd_spec))); + +DECLARE_UVERBS_TYPE(uverbs_type_device, NULL, + &UVERBS_ACTIONS( + ADD_UVERBS_CTX_ACTION(UVERBS_DEVICE_ALLOC_CONTEXT, + uverbs_get_context, + &uverbs_get_context_spec, + &uverbs_uhw_compat_spec), + ADD_UVERBS_ACTION(UVERBS_DEVICE_QUERY, + &uverbs_query_device_handler, + &uverbs_query_device_spec, + &uverbs_uhw_compat_spec), + ADD_UVERBS_ACTION(UVERBS_DEVICE_PORT_QUERY, + &uverbs_query_port_handler, + &uverbs_query_port_spec))); + +DECLARE_UVERBS_TYPES(uverbs_common_types, + ADD_UVERBS_TYPE(UVERBS_TYPE_DEVICE, uverbs_type_device), + ADD_UVERBS_TYPE(UVERBS_TYPE_PD, uverbs_type_pd), + ADD_UVERBS_TYPE(UVERBS_TYPE_MR, uverbs_type_mr), + ADD_UVERBS_TYPE(UVERBS_TYPE_COMP_CHANNEL, uverbs_type_comp_channel), + ADD_UVERBS_TYPE(UVERBS_TYPE_CQ, uverbs_type_cq), + ADD_UVERBS_TYPE(UVERBS_TYPE_QP, uverbs_type_qp), + ADD_UVERBS_TYPE(UVERBS_TYPE_AH, uverbs_type_ah), + ADD_UVERBS_TYPE(UVERBS_TYPE_MW, uverbs_type_mw), + ADD_UVERBS_TYPE(UVERBS_TYPE_SRQ, uverbs_type_srq), + ADD_UVERBS_TYPE(UVERBS_TYPE_FLOW, uverbs_type_flow), + ADD_UVERBS_TYPE(UVERBS_TYPE_WQ, uverbs_type_wq), + ADD_UVERBS_TYPE(UVERBS_TYPE_RWQ_IND_TBL, + uverbs_type_rwq_ind_table), + ADD_UVERBS_TYPE(UVERBS_TYPE_XRCD, uverbs_type_xrcd), +); +EXPORT_SYMBOL(uverbs_common_types); diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 9d5fe1853da46e..76beb7a7468149 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -53,6 +53,8 @@ #include #include #include +#include +#include #include "cxio_hal.h" #include "iwch.h" @@ -1361,6 +1363,8 @@ static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str, snprintf(str, str_len, "%s", info.fw_version); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + int iwch_register_device(struct iwch_dev *dev) { int ret; @@ -1454,6 +1458,7 @@ int iwch_register_device(struct iwch_dev *dev) memcpy(dev->ibdev.iwcm->ifname, dev->rdev.t3cdev_p->lldev->name, sizeof(dev->ibdev.iwcm->ifname)); + dev->ibdev.specs_root = (struct uverbs_root *)&root; ret = ib_register_device(&dev->ibdev, NULL); if (ret) goto bail1; diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 49b51b7e0fd786..0351d8622419a2 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -51,6 +51,8 @@ #include #include #include +#include +#include #include "iw_cxgb4.h" @@ -530,6 +532,8 @@ static void get_dev_fw_str(struct ib_device *dev, char *str, FW_HDR_FW_VER_BUILD_G(c4iw_dev->rdev.lldi.fw_vers)); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + int c4iw_register_device(struct c4iw_dev *dev) { int ret; @@ -625,6 +629,7 @@ int c4iw_register_device(struct c4iw_dev *dev) memcpy(dev->ibdev.iwcm->ifname, dev->rdev.lldi.ports[0]->name, sizeof(dev->ibdev.iwcm->ifname)); + dev->ibdev.specs_root = (struct uverbs_root *)&root; ret = ib_register_device(&dev->ibdev, NULL); if (ret) goto bail1; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 4953d9cb83a7ff..61133103b5a2b9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include "hns_roce_common.h" #include "hns_roce_device.h" #include @@ -422,6 +424,8 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) ib_unregister_device(&hr_dev->ib_dev); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + static int hns_roce_register_device(struct hns_roce_dev *hr_dev) { int ret; @@ -505,6 +509,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) /* OTHERS */ ib_dev->get_port_immutable = hns_roce_port_immutable; + dev->ib_dev.specs_root = (struct uverbs_root *)&root; ret = ib_register_device(ib_dev, NULL); if (ret) { dev_err(dev, "ib_register_device failed!\n"); diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 29e97df9e1a7f8..62d6e0ab74ee8a 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -43,6 +43,8 @@ #include #include #include +#include +#include #include #include "i40iw.h" @@ -2869,6 +2871,8 @@ void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev) ib_dealloc_device(&iwibdev->ibdev); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + /** * i40iw_register_rdma_device - register iwarp device to IB * @iwdev: iwarp device @@ -2883,6 +2887,7 @@ int i40iw_register_rdma_device(struct i40iw_device *iwdev) return -ENOMEM; iwibdev = iwdev->iwibdev; + iwibdev->ibdev.specs_root = (struct uverbs_root *)&root; ret = ib_register_device(&iwibdev->ibdev, NULL); if (ret) goto error; diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 7031a8dd4d1404..c3e126fd2a506c 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -45,6 +45,8 @@ #include #include +#include +#include #include #include @@ -2570,6 +2572,8 @@ static void get_fw_ver_str(struct ib_device *device, char *str, (int) dev->dev->caps.fw_ver & 0xffff); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + static void *mlx4_ib_add(struct mlx4_dev *dev) { struct mlx4_ib_dev *ibdev; @@ -2852,6 +2856,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) if (mlx4_ib_alloc_diag_counters(ibdev)) goto err_steer_free_bitmap; + ibdev->ib_dev.specs_root = (struct uverbs_root *)&root; if (ib_register_device(&ibdev->ib_dev, NULL)) goto err_diag_counters; diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index d566f673883348..a1af6dc67f316a 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -54,6 +54,8 @@ #include #include #include "mlx5_ib.h" +#include +#include #define DRIVER_NAME "mlx5_ib" #define DRIVER_VERSION "2.2-1" @@ -3027,6 +3029,8 @@ static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, return ARRAY_SIZE(names); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + static void *mlx5_ib_add(struct mlx5_core_dev *mdev) { struct mlx5_ib_dev *dev; @@ -3237,6 +3241,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev) if (err) goto err_odp; + dev->ib_dev.specs_root = (struct uverbs_root *)&root; err = ib_register_device(&dev->ib_dev, NULL); if (err) goto err_q_cnt; diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index d31708742ba5b1..12261df3197918 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -37,6 +37,8 @@ #include #include #include +#include +#include #include #include @@ -1189,6 +1191,8 @@ static void get_dev_fw_str(struct ib_device *device, char *str, (int) dev->fw_ver & 0xffff); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + int mthca_register_device(struct mthca_dev *dev) { int ret; @@ -1296,6 +1300,7 @@ int mthca_register_device(struct mthca_dev *dev) mutex_init(&dev->cap_mask_mutex); + dev->ib_dev.specs_root = (struct uverbs_root *)&root; ret = ib_register_device(&dev->ib_dev, NULL); if (ret) return ret; diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index aff9fb14768be9..c75560defe7654 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -41,6 +41,8 @@ #include #include #include +#include +#include #include "nes.h" @@ -3862,6 +3864,8 @@ void nes_destroy_ofa_device(struct nes_ib_device *nesibdev) } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + /** * nes_register_ofa_device */ @@ -3872,6 +3876,7 @@ int nes_register_ofa_device(struct nes_ib_device *nesibdev) struct nes_adapter *nesadapter = nesdev->nesadapter; int i, ret; + nesvnic->nesibdev->ibdev.specs_root = (struct uverbs_root *)&root; ret = ib_register_device(&nesvnic->nesibdev->ibdev, NULL); if (ret) { return ret; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index 896071502739a8..0f43018f07b437 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -44,6 +44,8 @@ #include #include #include +#include +#include #include #include @@ -115,6 +117,8 @@ static void get_dev_fw_str(struct ib_device *device, char *str, snprintf(str, str_len, "%s", &dev->attr.fw_ver[0]); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + static int ocrdma_register_device(struct ocrdma_dev *dev) { strlcpy(dev->ibdev.name, "ocrdma%d", IB_DEVICE_NAME_MAX); @@ -218,6 +222,7 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) dev->ibdev.destroy_srq = ocrdma_destroy_srq; dev->ibdev.post_srq_recv = ocrdma_post_srq_recv; } + dev->ibdev.specs_root = (struct uverbs_root *)&root; return ib_register_device(&dev->ibdev, NULL); } diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index 0a89a955550b29..13939bf54271f2 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -48,6 +48,8 @@ #include #include +#include +#include #include #include "usnic_abi.h" @@ -346,6 +348,8 @@ static void usnic_get_dev_fw_str(struct ib_device *device, snprintf(str, str_len, "%s", info.fw_version); } +DECLARE_UVERBS_TYPES_GROUP(root, &uverbs_common_types); + /* Start of PF discovery section */ static void *usnic_ib_device_add(struct pci_dev *dev) { @@ -432,6 +436,7 @@ static void *usnic_ib_device_add(struct pci_dev *dev) us_ibdev->ib_dev.get_dev_fw_str = usnic_get_dev_fw_str; + us_ibdev->ib_dev.specs_root = (struct uverbs_root *)&root; if (ib_register_device(&us_ibdev->ib_dev, NULL)) goto err_fwd_dealloc; diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index b1ac9735fbbea5..50e3298d9bf39b 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1331,21 +1331,29 @@ struct ib_fmr_attr { struct ib_umem; +enum rdma_remove_reason { + /* Userspace requested uobject deletion. Call could fail */ + RDMA_REMOVE_DESTROY, + /* Context deletion. This call should delete the actual object itself */ + RDMA_REMOVE_CLOSE, + /* Driver is being hot-unplugged. This call should delete the actual object itself */ + RDMA_REMOVE_DRIVER_REMOVE, + /* Context is being cleaned-up, but commit was just completed */ + RDMA_REMOVE_DURING_CLEANUP, +}; + struct ib_ucontext { struct ib_device *device; - struct list_head pd_list; - struct list_head mr_list; - struct list_head mw_list; - struct list_head cq_list; - struct list_head qp_list; - struct list_head srq_list; - struct list_head ah_list; - struct list_head xrcd_list; - struct list_head rule_list; - struct list_head wq_list; - struct list_head rwq_ind_tbl_list; + struct ib_uverbs_file *ufile; int closing; + /* locking the uobjects_list */ + struct mutex uobjects_lock; + struct list_head uobjects; + /* protects cleanup process from other actions */ + struct rw_semaphore cleanup_rwsem; + enum rdma_remove_reason cleanup_reason; + struct pid *tgid; #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING struct rb_root umem_tree; @@ -1372,9 +1380,16 @@ struct ib_uobject { struct list_head list; /* link to context's list */ int id; /* index into kernel idr */ struct kref ref; - struct rw_semaphore mutex; /* protects .live */ + atomic_t usecnt; /* protects exclusive access */ struct rcu_head rcu; /* kfree_rcu() overhead */ - int live; + + const struct uverbs_obj_type *type; +}; + +struct ib_uobject_file { + struct ib_uobject uobj; + /* ufile contains the lock between context release and file close */ + struct ib_uverbs_file *ufile; }; struct ib_udata { @@ -2131,6 +2146,8 @@ struct ib_device { */ int (*get_port_immutable)(struct ib_device *, u8, struct ib_port_immutable *); void (*get_dev_fw_str)(struct ib_device *, char *str, size_t str_len); + + struct uverbs_root *specs_root; }; struct ib_client { diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h new file mode 100644 index 00000000000000..aae70d24a86b01 --- /dev/null +++ b/include/rdma/uverbs_ioctl.h @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2016, Mellanox Technologies inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _UVERBS_IOCTL_ +#define _UVERBS_IOCTL_ + +#include +#include +#include + +struct uverbs_object_type; +struct uverbs_uobject_type; + +/* + * ======================================= + * Verbs action specifications + * ======================================= + */ + +#define UVERBS_ID_RESERVED_MASK 0xF000 +#define UVERBS_ID_RESERVED_SHIFT 12 + +enum uverbs_attr_type { + UVERBS_ATTR_TYPE_NA, + UVERBS_ATTR_TYPE_PTR_IN, + UVERBS_ATTR_TYPE_PTR_OUT, + UVERBS_ATTR_TYPE_IDR, + UVERBS_ATTR_TYPE_FD, + UVERBS_ATTR_TYPE_FLAG, +}; + +enum uverbs_idr_access { + UVERBS_ACCESS_READ, + UVERBS_ACCESS_WRITE, + UVERBS_ACCESS_NEW, + UVERBS_ACCESS_DESTROY +}; + +enum uverbs_attr_spec_flags { + UVERBS_ATTR_SPEC_F_MANDATORY = 1U << 0, + UVERBS_ATTR_SPEC_F_MIN_SZ = 1U << 1, +}; + +struct uverbs_attr_spec { + enum uverbs_attr_type type; + u8 flags; + union { + u16 len; + struct { + u16 obj_type; + u8 access; + } obj; + struct { + /* flags are always 64bits */ + u64 mask; + } flag; + }; +}; + +struct uverbs_attr_spec_group { + struct uverbs_attr_spec *attrs; + size_t num_attrs; + /* populate at runtime */ + unsigned long *mandatory_attrs_bitmask; +}; + +struct uverbs_attr_array; +struct ib_uverbs_file; + +enum uverbs_action_flags { + UVERBS_ACTION_FLAG_CREATE_ROOT = 1 << 0, +}; + +struct uverbs_action { + struct uverbs_attr_spec_group **attr_groups; + size_t num_groups; + size_t num_child_attrs; + u32 flags; + int (*handler)(struct ib_device *ib_dev, struct ib_uverbs_file *ufile, + struct uverbs_attr_array *ctx, size_t num); +}; + +struct uverbs_action_group { + size_t num_actions; + struct uverbs_action **actions; +}; + +struct uverbs_type { + size_t num_groups; + const struct uverbs_action_group **action_groups; + const struct uverbs_obj_type *type_attrs; +}; + +struct uverbs_type_group { + size_t num_types; + const struct uverbs_type **types; +}; + +struct uverbs_root { + const struct uverbs_type_group **type_groups; + size_t num_groups; +}; + +#define UA_FLAGS(_flags) .flags = _flags +#define UVERBS_ATTR(_id, _len, _type, ...) \ + [_id] = {.len = _len, .type = _type, ##__VA_ARGS__} +#define UVERBS_ATTR_PTR_IN_SZ(_id, _len, ...) \ + UVERBS_ATTR(_id, _len, UVERBS_ATTR_TYPE_PTR_IN, ##__VA_ARGS__) +#define UVERBS_ATTR_PTR_IN(_id, _type, ...) \ + UVERBS_ATTR_PTR_IN_SZ(_id, sizeof(_type), ##__VA_ARGS__) +#define UVERBS_ATTR_PTR_OUT_SZ(_id, _len, ...) \ + UVERBS_ATTR(_id, _len, UVERBS_ATTR_TYPE_PTR_OUT, ##__VA_ARGS__) +#define UVERBS_ATTR_PTR_OUT(_id, _type, ...) \ + UVERBS_ATTR_PTR_OUT_SZ(_id, sizeof(_type), ##__VA_ARGS__) +#define UVERBS_ATTR_IDR(_id, _idr_type, _access, ...) \ + [_id] = {.type = UVERBS_ATTR_TYPE_IDR, \ + .obj = {.obj_type = _idr_type, \ + .access = _access \ + }, ##__VA_ARGS__ } +#define UVERBS_ATTR_FD(_id, _fd_type, _access, ...) \ + [_id] = {.type = UVERBS_ATTR_TYPE_FD, \ + .obj = {.obj_type = _fd_type, \ + .access = (_access) + BUILD_BUG_ON_ZERO( \ + (_access) != UVERBS_ACCESS_NEW && \ + (_access) != UVERBS_ACCESS_READ) \ + }, ##__VA_ARGS__ } +#define UVERBS_ATTR_FLAG(_id, _mask, ...) \ + [_id] = {.type = UVERBS_ATTR_TYPE_FLAG, \ + .flag = {.mask = _mask}, ##__VA_ARGS__ } +#define _UVERBS_ATTR_SPEC_SZ(...) \ + (sizeof((struct uverbs_attr_spec[]){__VA_ARGS__}) / \ + sizeof(struct uverbs_attr_spec)) +#define UVERBS_ATTR_SPEC(...) \ + ((struct uverbs_attr_spec_group) \ + {.attrs = (struct uverbs_attr_spec[]){__VA_ARGS__}, \ + .num_attrs = _UVERBS_ATTR_SPEC_SZ(__VA_ARGS__), \ + .mandatory_attrs_bitmask = \ + ((unsigned long[BITS_TO_LONGS(_UVERBS_ATTR_SPEC_SZ(__VA_ARGS__))]) {})}) +#define DECLARE_UVERBS_ATTR_SPEC(name, ...) \ + struct uverbs_attr_spec_group name = \ + UVERBS_ATTR_SPEC(__VA_ARGS__) +#define _UVERBS_ATTR_ACTION_SPEC_SZ(...) \ + (sizeof((struct uverbs_attr_spec_group *[]){__VA_ARGS__}) / \ + sizeof(struct uverbs_attr_spec_group *)) +#define _UVERBS_ACTION(_handler, _flags, ...) \ + ((struct uverbs_action) { \ + .flags = _flags, \ + .handler = _handler, \ + .num_groups = _UVERBS_ATTR_ACTION_SPEC_SZ(__VA_ARGS__), \ + .attr_groups = (struct uverbs_attr_spec_group *[]){__VA_ARGS__} }) +#define UVERBS_ACTION(_handler, ...) \ + _UVERBS_ACTION(_handler, 0, __VA_ARGS__) +#define UVERBS_CTX_ACTION(_handler, ...) \ + _UVERBS_ACTION(_handler, UVERBS_ACTION_FLAG_CREATE_ROOT, __VA_ARGS__) +#define _UVERBS_ACTIONS_SZ(...) \ + (sizeof((struct uverbs_action *[]){__VA_ARGS__}) / \ + sizeof(struct uverbs_action *)) +#define ADD_UVERBS_ACTION(action_idx, _handler, ...) \ + [action_idx] = &UVERBS_ACTION(_handler, __VA_ARGS__) +#define DECLARE_UVERBS_ACTION(name, _handler, ...) \ + struct uverbs_action name = \ + UVERBS_ACTION(_handler, __VA_ARGS__) +#define ADD_UVERBS_CTX_ACTION(action_idx, _handler, ...) \ + [action_idx] = &UVERBS_CTX_ACTION(_handler, __VA_ARGS__) +#define DECLARE_UVERBS_CTX_ACTION(name, _handler, ...) \ + struct uverbs_action name = \ + UVERBS_CTX_ACTION(_handler, __VA_ARGS__) +#define ADD_UVERBS_ACTION_PTR(idx, ptr) \ + [idx] = ptr +#define UVERBS_ACTIONS(...) \ + ((const struct uverbs_action_group) \ + {.num_actions = _UVERBS_ACTIONS_SZ(__VA_ARGS__), \ + .actions = (struct uverbs_action *[]){__VA_ARGS__} }) +#define DECLARE_UVERBS_ACTIONS(name, ...) \ + const struct uverbs_type_actions_group name = \ + UVERBS_ACTIONS(__VA_ARGS__) +#define _UVERBS_ACTIONS_GROUP_SZ(...) \ + (sizeof((const struct uverbs_action_group*[]){__VA_ARGS__}) / \ + sizeof(const struct uverbs_action_group *)) + +#define DECLARE_UVERBS_TYPE(name, _type_attrs, ...) \ + const struct uverbs_type name = { \ + .type_attrs = _type_attrs, \ + .num_groups = _UVERBS_ACTIONS_GROUP_SZ(__VA_ARGS__), \ + .action_groups = (const struct uverbs_action_group *[]){__VA_ARGS__} \ + } +#define _UVERBS_TYPE_SZ(...) \ + (sizeof((const struct uverbs_type *[]){__VA_ARGS__}) / \ + sizeof(const struct uverbs_type *)) +#define ADD_UVERBS_TYPE_ACTIONS(type_idx, ...) \ + [type_idx] = &UVERBS_ACTIONS(__VA_ARGS__) +#define ADD_UVERBS_TYPE(type_idx, type_ptr) \ + [type_idx] = ((const struct uverbs_type * const)&(type_ptr)) +#define UVERBS_TYPES(...) ((const struct uverbs_type_group) \ + {.num_types = _UVERBS_TYPE_SZ(__VA_ARGS__), \ + .types = (const struct uverbs_type *[]){__VA_ARGS__} }) +#define DECLARE_UVERBS_TYPES(name, ...) \ + const struct uverbs_type_group name = UVERBS_TYPES(__VA_ARGS__) + +#define _UVERBS_TYPES_SZ(...) \ + (sizeof((const struct uverbs_type_group *[]){__VA_ARGS__}) / \ + sizeof(const struct uverbs_type_group *)) + +#define UVERBS_TYPES_GROUP(...) \ + ((const struct uverbs_root){ \ + .type_groups = (const struct uverbs_type_group *[]){__VA_ARGS__},\ + .num_groups = _UVERBS_TYPES_SZ(__VA_ARGS__)}) +#define DECLARE_UVERBS_TYPES_GROUP(name, ...) \ + const struct uverbs_root name = UVERBS_TYPES_GROUP(__VA_ARGS__) + +/* ================================================= + * Parsing infrastructure + * ================================================= + */ + +struct uverbs_ptr_attr { + void * __user ptr; + u16 len; +}; + +struct uverbs_flag_attr { + u64 flags; +}; + +struct uverbs_obj_attr { + /* pointer to the kernel descriptor -> type, access, etc */ + struct ib_uverbs_attr __user *uattr; + const struct uverbs_obj_type *type; + struct ib_uobject *uobject; + int id; +}; + +struct uverbs_attr { + union { + struct uverbs_ptr_attr ptr_attr; + struct uverbs_obj_attr obj_attr; + struct uverbs_flag_attr flag_attr; + }; +}; + +/* output of one validator */ +struct uverbs_attr_array { + unsigned long *valid_bitmap; + size_t num_attrs; + /* arrays of attrubytes, index is the id i.e SEND_CQ */ + struct uverbs_attr *attrs; +}; + +static inline bool uverbs_is_valid(const struct uverbs_attr_array *attr_array, + unsigned int idx) +{ + return test_bit(idx, attr_array->valid_bitmap); +} + +/* TODO: Add debug version for these macros/inline func */ +static inline int uverbs_copy_to(struct uverbs_attr_array *attr_array, + size_t idx, const void *from) +{ + if (!uverbs_is_valid(attr_array, idx)) + return -ENOENT; + + return copy_to_user(attr_array->attrs[idx].ptr_attr.ptr, from, + attr_array->attrs[idx].ptr_attr.len) ? -EFAULT : 0; +} + +#define uverbs_copy_from(to, attr_array, idx) \ + (uverbs_is_valid((attr_array), idx) ? \ + (sizeof(*to) <= sizeof(((struct ib_uverbs_attr *)0)->data) ?\ + (memcpy(to, &(attr_array)->attrs[idx].ptr_attr.ptr, \ + (attr_array)->attrs[idx].ptr_attr.len), 0) : \ + (copy_from_user((to), (attr_array)->attrs[idx].ptr_attr.ptr, \ + (attr_array)->attrs[idx].ptr_attr.len) ? \ + -EFAULT : 0)) : -ENOENT) +#define uverbs_get_attr(to, attr_array, idx) \ + (uverbs_is_valid((attr_array), idx) ? \ + (sizeof(to) <= sizeof(((struct ib_uverbs_attr *)0)->data) ? \ + (sizeof(to) == sizeof((&(to))[0]) ? \ + ((to) = *(typeof(to) *)&(attr_array)->attrs[idx].ptr_attr.ptr, 0) :\ + (memcpy(&(to), &(attr_array)->attrs[idx].ptr_attr.ptr, \ + (attr_array)->attrs[idx].ptr_attr.len), 0)) : \ + (copy_from_user(&(to), (attr_array)->attrs[idx].ptr_attr.ptr, \ + (attr_array)->attrs[idx].ptr_attr.len) ? \ + -EFAULT : 0)) : -ENOENT) + +#endif + diff --git a/include/rdma/uverbs_std_types.h b/include/rdma/uverbs_std_types.h new file mode 100644 index 00000000000000..d85e337b9b1de6 --- /dev/null +++ b/include/rdma/uverbs_std_types.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _UVERBS_STD_TYPES__ +#define _UVERBS_STD_TYPES__ + +#include +#include +#define UVERBS_UDATA_DRIVER_DATA_GROUP 1 +#define UVERBS_UDATA_DRIVER_DATA_FLAG BIT(UVERBS_ID_RESERVED_SHIFT) + +enum { + UVERBS_UHW_IN, + UVERBS_UHW_OUT, + UVERBS_UHW_NUM +}; + + +extern const struct uverbs_action_group uverbs_actions_mr; +extern const struct uverbs_action_group uverbs_actions_comp_channel; +extern const struct uverbs_action_group uverbs_actions_cq; +extern const struct uverbs_action_group uverbs_actions_qp; +extern const struct uverbs_action_group uverbs_actions_pd; +extern const struct uverbs_action_group uverbs_actions_device; + +extern const struct uverbs_type uverbs_type_comp_channel; +extern const struct uverbs_type uverbs_type_cq; +extern const struct uverbs_type uverbs_type_qp; +extern const struct uverbs_type uverbs_type_rwq_ind_table; +extern const struct uverbs_type uverbs_type_wq; +extern const struct uverbs_type uverbs_type_srq; +extern const struct uverbs_type uverbs_type_ah; +extern const struct uverbs_type uverbs_type_flow; +extern const struct uverbs_type uverbs_type_mr; +extern const struct uverbs_type uverbs_type_mw; +extern const struct uverbs_type uverbs_type_pd; +extern const struct uverbs_type uverbs_type_xrcd; +extern const struct uverbs_type uverbs_type_device; +extern const struct uverbs_type_group uverbs_common_types; + +static inline struct ib_uobject *__uobj_get(const struct uverbs_obj_type *type, + bool write, + struct ib_ucontext *ucontext, + int id) +{ + return rdma_lookup_get_uobject(type, ucontext, id, write); +} + +#define uobj_get_type(_type) uverbs_type_##_type.type_attrs + +#define uobj_get_read(_type, _id, _ucontext) \ + __uobj_get(_type, false, _ucontext, _id) + +#define uobj_get_obj_read(_type, _id, _ucontext) \ +({ \ + struct ib_uobject *uobj = \ + __uobj_get(uverbs_type_##_type.type_attrs, \ + false, _ucontext, _id); \ + \ + (struct ib_##_type *)(IS_ERR(uobj) ? NULL : uobj->object); \ +}) + +#define uobj_get_write(_type, _id, _ucontext) \ + __uobj_get(_type, true, _ucontext, _id) + +static inline void uobj_put_read(struct ib_uobject *uobj) +{ + rdma_lookup_put_uobject(uobj, false); +} + +#define uobj_put_obj_read(_obj) \ + uobj_put_read((_obj)->uobject) + +static inline void uobj_put_write(struct ib_uobject *uobj) +{ + rdma_lookup_put_uobject(uobj, true); +} + +static inline int __must_check uobj_remove_commit(struct ib_uobject *uobj) +{ + return rdma_remove_commit_uobject(uobj); +} + +static inline void uobj_alloc_commit(struct ib_uobject *uobj) +{ + rdma_alloc_commit_uobject(uobj); +} + +static inline void uobj_alloc_abort(struct ib_uobject *uobj) +{ + rdma_alloc_abort_uobject(uobj); +} + +static inline struct ib_uobject *__uobj_alloc(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext) +{ + return rdma_alloc_begin_uobject(type, ucontext); +} + +#define uobj_alloc(_type, ucontext) \ + __uobj_alloc(_type, ucontext) + +#endif + diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h new file mode 100644 index 00000000000000..25c90c10729899 --- /dev/null +++ b/include/rdma/uverbs_types.h @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _UVERBS_TYPES_ +#define _UVERBS_TYPES_ + +#include +#include + +struct uverbs_obj_type; + +struct uverbs_obj_type_class { + /* + * Get an ib_uobject that corresponds to the given id from ucontext, + * These functions could create or destroy objects if required. + * The action will be finalized only when commit, abort or put fops are + * called. + * The flow of the different actions is: + * [alloc]: Starts with alloc_begin. The handlers logic is than + * executed. If the handler is successful, alloc_commit + * is called and the object is inserted to the repository. + * Once alloc_commit completes the object is visible to + * other threads and userspace. + e Otherwise, alloc_abort is called and the object is + * destroyed. + * [lookup]: Starts with lookup_get which fetches and locks the + * object. After the handler finished using the object, it + * needs to call lookup_put to unlock it. The write flag + * indicates if the object is locked for exclusive access. + * [remove]: Starts with lookup_get with write flag set. This locks + * the object for exclusive access. If the handler code + * completed successfully, remove_commit is called and + * the ib_uobject is removed from the context's uobjects + * repository and put. The object itself is destroyed as + * well. Once remove succeeds new krefs to the object + * cannot be acquired by other threads or userspace and + * the hardware driver is removed from the object. + * Other krefs on the object may still exist. + * If the handler code failed, lookup_put should be + * called. This callback is used when the context + * is destroyed as well (process termination, + * reset flow). + */ + struct ib_uobject *(*alloc_begin)(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext); + void (*alloc_commit)(struct ib_uobject *uobj); + void (*alloc_abort)(struct ib_uobject *uobj); + + struct ib_uobject *(*lookup_get)(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext, int id, + bool write); + void (*lookup_put)(struct ib_uobject *uobj, bool write); + /* + * Must be called with the write lock held. If successful uobj is + * invalid on return. On failure uobject is left completely + * unchanged + */ + int __must_check (*remove_commit)(struct ib_uobject *uobj, + enum rdma_remove_reason why); + u8 needs_rcu; +}; + +struct uverbs_obj_type { + const struct uverbs_obj_type_class * const type_class; + size_t obj_size; + unsigned int destroy_order; +}; + +/* + * Objects type classes which support a detach state (object is still alive but + * it's not attached to any context need to make sure: + * (a) If function pointers are used in any modules, module_get was called on + * this module. + * (b) detach isn't called concurrently with context_cleanup + */ + +struct uverbs_obj_idr_type { + /* + * In idr based objects, uverbs_obj_type_class points to a generic + * idr operations. In order to specialize the underlying types (e.g. CQ, + * QPs, etc.), we add destroy_object specific callbacks. + */ + struct uverbs_obj_type type; + + /* Free driver resources from the uobject, make the driver uncallable, + * and move the uobject to the detached state. If the object was + * destroyed by the user's request, a failure should leave the uobject + * completely unchanged. + */ + int __must_check (*destroy_object)(struct ib_uobject *uobj, + enum rdma_remove_reason why); +}; + +struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext, + int id, bool write); +void rdma_lookup_put_uobject(struct ib_uobject *uobj, bool write); +struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_obj_type *type, + struct ib_ucontext *ucontext); +void rdma_alloc_abort_uobject(struct ib_uobject *uobj); +int __must_check rdma_remove_commit_uobject(struct ib_uobject *uobj); +int rdma_alloc_commit_uobject(struct ib_uobject *uobj); + +struct uverbs_obj_fd_type { + /* + * In fd based objects, uverbs_obj_type_ops points to generic + * fd operations. In order to specialize the underlying types (e.g. + * completion_channel), we use fops, name and flags for fd creation. + * context_closed is called when the context is closed either when + * the driver is removed or the process terminated. + */ + struct uverbs_obj_type type; + int (*context_closed)(struct ib_uobject_file *uobj_file, + enum rdma_remove_reason why); + const struct file_operations *fops; + const char *name; + int flags; +}; + +extern const struct uverbs_obj_type_class uverbs_idr_class; +extern const struct uverbs_obj_type_class uverbs_fd_class; + +#define UVERBS_BUILD_BUG_ON(cond) (sizeof(char[1 - 2 * !!(cond)]) - \ + sizeof(char)) +#define UVERBS_TYPE_ALLOC_FD(_order, _obj_size, _context_closed, _fops, _name, _flags)\ + ((&((const struct uverbs_obj_fd_type) \ + {.type = { \ + .destroy_order = _order, \ + .type_class = &uverbs_fd_class, \ + .obj_size = (_obj_size) + \ + UVERBS_BUILD_BUG_ON((_obj_size) < sizeof(struct ib_uobject_file)), \ + }, \ + .context_closed = _context_closed, \ + .fops = _fops, \ + .name = _name, \ + .flags = _flags}))->type) +#define UVERBS_TYPE_ALLOC_IDR_SZ(_size, _order, _destroy_object) \ + ((&((const struct uverbs_obj_idr_type) \ + {.type = { \ + .destroy_order = _order, \ + .type_class = &uverbs_idr_class, \ + .obj_size = (_size) + \ + UVERBS_BUILD_BUG_ON((_size) < \ + sizeof(struct ib_uobject)) \ + }, \ + .destroy_object = _destroy_object,}))->type) +#define UVERBS_TYPE_ALLOC_IDR(_order, _destroy_object) \ + UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uobject), _order, \ + _destroy_object) + +#endif diff --git a/include/uapi/rdma/Kbuild b/include/uapi/rdma/Kbuild index 541311115505d8..71140328c9573b 100644 --- a/include/uapi/rdma/Kbuild +++ b/include/uapi/rdma/Kbuild @@ -1,5 +1,6 @@ # UAPI Header export list header-y += ib_user_cm.h +header-y += rdma_user_ioctl_cmd.h header-y += rdma_user_ioctl.h header-y += ib_user_mad.h header-y += ib_user_sa.h diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h index dfdfe4e92d3118..d7dd73e2b225de 100644 --- a/include/uapi/rdma/ib_user_verbs.h +++ b/include/uapi/rdma/ib_user_verbs.h @@ -319,12 +319,25 @@ struct ib_uverbs_reg_mr { __u64 driver_data[0]; }; +struct ib_uverbs_ioctl_reg_mr { + __u64 start; + __u64 length; + __u64 hca_va; + __u32 access_flags; + __u32 reserved; +}; + struct ib_uverbs_reg_mr_resp { __u32 mr_handle; __u32 lkey; __u32 rkey; }; +struct ib_uverbs_ioctl_reg_mr_resp { + __u32 lkey; + __u32 rkey; +}; + struct ib_uverbs_rereg_mr { __u64 response; __u32 mr_handle; @@ -576,6 +589,17 @@ struct ib_uverbs_ex_create_qp { __u32 reserved1; }; +struct ib_uverbs_ioctl_create_qp { + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + __u8 sq_sig_all; + __u8 qp_type; + __u16 reserved; +}; + struct ib_uverbs_open_qp { __u64 response; __u64 user_handle; @@ -598,6 +622,15 @@ struct ib_uverbs_create_qp_resp { __u32 reserved; }; +struct ib_uverbs_ioctl_create_qp_resp { + __u32 qpn; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; +}; + struct ib_uverbs_ex_create_qp_resp { struct ib_uverbs_create_qp_resp base; __u32 comp_mask; @@ -623,6 +656,13 @@ struct ib_uverbs_qp_dest { __u8 port_num; }; +struct ib_uverbs_qp_alt_path { + struct ib_uverbs_qp_dest dest; + __u16 pkey_index; + __u8 port_num; + __u8 timeout; +}; + struct ib_uverbs_query_qp { __u64 response; __u32 qp_handle; diff --git a/include/uapi/rdma/rdma_user_ioctl.h b/include/uapi/rdma/rdma_user_ioctl.h index 9388125ad51ba0..12663f6700a0ce 100644 --- a/include/uapi/rdma/rdma_user_ioctl.h +++ b/include/uapi/rdma/rdma_user_ioctl.h @@ -43,6 +43,31 @@ /* Legacy name, for user space application which already use it */ #define IB_IOCTL_MAGIC RDMA_IOCTL_MAGIC +#define RDMA_VERBS_IOCTL \ + _IOWR(RDMA_IOCTL_MAGIC, 1, struct ib_uverbs_ioctl_hdr) + +enum ib_uverbs_attr_flags { + UVERBS_ATTR_F_MANDATORY = 1U << 0, +}; + +struct ib_uverbs_attr { + __u16 attr_id; /* command specific type attribute */ + __u16 len; /* NA for idr */ + __u16 flags; /* combination of uverbs_attr_flags */ + __u16 reserved; + __u64 data; /* ptr to command, inline data or idr/fd */ +}; + +struct ib_uverbs_ioctl_hdr { + __u16 length; + __u16 flags; + __u16 object_type; + __u16 reserved; /* future use for driver_id */ + __u16 action; + __u16 num_attrs; + struct ib_uverbs_attr attrs[0]; +}; + /* * General blocks assignments * It is closed on purpose do not expose it it user space diff --git a/include/uapi/rdma/rdma_user_ioctl_cmd.h b/include/uapi/rdma/rdma_user_ioctl_cmd.h new file mode 100644 index 00000000000000..727f17288c19a5 --- /dev/null +++ b/include/uapi/rdma/rdma_user_ioctl_cmd.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2016 Mellanox Technologies, LTD. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef RDMA_USER_IOCTL_CMD_H +#define RDMA_USER_IOCTL_CMD_H + +#include + +enum uverbs_common_types { + UVERBS_TYPE_DEVICE, /* No instances of DEVICE are allowed */ + UVERBS_TYPE_PD, + UVERBS_TYPE_COMP_CHANNEL, + UVERBS_TYPE_CQ, + UVERBS_TYPE_QP, + UVERBS_TYPE_SRQ, + UVERBS_TYPE_AH, + UVERBS_TYPE_MR, + UVERBS_TYPE_MW, + UVERBS_TYPE_FLOW, + UVERBS_TYPE_XRCD, + UVERBS_TYPE_RWQ_IND_TBL, + UVERBS_TYPE_WQ, + UVERBS_TYPE_LAST, +}; + +enum uverbs_create_qp_cmd_attr_ids { + CREATE_QP_HANDLE, + CREATE_QP_PD_HANDLE, + CREATE_QP_SEND_CQ, + CREATE_QP_RECV_CQ, + CREATE_QP_SRQ, + CREATE_QP_USER_HANDLE, + CREATE_QP_CMD, + CREATE_QP_CMD_FLAGS, + CREATE_QP_RESP, + CREATE_QP_RESERVED +}; + +enum uverbs_destroy_qp_cmd_attr_ids { + DESTROY_QP_HANDLE, + DESTROY_QP_EVENTS_REPORTED, + DESTROY_QP_RESERVED +}; + +enum uverbs_create_cq_cmd_attr_ids { + CREATE_CQ_HANDLE, + CREATE_CQ_CQE, + CREATE_CQ_USER_HANDLE, + CREATE_CQ_COMP_CHANNEL, + CREATE_CQ_COMP_VECTOR, + CREATE_CQ_FLAGS, + CREATE_CQ_RESP_CQE, + CREATE_CQ_RESERVED +}; + +enum uverbs_destroy_cq_cmd_attr_ids { + DESTROY_CQ_HANDLE, + DESTROY_CQ_RESP, + DESTROY_CQ_RESERVED +}; + +enum uverbs_query_qp_cmd_attr_ids { + QUERY_QP_HANDLE, + QUERY_QP_ATTR_MASK, + QUERY_QP_RESP, + QUERY_QP_RESERVED +}; + +enum uverbs_create_qp_xrc_tgt_cmd_attr_ids { + CREATE_QP_XRC_TGT_HANDLE, + CREATE_QP_XRC_TGT_XRCD, + CREATE_QP_XRC_TGT_USER_HANDLE, + CREATE_QP_XRC_TGT_CMD, + CREATE_QP_XRC_TGT_CMD_FLAGS, + CREATE_QP_XRC_TGT_RESP, + CREATE_QP_XRC_TGT_RESERVED +}; + +enum uverbs_modify_qp_cmd_attr_ids { + MODIFY_QP_HANDLE, + MODIFY_QP_STATE, + MODIFY_QP_CUR_STATE, + MODIFY_QP_EN_SQD_ASYNC_NOTIFY, + MODIFY_QP_ACCESS_FLAGS, + MODIFY_QP_PKEY_INDEX, + MODIFY_QP_PORT, + MODIFY_QP_QKEY, + MODIFY_QP_AV, + MODIFY_QP_PATH_MTU, + MODIFY_QP_TIMEOUT, + MODIFY_QP_RETRY_CNT, + MODIFY_QP_RNR_RETRY, + MODIFY_QP_RQ_PSN, + MODIFY_QP_MAX_RD_ATOMIC, + MODIFY_QP_ALT_PATH, + MODIFY_QP_MIN_RNR_TIMER, + MODIFY_QP_SQ_PSN, + MODIFY_QP_MAX_DEST_RD_ATOMIC, + MODIFY_QP_PATH_MIG_STATE, + MODIFY_QP_DEST_QPN, + MODIFY_QP_RATE_LIMIT, + MODIFY_QP_RESERVED +}; + +enum uverbs_create_comp_channel_cmd_attr_ids { + CREATE_COMP_CHANNEL_FD, + CREATE_COMP_CHANNEL_RESERVED +}; + +enum uverbs_get_context_cmd_attr_ids { + GET_CONTEXT_RESP, + GET_CONTEXT_RESERVED +}; + +enum uverbs_query_device_cmd_attr_ids { + QUERY_DEVICE_RESP, + QUERY_DEVICE_ODP, + QUERY_DEVICE_TIMESTAMP_MASK, + QUERY_DEVICE_HCA_CORE_CLOCK, + QUERY_DEVICE_CAP_FLAGS, + QUERY_DEVICE_RESERVED +}; + +enum uverbs_query_port_cmd_attr_ids { + QUERY_PORT_PORT_NUM, + QUERY_PORT_RESP, + QUERY_PORT_RESERVED +}; + +enum uverbs_alloc_pd_cmd_attr_ids { + ALLOC_PD_HANDLE, + ALLOC_PD_RESERVED +}; + +enum uverbs_dealloc_pd_cmd_attr_ids { + DEALLOC_PD_HANDLE, + DEALLOC_PD_RESERVED +}; + +enum uverbs_reg_mr_cmd_attr_ids { + REG_MR_HANDLE, + REG_MR_PD_HANDLE, + REG_MR_CMD, + REG_MR_RESP, + REG_MR_RESERVED +}; + +enum uverbs_rereg_mr_cmd_attr_ids { + REREG_MR_HANDLE, + REREG_MR_PD_HANDLE, + REREG_MR_CMD, + REREG_MR_RESP, + REREG_MR_RESERVED +}; + +enum uverbs_dereg_mr_cmd_attr_ids { + DEREG_MR_HANDLE, + DEREG_MR_RESERVED +}; + +enum uverbs_actions_mr_ops { + UVERBS_MR_REG, + UVERBS_MR_DEREG, + UVERBS_MR_REREG, +}; + +enum uverbs_actions_comp_channel_ops { + UVERBS_COMP_CHANNEL_CREATE, +}; + +enum uverbs_actions_cq_ops { + UVERBS_CQ_CREATE, + UVERBS_CQ_DESTROY, +}; + +enum uverbs_actions_qp_ops { + UVERBS_QP_CREATE, + UVERBS_QP_CREATE_XRC_TGT, + UVERBS_QP_MODIFY, + UVERBS_QP_DESTROY, + UVERBS_QP_QUERY, +}; + +enum uverbs_actions_pd_ops { + UVERBS_PD_ALLOC, + UVERBS_PD_DEALLOC +}; + +enum uverbs_actions_device_ops { + UVERBS_DEVICE_ALLOC_CONTEXT, + UVERBS_DEVICE_QUERY, + UVERBS_DEVICE_PORT_QUERY, +}; +#endif /* RDMA_USER_IOCTL_CMD_H */ +