From 2f16752cc6aa99b6f7fdbdc620cf7f78314ad673 Mon Sep 17 00:00:00 2001 From: Volodymyr Babchuk Date: Fri, 24 Dec 2021 02:41:05 +0200 Subject: [PATCH 1/5] [FIXUP ME][DOMD] tools/light: pci: use strict JSON types This will prevent vulnerability, when other side sends object of different type, which will be incorrectly interpolated by the code. The most grave example is when other side send a number, and local code interprets it as an pointer to a string while accessing ->u. Signed-off-by: Volodymyr Babchuk --- tools/libs/light/libxl_pci.c | 8 ++++---- tools/libs/light/libxl_pcid.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/libs/light/libxl_pci.c b/tools/libs/light/libxl_pci.c index 48a3908481e7..610a09ba5364 100644 --- a/tools/libs/light/libxl_pci.c +++ b/tools/libs/light/libxl_pci.c @@ -34,7 +34,7 @@ static int process_list_assignable(libxl__gc *gc, libxl__json_object **result) { *result = (libxl__json_object *)libxl__json_map_get(PCID_MSG_FIELD_DEVICES, - response, JSON_ANY); + response, JSON_ARRAY); if (!*result) return ERROR_INVAL; @@ -52,13 +52,13 @@ static int pci_handle_response(libxl__gc *gc, *result = NULL; - command_obj = libxl__json_map_get(PCID_MSG_FIELD_RESP, response, JSON_ANY); + command_obj = libxl__json_map_get(PCID_MSG_FIELD_RESP, response, JSON_STRING); if (!command_obj) { /* This is an unsupported or bad response. */ return 0; } - err_obj = libxl__json_map_get(PCID_MSG_FIELD_ERR, response, JSON_ANY); + err_obj = libxl__json_map_get(PCID_MSG_FIELD_ERR, response, JSON_STRING); if (!err_obj) { /* Bad packet without error code field. */ return 0; @@ -69,7 +69,7 @@ static int pci_handle_response(libxl__gc *gc, /* The response may contain an optional error string. */ err_desc_obj = libxl__json_map_get(PCID_MSG_FIELD_ERR_DESC, - response, JSON_ANY); + response, JSON_STRING); if (err_desc_obj) LOG(ERROR, "%s", err_desc_obj->u.string); else diff --git a/tools/libs/light/libxl_pcid.c b/tools/libs/light/libxl_pcid.c index 5a9ef2e4cb44..96f306acf2b3 100644 --- a/tools/libs/light/libxl_pcid.c +++ b/tools/libs/light/libxl_pcid.c @@ -129,7 +129,7 @@ static int pcid_handle_request(libxl__gc *gc, yajl_gen gen, yajl_gen_map_open(gen); - command_obj = libxl__json_map_get(PCID_MSG_FIELD_CMD, request, JSON_ANY); + command_obj = libxl__json_map_get(PCID_MSG_FIELD_CMD, request, JSON_STRING); if (!command_obj) { /* This is an unsupported or bad request. */ ret = make_error_reply(gc, gen, "Unsupported request or bad packet", From d89bd398a0f834ab82acc7acd90a9981ca20139c Mon Sep 17 00:00:00 2001 From: Volodymyr Babchuk Date: Fri, 24 Dec 2021 02:43:56 +0200 Subject: [PATCH 2/5] [DOMD] tools/light: pci: describe [MAKE|REVERT]_ASSIGNABLE commands Add protocol for two more commands, one to make a PCI device assignable, and other - to revert its state back. Signed-off-by: Volodymyr Babchuk --- tools/include/pcid.h | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tools/include/pcid.h b/tools/include/pcid.h index 452bdc11cf63..118f8105cf46 100644 --- a/tools/include/pcid.h +++ b/tools/include/pcid.h @@ -98,6 +98,44 @@ #define PCID_CMD_LIST_ASSIGNABLE "list_assignable" #define PCID_MSG_FIELD_DEVICES "devices" +/* + ******************************************************************************* + * Make device assignable + * + * This command makes given device assignable by ensuring that OS + * will not try to access it. + * + * Request (see other mandatory fields above): + * - "cmd" field of the request must be set to "make_assignable". + * - "sbdf" SBDF of the device in format defined by PCID_SBDF_FMT. + * - "rebind" = true if daemon needs to save original driver name, + * so device later can be rebound back. + * + * Response (see other mandatory fields above): + * - "resp" field of the response must be set to "make_assignable". + */ +#define PCID_CMD_MAKE_ASSIGNABLE "make_assignable" +#define PCID_MSG_FIELD_REBIND "rebind" + +/* + ******************************************************************************* + * Revert device from assignable state + * + * This command reverts effect of "make_assignable" command. Basically, + * now device can be used by OS again. + * + * Request (see other mandatory fields above): + * - "cmd" field of the request must be set to "revert_assignable". + * - "sbdf" SBDF of the device in format defined by PCID_SBDF_FMT. + * - "rebind" = true if daemon needs to rebind device back to it's + * original driver, which name was saved by "make_assignable" command + * + * Response (see other mandatory fields above): + * - "resp" field of the response must be set to "revert_assignable". + */ +#define PCID_CMD_REVERT_ASSIGNABLE "revert_assignable" + + int libxl_pcid_process(libxl_ctx *ctx); #endif /* PCID_H */ From 6757aa5ee346dddb38d1c6ff8df436d093a8401a Mon Sep 17 00:00:00 2001 From: Volodymyr Babchuk Date: Fri, 24 Dec 2021 02:45:43 +0200 Subject: [PATCH 3/5] [DOMD] tools/light: pci: add arguments to a JSON request Signed-off-by: Volodymyr Babchuk --- tools/libs/light/libxl_pci.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/libs/light/libxl_pci.c b/tools/libs/light/libxl_pci.c index 610a09ba5364..14e5cbe2562b 100644 --- a/tools/libs/light/libxl_pci.c +++ b/tools/libs/light/libxl_pci.c @@ -101,6 +101,13 @@ static char *pci_prepare_request(libxl__gc *gc, yajl_gen gen, char *cmd, if (rc) return NULL; + if (args) { + rc = libxl__json_object_to_yajl_gen(gc, gen, args); + /* TODO: close gen? */ + if (rc) + return NULL; + } + yajl_gen_map_close(gen); sts = yajl_gen_get_buf(gen, &buf, &len); From 38608f9e8ac1fdd94b6bb6a8cba4223dcf7f0bd7 Mon Sep 17 00:00:00 2001 From: Volodymyr Babchuk Date: Fri, 24 Dec 2021 02:47:09 +0200 Subject: [PATCH 4/5] [DOMD] tools/light: pcid: allow response to be empty Not all commands provide additional response fields, so make this an optional parameter. Signed-off-by: Volodymyr Babchuk --- tools/libs/light/libxl_pcid.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/libs/light/libxl_pcid.c b/tools/libs/light/libxl_pcid.c index 96f306acf2b3..9eb50c776ef5 100644 --- a/tools/libs/light/libxl_pcid.c +++ b/tools/libs/light/libxl_pcid.c @@ -123,7 +123,7 @@ static int pcid_handle_request(libxl__gc *gc, yajl_gen gen, const libxl__json_object *request) { const libxl__json_object *command_obj; - libxl__json_object *command_response; + libxl__json_object *command_response = NULL; char *command_name; int ret = 0; @@ -162,9 +162,11 @@ static int pcid_handle_request(libxl__gc *gc, yajl_gen gen, goto out; } - ret = libxl__json_object_to_yajl_gen(gc, gen, command_response); - if (ret) - goto out; + if (command_response) { + ret = libxl__json_object_to_yajl_gen(gc, gen, command_response); + if (ret) + goto out; + } ret = libxl__vchan_field_add_string(gc, gen, PCID_MSG_FIELD_RESP, command_name); From 05e4705f6bae9c7ac49e0030e1d343feb6962a3c Mon Sep 17 00:00:00 2001 From: Volodymyr Babchuk Date: Fri, 24 Dec 2021 02:48:24 +0200 Subject: [PATCH 5/5] [NOT TESTED][DOMD] tools/light: pci: move assign/revert logic to pcid Implement commands MAKE_ASSIGNABLE and REVERT_ASSIGNABLE in pcid in the same way as they were implemented in libxl_pci.c Replace original logic in libxl_pci.c by calling appropriate functions from pcid. This is quite huge patch, as lots of code were moved from lixbl_pci.c to libxl_pcid.c. Signed-off-by: Volodymyr Babchuk --- Not tested. Use at own risk. Also, we need to reconsider storing old driver name in xenstore. As pcid is lasting process, we can implement some map and store this information in memory. --- tools/libs/light/libxl_pci.c | 300 ++++++-------------------- tools/libs/light/libxl_pcid.c | 396 ++++++++++++++++++++++++++++++++++ 2 files changed, 464 insertions(+), 232 deletions(-) diff --git a/tools/libs/light/libxl_pci.c b/tools/libs/light/libxl_pci.c index 14e5cbe2562b..1b60e513a932 100644 --- a/tools/libs/light/libxl_pci.c +++ b/tools/libs/light/libxl_pci.c @@ -82,6 +82,10 @@ static int pci_handle_response(libxl__gc *gc, if (strcmp(command_name, PCID_CMD_LIST_ASSIGNABLE) == 0) ret = process_list_assignable(gc, response, result); + else if (strcmp(command_name, PCID_CMD_MAKE_ASSIGNABLE) == 0) + *result = libxl__json_object_alloc(gc, JSON_NULL); + else if (strcmp(command_name, PCID_CMD_REVERT_ASSIGNABLE) == 0) + *result = libxl__json_object_alloc(gc, JSON_NULL); return ret; } @@ -617,44 +621,6 @@ void libxl_device_pci_assignable_list_free(libxl_device_pci *list, int num) free(list); } -/* Unbind device from its current driver, if any. If driver_path is non-NULL, - * store the path to the original driver in it. */ -static int sysfs_dev_unbind(libxl__gc *gc, libxl_device_pci *pci, - char **driver_path) -{ - char * spath, *dp = NULL; - struct stat st; - - spath = GCSPRINTF(SYSFS_PCI_DEV"/"PCI_BDF"/driver", - pci->domain, - pci->bus, - pci->dev, - pci->func); - if ( !lstat(spath, &st) ) { - /* Find the canonical path to the driver. */ - dp = libxl__zalloc(gc, PATH_MAX); - dp = realpath(spath, dp); - if ( !dp ) { - LOGE(ERROR, "realpath() failed"); - return -1; - } - - LOG(DEBUG, "Driver re-plug path: %s", dp); - - /* Unbind from the old driver */ - spath = GCSPRINTF("%s/unbind", dp); - if ( sysfs_write_bdf(gc, spath, pci) < 0 ) { - LOGE(ERROR, "Couldn't unbind device"); - return -1; - } - } - - if ( driver_path ) - *driver_path = dp; - - return 0; -} - static uint16_t sysfs_dev_get_vendor(libxl__gc *gc, libxl_device_pci *pci) { char *pci_device_vendor_path = @@ -766,49 +732,6 @@ bool libxl__is_igd_vga_passthru(libxl__gc *gc, return false; } -/* - * A brief comment about slots. I don't know what slots are for; however, - * I have by experimentation determined: - * - Before a device can be bound to pciback, its BDF must first be listed - * in pciback/slots - * - The way to get the BDF listed there is to write BDF to - * pciback/new_slot - * - Writing the same BDF to pciback/new_slot is not idempotent; it results - * in two entries of the BDF in pciback/slots - * It's not clear whether having two entries in pciback/slots is a problem - * or not. Just to be safe, this code does the conservative thing, and - * first checks to see if there is a slot, adding one only if one does not - * already exist. - */ - -/* Scan through /sys/.../pciback/slots looking for pci's BDF */ -static int pciback_dev_has_slot(libxl__gc *gc, libxl_device_pci *pci) -{ - FILE *f; - int rc = 0; - unsigned dom, bus, dev, func; - - f = fopen(SYSFS_PCIBACK_DRIVER"/slots", "r"); - - if (f == NULL) { - LOGE(ERROR, "Couldn't open %s", SYSFS_PCIBACK_DRIVER"/slots"); - return ERROR_FAIL; - } - - while (fscanf(f, "%x:%x:%x.%d\n", &dom, &bus, &dev, &func) == 4) { - if (dom == pci->domain - && bus == pci->bus - && dev == pci->dev - && func == pci->func) { - rc = 1; - goto out; - } - } -out: - fclose(f); - return rc; -} - static int pciback_dev_is_assigned(libxl__gc *gc, libxl_device_pci *pci) { char * spath; @@ -837,133 +760,46 @@ static int pciback_dev_is_assigned(libxl__gc *gc, libxl_device_pci *pci) return -1; } -static int pciback_dev_assign(libxl__gc *gc, libxl_device_pci *pci) -{ - int rc; - - if ( (rc = pciback_dev_has_slot(gc, pci)) < 0 ) { - LOGE(ERROR, "Error checking for pciback slot"); - return ERROR_FAIL; - } else if (rc == 0) { - if ( sysfs_write_bdf(gc, SYSFS_PCIBACK_DRIVER"/new_slot", - pci) < 0 ) { - LOGE(ERROR, "Couldn't bind device to pciback!"); - return ERROR_FAIL; - } - } - - if ( sysfs_write_bdf(gc, SYSFS_PCIBACK_DRIVER"/bind", pci) < 0 ) { - LOGE(ERROR, "Couldn't bind device to pciback!"); - return ERROR_FAIL; - } - return 0; -} - -static int pciback_dev_unassign(libxl__gc *gc, libxl_device_pci *pci) -{ - /* Remove from pciback */ - if ( sysfs_dev_unbind(gc, pci, NULL) < 0 ) { - LOG(ERROR, "Couldn't unbind device!"); - return ERROR_FAIL; - } - - /* Remove slot if necessary */ - if ( pciback_dev_has_slot(gc, pci) > 0 ) { - if ( sysfs_write_bdf(gc, SYSFS_PCIBACK_DRIVER"/remove_slot", - pci) < 0 ) { - LOGE(ERROR, "Couldn't remove pciback slot"); - return ERROR_FAIL; - } - } - return 0; -} - static int libxl__device_pci_assignable_add(libxl__gc *gc, libxl_device_pci *pci, int rebind) { libxl_ctx *ctx = libxl__gc_owner(gc); - unsigned dom, bus, dev, func; - char *spath, *driver_path = NULL; - const char *name; + struct vchan_info *vchan; int rc; - struct stat st; - - /* Local copy for convenience */ - dom = pci->domain; - bus = pci->bus; - dev = pci->dev; - func = pci->func; - name = pci->name; + libxl__json_object *args, *temp_obj, *result; - /* Sanitise any name that is set */ - if (name) { - unsigned int i, n = strlen(name); - - if (n > 64) { /* Reasonable upper bound on name length */ - LOG(ERROR, "Name too long"); - return ERROR_FAIL; - } - - for (i = 0; i < n; i++) { - if (!isgraph(name[i])) { - LOG(ERROR, "Names may only include printable characters"); - return ERROR_FAIL; - } - } - } - - /* See if the device exists */ - spath = GCSPRINTF(SYSFS_PCI_DEV"/"PCI_BDF, dom, bus, dev, func); - if ( lstat(spath, &st) ) { - LOGE(ERROR, "Couldn't lstat %s", spath); - return ERROR_FAIL; - } - - /* Check to see if it's already assigned to pciback */ - rc = pciback_dev_is_assigned(gc, pci); - if ( rc < 0 ) { - return ERROR_FAIL; - } - if ( rc ) { - LOG(WARN, PCI_BDF" already assigned to pciback", dom, bus, dev, func); - goto name; + vchan = pci_vchan_get_client(gc); + if (!vchan) { + rc = ERROR_NOT_READY; + goto out; } - /* Check to see if there's already a driver that we need to unbind from */ - if ( sysfs_dev_unbind(gc, pci, &driver_path ) ) { - LOG(ERROR, "Couldn't unbind "PCI_BDF" from driver", - dom, bus, dev, func); - return ERROR_FAIL; + args = libxl__json_object_alloc(gc, JSON_MAP); + temp_obj = libxl__json_object_alloc(gc, JSON_STRING); + if (!temp_obj) { + rc = ERROR_NOMEM; + goto vchan_free; } + temp_obj->u.string = GCSPRINTF(PCID_SBDF_FMT, pci->domain, pci->bus, + pci->dev, pci->func); + flexarray_append_pair(args->u.map, PCID_MSG_FIELD_SBDF, temp_obj); - /* Store driver_path for rebinding to dom0 */ - if ( rebind ) { - if ( driver_path ) { - pci_info_xs_write(gc, pci, "driver_path", driver_path); - } else if ( (driver_path = - pci_info_xs_read(gc, pci, "driver_path")) != NULL ) { - LOG(INFO, PCI_BDF" not bound to a driver, will be rebound to %s", - dom, bus, dev, func, driver_path); - } else { - LOG(WARN, PCI_BDF" not bound to a driver, will not be rebound.", - dom, bus, dev, func); - } - } else { - pci_info_xs_remove(gc, pci, "driver_path"); + args = libxl__json_object_alloc(gc, JSON_MAP); + temp_obj = libxl__json_object_alloc(gc, JSON_BOOL); + if (!temp_obj) { + rc = ERROR_NOMEM; + goto vchan_free; } + temp_obj->u.b = rebind; + flexarray_append_pair(args->u.map, PCID_MSG_FIELD_REBIND, temp_obj); - if ( pciback_dev_assign(gc, pci) ) { - LOG(ERROR, "Couldn't bind device to pciback!"); - return ERROR_FAIL; + result = vchan_send_command(gc, vchan, PCID_CMD_MAKE_ASSIGNABLE, args); + if (!result) { + rc = ERROR_FAIL; + goto vchan_free; } -name: - if (name) - pci_info_xs_write(gc, pci, "name", name); - else - pci_info_xs_remove(gc, pci, "name"); - /* * DOMID_IO is just a sentinel domain, without any actual mappings, * so always pass XEN_DOMCTL_DEV_RDM_RELAXED to avoid assignment being @@ -971,12 +807,15 @@ static int libxl__device_pci_assignable_add(libxl__gc *gc, */ rc = xc_assign_device(ctx->xch, DOMID_IO, pci_encode_bdf(pci), XEN_DOMCTL_DEV_RDM_RELAXED); - if ( rc < 0 ) { - LOG(ERROR, "failed to quarantine "PCI_BDF, dom, bus, dev, func); - return ERROR_FAIL; - } + if ( rc < 0 ) + LOG(ERROR, "failed to quarantine "PCI_BDF, pci->domain, pci->bus, + pci->dev, pci->func); - return 0; +vchan_free: + pci_vchan_free(gc, vchan); + +out: + return rc; } static int name2bdf(libxl__gc *gc, libxl_device_pci *pci) @@ -1019,13 +858,8 @@ static int libxl__device_pci_assignable_remove(libxl__gc *gc, { libxl_ctx *ctx = libxl__gc_owner(gc); int rc; - char *driver_path; - - /* If the device is named then we need to look up the BDF */ - if (pci->name) { - rc = name2bdf(gc, pci); - if (rc) return rc; - } + struct vchan_info *vchan; + libxl__json_object *args, *temp_obj, *result; /* De-quarantine */ rc = xc_deassign_device(ctx->xch, DOMID_IO, pci_encode_bdf(pci)); @@ -1035,41 +869,43 @@ static int libxl__device_pci_assignable_remove(libxl__gc *gc, return ERROR_FAIL; } - /* Unbind from pciback */ - if ( (rc = pciback_dev_is_assigned(gc, pci)) < 0 ) { - return ERROR_FAIL; - } else if ( rc ) { - pciback_dev_unassign(gc, pci); - } else { - LOG(WARN, "Not bound to pciback"); + vchan = pci_vchan_get_client(gc); + if (!vchan) { + rc = ERROR_NOT_READY; + goto out; } - /* Rebind if necessary */ - driver_path = pci_info_xs_read(gc, pci, "driver_path"); + args = libxl__json_object_alloc(gc, JSON_MAP); + temp_obj = libxl__json_object_alloc(gc, JSON_STRING); + if (!temp_obj) { + rc = ERROR_NOMEM; + goto vchan_free; + } + temp_obj->u.string = GCSPRINTF(PCID_SBDF_FMT, pci->domain, pci->bus, + pci->dev, pci->func); + flexarray_append_pair(args->u.map, PCID_MSG_FIELD_SBDF, temp_obj); - if ( driver_path ) { - if ( rebind ) { - LOG(INFO, "Rebinding to driver at %s", driver_path); + args = libxl__json_object_alloc(gc, JSON_MAP); + temp_obj = libxl__json_object_alloc(gc, JSON_BOOL); + if (!temp_obj) { + rc = ERROR_NOMEM; + goto vchan_free; + } - if ( sysfs_write_bdf(gc, - GCSPRINTF("%s/bind", driver_path), - pci) < 0 ) { - LOGE(ERROR, "Couldn't bind device to %s", driver_path); - return -1; - } + temp_obj->u.b = rebind; + flexarray_append_pair(args->u.map, PCID_MSG_FIELD_REBIND, temp_obj); - pci_info_xs_remove(gc, pci, "driver_path"); - } - } else { - if ( rebind ) { - LOG(WARN, - "Couldn't find path for original driver; not rebinding"); - } + result = vchan_send_command(gc, vchan, PCID_CMD_REVERT_ASSIGNABLE, args); + if (!result) { + rc = ERROR_FAIL; + goto vchan_free; } - pci_info_xs_remove(gc, pci, "name"); +vchan_free: + pci_vchan_free(gc, vchan); - return 0; +out: + return rc; } int libxl_device_pci_assignable_add(libxl_ctx *ctx, libxl_device_pci *pci, diff --git a/tools/libs/light/libxl_pcid.c b/tools/libs/light/libxl_pcid.c index 9eb50c776ef5..f7c3b419da54 100644 --- a/tools/libs/light/libxl_pcid.c +++ b/tools/libs/light/libxl_pcid.c @@ -38,6 +38,8 @@ #define DOM0_ID 0 +#define PCI_BDF "%04x:%02x:%02x.%01x" + struct vchan_client { LIBXL_LIST_ENTRY(struct vchan_client) list; @@ -119,6 +121,394 @@ static int process_list_assignable(libxl__gc *gc, yajl_gen gen, return 0; } +static int pciback_dev_is_assigned(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func) +{ + char * spath; + int rc; + struct stat st; + + if (access(SYSFS_PCIBACK_DRIVER, F_OK) < 0) { + if (errno == ENOENT) { + LOG(ERROR, "Looks like pciback driver is not loaded"); + } else { + LOGE(ERROR, "Can't access "SYSFS_PCIBACK_DRIVER); + } + return -1; + } + + spath = GCSPRINTF(SYSFS_PCIBACK_DRIVER"/"PCI_BDF, + domain, bus, dev, func); + rc = lstat(spath, &st); + + if (rc == 0) + return 1; + if (rc < 0 && errno == ENOENT) + return 0; + LOGE(ERROR, "Accessing %s", spath); + return -1; +} + +#define PCID_INFO_PATH "pcid" +#define PCID_BDF_XSPATH "%04x-%02x-%02x-%01x" + +static char *pcid_info_xs_path(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func, const char *node) +{ + return node ? + GCSPRINTF(PCID_INFO_PATH"/"PCID_BDF_XSPATH"/%s", + domain, bus, dev, func, node) : + GCSPRINTF(PCID_INFO_PATH"/"PCID_BDF_XSPATH, + domain, bus, dev, func); +} + + +static int pcid_info_xs_write(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func, const char *node, + const char *val) +{ + char *path = pcid_info_xs_path(gc, domain, bus, dev, func, node); + int rc = libxl__xs_printf(gc, XBT_NULL, path, "%s", val); + + if (rc) LOGE(WARN, "Write of %s to node %s failed.", val, path); + + return rc; +} + +static char *pcid_info_xs_read(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func, const char *node) +{ + char *path = pcid_info_xs_path(gc, domain, bus, dev, func, node); + + return libxl__xs_read(gc, XBT_NULL, path); +} + +static void pcid_info_xs_remove(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func, const char *node) +{ + char *path = pcid_info_xs_path(gc, domain, bus, dev, func, node); + libxl_ctx *ctx = libxl__gc_owner(gc); + + /* Remove the xenstore entry */ + xs_rm(ctx->xsh, XBT_NULL, path); +} + + +/* Write the standard BDF into the sysfs path given by sysfs_path. */ +static int sysfs_write_bdf(libxl__gc *gc, const char * sysfs_path, + unsigned int domain, unsigned int bus, + unsigned int dev, unsigned int func) +{ + int rc, fd; + char *buf; + + fd = open(sysfs_path, O_WRONLY); + if (fd < 0) { + LOGE(ERROR, "Couldn't open %s", sysfs_path); + return ERROR_FAIL; + } + + buf = GCSPRINTF(PCI_BDF, domain, bus, dev, func); + rc = write(fd, buf, strlen(buf)); + /* Annoying to have two if's, but we need the errno */ + if (rc < 0) + LOGE(ERROR, "write to %s returned %d", sysfs_path, rc); + close(fd); + + if (rc < 0) + return ERROR_FAIL; + + return 0; +} + + +/* Unbind device from its current driver, if any. If driver_path is non-NULL, + * store the path to the original driver in it. */ +static int sysfs_dev_unbind(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func, + char **driver_path) +{ + char * spath, *dp = NULL; + struct stat st; + + spath = GCSPRINTF(SYSFS_PCI_DEV"/"PCI_BDF"/driver", + domain, bus, dev, func); + if (!lstat(spath, &st)) { + /* Find the canonical path to the driver. */ + dp = libxl__zalloc(gc, PATH_MAX); + dp = realpath(spath, dp); + if ( !dp ) { + LOGE(ERROR, "realpath() failed"); + return -1; + } + + LOG(DEBUG, "Driver re-plug path: %s", dp); + + /* Unbind from the old driver */ + spath = GCSPRINTF("%s/unbind", dp); + if (sysfs_write_bdf(gc, spath, domain, bus, dev, func) < 0) { + LOGE(ERROR, "Couldn't unbind device"); + return -1; + } + } + + if (driver_path) + *driver_path = dp; + + return 0; +} + +/* + * A brief comment about slots. I don't know what slots are for; however, + * I have by experimentation determined: + * - Before a device can be bound to pciback, its BDF must first be listed + * in pciback/slots + * - The way to get the BDF listed there is to write BDF to + * pciback/new_slot + * - Writing the same BDF to pciback/new_slot is not idempotent; it results + * in two entries of the BDF in pciback/slots + * It's not clear whether having two entries in pciback/slots is a problem + * or not. Just to be safe, this code does the conservative thing, and + * first checks to see if there is a slot, adding one only if one does not + * already exist. + */ + +/* Scan through /sys/.../pciback/slots looking for pci's BDF */ +static int pciback_dev_has_slot(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func) +{ + FILE *f; + int rc = 0; + unsigned s_domain, s_bus, s_dev, s_func; + + f = fopen(SYSFS_PCIBACK_DRIVER"/slots", "r"); + + if (f == NULL) { + LOGE(ERROR, "Couldn't open %s", SYSFS_PCIBACK_DRIVER"/slots"); + return ERROR_FAIL; + } + + while (fscanf(f, "%x:%x:%x.%d\n", + &s_domain, &s_bus, &s_dev, &s_func) == 4) { + if (s_domain == domain && + s_bus == bus && + s_dev == dev && + s_func == func) { + rc = 1; + goto out; + } + } +out: + fclose(f); + return rc; +} + +static int pciback_dev_assign(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func) +{ + int rc; + + if ( (rc = pciback_dev_has_slot(gc, domain, bus, dev, func)) < 0 ) { + LOGE(ERROR, "Error checking for pciback slot"); + return ERROR_FAIL; + } else if (rc == 0) { + if ( sysfs_write_bdf(gc, SYSFS_PCIBACK_DRIVER"/new_slot", + domain, bus, dev, func) < 0 ) { + LOGE(ERROR, "Couldn't bind device to pciback!"); + return ERROR_FAIL; + } + } + + if ( sysfs_write_bdf(gc, SYSFS_PCIBACK_DRIVER"/bind", + domain, bus, dev, func) < 0 ) { + LOGE(ERROR, "Couldn't bind device to pciback!"); + return ERROR_FAIL; + } + return 0; +} + +static int process_make_assignable(libxl__gc *gc, yajl_gen gen, + char *command_name, + const struct libxl__json_object *request, + struct libxl__json_object **response) +{ + struct stat st; + const struct libxl__json_object *json_o; + unsigned int dom, bus, dev, func; + int rc; + bool rebind; + char *spath, *driver_path = NULL; + + json_o = libxl__json_map_get(PCID_MSG_FIELD_SBDF, request, JSON_STRING); + if (!json_o) { + make_error_reply(gc, gen, "No mandatory parameter 'sbdf'", command_name); + return ERROR_FAIL; + } + + if (sscanf(libxl__json_object_get_string(json_o), PCID_SBDF_FMT, + &dom, &bus, &dev, &func) != 4) { + make_error_reply(gc, gen, "Can't parse SBDF", command_name); + return ERROR_FAIL; + } + + json_o = libxl__json_map_get(PCID_MSG_FIELD_REBIND, request, JSON_BOOL); + if (!json_o) { + make_error_reply(gc, gen, "No mandatory parameter 'rebind'", command_name); + return ERROR_FAIL; + } + + rebind = libxl__json_object_get_bool(json_o); + + /* See if the device exists */ + spath = GCSPRINTF(SYSFS_PCI_DEV"/"PCI_BDF, dom, bus, dev, func); + if ( lstat(spath, &st) ) { + make_error_reply(gc, gen, strerror(errno), command_name); + LOGE(ERROR, "Couldn't lstat %s", spath); + return ERROR_FAIL; + } + + /* Check to see if it's already assigned to pciback */ + rc = pciback_dev_is_assigned(gc, dom, bus, dev, func); + if (rc < 0) { + make_error_reply(gc, gen, "Can't check if device is assigned", + command_name); + return ERROR_FAIL; + } + if (rc) { + LOG(WARN, PCI_BDF" already assigned to pciback", dom, bus, dev, func); + goto done; + } + + /* Check to see if there's already a driver that we need to unbind from */ + if (sysfs_dev_unbind(gc, dom, bus, dev, func, &driver_path)) { + LOG(ERROR, "Couldn't unbind "PCI_BDF" from driver", + dom, bus, dev, func); + return ERROR_FAIL; + } + + /* Store driver_path for rebinding back */ + if (rebind) { + if (driver_path) { + pcid_info_xs_write(gc, dom, bus, dev, func, "driver_path", + driver_path); + } else if ( (driver_path = + pcid_info_xs_read(gc, dom, bus, dev, func, + "driver_path")) != NULL ) { + LOG(INFO, PCI_BDF" not bound to a driver, will be rebound to %s", + dom, bus, dev, func, driver_path); + } else { + LOG(WARN, PCI_BDF" not bound to a driver, will not be rebound.", + dom, bus, dev, func); + } + } else { + pcid_info_xs_remove(gc, dom, bus, dev, func, "driver_path"); + } + + if (pciback_dev_assign(gc, dom, bus, dev, func)) { + LOG(ERROR, "Couldn't bind device to pciback!"); + return ERROR_FAIL; + } + +done: + return 0; +} + +static int pciback_dev_unassign(libxl__gc *gc, unsigned int domain, + unsigned int bus, unsigned int dev, + unsigned int func) +{ + /* Remove from pciback */ + if ( sysfs_dev_unbind(gc, domain, bus, dev, func, NULL) < 0 ) { + LOG(ERROR, "Couldn't unbind device!"); + return ERROR_FAIL; + } + + /* Remove slot if necessary */ + if ( pciback_dev_has_slot(gc, domain, bus, dev, func) > 0 ) { + if ( sysfs_write_bdf(gc, SYSFS_PCIBACK_DRIVER"/remove_slot", + domain, bus, dev, func) < 0 ) { + LOGE(ERROR, "Couldn't remove pciback slot"); + return ERROR_FAIL; + } + } + return 0; +} + +static int process_revert_assignable(libxl__gc *gc, yajl_gen gen, + char *command_name, + const struct libxl__json_object *request, + struct libxl__json_object **response) +{ + const struct libxl__json_object *json_o; + unsigned int dom, bus, dev, func; + int rc; + bool rebind; + char *driver_path = NULL; + + json_o = libxl__json_map_get(PCID_MSG_FIELD_SBDF, request, JSON_STRING); + if (!json_o) { + make_error_reply(gc, gen, "No mandatory parameter 'sbdf'", command_name); + return ERROR_FAIL; + } + + if (sscanf(libxl__json_object_get_string(json_o), PCID_SBDF_FMT, + &dom, &bus, &dev, &func) != 4) { + make_error_reply(gc, gen, "Can't parse SBDF", command_name); + return ERROR_FAIL; + } + + json_o = libxl__json_map_get(PCID_MSG_FIELD_REBIND, request, JSON_BOOL); + if (!json_o) { + make_error_reply(gc, gen, "No mandatory parameter 'rebind'", command_name); + return ERROR_FAIL; + } + + rebind = libxl__json_object_get_bool(json_o); + + /* Unbind from pciback */ + if ( (rc = pciback_dev_is_assigned(gc, dom, bus, dev, func)) < 0 ) { + make_error_reply(gc, gen, "Can't unbind from pciback", command_name); + return ERROR_FAIL; + } else if ( rc ) { + pciback_dev_unassign(gc, dom, bus, dev, func); + } else { + LOG(WARN, "Not bound to pciback"); + } + + /* Rebind if necessary */ + driver_path = pcid_info_xs_read(gc, dom, bus, dev, func, "driver_path"); + + if ( driver_path ) { + if ( rebind ) { + LOG(INFO, "Rebinding to driver at %s", driver_path); + + if ( sysfs_write_bdf(gc, + GCSPRINTF("%s/bind", driver_path), + dom, bus, dev, func) < 0 ) { + LOGE(ERROR, "Couldn't bind device to %s", driver_path); + return -1; + } + + pcid_info_xs_remove(gc, dom, bus, dev, func, "driver_path"); + } + } else { + if ( rebind ) { + LOG(WARN, + "Couldn't find path for original driver; not rebinding"); + } + } + + return 0; +} + static int pcid_handle_request(libxl__gc *gc, yajl_gen gen, const libxl__json_object *request) { @@ -142,6 +532,12 @@ static int pcid_handle_request(libxl__gc *gc, yajl_gen gen, if (strcmp(command_name, PCID_CMD_LIST_ASSIGNABLE) == 0) ret = process_list_assignable(gc, gen, command_name, request, &command_response); + else if (strcmp(command_name, PCID_CMD_MAKE_ASSIGNABLE) == 0) + ret = process_make_assignable(gc, gen, command_name, + request, &command_response); + else if (strcmp(command_name, PCID_CMD_REVERT_ASSIGNABLE) == 0) + ret = process_revert_assignable(gc, gen, command_name, + request, &command_response); else { /* * This is an unsupported command: make a reply and proceed over