diff --git a/app/boards/intel_adsp_ace15_mtpm.conf b/app/boards/intel_adsp_ace15_mtpm.conf index 288410192b81..8eb485b30250 100644 --- a/app/boards/intel_adsp_ace15_mtpm.conf +++ b/app/boards/intel_adsp_ace15_mtpm.conf @@ -83,6 +83,7 @@ CONFIG_WATCHDOG=y CONFIG_TIMESLICE_PER_THREAD=y CONFIG_THREAD_RUNTIME_STATS=y CONFIG_SCHED_THREAD_USAGE=y +CONFIG_RTIO=y # Zephyr / device drivers CONFIG_CLOCK_CONTROL=y diff --git a/app/boards/intel_adsp_ace20_lnl.conf b/app/boards/intel_adsp_ace20_lnl.conf index 2d264293f66f..5ec251eee22a 100644 --- a/app/boards/intel_adsp_ace20_lnl.conf +++ b/app/boards/intel_adsp_ace20_lnl.conf @@ -63,6 +63,7 @@ CONFIG_LLEXT_STORAGE_WRITABLE=y CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y CONFIG_TIMING_FUNCTIONS=y +CONFIG_RTIO=y # Zephyr / device drivers CONFIG_CLOCK_CONTROL=y diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index 6eac290aa74e..bb88c5a98182 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -63,6 +63,7 @@ CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y +CONFIG_RTIO=y # Zephyr / device drivers CONFIG_CLOCK_CONTROL=y diff --git a/src/audio/aria/aria.c b/src/audio/aria/aria.c index 2f558eb2d6d9..ef9ced95adde 100644 --- a/src/audio/aria/aria.c +++ b/src/audio/aria/aria.c @@ -320,10 +320,8 @@ static const struct module_interface aria_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(aria, &aria_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("ARIA", aria_llext_entry, 1, SOF_REG_UUID(aria), 8); + SOF_LLEXT_MODULE_MANIFEST("ARIA", &aria_interface, 1, SOF_REG_UUID(aria), 8); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/asrc/asrc.c b/src/audio/asrc/asrc.c index 2d77a04c380f..281d2f82aaf7 100644 --- a/src/audio/asrc/asrc.c +++ b/src/audio/asrc/asrc.c @@ -892,10 +892,8 @@ static const struct module_interface asrc_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(asrc, &asrc_interface); - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("ASRC", asrc_llext_entry, 1, SOF_REG_UUID(asrc4), 2), + SOF_LLEXT_MODULE_MANIFEST("ASRC", &asrc_interface, 1, SOF_REG_UUID(asrc4), 2), }; SOF_LLEXT_BUILDINFO; diff --git a/src/audio/codec/dts/dts.c b/src/audio/codec/dts/dts.c index cae6aeb0938b..0bc590017004 100644 --- a/src/audio/codec/dts/dts.c +++ b/src/audio/codec/dts/dts.c @@ -473,10 +473,8 @@ static const struct module_interface dts_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(dts, &dts_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("DTS", dts_llext_entry, 1, SOF_REG_UUID(dts), 40); + SOF_LLEXT_MODULE_MANIFEST("DTS", &dts_interface, 1, SOF_REG_UUID(dts), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/crossover/crossover.c b/src/audio/crossover/crossover.c index 7ae8dd47c6a2..d1c0eefa4d7f 100644 --- a/src/audio/crossover/crossover.c +++ b/src/audio/crossover/crossover.c @@ -642,10 +642,8 @@ static const struct module_interface crossover_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(crossover, &crossover_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("XOVER", crossover_llext_entry, 1, SOF_REG_UUID(crossover), 40); + SOF_LLEXT_MODULE_MANIFEST("XOVER", &crossover_interface, 1, SOF_REG_UUID(crossover), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/dcblock/dcblock.c b/src/audio/dcblock/dcblock.c index 1576835d9ee1..d266f357554f 100644 --- a/src/audio/dcblock/dcblock.c +++ b/src/audio/dcblock/dcblock.c @@ -265,10 +265,8 @@ static const struct module_interface dcblock_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(dcblock, &dcblock_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("DCBLOCK", dcblock_llext_entry, 1, SOF_REG_UUID(dcblock), 40); + SOF_LLEXT_MODULE_MANIFEST("DCBLOCK", &dcblock_interface, 1, SOF_REG_UUID(dcblock), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/drc/drc.c b/src/audio/drc/drc.c index 5a6397291eda..6219f54c8842 100644 --- a/src/audio/drc/drc.c +++ b/src/audio/drc/drc.c @@ -424,10 +424,8 @@ static const struct module_interface drc_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(drc, &drc_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("DRC", drc_llext_entry, 1, SOF_REG_UUID(drc), 40); + SOF_LLEXT_MODULE_MANIFEST("DRC", &drc_interface, 1, SOF_REG_UUID(drc), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/eq_fir/eq_fir.c b/src/audio/eq_fir/eq_fir.c index 0269c783a331..ed946157ed4c 100644 --- a/src/audio/eq_fir/eq_fir.c +++ b/src/audio/eq_fir/eq_fir.c @@ -490,10 +490,8 @@ static const struct module_interface eq_fir_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(eq_fir, &eq_fir_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("EQFIR", eq_fir_llext_entry, 1, SOF_REG_UUID(eq_fir), 40); + SOF_LLEXT_MODULE_MANIFEST("EQFIR", &eq_fir_interface, 1, SOF_REG_UUID(eq_fir), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/eq_iir/eq_iir.c b/src/audio/eq_iir/eq_iir.c index 0e7d3a9b541d..4b70fe355c77 100644 --- a/src/audio/eq_iir/eq_iir.c +++ b/src/audio/eq_iir/eq_iir.c @@ -260,10 +260,8 @@ static const struct module_interface eq_iir_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(eq_iir, &eq_iir_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("EQIIR", eq_iir_llext_entry, 1, SOF_REG_UUID(eq_iir), 40); + SOF_LLEXT_MODULE_MANIFEST("EQIIR", &eq_iir_interface, 1, SOF_REG_UUID(eq_iir), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/google/google_ctc_audio_processing.c b/src/audio/google/google_ctc_audio_processing.c index 3e3e12224457..4b01feea1efa 100644 --- a/src/audio/google/google_ctc_audio_processing.c +++ b/src/audio/google/google_ctc_audio_processing.c @@ -466,10 +466,8 @@ static const struct module_interface google_ctc_audio_processing_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(google_ctc_audio_processing, &google_ctc_audio_processing_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("CTC", google_ctc_audio_processing_llext_entry, + SOF_LLEXT_MODULE_MANIFEST("CTC", &google_ctc_audio_processing_interface, 1, SOF_REG_UUID(google_ctc_audio_processing), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/google/google_rtc_audio_processing.c b/src/audio/google/google_rtc_audio_processing.c index f98b2de7322c..4e9a55f39516 100644 --- a/src/audio/google/google_rtc_audio_processing.c +++ b/src/audio/google/google_rtc_audio_processing.c @@ -856,10 +856,8 @@ static const struct module_interface google_rtc_audio_processing_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(google_rtc_audio_processing, &google_rtc_audio_processing_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("RTC_AEC", google_rtc_audio_processing_llext_entry, + SOF_LLEXT_MODULE_MANIFEST("RTC_AEC", &google_rtc_audio_processing_interface, 7, SOF_REG_UUID(google_rtc_audio_processing), 1); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/igo_nr/igo_nr.c b/src/audio/igo_nr/igo_nr.c index 5ab5424cdff3..fd6d900eaf82 100644 --- a/src/audio/igo_nr/igo_nr.c +++ b/src/audio/igo_nr/igo_nr.c @@ -895,10 +895,8 @@ static const struct module_interface igo_nr_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(igo_nr, &igo_nr_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("IGO_NR", igo_nr_llext_entry, 1, SOF_REG_UUID(igo_nr), 40); + SOF_LLEXT_MODULE_MANIFEST("IGO_NR", &igo_nr_interface, 1, SOF_REG_UUID(igo_nr), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/mfcc/mfcc.c b/src/audio/mfcc/mfcc.c index 6437c187e29e..da1fa9f3414e 100644 --- a/src/audio/mfcc/mfcc.c +++ b/src/audio/mfcc/mfcc.c @@ -267,10 +267,8 @@ static const struct module_interface mfcc_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(mfcc, &mfcc_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("MFCC", mfcc_llext_entry, 1, SOF_REG_UUID(mfcc), 40); + SOF_LLEXT_MODULE_MANIFEST("MFCC", &mfcc_interface, 1, SOF_REG_UUID(mfcc), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/mixin_mixout/mixin_mixout.c b/src/audio/mixin_mixout/mixin_mixout.c index 650d790dff7b..5043d02920f0 100644 --- a/src/audio/mixin_mixout/mixin_mixout.c +++ b/src/audio/mixin_mixout/mixin_mixout.c @@ -1020,13 +1020,10 @@ static const struct module_interface mixout_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(mixin, &mixin_interface); -SOF_LLEXT_MOD_ENTRY(mixout, &mixout_interface); - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("MIXIN", mixin_llext_entry, 1, SOF_REG_UUID(mixin), 30), - SOF_LLEXT_MODULE_MANIFEST("MIXOUT", mixout_llext_entry, 1, SOF_REG_UUID(mixout), 30), + SOF_LLEXT_MODULE_MANIFEST("MIXIN", &mixin_interface, 1, SOF_REG_UUID(mixin), 30), + SOF_LLEXT_MODULE_MANIFEST("MIXOUT", &mixout_interface, 1, SOF_REG_UUID(mixout), 30), }; SOF_LLEXT_BUILDINFO; diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 03a345552f71..f8c796764029 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -12,8 +12,13 @@ */ #include - #include +#include +#if CONFIG_IPC_MAJOR_4 +#include +#include +#include +#endif LOG_MODULE_DECLARE(module_adapter, CONFIG_SOF_LOG_LEVEL); @@ -95,7 +100,13 @@ int module_init(struct processing_module *mod) list_init(&md->memory.mem_list); /* Now we can proceed with module specific initialization */ - ret = interface->init(mod); +#if CONFIG_IPC_MAJOR_4 + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + ret = scheduler_dp_rtio_ipc(mod, SOF_IPC4_MOD_INIT_INSTANCE, NULL); + else +#endif + ret = interface->init(mod); + if (ret) { comp_err(dev, "module_init() error %d: module specific init failed, comp id %d", ret, dev_comp_id(dev)); @@ -242,7 +253,27 @@ int module_prepare(struct processing_module *mod, return -EPERM; #endif if (ops->prepare) { - int ret = ops->prepare(mod, sources, num_of_sources, sinks, num_of_sinks); + int ret; + + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { +#if CONFIG_IPC_MAJOR_4 + union scheduler_dp_rtio_ipc_param param = { + .pipeline_state = { + .trigger_cmd = COMP_TRIGGER_PREPARE, + .state = SOF_IPC4_PIPELINE_STATE_RUNNING, + .n_sources = num_of_sources, + .sources = sources, + .n_sinks = num_of_sinks, + .sinks = sinks, + }, + }; + ret = scheduler_dp_rtio_ipc(mod, SOF_IPC4_GLB_SET_PIPELINE_STATE, ¶m); +#else + ret = 0; +#endif + } else { + ret = ops->prepare(mod, sources, num_of_sources, sinks, num_of_sinks); + } if (ret) { comp_err(dev, "module_prepare() error %d: module specific prepare failed, comp_id %d", @@ -366,11 +397,21 @@ int module_reset(struct processing_module *mod) if (md->state < MODULE_IDLE) return 0; #endif - /* cancel task if DP task*/ - if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP && mod->dev->task) - schedule_task_cancel(mod->dev->task); + if (ops->reset) { - ret = ops->reset(mod); + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { +#if CONFIG_IPC_MAJOR_4 + union scheduler_dp_rtio_ipc_param param = { + .pipeline_state.trigger_cmd = COMP_TRIGGER_STOP, + }; + ret = scheduler_dp_rtio_ipc(mod, SOF_IPC4_GLB_SET_PIPELINE_STATE, ¶m); +#else + ret = 0; +#endif + } else { + ret = ops->reset(mod); + } + if (ret) { if (ret != PPL_STATUS_PATH_STOP) comp_err(mod->dev, @@ -423,7 +464,7 @@ int module_free(struct processing_module *mod) struct module_data *md = &mod->priv; int ret = 0; - if (ops->free) { + if (ops->free && mod->dev->ipc_config.proc_domain != COMP_PROCESSING_DOMAIN_DP) { ret = ops->free(mod); if (ret) comp_warn(mod->dev, "error: %d for %d", @@ -570,8 +611,18 @@ int module_bind(struct processing_module *mod, struct bind_info *bind_data) if (ret) return ret; - if (ops->bind) - ret = ops->bind(mod, bind_data); + if (ops->bind) { + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { +#if CONFIG_IPC_MAJOR_4 + union scheduler_dp_rtio_ipc_param param = { + .bind_data = bind_data, + }; + ret = scheduler_dp_rtio_ipc(mod, SOF_IPC4_MOD_BIND, ¶m); +#endif + } else { + ret = ops->bind(mod, bind_data); + } + } return ret; } diff --git a/src/audio/module_adapter/module/waves/waves.c b/src/audio/module_adapter/module/waves/waves.c index 5bb6dcd1a99a..624b53f5f75c 100644 --- a/src/audio/module_adapter/module/waves/waves.c +++ b/src/audio/module_adapter/module/waves/waves.c @@ -913,10 +913,8 @@ static const struct module_interface waves_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(waves, &waves_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("WAVES", waves_llext_entry, 7, SOF_REG_UUID(waves), 8); + SOF_LLEXT_MODULE_MANIFEST("WAVES", &waves_interface, 7, SOF_REG_UUID(waves), 8); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index da0353d892ab..27db61977cfa 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -18,10 +18,16 @@ #include #include #include +#include #include #include #include #include +#if CONFIG_IPC_MAJOR_4 +#include +#include +#include +#endif #include #include #include @@ -102,6 +108,14 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, mod->dev = dev; dev->mod = mod; +#if CONFIG_ZEPHYR_DP_SCHEDULER + /* create a task for DP processing */ + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) { + /* All data allocated, create a thread */ + pipeline_comp_dp_task_init(dev); + } +#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ + list_init(&mod->raw_data_buffers_list); ret = module_adapter_init_data(dev, dst, config, spec); @@ -133,12 +147,6 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, goto err; } -#if CONFIG_ZEPHYR_DP_SCHEDULER - /* create a task for DP processing */ - if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) - pipeline_comp_dp_task_init(dev); -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ - module_adapter_reset_data(dst); dev->state = COMP_STATE_READY; @@ -159,7 +167,12 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, comp_dbg(dev, "module_adapter_new() done"); return dev; + err: +#if CONFIG_ZEPHYR_DP_SCHEDULER + if (dev->task) + schedule_task_free(dev->task); +#endif #if CONFIG_IPC_MAJOR_4 if (mod) rfree(mod->priv.cfg.input_pins); @@ -1200,8 +1213,19 @@ int module_adapter_trigger(struct comp_dev *dev, int cmd) dev->state = COMP_STATE_ACTIVE; return PPL_STATUS_PATH_STOP; } - if (interface->trigger) + + if (interface->trigger) { +#if CONFIG_IPC_MAJOR_4 + if (dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { + /* Process DP module's trigger */ + union scheduler_dp_rtio_ipc_param param = { + .pipeline_state.trigger_cmd = cmd, + }; + return scheduler_dp_rtio_ipc(mod, SOF_IPC4_GLB_SET_PIPELINE_STATE, ¶m); + } +#endif return interface->trigger(mod, cmd); + } return module_adapter_set_state(mod, dev, cmd); } @@ -1263,6 +1287,14 @@ void module_adapter_free(struct comp_dev *dev) comp_dbg(dev, "start"); + if (dev->task) { + /* Run DP module's .free() method in its thread context */ +#if CONFIG_IPC_MAJOR_4 + scheduler_dp_rtio_ipc(mod, SOF_IPC4_MOD_DELETE_INSTANCE, NULL); +#endif + schedule_task_cancel(dev->task); + } + ret = module_free(mod); if (ret) comp_err(dev, "failed with error: %d", ret); diff --git a/src/audio/multiband_drc/multiband_drc.c b/src/audio/multiband_drc/multiband_drc.c index 74cdb0ebcdd4..cca2850cfe12 100644 --- a/src/audio/multiband_drc/multiband_drc.c +++ b/src/audio/multiband_drc/multiband_drc.c @@ -441,10 +441,8 @@ static const struct module_interface multiband_drc_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(multiband_drc, &multiband_drc_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("MB_DRC", multiband_drc_llext_entry, 1, + SOF_LLEXT_MODULE_MANIFEST("MB_DRC", &multiband_drc_interface, 1, SOF_REG_UUID(multiband_drc), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/mux/mux.c b/src/audio/mux/mux.c index a4313e45f4ea..bbcf1b544e90 100644 --- a/src/audio/mux/mux.c +++ b/src/audio/mux/mux.c @@ -479,19 +479,12 @@ static const struct module_interface demux_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(mux, &mux_interface); - -/* - * The demux entry is removed because mtl.toml doesn't have an entry - * for it. Once that is fixed, the manifest line below can be - * re-activated: - * SOF_LLEXT_MOD_ENTRY(demux, &demux_interface); - */ - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("MUX", mux_llext_entry, 1, SOF_REG_UUID(mux4), 15), + SOF_LLEXT_MODULE_MANIFEST("MUX", &mux_interface, 1, SOF_REG_UUID(mux4), 15), /* - * See comment above for a demux deactivation reason + * The demux entry is removed because mtl.toml doesn't have an entry + * for it. Once that is fixed, the manifest line below can be + * re-activated: * SOF_LLEXT_MODULE_MANIFEST("DEMUX", demux_llext_entry, 1, SOF_REG_UUID(demux), 15), */ }; diff --git a/src/audio/rtnr/rtnr.c b/src/audio/rtnr/rtnr.c index 0078ce275b33..541f985753a1 100644 --- a/src/audio/rtnr/rtnr.c +++ b/src/audio/rtnr/rtnr.c @@ -880,10 +880,8 @@ static const struct module_interface rtnr_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(rtnr, &rtnr_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("RTNR", rtnr_llext_entry, 1, SOF_REG_UUID(rtnr), 40); + SOF_LLEXT_MODULE_MANIFEST("RTNR", &rtnr_interface, 1, SOF_REG_UUID(rtnr), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/selector/selector.c b/src/audio/selector/selector.c index d0f52a92303b..201ed9fa6567 100644 --- a/src/audio/selector/selector.c +++ b/src/audio/selector/selector.c @@ -930,11 +930,8 @@ static const struct module_interface selector_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(selector, &selector_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("MICSEL", selector_llext_entry, 1, SOF_REG_UUID(selector4), - 8); + SOF_LLEXT_MODULE_MANIFEST("MICSEL", &selector_interface, 1, SOF_REG_UUID(selector4), 8); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/src/src.c b/src/audio/src/src.c index 7f744f64d43d..1b3d6b3d3ca8 100644 --- a/src/audio/src/src.c +++ b/src/audio/src/src.c @@ -78,8 +78,6 @@ static const struct module_interface src_interface = { .prepare = src_prepare, .process = src_process, .is_ready_to_process = src_is_ready_to_process, - .set_configuration = src_set_config, - .get_configuration = src_get_config, .reset = src_reset, .free = src_free, }; @@ -91,18 +89,14 @@ static const struct module_interface src_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(src, &src_interface); - #if CONFIG_COMP_SRC_LITE extern const struct module_interface src_lite_interface; -SOF_LLEXT_MOD_ENTRY(src_lite, &src_lite_interface); #endif static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("SRC", src_llext_entry, 1, SOF_REG_UUID(src4), 1), + SOF_LLEXT_MODULE_MANIFEST("SRC", &src_interface, 1, SOF_REG_UUID(src4), 1), #if CONFIG_COMP_SRC_LITE - SOF_LLEXT_MODULE_MANIFEST("SRC_LITE", src_lite_llext_entry, 1, SOF_REG_UUID(src_lite), - 1), + SOF_LLEXT_MODULE_MANIFEST("SRC_LITE", &src_lite_interface, 1, SOF_REG_UUID(src_lite), 1), #endif }; diff --git a/src/audio/src/src_common.c b/src/audio/src/src_common.c index f96b3bc6e358..cd9bbf97412f 100644 --- a/src/audio/src/src_common.c +++ b/src/audio/src/src_common.c @@ -672,24 +672,6 @@ int src_process(struct processing_module *mod, return cd->src_func(cd, sources[0], sinks[0]); } -__cold int src_set_config(struct processing_module *mod, uint32_t config_id, - enum module_cfg_fragment_position pos, uint32_t data_offset_size, - const uint8_t *fragment, size_t fragment_size, uint8_t *response, - size_t response_size) -{ - assert_can_be_cold(); - - return -EINVAL; -} - -__cold int src_get_config(struct processing_module *mod, uint32_t config_id, - uint32_t *data_offset_size, uint8_t *fragment, size_t fragment_size) -{ - assert_can_be_cold(); - - return -EINVAL; -} - int src_reset(struct processing_module *mod) { struct comp_data *cd = module_get_private_data(mod); diff --git a/src/audio/src/src_common.h b/src/audio/src/src_common.h index 6a8028662ffd..ae3c60574eec 100644 --- a/src/audio/src/src_common.h +++ b/src/audio/src/src_common.h @@ -242,12 +242,6 @@ int src_process(struct processing_module *mod, struct sof_source **sources, int num_of_sources, struct sof_sink **sinks, int num_of_sinks); -int src_set_config(struct processing_module *mod, uint32_t config_id, - enum module_cfg_fragment_position pos, uint32_t data_offset_size, - const uint8_t *fragment, size_t fragment_size, uint8_t *response, - size_t response_size); -int src_get_config(struct processing_module *mod, uint32_t config_id, - uint32_t *data_offset_size, uint8_t *fragment, size_t fragment_size); int src_free(struct processing_module *mod); int src_reset(struct processing_module *mod); extern struct tr_ctx src_tr; diff --git a/src/audio/src/src_lite.c b/src/audio/src/src_lite.c index 4cbedea76ad2..d6704374d8cc 100644 --- a/src/audio/src/src_lite.c +++ b/src/audio/src/src_lite.c @@ -61,8 +61,6 @@ const struct module_interface src_lite_interface = { .prepare = src_lite_prepare, .process = src_process, .is_ready_to_process = src_is_ready_to_process, - .set_configuration = src_set_config, - .get_configuration = src_get_config, .reset = src_reset, .free = src_free, }; diff --git a/src/audio/tdfb/tdfb.c b/src/audio/tdfb/tdfb.c index e849b1dd0781..9fc04820ff75 100644 --- a/src/audio/tdfb/tdfb.c +++ b/src/audio/tdfb/tdfb.c @@ -833,10 +833,8 @@ static const struct module_interface tdfb_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(tdfb, &tdfb_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("TDFB", tdfb_llext_entry, 1, SOF_REG_UUID(tdfb), 40); + SOF_LLEXT_MODULE_MANIFEST("TDFB", &tdfb_interface, 1, SOF_REG_UUID(tdfb), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/template_comp/template.c b/src/audio/template_comp/template.c index 7b15f1fbf591..fcd3fa294d03 100644 --- a/src/audio/template_comp/template.c +++ b/src/audio/template_comp/template.c @@ -197,10 +197,8 @@ static const struct module_interface template_comp_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(template_comp, &template_comp_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("TEMPLATE", template_comp_llext_entry, 1, + SOF_LLEXT_MODULE_MANIFEST("TEMPLATE", &template_comp_interface, 1, SOF_REG_UUID(template_comp), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/tensorflow/tflm-classify.c b/src/audio/tensorflow/tflm-classify.c index d24099757c2d..29b4b6a1af74 100644 --- a/src/audio/tensorflow/tflm-classify.c +++ b/src/audio/tensorflow/tflm-classify.c @@ -243,10 +243,8 @@ SOF_MODULE_INIT(tflmcly, sys_comp_module_tflmcly_interface_init); #include #include -SOF_LLEXT_MOD_ENTRY(tflmcly, &tflmcly_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("TFLMCLY", tflmcly_llext_entry, 1, SOF_REG_UUID(tflmcly), 40); + SOF_LLEXT_MODULE_MANIFEST("TFLMCLY", &tflmcly_interface, 1, SOF_REG_UUID(tflmcly), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/volume/volume.c b/src/audio/volume/volume.c index 50aa4a4750d5..9f8aff265226 100644 --- a/src/audio/volume/volume.c +++ b/src/audio/volume/volume.c @@ -812,20 +812,12 @@ static const struct module_interface gain_interface = { #include #include -#if CONFIG_COMP_PEAK_VOL -SOF_LLEXT_MOD_ENTRY(peakvol, &volume_interface); -#endif - -#if CONFIG_COMP_GAIN -SOF_LLEXT_MOD_ENTRY(gain, &gain_interface); -#endif - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { #if CONFIG_COMP_PEAK_VOL - SOF_LLEXT_MODULE_MANIFEST("PEAKVOL", peakvol_llext_entry, 1, SOF_REG_UUID(volume4), 10), + SOF_LLEXT_MODULE_MANIFEST("PEAKVOL", &volume_interface, 1, SOF_REG_UUID(volume4), 10), #endif #if CONFIG_COMP_GAIN - SOF_LLEXT_MODULE_MANIFEST("GAIN", gain_llext_entry, 1, SOF_REG_UUID(gain), 40), + SOF_LLEXT_MODULE_MANIFEST("GAIN", &gain_interface, 1, SOF_REG_UUID(gain), 40), #endif }; diff --git a/src/debug/tester/tester.c b/src/debug/tester/tester.c index 73686ff49652..869b7d9b2a7a 100644 --- a/src/debug/tester/tester.c +++ b/src/debug/tester/tester.c @@ -243,10 +243,8 @@ static const struct module_interface tester_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(tester, &tester_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("TESTER", tester_llext_entry, 1, SOF_REG_UUID(tester), 40); + SOF_LLEXT_MODULE_MANIFEST("TESTER", &tester_interface, 1, SOF_REG_UUID(tester), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/include/module/module/base.h b/src/include/module/module/base.h index 2371d77124f5..b5b46b6a5054 100644 --- a/src/include/module/module/base.h +++ b/src/include/module/module/base.h @@ -70,6 +70,8 @@ enum module_processing_type { MODULE_PROCESS_TYPE_RAW, }; +struct rtio_sqe; + /* * A pointer to this structure is passed to module API functions (from struct module_interface). * This structure should contain only fields that should be available to a module. @@ -103,6 +105,10 @@ struct processing_module { uint32_t num_of_sources; uint32_t num_of_sinks; + /* DP module RTIO SQE pointers, used to signal module's DP thread */ + struct rtio_sqe *ipc_sqe; + struct rtio_sqe *audio_sqe; + /* sink and source handlers for the module */ struct sof_sink *sinks[CONFIG_MODULE_MAX_CONNECTIONS]; struct sof_source *sources[CONFIG_MODULE_MAX_CONNECTIONS]; diff --git a/src/include/module/module/llext.h b/src/include/module/module/llext.h index 7b5d00feeb30..2f05cb6c692a 100644 --- a/src/include/module/module/llext.h +++ b/src/include/module/module/llext.h @@ -33,13 +33,6 @@ } \ } -#define SOF_LLEXT_MOD_ENTRY(name, interface) \ -static const struct module_interface *name##_llext_entry(void *mod_cfg, \ - void *parent_ppl, void **mod_ptr) \ -{ \ - return interface; \ -} - #define SOF_LLEXT_BUILDINFO \ static const struct sof_module_api_build_info buildinfo __section(".mod_buildinfo") __used = { \ .format = SOF_MODULE_API_BUILD_INFO_FORMAT, \ diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 3370cc110d2e..b41a8e679b35 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -46,6 +46,8 @@ static inline void comp_free(struct comp_dev *dev) { assert(dev->drv->ops.free); + dev->drv->ops.free(dev); + /* free task if shared component or DP task*/ if ((dev->is_shared || dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) && dev->task) { @@ -53,8 +55,6 @@ static inline void comp_free(struct comp_dev *dev) rfree(dev->task); dev->task = NULL; } - - dev->drv->ops.free(dev); } /** @@ -164,7 +164,6 @@ static inline int comp_trigger_local(struct comp_dev *dev, int cmd) case COMP_TRIGGER_XRUN: case COMP_TRIGGER_PAUSE: case COMP_TRIGGER_STOP: - schedule_task_cancel(dev->task); break; } } diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 0a064204e8d1..3b6472a3e6b2 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include struct processing_module; @@ -84,4 +86,31 @@ int scheduler_dp_task_init(struct task **task, void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_t *data_off_size); +struct bind_info; +struct sof_source; +struct sof_sink; +union scheduler_dp_rtio_ipc_param { + struct bind_info *bind_data; + struct { + unsigned int trigger_cmd; + enum ipc4_pipeline_state state; + int n_sources; + struct sof_source **sources; + int n_sinks; + struct sof_sink **sinks; + } pipeline_state; +}; + +#if CONFIG_ZEPHYR_DP_SCHEDULER +int scheduler_dp_rtio_ipc(struct processing_module *pmod, enum sof_ipc4_module_type cmd, + union scheduler_dp_rtio_ipc_param *param); +#else +static inline int scheduler_dp_rtio_ipc(struct processing_module *pmod, + enum sof_ipc4_module_type cmd, + union scheduler_dp_rtio_ipc_param *param) +{ + return 0; +} +#endif + #endif /* __SOF_SCHEDULE_DP_SCHEDULE_H__ */ diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index c153c61a308b..fbf87ebdb142 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -603,8 +603,18 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, mod = (const struct sof_man_module *) ((const uint8_t *)desc + SOF_MAN_MODULE_OFFSET(entry_index)); + const uintptr_t module_entry_point = lib_manager_allocate_module(mod, config, args->data); + + if (!module_entry_point) { + tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); + return NULL; + } + switch (lib_manager_get_module_type(desc, mod)) { case MOD_TYPE_LLEXT: + agent = NULL; + ops = (const struct module_interface *)module_entry_point; + break; case MOD_TYPE_LMDK: agent = &native_system_agent_start; agent_iface = (const void **)&ops; @@ -617,21 +627,16 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, agent_iface = (const void **)&adapter_priv; break; case MOD_TYPE_INVALID: - return NULL; - } - - const uintptr_t module_entry_point = lib_manager_allocate_module(mod, config, args->data); - if (!module_entry_point) { - tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); - return NULL; + goto err; } /* At this point module resources are allocated and it is moved to L2 memory. */ - - ret = lib_manager_start_agent(drv, config->id, args, module_entry_point, agent, - agent_iface); - if (ret) - goto err; + if (agent) { + ret = lib_manager_start_agent(drv, config->id, args, module_entry_point, agent, + agent_iface); + if (ret) + goto err; + } if (ops && comp_set_adapter_ops(drv, ops) < 0) goto err; diff --git a/src/probe/probe.c b/src/probe/probe.c index 7a9cb4121573..339935993ef1 100644 --- a/src/probe/probe.c +++ b/src/probe/probe.c @@ -1660,10 +1660,8 @@ static const struct module_interface probe_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(probe, &probe_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("PROBE", probe_llext_entry, 1, SOF_REG_UUID(probe4), 40); + SOF_LLEXT_MODULE_MANIFEST("PROBE", &probe_interface, 1, SOF_REG_UUID(probe4), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/samples/audio/smart_amp_test_ipc4.c b/src/samples/audio/smart_amp_test_ipc4.c index 53f42b43f03b..52e92699e669 100644 --- a/src/samples/audio/smart_amp_test_ipc4.c +++ b/src/samples/audio/smart_amp_test_ipc4.c @@ -381,10 +381,8 @@ static const struct module_interface smart_amp_test_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(smart_amp_test, &smart_amp_test_interface); - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("SMATEST", smart_amp_test_llext_entry, 1, + SOF_LLEXT_MODULE_MANIFEST("SMATEST", &smart_amp_test_interface, 1, SOF_REG_UUID(smart_amp_test), 1), }; diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index cbb36391b310..6a0517eed50c 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -15,18 +15,28 @@ #include #include #include +#include #include #include #include #include #include +#include LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); SOF_DEFINE_REG_UUID(dp_sched); DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); +/* RTIO context with 2 submit and 2 complete entries for IPC and audio */ +// FIXME: should be per-DP task +RTIO_DEFINE(dp_consumer, 2, 2); +K_SEM_DEFINE(dp_rtio_ipc_sem, 0, K_SEM_MAX_LIMIT); +K_SEM_DEFINE(dp_rtio_sync_sem, 0, 1); + +static struct k_spinlock rtio_lock; + struct scheduler_dp_data { struct list_item tasks; /* list of active dp tasks */ struct task ll_tick_src; /* LL task - source of DP tick */ @@ -38,7 +48,6 @@ struct task_dp_pdata { uint32_t deadline_clock_ticks; /* dp module deadline in Zephyr ticks */ k_thread_stack_t __sparse_cache *p_stack; /* pointer to thread stack */ size_t stack_size; /* size of the stack in bytes */ - struct k_sem sem; /* semaphore for task scheduling */ struct processing_module *mod; /* the module to be scheduled */ uint32_t ll_cycles_to_start; /* current number of LL cycles till delayed start */ }; @@ -242,33 +251,49 @@ void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void * mod->dp_startup_delay = false; } - if (curr_task->state == SOF_TASK_STATE_QUEUED) { - bool mod_ready; - - mod_ready = module_is_ready_to_process(mod, mod->sources, - mod->num_of_sources, - mod->sinks, - mod->num_of_sinks); - if (mod_ready) { - /* set a deadline for given num of ticks, starting now */ - k_thread_deadline_set(pdata->thread_id, - pdata->deadline_clock_ticks); - - /* trigger the task */ - curr_task->state = SOF_TASK_STATE_RUNNING; - k_sem_give(&pdata->sem); - } + if (curr_task->state != SOF_TASK_STATE_QUEUED || + mod->dev->state < COMP_STATE_ACTIVE) + continue; + + /* set a deadline for given num of ticks, starting now */ + k_thread_deadline_set(pdata->thread_id, + pdata->deadline_clock_ticks); + + /* trigger the task */ + curr_task->state = SOF_TASK_STATE_RUNNING; + + k_spinlock_key_t key = k_spin_lock(&rtio_lock); + + struct rtio_sqe *sqe = mod->audio_sqe; + + if (sqe) { + mod->audio_sqe = NULL; + rtio_iodev_sqe_ok(CONTAINER_OF(sqe, struct rtio_iodev_sqe, sqe), 0); } + k_spin_unlock(&rtio_lock, key); } scheduler_dp_unlock(lock_key); } +static void dp_rtio_drop_all(struct rtio *r) +{ + struct rtio_sqe_pool *pool = r->sqe_pool; + + mpsc_init(&pool->free_q); + for (unsigned int i = 0; i < pool->pool_size; i++) + mpsc_push(&pool->free_q, &pool->pool[i].q); + + pool->pool_free = pool->pool_size; +} + static int scheduler_dp_task_cancel(void *data, struct task *task) { unsigned int lock_key; struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; struct task_dp_pdata *pdata = task->priv_data; + if (!pdata->thread_id) + return 0; /* this is asyn cancel - mark the task as canceled and remove it from scheduling */ lock_key = scheduler_dp_lock(); @@ -276,17 +301,21 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) task->state = SOF_TASK_STATE_CANCEL; list_item_del(&task->list); - /* if there're no more DP task, stop LL tick source */ - if (list_is_empty(&dp_sch->tasks)) + /* if there're no more DP task, stop LL tick source */ + if (list_is_empty(&dp_sch->tasks)) { schedule_task_cancel(&dp_sch->ll_tick_src); + /* Move all SQEs to the free list */ + dp_rtio_drop_all(&dp_consumer); + } + /* if the task is waiting on a semaphore - let it run and self-terminate */ - k_sem_give(&pdata->sem); scheduler_dp_unlock(lock_key); /* wait till the task has finished, if there was any task created */ - if (pdata->thread_id) - k_thread_join(pdata->thread_id, K_FOREVER); + k_thread_abort(pdata->thread_id); + + pdata->thread_id = NULL; return 0; } @@ -297,13 +326,7 @@ static int scheduler_dp_task_free(void *data, struct task *task) scheduler_dp_task_cancel(data, task); - /* the thread should be terminated at this moment, - * abort is safe and will ensure no use after free - */ - if (pdata->thread_id) { - k_thread_abort(pdata->thread_id); - pdata->thread_id = NULL; - } + pdata->thread_id = NULL; /* free task stack */ rfree((__sparse_force void *)pdata->p_stack); @@ -313,52 +336,322 @@ static int scheduler_dp_task_free(void *data, struct task *task) return 0; } +/* TODO: make this a shared kernel->module buffer for IPC parameters */ +static uint8_t ipc_buf[4096]; + +struct ipc4_mod_bind { + struct ipc4_module_bind_unbind bu; + enum bind_type type; +}; + +struct ipc4_flat { + unsigned int cmd; + int ret; + union { + struct ipc4_mod_bind bind; + struct { + unsigned int trigger_cmd; + enum ipc4_pipeline_state state; + int n_sources; + int n_sinks; + void *source_sink[]; + } pipeline_state; + }; +}; + +/* Pack IPC input data */ +static int ipc_rtio_flatten(unsigned int cmd, union scheduler_dp_rtio_ipc_param *param, + struct rtio_iodev_sqe *iodev_sqe, struct ipc4_flat *flat) +{ + flat->cmd = cmd; + + /* + * FIXME: SOF_IPC4_MOD_* and SOF_IPC4_GLB_* aren't fully orthogonal, but + * so far none of the used ones overlap + */ + switch (cmd) { + case SOF_IPC4_MOD_BIND: + flat->bind.bu = *param->bind_data->ipc4_data; + flat->bind.type = param->bind_data->bind_type; + break; + case SOF_IPC4_GLB_SET_PIPELINE_STATE: + flat->pipeline_state.trigger_cmd = param->pipeline_state.trigger_cmd; + switch (param->pipeline_state.trigger_cmd) { + case COMP_TRIGGER_STOP: + break; + case COMP_TRIGGER_PREPARE: + if (sizeof(flat->cmd) + sizeof(flat->ret) + sizeof(flat->pipeline_state) + + sizeof(void *) * (param->pipeline_state.n_sources + + param->pipeline_state.n_sinks) > + sizeof(ipc_buf)) + return -ENOMEM; + + flat->pipeline_state.state = param->pipeline_state.state; + flat->pipeline_state.n_sources = param->pipeline_state.n_sources; + flat->pipeline_state.n_sinks = param->pipeline_state.n_sinks; + memcpy(flat->pipeline_state.source_sink, param->pipeline_state.sources, + flat->pipeline_state.n_sources * + sizeof(flat->pipeline_state.source_sink[0])); + memcpy(flat->pipeline_state.source_sink + flat->pipeline_state.n_sources, + param->pipeline_state.sinks, + flat->pipeline_state.n_sinks * + sizeof(flat->pipeline_state.source_sink[0])); + } + } + + return 0; +} + +/* Unpack IPC data and execute a callback */ +static void ipc_rtio_unflatten_run(struct processing_module *pmod, struct ipc4_flat *flat) +{ + const struct module_interface *const ops = pmod->dev->drv->adapter_ops; + + switch (flat->cmd) { + case SOF_IPC4_MOD_BIND: + if (ops->bind) { + struct bind_info bind_data = { + .ipc4_data = &flat->bind.bu, + .bind_type = flat->bind.type, + }; + + flat->ret = ops->bind(pmod, &bind_data); + } else { + flat->ret = 0; + } + break; + case SOF_IPC4_MOD_DELETE_INSTANCE: + flat->ret = ops->free(pmod); + break; + case SOF_IPC4_MOD_INIT_INSTANCE: + flat->ret = ops->init(pmod); + break; + case SOF_IPC4_GLB_SET_PIPELINE_STATE: + switch (flat->pipeline_state.trigger_cmd) { + case COMP_TRIGGER_STOP: + flat->ret = ops->reset(pmod); + break; + case COMP_TRIGGER_PREPARE: + flat->ret = ops->prepare(pmod, + (struct sof_source **)flat->pipeline_state.source_sink, + flat->pipeline_state.n_sources, + (struct sof_sink **)(flat->pipeline_state.source_sink + + flat->pipeline_state.n_sources), + flat->pipeline_state.n_sinks); + } + } +} + +/* Signal an IPC and wait for processing completion */ +int scheduler_dp_rtio_ipc(struct processing_module *pmod, enum sof_ipc4_module_type cmd, + union scheduler_dp_rtio_ipc_param *param) +{ + int ret; + + if (!pmod) { + tr_err(&dp_tr, "no RTIO mod"); + return -EINVAL; + } + + if (cmd == SOF_IPC4_MOD_INIT_INSTANCE) { + /* Wait for the DP thread to start */ + ret = k_sem_take(&dp_rtio_sync_sem, K_MSEC(100)); + if (ret == -EAGAIN) + return -ETIMEDOUT; + } + + k_spinlock_key_t key = k_spin_lock(&rtio_lock); + + struct rtio_sqe *sqe = pmod->ipc_sqe; + struct ipc4_flat *flat = (struct ipc4_flat *)ipc_buf; + + if (!sqe) { + tr_err(&dp_tr, "no RTIO on %p", pmod); + ret = -ENOENT; + } else { + /* IPCs are serialised */ + flat->ret = -ENOSYS; + + pmod->ipc_sqe = NULL; + + struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(sqe, struct rtio_iodev_sqe, sqe); + + tr_dbg(&dp_tr, "RTIO signal %p IPC cmd %d trig %d", pmod, cmd, + param ? param->pipeline_state.trigger_cmd : -EINVAL); + + ret = ipc_rtio_flatten(cmd, param, iodev_sqe, flat); + if (!ret) + rtio_iodev_sqe_ok(iodev_sqe, 0); + } + + k_spin_unlock(&rtio_lock, key); + + if (sqe && !ret) { + k_sem_take(&dp_rtio_ipc_sem, K_FOREVER); + ret = flat->ret; + } + + return ret; +} + +static void producer_submit(struct rtio_iodev_sqe *iodev_sqe) +{ +} + +/* An API and a .submit method are compulsory */ +const struct rtio_iodev_api producer_api = {.submit = producer_submit}; + +/* As long as .data == NULL one RTIO IO-device should suffice for all DP tasks */ +RTIO_IODEV_DEFINE(producer_iodev, &producer_api, NULL); + +enum sof_rtio_read_id { + IPC_READ_SQE, + AUDIO_READ_SQE, + N_READ_SQE, +}; + +struct sof_user_data { + enum sof_rtio_read_id id; +}; + +static struct rtio_sqe *zephyr_dp_rtio_handle(struct rtio_iodev_sqe *iodev_sqe, + struct sof_user_data *udata, bool *new) +{ + struct rtio_sqe *sqe_handle = NULL; + + rtio_sqe_prep_read(&iodev_sqe->sqe, &producer_iodev, + RTIO_PRIO_NORM, NULL, 0, udata); + + /* Copy sqe into the kernel mode and get a handle out */ + int ret = rtio_sqe_copy_in_get_handles(&dp_consumer, &iodev_sqe->sqe, + &sqe_handle, 1); + if (ret < 0) + tr_warn(&dp_tr, "DP RTIO: no SQE handle!"); + else + *new = false; + + return sqe_handle; +} + /* Thread function called in component context, on target core */ static void dp_thread_fn(void *p1, void *p2, void *p3) { struct task *task = p1; - (void)p2; - (void)p3; struct task_dp_pdata *task_pdata = task->priv_data; + struct processing_module *pmod = task_pdata->mod; + bool ipc_new = true, audio_new = true, first = true; + struct sof_user_data sof_rtio_ipc = {IPC_READ_SQE,}; + struct sof_user_data sof_rtio_audio = {AUDIO_READ_SQE,}; + struct rtio_iodev_sqe mod_sqe[N_READ_SQE]; unsigned int lock_key; enum task_state state; bool task_stop; + ARG_UNUSED(p2); + ARG_UNUSED(p3); + + for (unsigned int i = 0; i < N_READ_SQE; i++) { + mod_sqe[i].next = NULL; + mod_sqe[i].r = &dp_consumer; + mod_sqe[i].q.next = NULL; + } + do { + k_spinlock_key_t key = k_spin_lock(&rtio_lock); + bool ready; + + /* prepare and submit SQEs */ + if (audio_new) + pmod->audio_sqe = zephyr_dp_rtio_handle(mod_sqe + AUDIO_READ_SQE, + &sof_rtio_audio, &audio_new); + if (ipc_new) + pmod->ipc_sqe = zephyr_dp_rtio_handle(mod_sqe + IPC_READ_SQE, + &sof_rtio_ipc, &ipc_new); + + k_spin_unlock(&rtio_lock, key); + + /* Submit SQEs without waiting */ + rtio_submit(&dp_consumer, 0); + + if (first) { + /* + * The IPC RTIO completion function is waiting for SQEs to be submitted, + * it can proceed now. + */ + first = false; + k_sem_give(&dp_rtio_sync_sem); + } + /* - * the thread is started immediately after creation, it will stop on semaphore - * Semaphore will be released once the task is ready to process + * The thread is started immediately after creation, it stops on + * RTIO. RTIO is signaled once the task is ready to process */ - k_sem_take(&task_pdata->sem, K_FOREVER); + struct rtio_cqe *cqe = rtio_cqe_consume_block(&dp_consumer); + struct sof_user_data *completion = cqe->userdata; - if (task->state == SOF_TASK_STATE_RUNNING) - state = task_run(task); - else - state = task->state; /* to avoid undefined variable warning */ + rtio_cqe_release(&dp_consumer, cqe); - lock_key = scheduler_dp_lock(); - /* - * check if task is still running, may have been canceled by external call - * if not, set the state returned by run procedure - */ - if (task->state == SOF_TASK_STATE_RUNNING) { - task->state = state; - switch (state) { - case SOF_TASK_STATE_RESCHEDULE: - /* mark to reschedule, schedule time is already calculated */ - task->state = SOF_TASK_STATE_QUEUED; - break; + if (!completion) { + tr_warn(&dp_tr, "No RTIO completion"); + continue; + } - case SOF_TASK_STATE_CANCEL: - case SOF_TASK_STATE_COMPLETED: - /* remove from scheduling */ - list_item_del(&task->list); - break; + switch (completion->id) { + case IPC_READ_SQE: + /* handle IPC */ + tr_dbg(&dp_tr, "got IPC RTIO for %p state %d", pmod, task->state); + ipc_new = true; + ipc_rtio_unflatten_run(pmod, (struct ipc4_flat *)ipc_buf); + k_sem_give(&dp_rtio_ipc_sem); + + lock_key = scheduler_dp_lock(); + break; + case AUDIO_READ_SQE: + /* handle audio */ + audio_new = true; + + ready = module_is_ready_to_process(pmod, + pmod->sources, pmod->num_of_sources, + pmod->sinks, pmod->num_of_sinks); + + if (ready) { + if (task->state == SOF_TASK_STATE_RUNNING) + state = task_run(task); + else + state = task->state; /* to avoid undefined variable warning */ + } - default: - /* illegal state, serious defect, won't happen */ - k_panic(); + lock_key = scheduler_dp_lock(); + + /* + * check if task is still running, may have been canceled by external call + * if not, set the state returned by run procedure + */ + if (ready && task->state == SOF_TASK_STATE_RUNNING) { + task->state = state; + switch (state) { + case SOF_TASK_STATE_RESCHEDULE: + /* mark to reschedule, schedule time is already calculated */ + task->state = SOF_TASK_STATE_QUEUED; + break; + + case SOF_TASK_STATE_CANCEL: + case SOF_TASK_STATE_COMPLETED: + /* remove from scheduling */ + list_item_del(&task->list); + break; + + default: + /* illegal state, serious defect, won't happen */ + k_panic(); + } + } else { + task->state = SOF_TASK_STATE_QUEUED; } + break; + default: + tr_err(&dp_tr, "Invalid RTIO completion %d", completion->id); + continue; } /* if true exit the while loop, terminate the thread */ @@ -380,7 +673,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta struct task_dp_pdata *pdata = task->priv_data; unsigned int lock_key; uint64_t deadline_clock_ticks; - int ret; lock_key = scheduler_dp_lock(); @@ -391,22 +683,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return -EINVAL; } - /* create a zephyr thread for the task */ - pdata->thread_id = k_thread_create(&pdata->thread, (__sparse_force void *)pdata->p_stack, - pdata->stack_size, dp_thread_fn, task, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, K_USER, K_FOREVER); - - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(pdata->thread_id, task->core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto err; - } - - /* start the thread, it should immediately stop at a semaphore, so clean it */ - k_sem_reset(&pdata->sem); - k_thread_start(pdata->thread_id); - /* if there's no DP tasks scheduled yet, run ll tick source task */ if (list_is_empty(&dp_sch->tasks)) schedule_task(&dp_sch->ll_tick_src, 0, 0); @@ -429,11 +705,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta tr_dbg(&dp_tr, "DP task scheduled with period %u [us]", (uint32_t)period); return 0; -err: - /* cleanup - unlock and free all allocated resources */ - scheduler_dp_unlock(lock_key); - k_thread_abort(pdata->thread_id); - return ret; } static struct scheduler_ops schedule_dp_ops = { @@ -450,6 +721,8 @@ int scheduler_dp_init(void) if (!dp_sch) return -ENOMEM; + k_spinlock_init(&rtio_lock); + list_init(&dp_sch->tasks); scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); @@ -521,24 +794,39 @@ int scheduler_dp_task_init(struct task **task, goto err; } + struct task_dp_pdata *pdata = &task_memory->pdata; + /* initialize other task structures */ task_memory->task.ops.complete = ops->complete; task_memory->task.ops.get_deadline = ops->get_deadline; task_memory->task.state = SOF_TASK_STATE_INIT; task_memory->task.core = core; - - /* initialize semaprhore */ - k_sem_init(&task_memory->pdata.sem, 0, 1); + task_memory->task.priv_data = pdata; /* success, fill the structures */ - task_memory->task.priv_data = &task_memory->pdata; - task_memory->pdata.p_stack = p_stack; - task_memory->pdata.stack_size = stack_size; - task_memory->pdata.mod = mod; + pdata->p_stack = p_stack; + pdata->stack_size = stack_size; + pdata->mod = mod; *task = &task_memory->task; + /* create a zephyr thread for the task */ + pdata->thread_id = k_thread_create(&pdata->thread, (__sparse_force void *)p_stack, + stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, K_USER, K_FOREVER); + + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(pdata->thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr_dp_task_init(): zephyr task pin to core failed"); + goto e_thread; + } + + k_thread_start(pdata->thread_id); return 0; + +e_thread: + k_thread_abort(pdata->thread_id); err: /* cleanup - free all allocated resources */ rfree((__sparse_force void *)p_stack);