diff --git a/Kconfig b/Kconfig index 8cff91401937..0ec0e0dbff91 100644 --- a/Kconfig +++ b/Kconfig @@ -85,16 +85,16 @@ config HW_LLI Any platforms with hardware linked list support should set this. -config DMA_AGGREGATED_IRQ +config DW_DMA_AGGREGATED_IRQ bool default n help - Some platforms cannot register interrupt per DMA channel + Some platforms cannot register interrupt per DW-DMA channel and have the possibility only to register interrupts per DMA controller, which require manual handling of aggregated irq. - Any platforms with DMA aggregated interrupts support + Any platforms with DW-DMA aggregated interrupts support should set this. config DMA_SUSPEND_DRAIN diff --git a/src/arch/xtensa/init.c b/src/arch/xtensa/init.c index 3447cd2846ba..c1016f5a7148 100644 --- a/src/arch/xtensa/init.c +++ b/src/arch/xtensa/init.c @@ -132,6 +132,7 @@ int slave_core_init(struct sof *sof) trace_point(TRACE_BOOT_PLATFORM_SCHED); scheduler_init_edf(sof); scheduler_init_ll(platform_timer_domain); + scheduler_init_ll(platform_dma_domain); /* initialize IDC mechanism */ trace_point(TRACE_BOOT_PLATFORM_IDC); diff --git a/src/audio/dai.c b/src/audio/dai.c index be23cc858e15..592c5f4e28b3 100644 --- a/src/audio/dai.c +++ b/src/audio/dai.c @@ -73,18 +73,17 @@ struct dai_data { uint64_t wallclock; /* wall clock at stream start */ }; -static void dai_buffer_process(struct comp_dev *dev, struct dma_cb_data *next) +/* this is called by DMA driver every time descriptor has completed */ +static void dai_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) { + struct comp_dev *dev = (struct comp_dev *)data; struct dai_data *dd = comp_get_drvdata(dev); - struct dma_sg_config *config = &dd->config; uint32_t bytes = next->elem.size; void *buffer_ptr; - tracev_dai_with_ids(dev, "dai_buffer_process()"); + tracev_dai_with_ids(dev, "dai_dma_cb()"); - /* lli needs to be reloaded if irq is disabled */ - next->status = config->irq_disabled ? DMA_CB_STATUS_RELOAD : - DMA_CB_STATUS_IGNORE; + next->status = DMA_CB_STATUS_RELOAD; /* stop dma copy for pause/stop/xrun */ if (dev->state != COMP_STATE_ACTIVE || dd->xrun) { @@ -126,27 +125,6 @@ static void dai_buffer_process(struct comp_dev *dev, struct dma_cb_data *next) } } -/* this is called by DMA driver every time descriptor has completed */ -static void dai_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) -{ - struct comp_dev *dev = (struct comp_dev *)data; - - tracev_dai_with_ids(dev, "dai_dma_cb()"); - - switch (type) { - case DMA_CB_TYPE_IRQ: - pipeline_schedule_copy(dev->pipeline, 0); - break; - case DMA_CB_TYPE_COPY: - dai_buffer_process(dev, next); - break; - default: - trace_dai_error_with_ids(dev, "dai_dma_cb() error: Wrong " - "callback type = %u", type); - break; - } -} - static struct comp_dev *dai_new(struct sof_ipc_comp *comp) { struct comp_dev *dev; @@ -248,6 +226,7 @@ static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes) config->irq_disabled = pipeline_is_timer_driven(dev->pipeline); config->dest_dev = dai_get_handshake(dd->dai, dev->params.direction, dd->stream_id); + config->is_scheduling_source = comp_is_scheduling_source(dev); trace_dai_with_ids(dev, "dai_playback_params() " "dest_dev = %d stream_id = %d " @@ -313,6 +292,7 @@ static int dai_capture_params(struct comp_dev *dev, uint32_t period_bytes) config->irq_disabled = pipeline_is_timer_driven(dev->pipeline); config->src_dev = dai_get_handshake(dd->dai, dev->params.direction, dd->stream_id); + config->is_scheduling_source = comp_is_scheduling_source(dev); /* TODO: Make this code platform-specific or move it driver callback */ if (dai_get_info(dd->dai, DAI_INFO_TYPE) == SOF_DAI_INTEL_DMIC) { @@ -820,8 +800,7 @@ static int dai_config(struct comp_dev *dev, struct sof_ipc_dai_config *config) } /* set up callback */ - dma_set_cb(dd->chan, DMA_CB_TYPE_IRQ | DMA_CB_TYPE_COPY, - dai_dma_cb, dev); + dma_set_cb(dd->chan, DMA_CB_TYPE_COPY, dai_dma_cb, dev); dev->is_dma_connected = 1; } diff --git a/src/audio/host.c b/src/audio/host.c index 16795fa9c7f0..0e2e46c0a3a4 100644 --- a/src/audio/host.c +++ b/src/audio/host.c @@ -95,9 +95,8 @@ struct host_data { struct hc_buf *source; struct hc_buf *sink; - /* helpers used in split transactions */ + /* helper used in split transactions */ uint32_t split_value; - uint32_t last_split_value; uint32_t dma_copy_align; /**< Minimal chunk of data possible to be * copied by dma connected to host @@ -176,21 +175,12 @@ static void host_update_position(struct comp_dev *dev, uint32_t bytes) * This means we must check we do not overflow host period/buffer/page * boundaries on each transfer and split the DMA transfer if we do overflow. */ -static void host_dma_cb_irq(struct comp_dev *dev, struct dma_cb_data *next) +static void host_one_shot_cb(struct comp_dev *dev, uint32_t bytes) { struct host_data *hd = comp_get_drvdata(dev); struct dma_sg_elem *local_elem = hd->config.elem_array.elems; struct dma_sg_elem *source_elem; struct dma_sg_elem *sink_elem; - uint32_t curr_split_value = 0; - uint32_t next_bytes = 0; - uint32_t bytes = hd->last_split_value ? hd->last_split_value : - local_elem->size; - - tracev_host("host_dma_cb_irq()"); - - /* update position */ - host_update_position(dev, bytes); /* update src and dest positions and check for overflow */ local_elem->src += bytes; @@ -215,31 +205,6 @@ static void host_dma_cb_irq(struct comp_dev *dev, struct dma_cb_data *next) local_elem->dest = sink_elem->dest; } } - - /* we need to perform split transfer */ - if (hd->split_value) { - /* check for possible double split */ - curr_split_value = host_dma_get_split(hd, hd->split_value); - if (curr_split_value) { - next_bytes = hd->split_value - curr_split_value; - hd->split_value = curr_split_value; - } else { - next_bytes = hd->split_value; - hd->split_value = 0; - } - - hd->last_split_value = next_bytes; - - next->elem.src = local_elem->src; - next->elem.dest = local_elem->dest; - next->elem.size = next_bytes; - next->status = DMA_CB_STATUS_SPLIT; - return; - } - - hd->last_split_value = 0; - - next->status = DMA_CB_STATUS_END; } /* This is called by DMA driver every time when DMA completes its current @@ -248,21 +213,17 @@ static void host_dma_cb_irq(struct comp_dev *dev, struct dma_cb_data *next) static void host_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) { struct comp_dev *dev = (struct comp_dev *)data; + struct host_data *hd = comp_get_drvdata(dev); + uint32_t bytes = next->elem.size; tracev_host("host_dma_cb()"); - switch (type) { - case DMA_CB_TYPE_IRQ: - host_dma_cb_irq(dev, next); - break; - case DMA_CB_TYPE_COPY: - host_update_position(dev, next->elem.size); - break; - default: - trace_host_error("host_dma_cb() error: Wrong callback type " - "= %u", type); - break; - } + /* update position */ + host_update_position(dev, bytes); + + /* callback for one shot copy */ + if (hd->copy_type == COMP_COPY_ONE_SHOT) + host_one_shot_cb(dev, bytes); } static int create_local_elems(struct comp_dev *dev, uint32_t buffer_count, @@ -522,7 +483,7 @@ static void host_buffer_cb(void *data, uint32_t bytes) { struct comp_dev *dev = (struct comp_dev *)data; struct host_data *hd = comp_get_drvdata(dev); - uint32_t copy_bytes = host_buffer_get_copy_bytes(dev); + uint32_t copy_bytes = 0; uint32_t flags = 0; int ret; @@ -533,18 +494,28 @@ static void host_buffer_cb(void *data, uint32_t bytes) else if (hd->copy_type == COMP_COPY_ONE_SHOT) flags |= DMA_COPY_ONE_SHOT; - /* reconfigure transfer */ - ret = dma_set_config(hd->chan, &hd->config); - if (ret < 0) { - trace_host_error("host_buffer_cb() error: dma_set_config() " - "failed, ret = %u", ret); - return; - } + /* copy including all split transfers */ + do { + copy_bytes = host_buffer_get_copy_bytes(dev); - ret = dma_copy(hd->chan, copy_bytes, flags); - if (ret < 0) - trace_host_error("host_buffer_cb() error: dma_copy() failed, " - "ret = %u", ret); + /* reconfigure transfer */ + ret = dma_set_config(hd->chan, &hd->config); + if (ret < 0) { + trace_host_error("host_buffer_cb() error: " + "dma_set_config() failed, ret = %u", + ret); + hd->split_value = 0; + return; + } + + ret = dma_copy(hd->chan, copy_bytes, flags); + if (ret < 0) { + trace_host_error("host_buffer_cb() error: dma_copy() " + "failed, ret = %u", ret); + hd->split_value = 0; + return; + } + } while (hd->split_value); } /* configure the DMA params and descriptors for host buffer IO */ @@ -654,6 +625,7 @@ static int host_params(struct comp_dev *dev) config->dest_width = comp_sample_bytes(dev); config->cyclic = 0; config->irq_disabled = pipeline_is_timer_driven(dev->pipeline); + config->is_scheduling_source = comp_is_scheduling_source(dev); host_elements_reset(dev); @@ -688,8 +660,7 @@ static int host_params(struct comp_dev *dev) } /* set up callback */ - dma_set_cb(hd->chan, DMA_CB_TYPE_IRQ | - DMA_CB_TYPE_COPY, host_dma_cb, dev); + dma_set_cb(hd->chan, DMA_CB_TYPE_COPY, host_dma_cb, dev); return 0; } @@ -711,7 +682,6 @@ static int host_prepare(struct comp_dev *dev) hd->local_pos = 0; hd->report_pos = 0; hd->split_value = 0; - hd->last_split_value = 0; dev->position = 0; return 0; diff --git a/src/audio/pipeline.c b/src/audio/pipeline.c index 1942294d1e6e..062e2a176eff 100644 --- a/src/audio/pipeline.c +++ b/src/audio/pipeline.c @@ -38,13 +38,14 @@ struct pipeline_data { }; static enum task_state pipeline_task(void *arg); +static enum task_state pipeline_preload_task(void *arg); +static void pipeline_schedule_preload(struct pipeline *p); /* create new pipeline - returns pipeline id or negative error */ struct pipeline *pipeline_new(struct sof_ipc_pipe_new *pipe_desc, struct comp_dev *cd) { struct pipeline *p; - uint32_t type; trace_pipe("pipeline_new()"); @@ -62,12 +63,6 @@ struct pipeline *pipeline_new(struct sof_ipc_pipe_new *pipe_desc, assert(!memcpy_s(&p->ipc_pipe, sizeof(p->ipc_pipe), pipe_desc, sizeof(*pipe_desc))); - /* get pipeline task type */ - type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL : - SOF_SCHEDULE_EDF; - schedule_task_init(&p->pipe_task, type, pipe_desc->priority, - pipeline_task, NULL, p, pipe_desc->core, 0); - return p; } @@ -236,7 +231,11 @@ int pipeline_free(struct pipeline *p) } /* remove from any scheduling */ - schedule_task_free(&p->pipe_task); + if (p->pipe_task) + schedule_task_free(p->pipe_task); + + if (p->preload_task) + schedule_task_free(p->preload_task); /* now free the pipeline */ rfree(p); @@ -390,10 +389,25 @@ static int pipeline_comp_prepare(struct comp_dev *current, void *data, int dir) &buffer_reset_pos, dir); } +static struct task *pipeline_task_init(struct pipeline *p, uint32_t type, + enum task_state (*func)(void *data)) +{ + struct task *task = NULL; + + task = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(*task)); + + if (task) + schedule_task_init(task, type, p->ipc_pipe.priority, func, NULL, + p, p->ipc_pipe.core, 0); + + return task; +} + /* prepare the pipeline for usage - preload host buffers here */ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) { struct pipeline_data ppl_data; + uint32_t type; int ret = 0; trace_pipe_with_ids(p, "pipeline_prepare()"); @@ -412,6 +426,35 @@ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) */ p->preload = dev->params.direction == SOF_IPC_STREAM_PLAYBACK && p->sink_comp->state != COMP_STATE_ACTIVE; + + /* initialize task if necessary */ + if (!p->pipe_task) { + /* right now we always consider pipeline as a low latency + * component, but it may change in the future + */ + type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL_TIMER : + SOF_SCHEDULE_LL_DMA; + + p->pipe_task = pipeline_task_init(p, type, pipeline_task); + if (!p->pipe_task) { + trace_pipe_error("pipeline_prepare() error: task init " + "failed"); + return -ENOMEM; + } + } + + /* initialize preload task if necessary */ + if (p->preload && !p->preload_task) { + /* preload task is always EDF as it guarantees scheduling */ + p->preload_task = pipeline_task_init(p, SOF_SCHEDULE_EDF, + pipeline_preload_task); + if (!p->preload_task) { + trace_pipe_error("pipeline_prepare() error: preload " + "task init failed"); + return -ENOMEM; + } + } + p->status = COMP_STATE_PREPARE; return ret; @@ -444,8 +487,12 @@ void pipeline_cache(struct pipeline *p, struct comp_dev *dev, int cmd) uint32_t flags; /* pipeline needs to be invalidated before usage */ - if (cmd == CACHE_INVALIDATE) + if (cmd == CACHE_INVALIDATE) { dcache_invalidate_region(p, sizeof(*p)); + dcache_invalidate_region(p->pipe_task, sizeof(*p->pipe_task)); + dcache_invalidate_region(p->preload_task, + sizeof(*p->preload_task)); + } trace_pipe_with_ids(p, "pipeline_cache()"); @@ -458,8 +505,13 @@ void pipeline_cache(struct pipeline *p, struct comp_dev *dev, int cmd) pipeline_comp_cache(dev, &data, dev->params.direction); /* pipeline needs to be flushed after usage */ - if (cmd == CACHE_WRITEBACK_INV) + if (cmd == CACHE_WRITEBACK_INV) { + dcache_writeback_invalidate_region(p->pipe_task, + sizeof(*p->pipe_task)); + dcache_writeback_invalidate_region(p->preload_task, + sizeof(*p->preload_task)); dcache_writeback_invalidate_region(p, sizeof(*p)); + } irq_local_enable(flags); } @@ -482,15 +534,13 @@ static void pipeline_comp_trigger_sched_comp(struct pipeline *p, case COMP_TRIGGER_START: p->xrun_bytes = 0; - /* playback pipelines need to be scheduled now, - * capture pipelines are scheduled only for - * timer driven scheduling - */ - if (comp->params.direction == SOF_IPC_STREAM_PLAYBACK || - pipeline_is_timer_driven(p)) { - /* schedule initial pipeline fill */ + /* schedule preload if pipeline is not timer driven */ + if (p->preload && !pipeline_is_timer_driven(p)) + /* schedule pipeline preload */ + pipeline_schedule_preload(p); + else pipeline_schedule_copy(p, 0); - } + p->status = COMP_STATE_ACTIVE; break; case COMP_TRIGGER_SUSPEND: @@ -801,8 +851,10 @@ static int pipeline_copy(struct pipeline *p) start->comp.id, dir); /* stop preload only after full walkthrough */ - if (ret != PPL_STATUS_PATH_STOP) + if (ret != PPL_STATUS_PATH_STOP && p->preload) { p->preload = false; + pipeline_schedule_copy(p, 0); + } return ret; } @@ -943,12 +995,17 @@ static int pipeline_xrun_recover(struct pipeline *p) void pipeline_schedule_copy(struct pipeline *p, uint64_t start) { if (p->sched_comp->state == COMP_STATE_ACTIVE) - schedule_task(&p->pipe_task, start, p->ipc_pipe.period); + schedule_task(p->pipe_task, start, p->ipc_pipe.period); } void pipeline_schedule_cancel(struct pipeline *p) { - schedule_task_cancel(&p->pipe_task); + schedule_task_cancel(p->pipe_task); +} + +static void pipeline_schedule_preload(struct pipeline *p) +{ + schedule_task(p->preload_task, 0, p->ipc_pipe.period); } static enum task_state pipeline_task(void *arg) @@ -982,7 +1039,19 @@ static enum task_state pipeline_task(void *arg) tracev_pipe("pipeline_task() sched"); - /* automatically reschedule for timer or not finished preload */ - return (pipeline_is_timer_driven(p) || p->preload) ? - SOF_TASK_STATE_RESCHEDULE : SOF_TASK_STATE_COMPLETED; + return SOF_TASK_STATE_RESCHEDULE; +} + +static enum task_state pipeline_preload_task(void *arg) +{ + struct pipeline *p = arg; + + if (pipeline_copy(p) < 0) { + trace_pipe_error_with_ids(p, "pipeline_preload_task(): preload " + "has failed"); + return SOF_TASK_STATE_COMPLETED; + } + + return p->preload ? SOF_TASK_STATE_RESCHEDULE : + SOF_TASK_STATE_COMPLETED; } diff --git a/src/audio/volume/volume.c b/src/audio/volume/volume.c index cb80545a4c15..5ed4368a0eb3 100644 --- a/src/audio/volume/volume.c +++ b/src/audio/volume/volume.c @@ -163,8 +163,8 @@ static struct comp_dev *volume_new(struct sof_ipc_comp *comp) } comp_set_drvdata(dev, cd); - schedule_task_init(&cd->volwork, SOF_SCHEDULE_LL, SOF_TASK_PRI_MED, - vol_work, NULL, dev, 0, 0); + schedule_task_init(&cd->volwork, SOF_SCHEDULE_LL_TIMER, + SOF_TASK_PRI_MED, vol_work, NULL, dev, 0, 0); /* Set the default volumes. If IPC sets min_value or max_value to * not-zero, use them. Otherwise set to internal limits and notify diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index 5ba32737fe1e..2d085ac1fe9a 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -57,22 +56,8 @@ struct dw_dma_chan_data { struct dw_lli *lli_current; uint32_t cfg_lo; uint32_t cfg_hi; - bool irq_disabled; -#if !CONFIG_DMA_AGGREGATED_IRQ - int irq; -#endif - - /* pointer data */ - struct dw_dma_ptr_data ptr_data; -}; - -#if CONFIG_DMA_AGGREGATED_IRQ -/* private data for DW DMA engine */ -struct dma_pdata { - uint32_t mask_irq_channels[PLATFORM_CORE_COUNT]; - int irq; + struct dw_dma_ptr_data ptr_data; /* pointer data */ }; -#endif /* use array to get burst_elems for specific slot number setting. * the relation between msize and burst_elems should be @@ -80,28 +65,8 @@ struct dma_pdata { */ static const uint32_t burst_elems[] = {1, 2, 4, 8}; -#if !CONFIG_HW_LLI -static inline void dw_dma_chan_reload_lli(struct dma_chan_data *channel); -static inline void dw_dma_chan_reload_next(struct dma_chan_data *channel, - struct dma_sg_elem *next, - int direction); -#endif -static inline int dw_dma_interrupt_register(struct dma_chan_data *channel); -static inline void dw_dma_interrupt_unregister(struct dma_chan_data *channel); -static int dw_dma_copy(struct dma_chan_data *channel, int bytes, - uint32_t flags); - static void dw_dma_interrupt_mask(struct dma_chan_data *channel) { - struct dw_dma_chan_data *dw_chan_data = dma_chan_get_data(channel); - - if (dw_chan_data->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_mask(): dma %d channel %d " - "not working in irq mode", - channel->dma->plat_data.id, channel->index); - return; - } - /* mask block, transfer and error interrupts for channel */ dma_reg_write(channel->dma, DW_MASK_TFR, DW_CHAN_MASK(channel->index)); dma_reg_write(channel->dma, DW_MASK_BLOCK, @@ -111,15 +76,6 @@ static void dw_dma_interrupt_mask(struct dma_chan_data *channel) static void dw_dma_interrupt_unmask(struct dma_chan_data *channel) { - struct dw_dma_chan_data *dw_chan_data = dma_chan_get_data(channel); - - if (dw_chan_data->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_mask(): dma %d channel %d " - "not working in irq mode", - channel->dma->plat_data.id, channel->index); - return; - } - /* unmask block, transfer and error interrupts for channel */ #if CONFIG_HW_LLI dma_reg_write(channel->dma, DW_MASK_BLOCK, @@ -134,31 +90,25 @@ static void dw_dma_interrupt_unmask(struct dma_chan_data *channel) static void dw_dma_interrupt_clear(struct dma_chan_data *channel) { -#if CONFIG_DMA_AGGREGATED_IRQ - const struct dma_pdata *p = dma_get_drvdata(channel->dma); -#endif - const struct dw_dma_chan_data *chan = dma_chan_get_data(channel); - - if (chan->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_mask(): dma %d channel %d " - "not working in irq mode", - channel->dma->plat_data.id, channel->index); - return; - } - /* clear transfer, block, src, dst and error interrupts for channel */ dma_reg_write(channel->dma, DW_CLEAR_TFR, DW_CHAN(channel->index)); dma_reg_write(channel->dma, DW_CLEAR_BLOCK, DW_CHAN(channel->index)); dma_reg_write(channel->dma, DW_CLEAR_SRC_TRAN, DW_CHAN(channel->index)); dma_reg_write(channel->dma, DW_CLEAR_DST_TRAN, DW_CHAN(channel->index)); dma_reg_write(channel->dma, DW_CLEAR_ERR, DW_CHAN(channel->index)); +} + +static uint32_t dw_dma_interrupt_status(struct dma_chan_data *channel) +{ + uint32_t status; - /* clear platform interrupt */ -#if CONFIG_DMA_AGGREGATED_IRQ - interrupt_clear_mask(p->irq, DW_CHAN(channel->index)); +#if CONFIG_HW_LLI + status = dma_reg_read(channel->dma, DW_STATUS_BLOCK); #else - interrupt_clear_mask(chan->irq, DW_CHAN(channel->index)); + status = dma_reg_read(channel->dma, DW_STATUS_TFR); #endif + + return status & DW_CHAN(channel->index); } static void dw_dma_increment_pointer(struct dw_dma_chan_data *chan, int bytes) @@ -279,8 +229,6 @@ static int dw_dma_start(struct dma_chan_data *channel) goto out; } - dw_dma_interrupt_clear(channel); - #if CONFIG_HW_LLI /* LLP mode - write LLP pointer unless in scatter mode */ dma_reg_write(dma, DW_LLP(channel->index), lli->ctrl_lo & @@ -310,16 +258,12 @@ static int dw_dma_start(struct dma_chan_data *channel) } #endif - /* enable interrupt only for the first start */ - if (channel->status == COMP_STATE_PREPARE) - ret = dw_dma_interrupt_register(channel); + /* assign core */ + channel->core = cpu_get_id(); - if (!ret) { - /* enable the channel */ - channel->status = COMP_STATE_ACTIVE; - dma_reg_write(dma, DW_DMA_CHAN_EN, - DW_CHAN_UNMASK(channel->index)); - } + /* enable the channel */ + channel->status = COMP_STATE_ACTIVE; + dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_UNMASK(channel->index)); out: irq_local_enable(flags); @@ -433,13 +377,7 @@ static int dw_dma_stop(struct dma_chan_data *channel) dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_MASK(channel->index)); - /* disable interrupt */ - dw_dma_interrupt_unregister(channel); - #if CONFIG_HW_LLI - /* clear block interrupt */ - dma_reg_write(dma, DW_CLEAR_BLOCK, DW_CHAN(channel->index)); - for (i = 0; i < channel->desc_count; i++) { lli->ctrl_hi &= ~DW_CTLH_DONE(1); lli++; @@ -523,7 +461,7 @@ static int dw_dma_set_config(struct dma_chan_data *channel, /* default channel config */ channel->direction = config->direction; - dw_chan->irq_disabled = config->irq_disabled; + channel->is_scheduling_source = config->is_scheduling_source; dw_chan->cfg_lo = DW_CFG_LOW_DEF; dw_chan->cfg_hi = DW_CFG_HIGH_DEF; @@ -583,8 +521,6 @@ static int dw_dma_set_config(struct dma_chan_data *channel, } } - dw_dma_interrupt_unmask(channel); - dw_chan->ptr_data.buffer_bytes = 0; /* fill in lli for the elems in the list */ @@ -852,45 +788,6 @@ static inline void dw_dma_chan_reload_lli(struct dma_chan_data *channel) /* enable the channel */ dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_UNMASK(channel->index)); } - -/* reload using callback data */ -static inline void dw_dma_chan_reload_next(struct dma_chan_data *channel, - struct dma_sg_elem *next, - int direction) -{ - struct dma *dma = channel->dma; - struct dw_drv_plat_data *dp = dma->plat_data.drv_plat_data; - struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); - uint16_t class = dp->chan[channel->index].class; - struct dw_lli *lli = dw_chan->lli_current; - uint32_t sar; - uint32_t dar; - - dw_dma_mask_address(next, &sar, &dar, direction); - - /* channel needs to start from scratch, so write SAR and DAR */ - dma_reg_write(dma, DW_SAR(channel->index), sar); - dma_reg_write(dma, DW_DAR(channel->index), dar); - - lli->ctrl_hi = 0; - - /* set channel class */ - platform_dw_dma_set_class(dw_chan, lli, class); - - /* set transfer size of element */ - platform_dw_dma_set_transfer_size(dw_chan, lli, next->size); - - /* program CTL_LO and CTL_HI */ - dma_reg_write(dma, DW_CTRL_LOW(channel->index), lli->ctrl_lo); - dma_reg_write(dma, DW_CTRL_HIGH(channel->index), lli->ctrl_hi); - - /* program CFG_LO and CFG_HI */ - dma_reg_write(dma, DW_CFG_LOW(channel->index), dw_chan->cfg_lo); - dma_reg_write(dma, DW_CFG_HIGH(channel->index), dw_chan->cfg_hi); - - /* enable the channel */ - dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_UNMASK(channel->index)); -} #endif static void dw_dma_verify_transfer(struct dma_chan_data *channel, @@ -918,7 +815,6 @@ static void dw_dma_verify_transfer(struct dma_chan_data *channel, #else /* check for reload channel: * next->status is DMA_CB_STATUS_END, stop this dma copy; - * next->status is DMA_CB_STATUS_SPLIT, use next element for next copy; * otherwise, reload lli */ switch (next->status) { @@ -927,10 +823,6 @@ static void dw_dma_verify_transfer(struct dma_chan_data *channel, dw_chan->lli_current = (struct dw_lli *)dw_chan->lli_current->llp; break; - case DMA_CB_STATUS_SPLIT: - dw_dma_chan_reload_next(channel, &next->elem, - channel->direction); - break; default: dw_dma_chan_reload_lli(channel); break; @@ -938,182 +830,42 @@ static void dw_dma_verify_transfer(struct dma_chan_data *channel, #endif } -static void dw_dma_irq_callback(struct dma_chan_data *channel, - struct dma_cb_data *next, uint32_t type) -{ - if (channel->cb && channel->cb_type & type) - channel->cb(channel->cb_data, type, next); - - if (next->status != DMA_CB_STATUS_IGNORE) - dw_dma_verify_transfer(channel, next); -} - static int dw_dma_copy(struct dma_chan_data *channel, int bytes, uint32_t flags) { struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); + int ret = 0; struct dma_cb_data next = { .elem = { .size = bytes }, - .status = DMA_CB_STATUS_RELOAD + .status = DMA_CB_STATUS_END }; - /* for preload or one shot copy just start DMA */ - if (flags & (DMA_COPY_PRELOAD | DMA_COPY_ONE_SHOT)) - return dw_dma_start(channel); - - tracev_dwdma("dw_dma_copy(): dma %d channel %d copy", - channel->dma->plat_data.id, channel->index); - - dw_dma_irq_callback(channel, &next, DMA_CB_TYPE_COPY); - - /* increment current pointer */ - dw_dma_increment_pointer(dw_chan, bytes); - - return 0; -} - -/* interrupt handler for DMA */ -static void dw_dma_irq_handler(void *data) -{ - struct dma_chan_data *chan = data; - struct dma *dma = chan->dma; -#if CONFIG_DMA_AGGREGATED_IRQ - struct dma_pdata *p = dma_get_drvdata(dma); -#endif - uint32_t status_intr; - uint32_t status_err; - uint32_t status_src; - uint32_t mask; - int i; - struct dma_cb_data next = { .status = DMA_CB_STATUS_RELOAD }; - - status_intr = dma_reg_read(dma, DW_INTR_STATUS); - if (!status_intr) - return; - - tracev_dwdma("dw_dma_irq_handler(): dma %d IRQ status 0x%x", - dma->plat_data.id, status_intr); - - /* get the source of our IRQ and clear it */ -#if CONFIG_HW_LLI -#if CONFIG_DMA_AGGREGATED_IRQ - /* skip if channel is not registered on this core */ - mask = BIT(chan - dma->chan); -#else - mask = ~0; -#endif - - status_src = dma_reg_read(dma, DW_STATUS_BLOCK) & mask; - dma_reg_write(dma, DW_CLEAR_BLOCK, status_src); -#else - status_src = dma_reg_read(dma, DW_STATUS_TFR); - dma_reg_write(dma, DW_CLEAR_TFR, status_src); -#endif - - /* TODO: handle errors, just clear them atm */ - status_err = dma_reg_read(dma, DW_STATUS_ERR); - if (status_err) { - trace_dwdma_error("dw_dma_irq_handler() error: dma %d status " - "error 0x%x", dma->plat_data.id, status_err); - dma_reg_write(dma, DW_CLEAR_ERR, status_err); - } - -#if CONFIG_DMA_AGGREGATED_IRQ - /* clear platform and DSP interrupt */ - interrupt_clear_mask(p->irq, status_src | status_err); -#endif - - for (i = 0; i < dma->plat_data.channels; i++) { - /* skip if channel is not running */ - if (dma->chan[i].status != COMP_STATE_ACTIVE) - continue; - - mask = 0x1 << i; - - if (status_src & mask) - dw_dma_irq_callback(&dma->chan[i], &next, - DMA_CB_TYPE_IRQ); - } -} - -static inline int dw_dma_interrupt_register(struct dma_chan_data *channel) -{ - struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); - uint32_t irq = dma_chan_irq(channel->dma, channel->index); - int logical_irq = interrupt_get_irq(irq, dma_irq_name(channel->dma)); -#if CONFIG_DMA_AGGREGATED_IRQ - struct dma_pdata *p = dma_get_drvdata(channel->dma); - int cpu = cpu_get_id(); -#endif - int ret; - - if (dw_chan->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_register(): dma %d channel %d " - "not working in irq mode", - channel->dma->plat_data.id, channel->index); - return 0; - } - - if (logical_irq < 0) - return logical_irq; -#if !CONFIG_DMA_AGGREGATED_IRQ - dw_chan->irq = logical_irq; -#else - p->irq = logical_irq; - - if (!p->mask_irq_channels[cpu]) { -#endif - ret = interrupt_register(logical_irq, IRQ_AUTO_UNMASK, - dw_dma_irq_handler, channel); - if (ret < 0) { - trace_dwdma_error("dw_dma_interrupt_register() error: " - "dma %d channel %d failed to " - "allocate IRQ", - channel->dma->plat_data.id, - channel->index); + /* for preload and one shot copy just start the DMA and wait */ + if (flags & (DMA_COPY_PRELOAD | DMA_COPY_ONE_SHOT)) { + ret = dw_dma_start(channel); + if (ret < 0) return ret; - } - interrupt_enable(logical_irq, channel); -#if CONFIG_DMA_AGGREGATED_IRQ + ret = poll_for_register_delay(dma_base(channel->dma) + + DW_DMA_CHAN_EN, + DW_CHAN(channel->index), 0, + PLATFORM_DMA_TIMEOUT); + if (ret < 0) + return ret; } - p->mask_irq_channels[cpu] |= BIT(channel->index); -#endif - - return 0; -} - -static inline void dw_dma_interrupt_unregister(struct dma_chan_data *channel) -{ - struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); - int logical_irq; -#if CONFIG_DMA_AGGREGATED_IRQ - struct dma_pdata *p = dma_get_drvdata(channel->dma); - int cpu = cpu_get_id(); -#endif + tracev_dwdma("dw_dma_copy(): dma %d channel %d copy", + channel->dma->plat_data.id, channel->index); - if (dw_chan->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_unregister(): dma %d channel %d" - " not working in irq mode", - channel->dma->plat_data.id, channel->index); - return; - } + if (channel->cb && channel->cb_type & DMA_CB_TYPE_COPY) + channel->cb(channel->cb_data, DMA_CB_TYPE_COPY, &next); -#if !CONFIG_DMA_AGGREGATED_IRQ - logical_irq = dw_chan->irq; -#else - logical_irq = p->irq; + dw_dma_verify_transfer(channel, &next); - p->mask_irq_channels[cpu] &= ~BIT(channel->index); + /* increment current pointer */ + dw_dma_increment_pointer(dw_chan, bytes); - if (!p->mask_irq_channels[cpu]) { -#endif - interrupt_disable(logical_irq, channel); - interrupt_unregister(logical_irq, channel); -#if CONFIG_DMA_AGGREGATED_IRQ - } -#endif + return ret; } static int dw_dma_setup(struct dma *dma) @@ -1166,9 +918,6 @@ static int dw_dma_setup(struct dma *dma) static int dw_dma_probe(struct dma *dma) { -#if CONFIG_DMA_AGGREGATED_IRQ - struct dma_pdata *dw_pdata; -#endif struct dma_chan_data *chan; struct dw_dma_chan_data *dw_chan; int ret; @@ -1180,18 +929,6 @@ static int dw_dma_probe(struct dma *dma) /* disable dynamic clock gating */ pm_runtime_get_sync(DW_DMAC_CLK, dma->plat_data.id); -#if CONFIG_DMA_AGGREGATED_IRQ - /* allocate private data */ - dw_pdata = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, - SOF_MEM_CAPS_RAM, sizeof(*dw_pdata)); - if (!dw_pdata) { - trace_dwdma_error("dw_dma_probe() error: dma %d alloc failed", - dma->plat_data.id); - return -ENOMEM; - } - dma_set_drvdata(dma, dw_pdata); -#endif - /* allocate dma channels */ dma->chan = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, sizeof(struct dma_chan_data) * @@ -1213,9 +950,9 @@ static int dw_dma_probe(struct dma *dma) for (i = 0, chan = dma->chan; i < dma->plat_data.channels; i++, chan++) { chan->status = COMP_STATE_INIT; - chan->index = i; chan->dma = dma; chan->index = i; + chan->core = DMA_CORE_INVALID; dw_chan = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, sizeof(*dw_chan)); @@ -1243,19 +980,11 @@ static int dw_dma_probe(struct dma *dma) dma->chan = NULL; } -#if CONFIG_DMA_AGGREGATED_IRQ - rfree(dw_pdata); - dma_set_drvdata(dma, NULL); -#endif - return -ENOMEM; } static int dw_dma_remove(struct dma *dma) { -#if CONFIG_DMA_AGGREGATED_IRQ - struct dma_pdata *p = dma_get_drvdata(dma); -#endif int i; tracev_dwdma("dw_dma_remove(): dma %d remove", dma->plat_data.id); @@ -1267,10 +996,6 @@ static int dw_dma_remove(struct dma *dma) rfree(dma->chan); dma->chan = NULL; -#if CONFIG_DMA_AGGREGATED_IRQ - rfree(p); - dma_set_drvdata(dma, NULL); -#endif return 0; } @@ -1278,7 +1003,11 @@ static int dw_dma_avail_data_size(struct dma_chan_data *channel) { struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); int32_t read_ptr = dw_chan->ptr_data.current_ptr; +#if CONFIG_HW_LLI int32_t write_ptr = dma_reg_read(channel->dma, DW_DAR(channel->index)); +#else + int32_t write_ptr = ((struct dw_lli *)dw_chan->lli_current->llp)->dar; +#endif int size; size = write_ptr - read_ptr; @@ -1296,7 +1025,11 @@ static int dw_dma_avail_data_size(struct dma_chan_data *channel) static int dw_dma_free_data_size(struct dma_chan_data *channel) { struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); +#if CONFIG_HW_LLI int32_t read_ptr = dma_reg_read(channel->dma, DW_SAR(channel->index)); +#else + int32_t read_ptr = ((struct dw_lli *)dw_chan->lli_current->llp)->sar; +#endif int32_t write_ptr = dw_chan->ptr_data.current_ptr; int size; @@ -1353,6 +1086,31 @@ static int dw_dma_get_attribute(struct dma *dma, uint32_t type, return ret; } +static int dw_dma_interrupt(struct dma_chan_data *channel, enum dma_irq_cmd cmd) +{ + int ret = 0; + + switch (cmd) { + case DMA_IRQ_STATUS_GET: + ret = dw_dma_interrupt_status(channel); + break; + case DMA_IRQ_CLEAR: + dw_dma_interrupt_clear(channel); + break; + case DMA_IRQ_MASK: + dw_dma_interrupt_mask(channel); + break; + case DMA_IRQ_UNMASK: + dw_dma_interrupt_unmask(channel); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + const struct dma_ops dw_dma_ops = { .channel_get = dw_dma_channel_get, .channel_put = dw_dma_channel_put, @@ -1370,4 +1128,5 @@ const struct dma_ops dw_dma_ops = { .remove = dw_dma_remove, .get_data_size = dw_dma_get_data_size, .get_attribute = dw_dma_get_attribute, + .interrupt = dw_dma_interrupt, }; diff --git a/src/drivers/imx/dummy-dma.c b/src/drivers/imx/dummy-dma.c index 28e7347dbb5c..03121bf51da0 100644 --- a/src/drivers/imx/dummy-dma.c +++ b/src/drivers/imx/dummy-dma.c @@ -96,6 +96,12 @@ static int dummy_dma_get_data_size(struct dma_chan_data *channel, return 0; } +static int dummy_dma_interrupt(struct dma_chan_data *channel, + enum dma_irq_cmd cmd) +{ + return 0; +} + const struct dma_ops dummy_dma_ops = { .channel_get = dummy_dma_channel_get, .channel_put = dummy_dma_channel_put, @@ -112,4 +118,5 @@ const struct dma_ops dummy_dma_ops = { .probe = dummy_dma_probe, .remove = dummy_dma_remove, .get_data_size = dummy_dma_get_data_size, + .interrupt = dummy_dma_interrupt, }; diff --git a/src/drivers/imx/edma.c b/src/drivers/imx/edma.c index 9ff65aca7e63..e095b698aa54 100644 --- a/src/drivers/imx/edma.c +++ b/src/drivers/imx/edma.c @@ -110,6 +110,11 @@ static int edma_remove(struct dma *dma) return 0; } +static int edma_interrupt(struct dma_chan_data *channel, enum dma_irq_cmd cmd) +{ + return 0; +} + const struct dma_ops edma_ops = { .channel_get = edma_channel_get, .channel_put = edma_channel_put, @@ -124,4 +129,5 @@ const struct dma_ops edma_ops = { .pm_context_store = edma_pm_context_store, .probe = edma_probe, .remove = edma_remove, + .interrupt = edma_interrupt, }; diff --git a/src/drivers/intel/cavs/dmic.c b/src/drivers/intel/cavs/dmic.c index a8be891d142d..822dd33731e0 100644 --- a/src/drivers/intel/cavs/dmic.c +++ b/src/drivers/intel/cavs/dmic.c @@ -1067,7 +1067,7 @@ static int dmic_set_config(struct dai *dai, struct sof_ipc_dai_config *config) trace_dmic("dmic_set_config(), dai->index = %d", di); /* Initialize start sequence handler */ - schedule_task_init(&dmic->dmicwork, SOF_SCHEDULE_LL, + schedule_task_init(&dmic->dmicwork, SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_MED, dmic_work, NULL, dai, 0, SOF_SCHEDULE_FLAG_ASYNC); diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index c3cf5fb46631..6e4eb015e4f6 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -59,8 +60,6 @@ /* DGBS */ #define DGBS_MASK 0xfffff0 -#define HDA_DMA_MAX_CHANS 9 - #define HDA_STATE_RELEASE BIT(0) /* DGMBS align value */ @@ -477,6 +476,8 @@ static int hda_dma_start(struct dma_chan_data *channel) hda_dma_enable_unlock(channel); channel->status = COMP_STATE_ACTIVE; + channel->core = cpu_get_id(); + out: irq_local_enable(flags); return ret; @@ -604,6 +605,7 @@ static int hda_dma_set_config(struct dma_chan_data *channel, /* default channel config */ channel->direction = config->direction; channel->desc_count = config->elem_array.count; + channel->is_scheduling_source = config->is_scheduling_source; /* validate - HDA only supports continuous elems of same size */ for (i = 0; i < config->elem_array.count; i++) { @@ -743,6 +745,7 @@ static int hda_dma_probe(struct dma *dma) chan->dma = dma; chan->index = i; chan->status = COMP_STATE_INIT; + chan->core = DMA_CORE_INVALID; hda_chan = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, @@ -778,7 +781,7 @@ static int hda_dma_remove(struct dma *dma) trace_hddma("hda-dmac :%d -> remove", dma->plat_data.id); - for (i = 0; i < HDA_DMA_MAX_CHANS; i++) + for (i = 0; i < dma->plat_data.channels; i++) rfree(dma_chan_get_data(&dma->chan[i])); rfree(dma->chan); @@ -880,6 +883,13 @@ static int hda_dma_get_attribute(struct dma *dma, uint32_t type, return ret; } +static int hda_dma_interrupt(struct dma_chan_data *channel, + enum dma_irq_cmd cmd) +{ + /* HDA-DMA doesn't support interrupts */ + return -EINVAL; +} + const struct dma_ops hda_host_dma_ops = { .channel_get = hda_dma_channel_get, .channel_put = hda_dma_channel_put, @@ -897,6 +907,7 @@ const struct dma_ops hda_host_dma_ops = { .remove = hda_dma_remove, .get_data_size = hda_dma_data_size, .get_attribute = hda_dma_get_attribute, + .interrupt = hda_dma_interrupt, }; const struct dma_ops hda_link_dma_ops = { @@ -916,4 +927,5 @@ const struct dma_ops hda_link_dma_ops = { .remove = hda_dma_remove, .get_data_size = hda_dma_data_size, .get_attribute = hda_dma_get_attribute, + .interrupt = hda_dma_interrupt, }; diff --git a/src/drivers/intel/cavs/idc.c b/src/drivers/intel/cavs/idc.c index 12c9435162b2..a4c1407f2a16 100644 --- a/src/drivers/intel/cavs/idc.c +++ b/src/drivers/intel/cavs/idc.c @@ -192,8 +192,8 @@ static int idc_pipeline_trigger(uint32_t cmd) * Should be removed after changing memory management for * slave cores. */ - if (pcm_dev->cd->pipeline->pipe_task.type == SOF_SCHEDULE_EDF) { - task = &pcm_dev->cd->pipeline->pipe_task; + if (pcm_dev->cd->pipeline->preload_task) { + task = pcm_dev->cd->pipeline->preload_task; edf_pdata = edf_sch_get_pdata(task); dcache_invalidate_region(edf_pdata, sizeof(*edf_pdata)); task_context_cache(edf_pdata->ctx, CACHE_INVALIDATE); @@ -213,8 +213,8 @@ static int idc_pipeline_trigger(uint32_t cmd) * Should be removed after changing memory management for * slave cores. */ - if (pcm_dev->cd->pipeline->pipe_task.type == SOF_SCHEDULE_EDF) { - task = &pcm_dev->cd->pipeline->pipe_task; + if (pcm_dev->cd->pipeline->preload_task) { + task = pcm_dev->cd->pipeline->preload_task; edf_pdata = edf_sch_get_pdata(task); task_context_cache(edf_pdata->ctx, CACHE_WRITEBACK_INV); dcache_writeback_invalidate_region(edf_pdata, diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 91c894714480..47d5265e108f 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -760,6 +761,16 @@ static inline void comp_overrun(struct comp_dev *dev, struct comp_buffer *sink, pipeline_xrun(dev->pipeline, dev, (int32_t)copy_bytes - sink->free); } +/** + * Called to check whether component schedules its pipeline. + * @param dev Component device. + * @return True if this is scheduling component, false otherwise. + */ +static inline bool comp_is_scheduling_source(struct comp_dev *dev) +{ + return dev == dev->pipeline->sched_comp; +} + /** * Called by component in copy. * @param dev Component device. diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 717345c5c56d..82e03115cc51 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -78,7 +78,8 @@ struct pipeline { bool preload; /* is pipeline preload needed */ /* scheduling */ - struct task pipe_task; /* pipeline processing task */ + struct task *pipe_task; /* pipeline processing task */ + struct task *preload_task; /* pipeline preload task */ /* component that drives scheduling in this pipe */ struct comp_dev *sched_comp; diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index f4957585880e..c36eb257d17b 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -74,12 +73,19 @@ */ enum dma_cb_status { DMA_CB_STATUS_RELOAD = 0, - DMA_CB_STATUS_SPLIT, - DMA_CB_STATUS_IGNORE, DMA_CB_STATUS_END, }; +/* DMA interrupt commands */ +enum dma_irq_cmd { + DMA_IRQ_STATUS_GET = 0, + DMA_IRQ_CLEAR, + DMA_IRQ_MASK, + DMA_IRQ_UNMASK +}; + #define DMA_CHAN_INVALID 0xFFFFFFFF +#define DMA_CORE_INVALID 0xFFFFFFFF /* DMA attributes */ #define DMA_ATTR_BUFFER_ALIGNMENT 0 @@ -124,6 +130,8 @@ struct dma_sg_config { struct dma_sg_elem_array elem_array; /* array of dma_sg elems */ bool scatter; bool irq_disabled; + /* true if configured DMA channel is the scheduling source */ + bool is_scheduling_source; }; struct dma_chan_status { @@ -166,6 +174,8 @@ struct dma_ops { uint32_t *free); int (*get_attribute)(struct dma *dma, uint32_t type, uint32_t *value); + + int (*interrupt)(struct dma_chan_data *channel, enum dma_irq_cmd cmd); }; /* DMA platform data */ @@ -199,6 +209,9 @@ struct dma_chan_data { uint32_t direction; uint32_t desc_count; uint32_t index; + uint32_t core; + /* true if this DMA channel is the scheduling source */ + bool is_scheduling_source; /* client callback function */ void (*cb)(void *data, uint32_t type, struct dma_cb_data *next); @@ -210,6 +223,8 @@ struct dma_chan_data { void *private; }; +extern struct dma dma[]; + /** * \brief Plugs platform specific DMA array once initialized into the lib. * @@ -356,6 +371,12 @@ static inline int dma_get_attribute(struct dma *dma, uint32_t type, return dma->ops->get_attribute(dma, type, value); } +static inline int dma_interrupt(struct dma_chan_data *channel, + enum dma_irq_cmd cmd) +{ + return channel->dma->ops->interrupt(channel, cmd); +} + /* DMA hardware register operations */ static inline uint32_t dma_reg_read(struct dma *dma, uint32_t reg) { @@ -417,6 +438,11 @@ static inline void dma_chan_reg_update_bits(struct dma_chan_data *channel, mask, value); } +static inline bool dma_is_scheduling_source(struct dma_chan_data *channel) +{ + return channel->is_scheduling_source; +} + static inline void dma_sg_init(struct dma_sg_elem_array *ea) { ea->count = 0; @@ -469,7 +495,6 @@ static inline uint32_t dma_sg_get_size(struct dma_sg_elem_array *ea) struct dma_copy { struct dma_chan_data *chan; struct dma *dmac; - completion_t complete; }; /* init dma copy context */ diff --git a/src/include/sof/lib/wait.h b/src/include/sof/lib/wait.h index 05a1620c5038..1fc0fafe2a85 100644 --- a/src/include/sof/lib/wait.h +++ b/src/include/sof/lib/wait.h @@ -69,7 +69,7 @@ static inline void wait_init(completion_t *comp) c->complete = 0; - schedule_task_init(&comp->work, SOF_SCHEDULE_LL, SOF_TASK_PRI_MED, + schedule_task_init(&comp->work, SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_MED, _wait_cb, NULL, comp, 0, 0); } diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index 6ab4f9f44e8e..dc5e0a43c02a 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -20,6 +20,7 @@ #include #include +struct dma; struct ll_schedule_domain; struct task; struct timer; @@ -126,4 +127,8 @@ static inline bool domain_is_pending(struct ll_schedule_domain *domain, struct ll_schedule_domain *timer_domain_init(struct timer *timer, int clk, uint64_t timeout); +struct ll_schedule_domain *dma_multi_chan_domain_init(struct dma *dma_array, + uint32_t num_dma, int clk, + bool aggregated_irq); + #endif /* __SOF_SCHEDULE_LL_SCHEDULE_DOMAIN_H__ */ diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index e8f6eccf1eaf..8fe2309c8afd 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -28,8 +28,18 @@ /* SOF_SCHEDULE_ type comes from topology */ enum { - SOF_SCHEDULE_EDF = 0, /* EDF scheduler */ - SOF_SCHEDULE_LL, /* Low latency scheduler */ + /* EDF scheduler - schedules based on task's deadline */ + SOF_SCHEDULE_EDF = 0, + + /* Low latency timer scheduler - schedules immediately on selected + * timer's tick + */ + SOF_SCHEDULE_LL_TIMER, + + /* Low latency DMA scheduler - schedules immediately on scheduling + * component's DMA interrupt + */ + SOF_SCHEDULE_LL_DMA, SOF_SCHEDULE_COUNT }; diff --git a/src/ipc/dma-copy.c b/src/ipc/dma-copy.c index 97af7ec46096..c6f058c0c2a0 100644 --- a/src/ipc/dma-copy.c +++ b/src/ipc/dma-copy.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -50,22 +49,6 @@ static struct dma_sg_elem *sg_get_elem_at(struct dma_sg_config *host_sg, } #endif -#if !CONFIG_DMA_GW - -static void dma_complete(void *data, uint32_t type, struct dma_cb_data *next) -{ - completion_t *comp = (completion_t *)data; - - if (type == DMA_CB_TYPE_IRQ) - wait_completed(comp); - - ipc_dma_trace_send_position(); - - next->status = DMA_CB_STATUS_END; -} - -#endif - /* Copy DSP memory to host memory. * Copies DSP memory to host in a single PAGE_SIZE or smaller block. Does not * waits/sleeps and can be used in IRQ context. @@ -129,10 +112,12 @@ int dma_copy_to_host_nowait(struct dma_copy *dc, struct dma_sg_config *host_sg, if (err < 0) return err; - err = dma_start(dc->chan); + err = dma_copy(dc->chan, local_sg_elem.size, DMA_COPY_ONE_SHOT); if (err < 0) return err; + ipc_dma_trace_send_position(); + /* bytes copied */ return local_sg_elem.size; } @@ -160,9 +145,6 @@ int dma_copy_new(struct dma_copy *dc) trace_dma_error("dma_copy_new() error: dc->chan is NULL"); return -ENODEV; } - - dc->complete.timeout = 100; /* wait 100 usecs for DMA to finish */ - dma_set_cb(dc->chan, DMA_CB_TYPE_IRQ, dma_complete, &dc->complete); #endif return 0; diff --git a/src/ipc/ipc-host-ptable.c b/src/ipc/ipc-host-ptable.c index 2e9f368dffb6..7b30cff8bca6 100644 --- a/src/ipc/ipc-host-ptable.c +++ b/src/ipc/ipc-host-ptable.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -76,14 +75,6 @@ static int ipc_parse_page_descriptors(uint8_t *page_table, return 0; } -static void dma_complete(void *data, uint32_t type, struct dma_cb_data *next) -{ - completion_t *complete = data; - - if (type == DMA_CB_TYPE_IRQ) - wait_completed(complete); -} - /* * Copy the audio buffer page tables from the host to the DSP max of 4K. */ @@ -92,7 +83,6 @@ static int ipc_get_page_descriptors(struct dma *dmac, uint8_t *page_table, { struct dma_sg_config config; struct dma_sg_elem elem; - completion_t complete; struct dma_chan_data *chan; int ret = 0; @@ -129,25 +119,14 @@ static int ipc_get_page_descriptors(struct dma *dmac, uint8_t *page_table, goto out; } - /* set up callback */ - dma_set_cb(chan, DMA_CB_TYPE_IRQ, dma_complete, &complete); - - wait_init(&complete); - /* start the copy of page table to DSP */ - ret = dma_start(chan); + ret = dma_copy(chan, elem.size, DMA_COPY_ONE_SHOT); if (ret < 0) { trace_ipc_error("ipc_get_page_descriptors() error: " "dma_start() failed"); goto out; } - /* wait for DMA to complete */ - ret = poll_for_completion_delay(&complete, PLATFORM_DMA_TIMEOUT); - if (ret < 0) - trace_ipc_error("ipc_get_page_descriptors() error: " - "poll_for_completion_delay() failed"); - /* compressed page tables now in buffer at _ipc->page_table */ out: dma_channel_put(chan); diff --git a/src/lib/agent.c b/src/lib/agent.c index 12af04f464b8..7e2d9b337f80 100644 --- a/src/lib/agent.c +++ b/src/lib/agent.c @@ -83,7 +83,7 @@ void sa_init(struct sof *sof) sa->last_idle = platform_timer_get(platform_timer) + sa->ticks; sa->is_active = true; - schedule_task_init(&sa->work, SOF_SCHEDULE_LL, SOF_TASK_PRI_HIGH, + schedule_task_init(&sa->work, SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_HIGH, validate, NULL, sa, 0, 0); schedule_task(&sa->work, PLATFORM_IDLE_TIME, PLATFORM_IDLE_TIME); diff --git a/src/platform/Kconfig b/src/platform/Kconfig index 1d62e59b1cfc..69f2f1d1875c 100644 --- a/src/platform/Kconfig +++ b/src/platform/Kconfig @@ -9,7 +9,7 @@ choice config BAYTRAIL bool "Build for Baytrail" select HOST_PTABLE - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_SUSPEND_DRAIN select DMA_FIFO_PARTITION select DW @@ -22,7 +22,7 @@ config BAYTRAIL config CHERRYTRAIL bool "Build for Cherrytrail" select HOST_PTABLE - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_SUSPEND_DRAIN select DMA_FIFO_PARTITION select DW @@ -35,7 +35,7 @@ config CHERRYTRAIL config HASWELL bool "Build for Haswell" select HOST_PTABLE - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DW select DW_DMA select HAVE_RESET_VECTOR_ROM @@ -46,7 +46,7 @@ config HASWELL config BROADWELL bool "Build for Broadwell" select HOST_PTABLE - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DW select DW_DMA select HAVE_RESET_VECTOR_ROM @@ -78,7 +78,7 @@ config CANNONLAKE select DW_DMA select MEM_WND select HW_LLI - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_FIFO_PARTITION select CAVS select CAVS_VERSION_1_8 @@ -96,7 +96,7 @@ config SUECREEK select INTEL_IOMUX select DW_GPIO select HW_LLI - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_FIFO_PARTITION select CAVS select CAVS_VERSION_2_0 @@ -113,7 +113,7 @@ config ICELAKE select DW_DMA select MEM_WND select HW_LLI - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_FIFO_PARTITION select CAVS select CAVS_VERSION_2_0 @@ -132,7 +132,7 @@ config TIGERLAKE select TASK_HAVE_PRIORITY_LOW select TASK_HAVE_PRIORITY_MEDIUM select HW_LLI - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_FIFO_PARTITION select CAVS select CAVS_VERSION_2_5 diff --git a/src/platform/apollolake/include/platform/lib/dma.h b/src/platform/apollolake/include/platform/lib/dma.h index 2af38d9ec1ad..dad10e016cdd 100644 --- a/src/platform/apollolake/include/platform/lib/dma.h +++ b/src/platform/apollolake/include/platform/lib/dma.h @@ -11,6 +11,12 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 6 + +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 8 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/apollolake/include/platform/platform.h b/src/platform/apollolake/include/platform/platform.h index dfbeef351b56..f0d0ae0e4fe7 100644 --- a/src/platform/apollolake/include/platform/platform.h +++ b/src/platform/apollolake/include/platform/platform.h @@ -143,6 +143,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/baytrail/include/platform/drivers/dw-dma.h b/src/platform/baytrail/include/platform/drivers/dw-dma.h index c18fe75c5e2f..4d9dcec17c7d 100644 --- a/src/platform/baytrail/include/platform/drivers/dw-dma.h +++ b/src/platform/baytrail/include/platform/drivers/dw-dma.h @@ -11,6 +11,17 @@ #define __PLATFORM_DRIVERS_DW_DMA_H__ #include +#include + +/* number of supported DW-DMACs */ +#if CONFIG_CHERRYTRAIL_EXTRA_DW_DMA +#define PLATFORM_NUM_DW_DMACS 3 +#else +#define PLATFORM_NUM_DW_DMACS 2 +#endif + +/* index of the first DW-DMAC in the array */ +#define PLATFORM_DW_DMA_INDEX 0 /* CTL_HI */ #define DW_CTLH_CLASS(x) SET_BITS(31, 29, x) diff --git a/src/platform/baytrail/include/platform/lib/dma.h b/src/platform/baytrail/include/platform/lib/dma.h index ff9b6780db3f..161321dbf5d1 100644 --- a/src/platform/baytrail/include/platform/lib/dma.h +++ b/src/platform/baytrail/include/platform/lib/dma.h @@ -19,6 +19,9 @@ #define PLATFORM_NUM_DMACS 2 #endif +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 8 + #define DMA_ID_DMAC0 0 #define DMA_ID_DMAC1 1 #define DMA_ID_DMAC2 2 diff --git a/src/platform/baytrail/include/platform/platform.h b/src/platform/baytrail/include/platform/platform.h index 16613f9ffa42..3ff76791b198 100644 --- a/src/platform/baytrail/include/platform/platform.h +++ b/src/platform/baytrail/include/platform/platform.h @@ -128,6 +128,7 @@ static inline void platform_wait_for_interrupt(int level) extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/baytrail/platform.c b/src/platform/baytrail/platform.c index 17d9e50b3979..ee6ab0642e46 100644 --- a/src/platform/baytrail/platform.c +++ b/src/platform/baytrail/platform.c @@ -6,6 +6,7 @@ // Keyon Jie #include +#include #include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include @@ -123,6 +125,7 @@ struct timer timer = { struct timer *platform_timer = &timer; struct ll_schedule_domain *platform_timer_domain; +struct ll_schedule_domain *platform_dma_domain; int platform_boot_complete(uint32_t boot_message) { @@ -189,12 +192,20 @@ int platform_init(struct sof *sof) trace_point(TRACE_BOOT_PLATFORM_SCHED); scheduler_init_edf(sof); - /* init low latency domains and schedulers */ + /* init low latency timer domain and scheduler */ platform_timer_domain = timer_domain_init(platform_timer, PLATFORM_DEFAULT_CLOCK, PLATFORM_LL_DEFAULT_TIMEOUT); scheduler_init_ll(platform_timer_domain); + /* init low latency multi channel DW-DMA domain and scheduler */ + platform_dma_domain = + dma_multi_chan_domain_init( + &dma[PLATFORM_DW_DMA_INDEX], + PLATFORM_NUM_DW_DMACS, + PLATFORM_DEFAULT_CLOCK, true); + scheduler_init_ll(platform_dma_domain); + /* init the system agent */ trace_point(TRACE_BOOT_PLATFORM_AGENT); sa_init(sof); diff --git a/src/platform/cannonlake/include/platform/lib/dma.h b/src/platform/cannonlake/include/platform/lib/dma.h index 3768296b9b4b..881d870e5b64 100644 --- a/src/platform/cannonlake/include/platform/lib/dma.h +++ b/src/platform/cannonlake/include/platform/lib/dma.h @@ -12,6 +12,12 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 6 + +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 9 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/cannonlake/include/platform/platform.h b/src/platform/cannonlake/include/platform/platform.h index 318e03af7363..eb0dce32a4ac 100644 --- a/src/platform/cannonlake/include/platform/platform.h +++ b/src/platform/cannonlake/include/platform/platform.h @@ -140,6 +140,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/haswell/include/platform/drivers/dw-dma.h b/src/platform/haswell/include/platform/drivers/dw-dma.h index 16e82f5d496f..dbf32d713e89 100644 --- a/src/platform/haswell/include/platform/drivers/dw-dma.h +++ b/src/platform/haswell/include/platform/drivers/dw-dma.h @@ -12,6 +12,12 @@ #include +/* number of supported DW-DMACs */ +#define PLATFORM_NUM_DW_DMACS 2 + +/* index of the first DW-DMAC in the array */ +#define PLATFORM_DW_DMA_INDEX 0 + /* CTL_HI */ #define DW_CTLH_DONE(x) SET_BIT(12, x) #define DW_CTLH_BLOCK_TS_MASK MASK(11, 0) diff --git a/src/platform/haswell/include/platform/lib/dma.h b/src/platform/haswell/include/platform/lib/dma.h index 526eb9c9bb15..ef214979bbbb 100644 --- a/src/platform/haswell/include/platform/lib/dma.h +++ b/src/platform/haswell/include/platform/lib/dma.h @@ -12,6 +12,9 @@ #define PLATFORM_NUM_DMACS 2 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 8 + #define DMA_ID_DMAC0 0 #define DMA_ID_DMAC1 1 diff --git a/src/platform/haswell/include/platform/lib/memory.h b/src/platform/haswell/include/platform/lib/memory.h index 325e23b8eb58..9b1c36f1b0b4 100644 --- a/src/platform/haswell/include/platform/lib/memory.h +++ b/src/platform/haswell/include/platform/lib/memory.h @@ -100,7 +100,7 @@ void platform_init_memmap(void); #define SOF_DATA_SIZE 0xC000 #define HEAP_SYSTEM_BASE (DRAM0_BASE + SOF_DATA_SIZE) -#define HEAP_SYSTEM_SIZE 0x3000 +#define HEAP_SYSTEM_SIZE 0x4000 #define HEAP_SYSTEM_0_BASE HEAP_SYSTEM_BASE diff --git a/src/platform/haswell/include/platform/platform.h b/src/platform/haswell/include/platform/platform.h index e68c56b74ef5..e8cfc4c36225 100644 --- a/src/platform/haswell/include/platform/platform.h +++ b/src/platform/haswell/include/platform/platform.h @@ -117,6 +117,7 @@ static inline void platform_wait_for_interrupt(int level) extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/haswell/platform.c b/src/platform/haswell/platform.c index 46cc6ed67b7d..0aeae47284f0 100644 --- a/src/platform/haswell/platform.c +++ b/src/platform/haswell/platform.c @@ -5,6 +5,7 @@ // Author: Liam Girdwood #include +#include #include #include #include @@ -120,6 +121,7 @@ struct timer timer = { struct timer *platform_timer = &timer; struct ll_schedule_domain *platform_timer_domain; +struct ll_schedule_domain *platform_dma_domain; int platform_boot_complete(uint32_t boot_message) { @@ -178,12 +180,20 @@ int platform_init(struct sof *sof) trace_point(TRACE_BOOT_PLATFORM_SCHED); scheduler_init_edf(sof); - /* init low latency domains and schedulers */ + /* init low latency timer domain and scheduler */ platform_timer_domain = timer_domain_init(platform_timer, PLATFORM_DEFAULT_CLOCK, PLATFORM_LL_DEFAULT_TIMEOUT); scheduler_init_ll(platform_timer_domain); + /* init low latency multi channel DW-DMA domain and scheduler */ + platform_dma_domain = + dma_multi_chan_domain_init( + &dma[PLATFORM_DW_DMA_INDEX], + PLATFORM_NUM_DW_DMACS, + PLATFORM_DEFAULT_CLOCK, true); + scheduler_init_ll(platform_dma_domain); + /* init the system agent */ trace_point(TRACE_BOOT_PLATFORM_AGENT); sa_init(sof); diff --git a/src/platform/icelake/include/platform/lib/dma.h b/src/platform/icelake/include/platform/lib/dma.h index 3768296b9b4b..881d870e5b64 100644 --- a/src/platform/icelake/include/platform/lib/dma.h +++ b/src/platform/icelake/include/platform/lib/dma.h @@ -12,6 +12,12 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 6 + +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 9 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/icelake/include/platform/platform.h b/src/platform/icelake/include/platform/platform.h index f81681950118..fb2d51b80207 100644 --- a/src/platform/icelake/include/platform/platform.h +++ b/src/platform/icelake/include/platform/platform.h @@ -140,6 +140,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/imx8/include/platform/lib/dma.h b/src/platform/imx8/include/platform/lib/dma.h index 7b7c26302478..af2e9e31b811 100644 --- a/src/platform/imx8/include/platform/lib/dma.h +++ b/src/platform/imx8/include/platform/lib/dma.h @@ -12,9 +12,14 @@ #define PLATFORM_NUM_DMACS 2 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 32 + #define DMA_ID_EDMA0 0 #define DMA_ID_HOST 1 +#define dma_chan_irq(dma, chan) (dma_irq(dma) + chan) + int dmac_init(void); #endif /* __PLATFORM_LIB_DMA_H__ */ diff --git a/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h b/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h index 75f0ded6b999..4f2bcaa4e393 100644 --- a/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h +++ b/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h @@ -11,6 +11,17 @@ #define __CAVS_LIB_DW_DMA_H__ #include +#include + +/* number of supported DW-DMACs */ +#if CONFIG_SUECREEK +#define PLATFORM_NUM_DW_DMACS 3 +#else +#define PLATFORM_NUM_DW_DMACS 2 +#endif + +/* index of the first DW-DMAC in the array */ +#define PLATFORM_DW_DMA_INDEX 0 /* CTL_HI */ #define DW_CTLH_CLASS(x) SET_BITS(31, 29, x) diff --git a/src/platform/intel/cavs/lib/dma.c b/src/platform/intel/cavs/lib/dma.c index 2fc25cc904f4..82dbd7060d52 100644 --- a/src/platform/intel/cavs/lib/dma.c +++ b/src/platform/intel/cavs/lib/dma.c @@ -22,18 +22,15 @@ #define DMAC_HOST_OUT_CHANNELS_COUNT 6 #define DMAC_LINK_IN_CHANNELS_COUNT 8 #define DMAC_LINK_OUT_CHANNELS_COUNT 8 -#define CAVS_PLATFORM_NUM_DMACS 6 #elif CONFIG_CANNONLAKE || CONFIG_ICELAKE || CONFIG_TIGERLAKE #define DMAC0_CLASS 6 #define DMAC1_CLASS 7 #define DMAC_HOST_OUT_CHANNELS_COUNT 9 #define DMAC_LINK_IN_CHANNELS_COUNT 9 #define DMAC_LINK_OUT_CHANNELS_COUNT 7 -#define CAVS_PLATFORM_NUM_DMACS 6 #elif CONFIG_SUECREEK #define DMAC0_CLASS 6 #define DMAC1_CLASS 7 -#define CAVS_PLATFORM_NUM_DMACS 3 #endif static struct dw_drv_plat_data dmac0 = { @@ -107,7 +104,7 @@ static struct dw_drv_plat_data dmac1 = { }; #if CONFIG_SUECREEK -static struct dma dma[CAVS_PLATFORM_NUM_DMACS] = { +struct dma dma[PLATFORM_NUM_DMACS] = { { /* LP GP DMAC 0 */ .plat_data = { .id = DMA_GP_LP_DMAC0, @@ -156,7 +153,7 @@ static struct dma dma[CAVS_PLATFORM_NUM_DMACS] = { }; #else -static struct dma dma[CAVS_PLATFORM_NUM_DMACS] = { +struct dma dma[PLATFORM_NUM_DMACS] = { { /* Low Power GP DMAC 0 */ .plat_data = { .id = DMA_GP_LP_DMAC0, diff --git a/src/platform/intel/cavs/platform.c b/src/platform/intel/cavs/platform.c index e7d7a1e4739f..d4b6a76f55b9 100644 --- a/src/platform/intel/cavs/platform.c +++ b/src/platform/intel/cavs/platform.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,8 @@ #include #include #include +#include +#include #include #include #include @@ -242,6 +245,7 @@ struct timer timer = { struct timer *platform_timer = &timer; struct ll_schedule_domain *platform_timer_domain; +struct ll_schedule_domain *platform_dma_domain; #if CONFIG_DW_SPI @@ -376,12 +380,21 @@ int platform_init(struct sof *sof) trace_point(TRACE_BOOT_PLATFORM_SCHED); scheduler_init_edf(sof); - /* init low latency domains and schedulers */ + /* init low latency timer domain and scheduler */ platform_timer_domain = timer_domain_init(platform_timer, PLATFORM_DEFAULT_CLOCK, PLATFORM_LL_DEFAULT_TIMEOUT); scheduler_init_ll(platform_timer_domain); + /* init low latency multi channel DW-DMA domain and scheduler */ + platform_dma_domain = + dma_multi_chan_domain_init( + &dma[PLATFORM_DW_DMA_INDEX], + PLATFORM_NUM_DW_DMACS, + PLATFORM_DEFAULT_CLOCK, + IS_ENABLED(CONFIG_DW_DMA_AGGREGATED_IRQ)); + scheduler_init_ll(platform_dma_domain); + /* init the system agent */ trace_point(TRACE_BOOT_PLATFORM_AGENT); sa_init(sof); diff --git a/src/platform/suecreek/include/platform/lib/dma.h b/src/platform/suecreek/include/platform/lib/dma.h index b14ab58e6fde..b6dfcd35b0cc 100644 --- a/src/platform/suecreek/include/platform/lib/dma.h +++ b/src/platform/suecreek/include/platform/lib/dma.h @@ -12,6 +12,12 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 3 + +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 8 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/suecreek/include/platform/platform.h b/src/platform/suecreek/include/platform/platform.h index ab59378da92c..61a03ea7562c 100644 --- a/src/platform/suecreek/include/platform/platform.h +++ b/src/platform/suecreek/include/platform/platform.h @@ -150,6 +150,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/tigerlake/include/platform/lib/dma.h b/src/platform/tigerlake/include/platform/lib/dma.h index 3768296b9b4b..881d870e5b64 100644 --- a/src/platform/tigerlake/include/platform/lib/dma.h +++ b/src/platform/tigerlake/include/platform/lib/dma.h @@ -12,6 +12,12 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 6 + +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 9 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/tigerlake/include/platform/platform.h b/src/platform/tigerlake/include/platform/platform.h index 9f483b446b8f..bf1c0783febd 100644 --- a/src/platform/tigerlake/include/platform/platform.h +++ b/src/platform/tigerlake/include/platform/platform.h @@ -140,6 +140,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/schedule/CMakeLists.txt b/src/schedule/CMakeLists.txt index 2817fbe1f259..4ffb0ef2ac56 100644 --- a/src/schedule/CMakeLists.txt +++ b/src/schedule/CMakeLists.txt @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause add_local_sources(sof + dma_multi_chan_domain.c edf_schedule.c ll_schedule.c schedule.c diff --git a/src/schedule/dma_multi_chan_domain.c b/src/schedule/dma_multi_chan_domain.c new file mode 100644 index 000000000000..11141e70aef8 --- /dev/null +++ b/src/schedule/dma_multi_chan_domain.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019 Intel Corporation. All rights reserved. +// +// Author: Tomasz Lauda + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct dma_domain_data { + int irq; + struct task *task; + void (*handler)(void *arg); + void *arg; +}; + +struct dma_domain { + struct dma *dma_array; /* pointer to scheduling DMAs */ + uint32_t num_dma; /* number of scheduling DMAs */ + bool aggregated_irq; /* true if aggregated interrupts */ + + /* mask of currently running channels */ + uint32_t channel_mask[PLATFORM_NUM_DMACS][PLATFORM_CORE_COUNT]; + /* array of arguments for aggregated mode */ + struct dma_domain_data *arg[PLATFORM_NUM_DMACS][PLATFORM_CORE_COUNT]; + /* array of registered channels data */ + struct dma_domain_data data[PLATFORM_NUM_DMACS][PLATFORM_MAX_DMA_CHAN]; +}; + +struct ll_schedule_domain_ops dma_multi_chan_domain_ops; + +/** + * \brief Generic DMA interrupt handler. + * \param[in,out] data Pointer to DMA domain data. + */ +static void dma_multi_chan_domain_irq_handler(void *data) +{ + struct dma_domain_data *domain_data = data; + + /* just call registered handler */ + domain_data->handler(domain_data->arg); +} + +/** + * \brief Registers and enables selected DMA interrupt. + * \param[in,out] data Pointer to DMA domain data. + * \param[in,out] handler Pointer to DMA interrupt handler. + * \return Error code. + */ +static int dma_multi_chan_domain_irq_register(struct dma_domain_data *data, + void (*handler)(void *arg)) +{ + int ret; + + /* always go through dma_multi_chan_domain_irq_handler, + * so we have different arg registered for every channel + */ + ret = interrupt_register(data->irq, IRQ_AUTO_UNMASK, + dma_multi_chan_domain_irq_handler, data); + if (ret < 0) + return ret; + + interrupt_enable(data->irq, data); + + return 0; +} + +/** + * \brief Registers task to DMA domain. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] period Period of the scheduled task. + * \param[in,out] task Task to be registered. + * \param[in,out] handler Pointer to DMA interrupt handler. + * \param[in,out] arg Pointer to DMA interrupt handler's argument. + * \return Error code. + * + * Keeps track of potential double registrations and handles non aggregated + * DMA interrupts (different irq number per DMA channel). + */ +static int dma_multi_chan_domain_register(struct ll_schedule_domain *domain, + uint64_t period, struct task *task, + void (*handler)(void *arg), void *arg) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma *dmas = dma_domain->dma_array; + int core = cpu_get_id(); + int ret; + int i; + int j; + + for (i = 0; i < dma_domain->num_dma; ++i) { + for (j = 0; j < dmas[i].plat_data.channels; ++j) { + /* channel not set as scheduling source */ + if (!dma_is_scheduling_source(&dmas[i].chan[j])) + continue; + + /* channel not running */ + if (dmas[i].chan[j].status != COMP_STATE_ACTIVE) + continue; + + /* channel owned by different core */ + if (core != dmas[i].chan[j].core) + continue; + + /* channel has been already running */ + if (dma_domain->channel_mask[i][core] & BIT(j)) + continue; + + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_CLEAR); + + /* register only if not aggregated or not registered */ + if (!dma_domain->aggregated_irq || + !dma_domain->channel_mask[i][core]) { + ret = dma_multi_chan_domain_irq_register( + &dma_domain->data[i][j], + handler); + if (ret < 0) + return ret; + + dma_domain->data[i][j].handler = handler; + dma_domain->data[i][j].arg = arg; + + /* needed to unregister aggregated interrupts */ + dma_domain->arg[i][core] = + &dma_domain->data[i][j]; + } + + interrupt_clear_mask(dma_domain->data[i][j].irq, + BIT(j)); + + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_UNMASK); + + dma_domain->data[i][j].task = task; + dma_domain->channel_mask[i][core] |= BIT(j); + + return 0; + } + } + + return 0; +} + +/** + * \brief Unregisters and disables selected DMA interrupt. + * \param[in,out] data data Pointer to DMA domain data. + */ +static void dma_multi_chan_domain_irq_unregister(struct dma_domain_data *data) +{ + interrupt_disable(data->irq, data); + + interrupt_unregister(data->irq, data); +} + +/** + * \brief Unregisters task from DMA domain. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] num_tasks Number of currently scheduled tasks. + */ +static void dma_multi_chan_domain_unregister(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma *dmas = dma_domain->dma_array; + int core = cpu_get_id(); + int i; + int j; + + for (i = 0; i < dma_domain->num_dma; ++i) { + for (j = 0; j < dmas[i].plat_data.channels; ++j) { + /* channel not set as scheduling source */ + if (!dma_is_scheduling_source(&dmas[i].chan[j])) + continue; + + /* channel still running */ + if (dmas[i].chan[j].status == COMP_STATE_ACTIVE) + continue; + + /* channel owned by different core */ + if (core != dmas[i].chan[j].core) + continue; + + /* channel hasn't been running */ + if (!(dma_domain->channel_mask[i][core] & BIT(j))) + continue; + + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_MASK); + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_CLEAR); + interrupt_clear_mask(dma_domain->data[i][j].irq, + BIT(j)); + + dma_domain->data[i][j].task = NULL; + dma_domain->channel_mask[i][core] &= ~BIT(j); + + /* unregister interrupt */ + if (!dma_domain->aggregated_irq) + dma_multi_chan_domain_irq_unregister( + &dma_domain->data[i][j]); + else if (!dma_domain->channel_mask[i][core]) + dma_multi_chan_domain_irq_unregister( + dma_domain->arg[i][core]); + + return; + } + } +} + +/** + * \brief Checks if given task should be executed. + * \param[in,out] domain Pointer to schedule domain. + * \param[in,out] task Task to be checked. + * \return True is task should be executed, false otherwise. + */ +static bool dma_multi_chan_domain_is_pending(struct ll_schedule_domain *domain, + struct task *task) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma *dmas = dma_domain->dma_array; + uint32_t status; + int i; + int j; + + for (i = 0; i < dma_domain->num_dma; ++i) { + for (j = 0; j < dmas[i].plat_data.channels; ++j) { + status = dma_interrupt(&dmas[i].chan[j], + DMA_IRQ_STATUS_GET); + if (!status) + continue; + + /* not this task */ + if (dma_domain->data[i][j].task != task) + continue; + + /* clear interrupt */ + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_CLEAR); + interrupt_clear_mask(dma_domain->data[i][j].irq, + BIT(j)); + + return true; + } + } + + return false; +} + +/** + * \brief Initializes DMA multichannel scheduling domain. + * \param[in,out] dma_array Array of DMAs to be scheduled on. + * \param[in] num_dma Number of DMAs passed as dma_array. + * \param[in] clk Platform clock to base calculations on. + * \param[in] aggregated_irq True if all DMAs share the same interrupt line. + * \return Pointer to initialized scheduling domain object. + */ +struct ll_schedule_domain *dma_multi_chan_domain_init(struct dma *dma_array, + uint32_t num_dma, int clk, + bool aggregated_irq) +{ + struct ll_schedule_domain *domain; + struct dma_domain *dma_domain; + struct dma *dma; + int i; + int j; + + domain = domain_init(SOF_SCHEDULE_LL_DMA, clk, + &dma_multi_chan_domain_ops); + + dma_domain = rzalloc(RZONE_SYS | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, + sizeof(*dma_domain)); + dma_domain->dma_array = dma_array; + dma_domain->num_dma = num_dma; + dma_domain->aggregated_irq = aggregated_irq; + + /* retrieve IRQ numbers for each DMA channel */ + for (i = 0; i < num_dma; ++i) { + dma = &dma_array[i]; + for (j = 0; j < dma->plat_data.channels; ++j) + dma_domain->data[i][j].irq = interrupt_get_irq( + dma_chan_irq(dma, j), dma_irq_name(dma)); + } + + ll_sch_domain_set_pdata(domain, dma_domain); + + return domain; +} + +struct ll_schedule_domain_ops dma_multi_chan_domain_ops = { + .domain_register = dma_multi_chan_domain_register, + .domain_unregister = dma_multi_chan_domain_unregister, + .domain_is_pending = dma_multi_chan_domain_is_pending, + .domain_set = NULL, + .domain_enable = NULL, + .domain_disable = NULL, + .domain_clear = NULL, +}; diff --git a/src/schedule/timer_domain.c b/src/schedule/timer_domain.c index ae818c4c7f19..214fb86e808b 100644 --- a/src/schedule/timer_domain.c +++ b/src/schedule/timer_domain.c @@ -117,7 +117,7 @@ struct ll_schedule_domain *timer_domain_init(struct timer *timer, int clk, struct ll_schedule_domain *domain; struct timer_domain *timer_domain; - domain = domain_init(SOF_SCHEDULE_LL, clk, &timer_domain_ops); + domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, &timer_domain_ops); timer_domain = rzalloc(RZONE_SYS | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, sizeof(*timer_domain)); diff --git a/src/trace/dma-trace.c b/src/trace/dma-trace.c index 07044d1b9994..63aaa0b76f72 100644 --- a/src/trace/dma-trace.c +++ b/src/trace/dma-trace.c @@ -138,7 +138,7 @@ int dma_trace_init_complete(struct dma_trace_data *d) return ret; } - schedule_task_init(&d->dmat_work, SOF_SCHEDULE_LL, + schedule_task_init(&d->dmat_work, SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_MED, trace_work, NULL, d, 0, 0); return 0; diff --git a/test/cmocka/src/audio/pipeline/pipeline_connection_mocks.c b/test/cmocka/src/audio/pipeline/pipeline_connection_mocks.c index c1aaf4743a68..07805615cda9 100644 --- a/test/cmocka/src/audio/pipeline/pipeline_connection_mocks.c +++ b/test/cmocka/src/audio/pipeline/pipeline_connection_mocks.c @@ -37,7 +37,9 @@ struct pipeline_connect_data *get_standard_connect_objects(void) pipe->ipc_pipe = pipe_desc; pipe->status = COMP_STATE_INIT; - pipe->pipe_task.type = SOF_SCHEDULE_EDF; + + pipe->pipe_task = calloc(sizeof(struct task), 1); + pipe->pipe_task->type = SOF_SCHEDULE_EDF; schedulers = calloc(sizeof(struct schedulers), 1); list_init(&schedulers->list); diff --git a/test/cmocka/src/audio/pipeline/pipeline_free.c b/test/cmocka/src/audio/pipeline/pipeline_free.c index 3409fc8f4105..5c616413129c 100644 --- a/test/cmocka/src/audio/pipeline/pipeline_free.c +++ b/test/cmocka/src/audio/pipeline/pipeline_free.c @@ -73,9 +73,9 @@ static void test_audio_pipeline_free_sheduler_task_free(void **state) /*Testing component*/ pipeline_free(&result); - assert_int_equal(result.pipe_task.state, SOF_TASK_STATE_FREE); - assert_ptr_equal(NULL, result.pipe_task.data); - assert_ptr_equal(NULL, result.pipe_task.run); + assert_int_equal(result.pipe_task->state, SOF_TASK_STATE_FREE); + assert_ptr_equal(NULL, result.pipe_task->data); + assert_ptr_equal(NULL, result.pipe_task->run); } static void test_audio_pipeline_free_disconnect_full(void **state)