audio: buffer: prepare to allocate on vregion

Buffers, that are accessible to userspace DP modules, have to migrate
to vregion together with the latter. Prepare them for the migration.

Signed-off-by: Guennadi Liakhovetski <guennadi.liakhovetski@linux.intel.com>

Author: lyakh

src/audio/buffers/comp_buffer.c

@@ -17,6 +17,7 @@
 #include <rtos/alloc.h>
 #include <rtos/cache.h>
 #include <sof/lib/notifier.h>
+#include <sof/lib/vregion.h>
 #include <sof/list.h>
 #include <sof/schedule/dp_schedule.h>
 #include <rtos/spinlock.h>
@@ -159,15 +160,19 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer)
 	/* In case some listeners didn't unregister from buffer's callbacks */
 	notifier_unregister_all(NULL, buffer);
 
-	struct k_heap *heap = buffer->audio_buffer.heap;
+	struct mod_alloc_ctx *alloc = buffer->audio_buffer.alloc;
 
 	rfree(buffer->stream.addr);
-	sof_heap_free(heap, buffer);
-	if (heap) {
-		struct dp_heap_user *mod_heap_user = container_of(heap, struct dp_heap_user, heap);
+	if (alloc && alloc->vreg)
+		vregion_free(alloc->vreg, buffer);
+	else
+		sof_heap_free(alloc ? alloc->heap : NULL, buffer);
 
-		if (!--mod_heap_user->client_count)
-			rfree(mod_heap_user);
+	if (alloc && alloc->client && !--alloc->client->client_count) {
+		rfree(alloc->client);
+		alloc->client = NULL;
+		/* NULL is allowed */
+		vregion_put(alloc->vreg);
 	}
 }
 
@@ -198,7 +203,7 @@ static const struct audio_buffer_ops audio_buffer_ops = {
 	.set_alignment_constants = comp_buffer_set_alignment_constants,
 };
 
-static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap,
+static struct comp_buffer *buffer_alloc_struct(struct mod_alloc_ctx *alloc,
 					       void *stream_addr, size_t size,
 					       uint32_t flags, bool is_shared)
 {
@@ -210,7 +215,12 @@ static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap,
 	if (is_shared)
 		flags |= SOF_MEM_FLAG_COHERENT;
 
-	buffer = sof_heap_alloc(heap, flags, sizeof(*buffer), 0);
+	if (!alloc || !alloc->vreg)
+		buffer = sof_heap_alloc(alloc ? alloc->heap : NULL, flags, sizeof(*buffer), 0);
+	else if (is_shared)
+		buffer = vregion_alloc_coherent(alloc->vreg, VREGION_MEM_TYPE_INTERIM, sizeof(*buffer));
+	else
+		buffer = vregion_alloc(alloc->vreg, VREGION_MEM_TYPE_INTERIM, sizeof(*buffer));
 	if (!buffer) {
 		tr_err(&buffer_tr, "could not alloc structure");
 		return NULL;
@@ -232,16 +242,16 @@ static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap,
 
 	audio_stream_set_underrun(&buffer->stream, !!(flags & SOF_BUF_UNDERRUN_PERMITTED));
 	audio_stream_set_overrun(&buffer->stream, !!(flags & SOF_BUF_OVERRUN_PERMITTED));
-	buffer->audio_buffer.heap = heap;
+	buffer->audio_buffer.alloc = alloc;
 
 	comp_buffer_reset_source_list(buffer);
 	comp_buffer_reset_sink_list(buffer);
 
 	return buffer;
 }
 
-struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flags, uint32_t align,
-				 bool is_shared)
+struct comp_buffer *buffer_alloc(struct mod_alloc_ctx *alloc, size_t size, uint32_t flags,
+				 uint32_t align, bool is_shared)
 {
 	struct comp_buffer *buffer;
 	void *stream_addr;
@@ -261,7 +271,7 @@ struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flag
 		return NULL;
 	}
 
-	buffer = buffer_alloc_struct(heap, stream_addr, size, flags, is_shared);
+	buffer = buffer_alloc_struct(alloc, stream_addr, size, flags, is_shared);
 	if (!buffer) {
 		tr_err(&buffer_tr, "could not alloc buffer structure");
 		rfree(stream_addr);
@@ -270,7 +280,7 @@ struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flag
 	return buffer;
 }
 
-struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_size,
+struct comp_buffer *buffer_alloc_range(struct mod_alloc_ctx *alloc, size_t preferred_size,
 				       size_t minimum_size,
 				       uint32_t flags, uint32_t align, bool is_shared)
 {
@@ -305,7 +315,7 @@ struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_siz
 		return NULL;
 	}
 
-	buffer = buffer_alloc_struct(heap, stream_addr, size, flags, is_shared);
+	buffer = buffer_alloc_struct(alloc, stream_addr, size, flags, is_shared);
 	if (!buffer) {
 		tr_err(&buffer_tr, "could not alloc buffer structure");
 		rfree(stream_addr);

src/audio/buffers/ring_buffer.c

@@ -6,6 +6,7 @@
 #include <sof/common.h>
 #include <sof/trace/trace.h>
 #include <sof/lib/uuid.h>
+#include <sof/lib/vregion.h>
 
 #include <sof/audio/module_adapter/module/generic.h>
 #include <sof/audio/ring_buffer.h>
@@ -96,9 +97,15 @@ static void ring_buffer_free(struct sof_audio_buffer *audio_buffer)
 
 	struct ring_buffer *ring_buffer = container_of(audio_buffer,
 						       struct ring_buffer, audio_buffer);
-
-	sof_heap_free(audio_buffer->heap, (__sparse_force void *)ring_buffer->_data_buffer);
-	sof_heap_free(audio_buffer->heap, ring_buffer);
+	struct mod_alloc_ctx *alloc = audio_buffer->alloc;
+
+	if (alloc->vreg) {
+		vregion_free(alloc->vreg, (__sparse_force void *)ring_buffer->_data_buffer);
+		vregion_free(alloc->vreg, ring_buffer);
+	} else {
+		sof_heap_free(alloc->heap, (__sparse_force void *)ring_buffer->_data_buffer);
+		sof_heap_free(alloc->heap, ring_buffer);
+	}
 }
 
 static void ring_buffer_reset(struct sof_audio_buffer *audio_buffer)
@@ -287,12 +294,19 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl
 				       uint32_t id)
 {
 	struct ring_buffer *ring_buffer;
-	struct k_heap *heap = dev->mod->priv.resources.heap;
+	struct mod_alloc_ctx *alloc = &dev->mod->priv.resources.alloc;
+	struct k_heap *heap = alloc->heap;
+	struct vregion *vreg = alloc->vreg;
 	int memory_flags = (is_shared ? SOF_MEM_FLAG_COHERENT : 0) |
 			   user_get_buffer_memory_region(dev->drv);
 
 	/* allocate ring_buffer structure */
-	ring_buffer = sof_heap_alloc(heap, memory_flags, sizeof(*ring_buffer), 0);
+	if (!vreg)
+		ring_buffer = sof_heap_alloc(heap, memory_flags, sizeof(*ring_buffer), 0);
+	else if (is_shared)
+		ring_buffer = vregion_alloc_coherent(vreg, VREGION_MEM_TYPE_INTERIM, sizeof(*ring_buffer));
+	else
+		ring_buffer = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, sizeof(*ring_buffer));
 	if (!ring_buffer)
 		return NULL;
 
@@ -307,7 +321,8 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl
 	audio_buffer_init(&ring_buffer->audio_buffer, BUFFER_TYPE_RING_BUFFER,
 			  is_shared, &ring_buffer_source_ops, &ring_buffer_sink_ops,
 			  &audio_buffer_ops, NULL);
-	ring_buffer->audio_buffer.heap = heap;
+	ring_buffer->audio_buffer.alloc = alloc;
+	ring_buffer->audio_buffer.alloc->heap = heap;
 
 	/* set obs/ibs in sink/source interfaces */
 	sink_set_min_free_space(audio_buffer_get_sink(&ring_buffer->audio_buffer),
@@ -364,12 +379,21 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl
 	/* allocate data buffer - always in cached memory alias */
 	ring_buffer->data_buffer_size = ALIGN_UP(ring_buffer->data_buffer_size,
 						 PLATFORM_DCACHE_ALIGN);
-	ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *)sof_heap_alloc(heap,
-					user_get_buffer_memory_region(dev->drv),
-					ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN);
-	if (!ring_buffer->_data_buffer)
+
+	void *data_buf;
+
+	if (vreg)
+		data_buf = vregion_alloc_align(vreg, VREGION_MEM_TYPE_INTERIM, ring_buffer->data_buffer_size,
+					       PLATFORM_DCACHE_ALIGN);
+	else
+		data_buf = sof_heap_alloc(heap, user_get_buffer_memory_region(dev->drv),
+					  ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN);
+
+	if (!data_buf)
 		goto err;
 
+	ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *)data_buf;
+
 	tr_info(&ring_buffer_tr, "Ring buffer created, id: %u shared: %u min_available: %u min_free_space %u, size %u",
 		id, ring_buffer_is_shared(ring_buffer), min_available, min_free_space,
 		ring_buffer->data_buffer_size);
@@ -378,6 +402,9 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl
 	return ring_buffer;
 err:
 	tr_err(&ring_buffer_tr, "Ring buffer creation failure");
-	sof_heap_free(heap, ring_buffer);
+	if (vreg)
+		vregion_free(vreg, ring_buffer);
+	else
+		sof_heap_free(heap, ring_buffer);
 	return NULL;
 }

src/audio/module_adapter/module/generic.c

@@ -86,7 +86,7 @@ void mod_resource_init(struct processing_module *mod)
 
 	/* Init memory list */
 	list_init(&md->resources.objpool.list);
-	md->resources.objpool.heap = md->resources.heap;
+	md->resources.objpool.heap = md->resources.alloc.heap;
 	md->resources.heap_usage = 0;
 	md->resources.heap_high_water_mark = 0;
 }
@@ -159,10 +159,10 @@ void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start
 	struct module_resources *res = &mod->priv.resources;
 
 	if (size)
-		*size = res->heap->heap.init_bytes;
+		*size = res->alloc.heap->heap.init_bytes;
 
 	if (start)
-		*start = (uintptr_t)container_of(res->heap, struct dp_heap_user, heap);
+		*start = (uintptr_t)res->alloc.client;
 }
 #endif
 
@@ -195,7 +195,7 @@ void *mod_balloc_align(struct processing_module *mod, size_t size, size_t alignm
 	}
 
 	/* Allocate buffer memory for module */
-	void *ptr = sof_heap_alloc(res->heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER,
+	void *ptr = sof_heap_alloc(res->alloc.heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_LARGE_BUFFER,
 				   size, alignment);
 
 	if (!ptr) {
@@ -246,7 +246,7 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t
 	}
 
 	/* Allocate memory for module */
-	void *ptr = sof_heap_alloc(res->heap, flags, size, alignment);
+	void *ptr = sof_heap_alloc(res->alloc.heap, flags, size, alignment);
 
 	if (!ptr) {
 		comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.",
@@ -323,7 +323,7 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons
 	if (!container)
 		return NULL;
 
-	ptr = fast_get(res->heap, dram_ptr, size);
+	ptr = fast_get(res->alloc.heap, dram_ptr, size);
 	if (!ptr) {
 		container_put(mod, container);
 		return NULL;
@@ -347,7 +347,7 @@ static int free_contents(struct processing_module *mod, struct module_resource *
 
 	switch (container->type) {
 	case MOD_RES_HEAP:
-		sof_heap_free(res->heap, container->ptr);
+		sof_heap_free(res->alloc.heap, container->ptr);
 		res->heap_usage -= container->size;
 		return 0;
 #if CONFIG_COMP_BLOB
@@ -362,7 +362,7 @@ static int free_contents(struct processing_module *mod, struct module_resource *
 #else
 		mdom = NULL;
 #endif
-		fast_put(res->heap, mdom, container->sram_ptr);
+		fast_put(res->alloc.heap, mdom, container->sram_ptr);
 		return 0;
 #endif
 	default:
@@ -429,7 +429,7 @@ const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * cons
 	struct module_resources *res = &mod->priv.resources;
 
 	K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod)));
-	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->heap, sizeof(*res->heap)));
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap)));
 	K_OOPS(K_SYSCALL_MEMORY_READ(dram_ptr, size));
 
 	return z_impl_mod_fast_get(mod, dram_ptr, size);
@@ -443,7 +443,7 @@ void *z_vrfy_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t
 	struct module_resources *res = &mod->priv.resources;
 
 	K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod)));
-	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->heap, sizeof(*res->heap)));
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap)));
 
 	return z_impl_mod_alloc_ext(mod, flags, size, alignment);
 }
@@ -454,7 +454,7 @@ int z_vrfy_mod_free(struct processing_module *mod, const void *ptr)
 	struct module_resources *res = &mod->priv.resources;
 
 	K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod)));
-	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->heap, sizeof(*res->heap)));
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc.heap, sizeof(*res->alloc.heap)));
 
 	return z_impl_mod_free(mod, ptr);
 }

src/audio/module_adapter/module_adapter.c

@@ -124,7 +124,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	}
 
 	memset(mod, 0, sizeof(*mod));
-	mod->priv.resources.heap = mod_heap;
+	mod->priv.resources.alloc.heap = mod_heap;
+	mod->priv.resources.alloc.client = mod_heap_user;
 	mod_resource_init(mod);
 
 	/*
@@ -161,7 +162,7 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 
 static void module_adapter_mem_free(struct processing_module *mod)
 {
-	struct k_heap *mod_heap = mod->priv.resources.heap;
+	struct k_heap *mod_heap = mod->priv.resources.alloc.heap;
 	unsigned int domain = mod->dev->ipc_config.proc_domain;
 
 	/*
@@ -611,8 +612,8 @@ int module_adapter_prepare(struct comp_dev *dev)
 	if (list_is_empty(&mod->raw_data_buffers_list)) {
 		for (i = 0; i < mod->num_of_sinks; i++) {
 			/* allocate not shared buffer */
-			struct comp_buffer *buffer = buffer_alloc(md->resources.heap, buff_size,
-								  memory_flags,
+			struct comp_buffer *buffer = buffer_alloc(&md->resources.alloc,
+								  buff_size, memory_flags,
 								  PLATFORM_DCACHE_ALIGN,
 								  BUFFER_USAGE_NOT_SHARED);
 			uint32_t flags;
@@ -623,13 +624,9 @@ int module_adapter_prepare(struct comp_dev *dev)
 				goto free;
 			}
 
-			if (md->resources.heap && md->resources.heap != dev->drv->user_heap) {
-				struct dp_heap_user *dp_user = container_of(md->resources.heap,
-									    struct dp_heap_user,
-									    heap);
-
-				dp_user->client_count++;
-			}
+			if (md->resources.alloc.heap &&
+			    md->resources.alloc.heap != dev->drv->user_heap)
+				md->resources.alloc.client->client_count++;
 
 			irq_local_disable(flags);
 			list_item_prepend(&buffer->buffers_list, &mod->raw_data_buffers_list);

src/audio/module_adapter/module_adapter_ipc4.c

@@ -147,7 +147,7 @@ int module_adapter_init_data(struct comp_dev *dev,
 		if (cfgsz == (sizeof(*cfg) + pinsz)) {
 			dst->nb_input_pins = n_in;
 			dst->nb_output_pins = n_out;
-			dst->input_pins = sof_heap_alloc(dev->mod->priv.resources.heap,
+			dst->input_pins = sof_heap_alloc(dev->mod->priv.resources.alloc.heap,
 							 SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
 							 pinsz, 0);
 			if (!dst->input_pins)

src/include/sof/audio/audio_buffer.h

@@ -111,7 +111,7 @@ struct sof_audio_buffer {
 	 */
 	bool walking;		/**< indicates if the buffer is being walked */
 
-	struct k_heap *heap;
+	struct mod_alloc_ctx *alloc;
 };
 
 #if CONFIG_PIPELINE_2_0

src/include/sof/audio/buffer.h

@@ -211,15 +211,15 @@ struct buffer_cb_free {
 		buffer->cb_type = type;	\
 	} while (0)
 
-struct k_heap;
+struct mod_alloc_ctx;
 
 /* pipeline buffer creation and destruction */
-struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flags, uint32_t align,
-				 bool is_shared);
-struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_size,
+struct comp_buffer *buffer_alloc(struct mod_alloc_ctx *alloc, size_t size, uint32_t flags,
+				 uint32_t align, bool is_shared);
+struct comp_buffer *buffer_alloc_range(struct mod_alloc_ctx *alloc, size_t preferred_size,
 				       size_t minimum_size,
 				       uint32_t flags, uint32_t align, bool is_shared);
-struct comp_buffer *buffer_new(struct k_heap *heap, const struct sof_ipc_buffer *desc,
+struct comp_buffer *buffer_new(struct mod_alloc_ctx *alloc, const struct sof_ipc_buffer *desc,
 			       bool is_shared);
 
 int buffer_set_size(struct comp_buffer *buffer, uint32_t size, uint32_t alignment);

src/include/sof/audio/component.h

@@ -578,6 +578,15 @@ struct comp_ops {
 	uint64_t (*get_total_data_processed)(struct comp_dev *dev, uint32_t stream_no, bool input);
 };
 
+struct k_heap;
+struct vregion;
+struct dp_heap_user;
+struct mod_alloc_ctx {
+	struct k_heap *heap;
+	struct vregion *vreg;
+	struct dp_heap_user *client;
+};
+
 /**
  * Audio component base driver "class"
  * - used by all other component types.