nathan/CoolPi-Armbian-Rockchip-RK3588-Upstream-Linux-6.1-LTS-Kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
2ee46a76e236dfcdeb8b1a815dde855fe54dee0c
CoolPi-Armbian-Rockchip-RK3…/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c
Nathan bc48dd73a4 Import Armbian Linux 6.1 LTS (rk-6.1-rkr5)
2025-02-04 12:41:12 -06:00

835 lines
28 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
* (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*/
#include "mali_kbase_csf_csg.h"
#include "mali_kbase_csf_sync.h"
#include "mali_kbase_csf_util.h"
#include <mali_kbase.h>
#include <linux/version_compat_defs.h>
#if IS_ENABLED(CONFIG_SYNC_FILE)
#include "mali_kbase_sync.h"
#endif
#define CQS_UNREADABLE_LIVE_VALUE "(unavailable)"
#define CSF_SYNC_DUMP_SIZE 256
/* Number of nearby commands around the "extract_ptr" of GPU queues.
*
* [extract_ptr - MAX_NR_NEARBY_INSTR, extract_ptr + MAX_NR_NEARBY_INSTR].
*/
#define MAX_NR_NEARBY_INSTR 32
/**
* kbasep_csf_sync_get_cqs_live_u32() - Obtain live (u32) value for a CQS object.
*
* @kctx: The context of the queue.
* @obj_addr: Pointer to the CQS live 32-bit value.
* @live_val: Pointer to the u32 that will be set to the CQS object's current, live
* value.
*
* Return: 0 if successful or a negative error code on failure.
*/
static int kbasep_csf_sync_get_cqs_live_u32(struct kbase_context *kctx, u64 obj_addr, u32 *live_val)
{
struct kbase_vmap_struct *mapping;
u32 *const cpu_ptr = (u32 *)kbase_phy_alloc_mapping_get(kctx, obj_addr, &mapping);
if (!cpu_ptr)
return -1;
*live_val = *cpu_ptr;
kbase_phy_alloc_mapping_put(kctx, mapping);
return 0;
}
/**
* kbasep_csf_sync_get_cqs_live_u64() - Obtain live (u64) value for a CQS object.
*
* @kctx: The context of the queue.
* @obj_addr: Pointer to the CQS live value (32 or 64-bit).
* @live_val: Pointer to the u64 that will be set to the CQS object's current, live
* value.
*
* Return: 0 if successful or a negative error code on failure.
*/
static int kbasep_csf_sync_get_cqs_live_u64(struct kbase_context *kctx, u64 obj_addr, u64 *live_val)
{
struct kbase_vmap_struct *mapping;
u64 *cpu_ptr = (u64 *)kbase_phy_alloc_mapping_get(kctx, obj_addr, &mapping);
if (!cpu_ptr)
return -1;
*live_val = *cpu_ptr;
kbase_phy_alloc_mapping_put(kctx, mapping);
return 0;
}
/**
* kbasep_csf_sync_print_kcpu_fence_wait_or_signal() - Print details of a CSF SYNC Fence Wait
* or Fence Signal command, contained in a
* KCPU queue.
*
* @buffer: The buffer to write to.
* @length: The length of text in the buffer.
* @cmd: The KCPU Command to be printed.
* @cmd_name: The name of the command: indicates either a fence SIGNAL or WAIT.
*/
static void kbasep_csf_sync_print_kcpu_fence_wait_or_signal(char *buffer, int *length,
struct kbase_kcpu_command *cmd,
const char *cmd_name)
{
struct dma_fence *fence = NULL;
struct kbase_kcpu_command_fence_info *fence_info;
struct kbase_sync_fence_info info;
const char *timeline_name = NULL;
bool is_signaled = false;
fence_info = &cmd->info.fence;
if (kbase_kcpu_command_fence_has_force_signaled(fence_info))
return;
fence = kbase_fence_get(fence_info);
if (WARN_ON(!fence))
return;
kbase_sync_fence_info_get(fence, &info);
timeline_name = fence->ops->get_timeline_name(fence);
is_signaled = info.status > 0;
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:%s obj:0x%pK live_value:0x%.8x | ", cmd_name, fence, is_signaled);
/* Note: fence->seqno was u32 until 5.1 kernel, then u64 */
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"timeline_name:%s timeline_context:0x%.16llx fence_seqno:0x%.16llx",
timeline_name, fence->context, (u64)fence->seqno);
kbase_fence_put(fence);
}
/**
* kbasep_csf_sync_print_kcpu_cqs_wait() - Print details of a CSF SYNC CQS Wait command,
* contained in a KCPU queue.
*
* @kctx: The kbase context.
* @buffer: The buffer to write to.
* @length: The length of text in the buffer.
* @cmd: The KCPU Command to be printed.
*/
static void kbasep_csf_sync_print_kcpu_cqs_wait(struct kbase_context *kctx, char *buffer,
int *length, struct kbase_kcpu_command *cmd)
{
size_t i;
for (i = 0; i < cmd->info.cqs_wait.nr_objs; i++) {
struct base_cqs_wait_info *cqs_obj = &cmd->info.cqs_wait.objs[i];
u32 live_val;
int ret = kbasep_csf_sync_get_cqs_live_u32(kctx, cqs_obj->addr, &live_val);
bool live_val_valid = (ret >= 0);
*length += scnprintf(
buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr);
if (live_val_valid)
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", (u64)live_val);
else
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:gt arg_value:0x%.8x", cqs_obj->val);
}
}
/**
* kbasep_csf_sync_print_kcpu_cqs_set() - Print details of a CSF SYNC CQS
* Set command, contained in a KCPU queue.
*
* @kctx: The kbase context.
* @buffer: The buffer to write to.
* @length: The length of text in the buffer.
* @cmd: The KCPU Command to be printed.
*/
static void kbasep_csf_sync_print_kcpu_cqs_set(struct kbase_context *kctx, char *buffer,
int *length, struct kbase_kcpu_command *cmd)
{
size_t i;
for (i = 0; i < cmd->info.cqs_set.nr_objs; i++) {
struct base_cqs_set *cqs_obj = &cmd->info.cqs_set.objs[i];
u32 live_val;
int ret = kbasep_csf_sync_get_cqs_live_u32(kctx, cqs_obj->addr, &live_val);
bool live_val_valid = (ret >= 0);
*length +=
scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", cqs_obj->addr);
if (live_val_valid)
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", (u64)live_val);
else
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:add arg_value:0x%.8x", 1);
}
}
/**
* kbasep_csf_sync_get_wait_op_name() - Print the name of a CQS Wait Operation.
*
* @op: The numerical value of operation.
*
* Return: const static pointer to the command name, or '??' if unknown.
*/
static const char *kbasep_csf_sync_get_wait_op_name(basep_cqs_wait_operation_op op)
{
const char *string;
switch (op) {
case BASEP_CQS_WAIT_OPERATION_LE:
string = "le";
break;
case BASEP_CQS_WAIT_OPERATION_GT:
string = "gt";
break;
default:
string = "??";
break;
}
return string;
}
/**
* kbasep_csf_sync_get_set_op_name() - Print the name of a CQS Set Operation.
*
* @op: The numerical value of operation.
*
* Return: const static pointer to the command name, or '??' if unknown.
*/
static const char *kbasep_csf_sync_get_set_op_name(basep_cqs_set_operation_op op)
{
const char *string;
switch (op) {
case BASEP_CQS_SET_OPERATION_ADD:
string = "add";
break;
case BASEP_CQS_SET_OPERATION_SET:
string = "set";
break;
default:
string = "???";
break;
}
return string;
}
/**
* kbasep_csf_sync_print_kcpu_cqs_wait_op() - Print details of a CSF SYNC CQS
* Wait Operation command, contained
* in a KCPU queue.
*
* @kctx: The kbase context.
* @buffer: The buffer to write to.
* @length: The length of text in the buffer.
* @cmd: The KCPU Command to be printed.
*/
static void kbasep_csf_sync_print_kcpu_cqs_wait_op(struct kbase_context *kctx, char *buffer,
int *length, struct kbase_kcpu_command *cmd)
{
size_t i;
for (i = 0; i < cmd->info.cqs_wait.nr_objs; i++) {
struct base_cqs_wait_operation_info *wait_op =
&cmd->info.cqs_wait_operation.objs[i];
const char *op_name = kbasep_csf_sync_get_wait_op_name(wait_op->operation);
u64 live_val;
int ret = kbasep_csf_sync_get_cqs_live_u64(kctx, wait_op->addr, &live_val);
bool live_val_valid = (ret >= 0);
*length += scnprintf(
buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_WAIT_OPERATION obj:0x%.16llx live_value:", wait_op->addr);
if (live_val_valid)
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", live_val);
else
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:%s arg_value:0x%.16llx", op_name, wait_op->val);
}
}
/**
* kbasep_csf_sync_print_kcpu_cqs_set_op() - Print details of a CSF SYNC CQS
* Set Operation command, contained
* in a KCPU queue.
*
* @kctx: The kbase context.
* @buffer: The buffer to write to.
* @length: The length of text in the buffer.
* @cmd: The KCPU Command to be printed.
*/
static void kbasep_csf_sync_print_kcpu_cqs_set_op(struct kbase_context *kctx, char *buffer,
int *length, struct kbase_kcpu_command *cmd)
{
size_t i;
for (i = 0; i < cmd->info.cqs_set_operation.nr_objs; i++) {
struct base_cqs_set_operation_info *set_op = &cmd->info.cqs_set_operation.objs[i];
const char *op_name = kbasep_csf_sync_get_set_op_name(
(basep_cqs_set_operation_op)set_op->operation);
u64 live_val;
int ret = kbasep_csf_sync_get_cqs_live_u64(kctx, set_op->addr, &live_val);
bool live_val_valid = (ret >= 0);
*length +=
scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"cmd:CQS_SET_OPERATION obj:0x%.16llx live_value:", set_op->addr);
if (live_val_valid)
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
"0x%.16llx", live_val);
else
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
CQS_UNREADABLE_LIVE_VALUE);
*length += scnprintf(buffer + *length, CSF_SYNC_DUMP_SIZE - *length,
" | op:%s arg_value:0x%.16llx", op_name, set_op->val);
}
}
/**
* kbasep_csf_sync_kcpu_print_queue() - Print debug data for a KCPU queue
*
* @kctx: The kbase context.
* @kbpr: Pointer to printer instance.
* @queue: Pointer to the KCPU queue.
*/
static void kbasep_csf_sync_kcpu_print_queue(struct kbase_context *kctx,
struct kbase_kcpu_command_queue *queue,
struct kbasep_printer *kbpr)
{
char started_or_pending;
struct kbase_kcpu_command *cmd;
size_t i;
if (WARN_ON(!queue))
return;
lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
mutex_lock(&queue->lock);
for (i = 0; i != queue->num_pending_cmds; ++i) {
char buffer[CSF_SYNC_DUMP_SIZE];
int length = 0;
started_or_pending = ((i == 0) && queue->command_started) ? 'S' : 'P';
length += scnprintf(buffer, CSF_SYNC_DUMP_SIZE, "queue:KCPU-%d-%d exec:%c ",
kctx->id, queue->id, started_or_pending);
cmd = &queue->commands[(u8)(queue->start_offset + i)];
switch (cmd->type) {
#if IS_ENABLED(CONFIG_SYNC_FILE)
case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL:
kbasep_csf_sync_print_kcpu_fence_wait_or_signal(buffer, &length, cmd,
"FENCE_SIGNAL");
break;
case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT:
kbasep_csf_sync_print_kcpu_fence_wait_or_signal(buffer, &length, cmd,
"FENCE_WAIT");
break;
#endif
case BASE_KCPU_COMMAND_TYPE_CQS_WAIT:
kbasep_csf_sync_print_kcpu_cqs_wait(kctx, buffer, &length, cmd);
break;
case BASE_KCPU_COMMAND_TYPE_CQS_SET:
kbasep_csf_sync_print_kcpu_cqs_set(kctx, buffer, &length, cmd);
break;
case BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION:
kbasep_csf_sync_print_kcpu_cqs_wait_op(kctx, buffer, &length, cmd);
break;
case BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION:
kbasep_csf_sync_print_kcpu_cqs_set_op(kctx, buffer, &length, cmd);
break;
default:
length += scnprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length,
", U, Unknown blocking command");
break;
}
length += scnprintf(buffer + length, CSF_SYNC_DUMP_SIZE - length, "\n");
kbasep_print(kbpr, buffer);
}
mutex_unlock(&queue->lock);
}
int kbasep_csf_sync_kcpu_dump_print(struct kbase_context *kctx, struct kbasep_printer *kbpr)
{
unsigned long queue_idx;
mutex_lock(&kctx->csf.kcpu_queues.lock);
kbasep_print(kbpr, "CSF KCPU queues sync info (version: v" __stringify(
MALI_CSF_SYNC_DUMP_VERSION) "):\n");
kbasep_print(kbpr, "KCPU queues for ctx %d:\n", kctx->id);
queue_idx = find_first_bit(kctx->csf.kcpu_queues.in_use, KBASEP_MAX_KCPU_QUEUES);
while (queue_idx < KBASEP_MAX_KCPU_QUEUES) {
kbasep_csf_sync_kcpu_print_queue(kctx, kctx->csf.kcpu_queues.array[queue_idx],
kbpr);
queue_idx = find_next_bit(kctx->csf.kcpu_queues.in_use, KBASEP_MAX_KCPU_QUEUES,
queue_idx + 1);
}
mutex_unlock(&kctx->csf.kcpu_queues.lock);
return 0;
}
/* GPU queue related values */
#define GPU_CSF_MOVE_OPCODE ((u64)0x1)
#define GPU_CSF_MOVE32_OPCODE ((u64)0x2)
#define GPU_CSF_SYNC_ADD_OPCODE ((u64)0x25)
#define GPU_CSF_SYNC_SET_OPCODE ((u64)0x26)
#define GPU_CSF_SYNC_WAIT_OPCODE ((u64)0x27)
#define GPU_CSF_SYNC_ADD64_OPCODE ((u64)0x33)
#define GPU_CSF_SYNC_SET64_OPCODE ((u64)0x34)
#define GPU_CSF_SYNC_WAIT64_OPCODE ((u64)0x35)
#define GPU_CSF_CALL_OPCODE ((u64)0x20)
#define MAX_NR_GPU_CALLS (5)
#define INSTR_OPCODE_MASK ((u64)0xFF << 56)
#define INSTR_OPCODE_GET(value) ((value & INSTR_OPCODE_MASK) >> 56)
#define MOVE32_IMM_MASK ((u64)0xFFFFFFFFFUL)
#define MOVE_DEST_MASK ((u64)0xFF << 48)
#define MOVE_DEST_GET(value) ((value & MOVE_DEST_MASK) >> 48)
#define MOVE_IMM_MASK ((u64)0xFFFFFFFFFFFFUL)
#define SYNC_SRC0_MASK ((u64)0xFF << 40)
#define SYNC_SRC1_MASK ((u64)0xFF << 32)
#define SYNC_SRC0_GET(value) (u8)((value & SYNC_SRC0_MASK) >> 40)
#define SYNC_SRC1_GET(value) (u8)((value & SYNC_SRC1_MASK) >> 32)
#define SYNC_WAIT_CONDITION_MASK ((u64)0xF << 28)
#define SYNC_WAIT_CONDITION_GET(value) (u8)((value & SYNC_WAIT_CONDITION_MASK) >> 28)
/* Enumeration for types of GPU queue sync events for
* the purpose of dumping them through sync.
*/
enum sync_gpu_sync_type {
CSF_GPU_SYNC_WAIT,
CSF_GPU_SYNC_SET,
CSF_GPU_SYNC_ADD,
NUM_CSF_GPU_SYNC_TYPES
};
/**
* kbasep_csf_get_move_immediate_value() - Get the immediate values for sync operations
* from a MOVE instruction.
*
* @move_cmd: Raw MOVE instruction.
* @sync_addr_reg: Register identifier from SYNC_* instruction.
* @compare_val_reg: Register identifier from SYNC_* instruction.
* @sync_val: Pointer to store CQS object address for sync operation.
* @compare_val: Pointer to store compare value for sync operation.
*
* Return: True if value is obtained by checking for correct register identifier,
* or false otherwise.
*/
static bool kbasep_csf_get_move_immediate_value(u64 move_cmd, u64 sync_addr_reg,
u64 compare_val_reg, u64 *sync_val,
u64 *compare_val)
{
u64 imm_mask;
/* Verify MOVE instruction and get immediate mask */
if (INSTR_OPCODE_GET(move_cmd) == GPU_CSF_MOVE32_OPCODE)
imm_mask = MOVE32_IMM_MASK;
else if (INSTR_OPCODE_GET(move_cmd) == GPU_CSF_MOVE_OPCODE)
imm_mask = MOVE_IMM_MASK;
else
/* Error return */
return false;
/* Verify value from MOVE instruction and assign to variable */
if (sync_addr_reg == MOVE_DEST_GET(move_cmd))
*sync_val = move_cmd & imm_mask;
else if (compare_val_reg == MOVE_DEST_GET(move_cmd))
*compare_val = move_cmd & imm_mask;
else
/* Error return */
return false;
return true;
}
/** kbasep_csf_read_ringbuffer_value() - Reads a u64 from the ringbuffer at a provided
* offset.
*
* @queue: Pointer to the queue.
* @ringbuff_offset: Ringbuffer offset.
*
* Return: the u64 in the ringbuffer at the desired offset.
*/
static u64 kbasep_csf_read_ringbuffer_value(struct kbase_queue *queue, u32 ringbuff_offset)
{
u64 page_off = ringbuff_offset >> PAGE_SHIFT;
u64 offset_within_page = ringbuff_offset & ~PAGE_MASK;
struct page *page = as_page(queue->queue_reg->gpu_alloc->pages[page_off]);
u64 *ringbuffer = vmap(&page, 1, VM_MAP, pgprot_noncached(PAGE_KERNEL));
u64 value;
if (!ringbuffer) {
struct kbase_context *kctx = queue->kctx;
dev_err(kctx->kbdev->dev, "%s failed to map the buffer page for read a command!",
__func__);
/* Return an alternative 0 for dumping operation*/
value = 0;
} else {
value = ringbuffer[offset_within_page / sizeof(u64)];
vunmap(ringbuffer);
}
return value;
}
/**
* kbasep_csf_print_gpu_sync_op() - Print sync operation info for given sync command.
*
* @kbpr: Pointer to printer instance.
* @kctx: Pointer to kbase context.
* @queue: Pointer to the GPU command queue.
* @ringbuff_offset: Offset to index the ring buffer with, for the given sync command.
* (Useful for finding preceding MOVE commands)
* @instr_addr: GPU command address.
* @sync_cmd: Entire u64 of the sync command, which has both sync address and
* comparison-value encoded in it.
* @type: Type of GPU sync command (e.g. SYNC_SET, SYNC_ADD, SYNC_WAIT).
* @is_64bit: Bool to indicate if operation is 64 bit (true) or 32 bit (false).
* @follows_wait: Bool to indicate if the operation follows at least one wait
* operation. Used to determine whether it's pending or started.
*/
static void kbasep_csf_print_gpu_sync_op(struct kbasep_printer *kbpr, struct kbase_context *kctx,
struct kbase_queue *queue, u32 ringbuff_offset,
u64 instr_addr, u64 sync_cmd, enum sync_gpu_sync_type type,
bool is_64bit, bool follows_wait)
{
u64 sync_addr = 0, compare_val = 0, live_val = 0, ringbuffer_boundary_check;
u64 move_cmd;
u8 sync_addr_reg, compare_val_reg, wait_condition = 0;
int err;
static const char *const gpu_sync_type_name[] = { "SYNC_WAIT", "SYNC_SET", "SYNC_ADD" };
static const char *const gpu_sync_type_op[] = {
"wait", /* This should never be printed, only included to simplify indexing */
"set", "add"
};
if (type >= NUM_CSF_GPU_SYNC_TYPES) {
dev_warn(kctx->kbdev->dev, "Expected GPU queue sync type is unknown!");
return;
}
/* 1. Get Register identifiers from SYNC_* instruction */
sync_addr_reg = SYNC_SRC0_GET(sync_cmd);
compare_val_reg = SYNC_SRC1_GET(sync_cmd);
if (ringbuff_offset < sizeof(u64)) {
dev_warn(kctx->kbdev->dev,
"Unexpected wraparound detected between %s & MOVE instruction",
gpu_sync_type_name[type]);
return;
}
/* 2. Get values from first MOVE command */
ringbuff_offset -= sizeof(u64);
move_cmd = kbasep_csf_read_ringbuffer_value(queue, ringbuff_offset);
/* We expect there to be at least 2 preceding MOVE instructions for CQS, or 3 preceding
* MOVE instructions for Timeline CQS, and Base will always arrange for these
* MOVE + SYNC instructions to be contiguously located, and is therefore never expected
* to be wrapped around the ringbuffer boundary. The following check takes place after
* the ringbuffer has been decremented, and already points to the first MOVE command,
* so that it can be determined if it's a 32-bit MOVE (so 2 vs 1 preceding MOVE commands
* will be checked).
* This is to maintain compatibility with older userspace; a check is done to ensure that
* the MOVE opcode found was a 32-bit MOVE, and if so, it has determined that a newer
* userspace is being used and will continue to read the next 32-bit MOVE to recover the
* compare/set value in the wait/set operation. If not, the single 48-bit value found
* will be used.
*/
ringbuffer_boundary_check =
(INSTR_OPCODE_GET(move_cmd) == GPU_CSF_MOVE32_OPCODE && is_64bit) ? 2 : 1;
if (unlikely(ringbuff_offset < (ringbuffer_boundary_check * sizeof(u64)))) {
dev_warn(kctx->kbdev->dev,
"Unexpected wraparound detected between %s & MOVE instruction",
gpu_sync_type_name[type]);
return;
}
/* For 64-bit SYNC commands, the first MOVE command read in will actually use 1 register
* above the compare value register in the sync command, as this will store the higher
* 32-bits of 64-bit compare value. The compare value register read above will be read
* afterwards.
*/
if (!kbasep_csf_get_move_immediate_value(move_cmd, sync_addr_reg,
compare_val_reg + (is_64bit ? 1 : 0), &sync_addr,
&compare_val))
return;
/* 64-bit WAITs or SETs are split into 2 32-bit MOVEs. sync_val would contain the higher
* 32 bits, so the lower 32-bits are retrieved afterwards, to recover the full u64 value.
*/
if (INSTR_OPCODE_GET(move_cmd) == GPU_CSF_MOVE32_OPCODE && is_64bit) {
u64 compare_val_lower = 0;
ringbuff_offset -= sizeof(u64);
move_cmd = kbasep_csf_read_ringbuffer_value(queue, ringbuff_offset);
if (!kbasep_csf_get_move_immediate_value(move_cmd, sync_addr_reg, compare_val_reg,
&sync_addr, &compare_val_lower))
return;
/* Mask off upper 32 bits of compare_val_lower, and combine with the higher 32 bits
* to restore the original u64 compare value.
*/
compare_val = (compare_val << 32) | (compare_val_lower & ((u64)U32_MAX));
}
/* 3. Get values from next MOVE command, which should be the CQS object address */
ringbuff_offset -= sizeof(u64);
move_cmd = kbasep_csf_read_ringbuffer_value(queue, ringbuff_offset);
if (!kbasep_csf_get_move_immediate_value(move_cmd, sync_addr_reg, compare_val_reg,
&sync_addr, &compare_val))
return;
/* 4. Get CQS object value */
if (is_64bit)
err = kbasep_csf_sync_get_cqs_live_u64(kctx, sync_addr, &live_val);
else
err = kbasep_csf_sync_get_cqs_live_u32(kctx, sync_addr, (u32 *)(&live_val));
if (err)
return;
/* 5. Print info */
kbasep_print(kbpr, "queue:GPU-%u-%u-%u exec:%c at:0x%.16llx cmd:%s ", kctx->id,
queue->group->handle, queue->csi_index,
queue->enabled && !follows_wait ? 'S' : 'P', instr_addr,
gpu_sync_type_name[type]);
if (queue->group->csg_nr == KBASEP_CSG_NR_INVALID)
kbasep_print(kbpr, "slot:-");
else
kbasep_print(kbpr, "slot:%d", (int)queue->group->csg_nr);
kbasep_print(kbpr, " obj:0x%.16llx live_value:0x%.16llx | ", sync_addr, live_val);
if (type == CSF_GPU_SYNC_WAIT) {
wait_condition = SYNC_WAIT_CONDITION_GET(sync_cmd);
kbasep_print(kbpr, "op:%s ", kbasep_csf_sync_get_wait_op_name(wait_condition));
} else
kbasep_print(kbpr, "op:%s ", gpu_sync_type_op[type]);
kbasep_print(kbpr, "arg_value:0x%.16llx\n", compare_val);
}
/**
* kbasep_csf_dump_active_queue_sync_info() - Print GPU command queue sync information.
*
* @kbpr: Pointer to printer instance.
* @queue: Address of a GPU command queue to examine.
*
* This function will iterate through each command in the ring buffer of the given GPU queue from
* CS_EXTRACT, and if is a SYNC_* instruction it will attempt to decode the sync operation and
* print relevant information to the sync file.
* This function will stop iterating once the CS_INSERT address is reached by the cursor (i.e.
* when there are no more commands to view) or a number of consumed GPU CALL commands have
* been observed.
*/
static void kbasep_csf_dump_active_queue_sync_info(struct kbasep_printer *kbpr,
struct kbase_queue *queue)
{
struct kbase_context *kctx;
u64 *addr;
u64 cs_extract, cs_insert, instr, cursor, end_cursor;
u32 nr_nearby_instr_size;
bool follows_wait = false;
int nr_calls = 0;
if (!queue)
return;
kctx = queue->kctx;
addr = queue->user_io_addr;
cs_insert = addr[CS_INSERT_LO / sizeof(*addr)];
addr = queue->user_io_addr + PAGE_SIZE / sizeof(*addr);
cs_extract = addr[CS_EXTRACT_LO / sizeof(*addr)];
nr_nearby_instr_size =
min((MAX_NR_NEARBY_INSTR * (u32)sizeof(u64)), ((queue->size / 2) & ~(0x7u)));
cursor = (cs_extract + queue->size - nr_nearby_instr_size) & ((u64)queue->size - 1);
end_cursor = min(cs_insert, ((cs_extract + nr_nearby_instr_size) & ((u64)queue->size - 1)));
if (!is_power_of_2(queue->size)) {
dev_warn(kctx->kbdev->dev, "GPU queue %u size of %u not a power of 2",
queue->csi_index, queue->size);
return;
}
kbasep_print(
kbpr,
"queue:GPU-%u-%u-%u size:%u cs_insert:%8llx cs_extract:%8llx dump_begin:%8llx dump_end:%8llx\n",
kctx->id, queue->group->handle, queue->csi_index, queue->size, cs_insert,
cs_extract, cursor, end_cursor);
while ((cs_insert != cs_extract) && (cursor != end_cursor) &&
(nr_calls < MAX_NR_GPU_CALLS)) {
bool instr_is_64_bit = false;
u32 cursor_ringbuff_offset = (u32)cursor;
/* Find instruction that cursor is currently on */
instr = kbasep_csf_read_ringbuffer_value(queue, cursor_ringbuff_offset);
switch (INSTR_OPCODE_GET(instr)) {
case GPU_CSF_SYNC_ADD64_OPCODE:
case GPU_CSF_SYNC_SET64_OPCODE:
case GPU_CSF_SYNC_WAIT64_OPCODE:
instr_is_64_bit = true;
break;
default:
break;
}
switch (INSTR_OPCODE_GET(instr)) {
case GPU_CSF_SYNC_ADD_OPCODE:
case GPU_CSF_SYNC_ADD64_OPCODE:
kbasep_csf_print_gpu_sync_op(kbpr, kctx, queue, cursor_ringbuff_offset,
cursor, instr, CSF_GPU_SYNC_ADD,
instr_is_64_bit, follows_wait);
break;
case GPU_CSF_SYNC_SET_OPCODE:
case GPU_CSF_SYNC_SET64_OPCODE:
kbasep_csf_print_gpu_sync_op(kbpr, kctx, queue, cursor_ringbuff_offset,
cursor, instr, CSF_GPU_SYNC_SET,
instr_is_64_bit, follows_wait);
break;
case GPU_CSF_SYNC_WAIT_OPCODE:
case GPU_CSF_SYNC_WAIT64_OPCODE:
kbasep_csf_print_gpu_sync_op(kbpr, kctx, queue, cursor_ringbuff_offset,
cursor, instr, CSF_GPU_SYNC_WAIT,
instr_is_64_bit, follows_wait);
follows_wait = true; /* Future commands will follow at least one wait */
break;
case GPU_CSF_CALL_OPCODE:
nr_calls++;
kbasep_print(kbpr,
"queue:GPU-%u-%u-%u exec:%c at:0x%.16llx cmd:0x%.16llx\n",
kctx->id, queue->group->handle, queue->csi_index,
queue->enabled && !follows_wait ? 'S' : 'P', cursor, instr);
break;
default:
/* NOP instructions without metadata are not printed. */
if (instr) {
kbasep_print(
kbpr,
"queue:GPU-%u-%u-%u exec:%c at:0x%.16llx cmd:0x%.16llx\n",
kctx->id, queue->group->handle, queue->csi_index,
queue->enabled && !follows_wait ? 'S' : 'P', cursor, instr);
}
break;
}
cursor = (cursor + sizeof(u64)) & ((u64)queue->size - 1);
}
}
/**
* kbasep_csf_dump_active_group_sync_state() - Prints SYNC commands in all GPU queues of
* the provided queue group.
*
* @kctx: The kbase context
* @kbpr: Pointer to printer instance.
* @group: Address of a GPU command group to iterate through.
*
* This function will iterate through each queue in the provided GPU queue group and
* print its SYNC related commands.
*/
static void kbasep_csf_dump_active_group_sync_state(struct kbase_context *kctx,
struct kbasep_printer *kbpr,
struct kbase_queue_group *const group)
{
unsigned int i;
kbasep_print(kbpr, "GPU queues for group %u (slot %d) of ctx %d_%d\n", group->handle,
group->csg_nr, kctx->tgid, kctx->id);
for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++)
kbasep_csf_dump_active_queue_sync_info(kbpr, group->bound_queues[i]);
}
int kbasep_csf_sync_gpu_dump_print(struct kbase_context *kctx, struct kbasep_printer *kbpr)
{
u32 gr;
struct kbase_device *kbdev;
if (WARN_ON(!kctx))
return -EINVAL;
kbdev = kctx->kbdev;
kbase_csf_scheduler_lock(kbdev);
kbase_csf_csg_update_status(kbdev);
kbasep_print(kbpr, "CSF GPU queues sync info (version: v" __stringify(
MALI_CSF_SYNC_DUMP_VERSION) "):\n");
for (gr = 0; gr < kbdev->csf.global_iface.group_num; gr++) {
struct kbase_queue_group *const group =
kbdev->csf.scheduler.csg_slots[gr].resident_group;
if (!group || group->kctx != kctx)
continue;
kbasep_csf_dump_active_group_sync_state(kctx, kbpr, group);
}
kbase_csf_scheduler_unlock(kbdev);
return 0;
}
Reference in New Issue View Git Blame Copy Permalink