GK SDK 源码库: XMIPCLinuxV100R005C00SPC030 (kernel/tools/open_source excluded)
This commit is contained in:
+345
@@ -0,0 +1,345 @@
|
||||
/*
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <bpmp_ipc.h>
|
||||
#include <common/debug.h>
|
||||
#include <drivers/delay_timer.h>
|
||||
#include <errno.h>
|
||||
#include <lib/mmio.h>
|
||||
#include <lib/utils_def.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <tegra_def.h>
|
||||
|
||||
#include "intf.h"
|
||||
#include "ivc.h"
|
||||
|
||||
/**
|
||||
* Holds IVC channel data
|
||||
*/
|
||||
struct ccplex_bpmp_channel_data {
|
||||
/* Buffer for incoming data */
|
||||
struct frame_data *ib;
|
||||
|
||||
/* Buffer for outgoing data */
|
||||
struct frame_data *ob;
|
||||
};
|
||||
|
||||
static struct ccplex_bpmp_channel_data s_channel;
|
||||
static struct ivc ivc_ccplex_bpmp_channel;
|
||||
|
||||
/*
|
||||
* Helper functions to access the HSP doorbell registers
|
||||
*/
|
||||
static inline uint32_t hsp_db_read(uint32_t reg)
|
||||
{
|
||||
return mmio_read_32((uint32_t)(TEGRA_HSP_DBELL_BASE + reg));
|
||||
}
|
||||
|
||||
static inline void hsp_db_write(uint32_t reg, uint32_t val)
|
||||
{
|
||||
mmio_write_32((uint32_t)(TEGRA_HSP_DBELL_BASE + reg), val);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
* IVC wrappers for CCPLEX <-> BPMP communication.
|
||||
******************************************************************************/
|
||||
|
||||
static void tegra_bpmp_ring_bpmp_doorbell(void);
|
||||
|
||||
/*
|
||||
* Get the next frame where data can be written.
|
||||
*/
|
||||
static struct frame_data *tegra_bpmp_get_next_out_frame(void)
|
||||
{
|
||||
struct frame_data *frame;
|
||||
const struct ivc *ch = &ivc_ccplex_bpmp_channel;
|
||||
|
||||
frame = (struct frame_data *)tegra_ivc_write_get_next_frame(ch);
|
||||
if (frame == NULL) {
|
||||
ERROR("%s: Error in getting next frame, exiting\n", __func__);
|
||||
} else {
|
||||
s_channel.ob = frame;
|
||||
}
|
||||
|
||||
return frame;
|
||||
}
|
||||
|
||||
static void tegra_bpmp_signal_slave(void)
|
||||
{
|
||||
(void)tegra_ivc_write_advance(&ivc_ccplex_bpmp_channel);
|
||||
tegra_bpmp_ring_bpmp_doorbell();
|
||||
}
|
||||
|
||||
static int32_t tegra_bpmp_free_master(void)
|
||||
{
|
||||
return tegra_ivc_read_advance(&ivc_ccplex_bpmp_channel);
|
||||
}
|
||||
|
||||
static bool tegra_bpmp_slave_acked(void)
|
||||
{
|
||||
struct frame_data *frame;
|
||||
bool ret = true;
|
||||
|
||||
frame = (struct frame_data *)tegra_ivc_read_get_next_frame(&ivc_ccplex_bpmp_channel);
|
||||
if (frame == NULL) {
|
||||
ret = false;
|
||||
} else {
|
||||
s_channel.ib = frame;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static struct frame_data *tegra_bpmp_get_cur_in_frame(void)
|
||||
{
|
||||
return s_channel.ib;
|
||||
}
|
||||
|
||||
/*
|
||||
* Enables BPMP to ring CCPlex doorbell
|
||||
*/
|
||||
static void tegra_bpmp_enable_ccplex_doorbell(void)
|
||||
{
|
||||
uint32_t reg;
|
||||
|
||||
reg = hsp_db_read(HSP_DBELL_1_ENABLE);
|
||||
reg |= HSP_MASTER_BPMP_BIT;
|
||||
hsp_db_write(HSP_DBELL_1_ENABLE, reg);
|
||||
}
|
||||
|
||||
/*
|
||||
* CCPlex rings the BPMP doorbell
|
||||
*/
|
||||
static void tegra_bpmp_ring_bpmp_doorbell(void)
|
||||
{
|
||||
/*
|
||||
* Any writes to this register has the same effect,
|
||||
* uses master ID of the write transaction and set
|
||||
* corresponding flag.
|
||||
*/
|
||||
hsp_db_write(HSP_DBELL_3_TRIGGER, HSP_MASTER_CCPLEX_BIT);
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns true if CCPLex can ring BPMP doorbell, otherwise false.
|
||||
* This also signals that BPMP is up and ready.
|
||||
*/
|
||||
static bool tegra_bpmp_can_ccplex_ring_doorbell(void)
|
||||
{
|
||||
uint32_t reg;
|
||||
|
||||
/* check if ccplex can communicate with bpmp */
|
||||
reg = hsp_db_read(HSP_DBELL_3_ENABLE);
|
||||
|
||||
return ((reg & HSP_MASTER_CCPLEX_BIT) != 0U);
|
||||
}
|
||||
|
||||
static int32_t tegra_bpmp_wait_for_slave_ack(void)
|
||||
{
|
||||
uint32_t timeout = TIMEOUT_RESPONSE_FROM_BPMP_US;
|
||||
|
||||
while (!tegra_bpmp_slave_acked() && (timeout != 0U)) {
|
||||
udelay(1);
|
||||
timeout--;
|
||||
};
|
||||
|
||||
return ((timeout == 0U) ? -ETIMEDOUT : 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Notification from the ivc layer
|
||||
*/
|
||||
static void tegra_bpmp_ivc_notify(const struct ivc *ivc)
|
||||
{
|
||||
(void)(ivc);
|
||||
|
||||
tegra_bpmp_ring_bpmp_doorbell();
|
||||
}
|
||||
|
||||
/*
|
||||
* Atomic send/receive API, which means it waits until slave acks
|
||||
*/
|
||||
static int32_t tegra_bpmp_ipc_send_req_atomic(uint32_t mrq, void *p_out,
|
||||
uint32_t size_out, void *p_in, uint32_t size_in)
|
||||
{
|
||||
struct frame_data *frame = tegra_bpmp_get_next_out_frame();
|
||||
const struct frame_data *f_in = NULL;
|
||||
int32_t ret = 0;
|
||||
void *p_fdata;
|
||||
|
||||
if ((p_out == NULL) || (size_out > IVC_DATA_SZ_BYTES) ||
|
||||
(frame == NULL)) {
|
||||
ERROR("%s: invalid parameters, exiting\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* prepare the command frame */
|
||||
frame->mrq = mrq;
|
||||
frame->flags = FLAG_DO_ACK;
|
||||
p_fdata = frame->data;
|
||||
(void)memcpy(p_fdata, p_out, (size_t)size_out);
|
||||
|
||||
/* signal the slave */
|
||||
tegra_bpmp_signal_slave();
|
||||
|
||||
/* wait for slave to ack */
|
||||
ret = tegra_bpmp_wait_for_slave_ack();
|
||||
if (ret < 0) {
|
||||
ERROR("%s: wait for slave failed (%d)\n", __func__, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* retrieve the response frame */
|
||||
if ((size_in <= IVC_DATA_SZ_BYTES) && (p_in != NULL)) {
|
||||
|
||||
f_in = tegra_bpmp_get_cur_in_frame();
|
||||
if (f_in != NULL) {
|
||||
ERROR("Failed to get next input frame!\n");
|
||||
} else {
|
||||
(void)memcpy(p_in, p_fdata, (size_t)size_in);
|
||||
}
|
||||
}
|
||||
|
||||
ret = tegra_bpmp_free_master();
|
||||
if (ret < 0) {
|
||||
ERROR("%s: free master failed (%d)\n", __func__, ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Initializes the BPMP<--->CCPlex communication path.
|
||||
*/
|
||||
int32_t tegra_bpmp_ipc_init(void)
|
||||
{
|
||||
size_t msg_size;
|
||||
uint32_t frame_size, timeout;
|
||||
int32_t error = 0;
|
||||
|
||||
/* allow bpmp to ring CCPLEX's doorbell */
|
||||
tegra_bpmp_enable_ccplex_doorbell();
|
||||
|
||||
/* wait for BPMP to actually ring the doorbell */
|
||||
timeout = TIMEOUT_RESPONSE_FROM_BPMP_US;
|
||||
while ((timeout != 0U) && !tegra_bpmp_can_ccplex_ring_doorbell()) {
|
||||
udelay(1); /* bpmp turn-around time */
|
||||
timeout--;
|
||||
}
|
||||
|
||||
if (timeout == 0U) {
|
||||
ERROR("%s: BPMP firmware is not ready\n", __func__);
|
||||
return -ENOTSUP;
|
||||
}
|
||||
|
||||
INFO("%s: BPMP handshake completed\n", __func__);
|
||||
|
||||
msg_size = tegra_ivc_align(IVC_CMD_SZ_BYTES);
|
||||
frame_size = (uint32_t)tegra_ivc_total_queue_size(msg_size);
|
||||
if (frame_size > TEGRA_BPMP_IPC_CH_MAP_SIZE) {
|
||||
ERROR("%s: carveout size is not sufficient\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
error = tegra_ivc_init(&ivc_ccplex_bpmp_channel,
|
||||
(uint32_t)TEGRA_BPMP_IPC_RX_PHYS_BASE,
|
||||
(uint32_t)TEGRA_BPMP_IPC_TX_PHYS_BASE,
|
||||
1U, frame_size, tegra_bpmp_ivc_notify);
|
||||
if (error != 0) {
|
||||
|
||||
ERROR("%s: IVC init failed (%d)\n", __func__, error);
|
||||
|
||||
} else {
|
||||
|
||||
/* reset channel */
|
||||
tegra_ivc_channel_reset(&ivc_ccplex_bpmp_channel);
|
||||
|
||||
/* wait for notification from BPMP */
|
||||
while (tegra_ivc_channel_notified(&ivc_ccplex_bpmp_channel) != 0) {
|
||||
/*
|
||||
* Interrupt BPMP with doorbell each time after
|
||||
* tegra_ivc_channel_notified() returns non zero
|
||||
* value.
|
||||
*/
|
||||
tegra_bpmp_ring_bpmp_doorbell();
|
||||
}
|
||||
|
||||
INFO("%s: All communication channels initialized\n", __func__);
|
||||
}
|
||||
|
||||
return error;
|
||||
}
|
||||
|
||||
/* Handler to reset a hardware module */
|
||||
int32_t tegra_bpmp_ipc_reset_module(uint32_t rst_id)
|
||||
{
|
||||
int32_t ret;
|
||||
struct mrq_reset_request req = {
|
||||
.cmd = (uint32_t)CMD_RESET_MODULE,
|
||||
.reset_id = rst_id
|
||||
};
|
||||
|
||||
/* only GPCDMA/XUSB_PADCTL resets are supported */
|
||||
assert((rst_id == TEGRA_RESET_ID_XUSB_PADCTL) ||
|
||||
(rst_id == TEGRA_RESET_ID_GPCDMA));
|
||||
|
||||
ret = tegra_bpmp_ipc_send_req_atomic(MRQ_RESET, &req,
|
||||
(uint32_t)sizeof(req), NULL, 0);
|
||||
if (ret != 0) {
|
||||
ERROR("%s: failed for module %d with error %d\n", __func__,
|
||||
rst_id, ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int tegra_bpmp_ipc_enable_clock(uint32_t clk_id)
|
||||
{
|
||||
int ret;
|
||||
struct mrq_clk_request req;
|
||||
|
||||
/* only SE clocks are supported */
|
||||
if (clk_id != TEGRA_CLK_SE) {
|
||||
return -ENOTSUP;
|
||||
}
|
||||
|
||||
/* prepare the MRQ_CLK command */
|
||||
req.cmd_and_id = make_mrq_clk_cmd(CMD_CLK_ENABLE, clk_id);
|
||||
|
||||
ret = tegra_bpmp_ipc_send_req_atomic(MRQ_CLK, &req, (uint32_t)sizeof(req),
|
||||
NULL, 0);
|
||||
if (ret != 0) {
|
||||
ERROR("%s: failed for module %d with error %d\n", __func__,
|
||||
clk_id, ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int tegra_bpmp_ipc_disable_clock(uint32_t clk_id)
|
||||
{
|
||||
int ret;
|
||||
struct mrq_clk_request req;
|
||||
|
||||
/* only SE clocks are supported */
|
||||
if (clk_id != TEGRA_CLK_SE) {
|
||||
return -ENOTSUP;
|
||||
}
|
||||
|
||||
/* prepare the MRQ_CLK command */
|
||||
req.cmd_and_id = make_mrq_clk_cmd(CMD_CLK_DISABLE, clk_id);
|
||||
|
||||
ret = tegra_bpmp_ipc_send_req_atomic(MRQ_CLK, &req, (uint32_t)sizeof(req),
|
||||
NULL, 0);
|
||||
if (ret != 0) {
|
||||
ERROR("%s: failed for module %d with error %d\n", __func__,
|
||||
clk_id, ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
#ifndef BPMP_INTF_H
|
||||
#define BPMP_INTF_H
|
||||
|
||||
/**
|
||||
* Flags used in IPC req
|
||||
*/
|
||||
#define FLAG_DO_ACK (U(1) << 0)
|
||||
#define FLAG_RING_DOORBELL (U(1) << 1)
|
||||
|
||||
/* Bit 1 is designated for CCPlex in secure world */
|
||||
#define HSP_MASTER_CCPLEX_BIT (U(1) << 1)
|
||||
/* Bit 19 is designated for BPMP in non-secure world */
|
||||
#define HSP_MASTER_BPMP_BIT (U(1) << 19)
|
||||
/* Timeout to receive response from BPMP is 1 sec */
|
||||
#define TIMEOUT_RESPONSE_FROM_BPMP_US U(1000000) /* in microseconds */
|
||||
|
||||
/**
|
||||
* IVC protocol defines and command/response frame
|
||||
*/
|
||||
|
||||
/**
|
||||
* IVC specific defines
|
||||
*/
|
||||
#define IVC_CMD_SZ_BYTES U(128)
|
||||
#define IVC_DATA_SZ_BYTES U(120)
|
||||
|
||||
/**
|
||||
* Holds frame data for an IPC request
|
||||
*/
|
||||
struct frame_data {
|
||||
/* Identification as to what kind of data is being transmitted */
|
||||
uint32_t mrq;
|
||||
|
||||
/* Flags for slave as to how to respond back */
|
||||
uint32_t flags;
|
||||
|
||||
/* Actual data being sent */
|
||||
uint8_t data[IVC_DATA_SZ_BYTES];
|
||||
};
|
||||
|
||||
/**
|
||||
* Commands send to the BPMP firmware
|
||||
*/
|
||||
|
||||
/**
|
||||
* MRQ command codes
|
||||
*/
|
||||
#define MRQ_RESET U(20)
|
||||
#define MRQ_CLK U(22)
|
||||
|
||||
/**
|
||||
* Reset sub-commands
|
||||
*/
|
||||
#define CMD_RESET_ASSERT U(1)
|
||||
#define CMD_RESET_DEASSERT U(2)
|
||||
#define CMD_RESET_MODULE U(3)
|
||||
|
||||
/**
|
||||
* Used by the sender of an #MRQ_RESET message to request BPMP to
|
||||
* assert or deassert a given reset line.
|
||||
*/
|
||||
struct __attribute__((packed)) mrq_reset_request {
|
||||
/* reset action to perform (mrq_reset_commands) */
|
||||
uint32_t cmd;
|
||||
/* id of the reset to affected */
|
||||
uint32_t reset_id;
|
||||
};
|
||||
|
||||
/**
|
||||
* MRQ_CLK sub-commands
|
||||
*
|
||||
*/
|
||||
enum {
|
||||
CMD_CLK_GET_RATE = U(1),
|
||||
CMD_CLK_SET_RATE = U(2),
|
||||
CMD_CLK_ROUND_RATE = U(3),
|
||||
CMD_CLK_GET_PARENT = U(4),
|
||||
CMD_CLK_SET_PARENT = U(5),
|
||||
CMD_CLK_IS_ENABLED = U(6),
|
||||
CMD_CLK_ENABLE = U(7),
|
||||
CMD_CLK_DISABLE = U(8),
|
||||
CMD_CLK_GET_ALL_INFO = U(14),
|
||||
CMD_CLK_GET_MAX_CLK_ID = U(15),
|
||||
CMD_CLK_MAX,
|
||||
};
|
||||
|
||||
/**
|
||||
* Used by the sender of an #MRQ_CLK message to control clocks. The
|
||||
* clk_request is split into several sub-commands. Some sub-commands
|
||||
* require no additional data. Others have a sub-command specific
|
||||
* payload
|
||||
*
|
||||
* |sub-command |payload |
|
||||
* |----------------------------|-----------------------|
|
||||
* |CMD_CLK_GET_RATE |- |
|
||||
* |CMD_CLK_SET_RATE |clk_set_rate |
|
||||
* |CMD_CLK_ROUND_RATE |clk_round_rate |
|
||||
* |CMD_CLK_GET_PARENT |- |
|
||||
* |CMD_CLK_SET_PARENT |clk_set_parent |
|
||||
* |CMD_CLK_IS_ENABLED |- |
|
||||
* |CMD_CLK_ENABLE |- |
|
||||
* |CMD_CLK_DISABLE |- |
|
||||
* |CMD_CLK_GET_ALL_INFO |- |
|
||||
* |CMD_CLK_GET_MAX_CLK_ID |- |
|
||||
*
|
||||
*/
|
||||
struct mrq_clk_request {
|
||||
/**
|
||||
* sub-command and clock id concatenated to 32-bit word.
|
||||
* - bits[31..24] is the sub-cmd.
|
||||
* - bits[23..0] is the clock id
|
||||
*/
|
||||
uint32_t cmd_and_id;
|
||||
};
|
||||
|
||||
/**
|
||||
* Macro to prepare the MRQ_CLK sub-command
|
||||
*/
|
||||
#define make_mrq_clk_cmd(cmd, id) (((cmd) << 24) | (id & 0xFFFFFF))
|
||||
|
||||
#endif /* BPMP_INTF_H */
|
||||
+654
@@ -0,0 +1,654 @@
|
||||
/*
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
#include <arch_helpers.h>
|
||||
#include <assert.h>
|
||||
#include <common/debug.h>
|
||||
#include <errno.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "ivc.h"
|
||||
|
||||
/*
|
||||
* IVC channel reset protocol.
|
||||
*
|
||||
* Each end uses its tx_channel.state to indicate its synchronization state.
|
||||
*/
|
||||
enum {
|
||||
/*
|
||||
* This value is zero for backwards compatibility with services that
|
||||
* assume channels to be initially zeroed. Such channels are in an
|
||||
* initially valid state, but cannot be asynchronously reset, and must
|
||||
* maintain a valid state at all times.
|
||||
*
|
||||
* The transmitting end can enter the established state from the sync or
|
||||
* ack state when it observes the receiving endpoint in the ack or
|
||||
* established state, indicating that has cleared the counters in our
|
||||
* rx_channel.
|
||||
*/
|
||||
ivc_state_established = U(0),
|
||||
|
||||
/*
|
||||
* If an endpoint is observed in the sync state, the remote endpoint is
|
||||
* allowed to clear the counters it owns asynchronously with respect to
|
||||
* the current endpoint. Therefore, the current endpoint is no longer
|
||||
* allowed to communicate.
|
||||
*/
|
||||
ivc_state_sync = U(1),
|
||||
|
||||
/*
|
||||
* When the transmitting end observes the receiving end in the sync
|
||||
* state, it can clear the w_count and r_count and transition to the ack
|
||||
* state. If the remote endpoint observes us in the ack state, it can
|
||||
* return to the established state once it has cleared its counters.
|
||||
*/
|
||||
ivc_state_ack = U(2)
|
||||
};
|
||||
|
||||
/*
|
||||
* This structure is divided into two-cache aligned parts, the first is only
|
||||
* written through the tx_channel pointer, while the second is only written
|
||||
* through the rx_channel pointer. This delineates ownership of the cache lines,
|
||||
* which is critical to performance and necessary in non-cache coherent
|
||||
* implementations.
|
||||
*/
|
||||
struct ivc_channel_header {
|
||||
struct {
|
||||
/* fields owned by the transmitting end */
|
||||
uint32_t w_count;
|
||||
uint32_t state;
|
||||
uint32_t w_rsvd[IVC_CHHDR_TX_FIELDS - 2];
|
||||
};
|
||||
struct {
|
||||
/* fields owned by the receiving end */
|
||||
uint32_t r_count;
|
||||
uint32_t r_rsvd[IVC_CHHDR_RX_FIELDS - 1];
|
||||
};
|
||||
};
|
||||
|
||||
static inline bool ivc_channel_empty(const struct ivc *ivc,
|
||||
volatile const struct ivc_channel_header *ch)
|
||||
{
|
||||
/*
|
||||
* This function performs multiple checks on the same values with
|
||||
* security implications, so sample the counters' current values in
|
||||
* shared memory to ensure that these checks use the same values.
|
||||
*/
|
||||
uint32_t wr_count = ch->w_count;
|
||||
uint32_t rd_count = ch->r_count;
|
||||
bool ret = false;
|
||||
|
||||
(void)ivc;
|
||||
|
||||
/*
|
||||
* Perform an over-full check to prevent denial of service attacks where
|
||||
* a server could be easily fooled into believing that there's an
|
||||
* extremely large number of frames ready, since receivers are not
|
||||
* expected to check for full or over-full conditions.
|
||||
*
|
||||
* Although the channel isn't empty, this is an invalid case caused by
|
||||
* a potentially malicious peer, so returning empty is safer, because it
|
||||
* gives the impression that the channel has gone silent.
|
||||
*/
|
||||
if (((wr_count - rd_count) > ivc->nframes) || (wr_count == rd_count)) {
|
||||
ret = true;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline bool ivc_channel_full(const struct ivc *ivc,
|
||||
volatile const struct ivc_channel_header *ch)
|
||||
{
|
||||
uint32_t wr_count = ch->w_count;
|
||||
uint32_t rd_count = ch->r_count;
|
||||
|
||||
(void)ivc;
|
||||
|
||||
/*
|
||||
* Invalid cases where the counters indicate that the queue is over
|
||||
* capacity also appear full.
|
||||
*/
|
||||
return ((wr_count - rd_count) >= ivc->nframes);
|
||||
}
|
||||
|
||||
static inline uint32_t ivc_channel_avail_count(const struct ivc *ivc,
|
||||
volatile const struct ivc_channel_header *ch)
|
||||
{
|
||||
uint32_t wr_count = ch->w_count;
|
||||
uint32_t rd_count = ch->r_count;
|
||||
|
||||
(void)ivc;
|
||||
|
||||
/*
|
||||
* This function isn't expected to be used in scenarios where an
|
||||
* over-full situation can lead to denial of service attacks. See the
|
||||
* comment in ivc_channel_empty() for an explanation about special
|
||||
* over-full considerations.
|
||||
*/
|
||||
return (wr_count - rd_count);
|
||||
}
|
||||
|
||||
static inline void ivc_advance_tx(struct ivc *ivc)
|
||||
{
|
||||
ivc->tx_channel->w_count++;
|
||||
|
||||
if (ivc->w_pos == (ivc->nframes - (uint32_t)1U)) {
|
||||
ivc->w_pos = 0U;
|
||||
} else {
|
||||
ivc->w_pos++;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void ivc_advance_rx(struct ivc *ivc)
|
||||
{
|
||||
ivc->rx_channel->r_count++;
|
||||
|
||||
if (ivc->r_pos == (ivc->nframes - (uint32_t)1U)) {
|
||||
ivc->r_pos = 0U;
|
||||
} else {
|
||||
ivc->r_pos++;
|
||||
}
|
||||
}
|
||||
|
||||
static inline int32_t ivc_check_read(const struct ivc *ivc)
|
||||
{
|
||||
/*
|
||||
* tx_channel->state is set locally, so it is not synchronized with
|
||||
* state from the remote peer. The remote peer cannot reset its
|
||||
* transmit counters until we've acknowledged its synchronization
|
||||
* request, so no additional synchronization is required because an
|
||||
* asynchronous transition of rx_channel->state to ivc_state_ack is not
|
||||
* allowed.
|
||||
*/
|
||||
if (ivc->tx_channel->state != ivc_state_established) {
|
||||
return -ECONNRESET;
|
||||
}
|
||||
|
||||
/*
|
||||
* Avoid unnecessary invalidations when performing repeated accesses to
|
||||
* an IVC channel by checking the old queue pointers first.
|
||||
* Synchronization is only necessary when these pointers indicate empty
|
||||
* or full.
|
||||
*/
|
||||
if (!ivc_channel_empty(ivc, ivc->rx_channel)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0;
|
||||
}
|
||||
|
||||
static inline int32_t ivc_check_write(const struct ivc *ivc)
|
||||
{
|
||||
if (ivc->tx_channel->state != ivc_state_established) {
|
||||
return -ECONNRESET;
|
||||
}
|
||||
|
||||
if (!ivc_channel_full(ivc, ivc->tx_channel)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0;
|
||||
}
|
||||
|
||||
bool tegra_ivc_can_read(const struct ivc *ivc)
|
||||
{
|
||||
return ivc_check_read(ivc) == 0;
|
||||
}
|
||||
|
||||
bool tegra_ivc_can_write(const struct ivc *ivc)
|
||||
{
|
||||
return ivc_check_write(ivc) == 0;
|
||||
}
|
||||
|
||||
bool tegra_ivc_tx_empty(const struct ivc *ivc)
|
||||
{
|
||||
return ivc_channel_empty(ivc, ivc->tx_channel);
|
||||
}
|
||||
|
||||
static inline uintptr_t calc_frame_offset(uint32_t frame_index,
|
||||
uint32_t frame_size, uint32_t frame_offset)
|
||||
{
|
||||
return ((uintptr_t)frame_index * (uintptr_t)frame_size) +
|
||||
(uintptr_t)frame_offset;
|
||||
}
|
||||
|
||||
static void *ivc_frame_pointer(const struct ivc *ivc,
|
||||
volatile const struct ivc_channel_header *ch,
|
||||
uint32_t frame)
|
||||
{
|
||||
assert(frame < ivc->nframes);
|
||||
return (void *)((uintptr_t)(&ch[1]) +
|
||||
calc_frame_offset(frame, ivc->frame_size, 0));
|
||||
}
|
||||
|
||||
int32_t tegra_ivc_read(struct ivc *ivc, void *buf, size_t max_read)
|
||||
{
|
||||
const void *src;
|
||||
int32_t result;
|
||||
|
||||
if (buf == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (max_read > ivc->frame_size) {
|
||||
return -E2BIG;
|
||||
}
|
||||
|
||||
result = ivc_check_read(ivc);
|
||||
if (result != 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Order observation of w_pos potentially indicating new data before
|
||||
* data read.
|
||||
*/
|
||||
dmbish();
|
||||
|
||||
src = ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);
|
||||
|
||||
(void)memcpy(buf, src, max_read);
|
||||
|
||||
ivc_advance_rx(ivc);
|
||||
|
||||
/*
|
||||
* Ensure our write to r_pos occurs before our read from w_pos.
|
||||
*/
|
||||
dmbish();
|
||||
|
||||
/*
|
||||
* Notify only upon transition from full to non-full.
|
||||
* The available count can only asynchronously increase, so the
|
||||
* worst possible side-effect will be a spurious notification.
|
||||
*/
|
||||
if (ivc_channel_avail_count(ivc, ivc->rx_channel) == (ivc->nframes - (uint32_t)1U)) {
|
||||
ivc->notify(ivc);
|
||||
}
|
||||
|
||||
return (int32_t)max_read;
|
||||
}
|
||||
|
||||
/* directly peek at the next frame rx'ed */
|
||||
void *tegra_ivc_read_get_next_frame(const struct ivc *ivc)
|
||||
{
|
||||
if (ivc_check_read(ivc) != 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Order observation of w_pos potentially indicating new data before
|
||||
* data read.
|
||||
*/
|
||||
dmbld();
|
||||
|
||||
return ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);
|
||||
}
|
||||
|
||||
int32_t tegra_ivc_read_advance(struct ivc *ivc)
|
||||
{
|
||||
/*
|
||||
* No read barriers or synchronization here: the caller is expected to
|
||||
* have already observed the channel non-empty. This check is just to
|
||||
* catch programming errors.
|
||||
*/
|
||||
int32_t result = ivc_check_read(ivc);
|
||||
if (result != 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
ivc_advance_rx(ivc);
|
||||
|
||||
/*
|
||||
* Ensure our write to r_pos occurs before our read from w_pos.
|
||||
*/
|
||||
dmbish();
|
||||
|
||||
/*
|
||||
* Notify only upon transition from full to non-full.
|
||||
* The available count can only asynchronously increase, so the
|
||||
* worst possible side-effect will be a spurious notification.
|
||||
*/
|
||||
if (ivc_channel_avail_count(ivc, ivc->rx_channel) == (ivc->nframes - (uint32_t)1U)) {
|
||||
ivc->notify(ivc);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tegra_ivc_write(struct ivc *ivc, const void *buf, size_t size)
|
||||
{
|
||||
void *p;
|
||||
int32_t result;
|
||||
|
||||
if ((buf == NULL) || (ivc == NULL)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (size > ivc->frame_size) {
|
||||
return -E2BIG;
|
||||
}
|
||||
|
||||
result = ivc_check_write(ivc);
|
||||
if (result != 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
p = ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);
|
||||
|
||||
(void)memset(p, 0, ivc->frame_size);
|
||||
(void)memcpy(p, buf, size);
|
||||
|
||||
/*
|
||||
* Ensure that updated data is visible before the w_pos counter
|
||||
* indicates that it is ready.
|
||||
*/
|
||||
dmbst();
|
||||
|
||||
ivc_advance_tx(ivc);
|
||||
|
||||
/*
|
||||
* Ensure our write to w_pos occurs before our read from r_pos.
|
||||
*/
|
||||
dmbish();
|
||||
|
||||
/*
|
||||
* Notify only upon transition from empty to non-empty.
|
||||
* The available count can only asynchronously decrease, so the
|
||||
* worst possible side-effect will be a spurious notification.
|
||||
*/
|
||||
if (ivc_channel_avail_count(ivc, ivc->tx_channel) == 1U) {
|
||||
ivc->notify(ivc);
|
||||
}
|
||||
|
||||
return (int32_t)size;
|
||||
}
|
||||
|
||||
/* directly poke at the next frame to be tx'ed */
|
||||
void *tegra_ivc_write_get_next_frame(const struct ivc *ivc)
|
||||
{
|
||||
if (ivc_check_write(ivc) != 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);
|
||||
}
|
||||
|
||||
/* advance the tx buffer */
|
||||
int32_t tegra_ivc_write_advance(struct ivc *ivc)
|
||||
{
|
||||
int32_t result = ivc_check_write(ivc);
|
||||
|
||||
if (result != 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Order any possible stores to the frame before update of w_pos.
|
||||
*/
|
||||
dmbst();
|
||||
|
||||
ivc_advance_tx(ivc);
|
||||
|
||||
/*
|
||||
* Ensure our write to w_pos occurs before our read from r_pos.
|
||||
*/
|
||||
dmbish();
|
||||
|
||||
/*
|
||||
* Notify only upon transition from empty to non-empty.
|
||||
* The available count can only asynchronously decrease, so the
|
||||
* worst possible side-effect will be a spurious notification.
|
||||
*/
|
||||
if (ivc_channel_avail_count(ivc, ivc->tx_channel) == (uint32_t)1U) {
|
||||
ivc->notify(ivc);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tegra_ivc_channel_reset(const struct ivc *ivc)
|
||||
{
|
||||
ivc->tx_channel->state = ivc_state_sync;
|
||||
ivc->notify(ivc);
|
||||
}
|
||||
|
||||
/*
|
||||
* ===============================================================
|
||||
* IVC State Transition Table - see tegra_ivc_channel_notified()
|
||||
* ===============================================================
|
||||
*
|
||||
* local remote action
|
||||
* ----- ------ -----------------------------------
|
||||
* SYNC EST <none>
|
||||
* SYNC ACK reset counters; move to EST; notify
|
||||
* SYNC SYNC reset counters; move to ACK; notify
|
||||
* ACK EST move to EST; notify
|
||||
* ACK ACK move to EST; notify
|
||||
* ACK SYNC reset counters; move to ACK; notify
|
||||
* EST EST <none>
|
||||
* EST ACK <none>
|
||||
* EST SYNC reset counters; move to ACK; notify
|
||||
*
|
||||
* ===============================================================
|
||||
*/
|
||||
int32_t tegra_ivc_channel_notified(struct ivc *ivc)
|
||||
{
|
||||
uint32_t peer_state;
|
||||
|
||||
/* Copy the receiver's state out of shared memory. */
|
||||
peer_state = ivc->rx_channel->state;
|
||||
|
||||
if (peer_state == (uint32_t)ivc_state_sync) {
|
||||
/*
|
||||
* Order observation of ivc_state_sync before stores clearing
|
||||
* tx_channel.
|
||||
*/
|
||||
dmbld();
|
||||
|
||||
/*
|
||||
* Reset tx_channel counters. The remote end is in the SYNC
|
||||
* state and won't make progress until we change our state,
|
||||
* so the counters are not in use at this time.
|
||||
*/
|
||||
ivc->tx_channel->w_count = 0U;
|
||||
ivc->rx_channel->r_count = 0U;
|
||||
|
||||
ivc->w_pos = 0U;
|
||||
ivc->r_pos = 0U;
|
||||
|
||||
/*
|
||||
* Ensure that counters appear cleared before new state can be
|
||||
* observed.
|
||||
*/
|
||||
dmbst();
|
||||
|
||||
/*
|
||||
* Move to ACK state. We have just cleared our counters, so it
|
||||
* is now safe for the remote end to start using these values.
|
||||
*/
|
||||
ivc->tx_channel->state = ivc_state_ack;
|
||||
|
||||
/*
|
||||
* Notify remote end to observe state transition.
|
||||
*/
|
||||
ivc->notify(ivc);
|
||||
|
||||
} else if ((ivc->tx_channel->state == (uint32_t)ivc_state_sync) &&
|
||||
(peer_state == (uint32_t)ivc_state_ack)) {
|
||||
/*
|
||||
* Order observation of ivc_state_sync before stores clearing
|
||||
* tx_channel.
|
||||
*/
|
||||
dmbld();
|
||||
|
||||
/*
|
||||
* Reset tx_channel counters. The remote end is in the ACK
|
||||
* state and won't make progress until we change our state,
|
||||
* so the counters are not in use at this time.
|
||||
*/
|
||||
ivc->tx_channel->w_count = 0U;
|
||||
ivc->rx_channel->r_count = 0U;
|
||||
|
||||
ivc->w_pos = 0U;
|
||||
ivc->r_pos = 0U;
|
||||
|
||||
/*
|
||||
* Ensure that counters appear cleared before new state can be
|
||||
* observed.
|
||||
*/
|
||||
dmbst();
|
||||
|
||||
/*
|
||||
* Move to ESTABLISHED state. We know that the remote end has
|
||||
* already cleared its counters, so it is safe to start
|
||||
* writing/reading on this channel.
|
||||
*/
|
||||
ivc->tx_channel->state = ivc_state_established;
|
||||
|
||||
/*
|
||||
* Notify remote end to observe state transition.
|
||||
*/
|
||||
ivc->notify(ivc);
|
||||
|
||||
} else if (ivc->tx_channel->state == (uint32_t)ivc_state_ack) {
|
||||
/*
|
||||
* At this point, we have observed the peer to be in either
|
||||
* the ACK or ESTABLISHED state. Next, order observation of
|
||||
* peer state before storing to tx_channel.
|
||||
*/
|
||||
dmbld();
|
||||
|
||||
/*
|
||||
* Move to ESTABLISHED state. We know that we have previously
|
||||
* cleared our counters, and we know that the remote end has
|
||||
* cleared its counters, so it is safe to start writing/reading
|
||||
* on this channel.
|
||||
*/
|
||||
ivc->tx_channel->state = ivc_state_established;
|
||||
|
||||
/*
|
||||
* Notify remote end to observe state transition.
|
||||
*/
|
||||
ivc->notify(ivc);
|
||||
|
||||
} else {
|
||||
/*
|
||||
* There is no need to handle any further action. Either the
|
||||
* channel is already fully established, or we are waiting for
|
||||
* the remote end to catch up with our current state. Refer
|
||||
* to the diagram in "IVC State Transition Table" above.
|
||||
*/
|
||||
}
|
||||
|
||||
return ((ivc->tx_channel->state == (uint32_t)ivc_state_established) ? 0 : -EAGAIN);
|
||||
}
|
||||
|
||||
size_t tegra_ivc_align(size_t size)
|
||||
{
|
||||
return (size + (IVC_ALIGN - 1U)) & ~(IVC_ALIGN - 1U);
|
||||
}
|
||||
|
||||
size_t tegra_ivc_total_queue_size(size_t queue_size)
|
||||
{
|
||||
if ((queue_size & (IVC_ALIGN - 1U)) != 0U) {
|
||||
ERROR("queue_size (%d) must be %d-byte aligned\n",
|
||||
(int32_t)queue_size, IVC_ALIGN);
|
||||
return 0;
|
||||
}
|
||||
return queue_size + sizeof(struct ivc_channel_header);
|
||||
}
|
||||
|
||||
static int32_t check_ivc_params(uintptr_t queue_base1, uintptr_t queue_base2,
|
||||
uint32_t nframes, uint32_t frame_size)
|
||||
{
|
||||
assert((offsetof(struct ivc_channel_header, w_count)
|
||||
& (IVC_ALIGN - 1U)) == 0U);
|
||||
assert((offsetof(struct ivc_channel_header, r_count)
|
||||
& (IVC_ALIGN - 1U)) == 0U);
|
||||
assert((sizeof(struct ivc_channel_header) & (IVC_ALIGN - 1U)) == 0U);
|
||||
|
||||
if (((uint64_t)nframes * (uint64_t)frame_size) >= 0x100000000ULL) {
|
||||
ERROR("nframes * frame_size overflows\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/*
|
||||
* The headers must at least be aligned enough for counters
|
||||
* to be accessed atomically.
|
||||
*/
|
||||
if ((queue_base1 & (IVC_ALIGN - 1U)) != 0U) {
|
||||
ERROR("ivc channel start not aligned: %lx\n", queue_base1);
|
||||
return -EINVAL;
|
||||
}
|
||||
if ((queue_base2 & (IVC_ALIGN - 1U)) != 0U) {
|
||||
ERROR("ivc channel start not aligned: %lx\n", queue_base2);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if ((frame_size & (IVC_ALIGN - 1U)) != 0U) {
|
||||
ERROR("frame size not adequately aligned: %u\n",
|
||||
frame_size);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (queue_base1 < queue_base2) {
|
||||
if ((queue_base1 + ((uint64_t)frame_size * nframes)) > queue_base2) {
|
||||
ERROR("queue regions overlap: %lx + %x, %x\n",
|
||||
queue_base1, frame_size,
|
||||
frame_size * nframes);
|
||||
return -EINVAL;
|
||||
}
|
||||
} else {
|
||||
if ((queue_base2 + ((uint64_t)frame_size * nframes)) > queue_base1) {
|
||||
ERROR("queue regions overlap: %lx + %x, %x\n",
|
||||
queue_base2, frame_size,
|
||||
frame_size * nframes);
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tegra_ivc_init(struct ivc *ivc, uintptr_t rx_base, uintptr_t tx_base,
|
||||
uint32_t nframes, uint32_t frame_size,
|
||||
ivc_notify_function notify)
|
||||
{
|
||||
int32_t result;
|
||||
|
||||
/* sanity check input params */
|
||||
if ((ivc == NULL) || (notify == NULL)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
result = check_ivc_params(rx_base, tx_base, nframes, frame_size);
|
||||
if (result != 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* All sizes that can be returned by communication functions should
|
||||
* fit in a 32-bit integer.
|
||||
*/
|
||||
if (frame_size > (1u << 31)) {
|
||||
return -E2BIG;
|
||||
}
|
||||
|
||||
ivc->rx_channel = (struct ivc_channel_header *)rx_base;
|
||||
ivc->tx_channel = (struct ivc_channel_header *)tx_base;
|
||||
ivc->notify = notify;
|
||||
ivc->frame_size = frame_size;
|
||||
ivc->nframes = nframes;
|
||||
ivc->w_pos = 0U;
|
||||
ivc->r_pos = 0U;
|
||||
|
||||
INFO("%s: done\n", __func__);
|
||||
|
||||
return 0;
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright (c) 2017-2020, NVIDIA Corporation. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
#ifndef BPMP_IVC_H
|
||||
#define BPMP_IVC_H
|
||||
|
||||
#include <lib/utils_def.h>
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#define IVC_ALIGN U(64)
|
||||
#define IVC_CHHDR_TX_FIELDS U(16)
|
||||
#define IVC_CHHDR_RX_FIELDS U(16)
|
||||
|
||||
struct ivc_channel_header;
|
||||
|
||||
struct ivc {
|
||||
struct ivc_channel_header *rx_channel;
|
||||
struct ivc_channel_header *tx_channel;
|
||||
uint32_t w_pos;
|
||||
uint32_t r_pos;
|
||||
void (*notify)(const struct ivc *);
|
||||
uint32_t nframes;
|
||||
uint32_t frame_size;
|
||||
};
|
||||
|
||||
/* callback handler for notify on receiving a response */
|
||||
typedef void (* ivc_notify_function)(const struct ivc *);
|
||||
|
||||
int32_t tegra_ivc_init(struct ivc *ivc, uintptr_t rx_base, uintptr_t tx_base,
|
||||
uint32_t nframes, uint32_t frame_size,
|
||||
ivc_notify_function notify);
|
||||
size_t tegra_ivc_total_queue_size(size_t queue_size);
|
||||
size_t tegra_ivc_align(size_t size);
|
||||
int32_t tegra_ivc_channel_notified(struct ivc *ivc);
|
||||
void tegra_ivc_channel_reset(const struct ivc *ivc);
|
||||
int32_t tegra_ivc_write_advance(struct ivc *ivc);
|
||||
void *tegra_ivc_write_get_next_frame(const struct ivc *ivc);
|
||||
int32_t tegra_ivc_write(struct ivc *ivc, const void *buf, size_t size);
|
||||
int32_t tegra_ivc_read_advance(struct ivc *ivc);
|
||||
void *tegra_ivc_read_get_next_frame(const struct ivc *ivc);
|
||||
int32_t tegra_ivc_read(struct ivc *ivc, void *buf, size_t max_read);
|
||||
bool tegra_ivc_tx_empty(const struct ivc *ivc);
|
||||
bool tegra_ivc_can_write(const struct ivc *ivc);
|
||||
bool tegra_ivc_can_read(const struct ivc *ivc);
|
||||
|
||||
#endif /* BPMP_IVC_H */
|
||||
Reference in New Issue
Block a user