diff options
Diffstat (limited to 'gpxe/src/drivers/net/mlx_ipoib/ib_mt23108.c')
| -rw-r--r-- | gpxe/src/drivers/net/mlx_ipoib/ib_mt23108.c | 1701 |
1 files changed, 1701 insertions, 0 deletions
diff --git a/gpxe/src/drivers/net/mlx_ipoib/ib_mt23108.c b/gpxe/src/drivers/net/mlx_ipoib/ib_mt23108.c new file mode 100644 index 00000000..ca3abb10 --- /dev/null +++ b/gpxe/src/drivers/net/mlx_ipoib/ib_mt23108.c @@ -0,0 +1,1701 @@ +/* + This software is available to you under a choice of one of two + licenses. You may choose to be licensed under the terms of the GNU + General Public License (GPL) Version 2, available at + <http://www.fsf.org/copyleft/gpl.html>, or the OpenIB.org BSD + license, available in the LICENSE.TXT file accompanying this + software. These details are also available at + <http://openib.org/license.html>. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. +*/ + +#include "mt23108.h" +#include "ib_driver.h" +#include <gpxe/pci.h> + +struct device_buffers_st { + union recv_wqe_u mads_qp_rcv_queue[NUM_MADS_RCV_WQES] + __attribute__ ((aligned(RECV_WQE_U_ALIGN))); + union recv_wqe_u ipoib_qp_rcv_queue[NUM_IPOIB_RCV_WQES] + __attribute__ ((aligned(RECV_WQE_U_ALIGN))); + union ud_send_wqe_u mads_qp_snd_queue[NUM_MADS_SND_WQES] + __attribute__ ((aligned(UD_SEND_WQE_U_ALIGN))); + union ud_send_wqe_u ipoib_qp_snd_queue[NUM_IPOIB_SND_WQES] + __attribute__ ((aligned(UD_SEND_WQE_U_ALIGN))); + u8 inprm_buf[INPRM_BUF_SZ] __attribute__ ((aligned(INPRM_BUF_ALIGN))); + u8 outprm_buf[OUTPRM_BUF_SZ] + __attribute__ ((aligned(OUTPRM_BUF_ALIGN))); + struct eqe_t eq_buf[1 << LOG2_EQ_SZ] + __attribute__ ((aligned(sizeof(struct eqe_t)))); + union cqe_st mads_snd_cq_buf[NUM_MADS_SND_CQES] + __attribute__ ((aligned(sizeof(union cqe_st)))); + union cqe_st ipoib_snd_cq_buf[NUM_IPOIB_SND_CQES] + __attribute__ ((aligned(sizeof(union cqe_st)))); + union cqe_st mads_rcv_cq_buf[NUM_MADS_RCV_CQES] + __attribute__ ((aligned(sizeof(union cqe_st)))); + union cqe_st ipoib_rcv_cq_buf[NUM_IPOIB_RCV_CQES] + __attribute__ ((aligned(sizeof(union cqe_st)))); + union ud_av_u av_array[NUM_AVS] + __attribute__ ((aligned(ADDRESS_VECTOR_ST_ALIGN))); +} __attribute__ ((packed)); + +#define STRUCT_ALIGN_SZ 4096 +#define SRC_BUF_SZ (sizeof(struct device_buffers_st) + STRUCT_ALIGN_SZ - 1) + +/* the following must be kept in this order + for the memory region to cover the buffers */ +static u8 src_buf[SRC_BUF_SZ]; +static struct ib_buffers_st ib_buffers; +static __u32 memreg_size; +/* end of order constraint */ + +static struct dev_pci_struct tavor_pci_dev; +static struct device_buffers_st *dev_buffers_p; +static struct device_ib_data_st dev_ib_data; + +static int gw_write_cr(__u32 addr, __u32 data) +{ + writel(htonl(data), tavor_pci_dev.cr_space + addr); + return 0; +} + +static int gw_read_cr(__u32 addr, __u32 * result) +{ + *result = ntohl(readl(tavor_pci_dev.cr_space + addr)); + return 0; +} + +static int reset_hca(void) +{ + return gw_write_cr(TAVOR_RESET_OFFSET, 1); +} + +static int find_mlx_bridge(__u8 hca_bus, __u8 * br_bus_p, __u8 * br_devfn_p) +{ + int bus; + int dev; + int devfn; + int rc; + __u16 vendor, dev_id; + __u8 sec_bus; + + for (bus = 0; bus < 256; ++bus) { + for (dev = 0; dev < 32; ++dev) { + devfn = (dev << 3); + rc = pcibios_read_config_word(bus, devfn, PCI_VENDOR_ID, + &vendor); + if (rc) + return rc; + + if (vendor != MELLANOX_VENDOR_ID) + continue; + + rc = pcibios_read_config_word(bus, devfn, PCI_DEVICE_ID, + &dev_id); + if (rc) + return rc; + + if (dev_id != TAVOR_BRIDGE_DEVICE_ID) + continue; + + rc = pcibios_read_config_byte(bus, devfn, + PCI_SECONDARY_BUS, + &sec_bus); + if (rc) + return rc; + + if (sec_bus == hca_bus) { + *br_bus_p = bus; + *br_devfn_p = devfn; + return 0; + } + } + } + + return -1; +} + +static int ib_device_init(struct pci_device *dev) +{ + int i; + int rc; + __u8 br_bus, br_devfn; + + tprintf(""); + + memset(&dev_ib_data, 0, sizeof dev_ib_data); + + /* save bars */ + tprintf("bus=%d devfn=0x%x", dev->bus, dev->devfn); + for (i = 0; i < 6; ++i) { + tavor_pci_dev.dev.bar[i] = + pci_bar_start(dev, PCI_BASE_ADDRESS_0 + (i << 2)); + tprintf("bar[%d]= 0x%08lx", i, tavor_pci_dev.dev.bar[i]); + } + + tprintf(""); + /* save config space */ + for (i = 0; i < 64; ++i) { + rc = pci_read_config_dword(dev, i << 2, + &tavor_pci_dev.dev. + dev_config_space[i]); + if (rc) { + eprintf(""); + return rc; + } + tprintf("config[%d]= 0x%08lx", i << 2, + tavor_pci_dev.dev.dev_config_space[i]); + } + + tprintf(""); + tavor_pci_dev.dev.dev = dev; + + tprintf(""); + if (dev->dev_id == TAVOR_DEVICE_ID) { + + rc = find_mlx_bridge(dev->bus, &br_bus, &br_devfn); + if (rc) { + eprintf(""); + return rc; + } + + tavor_pci_dev.br.bus = br_bus; + tavor_pci_dev.br.devfn = br_devfn; + + tprintf("bus=%d devfn=0x%x", br_bus, br_devfn); + /* save config space */ + for (i = 0; i < 64; ++i) { + rc = pcibios_read_config_dword(br_bus, br_devfn, i << 2, + &tavor_pci_dev.br. + dev_config_space[i]); + if (rc) { + eprintf(""); + return rc; + } + tprintf("config[%d]= 0x%08lx", i << 2, + tavor_pci_dev.br.dev_config_space[i]); + } + } + + tprintf(""); + + /* map cr-space */ + tavor_pci_dev.cr_space = ioremap(tavor_pci_dev.dev.bar[0], 0x100000); + if (!tavor_pci_dev.cr_space) { + eprintf(""); + return -1; + } + + /* map uar */ + tavor_pci_dev.uar = + ioremap(tavor_pci_dev.dev.bar[2] + UAR_IDX * 0x1000, 0x1000); + if (!tavor_pci_dev.uar) { + eprintf(""); + return -1; + } + tprintf("uar_base (pa:va) = 0x%lx 0x%lx", + tavor_pci_dev.dev.bar[2] + UAR_IDX * 0x1000, tavor_pci_dev.uar); + + tprintf(""); + + return 0; +} + +static inline unsigned long lalign(unsigned long buf, unsigned long align) +{ + return (unsigned long)((buf + align - 1) & + (~(((unsigned long)align) - 1))); +} + +static int init_dev_data(void) +{ + unsigned long tmp; + + tmp = lalign(virt_to_bus(src_buf), STRUCT_ALIGN_SZ); + + dev_buffers_p = bus_to_virt(tmp); + memreg_size = (__u32) (&memreg_size) - (__u32) dev_buffers_p; + tprintf("src_buf=0x%lx, dev_buffers_p=0x%lx, memreg_size=0x%x", src_buf, + dev_buffers_p, memreg_size); + + return 0; +} + +static int restore_config(void) +{ + int i; + int rc; + + if (tavor_pci_dev.dev.dev->dev_id == TAVOR_DEVICE_ID) { + for (i = 0; i < 64; ++i) { + rc = pcibios_write_config_dword(tavor_pci_dev.br.bus, + tavor_pci_dev.br.devfn, + i << 2, + tavor_pci_dev.br. + dev_config_space[i]); + if (rc) { + return rc; + } + } + } + + for (i = 0; i < 64; ++i) { + if (i != 22 && i != 23) { + rc = pci_write_config_dword(tavor_pci_dev.dev.dev, + i << 2, + tavor_pci_dev.dev. + dev_config_space[i]); + if (rc) { + return rc; + } + } + } + return 0; +} + +static void prep_init_hca_buf(const struct init_hca_st *init_hca_p, void *buf) +{ + /*struct init_hca_param_st */ void *p = buf; + void *tmp; + + memset(buf, 0, MT_STRUCT_SIZE(tavorprm_init_hca_st)); + + tmp = + p + MT_BYTE_OFFSET(tavorprm_init_hca_st, + qpc_eec_cqc_eqc_rdb_parameters); + + INS_FLD(init_hca_p->qpc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, + qpc_base_addr_h); + INS_FLD(init_hca_p-> + qpc_base_addr_l >> (32 - + (MT_BIT_SIZE + (tavorprm_qpcbaseaddr_st, + qpc_base_addr_l))), tmp, + tavorprm_qpcbaseaddr_st, qpc_base_addr_l); + INS_FLD(init_hca_p->log_num_of_qp, tmp, tavorprm_qpcbaseaddr_st, + log_num_of_qp); + + INS_FLD(init_hca_p->cqc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, + cqc_base_addr_h); + INS_FLD(init_hca_p-> + cqc_base_addr_l >> (32 - + (MT_BIT_SIZE + (tavorprm_qpcbaseaddr_st, + cqc_base_addr_l))), tmp, + tavorprm_qpcbaseaddr_st, cqc_base_addr_l); + INS_FLD(init_hca_p->log_num_of_cq, tmp, tavorprm_qpcbaseaddr_st, + log_num_of_cq); + + INS_FLD(init_hca_p->eqc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, + eqc_base_addr_h); + INS_FLD(init_hca_p-> + eqc_base_addr_l >> (32 - + (MT_BIT_SIZE + (tavorprm_qpcbaseaddr_st, + eqc_base_addr_l))), tmp, + tavorprm_qpcbaseaddr_st, eqc_base_addr_l); + INS_FLD(LOG2_EQS, tmp, tavorprm_qpcbaseaddr_st, log_num_eq); + + INS_FLD(init_hca_p->srqc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, + srqc_base_addr_h); + INS_FLD(init_hca_p-> + srqc_base_addr_l >> (32 - + (MT_BIT_SIZE + (tavorprm_qpcbaseaddr_st, + srqc_base_addr_l))), tmp, + tavorprm_qpcbaseaddr_st, srqc_base_addr_l); + INS_FLD(init_hca_p->log_num_of_srq, tmp, tavorprm_qpcbaseaddr_st, + log_num_of_srq); + + INS_FLD(init_hca_p->eqpc_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, + eqpc_base_addr_h); + INS_FLD(init_hca_p->eqpc_base_addr_l, tmp, tavorprm_qpcbaseaddr_st, + eqpc_base_addr_l); + + INS_FLD(init_hca_p->eeec_base_addr_h, tmp, tavorprm_qpcbaseaddr_st, + eeec_base_addr_h); + INS_FLD(init_hca_p->eeec_base_addr_l, tmp, tavorprm_qpcbaseaddr_st, + eeec_base_addr_l); + + tmp = p + MT_BYTE_OFFSET(tavorprm_init_hca_st, multicast_parameters); + + INS_FLD(init_hca_p->mc_base_addr_h, tmp, tavorprm_multicastparam_st, + mc_base_addr_h); + INS_FLD(init_hca_p->mc_base_addr_l, tmp, tavorprm_multicastparam_st, + mc_base_addr_l); + + INS_FLD(init_hca_p->log_mc_table_entry_sz, tmp, + tavorprm_multicastparam_st, log_mc_table_entry_sz); + INS_FLD(init_hca_p->log_mc_table_sz, tmp, tavorprm_multicastparam_st, + log_mc_table_sz); + INS_FLD(init_hca_p->mc_table_hash_sz, tmp, tavorprm_multicastparam_st, + mc_table_hash_sz); + + tmp = p + MT_BYTE_OFFSET(tavorprm_init_hca_st, tpt_parameters); + + INS_FLD(init_hca_p->mpt_base_addr_h, tmp, tavorprm_tptparams_st, + mpt_base_adr_h); + INS_FLD(init_hca_p->mpt_base_addr_l, tmp, tavorprm_tptparams_st, + mpt_base_adr_l); + INS_FLD(init_hca_p->log_mpt_sz, tmp, tavorprm_tptparams_st, log_mpt_sz); + + INS_FLD(init_hca_p->mtt_base_addr_h, tmp, tavorprm_tptparams_st, + mtt_base_addr_h); + INS_FLD(init_hca_p->mtt_base_addr_l, tmp, tavorprm_tptparams_st, + mtt_base_addr_l); + + tmp = p + MT_BYTE_OFFSET(tavorprm_init_hca_st, uar_parameters); + INS_FLD(tavor_pci_dev.dev.bar[3], tmp, tavorprm_uar_params_st, + uar_base_addr_h); + INS_FLD(tavor_pci_dev.dev.bar[2] & 0xfff00000, tmp, + tavorprm_uar_params_st, uar_base_addr_l); + +} + +static void prep_sw2hw_mpt_buf(void *buf, __u32 mkey) +{ + INS_FLD(1, buf, tavorprm_mpt_st, m_io); + INS_FLD(1, buf, tavorprm_mpt_st, lw); + INS_FLD(1, buf, tavorprm_mpt_st, lr); + INS_FLD(1, buf, tavorprm_mpt_st, pa); + INS_FLD(1, buf, tavorprm_mpt_st, r_w); + + INS_FLD(mkey, buf, tavorprm_mpt_st, mem_key); + INS_FLD(GLOBAL_PD, buf, tavorprm_mpt_st, pd); + + INS_FLD(virt_to_bus(dev_buffers_p), buf, tavorprm_mpt_st, + start_address_l); + INS_FLD(memreg_size, buf, tavorprm_mpt_st, reg_wnd_len_l); +} + +static void prep_sw2hw_eq_buf(void *buf, struct eqe_t *eq) +{ + memset(buf, 0, MT_STRUCT_SIZE(tavorprm_eqc_st)); + + INS_FLD(2, buf, tavorprm_eqc_st, st); /* fired */ + INS_FLD(virt_to_bus(eq), buf, tavorprm_eqc_st, start_address_l); + INS_FLD(LOG2_EQ_SZ, buf, tavorprm_eqc_st, log_eq_size); + INS_FLD(UAR_IDX, buf, tavorprm_eqc_st, usr_page); + INS_FLD(GLOBAL_PD, buf, tavorprm_eqc_st, pd); + INS_FLD(dev_ib_data.mkey, buf, tavorprm_eqc_st, lkey); +} + +static void init_eq_buf(void *eq_buf) +{ + int num_eqes = 1 << LOG2_EQ_SZ; + + memset(eq_buf, 0xff, num_eqes * sizeof(struct eqe_t)); +} + +static void prep_init_ib_buf(void *buf) +{ + __u32 *ptr = (__u32 *) buf; + + ptr[0] = 0x4310; + ptr[1] = 1; + ptr[2] = 64; +} + +static void prep_sw2hw_cq_buf(void *buf, __u8 eqn, __u32 cqn, + union cqe_st *cq_buf) +{ + __u32 *ptr = (__u32 *) buf; + + ptr[2] = virt_to_bus(cq_buf); + ptr[3] = (LOG2_CQ_SZ << 24) | UAR_IDX; + ptr[4] = eqn; + ptr[5] = eqn; + ptr[6] = dev_ib_data.pd; + ptr[7] = dev_ib_data.mkey; + ptr[12] = cqn; +} + +static void prep_rst2init_qpee_buf(void *buf, __u32 snd_cqn, __u32 rcv_cqn, + __u32 qkey) +{ + struct qp_ee_state_tarnisition_st *prm; + void *tmp; + + prm = (struct qp_ee_state_tarnisition_st *)buf; + + INS_FLD(3, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, st); /* service type = UD */ + INS_FLD(3, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, pm_state); /* required for UD QP */ + INS_FLD(UAR_IDX, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, + usr_page); + INS_FLD(dev_ib_data.pd, &prm->ctx, + tavorprm_queue_pair_ee_context_entry_st, pd); + INS_FLD(dev_ib_data.mkey, &prm->ctx, + tavorprm_queue_pair_ee_context_entry_st, wqe_lkey); + INS_FLD(1, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, ssc); /* generate send CQE */ + INS_FLD(1, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, rsc); /* generate receive CQE */ + INS_FLD(snd_cqn, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, + cqn_snd); + INS_FLD(rcv_cqn, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, + cqn_rcv); + INS_FLD(qkey, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, + q_key); + + tmp = + (void *)(&prm->ctx) + + MT_BYTE_OFFSET(tavorprm_queue_pair_ee_context_entry_st, + primary_address_path); + INS_FLD(dev_ib_data.port, tmp, tavorprm_address_path_st, port_number); + + INS_FLD(4, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, mtu); + INS_FLD(0xb, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, + msg_max); +} + +static void prep_init2rtr_qpee_buf(void *buf) +{ + struct qp_ee_state_tarnisition_st *prm; + + prm = (struct qp_ee_state_tarnisition_st *)buf; + + INS_FLD(4, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, mtu); + INS_FLD(0xb, &prm->ctx, tavorprm_queue_pair_ee_context_entry_st, + msg_max); +} + +static void init_av_array() +{ + int i; + + dev_ib_data.udav.av_array = dev_buffers_p->av_array; + dev_ib_data.udav.udav_next_free = FL_EOL; + for (i = 0; i < NUM_AVS; ++i) { + dev_ib_data.udav.av_array[i].ud_av.next_free = + dev_ib_data.udav.udav_next_free; + dev_ib_data.udav.udav_next_free = i; + } + tprintf("dev_ib_data.udav.udav_next_free=%d", i); +} + +static int setup_hca(__u8 port, void **eq_p) +{ + int rc; + __u32 key, in_key; + __u32 *inprm; + struct eqe_t *eq_buf; + __u32 event_mask; + void *cfg; + int ret = 0; + __u8 eqn; + struct dev_lim_st dev_lim; + struct init_hca_st init_hca; + __u32 offset, base_h, base_l; + const __u32 delta = 0x400000; + struct query_fw_st qfw; + + tprintf("called"); + + init_dev_data(); + + rc = reset_hca(); + if (rc) { + ret = -1; + eprintf(""); + goto exit; + } else { + tprintf("reset_hca() success"); + } + + mdelay(1000); /* wait for 1 sec */ + + rc = restore_config(); + if (rc) { + ret = -1; + eprintf(""); + goto exit; + } else { + tprintf("restore_config() success"); + } + + dev_ib_data.pd = GLOBAL_PD; + dev_ib_data.port = port; + + /* execute system enable command */ + rc = cmd_sys_en(); + if (rc) { + ret = -1; + eprintf(""); + goto exit; + } else { + tprintf("cmd_sys_en() success"); + } + + rc= cmd_query_fw(&qfw); + if (rc) { + ret = -1; + eprintf(""); + goto exit; + } else { + tprintf("cmd_query_fw() success"); + + if (print_info) { + printf("FW ver = %d.%d.%d\n", + qfw.fw_rev_major, + qfw.fw_rev_minor, + qfw.fw_rev_subminor); + } + tprintf("fw_rev_major=%d", qfw.fw_rev_major); + tprintf("fw_rev_minor=%d", qfw.fw_rev_minor); + tprintf("fw_rev_subminor=%d", qfw.fw_rev_subminor); + tprintf("error_buf_start_h=0x%x", qfw.error_buf_start_h); + tprintf("error_buf_start_l=0x%x", qfw.error_buf_start_l); + tprintf("error_buf_size=%d", qfw.error_buf_size); + } + + if (qfw.error_buf_start_h) { + eprintf("too high physical address"); + ret = -1; + goto exit; + } + + dev_ib_data.error_buf_addr= ioremap(qfw.error_buf_start_l, + qfw.error_buf_size*4); + dev_ib_data.error_buf_size= qfw.error_buf_size; + if (!dev_ib_data.error_buf_addr) { + eprintf(""); + ret = -1; + goto exit; + } + + + rc = cmd_query_dev_lim(&dev_lim); + if (rc) { + ret = -1; + eprintf(""); + goto exit; + } else { + tprintf("cmd_query_dev_lim() success"); + tprintf("log2_rsvd_qps=%x", dev_lim.log2_rsvd_qps); + tprintf("qpc_entry_sz=%x", dev_lim.qpc_entry_sz); + tprintf("log2_rsvd_srqs=%x", dev_lim.log2_rsvd_srqs); + tprintf("srq_entry_sz=%x", dev_lim.srq_entry_sz); + tprintf("log2_rsvd_ees=%x", dev_lim.log2_rsvd_ees); + tprintf("eec_entry_sz=%x", dev_lim.eec_entry_sz); + tprintf("log2_rsvd_cqs=%x", dev_lim.log2_rsvd_cqs); + tprintf("cqc_entry_sz=%x", dev_lim.cqc_entry_sz); + tprintf("log2_rsvd_mtts=%x", dev_lim.log2_rsvd_mtts); + tprintf("mtt_entry_sz=%x", dev_lim.mtt_entry_sz); + tprintf("log2_rsvd_mrws=%x", dev_lim.log2_rsvd_mrws); + tprintf("mpt_entry_sz=%x", dev_lim.mpt_entry_sz); + tprintf("eqc_entry_sz=%x", dev_lim.eqc_entry_sz); + } + + /* set the qp and cq numbers according + to the results of query_dev_lim */ + dev_ib_data.mads_qp.qpn = (1 << dev_lim.log2_rsvd_qps) + + +QPN_BASE + MADS_QPN_SN; + dev_ib_data.ipoib_qp.qpn = (1 << dev_lim.log2_rsvd_qps) + + +QPN_BASE + IPOIB_QPN_SN; + + dev_ib_data.mads_qp.snd_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) + + MADS_SND_CQN_SN; + dev_ib_data.mads_qp.rcv_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) + + MADS_RCV_CQN_SN; + + dev_ib_data.ipoib_qp.snd_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) + + IPOIB_SND_CQN_SN; + dev_ib_data.ipoib_qp.rcv_cq.cqn = (1 << dev_lim.log2_rsvd_cqs) + + IPOIB_RCV_CQN_SN; + + /* disable SRQ */ + cfg = (void *)dev_buffers_p->inprm_buf; + memset(cfg, 0, MT_STRUCT_SIZE(tavorprm_mod_stat_cfg_st)); + INS_FLD(1, cfg, tavorprm_mod_stat_cfg_st, srq_m); //cfg->srq_m = 1; + rc = cmd_mod_stat_cfg(cfg); + if (rc) { + ret = -1; + eprintf(""); + goto exit; + } else { + tprintf("cmd_mod_stat_cfg() success"); + } + + /* prepare the init_hca params to pass + to prep_init_hca_buf */ + memset(&init_hca, 0, sizeof init_hca); + offset = 0; + base_h = tavor_pci_dev.dev.bar[5] & 0xfffffff0; + base_l = tavor_pci_dev.dev.bar[4] & 0xfffffff0; + + tprintf("base_h=0x%lx, base_l=0x%lx", base_h, base_l); + + init_hca.qpc_base_addr_h = base_h; + init_hca.qpc_base_addr_l = base_l + offset; + init_hca.log_num_of_qp = dev_lim.log2_rsvd_qps + 1; + offset += delta; + + init_hca.eec_base_addr_h = base_h; + init_hca.eec_base_addr_l = base_l + offset; + init_hca.log_num_of_ee = dev_lim.log2_rsvd_ees; + offset += delta; + + init_hca.srqc_base_addr_h = base_h; + init_hca.srqc_base_addr_l = base_l + offset; + init_hca.log_num_of_srq = dev_lim.log2_rsvd_srqs; + offset += delta; + + init_hca.cqc_base_addr_h = base_h; + init_hca.cqc_base_addr_l = base_l + offset; + init_hca.log_num_of_cq = dev_lim.log2_rsvd_cqs + 1; + offset += delta; + + init_hca.eqpc_base_addr_h = base_h; + init_hca.eqpc_base_addr_l = base_l + offset; + offset += delta; + + init_hca.eeec_base_addr_h = base_h; + init_hca.eeec_base_addr_l = base_l + offset; + offset += delta; + + init_hca.eqc_base_addr_h = base_h; + init_hca.eqc_base_addr_l = base_l + offset; + init_hca.log_num_of_eq = LOG2_EQS; + offset += delta; + + init_hca.rdb_base_addr_h = base_h; + init_hca.rdb_base_addr_l = base_l + offset; + offset += delta; + + init_hca.mc_base_addr_h = base_h; + init_hca.mc_base_addr_l = base_l + offset; + init_hca.log_mc_table_entry_sz = LOG2_MC_ENTRY; + init_hca.mc_table_hash_sz = 0; + init_hca.log_mc_table_sz = LOG2_MC_GROUPS; + offset += delta; + + init_hca.mpt_base_addr_h = base_h; + init_hca.mpt_base_addr_l = base_l + offset; + init_hca.log_mpt_sz = dev_lim.log2_rsvd_mrws + 1; + offset += delta; + + init_hca.mtt_base_addr_h = base_h; + init_hca.mtt_base_addr_l = base_l + offset; + + /* this buffer is used for all the commands */ + inprm = (void *)dev_buffers_p->inprm_buf; + /* excute init_hca command */ + prep_init_hca_buf(&init_hca, inprm); + + rc = cmd_init_hca(inprm, MT_STRUCT_SIZE(tavorprm_init_hca_st)); + if (rc) { + ret = -1; + eprintf(""); + goto undo_sys_en; + } else + tprintf("cmd_init_hca() success"); + + /* register a single memory region which covers + 4 GB of the address space which will be used + throughout the driver */ + memset(inprm, 0, SW2HW_MPT_IBUF_SZ); + in_key = MKEY_PREFIX + (1 << dev_lim.log2_rsvd_mrws); + prep_sw2hw_mpt_buf(inprm, in_key); + rc = cmd_sw2hw_mpt(&key, in_key, inprm, SW2HW_MPT_IBUF_SZ); + if (rc) { + ret = -1; + eprintf(""); + goto undo_init_hca; + } else { + tprintf("cmd_sw2hw_mpt() success, key=0x%lx", key); + } + dev_ib_data.mkey = key; + + eqn = EQN; + /* allocate a single EQ which will receive + all the events */ + eq_buf = dev_buffers_p->eq_buf; + init_eq_buf(eq_buf); /* put in HW ownership */ + prep_sw2hw_eq_buf(inprm, eq_buf); + rc = cmd_sw2hw_eq(SW2HW_EQ_IBUF_SZ); + if (rc) { + ret = -1; + eprintf(""); + goto undo_sw2hw_mpt; + } else + tprintf("cmd_sw2hw_eq() success"); + + event_mask = (1 << XDEV_EV_TYPE_CQ_COMP) | + (1 << XDEV_EV_TYPE_CQ_ERR) | + (1 << XDEV_EV_TYPE_LOCAL_WQ_CATAS_ERR) | + (1 << XDEV_EV_TYPE_PORT_ERR) | + (1 << XDEV_EV_TYPE_LOCAL_WQ_INVALID_REQ_ERR) | + (1 << XDEV_EV_TYPE_LOCAL_WQ_ACCESS_VIOL_ERR) | + (1 << TAVOR_IF_EV_TYPE_OVERRUN); + rc = cmd_map_eq(eqn, event_mask, 1); + if (rc) { + ret = -1; + eprintf(""); + goto undo_sw2hw_eq; + } else + tprintf("cmd_map_eq() success"); + + dev_ib_data.eq.eqn = eqn; + dev_ib_data.eq.eq_buf = eq_buf; + dev_ib_data.eq.cons_idx = 0; + dev_ib_data.eq.eq_size = 1 << LOG2_EQ_SZ; + *eq_p = &dev_ib_data.eq; + + memset(inprm, 0, INIT_IB_IBUF_SZ); + prep_init_ib_buf(inprm); + rc = cmd_init_ib(port, inprm, INIT_IB_IBUF_SZ); + if (rc) { + ret = -1; + eprintf(""); + goto undo_sw2hw_eq; + } else + tprintf("cmd_init_ib() success"); + + init_av_array(); + tprintf("init_av_array() done"); + + goto exit; + + undo_sw2hw_eq: + rc = cmd_hw2sw_eq(EQN); + if (rc) { + eprintf(""); + } else + tprintf("cmd_hw2sw_eq() success"); + + undo_sw2hw_mpt: + rc = cmd_hw2sw_mpt(key); + if (rc) + eprintf(""); + else + tprintf("cmd_hw2sw_mpt() success key=0x%lx", key); + + undo_init_hca: + rc = cmd_close_hca(0); + if (rc) { + eprintf(""); + goto undo_sys_en; + } else + tprintf("cmd_close_hca() success"); + + undo_sys_en: + rc = cmd_sys_dis(); + if (rc) { + eprintf(""); + goto undo_sys_en; + } else + tprintf("cmd_sys_dis() success"); + goto exit; + + exit: + return ret; +} + +static void *get_inprm_buf(void) +{ + return dev_buffers_p->inprm_buf; +} + +static void *get_outprm_buf(void) +{ + return dev_buffers_p->outprm_buf; +} + +static void *get_send_wqe_buf(void *wqe, __u8 index) +{ + struct ud_send_wqe_st *snd_wqe = wqe; + + return bus_to_virt(snd_wqe->mpointer[index].local_addr_l); +} + +static void *get_rcv_wqe_buf(void *wqe, __u8 index) +{ + struct recv_wqe_st *rcv_wqe = wqe; + + return bus_to_virt(be32_to_cpu(rcv_wqe->mpointer[index].local_addr_l)); +} + +static void modify_av_params(struct ud_av_st *av, + __u16 dlid, + __u8 g, + __u8 sl, __u8 rate, union ib_gid_u *gid, __u32 qpn) +{ + memset(&av->av, 0, sizeof av->av); + INS_FLD(dev_ib_data.port, &av->av, tavorprm_ud_address_vector_st, + port_number); + INS_FLD(dev_ib_data.pd, &av->av, tavorprm_ud_address_vector_st, pd); + INS_FLD(dlid, &av->av, tavorprm_ud_address_vector_st, rlid); + INS_FLD(g, &av->av, tavorprm_ud_address_vector_st, g); + INS_FLD(sl, &av->av, tavorprm_ud_address_vector_st, sl); + INS_FLD(3, &av->av, tavorprm_ud_address_vector_st, msg); + + if (rate >= 3) + INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, max_stat_rate); /* 4x */ + else + INS_FLD(1, &av->av, tavorprm_ud_address_vector_st, max_stat_rate); /* 1x */ + + cpu_to_be_buf(&av->av, sizeof(av->av)); + if (g) { + if (gid) { + INS_FLD(*((__u32 *) (&gid->raw[0])), &av->av, + tavorprm_ud_address_vector_st, rgid_127_96); + INS_FLD(*((__u32 *) (&gid->raw[4])), &av->av, + tavorprm_ud_address_vector_st, rgid_95_64); + INS_FLD(*((__u32 *) (&gid->raw[8])), &av->av, + tavorprm_ud_address_vector_st, rgid_63_32); + INS_FLD(*((__u32 *) (&gid->raw[12])), &av->av, + tavorprm_ud_address_vector_st, rgid_31_0); + } else { + INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, + rgid_127_96); + INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, + rgid_95_64); + INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, + rgid_63_32); + INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, + rgid_31_0); + } + } else { + INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, rgid_127_96); + INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, rgid_95_64); + INS_FLD(0, &av->av, tavorprm_ud_address_vector_st, rgid_63_32); + INS_FLD(2, &av->av, tavorprm_ud_address_vector_st, rgid_31_0); + } + av->dest_qp = qpn; +} + +static void init_cq_buf(union cqe_st *cq_buf, __u8 num_cqes) +{ + memset(cq_buf, 0xff, num_cqes * sizeof cq_buf[0]); +} + +static int post_rcv_buf(struct udqp_st *qp, struct recv_wqe_st *rcv_wqe) +{ + struct recv_doorbell_st dbell; + int rc; + __u32 tmp[2]; + struct recv_wqe_st *tmp_wqe = (struct recv_wqe_st *)tmp; + __u32 *ptr_dst; + + memset(&dbell, 0, sizeof dbell); + INS_FLD(sizeof(*rcv_wqe) >> 4, &dbell, tavorprm_receive_doorbell_st, + nds); + INS_FLD(virt_to_bus(rcv_wqe) >> 6, &dbell, tavorprm_receive_doorbell_st, + nda); + INS_FLD(qp->qpn, &dbell, tavorprm_receive_doorbell_st, qpn); + INS_FLD(1, &dbell, tavorprm_receive_doorbell_st, credits); + + if (qp->last_posted_rcv_wqe) { + memcpy(tmp, qp->last_posted_rcv_wqe, sizeof(tmp)); + be_to_cpu_buf(tmp, sizeof(tmp)); + INS_FLD(1, tmp_wqe->next, wqe_segment_next_st, dbd); + INS_FLD(sizeof(*rcv_wqe) >> 4, tmp_wqe->next, + wqe_segment_next_st, nds); + INS_FLD(virt_to_bus(rcv_wqe) >> 6, tmp_wqe->next, + wqe_segment_next_st, nda_31_6); + /* this is not really opcode but since the struct + is used for both send and receive, in receive this bit must be 1 + which coinsides with nopcode */ + INS_FLD(1, tmp_wqe->next, wqe_segment_next_st, nopcode); + + cpu_to_be_buf(tmp, sizeof(tmp)); + + ptr_dst = (__u32 *) (qp->last_posted_rcv_wqe); + ptr_dst[0] = tmp[0]; + ptr_dst[1] = tmp[1]; + } + rc = cmd_post_doorbell(&dbell, POST_RCV_OFFSET); + if (!rc) { + qp->last_posted_rcv_wqe = rcv_wqe; + } + + return rc; +} + +static int post_send_req(void *qph, void *wqeh, __u8 num_gather) +{ + struct send_doorbell_st dbell; + int rc; + struct udqp_st *qp = qph; + struct ud_send_wqe_st *snd_wqe = wqeh; + struct next_control_seg_st tmp; + __u32 *psrc, *pdst; + __u32 nds; + + tprintf("snd_wqe=0x%lx, virt_to_bus(snd_wqe)=0x%lx", snd_wqe, + virt_to_bus(snd_wqe)); + + memset(&dbell, 0, sizeof dbell); + INS_FLD(XDEV_NOPCODE_SEND, &dbell, tavorprm_send_doorbell_st, nopcode); + INS_FLD(1, &dbell, tavorprm_send_doorbell_st, f); + INS_FLD(virt_to_bus(snd_wqe) >> 6, &dbell, tavorprm_send_doorbell_st, + nda); + nds = + (sizeof(snd_wqe->next) + sizeof(snd_wqe->udseg) + + sizeof(snd_wqe->mpointer[0]) * num_gather) >> 4; + INS_FLD(nds, &dbell, tavorprm_send_doorbell_st, nds); + INS_FLD(qp->qpn, &dbell, tavorprm_send_doorbell_st, qpn); + + tprintf("0= %lx", ((__u32 *) ((void *)(&dbell)))[0]); + tprintf("1= %lx", ((__u32 *) ((void *)(&dbell)))[1]); + + if (qp->last_posted_snd_wqe) { + memcpy(&tmp, &qp->last_posted_snd_wqe->next, sizeof tmp); + be_to_cpu_buf(&tmp, sizeof tmp); + INS_FLD(1, &tmp, wqe_segment_next_st, dbd); + INS_FLD(virt_to_bus(snd_wqe) >> 6, &tmp, wqe_segment_next_st, + nda_31_6); + INS_FLD(nds, &tmp, wqe_segment_next_st, nds); + + psrc = (__u32 *) (&tmp); + pdst = (__u32 *) (&qp->last_posted_snd_wqe->next); + pdst[0] = htonl(psrc[0]); + pdst[1] = htonl(psrc[1]); + } + + rc = cmd_post_doorbell(&dbell, POST_SND_OFFSET); + if (!rc) { + qp->last_posted_snd_wqe = snd_wqe; + } + + return rc; +} + +static int create_mads_qp(void **qp_pp, void **snd_cq_pp, void **rcv_cq_pp) +{ + __u8 i; + int rc; + struct udqp_st *qp; + + qp = &dev_ib_data.mads_qp; + + /* set the pointer to the receive WQEs buffer */ + qp->rcv_wq = dev_buffers_p->mads_qp_rcv_queue; + + qp->send_buf_sz = MAD_BUF_SZ; + qp->rcv_buf_sz = MAD_BUF_SZ; + + qp->recv_wqe_alloc_idx = 0; + qp->max_recv_wqes = NUM_MADS_RCV_WQES; + qp->recv_wqe_cur_free = NUM_MADS_RCV_WQES; + + /* iterrate through the list */ + for (i = 0; i < NUM_MADS_RCV_WQES; ++i) { + /* clear the WQE */ + memset(&qp->rcv_wq[i], 0, sizeof(qp->rcv_wq[i])); + + qp->rcv_wq[i].wqe_cont.qp = qp; + qp->rcv_bufs[i] = ib_buffers.rcv_mad_buf[i]; + } + + /* set the pointer to the send WQEs buffer */ + qp->snd_wq = dev_buffers_p->mads_qp_snd_queue; + + qp->snd_wqe_alloc_idx = 0; + qp->max_snd_wqes = NUM_MADS_SND_WQES; + qp->snd_wqe_cur_free = NUM_MADS_SND_WQES; + + /* iterrate through the list */ + for (i = 0; i < NUM_MADS_SND_WQES; ++i) { + /* clear the WQE */ + memset(&qp->snd_wq[i], 0, sizeof(qp->snd_wq[i])); + + /* link the WQE to the free list */ + qp->snd_wq[i].wqe_cont.qp = qp; + qp->snd_bufs[i] = ib_buffers.send_mad_buf[i]; + } + + /* qp number and cq numbers are already set up */ + qp->snd_cq.cq_buf = dev_buffers_p->mads_snd_cq_buf; + qp->rcv_cq.cq_buf = dev_buffers_p->mads_rcv_cq_buf; + qp->snd_cq.num_cqes = NUM_MADS_SND_CQES; + qp->rcv_cq.num_cqes = NUM_MADS_RCV_CQES; + qp->qkey = GLOBAL_QKEY; + rc = create_udqp(qp); + if (!rc) { + *qp_pp = qp; + *snd_cq_pp = &qp->snd_cq; + *rcv_cq_pp = &qp->rcv_cq; + } + + return rc; +} + +static int create_ipoib_qp(void **qp_pp, + void **snd_cq_pp, void **rcv_cq_pp, __u32 qkey) +{ + __u8 i; + int rc; + struct udqp_st *qp; + qp = &dev_ib_data.ipoib_qp; + + /* set the pointer to the receive WQEs buffer */ + qp->rcv_wq = dev_buffers_p->ipoib_qp_rcv_queue; + + qp->rcv_buf_sz = IPOIB_RCV_BUF_SZ; + + qp->recv_wqe_alloc_idx = 0; + qp->max_recv_wqes = NUM_IPOIB_RCV_WQES; + qp->recv_wqe_cur_free = NUM_IPOIB_RCV_WQES; + + /* iterrate through the list */ + for (i = 0; i < NUM_IPOIB_RCV_WQES; ++i) { + /* clear the WQE */ + memset(&qp->rcv_wq[i], 0, sizeof(qp->rcv_wq[i])); + + /* update data */ + qp->rcv_wq[i].wqe_cont.qp = qp; + qp->rcv_bufs[i] = ib_buffers.ipoib_rcv_buf[i]; + tprintf("rcv_buf=%lx", qp->rcv_bufs[i]); + } + + /* init send queue WQEs list */ + /* set the list empty */ + qp->snd_wqe_alloc_idx = 0; + qp->max_snd_wqes = NUM_IPOIB_SND_WQES; + qp->snd_wqe_cur_free = NUM_IPOIB_SND_WQES; + + /* set the pointer to the send WQEs buffer */ + qp->snd_wq = dev_buffers_p->ipoib_qp_snd_queue; + + /* iterrate through the list */ + for (i = 0; i < NUM_IPOIB_SND_WQES; ++i) { + /* clear the WQE */ + memset(&qp->snd_wq[i], 0, sizeof(qp->snd_wq[i])); + + /* update data */ + qp->snd_wq[i].wqe_cont.qp = qp; + qp->snd_bufs[i] = ib_buffers.send_ipoib_buf[i]; + qp->send_buf_sz = 4; + } + + /* qp number and cq numbers are already set up */ + + qp->snd_cq.cq_buf = dev_buffers_p->ipoib_snd_cq_buf; + qp->rcv_cq.cq_buf = dev_buffers_p->ipoib_rcv_cq_buf; + qp->snd_cq.num_cqes = NUM_IPOIB_SND_CQES; + qp->rcv_cq.num_cqes = NUM_IPOIB_RCV_CQES; + qp->qkey = qkey; + rc = create_udqp(qp); + if (!rc) { + *qp_pp = qp; + *snd_cq_pp = &qp->snd_cq; + *rcv_cq_pp = &qp->rcv_cq; + } + + return rc; +} + +static int create_udqp(struct udqp_st *qp) +{ + int rc, ret = 0; + void *inprm; + struct recv_wqe_st *rcv_wqe; + + inprm = dev_buffers_p->inprm_buf; + + /* create send CQ */ + init_cq_buf(qp->snd_cq.cq_buf, qp->snd_cq.num_cqes); + qp->snd_cq.cons_idx = 0; + memset(inprm, 0, SW2HW_CQ_IBUF_SZ); + prep_sw2hw_cq_buf(inprm, dev_ib_data.eq.eqn, qp->snd_cq.cqn, + qp->snd_cq.cq_buf); + rc = cmd_sw2hw_cq(qp->snd_cq.cqn, inprm, SW2HW_CQ_IBUF_SZ); + if (rc) { + ret = -1; + eprintf(""); + goto exit; + } + + /* create receive CQ */ + init_cq_buf(qp->rcv_cq.cq_buf, qp->rcv_cq.num_cqes); + qp->rcv_cq.cons_idx = 0; + memset(inprm, 0, SW2HW_CQ_IBUF_SZ); + prep_sw2hw_cq_buf(inprm, dev_ib_data.eq.eqn, qp->rcv_cq.cqn, + qp->rcv_cq.cq_buf); + rc = cmd_sw2hw_cq(qp->rcv_cq.cqn, inprm, SW2HW_CQ_IBUF_SZ); + if (rc) { + ret = -1; + eprintf(""); + goto undo_snd_cq; + } + + memset(inprm, 0, QPCTX_IBUF_SZ); + prep_rst2init_qpee_buf(inprm, qp->snd_cq.cqn, qp->rcv_cq.cqn, qp->qkey); + rc = cmd_rst2init_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ); + if (rc) { + ret = -1; + eprintf(""); + goto undo_rcv_cq; + } + + qp->last_posted_rcv_wqe = NULL; + qp->last_posted_snd_wqe = NULL; + + /* post all the buffers to the receive queue */ + while (1) { + /* allocate wqe */ + rcv_wqe = alloc_rcv_wqe(qp); + if (!rcv_wqe) + break; + + /* post the buffer */ + rc = post_rcv_buf(qp, rcv_wqe); + if (rc) { + ret = -1; + eprintf(""); + goto undo_rcv_cq; + } + } + + memset(inprm, 0, QPCTX_IBUF_SZ); + prep_init2rtr_qpee_buf(inprm); + rc = cmd_init2rtr_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ); + if (rc) { + ret = -1; + eprintf(""); + goto undo_rcv_cq; + } + + memset(inprm, 0, QPCTX_IBUF_SZ); + rc = cmd_rtr2rts_qpee(qp->qpn, inprm, QPCTX_IBUF_SZ); + if (rc) { + ret = -1; + eprintf(""); + goto undo_rcv_cq; + } + + goto exit; + + undo_rcv_cq: + rc = cmd_hw2sw_cq(qp->rcv_cq.cqn); + if (rc) + eprintf(""); + + undo_snd_cq: + rc = cmd_hw2sw_cq(qp->snd_cq.cqn); + if (rc) + eprintf(""); + + exit: + return ret; +} + +static int destroy_udqp(struct udqp_st *qp) +{ + int rc; + + rc = cmd_2err_qpee(qp->qpn); + if (rc) { + eprintf(""); + return rc; + } + tprintf("cmd_2err_qpee(0x%lx) success", qp->qpn); + + rc = cmd_2rst_qpee(qp->qpn); + if (rc) { + eprintf(""); + return rc; + } + tprintf("cmd_2rst_qpee(0x%lx) success", qp->qpn); + + rc = cmd_hw2sw_cq(qp->rcv_cq.cqn); + if (rc) { + eprintf(""); + return rc; + } + tprintf("cmd_hw2sw_cq(0x%lx) success", qp->snd_cq.cqn); + + rc = cmd_hw2sw_cq(qp->snd_cq.cqn); + if (rc) { + eprintf(""); + return rc; + } + tprintf("cmd_hw2sw_cq(0x%lx) success", qp->rcv_cq.cqn); + + return rc; +} + +static void prep_send_wqe_buf(void *qph, + void *avh, + void *wqeh, + const void *buf, + unsigned int offset, __u16 len, __u8 e) +{ + struct udqp_st *qp = qph; + struct ud_av_st *av = avh; + struct ud_send_wqe_st *wqe = wqeh; + + INS_FLD(e, wqe->next.control, wqe_segment_ctrl_send_st, e); + INS_FLD(1, wqe->next.control, wqe_segment_ctrl_send_st, always1); + + wqe->udseg.av_add_h = 0; + wqe->udseg.av_add_l = virt_to_bus(&av->av); + wqe->udseg.dest_qp = av->dest_qp; + wqe->udseg.lkey = dev_ib_data.mkey; + wqe->udseg.qkey = qp->qkey; + + if (buf) { + memcpy(bus_to_virt(wqe->mpointer[0].local_addr_l) + offset, buf, + len); + len += offset; + } + wqe->mpointer[0].byte_count = len; + wqe->mpointer[0].lkey = dev_ib_data.mkey; + + cpu_to_be_buf(wqe, sizeof *wqe); +} + +static void *alloc_ud_av(void) +{ + u8 next_free; + + if (dev_ib_data.udav.udav_next_free == FL_EOL) { + return NULL; + } + + next_free = dev_ib_data.udav.udav_next_free; + dev_ib_data.udav.udav_next_free = + dev_buffers_p->av_array[next_free].ud_av.next_free; + tprintf("allocated udav %d", next_free); + return &dev_buffers_p->av_array[next_free].ud_av; +} + +static void free_ud_av(void *avh) +{ + union ud_av_u *avu; + __u8 idx, old_idx; + struct ud_av_st *av = avh; + + avu = (union ud_av_u *)av; + + idx = avu - dev_buffers_p->av_array; + tprintf("freeing udav idx=%d", idx); + old_idx = dev_ib_data.udav.udav_next_free; + dev_ib_data.udav.udav_next_free = idx; + avu->ud_av.next_free = old_idx; +} + +static int update_cq_cons_idx(struct cq_st *cq) +{ + struct cq_dbell_st dbell; + int rc; + + memset(&dbell, 0, sizeof dbell); + INS_FLD(cq->cqn, &dbell, tavorprm_cq_cmd_doorbell_st, cqn); + INS_FLD(CQ_DBELL_CMD_INC_CONS_IDX, &dbell, tavorprm_cq_cmd_doorbell_st, + cq_cmd); + rc = cmd_post_doorbell(&dbell, CQ_DBELL_OFFSET); + return rc; +} + +static int poll_cq(void *cqh, union cqe_st *cqe_p, u8 * num_cqes) +{ + union cqe_st cqe; + int rc; + u32 *ptr; + struct cq_st *cq = cqh; + + if (cq->cqn < 0x80 || cq->cqn > 0x83) { + eprintf(""); + return -1; + } + ptr = (u32 *) (&(cq->cq_buf[cq->cons_idx])); + barrier(); + if ((ptr[7] & 0x80000000) == 0) { + cqe = cq->cq_buf[cq->cons_idx]; + be_to_cpu_buf(&cqe, sizeof(cqe)); + *cqe_p = cqe; + ptr[7] = 0x80000000; + barrier(); + cq->cons_idx = (cq->cons_idx + 1) % cq->num_cqes; + rc = update_cq_cons_idx(cq); + if (rc) { + return rc; + } + *num_cqes = 1; + } else + *num_cqes = 0; + + return 0; +} + +static void dev2ib_cqe(struct ib_cqe_st *ib_cqe_p, union cqe_st *cqe_p) +{ + __u8 opcode; + __u32 wqe_addr_ba; + + opcode = + EX_FLD(cqe_p->good_cqe, tavorprm_completion_queue_entry_st, opcode); + if (opcode >= CQE_ERROR_OPCODE) + ib_cqe_p->is_error = 1; + else + ib_cqe_p->is_error = 0; + + ib_cqe_p->is_send = + EX_FLD(cqe_p->good_cqe, tavorprm_completion_queue_entry_st, s); + wqe_addr_ba = + EX_FLD(cqe_p->good_cqe, tavorprm_completion_queue_entry_st, + wqe_adr) << 6; + ib_cqe_p->wqe = bus_to_virt(wqe_addr_ba); + +// if (ib_cqe_p->is_send) { +// be_to_cpu_buf(ib_cqe_p->wqe, sizeof(struct ud_send_wqe_st)); +// } +// else { +// be_to_cpu_buf(ib_cqe_p->wqe, sizeof(struct recv_wqe_st)); +// } + ib_cqe_p->count = + EX_FLD(cqe_p->good_cqe, tavorprm_completion_queue_entry_st, + byte_cnt); +} + +static int ib_poll_cq(void *cqh, struct ib_cqe_st *ib_cqe_p, u8 * num_cqes) +{ + int rc; + union cqe_st cqe; + struct cq_st *cq = cqh; + __u8 opcode; + + rc = poll_cq(cq, &cqe, num_cqes); + if (rc || ((*num_cqes) == 0)) { + return rc; + } + + dev2ib_cqe(ib_cqe_p, &cqe); + + opcode = + EX_FLD(cqe.good_cqe, tavorprm_completion_queue_entry_st, opcode); + if (opcode >= CQE_ERROR_OPCODE) { + struct ud_send_wqe_st *wqe_p, wqe; + __u32 *ptr; + unsigned int i; + + wqe_p = + bus_to_virt(EX_FLD + (cqe.error_cqe, + tavorprm_completion_with_error_st, + wqe_addr) << 6); + eprintf("syndrome=0x%lx", + EX_FLD(cqe.error_cqe, tavorprm_completion_with_error_st, + syndrome)); + eprintf("wqe_addr=0x%lx", wqe_p); + eprintf("wqe_size=0x%lx", + EX_FLD(cqe.error_cqe, tavorprm_completion_with_error_st, + wqe_size)); + eprintf("myqpn=0x%lx", + EX_FLD(cqe.error_cqe, tavorprm_completion_with_error_st, + myqpn)); + eprintf("db_cnt=0x%lx", + EX_FLD(cqe.error_cqe, tavorprm_completion_with_error_st, + db_cnt)); + memcpy(&wqe, wqe_p, sizeof wqe); + be_to_cpu_buf(&wqe, sizeof wqe); + + eprintf("dumping wqe..."); + ptr = (__u32 *) (&wqe); + for (i = 0; i < sizeof wqe; i += 4) { + printf("%lx : ", ptr[i >> 2]); + } + + } + + return rc; +} + +/* always work on ipoib qp */ +static int add_qp_to_mcast_group(union ib_gid_u mcast_gid, __u8 add) +{ + void *mg; + __u8 *tmp; + int rc; + __u16 mgid_hash; + void *mgmqp_p; + + tmp = dev_buffers_p->inprm_buf; + memcpy(tmp, mcast_gid.raw, 16); + be_to_cpu_buf(tmp, 16); + rc = cmd_mgid_hash(tmp, &mgid_hash); + if (!rc) { + mg = (void *)dev_buffers_p->inprm_buf; + memset(mg, 0, MT_STRUCT_SIZE(tavorprm_mgm_entry_st)); + INS_FLD(mcast_gid.as_u32.dw[0], mg, tavorprm_mgm_entry_st, mgid_128_96); // memcpy(&mg->mgid_128_96, &mcast_gid.raw[0], 4); + INS_FLD(mcast_gid.as_u32.dw[1], mg, tavorprm_mgm_entry_st, mgid_95_64); // memcpy(&mg->mgid_95_64, &mcast_gid.raw[4], 4); + INS_FLD(mcast_gid.as_u32.dw[2], mg, tavorprm_mgm_entry_st, mgid_63_32); //memcpy(&mg->mgid_63_32, &mcast_gid.raw[8], 4); + INS_FLD(mcast_gid.as_u32.dw[3], mg, tavorprm_mgm_entry_st, mgid_31_0); //memcpy(&mg->mgid_31_0, &mcast_gid.raw[12], 4); + be_to_cpu_buf(mg + MT_BYTE_OFFSET(tavorprm_mgm_entry_st, mgid_128_96), 16); //be_to_cpu_buf(&mg->mgid_128_96, 16); + mgmqp_p = mg + MT_BYTE_OFFSET(tavorprm_mgm_entry_st, mgmqp_0); + INS_FLD(dev_ib_data.ipoib_qp.qpn, mgmqp_p, tavorprm_mgmqp_st, qpn_i); //mg->mgmqp[0].qpn = dev_ib_data.ipoib_qp.qpn; + INS_FLD(add, mgmqp_p, tavorprm_mgmqp_st, qi); //mg->mgmqp[0].valid = add ? 1 : 0; + rc = cmd_write_mgm(mg, mgid_hash); + } + return rc; +} + +static int clear_interrupt(void) +{ + __u32 ecr; + int ret = 0; + + if (gw_read_cr(0x80704, &ecr)) { + eprintf(""); + } else { + if (ecr) { + ret = 1; + } + } + gw_write_cr(0xf00d8, 0x80000000); /* clear int */ + gw_write_cr(0x8070c, 0xffffffff); + + return ret; +} + +static struct ud_send_wqe_st *alloc_send_wqe(udqp_t qph) +{ + struct udqp_st *qp = qph; + __u8 new_entry; + struct ud_send_wqe_st *wqe; + + if (qp->snd_wqe_cur_free == 0) { + return NULL; + } + new_entry = qp->snd_wqe_alloc_idx; + + wqe = &qp->snd_wq[new_entry].wqe; + qp->snd_wqe_cur_free--; + qp->snd_wqe_alloc_idx = (qp->snd_wqe_alloc_idx + 1) % qp->max_snd_wqes; + + memset(wqe, 0, sizeof *wqe); + + wqe->mpointer[0].local_addr_l = virt_to_bus(qp->snd_bufs[new_entry]); + + return wqe; +} + +/* + * alloc_rcv_wqe + * + * Note: since we work directly on the work queue, wqes + * are left in big endian + */ +static struct recv_wqe_st *alloc_rcv_wqe(struct udqp_st *qp) +{ + __u8 new_entry; + struct recv_wqe_st *wqe; + + if (qp->recv_wqe_cur_free == 0) { + return NULL; + } + + new_entry = qp->recv_wqe_alloc_idx; + wqe = &qp->rcv_wq[new_entry].wqe; + + qp->recv_wqe_cur_free--; + qp->recv_wqe_alloc_idx = + (qp->recv_wqe_alloc_idx + 1) % qp->max_recv_wqes; + + memset(wqe, 0, sizeof *wqe); + + /* GRH is always required */ + wqe->mpointer[0].local_addr_h = 0; + wqe->mpointer[0].local_addr_l = virt_to_bus(qp->rcv_bufs[new_entry]); + wqe->mpointer[0].lkey = dev_ib_data.mkey; + wqe->mpointer[0].byte_count = GRH_SIZE; + + wqe->mpointer[1].local_addr_h = 0; + wqe->mpointer[1].local_addr_l = + virt_to_bus(qp->rcv_bufs[new_entry] + GRH_SIZE); + wqe->mpointer[1].lkey = dev_ib_data.mkey; + wqe->mpointer[1].byte_count = qp->rcv_buf_sz; + + tprintf("rcv_buf=%lx\n", qp->rcv_bufs[new_entry]); + + /* we do it only on the data segment since the control + segment is always owned by HW */ + cpu_to_be_buf(wqe, sizeof *wqe); + +// tprintf("alloc wqe= 0x%x", wqe); + return wqe; +} + +static int free_send_wqe(struct ud_send_wqe_st *wqe) +{ + union ud_send_wqe_u *wqe_u; + struct udqp_st *qp; + + wqe_u = (union ud_send_wqe_u *)wqe; + qp = wqe_u->wqe_cont.qp; + + if (qp->snd_wqe_cur_free >= qp->max_snd_wqes) { + return -1; + } + + qp->snd_wqe_cur_free++; + + return 0; +} + +static int free_rcv_wqe(struct recv_wqe_st *wqe) +{ + union recv_wqe_u *wqe_u; + struct udqp_st *qp; + + wqe_u = (union recv_wqe_u *)wqe; + qp = wqe_u->wqe_cont.qp; + + if (qp->recv_wqe_cur_free >= qp->max_recv_wqes) { + return -1; + } + + qp->recv_wqe_cur_free++; + + return 0; +} + +static int free_wqe(void *wqe) +{ + int rc = 0; + struct recv_wqe_st *rcv_wqe; + +// tprintf("free wqe= 0x%x", wqe); + if ((wqe >= (void *)(dev_ib_data.ipoib_qp.rcv_wq)) && + (wqe < + (void *)(&dev_ib_data.ipoib_qp.rcv_wq[NUM_IPOIB_RCV_WQES]))) { + /* ipoib receive wqe */ + free_rcv_wqe(wqe); + rcv_wqe = alloc_rcv_wqe(&dev_ib_data.ipoib_qp); + if (rcv_wqe) { + rc = post_rcv_buf(&dev_ib_data.ipoib_qp, rcv_wqe); + if (rc) { + eprintf(""); + } + } + } else if (wqe >= (void *)(dev_ib_data.ipoib_qp.snd_wq) && + wqe < + (void *)(&dev_ib_data.ipoib_qp.snd_wq[NUM_IPOIB_SND_WQES])) { + /* ipoib send wqe */ + free_send_wqe(wqe); + } else if (wqe >= (void *)(dev_ib_data.mads_qp.rcv_wq) && + wqe < + (void *)(&dev_ib_data.mads_qp.rcv_wq[NUM_MADS_RCV_WQES])) { + /* mads receive wqe */ + free_rcv_wqe(wqe); + rcv_wqe = alloc_rcv_wqe(&dev_ib_data.mads_qp); + if (rcv_wqe) { + rc = post_rcv_buf(&dev_ib_data.mads_qp, rcv_wqe); + if (rc) { + eprintf(""); + } + } + } else if (wqe >= (void *)(dev_ib_data.mads_qp.snd_wq) && + wqe < + (void *)(&dev_ib_data.mads_qp.snd_wq[NUM_MADS_SND_WQES])) { + /* mads send wqe */ + free_send_wqe(wqe); + } else { + rc = -1; + eprintf(""); + } + + return rc; +} + +static int update_eq_cons_idx(struct eq_st *eq) +{ + struct eq_dbell_st dbell; + int rc; + + memset(&dbell, 0, sizeof dbell); + INS_FLD(dev_ib_data.eq.eqn, &dbell, tavorprm_eq_cmd_doorbell_st, eqn); + INS_FLD(EQ_DBELL_CMD_SET_CONS_IDX, &dbell, tavorprm_eq_cmd_doorbell_st, + eq_cmd); + INS_FLD(eq->cons_idx, &dbell, tavorprm_eq_cmd_doorbell_st, eq_param); + rc = cmd_post_doorbell(&dbell, EQ_DBELL_OFFSET); + + return rc; +} + +static void dev2ib_eqe(struct ib_eqe_st *ib_eqe_p, void *eqe_p) +{ + void *tmp; + + ib_eqe_p->event_type = + EX_FLD(eqe_p, tavorprm_event_queue_entry_st, event_type); + + tmp = eqe_p + MT_BYTE_OFFSET(tavorprm_event_queue_entry_st, event_data); + ib_eqe_p->cqn = EX_FLD(tmp, tavorprm_completion_event_st, cqn); +} + +static int poll_eq(struct ib_eqe_st *ib_eqe_p, __u8 * num_eqes) +{ + struct eqe_t eqe; + __u8 owner; + int rc; + __u32 *ptr; + struct eq_st *eq = &dev_ib_data.eq; + + ptr = (__u32 *) (&(eq->eq_buf[eq->cons_idx])); + tprintf("cons)idx=%d, addr(eqe)=%x, val=0x%x", eq->cons_idx, virt_to_bus(ptr), ptr[7]); + owner = (ptr[7] & 0x80000000) ? OWNER_HW : OWNER_SW; + if (owner == OWNER_SW) { + tprintf("got eqe"); + eqe = eq->eq_buf[eq->cons_idx]; + be_to_cpu_buf(&eqe, sizeof(eqe)); + dev2ib_eqe(ib_eqe_p, &eqe); + ptr[7] |= 0x80000000; + eq->eq_buf[eq->cons_idx] = eqe; + eq->cons_idx = (eq->cons_idx + 1) % eq->eq_size; + rc = update_eq_cons_idx(eq); + if (rc) { + return -1; + } + *num_eqes = 1; + } else { + *num_eqes = 0; + } + return 0; +} + +static int ib_device_close(void) +{ + iounmap(tavor_pci_dev.uar); + iounmap(tavor_pci_dev.cr_space); + iounmap(dev_ib_data.error_buf_addr); + return 0; +} + +static __u32 dev_get_qpn(void *qph) +{ + struct udqp_st *qp = qph; + + return qp->qpn; +} + +static void dev_post_dbell(void *dbell, __u32 offset) +{ + __u32 *ptr; + unsigned long address; + + ptr = dbell; + tprintf("ptr[0]= 0x%lx", ptr[0]); + tprintf("ptr[1]= 0x%lx", ptr[1]); + address = (unsigned long)(tavor_pci_dev.uar) + offset; + tprintf("va=0x%lx pa=0x%lx", address, + virt_to_bus((const void *)address)); + writel(htonl(ptr[0]), tavor_pci_dev.uar + offset); + barrier(); + address += 4; + tprintf("va=0x%lx pa=0x%lx", address, + virt_to_bus((const void *)address)); + writel(htonl(ptr[1]), tavor_pci_dev.uar + offset + 4); +} + + |