diff options
Diffstat (limited to 'gpxe/src/drivers/infiniband/hermon.c')
-rw-r--r-- | gpxe/src/drivers/infiniband/hermon.c | 2752 |
1 files changed, 0 insertions, 2752 deletions
diff --git a/gpxe/src/drivers/infiniband/hermon.c b/gpxe/src/drivers/infiniband/hermon.c deleted file mode 100644 index b9c97f94..00000000 --- a/gpxe/src/drivers/infiniband/hermon.c +++ /dev/null @@ -1,2752 +0,0 @@ -/* - * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>. - * Copyright (C) 2008 Mellanox Technologies Ltd. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -FILE_LICENCE ( GPL2_OR_LATER ); - -#include <stdint.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <strings.h> -#include <unistd.h> -#include <errno.h> -#include <byteswap.h> -#include <gpxe/io.h> -#include <gpxe/pci.h> -#include <gpxe/pcibackup.h> -#include <gpxe/malloc.h> -#include <gpxe/umalloc.h> -#include <gpxe/iobuf.h> -#include <gpxe/netdevice.h> -#include <gpxe/infiniband.h> -#include <gpxe/ib_smc.h> -#include "hermon.h" - -/** - * @file - * - * Mellanox Hermon Infiniband HCA - * - */ - -/*************************************************************************** - * - * Queue number allocation - * - *************************************************************************** - */ - -/** - * Allocate offsets within usage bitmask - * - * @v bits Usage bitmask - * @v bits_len Length of usage bitmask - * @v num_bits Number of contiguous bits to allocate within bitmask - * @ret bit First free bit within bitmask, or negative error - */ -static int hermon_bitmask_alloc ( hermon_bitmask_t *bits, - unsigned int bits_len, - unsigned int num_bits ) { - unsigned int bit = 0; - hermon_bitmask_t mask = 1; - unsigned int found = 0; - - /* Search bits for num_bits contiguous free bits */ - while ( bit < bits_len ) { - if ( ( mask & *bits ) == 0 ) { - if ( ++found == num_bits ) - goto found; - } else { - found = 0; - } - bit++; - mask = ( mask << 1 ) | ( mask >> ( 8 * sizeof ( mask ) - 1 ) ); - if ( mask == 1 ) - bits++; - } - return -ENFILE; - - found: - /* Mark bits as in-use */ - do { - *bits |= mask; - if ( mask == 1 ) - bits--; - mask = ( mask >> 1 ) | ( mask << ( 8 * sizeof ( mask ) - 1 ) ); - } while ( --found ); - - return ( bit - num_bits + 1 ); -} - -/** - * Free offsets within usage bitmask - * - * @v bits Usage bitmask - * @v bit Starting bit within bitmask - * @v num_bits Number of contiguous bits to free within bitmask - */ -static void hermon_bitmask_free ( hermon_bitmask_t *bits, - int bit, unsigned int num_bits ) { - hermon_bitmask_t mask; - - for ( ; num_bits ; bit++, num_bits-- ) { - mask = ( 1 << ( bit % ( 8 * sizeof ( mask ) ) ) ); - bits[ ( bit / ( 8 * sizeof ( mask ) ) ) ] &= ~mask; - } -} - -/*************************************************************************** - * - * HCA commands - * - *************************************************************************** - */ - -/** - * Wait for Hermon command completion - * - * @v hermon Hermon device - * @v hcr HCA command registers - * @ret rc Return status code - */ -static int hermon_cmd_wait ( struct hermon *hermon, - struct hermonprm_hca_command_register *hcr ) { - unsigned int wait; - - for ( wait = HERMON_HCR_MAX_WAIT_MS ; wait ; wait-- ) { - hcr->u.dwords[6] = - readl ( hermon->config + HERMON_HCR_REG ( 6 ) ); - if ( ( MLX_GET ( hcr, go ) == 0 ) && - ( MLX_GET ( hcr, t ) == hermon->toggle ) ) - return 0; - mdelay ( 1 ); - } - return -EBUSY; -} - -/** - * Issue HCA command - * - * @v hermon Hermon device - * @v command Command opcode, flags and input/output lengths - * @v op_mod Opcode modifier (0 if no modifier applicable) - * @v in Input parameters - * @v in_mod Input modifier (0 if no modifier applicable) - * @v out Output parameters - * @ret rc Return status code - */ -static int hermon_cmd ( struct hermon *hermon, unsigned long command, - unsigned int op_mod, const void *in, - unsigned int in_mod, void *out ) { - struct hermonprm_hca_command_register hcr; - unsigned int opcode = HERMON_HCR_OPCODE ( command ); - size_t in_len = HERMON_HCR_IN_LEN ( command ); - size_t out_len = HERMON_HCR_OUT_LEN ( command ); - void *in_buffer; - void *out_buffer; - unsigned int status; - unsigned int i; - int rc; - - assert ( in_len <= HERMON_MBOX_SIZE ); - assert ( out_len <= HERMON_MBOX_SIZE ); - - DBGC2 ( hermon, "Hermon %p command %02x in %zx%s out %zx%s\n", - hermon, opcode, in_len, - ( ( command & HERMON_HCR_IN_MBOX ) ? "(mbox)" : "" ), out_len, - ( ( command & HERMON_HCR_OUT_MBOX ) ? "(mbox)" : "" ) ); - - /* Check that HCR is free */ - if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) { - DBGC ( hermon, "Hermon %p command interface locked\n", - hermon ); - return rc; - } - - /* Flip HCR toggle */ - hermon->toggle = ( 1 - hermon->toggle ); - - /* Prepare HCR */ - memset ( &hcr, 0, sizeof ( hcr ) ); - in_buffer = &hcr.u.dwords[0]; - if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) { - in_buffer = hermon->mailbox_in; - MLX_FILL_1 ( &hcr, 1, in_param_l, virt_to_bus ( in_buffer ) ); - } - memcpy ( in_buffer, in, in_len ); - MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod ); - out_buffer = &hcr.u.dwords[3]; - if ( out_len && ( command & HERMON_HCR_OUT_MBOX ) ) { - out_buffer = hermon->mailbox_out; - MLX_FILL_1 ( &hcr, 4, out_param_l, - virt_to_bus ( out_buffer ) ); - } - MLX_FILL_4 ( &hcr, 6, - opcode, opcode, - opcode_modifier, op_mod, - go, 1, - t, hermon->toggle ); - DBGC ( hermon, "Hermon %p issuing command %04x\n", - hermon, opcode ); - DBGC2_HDA ( hermon, virt_to_phys ( hermon->config + HERMON_HCR_BASE ), - &hcr, sizeof ( hcr ) ); - if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) { - DBGC2 ( hermon, "Input mailbox:\n" ); - DBGC2_HDA ( hermon, virt_to_phys ( in_buffer ), in_buffer, - ( ( in_len < 512 ) ? in_len : 512 ) ); - } - - /* Issue command */ - for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ; - i++ ) { - writel ( hcr.u.dwords[i], - hermon->config + HERMON_HCR_REG ( i ) ); - barrier(); - } - - /* Wait for command completion */ - if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) { - DBGC ( hermon, "Hermon %p timed out waiting for command:\n", - hermon ); - DBGC_HDA ( hermon, - virt_to_phys ( hermon->config + HERMON_HCR_BASE ), - &hcr, sizeof ( hcr ) ); - return rc; - } - - /* Check command status */ - status = MLX_GET ( &hcr, status ); - if ( status != 0 ) { - DBGC ( hermon, "Hermon %p command failed with status %02x:\n", - hermon, status ); - DBGC_HDA ( hermon, - virt_to_phys ( hermon->config + HERMON_HCR_BASE ), - &hcr, sizeof ( hcr ) ); - return -EIO; - } - - /* Read output parameters, if any */ - hcr.u.dwords[3] = readl ( hermon->config + HERMON_HCR_REG ( 3 ) ); - hcr.u.dwords[4] = readl ( hermon->config + HERMON_HCR_REG ( 4 ) ); - memcpy ( out, out_buffer, out_len ); - if ( out_len ) { - DBGC2 ( hermon, "Output%s:\n", - ( command & HERMON_HCR_OUT_MBOX ) ? " mailbox" : "" ); - DBGC2_HDA ( hermon, virt_to_phys ( out_buffer ), out_buffer, - ( ( out_len < 512 ) ? out_len : 512 ) ); - } - - return 0; -} - -static inline int -hermon_cmd_query_dev_cap ( struct hermon *hermon, - struct hermonprm_query_dev_cap *dev_cap ) { - return hermon_cmd ( hermon, - HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_DEV_CAP, - 1, sizeof ( *dev_cap ) ), - 0, NULL, 0, dev_cap ); -} - -static inline int -hermon_cmd_query_fw ( struct hermon *hermon, struct hermonprm_query_fw *fw ) { - return hermon_cmd ( hermon, - HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_FW, - 1, sizeof ( *fw ) ), - 0, NULL, 0, fw ); -} - -static inline int -hermon_cmd_init_hca ( struct hermon *hermon, - const struct hermonprm_init_hca *init_hca ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_INIT_HCA, - 1, sizeof ( *init_hca ) ), - 0, init_hca, 0, NULL ); -} - -static inline int -hermon_cmd_close_hca ( struct hermon *hermon ) { - return hermon_cmd ( hermon, - HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_HCA ), - 0, NULL, 0, NULL ); -} - -static inline int -hermon_cmd_init_port ( struct hermon *hermon, unsigned int port, - const struct hermonprm_init_port *init_port ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_INIT_PORT, - 1, sizeof ( *init_port ) ), - 0, init_port, port, NULL ); -} - -static inline int -hermon_cmd_close_port ( struct hermon *hermon, unsigned int port ) { - return hermon_cmd ( hermon, - HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_PORT ), - 0, NULL, port, NULL ); -} - -static inline int -hermon_cmd_sw2hw_mpt ( struct hermon *hermon, unsigned int index, - const struct hermonprm_mpt *mpt ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_MPT, - 1, sizeof ( *mpt ) ), - 0, mpt, index, NULL ); -} - -static inline int -hermon_cmd_write_mtt ( struct hermon *hermon, - const struct hermonprm_write_mtt *write_mtt ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MTT, - 1, sizeof ( *write_mtt ) ), - 0, write_mtt, 1, NULL ); -} - -static inline int -hermon_cmd_map_eq ( struct hermon *hermon, unsigned long index_map, - const struct hermonprm_event_mask *mask ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_EQ, - 0, sizeof ( *mask ) ), - 0, mask, index_map, NULL ); -} - -static inline int -hermon_cmd_sw2hw_eq ( struct hermon *hermon, unsigned int index, - const struct hermonprm_eqc *eqctx ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_EQ, - 1, sizeof ( *eqctx ) ), - 0, eqctx, index, NULL ); -} - -static inline int -hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index, - struct hermonprm_eqc *eqctx ) { - return hermon_cmd ( hermon, - HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_EQ, - 1, sizeof ( *eqctx ) ), - 1, NULL, index, eqctx ); -} - -static inline int -hermon_cmd_query_eq ( struct hermon *hermon, unsigned int index, - struct hermonprm_eqc *eqctx ) { - return hermon_cmd ( hermon, - HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_EQ, - 1, sizeof ( *eqctx ) ), - 0, NULL, index, eqctx ); -} - -static inline int -hermon_cmd_sw2hw_cq ( struct hermon *hermon, unsigned long cqn, - const struct hermonprm_completion_queue_context *cqctx ){ - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_CQ, - 1, sizeof ( *cqctx ) ), - 0, cqctx, cqn, NULL ); -} - -static inline int -hermon_cmd_hw2sw_cq ( struct hermon *hermon, unsigned long cqn, - struct hermonprm_completion_queue_context *cqctx) { - return hermon_cmd ( hermon, - HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_CQ, - 1, sizeof ( *cqctx ) ), - 0, NULL, cqn, cqctx ); -} - -static inline int -hermon_cmd_rst2init_qp ( struct hermon *hermon, unsigned long qpn, - const struct hermonprm_qp_ee_state_transitions *ctx ){ - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_RST2INIT_QP, - 1, sizeof ( *ctx ) ), - 0, ctx, qpn, NULL ); -} - -static inline int -hermon_cmd_init2rtr_qp ( struct hermon *hermon, unsigned long qpn, - const struct hermonprm_qp_ee_state_transitions *ctx ){ - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_INIT2RTR_QP, - 1, sizeof ( *ctx ) ), - 0, ctx, qpn, NULL ); -} - -static inline int -hermon_cmd_rtr2rts_qp ( struct hermon *hermon, unsigned long qpn, - const struct hermonprm_qp_ee_state_transitions *ctx ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_RTR2RTS_QP, - 1, sizeof ( *ctx ) ), - 0, ctx, qpn, NULL ); -} - -static inline int -hermon_cmd_rts2rts_qp ( struct hermon *hermon, unsigned long qpn, - const struct hermonprm_qp_ee_state_transitions *ctx ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_RTS2RTS_QP, - 1, sizeof ( *ctx ) ), - 0, ctx, qpn, NULL ); -} - -static inline int -hermon_cmd_2rst_qp ( struct hermon *hermon, unsigned long qpn ) { - return hermon_cmd ( hermon, - HERMON_HCR_VOID_CMD ( HERMON_HCR_2RST_QP ), - 0x03, NULL, qpn, NULL ); -} - -static inline int -hermon_cmd_query_qp ( struct hermon *hermon, unsigned long qpn, - struct hermonprm_qp_ee_state_transitions *ctx ) { - return hermon_cmd ( hermon, - HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_QP, - 1, sizeof ( *ctx ) ), - 0, NULL, qpn, ctx ); -} - -static inline int -hermon_cmd_conf_special_qp ( struct hermon *hermon, unsigned int internal_qps, - unsigned long base_qpn ) { - return hermon_cmd ( hermon, - HERMON_HCR_VOID_CMD ( HERMON_HCR_CONF_SPECIAL_QP ), - internal_qps, NULL, base_qpn, NULL ); -} - -static inline int -hermon_cmd_mad_ifc ( struct hermon *hermon, unsigned int port, - union hermonprm_mad *mad ) { - return hermon_cmd ( hermon, - HERMON_HCR_INOUT_CMD ( HERMON_HCR_MAD_IFC, - 1, sizeof ( *mad ), - 1, sizeof ( *mad ) ), - 0x03, mad, port, mad ); -} - -static inline int -hermon_cmd_read_mcg ( struct hermon *hermon, unsigned int index, - struct hermonprm_mcg_entry *mcg ) { - return hermon_cmd ( hermon, - HERMON_HCR_OUT_CMD ( HERMON_HCR_READ_MCG, - 1, sizeof ( *mcg ) ), - 0, NULL, index, mcg ); -} - -static inline int -hermon_cmd_write_mcg ( struct hermon *hermon, unsigned int index, - const struct hermonprm_mcg_entry *mcg ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MCG, - 1, sizeof ( *mcg ) ), - 0, mcg, index, NULL ); -} - -static inline int -hermon_cmd_mgid_hash ( struct hermon *hermon, const struct ib_gid *gid, - struct hermonprm_mgm_hash *hash ) { - return hermon_cmd ( hermon, - HERMON_HCR_INOUT_CMD ( HERMON_HCR_MGID_HASH, - 1, sizeof ( *gid ), - 0, sizeof ( *hash ) ), - 0, gid, 0, hash ); -} - -static inline int -hermon_cmd_run_fw ( struct hermon *hermon ) { - return hermon_cmd ( hermon, - HERMON_HCR_VOID_CMD ( HERMON_HCR_RUN_FW ), - 0, NULL, 0, NULL ); -} - -static inline int -hermon_cmd_unmap_icm ( struct hermon *hermon, unsigned int page_count, - const struct hermonprm_scalar_parameter *offset ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_UNMAP_ICM, - 0, sizeof ( *offset ) ), - 0, offset, page_count, NULL ); -} - -static inline int -hermon_cmd_map_icm ( struct hermon *hermon, - const struct hermonprm_virtual_physical_mapping *map ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM, - 1, sizeof ( *map ) ), - 0, map, 1, NULL ); -} - -static inline int -hermon_cmd_unmap_icm_aux ( struct hermon *hermon ) { - return hermon_cmd ( hermon, - HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_ICM_AUX ), - 0, NULL, 0, NULL ); -} - -static inline int -hermon_cmd_map_icm_aux ( struct hermon *hermon, - const struct hermonprm_virtual_physical_mapping *map ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM_AUX, - 1, sizeof ( *map ) ), - 0, map, 1, NULL ); -} - -static inline int -hermon_cmd_set_icm_size ( struct hermon *hermon, - const struct hermonprm_scalar_parameter *icm_size, - struct hermonprm_scalar_parameter *icm_aux_size ) { - return hermon_cmd ( hermon, - HERMON_HCR_INOUT_CMD ( HERMON_HCR_SET_ICM_SIZE, - 0, sizeof ( *icm_size ), - 0, sizeof (*icm_aux_size) ), - 0, icm_size, 0, icm_aux_size ); -} - -static inline int -hermon_cmd_unmap_fa ( struct hermon *hermon ) { - return hermon_cmd ( hermon, - HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_FA ), - 0, NULL, 0, NULL ); -} - -static inline int -hermon_cmd_map_fa ( struct hermon *hermon, - const struct hermonprm_virtual_physical_mapping *map ) { - return hermon_cmd ( hermon, - HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_FA, - 1, sizeof ( *map ) ), - 0, map, 1, NULL ); -} - -static inline int -hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port, - struct hermonprm_sense_port *port_type ) { - return hermon_cmd ( hermon, - HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT, - 1, sizeof ( *port_type ) ), - 0, NULL, port, port_type ); -} - - -/*************************************************************************** - * - * Memory translation table operations - * - *************************************************************************** - */ - -/** - * Allocate MTT entries - * - * @v hermon Hermon device - * @v memory Memory to map into MTT - * @v len Length of memory to map - * @v mtt MTT descriptor to fill in - * @ret rc Return status code - */ -static int hermon_alloc_mtt ( struct hermon *hermon, - const void *memory, size_t len, - struct hermon_mtt *mtt ) { - struct hermonprm_write_mtt write_mtt; - physaddr_t start; - unsigned int page_offset; - unsigned int num_pages; - int mtt_offset; - unsigned int mtt_base_addr; - unsigned int i; - int rc; - - /* Find available MTT entries */ - start = virt_to_phys ( memory ); - page_offset = ( start & ( HERMON_PAGE_SIZE - 1 ) ); - start -= page_offset; - len += page_offset; - num_pages = ( ( len + HERMON_PAGE_SIZE - 1 ) / HERMON_PAGE_SIZE ); - mtt_offset = hermon_bitmask_alloc ( hermon->mtt_inuse, HERMON_MAX_MTTS, - num_pages ); - if ( mtt_offset < 0 ) { - DBGC ( hermon, "Hermon %p could not allocate %d MTT entries\n", - hermon, num_pages ); - rc = mtt_offset; - goto err_mtt_offset; - } - mtt_base_addr = ( ( hermon->cap.reserved_mtts + mtt_offset ) * - hermon->cap.mtt_entry_size ); - - /* Fill in MTT structure */ - mtt->mtt_offset = mtt_offset; - mtt->num_pages = num_pages; - mtt->mtt_base_addr = mtt_base_addr; - mtt->page_offset = page_offset; - - /* Construct and issue WRITE_MTT commands */ - for ( i = 0 ; i < num_pages ; i++ ) { - memset ( &write_mtt, 0, sizeof ( write_mtt ) ); - MLX_FILL_1 ( &write_mtt.mtt_base_addr, 1, - value, mtt_base_addr ); - MLX_FILL_2 ( &write_mtt.mtt, 1, - p, 1, - ptag_l, ( start >> 3 ) ); - if ( ( rc = hermon_cmd_write_mtt ( hermon, - &write_mtt ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not write MTT at %x\n", - hermon, mtt_base_addr ); - goto err_write_mtt; - } - start += HERMON_PAGE_SIZE; - mtt_base_addr += hermon->cap.mtt_entry_size; - } - - return 0; - - err_write_mtt: - hermon_bitmask_free ( hermon->mtt_inuse, mtt_offset, num_pages ); - err_mtt_offset: - return rc; -} - -/** - * Free MTT entries - * - * @v hermon Hermon device - * @v mtt MTT descriptor - */ -static void hermon_free_mtt ( struct hermon *hermon, - struct hermon_mtt *mtt ) { - hermon_bitmask_free ( hermon->mtt_inuse, mtt->mtt_offset, - mtt->num_pages ); -} - -/*************************************************************************** - * - * MAD operations - * - *************************************************************************** - */ - -/** - * Issue management datagram - * - * @v ibdev Infiniband device - * @v mad Management datagram - * @ret rc Return status code - */ -static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - union hermonprm_mad mad_ifc; - int rc; - - linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ), - mad_size_mismatch ); - - /* Copy in request packet */ - memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) ); - - /* Issue MAD */ - if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port, - &mad_ifc ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - - /* Copy out reply packet */ - memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) ); - - if ( mad->hdr.status != 0 ) { - DBGC ( hermon, "Hermon %p MAD IFC status %04x\n", - hermon, ntohs ( mad->hdr.status ) ); - return -EIO; - } - return 0; -} - -/*************************************************************************** - * - * Completion queue operations - * - *************************************************************************** - */ - -/** - * Create completion queue - * - * @v ibdev Infiniband device - * @v cq Completion queue - * @ret rc Return status code - */ -static int hermon_create_cq ( struct ib_device *ibdev, - struct ib_completion_queue *cq ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_completion_queue *hermon_cq; - struct hermonprm_completion_queue_context cqctx; - int cqn_offset; - unsigned int i; - int rc; - - /* Find a free completion queue number */ - cqn_offset = hermon_bitmask_alloc ( hermon->cq_inuse, - HERMON_MAX_CQS, 1 ); - if ( cqn_offset < 0 ) { - DBGC ( hermon, "Hermon %p out of completion queues\n", - hermon ); - rc = cqn_offset; - goto err_cqn_offset; - } - cq->cqn = ( hermon->cap.reserved_cqs + cqn_offset ); - - /* Allocate control structures */ - hermon_cq = zalloc ( sizeof ( *hermon_cq ) ); - if ( ! hermon_cq ) { - rc = -ENOMEM; - goto err_hermon_cq; - } - - /* Allocate completion queue itself */ - hermon_cq->cqe_size = ( cq->num_cqes * sizeof ( hermon_cq->cqe[0] ) ); - hermon_cq->cqe = malloc_dma ( hermon_cq->cqe_size, - sizeof ( hermon_cq->cqe[0] ) ); - if ( ! hermon_cq->cqe ) { - rc = -ENOMEM; - goto err_cqe; - } - memset ( hermon_cq->cqe, 0, hermon_cq->cqe_size ); - for ( i = 0 ; i < cq->num_cqes ; i++ ) { - MLX_FILL_1 ( &hermon_cq->cqe[i].normal, 7, owner, 1 ); - } - barrier(); - - /* Allocate MTT entries */ - if ( ( rc = hermon_alloc_mtt ( hermon, hermon_cq->cqe, - hermon_cq->cqe_size, - &hermon_cq->mtt ) ) != 0 ) - goto err_alloc_mtt; - - /* Hand queue over to hardware */ - memset ( &cqctx, 0, sizeof ( cqctx ) ); - MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ ); - MLX_FILL_1 ( &cqctx, 2, - page_offset, ( hermon_cq->mtt.page_offset >> 5 ) ); - MLX_FILL_2 ( &cqctx, 3, - usr_page, HERMON_UAR_NON_EQ_PAGE, - log_cq_size, fls ( cq->num_cqes - 1 ) ); - MLX_FILL_1 ( &cqctx, 7, mtt_base_addr_l, - ( hermon_cq->mtt.mtt_base_addr >> 3 ) ); - MLX_FILL_1 ( &cqctx, 15, db_record_addr_l, - ( virt_to_phys ( &hermon_cq->doorbell ) >> 3 ) ); - if ( ( rc = hermon_cmd_sw2hw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) { - DBGC ( hermon, "Hermon %p SW2HW_CQ failed: %s\n", - hermon, strerror ( rc ) ); - goto err_sw2hw_cq; - } - - DBGC ( hermon, "Hermon %p CQN %#lx ring at [%p,%p)\n", - hermon, cq->cqn, hermon_cq->cqe, - ( ( ( void * ) hermon_cq->cqe ) + hermon_cq->cqe_size ) ); - ib_cq_set_drvdata ( cq, hermon_cq ); - return 0; - - err_sw2hw_cq: - hermon_free_mtt ( hermon, &hermon_cq->mtt ); - err_alloc_mtt: - free_dma ( hermon_cq->cqe, hermon_cq->cqe_size ); - err_cqe: - free ( hermon_cq ); - err_hermon_cq: - hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 ); - err_cqn_offset: - return rc; -} - -/** - * Destroy completion queue - * - * @v ibdev Infiniband device - * @v cq Completion queue - */ -static void hermon_destroy_cq ( struct ib_device *ibdev, - struct ib_completion_queue *cq ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq ); - struct hermonprm_completion_queue_context cqctx; - int cqn_offset; - int rc; - - /* Take ownership back from hardware */ - if ( ( rc = hermon_cmd_hw2sw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) { - DBGC ( hermon, "Hermon %p FATAL HW2SW_CQ failed on CQN %#lx: " - "%s\n", hermon, cq->cqn, strerror ( rc ) ); - /* Leak memory and return; at least we avoid corruption */ - return; - } - - /* Free MTT entries */ - hermon_free_mtt ( hermon, &hermon_cq->mtt ); - - /* Free memory */ - free_dma ( hermon_cq->cqe, hermon_cq->cqe_size ); - free ( hermon_cq ); - - /* Mark queue number as free */ - cqn_offset = ( cq->cqn - hermon->cap.reserved_cqs ); - hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 ); - - ib_cq_set_drvdata ( cq, NULL ); -} - -/*************************************************************************** - * - * Queue pair operations - * - *************************************************************************** - */ - -/** - * Assign queue pair number - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @ret rc Return status code - */ -static int hermon_alloc_qpn ( struct ib_device *ibdev, - struct ib_queue_pair *qp ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - unsigned int port_offset; - int qpn_offset; - - /* Calculate queue pair number */ - port_offset = ( ibdev->port - HERMON_PORT_BASE ); - - switch ( qp->type ) { - case IB_QPT_SMI: - qp->qpn = ( hermon->special_qpn_base + port_offset ); - return 0; - case IB_QPT_GSI: - qp->qpn = ( hermon->special_qpn_base + 2 + port_offset ); - return 0; - case IB_QPT_UD: - case IB_QPT_RC: - /* Find a free queue pair number */ - qpn_offset = hermon_bitmask_alloc ( hermon->qp_inuse, - HERMON_MAX_QPS, 1 ); - if ( qpn_offset < 0 ) { - DBGC ( hermon, "Hermon %p out of queue pairs\n", - hermon ); - return qpn_offset; - } - qp->qpn = ( ( random() & HERMON_QPN_RANDOM_MASK ) | - ( hermon->qpn_base + qpn_offset ) ); - return 0; - default: - DBGC ( hermon, "Hermon %p unsupported QP type %d\n", - hermon, qp->type ); - return -ENOTSUP; - } -} - -/** - * Free queue pair number - * - * @v ibdev Infiniband device - * @v qp Queue pair - */ -static void hermon_free_qpn ( struct ib_device *ibdev, - struct ib_queue_pair *qp ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - int qpn_offset; - - qpn_offset = ( ( qp->qpn & ~HERMON_QPN_RANDOM_MASK ) - - hermon->qpn_base ); - if ( qpn_offset >= 0 ) - hermon_bitmask_free ( hermon->qp_inuse, qpn_offset, 1 ); -} - -/** - * Calculate transmission rate - * - * @v av Address vector - * @ret hermon_rate Hermon rate - */ -static unsigned int hermon_rate ( struct ib_address_vector *av ) { - return ( ( ( av->rate >= IB_RATE_2_5 ) && ( av->rate <= IB_RATE_120 ) ) - ? ( av->rate + 5 ) : 0 ); -} - -/** - * Calculate schedule queue - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @ret sched_queue Schedule queue - */ -static unsigned int hermon_sched_queue ( struct ib_device *ibdev, - struct ib_queue_pair *qp ) { - return ( ( ( qp->type == IB_QPT_SMI ) ? - HERMON_SCHED_QP0 : HERMON_SCHED_DEFAULT ) | - ( ( ibdev->port - 1 ) << 6 ) ); -} - -/** Queue pair transport service type map */ -static uint8_t hermon_qp_st[] = { - [IB_QPT_SMI] = HERMON_ST_MLX, - [IB_QPT_GSI] = HERMON_ST_MLX, - [IB_QPT_UD] = HERMON_ST_UD, - [IB_QPT_RC] = HERMON_ST_RC, -}; - -/** - * Dump queue pair context (for debugging only) - * - * @v hermon Hermon device - * @v qp Queue pair - * @ret rc Return status code - */ -static inline int hermon_dump_qpctx ( struct hermon *hermon, - struct ib_queue_pair *qp ) { - struct hermonprm_qp_ee_state_transitions qpctx; - int rc; - - memset ( &qpctx, 0, sizeof ( qpctx ) ); - if ( ( rc = hermon_cmd_query_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ) { - DBGC ( hermon, "Hermon %p QUERY_QP failed: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - DBGC ( hermon, "Hermon %p QPN %lx context:\n", hermon, qp->qpn ); - DBGC_HDA ( hermon, 0, &qpctx.u.dwords[2], - ( sizeof ( qpctx ) - 8 ) ); - - return 0; -} - -/** - * Create queue pair - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @ret rc Return status code - */ -static int hermon_create_qp ( struct ib_device *ibdev, - struct ib_queue_pair *qp ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_queue_pair *hermon_qp; - struct hermonprm_qp_ee_state_transitions qpctx; - int rc; - - /* Calculate queue pair number */ - if ( ( rc = hermon_alloc_qpn ( ibdev, qp ) ) != 0 ) - goto err_alloc_qpn; - - /* Allocate control structures */ - hermon_qp = zalloc ( sizeof ( *hermon_qp ) ); - if ( ! hermon_qp ) { - rc = -ENOMEM; - goto err_hermon_qp; - } - - /* Calculate doorbell address */ - hermon_qp->send.doorbell = - ( hermon->uar + HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE + - HERMON_DB_POST_SND_OFFSET ); - - /* Allocate work queue buffer */ - hermon_qp->send.num_wqes = ( qp->send.num_wqes /* headroom */ + 1 + - ( 2048 / sizeof ( hermon_qp->send.wqe[0] ) ) ); - hermon_qp->send.num_wqes = - ( 1 << fls ( hermon_qp->send.num_wqes - 1 ) ); /* round up */ - hermon_qp->send.wqe_size = ( hermon_qp->send.num_wqes * - sizeof ( hermon_qp->send.wqe[0] ) ); - hermon_qp->recv.wqe_size = ( qp->recv.num_wqes * - sizeof ( hermon_qp->recv.wqe[0] ) ); - hermon_qp->wqe_size = ( hermon_qp->send.wqe_size + - hermon_qp->recv.wqe_size ); - hermon_qp->wqe = malloc_dma ( hermon_qp->wqe_size, - sizeof ( hermon_qp->send.wqe[0] ) ); - if ( ! hermon_qp->wqe ) { - rc = -ENOMEM; - goto err_alloc_wqe; - } - hermon_qp->send.wqe = hermon_qp->wqe; - memset ( hermon_qp->send.wqe, 0xff, hermon_qp->send.wqe_size ); - hermon_qp->recv.wqe = ( hermon_qp->wqe + hermon_qp->send.wqe_size ); - memset ( hermon_qp->recv.wqe, 0, hermon_qp->recv.wqe_size ); - - /* Allocate MTT entries */ - if ( ( rc = hermon_alloc_mtt ( hermon, hermon_qp->wqe, - hermon_qp->wqe_size, - &hermon_qp->mtt ) ) != 0 ) { - goto err_alloc_mtt; - } - - /* Transition queue to INIT state */ - memset ( &qpctx, 0, sizeof ( qpctx ) ); - MLX_FILL_2 ( &qpctx, 2, - qpc_eec_data.pm_state, HERMON_PM_STATE_MIGRATED, - qpc_eec_data.st, hermon_qp_st[qp->type] ); - MLX_FILL_1 ( &qpctx, 3, qpc_eec_data.pd, HERMON_GLOBAL_PD ); - MLX_FILL_4 ( &qpctx, 4, - qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ), - qpc_eec_data.log_rq_stride, - ( fls ( sizeof ( hermon_qp->recv.wqe[0] ) - 1 ) - 4 ), - qpc_eec_data.log_sq_size, - fls ( hermon_qp->send.num_wqes - 1 ), - qpc_eec_data.log_sq_stride, - ( fls ( sizeof ( hermon_qp->send.wqe[0] ) - 1 ) - 4 ) ); - MLX_FILL_1 ( &qpctx, 5, - qpc_eec_data.usr_page, HERMON_UAR_NON_EQ_PAGE ); - MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn ); - MLX_FILL_4 ( &qpctx, 38, - qpc_eec_data.rre, 1, - qpc_eec_data.rwe, 1, - qpc_eec_data.rae, 1, - qpc_eec_data.page_offset, - ( hermon_qp->mtt.page_offset >> 6 ) ); - MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn ); - MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.db_record_addr_l, - ( virt_to_phys ( &hermon_qp->recv.doorbell ) >> 2 ) ); - MLX_FILL_1 ( &qpctx, 53, qpc_eec_data.mtt_base_addr_l, - ( hermon_qp->mtt.mtt_base_addr >> 3 ) ); - if ( ( rc = hermon_cmd_rst2init_qp ( hermon, qp->qpn, - &qpctx ) ) != 0 ) { - DBGC ( hermon, "Hermon %p RST2INIT_QP failed: %s\n", - hermon, strerror ( rc ) ); - goto err_rst2init_qp; - } - hermon_qp->state = HERMON_QP_ST_INIT; - - DBGC ( hermon, "Hermon %p QPN %#lx send ring at [%p,%p)\n", - hermon, qp->qpn, hermon_qp->send.wqe, - ( ((void *)hermon_qp->send.wqe ) + hermon_qp->send.wqe_size ) ); - DBGC ( hermon, "Hermon %p QPN %#lx receive ring at [%p,%p)\n", - hermon, qp->qpn, hermon_qp->recv.wqe, - ( ((void *)hermon_qp->recv.wqe ) + hermon_qp->recv.wqe_size ) ); - ib_qp_set_drvdata ( qp, hermon_qp ); - return 0; - - hermon_cmd_2rst_qp ( hermon, qp->qpn ); - err_rst2init_qp: - hermon_free_mtt ( hermon, &hermon_qp->mtt ); - err_alloc_mtt: - free_dma ( hermon_qp->wqe, hermon_qp->wqe_size ); - err_alloc_wqe: - free ( hermon_qp ); - err_hermon_qp: - hermon_free_qpn ( ibdev, qp ); - err_alloc_qpn: - return rc; -} - -/** - * Modify queue pair - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @ret rc Return status code - */ -static int hermon_modify_qp ( struct ib_device *ibdev, - struct ib_queue_pair *qp ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp ); - struct hermonprm_qp_ee_state_transitions qpctx; - int rc; - - /* Transition queue to RTR state, if applicable */ - if ( hermon_qp->state < HERMON_QP_ST_RTR ) { - memset ( &qpctx, 0, sizeof ( qpctx ) ); - MLX_FILL_2 ( &qpctx, 4, - qpc_eec_data.mtu, HERMON_MTU_2048, - qpc_eec_data.msg_max, 31 ); - MLX_FILL_1 ( &qpctx, 7, - qpc_eec_data.remote_qpn_een, qp->av.qpn ); - MLX_FILL_1 ( &qpctx, 9, - qpc_eec_data.primary_address_path.rlid, - qp->av.lid ); - MLX_FILL_1 ( &qpctx, 10, - qpc_eec_data.primary_address_path.max_stat_rate, - hermon_rate ( &qp->av ) ); - memcpy ( &qpctx.u.dwords[12], &qp->av.gid, - sizeof ( qp->av.gid ) ); - MLX_FILL_1 ( &qpctx, 16, - qpc_eec_data.primary_address_path.sched_queue, - hermon_sched_queue ( ibdev, qp ) ); - MLX_FILL_1 ( &qpctx, 39, - qpc_eec_data.next_rcv_psn, qp->recv.psn ); - if ( ( rc = hermon_cmd_init2rtr_qp ( hermon, qp->qpn, - &qpctx ) ) != 0 ) { - DBGC ( hermon, "Hermon %p INIT2RTR_QP failed: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - hermon_qp->state = HERMON_QP_ST_RTR; - } - - /* Transition queue to RTS state */ - if ( hermon_qp->state < HERMON_QP_ST_RTS ) { - memset ( &qpctx, 0, sizeof ( qpctx ) ); - MLX_FILL_1 ( &qpctx, 10, - qpc_eec_data.primary_address_path.ack_timeout, - 14 /* 4.096us * 2^(14) = 67ms */ ); - MLX_FILL_2 ( &qpctx, 30, - qpc_eec_data.retry_count, HERMON_RETRY_MAX, - qpc_eec_data.rnr_retry, HERMON_RETRY_MAX ); - MLX_FILL_1 ( &qpctx, 32, - qpc_eec_data.next_send_psn, qp->send.psn ); - if ( ( rc = hermon_cmd_rtr2rts_qp ( hermon, qp->qpn, - &qpctx ) ) != 0 ) { - DBGC ( hermon, "Hermon %p RTR2RTS_QP failed: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - hermon_qp->state = HERMON_QP_ST_RTS; - } - - /* Update parameters in RTS state */ - memset ( &qpctx, 0, sizeof ( qpctx ) ); - MLX_FILL_1 ( &qpctx, 0, opt_param_mask, HERMON_QP_OPT_PARAM_QKEY ); - MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey ); - if ( ( rc = hermon_cmd_rts2rts_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ){ - DBGC ( hermon, "Hermon %p RTS2RTS_QP failed: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - - return 0; -} - -/** - * Destroy queue pair - * - * @v ibdev Infiniband device - * @v qp Queue pair - */ -static void hermon_destroy_qp ( struct ib_device *ibdev, - struct ib_queue_pair *qp ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp ); - int rc; - - /* Take ownership back from hardware */ - if ( ( rc = hermon_cmd_2rst_qp ( hermon, qp->qpn ) ) != 0 ) { - DBGC ( hermon, "Hermon %p FATAL 2RST_QP failed on QPN %#lx: " - "%s\n", hermon, qp->qpn, strerror ( rc ) ); - /* Leak memory and return; at least we avoid corruption */ - return; - } - - /* Free MTT entries */ - hermon_free_mtt ( hermon, &hermon_qp->mtt ); - - /* Free memory */ - free_dma ( hermon_qp->wqe, hermon_qp->wqe_size ); - free ( hermon_qp ); - - /* Mark queue number as free */ - hermon_free_qpn ( ibdev, qp ); - - ib_qp_set_drvdata ( qp, NULL ); -} - -/*************************************************************************** - * - * Work request operations - * - *************************************************************************** - */ - -/** - * Construct UD send work queue entry - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @v av Address vector - * @v iobuf I/O buffer - * @v wqe Send work queue entry - * @ret opcode Control opcode - */ -static unsigned int -hermon_fill_ud_send_wqe ( struct ib_device *ibdev, - struct ib_queue_pair *qp __unused, - struct ib_address_vector *av, - struct io_buffer *iobuf, - union hermon_send_wqe *wqe ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - - MLX_FILL_1 ( &wqe->ud.ctrl, 1, ds, - ( ( offsetof ( typeof ( wqe->ud ), data[1] ) / 16 ) ) ); - MLX_FILL_1 ( &wqe->ud.ctrl, 2, c, 0x03 /* generate completion */ ); - MLX_FILL_2 ( &wqe->ud.ud, 0, - ud_address_vector.pd, HERMON_GLOBAL_PD, - ud_address_vector.port_number, ibdev->port ); - MLX_FILL_2 ( &wqe->ud.ud, 1, - ud_address_vector.rlid, av->lid, - ud_address_vector.g, av->gid_present ); - MLX_FILL_1 ( &wqe->ud.ud, 2, - ud_address_vector.max_stat_rate, hermon_rate ( av ) ); - MLX_FILL_1 ( &wqe->ud.ud, 3, ud_address_vector.sl, av->sl ); - memcpy ( &wqe->ud.ud.u.dwords[4], &av->gid, sizeof ( av->gid ) ); - MLX_FILL_1 ( &wqe->ud.ud, 8, destination_qp, av->qpn ); - MLX_FILL_1 ( &wqe->ud.ud, 9, q_key, av->qkey ); - MLX_FILL_1 ( &wqe->ud.data[0], 0, byte_count, iob_len ( iobuf ) ); - MLX_FILL_1 ( &wqe->ud.data[0], 1, l_key, hermon->lkey ); - MLX_FILL_1 ( &wqe->ud.data[0], 3, - local_address_l, virt_to_bus ( iobuf->data ) ); - return HERMON_OPCODE_SEND; -} - -/** - * Construct MLX send work queue entry - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @v av Address vector - * @v iobuf I/O buffer - * @v wqe Send work queue entry - * @ret opcode Control opcode - */ -static unsigned int -hermon_fill_mlx_send_wqe ( struct ib_device *ibdev, - struct ib_queue_pair *qp, - struct ib_address_vector *av, - struct io_buffer *iobuf, - union hermon_send_wqe *wqe ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct io_buffer headers; - - /* Construct IB headers */ - iob_populate ( &headers, &wqe->mlx.headers, 0, - sizeof ( wqe->mlx.headers ) ); - iob_reserve ( &headers, sizeof ( wqe->mlx.headers ) ); - ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), av ); - - /* Fill work queue entry */ - MLX_FILL_1 ( &wqe->mlx.ctrl, 1, ds, - ( ( offsetof ( typeof ( wqe->mlx ), data[2] ) / 16 ) ) ); - MLX_FILL_5 ( &wqe->mlx.ctrl, 2, - c, 0x03 /* generate completion */, - icrc, 0 /* generate ICRC */, - max_statrate, hermon_rate ( av ), - slr, 0, - v15, ( ( qp->ext_qpn == IB_QPN_SMI ) ? 1 : 0 ) ); - MLX_FILL_1 ( &wqe->mlx.ctrl, 3, rlid, av->lid ); - MLX_FILL_1 ( &wqe->mlx.data[0], 0, - byte_count, iob_len ( &headers ) ); - MLX_FILL_1 ( &wqe->mlx.data[0], 1, l_key, hermon->lkey ); - MLX_FILL_1 ( &wqe->mlx.data[0], 3, - local_address_l, virt_to_bus ( headers.data ) ); - MLX_FILL_1 ( &wqe->mlx.data[1], 0, - byte_count, ( iob_len ( iobuf ) + 4 /* ICRC */ ) ); - MLX_FILL_1 ( &wqe->mlx.data[1], 1, l_key, hermon->lkey ); - MLX_FILL_1 ( &wqe->mlx.data[1], 3, - local_address_l, virt_to_bus ( iobuf->data ) ); - return HERMON_OPCODE_SEND; -} - -/** - * Construct RC send work queue entry - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @v av Address vector - * @v iobuf I/O buffer - * @v wqe Send work queue entry - * @ret opcode Control opcode - */ -static unsigned int -hermon_fill_rc_send_wqe ( struct ib_device *ibdev, - struct ib_queue_pair *qp __unused, - struct ib_address_vector *av __unused, - struct io_buffer *iobuf, - union hermon_send_wqe *wqe ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - - MLX_FILL_1 ( &wqe->rc.ctrl, 1, ds, - ( ( offsetof ( typeof ( wqe->rc ), data[1] ) / 16 ) ) ); - MLX_FILL_1 ( &wqe->rc.ctrl, 2, c, 0x03 /* generate completion */ ); - MLX_FILL_1 ( &wqe->rc.data[0], 0, byte_count, iob_len ( iobuf ) ); - MLX_FILL_1 ( &wqe->rc.data[0], 1, l_key, hermon->lkey ); - MLX_FILL_1 ( &wqe->rc.data[0], 3, - local_address_l, virt_to_bus ( iobuf->data ) ); - return HERMON_OPCODE_SEND; -} - -/** Work queue entry constructors */ -static unsigned int -( * hermon_fill_send_wqe[] ) ( struct ib_device *ibdev, - struct ib_queue_pair *qp, - struct ib_address_vector *av, - struct io_buffer *iobuf, - union hermon_send_wqe *wqe ) = { - [IB_QPT_SMI] = hermon_fill_mlx_send_wqe, - [IB_QPT_GSI] = hermon_fill_mlx_send_wqe, - [IB_QPT_UD] = hermon_fill_ud_send_wqe, - [IB_QPT_RC] = hermon_fill_rc_send_wqe, -}; - -/** - * Post send work queue entry - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @v av Address vector - * @v iobuf I/O buffer - * @ret rc Return status code - */ -static int hermon_post_send ( struct ib_device *ibdev, - struct ib_queue_pair *qp, - struct ib_address_vector *av, - struct io_buffer *iobuf ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp ); - struct ib_work_queue *wq = &qp->send; - struct hermon_send_work_queue *hermon_send_wq = &hermon_qp->send; - union hermon_send_wqe *wqe; - union hermonprm_doorbell_register db_reg; - unsigned int wqe_idx_mask; - unsigned int opcode; - - /* Allocate work queue entry */ - wqe_idx_mask = ( wq->num_wqes - 1 ); - if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) { - DBGC ( hermon, "Hermon %p send queue full", hermon ); - return -ENOBUFS; - } - wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf; - wqe = &hermon_send_wq->wqe[ wq->next_idx & - ( hermon_send_wq->num_wqes - 1 ) ]; - - /* Construct work queue entry */ - memset ( ( ( ( void * ) wqe ) + 4 /* avoid ctrl.owner */ ), 0, - ( sizeof ( *wqe ) - 4 ) ); - assert ( qp->type < ( sizeof ( hermon_fill_send_wqe ) / - sizeof ( hermon_fill_send_wqe[0] ) ) ); - assert ( hermon_fill_send_wqe[qp->type] != NULL ); - opcode = hermon_fill_send_wqe[qp->type] ( ibdev, qp, av, iobuf, wqe ); - barrier(); - MLX_FILL_2 ( &wqe->ctrl, 0, - opcode, opcode, - owner, - ( ( wq->next_idx & hermon_send_wq->num_wqes ) ? 1 : 0 ) ); - DBGCP ( hermon, "Hermon %p posting send WQE:\n", hermon ); - DBGCP_HD ( hermon, wqe, sizeof ( *wqe ) ); - barrier(); - - /* Ring doorbell register */ - MLX_FILL_1 ( &db_reg.send, 0, qn, qp->qpn ); - DBGCP ( hermon, "Ringing doorbell %08lx with %08x\n", - virt_to_phys ( hermon_send_wq->doorbell ), db_reg.dword[0] ); - writel ( db_reg.dword[0], ( hermon_send_wq->doorbell ) ); - - /* Update work queue's index */ - wq->next_idx++; - - return 0; -} - -/** - * Post receive work queue entry - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @v iobuf I/O buffer - * @ret rc Return status code - */ -static int hermon_post_recv ( struct ib_device *ibdev, - struct ib_queue_pair *qp, - struct io_buffer *iobuf ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp ); - struct ib_work_queue *wq = &qp->recv; - struct hermon_recv_work_queue *hermon_recv_wq = &hermon_qp->recv; - struct hermonprm_recv_wqe *wqe; - unsigned int wqe_idx_mask; - - /* Allocate work queue entry */ - wqe_idx_mask = ( wq->num_wqes - 1 ); - if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) { - DBGC ( hermon, "Hermon %p receive queue full", hermon ); - return -ENOBUFS; - } - wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf; - wqe = &hermon_recv_wq->wqe[wq->next_idx & wqe_idx_mask].recv; - - /* Construct work queue entry */ - MLX_FILL_1 ( &wqe->data[0], 0, byte_count, iob_tailroom ( iobuf ) ); - MLX_FILL_1 ( &wqe->data[0], 1, l_key, hermon->lkey ); - MLX_FILL_1 ( &wqe->data[0], 3, - local_address_l, virt_to_bus ( iobuf->data ) ); - - /* Update work queue's index */ - wq->next_idx++; - - /* Update doorbell record */ - barrier(); - MLX_FILL_1 ( &hermon_recv_wq->doorbell, 0, receive_wqe_counter, - ( wq->next_idx & 0xffff ) ); - - return 0; -} - -/** - * Handle completion - * - * @v ibdev Infiniband device - * @v cq Completion queue - * @v cqe Hardware completion queue entry - * @ret rc Return status code - */ -static int hermon_complete ( struct ib_device *ibdev, - struct ib_completion_queue *cq, - union hermonprm_completion_entry *cqe ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct ib_work_queue *wq; - struct ib_queue_pair *qp; - struct hermon_queue_pair *hermon_qp; - struct io_buffer *iobuf; - struct ib_address_vector recv_av; - struct ib_global_route_header *grh; - struct ib_address_vector *av; - unsigned int opcode; - unsigned long qpn; - int is_send; - unsigned int wqe_idx; - size_t len; - int rc = 0; - - /* Parse completion */ - qpn = MLX_GET ( &cqe->normal, qpn ); - is_send = MLX_GET ( &cqe->normal, s_r ); - opcode = MLX_GET ( &cqe->normal, opcode ); - if ( opcode >= HERMON_OPCODE_RECV_ERROR ) { - /* "s" field is not valid for error opcodes */ - is_send = ( opcode == HERMON_OPCODE_SEND_ERROR ); - DBGC ( hermon, "Hermon %p CQN %lx syndrome %x vendor %x\n", - hermon, cq->cqn, MLX_GET ( &cqe->error, syndrome ), - MLX_GET ( &cqe->error, vendor_error_syndrome ) ); - rc = -EIO; - /* Don't return immediately; propagate error to completer */ - } - - /* Identify work queue */ - wq = ib_find_wq ( cq, qpn, is_send ); - if ( ! wq ) { - DBGC ( hermon, "Hermon %p CQN %lx unknown %s QPN %lx\n", - hermon, cq->cqn, ( is_send ? "send" : "recv" ), qpn ); - return -EIO; - } - qp = wq->qp; - hermon_qp = ib_qp_get_drvdata ( qp ); - - /* Identify I/O buffer */ - wqe_idx = ( MLX_GET ( &cqe->normal, wqe_counter ) & - ( wq->num_wqes - 1 ) ); - iobuf = wq->iobufs[wqe_idx]; - if ( ! iobuf ) { - DBGC ( hermon, "Hermon %p CQN %lx QPN %lx empty WQE %x\n", - hermon, cq->cqn, qp->qpn, wqe_idx ); - return -EIO; - } - wq->iobufs[wqe_idx] = NULL; - - if ( is_send ) { - /* Hand off to completion handler */ - ib_complete_send ( ibdev, qp, iobuf, rc ); - } else { - /* Set received length */ - len = MLX_GET ( &cqe->normal, byte_cnt ); - assert ( len <= iob_tailroom ( iobuf ) ); - iob_put ( iobuf, len ); - switch ( qp->type ) { - case IB_QPT_SMI: - case IB_QPT_GSI: - case IB_QPT_UD: - assert ( iob_len ( iobuf ) >= sizeof ( *grh ) ); - grh = iobuf->data; - iob_pull ( iobuf, sizeof ( *grh ) ); - /* Construct address vector */ - av = &recv_av; - memset ( av, 0, sizeof ( *av ) ); - av->qpn = MLX_GET ( &cqe->normal, srq_rqpn ); - av->lid = MLX_GET ( &cqe->normal, slid_smac47_32 ); - av->sl = MLX_GET ( &cqe->normal, sl ); - av->gid_present = MLX_GET ( &cqe->normal, g ); - memcpy ( &av->gid, &grh->sgid, sizeof ( av->gid ) ); - break; - case IB_QPT_RC: - av = &qp->av; - break; - default: - assert ( 0 ); - return -EINVAL; - } - /* Hand off to completion handler */ - ib_complete_recv ( ibdev, qp, av, iobuf, rc ); - } - - return rc; -} - -/** - * Poll completion queue - * - * @v ibdev Infiniband device - * @v cq Completion queue - */ -static void hermon_poll_cq ( struct ib_device *ibdev, - struct ib_completion_queue *cq ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq ); - union hermonprm_completion_entry *cqe; - unsigned int cqe_idx_mask; - int rc; - - while ( 1 ) { - /* Look for completion entry */ - cqe_idx_mask = ( cq->num_cqes - 1 ); - cqe = &hermon_cq->cqe[cq->next_idx & cqe_idx_mask]; - if ( MLX_GET ( &cqe->normal, owner ) ^ - ( ( cq->next_idx & cq->num_cqes ) ? 1 : 0 ) ) { - /* Entry still owned by hardware; end of poll */ - break; - } - DBGCP ( hermon, "Hermon %p completion:\n", hermon ); - DBGCP_HD ( hermon, cqe, sizeof ( *cqe ) ); - - /* Handle completion */ - if ( ( rc = hermon_complete ( ibdev, cq, cqe ) ) != 0 ) { - DBGC ( hermon, "Hermon %p failed to complete: %s\n", - hermon, strerror ( rc ) ); - DBGC_HD ( hermon, cqe, sizeof ( *cqe ) ); - } - - /* Update completion queue's index */ - cq->next_idx++; - - /* Update doorbell record */ - MLX_FILL_1 ( &hermon_cq->doorbell, 0, update_ci, - ( cq->next_idx & 0x00ffffffUL ) ); - } -} - -/*************************************************************************** - * - * Event queues - * - *************************************************************************** - */ - -/** - * Create event queue - * - * @v hermon Hermon device - * @ret rc Return status code - */ -static int hermon_create_eq ( struct hermon *hermon ) { - struct hermon_event_queue *hermon_eq = &hermon->eq; - struct hermonprm_eqc eqctx; - struct hermonprm_event_mask mask; - unsigned int i; - int rc; - - /* Select event queue number */ - hermon_eq->eqn = ( 4 * hermon->cap.reserved_uars ); - if ( hermon_eq->eqn < hermon->cap.reserved_eqs ) - hermon_eq->eqn = hermon->cap.reserved_eqs; - - /* Calculate doorbell address */ - hermon_eq->doorbell = - ( hermon->uar + HERMON_DB_EQ_OFFSET ( hermon_eq->eqn ) ); - - /* Allocate event queue itself */ - hermon_eq->eqe_size = - ( HERMON_NUM_EQES * sizeof ( hermon_eq->eqe[0] ) ); - hermon_eq->eqe = malloc_dma ( hermon_eq->eqe_size, - sizeof ( hermon_eq->eqe[0] ) ); - if ( ! hermon_eq->eqe ) { - rc = -ENOMEM; - goto err_eqe; - } - memset ( hermon_eq->eqe, 0, hermon_eq->eqe_size ); - for ( i = 0 ; i < HERMON_NUM_EQES ; i++ ) { - MLX_FILL_1 ( &hermon_eq->eqe[i].generic, 7, owner, 1 ); - } - barrier(); - - /* Allocate MTT entries */ - if ( ( rc = hermon_alloc_mtt ( hermon, hermon_eq->eqe, - hermon_eq->eqe_size, - &hermon_eq->mtt ) ) != 0 ) - goto err_alloc_mtt; - - /* Hand queue over to hardware */ - memset ( &eqctx, 0, sizeof ( eqctx ) ); - MLX_FILL_1 ( &eqctx, 0, st, 0xa /* "Fired" */ ); - MLX_FILL_1 ( &eqctx, 2, - page_offset, ( hermon_eq->mtt.page_offset >> 5 ) ); - MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( HERMON_NUM_EQES - 1 ) ); - MLX_FILL_1 ( &eqctx, 7, mtt_base_addr_l, - ( hermon_eq->mtt.mtt_base_addr >> 3 ) ); - if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, hermon_eq->eqn, - &eqctx ) ) != 0 ) { - DBGC ( hermon, "Hermon %p SW2HW_EQ failed: %s\n", - hermon, strerror ( rc ) ); - goto err_sw2hw_eq; - } - - /* Map events to this event queue */ - memset ( &mask, 0, sizeof ( mask ) ); - MLX_FILL_1 ( &mask, 1, port_state_change, 1 ); - if ( ( rc = hermon_cmd_map_eq ( hermon, - ( HERMON_MAP_EQ | hermon_eq->eqn ), - &mask ) ) != 0 ) { - DBGC ( hermon, "Hermon %p MAP_EQ failed: %s\n", - hermon, strerror ( rc ) ); - goto err_map_eq; - } - - DBGC ( hermon, "Hermon %p EQN %#lx ring at [%p,%p])\n", - hermon, hermon_eq->eqn, hermon_eq->eqe, - ( ( ( void * ) hermon_eq->eqe ) + hermon_eq->eqe_size ) ); - return 0; - - err_map_eq: - hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, &eqctx ); - err_sw2hw_eq: - hermon_free_mtt ( hermon, &hermon_eq->mtt ); - err_alloc_mtt: - free_dma ( hermon_eq->eqe, hermon_eq->eqe_size ); - err_eqe: - memset ( hermon_eq, 0, sizeof ( *hermon_eq ) ); - return rc; -} - -/** - * Destroy event queue - * - * @v hermon Hermon device - */ -static void hermon_destroy_eq ( struct hermon *hermon ) { - struct hermon_event_queue *hermon_eq = &hermon->eq; - struct hermonprm_eqc eqctx; - struct hermonprm_event_mask mask; - int rc; - - /* Unmap events from event queue */ - memset ( &mask, 0, sizeof ( mask ) ); - MLX_FILL_1 ( &mask, 1, port_state_change, 1 ); - if ( ( rc = hermon_cmd_map_eq ( hermon, - ( HERMON_UNMAP_EQ | hermon_eq->eqn ), - &mask ) ) != 0 ) { - DBGC ( hermon, "Hermon %p FATAL MAP_EQ failed to unmap: %s\n", - hermon, strerror ( rc ) ); - /* Continue; HCA may die but system should survive */ - } - - /* Take ownership back from hardware */ - if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, - &eqctx ) ) != 0 ) { - DBGC ( hermon, "Hermon %p FATAL HW2SW_EQ failed: %s\n", - hermon, strerror ( rc ) ); - /* Leak memory and return; at least we avoid corruption */ - return; - } - - /* Free MTT entries */ - hermon_free_mtt ( hermon, &hermon_eq->mtt ); - - /* Free memory */ - free_dma ( hermon_eq->eqe, hermon_eq->eqe_size ); - memset ( hermon_eq, 0, sizeof ( *hermon_eq ) ); -} - -/** - * Handle port state event - * - * @v hermon Hermon device - * @v eqe Port state change event queue entry - */ -static void hermon_event_port_state_change ( struct hermon *hermon, - union hermonprm_event_entry *eqe){ - unsigned int port; - int link_up; - - /* Get port and link status */ - port = ( MLX_GET ( &eqe->port_state_change, data.p ) - 1 ); - link_up = ( MLX_GET ( &eqe->generic, event_sub_type ) & 0x04 ); - DBGC ( hermon, "Hermon %p port %d link %s\n", hermon, ( port + 1 ), - ( link_up ? "up" : "down" ) ); - - /* Sanity check */ - if ( port >= hermon->cap.num_ports ) { - DBGC ( hermon, "Hermon %p port %d does not exist!\n", - hermon, ( port + 1 ) ); - return; - } - - /* Update MAD parameters */ - ib_smc_update ( hermon->ibdev[port], hermon_mad ); - - /* Notify Infiniband core of link state change */ - ib_link_state_changed ( hermon->ibdev[port] ); -} - -/** - * Poll event queue - * - * @v ibdev Infiniband device - */ -static void hermon_poll_eq ( struct ib_device *ibdev ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermon_event_queue *hermon_eq = &hermon->eq; - union hermonprm_event_entry *eqe; - union hermonprm_doorbell_register db_reg; - unsigned int eqe_idx_mask; - unsigned int event_type; - - while ( 1 ) { - /* Look for event entry */ - eqe_idx_mask = ( HERMON_NUM_EQES - 1 ); - eqe = &hermon_eq->eqe[hermon_eq->next_idx & eqe_idx_mask]; - if ( MLX_GET ( &eqe->generic, owner ) ^ - ( ( hermon_eq->next_idx & HERMON_NUM_EQES ) ? 1 : 0 ) ) { - /* Entry still owned by hardware; end of poll */ - break; - } - DBGCP ( hermon, "Hermon %p event:\n", hermon ); - DBGCP_HD ( hermon, eqe, sizeof ( *eqe ) ); - - /* Handle event */ - event_type = MLX_GET ( &eqe->generic, event_type ); - switch ( event_type ) { - case HERMON_EV_PORT_STATE_CHANGE: - hermon_event_port_state_change ( hermon, eqe ); - break; - default: - DBGC ( hermon, "Hermon %p unrecognised event type " - "%#x:\n", hermon, event_type ); - DBGC_HD ( hermon, eqe, sizeof ( *eqe ) ); - break; - } - - /* Update event queue's index */ - hermon_eq->next_idx++; - - /* Ring doorbell */ - MLX_FILL_1 ( &db_reg.event, 0, - ci, ( hermon_eq->next_idx & 0x00ffffffUL ) ); - DBGCP ( hermon, "Ringing doorbell %08lx with %08x\n", - virt_to_phys ( hermon_eq->doorbell ), - db_reg.dword[0] ); - writel ( db_reg.dword[0], hermon_eq->doorbell ); - } -} - -/*************************************************************************** - * - * Infiniband link-layer operations - * - *************************************************************************** - */ - -/** - * Sense port type - * - * @v ibdev Infiniband device - * @ret port_type Port type, or negative error - */ -static int hermon_sense_port_type ( struct ib_device *ibdev ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermonprm_sense_port sense_port; - int port_type; - int rc; - - /* If DPDP is not supported, always assume Infiniband */ - if ( ! hermon->cap.dpdp ) - return HERMON_PORT_TYPE_IB; - - /* Sense the port type */ - if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port, - &sense_port ) ) != 0 ) { - DBGC ( hermon, "Hermon %p port %d sense failed: %s\n", - hermon, ibdev->port, strerror ( rc ) ); - return rc; - } - port_type = MLX_GET ( &sense_port, port_type ); - - DBGC ( hermon, "Hermon %p port %d type %d\n", - hermon, ibdev->port, port_type ); - return port_type; -} - -/** - * Initialise Infiniband link - * - * @v ibdev Infiniband device - * @ret rc Return status code - */ -static int hermon_open ( struct ib_device *ibdev ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermonprm_init_port init_port; - int port_type; - int rc; - - /* Check we are connected to an Infiniband network */ - if ( ( rc = port_type = hermon_sense_port_type ( ibdev ) ) < 0 ) - return rc; - if ( port_type != HERMON_PORT_TYPE_IB ) { - DBGC ( hermon, "Hermon %p port %d not connected to an " - "Infiniband network", hermon, ibdev->port ); - return -ENOTCONN; - } - - /* Init Port */ - memset ( &init_port, 0, sizeof ( init_port ) ); - MLX_FILL_2 ( &init_port, 0, - port_width_cap, 3, - vl_cap, 1 ); - MLX_FILL_2 ( &init_port, 1, - mtu, HERMON_MTU_2048, - max_gid, 1 ); - MLX_FILL_1 ( &init_port, 2, max_pkey, 64 ); - if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port, - &init_port ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not intialise port: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - - /* Update MAD parameters */ - ib_smc_update ( ibdev, hermon_mad ); - - return 0; -} - -/** - * Close Infiniband link - * - * @v ibdev Infiniband device - */ -static void hermon_close ( struct ib_device *ibdev ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - int rc; - - if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not close port: %s\n", - hermon, strerror ( rc ) ); - /* Nothing we can do about this */ - } -} - -/** - * Inform embedded subnet management agent of a received MAD - * - * @v ibdev Infiniband device - * @v mad MAD - * @ret rc Return status code - */ -static int hermon_inform_sma ( struct ib_device *ibdev, - union ib_mad *mad ) { - int rc; - - /* Send the MAD to the embedded SMA */ - if ( ( rc = hermon_mad ( ibdev, mad ) ) != 0 ) - return rc; - - /* Update parameters held in software */ - ib_smc_update ( ibdev, hermon_mad ); - - return 0; -} - -/*************************************************************************** - * - * Multicast group operations - * - *************************************************************************** - */ - -/** - * Attach to multicast group - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @v gid Multicast GID - * @ret rc Return status code - */ -static int hermon_mcast_attach ( struct ib_device *ibdev, - struct ib_queue_pair *qp, - struct ib_gid *gid ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermonprm_mgm_hash hash; - struct hermonprm_mcg_entry mcg; - unsigned int index; - int rc; - - /* Generate hash table index */ - if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not hash GID: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - index = MLX_GET ( &hash, hash ); - - /* Check for existing hash table entry */ - if ( ( rc = hermon_cmd_read_mcg ( hermon, index, &mcg ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not read MCG %#x: %s\n", - hermon, index, strerror ( rc ) ); - return rc; - } - if ( MLX_GET ( &mcg, hdr.members_count ) != 0 ) { - /* FIXME: this implementation allows only a single QP - * per multicast group, and doesn't handle hash - * collisions. Sufficient for IPoIB but may need to - * be extended in future. - */ - DBGC ( hermon, "Hermon %p MGID index %#x already in use\n", - hermon, index ); - return -EBUSY; - } - - /* Update hash table entry */ - MLX_FILL_1 ( &mcg, 1, hdr.members_count, 1 ); - MLX_FILL_1 ( &mcg, 8, qp[0].qpn, qp->qpn ); - memcpy ( &mcg.u.dwords[4], gid, sizeof ( *gid ) ); - if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n", - hermon, index, strerror ( rc ) ); - return rc; - } - - return 0; -} - -/** - * Detach from multicast group - * - * @v ibdev Infiniband device - * @v qp Queue pair - * @v gid Multicast GID - */ -static void hermon_mcast_detach ( struct ib_device *ibdev, - struct ib_queue_pair *qp __unused, - struct ib_gid *gid ) { - struct hermon *hermon = ib_get_drvdata ( ibdev ); - struct hermonprm_mgm_hash hash; - struct hermonprm_mcg_entry mcg; - unsigned int index; - int rc; - - /* Generate hash table index */ - if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not hash GID: %s\n", - hermon, strerror ( rc ) ); - return; - } - index = MLX_GET ( &hash, hash ); - - /* Clear hash table entry */ - memset ( &mcg, 0, sizeof ( mcg ) ); - if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n", - hermon, index, strerror ( rc ) ); - return; - } -} - -/** Hermon Infiniband operations */ -static struct ib_device_operations hermon_ib_operations = { - .create_cq = hermon_create_cq, - .destroy_cq = hermon_destroy_cq, - .create_qp = hermon_create_qp, - .modify_qp = hermon_modify_qp, - .destroy_qp = hermon_destroy_qp, - .post_send = hermon_post_send, - .post_recv = hermon_post_recv, - .poll_cq = hermon_poll_cq, - .poll_eq = hermon_poll_eq, - .open = hermon_open, - .close = hermon_close, - .mcast_attach = hermon_mcast_attach, - .mcast_detach = hermon_mcast_detach, - .set_port_info = hermon_inform_sma, - .set_pkey_table = hermon_inform_sma, -}; - -/*************************************************************************** - * - * Firmware control - * - *************************************************************************** - */ - -/** - * Map virtual to physical address for firmware usage - * - * @v hermon Hermon device - * @v map Mapping function - * @v va Virtual address - * @v pa Physical address - * @v len Length of region - * @ret rc Return status code - */ -static int hermon_map_vpm ( struct hermon *hermon, - int ( *map ) ( struct hermon *hermon, - const struct hermonprm_virtual_physical_mapping* ), - uint64_t va, physaddr_t pa, size_t len ) { - struct hermonprm_virtual_physical_mapping mapping; - int rc; - - assert ( ( va & ( HERMON_PAGE_SIZE - 1 ) ) == 0 ); - assert ( ( pa & ( HERMON_PAGE_SIZE - 1 ) ) == 0 ); - assert ( ( len & ( HERMON_PAGE_SIZE - 1 ) ) == 0 ); - - /* These mappings tend to generate huge volumes of - * uninteresting debug data, which basically makes it - * impossible to use debugging otherwise. - */ - DBG_DISABLE ( DBGLVL_LOG | DBGLVL_EXTRA ); - - while ( len ) { - memset ( &mapping, 0, sizeof ( mapping ) ); - MLX_FILL_1 ( &mapping, 0, va_h, ( va >> 32 ) ); - MLX_FILL_1 ( &mapping, 1, va_l, ( va >> 12 ) ); - MLX_FILL_2 ( &mapping, 3, - log2size, 0, - pa_l, ( pa >> 12 ) ); - if ( ( rc = map ( hermon, &mapping ) ) != 0 ) { - DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA ); - DBGC ( hermon, "Hermon %p could not map %llx => %lx: " - "%s\n", hermon, va, pa, strerror ( rc ) ); - return rc; - } - pa += HERMON_PAGE_SIZE; - va += HERMON_PAGE_SIZE; - len -= HERMON_PAGE_SIZE; - } - - DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA ); - return 0; -} - -/** - * Start firmware running - * - * @v hermon Hermon device - * @ret rc Return status code - */ -static int hermon_start_firmware ( struct hermon *hermon ) { - struct hermonprm_query_fw fw; - unsigned int fw_pages; - size_t fw_size; - physaddr_t fw_base; - int rc; - - /* Get firmware parameters */ - if ( ( rc = hermon_cmd_query_fw ( hermon, &fw ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not query firmware: %s\n", - hermon, strerror ( rc ) ); - goto err_query_fw; - } - DBGC ( hermon, "Hermon %p firmware version %d.%d.%d\n", hermon, - MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ), - MLX_GET ( &fw, fw_rev_subminor ) ); - fw_pages = MLX_GET ( &fw, fw_pages ); - DBGC ( hermon, "Hermon %p requires %d pages (%d kB) for firmware\n", - hermon, fw_pages, ( fw_pages * ( HERMON_PAGE_SIZE / 1024 ) ) ); - - /* Allocate firmware pages and map firmware area */ - fw_size = ( fw_pages * HERMON_PAGE_SIZE ); - hermon->firmware_area = umalloc ( fw_size ); - if ( ! hermon->firmware_area ) { - rc = -ENOMEM; - goto err_alloc_fa; - } - fw_base = user_to_phys ( hermon->firmware_area, 0 ); - DBGC ( hermon, "Hermon %p firmware area at physical [%lx,%lx)\n", - hermon, fw_base, ( fw_base + fw_size ) ); - if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_fa, - 0, fw_base, fw_size ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not map firmware: %s\n", - hermon, strerror ( rc ) ); - goto err_map_fa; - } - - /* Start firmware */ - if ( ( rc = hermon_cmd_run_fw ( hermon ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not run firmware: %s\n", - hermon, strerror ( rc ) ); - goto err_run_fw; - } - - DBGC ( hermon, "Hermon %p firmware started\n", hermon ); - return 0; - - err_run_fw: - err_map_fa: - hermon_cmd_unmap_fa ( hermon ); - ufree ( hermon->firmware_area ); - hermon->firmware_area = UNULL; - err_alloc_fa: - err_query_fw: - return rc; -} - -/** - * Stop firmware running - * - * @v hermon Hermon device - */ -static void hermon_stop_firmware ( struct hermon *hermon ) { - int rc; - - if ( ( rc = hermon_cmd_unmap_fa ( hermon ) ) != 0 ) { - DBGC ( hermon, "Hermon %p FATAL could not stop firmware: %s\n", - hermon, strerror ( rc ) ); - /* Leak memory and return; at least we avoid corruption */ - return; - } - ufree ( hermon->firmware_area ); - hermon->firmware_area = UNULL; -} - -/*************************************************************************** - * - * Infinihost Context Memory management - * - *************************************************************************** - */ - -/** - * Get device limits - * - * @v hermon Hermon device - * @ret rc Return status code - */ -static int hermon_get_cap ( struct hermon *hermon ) { - struct hermonprm_query_dev_cap dev_cap; - int rc; - - if ( ( rc = hermon_cmd_query_dev_cap ( hermon, &dev_cap ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not get device limits: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - - hermon->cap.cmpt_entry_size = MLX_GET ( &dev_cap, c_mpt_entry_sz ); - hermon->cap.reserved_qps = - ( 1 << MLX_GET ( &dev_cap, log2_rsvd_qps ) ); - hermon->cap.qpc_entry_size = MLX_GET ( &dev_cap, qpc_entry_sz ); - hermon->cap.altc_entry_size = MLX_GET ( &dev_cap, altc_entry_sz ); - hermon->cap.auxc_entry_size = MLX_GET ( &dev_cap, aux_entry_sz ); - hermon->cap.reserved_srqs = - ( 1 << MLX_GET ( &dev_cap, log2_rsvd_srqs ) ); - hermon->cap.srqc_entry_size = MLX_GET ( &dev_cap, srq_entry_sz ); - hermon->cap.reserved_cqs = - ( 1 << MLX_GET ( &dev_cap, log2_rsvd_cqs ) ); - hermon->cap.cqc_entry_size = MLX_GET ( &dev_cap, cqc_entry_sz ); - hermon->cap.reserved_eqs = MLX_GET ( &dev_cap, num_rsvd_eqs ); - hermon->cap.eqc_entry_size = MLX_GET ( &dev_cap, eqc_entry_sz ); - hermon->cap.reserved_mtts = - ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mtts ) ); - hermon->cap.mtt_entry_size = MLX_GET ( &dev_cap, mtt_entry_sz ); - hermon->cap.reserved_mrws = - ( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) ); - hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz ); - hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars ); - hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports ); - hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp ); - - /* Sanity check */ - if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) { - DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n", - hermon, hermon->cap.num_ports, HERMON_MAX_PORTS ); - hermon->cap.num_ports = HERMON_MAX_PORTS; - } - - return 0; -} - -/** - * Get ICM usage - * - * @v log_num_entries Log2 of the number of entries - * @v entry_size Entry size - * @ret usage Usage size in ICM - */ -static size_t icm_usage ( unsigned int log_num_entries, size_t entry_size ) { - size_t usage; - - usage = ( ( 1 << log_num_entries ) * entry_size ); - usage = ( ( usage + HERMON_PAGE_SIZE - 1 ) & - ~( HERMON_PAGE_SIZE - 1 ) ); - return usage; -} - -/** - * Allocate ICM - * - * @v hermon Hermon device - * @v init_hca INIT_HCA structure to fill in - * @ret rc Return status code - */ -static int hermon_alloc_icm ( struct hermon *hermon, - struct hermonprm_init_hca *init_hca ) { - struct hermonprm_scalar_parameter icm_size; - struct hermonprm_scalar_parameter icm_aux_size; - uint64_t icm_offset = 0; - unsigned int log_num_qps, log_num_srqs, log_num_cqs, log_num_eqs; - unsigned int log_num_mtts, log_num_mpts; - size_t cmpt_max_len; - size_t qp_cmpt_len, srq_cmpt_len, cq_cmpt_len, eq_cmpt_len; - size_t icm_len, icm_aux_len; - physaddr_t icm_phys; - int i; - int rc; - - /* - * Start by carving up the ICM virtual address space - * - */ - - /* Calculate number of each object type within ICM */ - log_num_qps = fls ( hermon->cap.reserved_qps + - HERMON_RSVD_SPECIAL_QPS + HERMON_MAX_QPS - 1 ); - log_num_srqs = fls ( hermon->cap.reserved_srqs - 1 ); - log_num_cqs = fls ( hermon->cap.reserved_cqs + HERMON_MAX_CQS - 1 ); - log_num_eqs = fls ( hermon->cap.reserved_eqs + HERMON_MAX_EQS - 1 ); - log_num_mtts = fls ( hermon->cap.reserved_mtts + HERMON_MAX_MTTS - 1 ); - - /* ICM starts with the cMPT tables, which are sparse */ - cmpt_max_len = ( HERMON_CMPT_MAX_ENTRIES * - ( ( uint64_t ) hermon->cap.cmpt_entry_size ) ); - qp_cmpt_len = icm_usage ( log_num_qps, hermon->cap.cmpt_entry_size ); - hermon->icm_map[HERMON_ICM_QP_CMPT].offset = icm_offset; - hermon->icm_map[HERMON_ICM_QP_CMPT].len = qp_cmpt_len; - icm_offset += cmpt_max_len; - srq_cmpt_len = icm_usage ( log_num_srqs, hermon->cap.cmpt_entry_size ); - hermon->icm_map[HERMON_ICM_SRQ_CMPT].offset = icm_offset; - hermon->icm_map[HERMON_ICM_SRQ_CMPT].len = srq_cmpt_len; - icm_offset += cmpt_max_len; - cq_cmpt_len = icm_usage ( log_num_cqs, hermon->cap.cmpt_entry_size ); - hermon->icm_map[HERMON_ICM_CQ_CMPT].offset = icm_offset; - hermon->icm_map[HERMON_ICM_CQ_CMPT].len = cq_cmpt_len; - icm_offset += cmpt_max_len; - eq_cmpt_len = icm_usage ( log_num_eqs, hermon->cap.cmpt_entry_size ); - hermon->icm_map[HERMON_ICM_EQ_CMPT].offset = icm_offset; - hermon->icm_map[HERMON_ICM_EQ_CMPT].len = eq_cmpt_len; - icm_offset += cmpt_max_len; - - hermon->icm_map[HERMON_ICM_OTHER].offset = icm_offset; - - /* Queue pair contexts */ - MLX_FILL_1 ( init_hca, 12, - qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_h, - ( icm_offset >> 32 ) ); - MLX_FILL_2 ( init_hca, 13, - qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_l, - ( icm_offset >> 5 ), - qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp, - log_num_qps ); - DBGC ( hermon, "Hermon %p ICM QPC base = %llx\n", hermon, icm_offset ); - icm_offset += icm_usage ( log_num_qps, hermon->cap.qpc_entry_size ); - - /* Extended alternate path contexts */ - MLX_FILL_1 ( init_hca, 24, - qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_h, - ( icm_offset >> 32 ) ); - MLX_FILL_1 ( init_hca, 25, - qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_l, - icm_offset ); - DBGC ( hermon, "Hermon %p ICM ALTC base = %llx\n", hermon, icm_offset); - icm_offset += icm_usage ( log_num_qps, - hermon->cap.altc_entry_size ); - - /* Extended auxiliary contexts */ - MLX_FILL_1 ( init_hca, 28, - qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_h, - ( icm_offset >> 32 ) ); - MLX_FILL_1 ( init_hca, 29, - qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_l, - icm_offset ); - DBGC ( hermon, "Hermon %p ICM AUXC base = %llx\n", hermon, icm_offset); - icm_offset += icm_usage ( log_num_qps, - hermon->cap.auxc_entry_size ); - - /* Shared receive queue contexts */ - MLX_FILL_1 ( init_hca, 18, - qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_h, - ( icm_offset >> 32 ) ); - MLX_FILL_2 ( init_hca, 19, - qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_l, - ( icm_offset >> 5 ), - qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq, - log_num_srqs ); - DBGC ( hermon, "Hermon %p ICM SRQC base = %llx\n", hermon, icm_offset); - icm_offset += icm_usage ( log_num_srqs, - hermon->cap.srqc_entry_size ); - - /* Completion queue contexts */ - MLX_FILL_1 ( init_hca, 20, - qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_h, - ( icm_offset >> 32 ) ); - MLX_FILL_2 ( init_hca, 21, - qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_l, - ( icm_offset >> 5 ), - qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq, - log_num_cqs ); - DBGC ( hermon, "Hermon %p ICM CQC base = %llx\n", hermon, icm_offset ); - icm_offset += icm_usage ( log_num_cqs, hermon->cap.cqc_entry_size ); - - /* Event queue contexts */ - MLX_FILL_1 ( init_hca, 32, - qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_h, - ( icm_offset >> 32 ) ); - MLX_FILL_2 ( init_hca, 33, - qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_l, - ( icm_offset >> 5 ), - qpc_eec_cqc_eqc_rdb_parameters.log_num_of_eq, - log_num_eqs ); - DBGC ( hermon, "Hermon %p ICM EQC base = %llx\n", hermon, icm_offset ); - icm_offset += icm_usage ( log_num_eqs, hermon->cap.eqc_entry_size ); - - /* Memory translation table */ - MLX_FILL_1 ( init_hca, 64, - tpt_parameters.mtt_base_addr_h, ( icm_offset >> 32 ) ); - MLX_FILL_1 ( init_hca, 65, - tpt_parameters.mtt_base_addr_l, icm_offset ); - DBGC ( hermon, "Hermon %p ICM MTT base = %llx\n", hermon, icm_offset ); - icm_offset += icm_usage ( log_num_mtts, - hermon->cap.mtt_entry_size ); - - /* Memory protection table */ - log_num_mpts = fls ( hermon->cap.reserved_mrws + 1 - 1 ); - MLX_FILL_1 ( init_hca, 60, - tpt_parameters.dmpt_base_adr_h, ( icm_offset >> 32 ) ); - MLX_FILL_1 ( init_hca, 61, - tpt_parameters.dmpt_base_adr_l, icm_offset ); - MLX_FILL_1 ( init_hca, 62, - tpt_parameters.log_dmpt_sz, log_num_mpts ); - DBGC ( hermon, "Hermon %p ICM DMPT base = %llx\n", hermon, icm_offset); - icm_offset += icm_usage ( log_num_mpts, - hermon->cap.dmpt_entry_size ); - - /* Multicast table */ - MLX_FILL_1 ( init_hca, 48, - multicast_parameters.mc_base_addr_h, - ( icm_offset >> 32 ) ); - MLX_FILL_1 ( init_hca, 49, - multicast_parameters.mc_base_addr_l, icm_offset ); - MLX_FILL_1 ( init_hca, 52, - multicast_parameters.log_mc_table_entry_sz, - fls ( sizeof ( struct hermonprm_mcg_entry ) - 1 ) ); - MLX_FILL_1 ( init_hca, 53, - multicast_parameters.log_mc_table_hash_sz, 3 ); - MLX_FILL_1 ( init_hca, 54, - multicast_parameters.log_mc_table_sz, 3 ); - DBGC ( hermon, "Hermon %p ICM MC base = %llx\n", hermon, icm_offset ); - icm_offset += ( ( 8 * sizeof ( struct hermonprm_mcg_entry ) + - HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) ); - - hermon->icm_map[HERMON_ICM_OTHER].len = - ( icm_offset - hermon->icm_map[HERMON_ICM_OTHER].offset ); - - /* - * Allocate and map physical memory for (portions of) ICM - * - * Map is: - * ICM AUX area (aligned to its own size) - * cMPT areas - * Other areas - */ - - /* Calculate physical memory required for ICM */ - icm_len = 0; - for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) { - icm_len += hermon->icm_map[i].len; - } - - /* Get ICM auxiliary area size */ - memset ( &icm_size, 0, sizeof ( icm_size ) ); - MLX_FILL_1 ( &icm_size, 0, value_hi, ( icm_offset >> 32 ) ); - MLX_FILL_1 ( &icm_size, 1, value, icm_offset ); - if ( ( rc = hermon_cmd_set_icm_size ( hermon, &icm_size, - &icm_aux_size ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not set ICM size: %s\n", - hermon, strerror ( rc ) ); - goto err_set_icm_size; - } - icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * HERMON_PAGE_SIZE ); - - /* Allocate ICM data and auxiliary area */ - DBGC ( hermon, "Hermon %p requires %zd kB ICM and %zd kB AUX ICM\n", - hermon, ( icm_len / 1024 ), ( icm_aux_len / 1024 ) ); - hermon->icm = umalloc ( icm_aux_len + icm_len ); - if ( ! hermon->icm ) { - rc = -ENOMEM; - goto err_alloc; - } - icm_phys = user_to_phys ( hermon->icm, 0 ); - - /* Map ICM auxiliary area */ - DBGC ( hermon, "Hermon %p mapping ICM AUX => %08lx\n", - hermon, icm_phys ); - if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm_aux, - 0, icm_phys, icm_aux_len ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not map AUX ICM: %s\n", - hermon, strerror ( rc ) ); - goto err_map_icm_aux; - } - icm_phys += icm_aux_len; - - /* MAP ICM area */ - for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) { - DBGC ( hermon, "Hermon %p mapping ICM %llx+%zx => %08lx\n", - hermon, hermon->icm_map[i].offset, - hermon->icm_map[i].len, icm_phys ); - if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm, - hermon->icm_map[i].offset, - icm_phys, - hermon->icm_map[i].len ) ) != 0 ){ - DBGC ( hermon, "Hermon %p could not map ICM: %s\n", - hermon, strerror ( rc ) ); - goto err_map_icm; - } - icm_phys += hermon->icm_map[i].len; - } - - return 0; - - err_map_icm: - assert ( i == 0 ); /* We don't handle partial failure at present */ - err_map_icm_aux: - hermon_cmd_unmap_icm_aux ( hermon ); - ufree ( hermon->icm ); - hermon->icm = UNULL; - err_alloc: - err_set_icm_size: - return rc; -} - -/** - * Free ICM - * - * @v hermon Hermon device - */ -static void hermon_free_icm ( struct hermon *hermon ) { - struct hermonprm_scalar_parameter unmap_icm; - int i; - - for ( i = ( HERMON_ICM_NUM_REGIONS - 1 ) ; i >= 0 ; i-- ) { - memset ( &unmap_icm, 0, sizeof ( unmap_icm ) ); - MLX_FILL_1 ( &unmap_icm, 0, value_hi, - ( hermon->icm_map[i].offset >> 32 ) ); - MLX_FILL_1 ( &unmap_icm, 1, value, - hermon->icm_map[i].offset ); - hermon_cmd_unmap_icm ( hermon, - ( 1 << fls ( ( hermon->icm_map[i].len / - HERMON_PAGE_SIZE ) - 1)), - &unmap_icm ); - } - hermon_cmd_unmap_icm_aux ( hermon ); - ufree ( hermon->icm ); - hermon->icm = UNULL; -} - -/*************************************************************************** - * - * PCI interface - * - *************************************************************************** - */ - -/** - * Set up memory protection table - * - * @v hermon Hermon device - * @ret rc Return status code - */ -static int hermon_setup_mpt ( struct hermon *hermon ) { - struct hermonprm_mpt mpt; - uint32_t key; - int rc; - - /* Derive key */ - key = ( hermon->cap.reserved_mrws | HERMON_MKEY_PREFIX ); - hermon->lkey = ( ( key << 8 ) | ( key >> 24 ) ); - - /* Initialise memory protection table */ - memset ( &mpt, 0, sizeof ( mpt ) ); - MLX_FILL_7 ( &mpt, 0, - atomic, 1, - rw, 1, - rr, 1, - lw, 1, - lr, 1, - pa, 1, - r_w, 1 ); - MLX_FILL_1 ( &mpt, 2, mem_key, key ); - MLX_FILL_1 ( &mpt, 3, - pd, HERMON_GLOBAL_PD ); - MLX_FILL_1 ( &mpt, 10, len64, 1 ); - if ( ( rc = hermon_cmd_sw2hw_mpt ( hermon, - hermon->cap.reserved_mrws, - &mpt ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not set up MPT: %s\n", - hermon, strerror ( rc ) ); - return rc; - } - - return 0; -} - -/** - * Configure special queue pairs - * - * @v hermon Hermon device - * @ret rc Return status code - */ -static int hermon_configure_special_qps ( struct hermon *hermon ) { - int rc; - - /* Special QP block must be aligned on its own size */ - hermon->special_qpn_base = ( ( hermon->cap.reserved_qps + - HERMON_NUM_SPECIAL_QPS - 1 ) - & ~( HERMON_NUM_SPECIAL_QPS - 1 ) ); - hermon->qpn_base = ( hermon->special_qpn_base + - HERMON_NUM_SPECIAL_QPS ); - DBGC ( hermon, "Hermon %p special QPs at [%lx,%lx]\n", hermon, - hermon->special_qpn_base, ( hermon->qpn_base - 1 ) ); - - /* Issue command to configure special QPs */ - if ( ( rc = hermon_cmd_conf_special_qp ( hermon, 0x00, - hermon->special_qpn_base ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not configure special QPs: " - "%s\n", hermon, strerror ( rc ) ); - return rc; - } - - return 0; -} - -/** - * Reset device - * - * @v hermon Hermon device - * @v pci PCI device - */ -static void hermon_reset ( struct hermon *hermon, - struct pci_device *pci ) { - struct pci_config_backup backup; - static const uint8_t backup_exclude[] = - PCI_CONFIG_BACKUP_EXCLUDE ( 0x58, 0x5c ); - - pci_backup ( pci, &backup, backup_exclude ); - writel ( HERMON_RESET_MAGIC, - ( hermon->config + HERMON_RESET_OFFSET ) ); - mdelay ( HERMON_RESET_WAIT_TIME_MS ); - pci_restore ( pci, &backup, backup_exclude ); -} - -/** - * Probe PCI device - * - * @v pci PCI device - * @v id PCI ID - * @ret rc Return status code - */ -static int hermon_probe ( struct pci_device *pci, - const struct pci_device_id *id __unused ) { - struct hermon *hermon; - struct ib_device *ibdev; - struct hermonprm_init_hca init_hca; - unsigned int i; - int rc; - - /* Allocate Hermon device */ - hermon = zalloc ( sizeof ( *hermon ) ); - if ( ! hermon ) { - rc = -ENOMEM; - goto err_alloc_hermon; - } - pci_set_drvdata ( pci, hermon ); - - /* Fix up PCI device */ - adjust_pci_device ( pci ); - - /* Get PCI BARs */ - hermon->config = ioremap ( pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR), - HERMON_PCI_CONFIG_BAR_SIZE ); - hermon->uar = ioremap ( pci_bar_start ( pci, HERMON_PCI_UAR_BAR ), - HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE ); - - /* Reset device */ - hermon_reset ( hermon, pci ); - - /* Allocate space for mailboxes */ - hermon->mailbox_in = malloc_dma ( HERMON_MBOX_SIZE, - HERMON_MBOX_ALIGN ); - if ( ! hermon->mailbox_in ) { - rc = -ENOMEM; - goto err_mailbox_in; - } - hermon->mailbox_out = malloc_dma ( HERMON_MBOX_SIZE, - HERMON_MBOX_ALIGN ); - if ( ! hermon->mailbox_out ) { - rc = -ENOMEM; - goto err_mailbox_out; - } - - /* Start firmware */ - if ( ( rc = hermon_start_firmware ( hermon ) ) != 0 ) - goto err_start_firmware; - - /* Get device limits */ - if ( ( rc = hermon_get_cap ( hermon ) ) != 0 ) - goto err_get_cap; - - /* Allocate Infiniband devices */ - for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { - ibdev = alloc_ibdev ( 0 ); - if ( ! ibdev ) { - rc = -ENOMEM; - goto err_alloc_ibdev; - } - hermon->ibdev[i] = ibdev; - ibdev->op = &hermon_ib_operations; - ibdev->dev = &pci->dev; - ibdev->port = ( HERMON_PORT_BASE + i ); - ib_set_drvdata ( ibdev, hermon ); - } - - /* Allocate ICM */ - memset ( &init_hca, 0, sizeof ( init_hca ) ); - if ( ( rc = hermon_alloc_icm ( hermon, &init_hca ) ) != 0 ) - goto err_alloc_icm; - - /* Initialise HCA */ - MLX_FILL_1 ( &init_hca, 0, version, 0x02 /* "Must be 0x02" */ ); - MLX_FILL_1 ( &init_hca, 5, udp, 1 ); - MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 8 ); - if ( ( rc = hermon_cmd_init_hca ( hermon, &init_hca ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not initialise HCA: %s\n", - hermon, strerror ( rc ) ); - goto err_init_hca; - } - - /* Set up memory protection */ - if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 ) - goto err_setup_mpt; - for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) - hermon->ibdev[i]->rdma_key = hermon->lkey; - - /* Set up event queue */ - if ( ( rc = hermon_create_eq ( hermon ) ) != 0 ) - goto err_create_eq; - - /* Configure special QPs */ - if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 ) - goto err_conf_special_qps; - - /* Update IPoIB MAC address */ - for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { - ib_smc_update ( hermon->ibdev[i], hermon_mad ); - } - - /* Register Infiniband devices */ - for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) { - if ( ( rc = register_ibdev ( hermon->ibdev[i] ) ) != 0 ) { - DBGC ( hermon, "Hermon %p could not register IB " - "device: %s\n", hermon, strerror ( rc ) ); - goto err_register_ibdev; - } - } - - return 0; - - i = hermon->cap.num_ports; - err_register_ibdev: - for ( i-- ; ( signed int ) i >= 0 ; i-- ) - unregister_ibdev ( hermon->ibdev[i] ); - err_conf_special_qps: - hermon_destroy_eq ( hermon ); - err_create_eq: - err_setup_mpt: - hermon_cmd_close_hca ( hermon ); - err_init_hca: - hermon_free_icm ( hermon ); - err_alloc_icm: - i = hermon->cap.num_ports; - err_alloc_ibdev: - for ( i-- ; ( signed int ) i >= 0 ; i-- ) - ibdev_put ( hermon->ibdev[i] ); - err_get_cap: - hermon_stop_firmware ( hermon ); - err_start_firmware: - free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE ); - err_mailbox_out: - free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE ); - err_mailbox_in: - free ( hermon ); - err_alloc_hermon: - return rc; -} - -/** - * Remove PCI device - * - * @v pci PCI device - */ -static void hermon_remove ( struct pci_device *pci ) { - struct hermon *hermon = pci_get_drvdata ( pci ); - int i; - - for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) - unregister_ibdev ( hermon->ibdev[i] ); - hermon_destroy_eq ( hermon ); - hermon_cmd_close_hca ( hermon ); - hermon_free_icm ( hermon ); - hermon_stop_firmware ( hermon ); - hermon_stop_firmware ( hermon ); - free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE ); - free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE ); - for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) - ibdev_put ( hermon->ibdev[i] ); - free ( hermon ); -} - -static struct pci_device_id hermon_nics[] = { - PCI_ROM ( 0x15b3, 0x6340, "mt25408", "MT25408 HCA driver", 0 ), - PCI_ROM ( 0x15b3, 0x634a, "mt25418", "MT25418 HCA driver", 0 ), - PCI_ROM ( 0x15b3, 0x6732, "mt26418", "MT26418 HCA driver", 0 ), - PCI_ROM ( 0x15b3, 0x673c, "mt26428", "MT26428 HCA driver", 0 ), -}; - -struct pci_driver hermon_driver __pci_driver = { - .ids = hermon_nics, - .id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ), - .probe = hermon_probe, - .remove = hermon_remove, -}; |