/* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: main (header) */ #ifndef __BNXT_RE_H__ #define __BNXT_RE_H__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bnxt.h" #include "bnxt_ulp.h" #include "hsi_struct_def.h" #include "qplib_res.h" #include "qplib_sp.h" #include "qplib_fp.h" #include "qplib_rcfw.h" #include "ib_verbs.h" #include "stats.h" #define ROCE_DRV_MODULE_NAME "bnxt_re" #define ROCE_DRV_MODULE_VERSION "230.0.133.0" #define ROCE_DRV_MODULE_RELDATE "April 22, 2024" #define BNXT_RE_REF_WAIT_COUNT 20 #define BNXT_RE_ROCE_V1_ETH_TYPE 0x8915 #define BNXT_RE_ROCE_V2_PORT_NO 4791 #define BNXT_RE_RES_FREE_WAIT_COUNT 1000 #define BNXT_RE_PAGE_SHIFT_4K (12) #define BNXT_RE_PAGE_SHIFT_8K (13) #define BNXT_RE_PAGE_SHIFT_64K (16) #define BNXT_RE_PAGE_SHIFT_2M (21) #define BNXT_RE_PAGE_SHIFT_8M (23) #define BNXT_RE_PAGE_SHIFT_1G (30) #define BNXT_RE_PAGE_SIZE_4K BIT(BNXT_RE_PAGE_SHIFT_4K) #define BNXT_RE_PAGE_SIZE_8K BIT(BNXT_RE_PAGE_SHIFT_8K) #define BNXT_RE_PAGE_SIZE_64K BIT(BNXT_RE_PAGE_SHIFT_64K) #define BNXT_RE_PAGE_SIZE_2M BIT(BNXT_RE_PAGE_SHIFT_2M) #define BNXT_RE_PAGE_SIZE_8M BIT(BNXT_RE_PAGE_SHIFT_8M) #define BNXT_RE_PAGE_SIZE_1G BIT(BNXT_RE_PAGE_SHIFT_1G) #define BNXT_RE_MAX_MR_SIZE_LOW BIT(BNXT_RE_PAGE_SHIFT_1G) #define BNXT_RE_MAX_MR_SIZE_HIGH BIT(39) #define BNXT_RE_MAX_MR_SIZE BNXT_RE_MAX_MR_SIZE_HIGH /* Number of MRs to reserve for PF, leaving remainder for VFs */ #define BNXT_RE_RESVD_MR_FOR_PF (32 * 1024) #define BNXT_RE_MAX_GID_PER_VF 128 #define BNXT_RE_MAX_VF_QPS_PER_PF (6 * 1024) /** * min_not_zero - return the minimum that is _not_ zero, unless both are zero * @x: value1 * @y: value2 */ #ifndef min_not_zero #define min_not_zero(x, y) ({ \ typeof(x) __x = (x); \ typeof(y) __y = (y); \ __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); }) #endif struct ib_mr_init_attr { int max_reg_descriptors; u32 flags; }; struct bnxt_re_dev; int bnxt_re_register_netdevice_notifier(struct notifier_block *nb); int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb); int ib_register_device_compat(struct bnxt_re_dev *rdev); #ifndef __struct_group #define __struct_group(TAG, NAME, ATTRS, MEMBERS...) \ union { \ struct { MEMBERS } ATTRS; \ struct TAG { MEMBERS } ATTRS NAME; \ } #endif /* __struct_group */ #ifndef struct_group_attr #define struct_group_attr(NAME, ATTRS, MEMBERS...) \ __struct_group(/* no tag */, NAME, ATTRS, MEMBERS) #endif /* struct_group_attr */ /* * Percentage of resources of each type reserved for PF. * Remaining resources are divided equally among VFs. * [0, 100] */ #define BNXT_RE_RQ_WQE_THRESHOLD 32 #define BNXT_RE_UD_QP_HW_STALL 0x400000 /* * Setting the default ack delay value to 16, which means * the default timeout is approx. 260ms(4 usec * 2 ^(timeout)) */ #define BNXT_RE_DEFAULT_ACK_DELAY 16 #define BNXT_RE_BOND_PF_MAX 2 #define BNXT_RE_STATS_CTX_UPDATE_TIMER 250 #define BNXT_RE_30SEC_MSEC (30 * 1000) #define BNXT_RE_BOND_RESCHED_CNT 10 #define BNXT_RE_CHIP_NUM_57454 0xC454 #define BNXT_RE_CHIP_NUM_57452 0xC452 #define BNXT_RE_CHIP_NUM_5745X(chip_num) \ ((chip_num) == BNXT_RE_CHIP_NUM_57454 || \ (chip_num) == BNXT_RE_CHIP_NUM_57452) #define BNXT_RE_MIN_KERNEL_QP_TX_DEPTH 4096 #define BNXT_RE_STOP_QPS_BUDGET 200 #define BNXT_RE_HWRM_CMD_TIMEOUT(rdev) \ ((rdev)->chip_ctx->hwrm_cmd_max_timeout * 1000) extern unsigned int min_tx_depth; extern struct mutex bnxt_re_dev_lock; extern struct mutex bnxt_re_mutex; extern struct list_head bnxt_re_dev_list; struct bnxt_re_ring_attr { dma_addr_t *dma_arr; int pages; int type; u32 depth; u32 lrid; /* Logical ring id */ u16 flags; u8 mode; u8 rsvd; }; #define BNXT_RE_MAX_DEVICES 256 #define BNXT_RE_MSIX_FROM_MOD_PARAM -1 #define BNXT_RE_MIN_MSIX 2 #define BNXT_RE_MAX_MSIX_VF 2 #define BNXT_RE_MAX_MSIX_PF 9 #define BNXT_RE_MAX_MSIX_NPAR_PF 5 #define BNXT_RE_MAX_MSIX 64 #define BNXT_RE_MAX_MSIX_GEN_P5_PF BNXT_RE_MAX_MSIX #define BNXT_RE_GEN_P5_MAX_VF 64 struct bnxt_re_nq_record { struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX]; /* FP Notification Queue (CQ & SRQ) */ struct bnxt_qplib_nq nq[BNXT_RE_MAX_MSIX]; int num_msix; int max_init; struct mutex load_lock; }; struct bnxt_re_work { struct work_struct work; unsigned long event; struct bnxt_re_dev *rdev; struct ifnet *vlan_dev; bool do_lag; /* netdev where we received the event */ struct ifnet *netdev; struct auxiliary_device *adev; }; /* * Data structure and defines to handle * recovery */ #define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY 20 #define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS 30000 /* 30sec timeout */ #define BNXT_RE_PRE_RECOVERY_REMOVE 0x1 #define BNXT_RE_COMPLETE_REMOVE 0x2 #define BNXT_RE_POST_RECOVERY_INIT 0x4 #define BNXT_RE_COMPLETE_INIT 0x8 #define BNXT_RE_COMPLETE_SHUTDOWN 0x10 /* QP1 SQ entry data strucutre */ struct bnxt_re_sqp_entries { u64 wrid; struct bnxt_qplib_sge sge; /* For storing the actual qp1 cqe */ struct bnxt_qplib_cqe cqe; struct bnxt_re_qp *qp1_qp; }; /* GSI QP mode enum */ enum bnxt_re_gsi_mode { BNXT_RE_GSI_MODE_INVALID = 0, BNXT_RE_GSI_MODE_ALL = 1, BNXT_RE_GSI_MODE_ROCE_V1, BNXT_RE_GSI_MODE_ROCE_V2_IPV4, BNXT_RE_GSI_MODE_ROCE_V2_IPV6, BNXT_RE_GSI_MODE_UD }; enum bnxt_re_roce_cap { BNXT_RE_FLAG_ROCEV1_CAP = 1, BNXT_RE_FLAG_ROCEV2_CAP, BNXT_RE_FLAG_ROCEV1_V2_CAP, }; #define BNXT_RE_MAX_GSI_SQP_ENTRIES 1024 struct bnxt_re_gsi_context { u8 gsi_qp_mode; bool first_cq_created; /* Start: used only in gsi_mode_all */ struct bnxt_re_qp *gsi_qp; struct bnxt_re_qp *gsi_sqp; struct bnxt_re_ah *gsi_sah; struct bnxt_re_sqp_entries *sqp_tbl; /* End: used only in gsi_mode_all */ }; struct bnxt_re_tc_rec { u8 cos_id_roce; u8 tc_roce; u8 cos_id_cnp; u8 tc_cnp; u8 tc_def; u8 cos_id_def; u8 max_tc; u8 roce_prio; u8 cnp_prio; u8 roce_dscp; u8 cnp_dscp; u8 prio_valid; u8 dscp_valid; bool ecn_enabled; bool serv_type_enabled; u64 cnp_dscp_bv; u64 roce_dscp_bv; }; struct bnxt_re_dscp2pri { u8 dscp; u8 mask; u8 pri; }; struct bnxt_re_cos2bw_cfg { u8 pad[3]; struct_group_attr(cfg, __packed, u8 queue_id; __le32 min_bw; __le32 max_bw; u8 tsa; u8 pri_lvl; u8 bw_weight; ); u8 unused; }; #define BNXT_RE_AEQ_IDX 0 #define BNXT_RE_MAX_SGID_ENTRIES 256 #define BNXT_RE_DBGFS_FILE_MEM 65536 enum { BNXT_RE_STATS_QUERY = 1, BNXT_RE_QP_QUERY = 2, BNXT_RE_SERVICE_FN_QUERY = 3, }; struct bnxt_re_dbg_file { struct bnxt_re_dev *rdev; u32 type; union { struct bnxt_qplib_query_stats_info sinfo; struct bnxt_qplib_query_fn_info fninfo; }params; char dbg_buf[BNXT_RE_DBGFS_FILE_MEM]; }; struct bnxt_re_debug_entries { /* Dir entries */ struct dentry *qpinfo_dir; struct dentry *service_fn_dir; /* file entries */ struct dentry *stat_query; struct bnxt_re_dbg_file stat_file; struct dentry *qplist_query; struct bnxt_re_dbg_file qp_file; struct dentry *service_fn_query; struct bnxt_re_dbg_file service_fn_file; }; struct bnxt_re_en_dev_info { struct list_head en_list; struct bnxt_en_dev *en_dev; struct bnxt_re_dev *rdev; unsigned long flags; #define BNXT_RE_FLAG_EN_DEV_NETDEV_REG 0 #define BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV 1 #define BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV 2 u8 wqe_mode; u8 gsi_mode; bool te_bypass; bool ib_uninit_done; u32 num_msix_requested; wait_queue_head_t waitq; }; #define BNXT_RE_DB_FIFO_ROOM_MASK_P5 0x1FFF8000 #define BNXT_RE_MAX_FIFO_DEPTH_P5 0x2c00 #define BNXT_RE_DB_FIFO_ROOM_SHIFT 15 #define BNXT_RE_DB_FIFO_ROOM_MASK_P7 0x3FFF8000 #define BNXT_RE_MAX_FIFO_DEPTH_P7 0x8000 #define BNXT_RE_DB_FIFO_ROOM_MASK(ctx) \ (_is_chip_p7((ctx)) ? \ BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\ BNXT_RE_DB_FIFO_ROOM_MASK_P5) #define BNXT_RE_MAX_FIFO_DEPTH(ctx) \ (_is_chip_p7((ctx)) ? \ BNXT_RE_MAX_FIFO_DEPTH_P7 :\ BNXT_RE_MAX_FIFO_DEPTH_P5) struct bnxt_dbq_nq_list { int num_nql_entries; u16 nq_id[16]; }; #define BNXT_RE_ASYNC_ERR_REP_BASE(_type) \ (ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_##_type) #define BNXT_RE_ASYNC_ERR_DBR_TRESH(_type) \ (ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_##_type) #define BNXT_RE_EVENT_DBR_EPOCH(data) \ (((data) & \ BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_MASK)) >> \ BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_SFT)) #define BNXT_RE_EVENT_ERROR_REPORT_TYPE(data1) \ (((data1) & \ BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_MASK)) >> \ BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_SFT)) #define BNXT_RE_DBR_LIST_ADD(_rdev, _res, _type) \ { \ spin_lock(&(_rdev)->res_list[_type].lock); \ list_add_tail(&(_res)->dbr_list, \ &(_rdev)->res_list[_type].head); \ spin_unlock(&(_rdev)->res_list[_type].lock); \ } #define BNXT_RE_DBR_LIST_DEL(_rdev, _res, _type) \ { \ spin_lock(&(_rdev)->res_list[_type].lock); \ list_del(&(_res)->dbr_list); \ spin_unlock(&(_rdev)->res_list[_type].lock); \ } #define BNXT_RE_CQ_PAGE_LIST_ADD(_uctx, _cq) \ { \ mutex_lock(&(_uctx)->cq_lock); \ list_add_tail(&(_cq)->cq_list, &(_uctx)->cq_list); \ mutex_unlock(&(_uctx)->cq_lock); \ } #define BNXT_RE_CQ_PAGE_LIST_DEL(_uctx, _cq) \ { \ mutex_lock(&(_uctx)->cq_lock); \ list_del(&(_cq)->cq_list); \ mutex_unlock(&(_uctx)->cq_lock); \ } #define BNXT_RE_NETDEV_EVENT(event, x) \ do { \ if ((event) == (x)) \ return #x; \ } while (0) /* Do not change the seq of this enum which is followed by dbr recov */ enum { BNXT_RE_RES_TYPE_CQ = 0, BNXT_RE_RES_TYPE_UCTX, BNXT_RE_RES_TYPE_QP, BNXT_RE_RES_TYPE_SRQ, BNXT_RE_RES_TYPE_MAX }; struct bnxt_re_dbr_res_list { struct list_head head; spinlock_t lock; }; struct bnxt_re_dbr_drop_recov_work { struct work_struct work; struct bnxt_re_dev *rdev; u32 curr_epoch; }; struct bnxt_re_aer_work { struct work_struct work; struct bnxt_re_dev *rdev; }; struct bnxt_re_dbq_stats { u64 fifo_occup_slab_1; u64 fifo_occup_slab_2; u64 fifo_occup_slab_3; u64 fifo_occup_slab_4; u64 fifo_occup_water_mark; u64 do_pacing_slab_1; u64 do_pacing_slab_2; u64 do_pacing_slab_3; u64 do_pacing_slab_4; u64 do_pacing_slab_5; u64 do_pacing_water_mark; }; /* Device debug statistics */ struct bnxt_re_drv_dbg_stats { struct bnxt_re_dbq_stats dbq; }; /* DB pacing counters */ struct bnxt_re_dbr_sw_stats { u64 dbq_int_recv; u64 dbq_int_en; u64 dbq_pacing_resched; u64 dbq_pacing_complete; u64 dbq_pacing_alerts; u64 dbr_drop_recov_events; u64 dbr_drop_recov_timeouts; u64 dbr_drop_recov_timeout_users; u64 dbr_drop_recov_event_skips; }; struct bnxt_re_dev { struct ib_device ibdev; struct list_head list; atomic_t ref_count; atomic_t sched_count; unsigned long flags; #define BNXT_RE_FLAG_NETDEV_REGISTERED 0 #define BNXT_RE_FLAG_IBDEV_REGISTERED 1 #define BNXT_RE_FLAG_GOT_MSIX 2 #define BNXT_RE_FLAG_HAVE_L2_REF 3 #define BNXT_RE_FLAG_ALLOC_RCFW 4 #define BNXT_RE_FLAG_NET_RING_ALLOC 5 #define BNXT_RE_FLAG_RCFW_CHANNEL_EN 6 #define BNXT_RE_FLAG_ALLOC_CTX 7 #define BNXT_RE_FLAG_STATS_CTX_ALLOC 8 #define BNXT_RE_FLAG_STATS_CTX2_ALLOC 9 #define BNXT_RE_FLAG_RCFW_CHANNEL_INIT 10 #define BNXT_RE_FLAG_WORKER_REG 11 #define BNXT_RE_FLAG_TBLS_ALLOCINIT 12 #define BNXT_RE_FLAG_SETUP_NQ 13 #define BNXT_RE_FLAG_BOND_DEV_REGISTERED 14 #define BNXT_RE_FLAG_PER_PORT_DEBUG_INFO 15 #define BNXT_RE_FLAG_DEV_LIST_INITIALIZED 16 #define BNXT_RE_FLAG_ERR_DEVICE_DETACHED 17 #define BNXT_RE_FLAG_INIT_DCBX_CC_PARAM 18 #define BNXT_RE_FLAG_STOP_IN_PROGRESS 20 #define BNXT_RE_FLAG_ISSUE_ROCE_STATS 29 #define BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS 30 struct ifnet *netdev; struct auxiliary_device *adev; struct bnxt_qplib_chip_ctx *chip_ctx; struct bnxt_en_dev *en_dev; struct bnxt_re_nq_record nqr; int id; struct delayed_work worker; u16 worker_30s; struct bnxt_re_tc_rec tc_rec[2]; u8 cur_prio_map; /* RCFW Channel */ struct bnxt_qplib_rcfw rcfw; /* Device Resources */ struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_res qplib_res; struct bnxt_qplib_dpi dpi_privileged; struct bnxt_qplib_cc_param cc_param; struct mutex cc_lock; struct mutex qp_lock; struct list_head qp_list; u8 roce_mode; /* Max of 2 lossless traffic class supported per port */ u16 cosq[2]; /* Start: QP for handling QP1 packets */ struct bnxt_re_gsi_context gsi_ctx; /* End: QP for handling QP1 packets */ bool is_virtfn; u32 num_vfs; u32 espeed; /* * For storing the speed of slave interfaces. * Same as espeed when bond is not configured */ u32 sl_espeed; /* To be used for a workaround for ISER stack */ u32 min_tx_depth; /* To enable qp debug info. Disabled during driver load */ u32 en_qp_dbg; /* Array to handle gid mapping */ char *gid_map; struct bnxt_re_device_stats stats; struct bnxt_re_drv_dbg_stats *dbg_stats; /* debugfs to expose per port information*/ struct dentry *port_debug_dir; struct dentry *info; struct dentry *drv_dbg_stats; struct dentry *sp_perf_stats; struct dentry *pdev_debug_dir; struct dentry *pdev_qpinfo_dir; struct bnxt_re_debug_entries *dbg_ent; struct workqueue_struct *resolve_wq; struct list_head mac_wq_list; struct workqueue_struct *dcb_wq; struct workqueue_struct *aer_wq; u32 event_bitmap[3]; bool unreg_sched; u64 dbr_throttling_reg_off; u64 dbr_aeq_arm_reg_off; u64 dbr_db_fifo_reg_off; void *dbr_page; u64 dbr_bar_addr; u32 pacing_algo_th; u32 pacing_en_int_th; u32 do_pacing_save; struct workqueue_struct *dbq_wq; struct workqueue_struct *dbr_drop_recov_wq; struct work_struct dbq_fifo_check_work; struct delayed_work dbq_pacing_work; /* protect DB pacing */ struct mutex dbq_lock; /* Control DBR pacing feature. Set if enabled */ bool dbr_pacing; /* Control DBR recovery feature. Set if enabled */ bool dbr_drop_recov; bool user_dbr_drop_recov; /* DBR recovery feature. Set if running */ bool dbr_recovery_on; u32 user_dbr_drop_recov_timeout; /* * Value used for pacing algo when pacing is active */ #define BNXT_RE_MAX_DBR_DO_PACING 0xFFFF u32 dbr_do_pacing; u32 dbq_watermark; /* Current watermark set in HW registers */ u32 dbq_nq_id; /* Current NQ ID for DBQ events */ u32 dbq_pacing_time; /* ms */ u32 dbr_def_do_pacing; /* do_pacing when no congestion */ u32 dbr_evt_curr_epoch; bool dbq_int_disable; bool mod_exit; struct bnxt_re_dbr_sw_stats *dbr_sw_stats; struct bnxt_re_dbr_res_list res_list[BNXT_RE_RES_TYPE_MAX]; struct bnxt_dbq_nq_list nq_list; char dev_name[IB_DEVICE_NAME_MAX]; atomic_t dbq_intr_running; u32 num_msix_requested; unsigned char *dev_addr; /* For netdev->dev_addr */ }; #define BNXT_RE_RESOLVE_RETRY_COUNT_US 5000000 /* 5 sec */ struct bnxt_re_resolve_dmac_work{ struct work_struct work; struct list_head list; struct bnxt_re_dev *rdev; struct ib_ah_attr *ah_attr; struct bnxt_re_ah_info *ah_info; atomic_t status_wait; }; static inline u8 bnxt_re_get_prio(u8 prio_map) { u8 prio = 0xFF; for (prio = 0; prio < 8; prio++) if (prio_map & (1UL << prio)) break; return prio; } /* This should be called with bnxt_re_dev_lock mutex held */ static inline bool __bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev) { struct bnxt_re_dev *tmp_rdev; list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) { if (rdev == tmp_rdev) return true; } return false; } static inline bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev) { struct bnxt_re_dev *tmp_rdev; mutex_lock(&bnxt_re_dev_lock); list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) { if (rdev == tmp_rdev) { mutex_unlock(&bnxt_re_dev_lock); return true; } } mutex_unlock(&bnxt_re_dev_lock); pr_debug("bnxt_re: %s : Invalid rdev received rdev = %p\n", __func__, rdev); return false; } int bnxt_re_send_hwrm_cmd(struct bnxt_re_dev *rdev, void *cmd, int cmdlen); void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev); int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, struct bnxt_re_dscp2pri *d2p, u16 count, u16 target_id); int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, struct bnxt_re_dscp2pri *d2p, u16 *count, u16 target_id); int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *cnprec, u16 tid); int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, struct bnxt_re_cos2bw_cfg *cfg); int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, struct bnxt_re_cos2bw_cfg *cfg); int bnxt_re_hwrm_pri2cos_cfg(struct bnxt_re_dev *rdev, u16 target_id, u16 port_id, u8 *cos_id_map, u8 pri_map); int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev); int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev); struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev); struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev); u8 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev, u8 selector); struct bnxt_qplib_nq * bnxt_re_get_nq(struct bnxt_re_dev *rdev); void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq); #define to_bnxt_re(ptr, type, member) \ container_of(ptr, type, member) #define to_bnxt_re_dev(ptr, member) \ container_of((ptr), struct bnxt_re_dev, member) /* Even number functions from port 0 and odd number from port 1 */ #define BNXT_RE_IS_PORT0(rdev) (!(rdev->en_dev->pdev->devfn & 1)) #define BNXT_RE_ROCE_V1_PACKET 0 #define BNXT_RE_ROCEV2_IPV4_PACKET 2 #define BNXT_RE_ROCEV2_IPV6_PACKET 3 #define BNXT_RE_ACTIVE_MAP_PORT1 0x1 /*port-1 active */ #define BNXT_RE_ACTIVE_MAP_PORT2 0x2 /*port-2 active */ #define BNXT_RE_MEMBER_PORT_MAP (BNXT_RE_ACTIVE_MAP_PORT1 | \ BNXT_RE_ACTIVE_MAP_PORT2) #define rdev_to_dev(rdev) ((rdev) ? (&(rdev)->ibdev.dev) : NULL) void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev); bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev); #define bnxt_re_rdev_ready(rdev) (bnxt_re_is_rdev_valid(rdev) && \ (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))) #define BNXT_RE_SRIOV_CFG_TIMEOUT 6 int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev); void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 removal_type, struct auxiliary_device *aux_dev); void bnxt_re_destroy_lag(struct bnxt_re_dev **rdev); int bnxt_re_add_device(struct bnxt_re_dev **rdev, struct ifnet *netdev, u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested, struct auxiliary_device *aux_dev); void bnxt_re_create_base_interface(bool primary); int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, struct ifnet *vlan_dev, struct ifnet *netdev, struct auxiliary_device *aux_dev); void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev); int _bnxt_re_ib_init(struct bnxt_re_dev *rdev); int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev); void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev); void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev); /* The rdev ref_count is to protect immature removal of the device */ static inline void bnxt_re_hold(struct bnxt_re_dev *rdev) { atomic_inc(&rdev->ref_count); dev_dbg(rdev_to_dev(rdev), "Hold ref_count = 0x%x", atomic_read(&rdev->ref_count)); } static inline void bnxt_re_put(struct bnxt_re_dev *rdev) { atomic_dec(&rdev->ref_count); dev_dbg(rdev_to_dev(rdev), "Put ref_count = 0x%x", atomic_read(&rdev->ref_count)); } /* * Responder Error reason codes * FIXME: Remove these when the defs * are properly included in hsi header */ enum res_err_state_reason { /* No error. */ CFCQ_RES_ERR_STATE_REASON_NO_ERROR = 0, /* * Incoming Send, RDMA write, or RDMA read exceeds the maximum * transfer length. Detected on RX first and only packets for * write. Detected on RX request for read. This is an RX * Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX, /* * RDMA write payload size does not match write length. Detected * when total write payload is not equal to the RDMA write * length that was given in the first or only packet of the * request. This is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH, /* * Send payload exceeds RQ/SRQ WQE buffer capacity. The total * send payload that arrived is more than the size of the WQE * buffer that was fetched from the RQ/SRQ. This is an RX * Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE, /* * Responder detected opcode error. * First, only, middle, last * for incoming requests are improperly ordered with respect to * previous (PSN) packet. * First or middle packet is not full * MTU size. This is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR, /* * PSN sequence error retry limit exceeded. The responder * encountered a PSN sequence error for the same PSN too many * times. This can occur via implicit or explicit NAK. This is * an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT, /* * Invalid R_Key. An incoming request contained an R_Key that * did not reference a valid MR/MW. This error may be detected * by the RX engine for RDMA write or by the TX engine for RDMA * read (detected while servicing IRRQ). This is an RX Detected * Error. */ CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY, /* * Domain error. An incoming request specified an R_Key which * referenced a MR/MW that was not in the same PD as the QP on * which the request arrived. This is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR, /* * No permission. An incoming request contained an R_Key that * referenced a MR/MW which did not have the access permission * needed for the operation. This is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION, /* * Range error. An incoming request had a combination of R_Key, * VA, and length that was out of bounds of the associated * MR/MW. This is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR, /* * Invalid R_Key. An incoming request contained an R_Key that * did not reference a valid MR/MW. This error may be detected * by the RX engine for RDMA write or by the TX engine for RDMA * read (detected while servicing IRRQ). This is a TX Detected * Error. */ CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY, /* * Domain error. An incoming request specified an R_Key which * referenced a MR/MW that was not in the same PD as the QP on * which the request arrived. This is a TX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR, /* * No permission. An incoming request contained an R_Key that * referenced a MR/MW which did not have the access permission * needed for the operation. This is a TX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION, /* * Range error. An incoming request had a combination of R_Key, * VA, and length that was out of bounds of the associated * MR/MW. This is a TX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR, /* * IRRQ overflow. The peer sent us more RDMA read or atomic * requests than the negotiated maximum. This is an RX Detected * Error. */ CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW, /* * Unsupported opcode. The peer sent us a request with an opcode * for a request type that is not supported on this QP. This is * an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE, /* * Unaligned atomic operation. The VA of an atomic request is on * a memory boundary that prevents atomic execution. This is an * RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC, /* * Remote invalidate error. A send with invalidate request * arrived in which the R_Key to invalidate did not describe a * MR/MW which could be invalidated. RQ WQE completes with error * status. This error is only reported if the send operation did * not fail. If the send operation failed then the remote * invalidate error is not reported. This is an RX Detected * Error. */ CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE, /* * Local memory error. An RQ/SRQ SGE described an inaccessible * memory. This is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR, /* * SRQ in error. The QP is moving to error state because it * found SRQ it uses in error. This is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR, /* * Completion error. No CQE space available on queue or CQ not * in VALID state. This is a Completion Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR, /* * Invalid R_Key while resending responses to duplicate request. * This is a TX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY, /* * Problem was found in the format of a WQE in the RQ/SRQ. This * is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR, /* * A load error occurred on an attempt to load the CQ Context. * This is a Completion Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR = 0x18, /* * A load error occurred on an attempt to load the SRQ Context. * This is an RX Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR, /* * A fatal error was detected on an attempt to read from or * write to PCIe on the transmit side. This error is detected by * the TX side, but has the priority of a Completion Detected * Error. */ CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR = 0x1b, /* * A fatal error was detected on an attempt to read from or * write to PCIe on the receive side. This error is detected by * the RX side (or CAGR), but has the priority of a Completion * Detected Error. */ CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR = 0x1c }; int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, struct bnxt_qplib_query_fn_info *fn_info, u32 *pf_mask, u32 *first_pf); /* Default DCBx and CC values */ #define BNXT_RE_DEFAULT_CNP_DSCP 48 #define BNXT_RE_DEFAULT_CNP_PRI 7 #define BNXT_RE_DEFAULT_ROCE_DSCP 26 #define BNXT_RE_DEFAULT_ROCE_PRI 3 #define BNXT_RE_DEFAULT_L2_BW 50 #define BNXT_RE_DEFAULT_ROCE_BW 50 #define ROCE_PRIO_VALID 0x0 #define CNP_PRIO_VALID 0x1 #define ROCE_DSCP_VALID 0x0 #define CNP_DSCP_VALID 0x1 int bnxt_re_get_pri_dscp_settings(struct bnxt_re_dev *rdev, u16 target_id, struct bnxt_re_tc_rec *tc_rec); int bnxt_re_setup_dscp(struct bnxt_re_dev *rdev); int bnxt_re_clear_dscp(struct bnxt_re_dev *rdev); int bnxt_re_setup_cnp_cos(struct bnxt_re_dev *rdev, bool reset); static inline enum ib_port_state bnxt_re_get_link_state(struct bnxt_re_dev *rdev) { if (if_getdrvflags(rdev->netdev) & IFF_DRV_RUNNING && if_getlinkstate(rdev->netdev) == LINK_STATE_UP) return IB_PORT_ACTIVE; return IB_PORT_DOWN; } static inline int bnxt_re_link_state(struct bnxt_re_dev *rdev) { return bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE ? 1:0; } static inline int is_cc_enabled(struct bnxt_re_dev *rdev) { return rdev->cc_param.enable; } static inline void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr, u16 opcd, u16 crid, u16 trid) { hdr->req_type = cpu_to_le16(opcd); hdr->cmpl_ring = cpu_to_le16(crid); hdr->target_id = cpu_to_le16(trid); } static inline void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg, int msg_len, void *resp, int resp_max_len, int timeout) { fw_msg->msg = msg; fw_msg->msg_len = msg_len; fw_msg->resp = resp; fw_msg->resp_max_len = resp_max_len; fw_msg->timeout = timeout; } static inline bool is_qport_service_type_supported(struct bnxt_re_dev *rdev) { return rdev->tc_rec[0].serv_type_enabled; } static inline bool is_bnxt_roce_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type) { if (is_qport_service_type_supported(rdev)) return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_ROCE); else return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS_ROCE); } static inline bool is_bnxt_cnp_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type) { if (is_qport_service_type_supported(rdev)) return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_CNP); else return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY_ROCE_CNP); } #define BNXT_RE_MAP_SH_PAGE 0x0 #define BNXT_RE_MAP_WC 0x1 #define BNXT_RE_DBR_PAGE 0x2 #define BNXT_RE_MAP_DB_RECOVERY_PAGE 0x3 #define BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT (20) /* 20 ms */ #define BNXT_RE_DBR_INT_TIME 5 /* ms */ #define BNXT_RE_PACING_EN_INT_THRESHOLD 50 /* Entries in DB FIFO */ #define BNXT_RE_PACING_ALGO_THRESHOLD 250 /* Entries in DB FIFO */ /* Percentage of DB FIFO depth */ #define BNXT_RE_PACING_DBQ_THRESHOLD BNXT_RE_PACING_DBQ_HIGH_WATERMARK #define BNXT_RE_PACING_ALARM_TH_MULTIPLE(ctx) (_is_chip_p7(ctx) ? 0 : 2) /* * Maximum Percentage of configurable DB FIFO depth. * The Doorbell FIFO depth is 0x2c00. But the DBR_REG_DB_THROTTLING register has only 12 bits * to program the high watermark. This means user can configure maximum 36% only(4095/11264). */ #define BNXT_RE_PACING_DBQ_HIGH_WATERMARK 36 /* Default do_pacing value when there is no congestion */ #define BNXT_RE_DBR_DO_PACING_NO_CONGESTION 0x7F /* 1 in 512 probability */ enum { BNXT_RE_DBQ_EVENT_SCHED = 0, BNXT_RE_DBR_PACING_EVENT = 1, BNXT_RE_DBR_NQ_PACING_NOTIFICATION = 2, }; struct bnxt_re_dbq_work { struct work_struct work; struct bnxt_re_dev *rdev; struct hwrm_async_event_cmpl cmpl; u32 event; }; int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev); int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev); int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev); int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle); void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev); int bnxt_re_hwrm_pri2cos_qcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *tc_rec, u16 target_id); void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); static inline unsigned int bnxt_re_get_total_mr_mw_count(struct bnxt_re_dev *rdev) { return (atomic_read(&rdev->stats.rsors.mr_count) + atomic_read(&rdev->stats.rsors.mw_count)); } static inline void bnxt_re_set_def_pacing_threshold(struct bnxt_re_dev *rdev) { rdev->qplib_res.pacing_data->pacing_th = rdev->pacing_algo_th; rdev->qplib_res.pacing_data->alarm_th = rdev->pacing_algo_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); } static inline void bnxt_re_set_def_do_pacing(struct bnxt_re_dev *rdev) { rdev->qplib_res.pacing_data->do_pacing = rdev->dbr_def_do_pacing; } static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev) { rdev->qplib_res.pacing_data->dev_err_state = test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); } #endif