1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
|
/*
* cxgb4_ppm.h: Chelsio common library for T4/T5 iSCSI ddp operation
*
* Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* Written by: Karen Xie (kxie@chelsio.com)
*/
#ifndef __CXGB4PPM_H__
#define __CXGB4PPM_H__
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/debugfs.h>
#include <linux/list.h>
#include <linux/netdevice.h>
#include <linux/scatterlist.h>
#include <linux/skbuff.h>
#include <linux/vmalloc.h>
#include <linux/bitmap.h>
struct cxgbi_pagepod_hdr {
u32 vld_tid;
u32 pgsz_tag_clr;
u32 max_offset;
u32 page_offset;
u64 rsvd;
};
#define PPOD_PAGES_MAX 4
struct cxgbi_pagepod {
struct cxgbi_pagepod_hdr hdr;
u64 addr[PPOD_PAGES_MAX + 1];
};
/* ddp tag format
* for a 32-bit tag:
* bit #
* 31 ..... ..... 0
* X Y...Y Z...Z, where
* ^ ^^^^^ ^^^^
* | | |____ when ddp bit = 0: color bits
* | |
* | |____ when ddp bit = 0: idx into the ddp memory region
* |
* |____ ddp bit: 0 - ddp tag, 1 - non-ddp tag
*
* [page selector:2] [sw/free bits] [0] [idx] [color:6]
*/
#define DDP_PGIDX_MAX 4
#define DDP_PGSZ_BASE_SHIFT 12 /* base page 4K */
struct cxgbi_task_tag_info {
unsigned char flags;
#define CXGBI_PPOD_INFO_FLAG_VALID 0x1
#define CXGBI_PPOD_INFO_FLAG_MAPPED 0x2
unsigned char cid;
unsigned short pg_shift;
unsigned int npods;
unsigned int idx;
unsigned int tag;
struct cxgbi_pagepod_hdr hdr;
int nents;
int nr_pages;
struct scatterlist *sgl;
};
struct cxgbi_tag_format {
unsigned char pgsz_order[DDP_PGIDX_MAX];
unsigned char pgsz_idx_dflt;
unsigned char free_bits:4;
unsigned char color_bits:4;
unsigned char idx_bits;
unsigned char rsvd_bits;
unsigned int no_ddp_mask;
unsigned int idx_mask;
unsigned int color_mask;
unsigned int idx_clr_mask;
unsigned int rsvd_mask;
};
struct cxgbi_ppod_data {
unsigned char pg_idx:2;
unsigned char color:6;
unsigned char chan_id;
unsigned short npods;
unsigned long caller_data;
};
/* per cpu ppm pool */
struct cxgbi_ppm_pool {
unsigned int base; /* base index */
unsigned int next; /* next possible free index */
spinlock_t lock; /* ppm pool lock */
unsigned long bmap[0];
} ____cacheline_aligned_in_smp;
struct cxgbi_ppm {
struct kref refcnt;
struct net_device *ndev; /* net_device, 1st port */
struct pci_dev *pdev;
void *lldev;
void **ppm_pp;
struct cxgbi_tag_format tformat;
unsigned int ppmax;
unsigned int llimit;
unsigned int base_idx;
unsigned int pool_rsvd;
unsigned int pool_index_max;
struct cxgbi_ppm_pool __percpu *pool;
/* map lock */
spinlock_t map_lock; /* ppm map lock */
unsigned int bmap_index_max;
unsigned int next;
unsigned long *ppod_bmap;
struct cxgbi_ppod_data ppod_data[0];
};
#define DDP_THRESHOLD 512
#define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */
#define IPPOD_SIZE sizeof(struct cxgbi_pagepod) /* 64 */
#define PPOD_SIZE_SHIFT 6
/* page pods are allocated in groups of this size (must be power of 2) */
#define PPOD_CLUSTER_SIZE 16U
#define ULPMEM_DSGL_MAX_NPPODS 16 /* 1024/PPOD_SIZE */
#define ULPMEM_IDATA_MAX_NPPODS 3 /* (PPOD_SIZE * 3 + ulptx hdr) < 256B */
#define PCIE_MEMWIN_MAX_NPPODS 16 /* 1024/PPOD_SIZE */
#define PPOD_COLOR_SHIFT 0
#define PPOD_COLOR(x) ((x) << PPOD_COLOR_SHIFT)
#define PPOD_IDX_SHIFT 6
#define PPOD_IDX_MAX_SIZE 24
#define PPOD_TID_SHIFT 0
#define PPOD_TID(x) ((x) << PPOD_TID_SHIFT)
#define PPOD_TAG_SHIFT 6
#define PPOD_TAG(x) ((x) << PPOD_TAG_SHIFT)
#define PPOD_VALID_SHIFT 24
#define PPOD_VALID(x) ((x) << PPOD_VALID_SHIFT)
#define PPOD_VALID_FLAG PPOD_VALID(1U)
#define PPOD_PI_EXTRACT_CTL_SHIFT 31
#define PPOD_PI_EXTRACT_CTL(x) ((x) << PPOD_PI_EXTRACT_CTL_SHIFT)
#define PPOD_PI_EXTRACT_CTL_FLAG V_PPOD_PI_EXTRACT_CTL(1U)
#define PPOD_PI_TYPE_SHIFT 29
#define PPOD_PI_TYPE_MASK 0x3
#define PPOD_PI_TYPE(x) ((x) << PPOD_PI_TYPE_SHIFT)
#define PPOD_PI_CHECK_CTL_SHIFT 27
#define PPOD_PI_CHECK_CTL_MASK 0x3
#define PPOD_PI_CHECK_CTL(x) ((x) << PPOD_PI_CHECK_CTL_SHIFT)
#define PPOD_PI_REPORT_CTL_SHIFT 25
#define PPOD_PI_REPORT_CTL_MASK 0x3
#define PPOD_PI_REPORT_CTL(x) ((x) << PPOD_PI_REPORT_CTL_SHIFT)
static inline int cxgbi_ppm_is_ddp_tag(struct cxgbi_ppm *ppm, u32 tag)
{
return !(tag & ppm->tformat.no_ddp_mask);
}
static inline int cxgbi_ppm_sw_tag_is_usable(struct cxgbi_ppm *ppm,
u32 tag)
{
/* the sw tag must be using <= 31 bits */
return !(tag & 0x80000000U);
}
static inline int cxgbi_ppm_make_non_ddp_tag(struct cxgbi_ppm *ppm,
u32 sw_tag,
u32 *final_tag)
{
struct cxgbi_tag_format *tformat = &ppm->tformat;
if (!cxgbi_ppm_sw_tag_is_usable(ppm, sw_tag)) {
pr_info("sw_tag 0x%x NOT usable.\n", sw_tag);
return -EINVAL;
}
if (!sw_tag) {
*final_tag = tformat->no_ddp_mask;
} else {
unsigned int shift = tformat->idx_bits + tformat->color_bits;
u32 lower = sw_tag & tformat->idx_clr_mask;
u32 upper = (sw_tag >> shift) << (shift + 1);
*final_tag = upper | tformat->no_ddp_mask | lower;
}
return 0;
}
static inline u32 cxgbi_ppm_decode_non_ddp_tag(struct cxgbi_ppm *ppm,
u32 tag)
{
struct cxgbi_tag_format *tformat = &ppm->tformat;
unsigned int shift = tformat->idx_bits + tformat->color_bits;
u32 lower = tag & tformat->idx_clr_mask;
u32 upper = (tag >> tformat->rsvd_bits) << shift;
return upper | lower;
}
static inline u32 cxgbi_ppm_ddp_tag_get_idx(struct cxgbi_ppm *ppm,
u32 ddp_tag)
{
u32 hw_idx = (ddp_tag >> PPOD_IDX_SHIFT) &
ppm->tformat.idx_mask;
return hw_idx - ppm->base_idx;
}
static inline u32 cxgbi_ppm_make_ddp_tag(unsigned int hw_idx,
unsigned char color)
{
return (hw_idx << PPOD_IDX_SHIFT) | ((u32)color);
}
static inline unsigned long
cxgbi_ppm_get_tag_caller_data(struct cxgbi_ppm *ppm,
u32 ddp_tag)
{
u32 idx = cxgbi_ppm_ddp_tag_get_idx(ppm, ddp_tag);
return ppm->ppod_data[idx].caller_data;
}
/* sw bits are the free bits */
static inline int cxgbi_ppm_ddp_tag_update_sw_bits(struct cxgbi_ppm *ppm,
u32 val, u32 orig_tag,
u32 *final_tag)
{
struct cxgbi_tag_format *tformat = &ppm->tformat;
u32 v = val >> tformat->free_bits;
if (v) {
pr_info("sw_bits 0x%x too large, avail bits %u.\n",
val, tformat->free_bits);
return -EINVAL;
}
if (!cxgbi_ppm_is_ddp_tag(ppm, orig_tag))
return -EINVAL;
*final_tag = (val << tformat->rsvd_bits) |
(orig_tag & ppm->tformat.rsvd_mask);
return 0;
}
static inline void cxgbi_ppm_ppod_clear(struct cxgbi_pagepod *ppod)
{
ppod->hdr.vld_tid = 0U;
}
static inline void cxgbi_tagmask_check(unsigned int tagmask,
struct cxgbi_tag_format *tformat)
{
unsigned int bits = fls(tagmask);
/* reserve top most 2 bits for page selector */
tformat->free_bits = 32 - 2 - bits;
tformat->rsvd_bits = bits;
tformat->color_bits = PPOD_IDX_SHIFT;
tformat->idx_bits = bits - 1 - PPOD_IDX_SHIFT;
tformat->no_ddp_mask = 1 << (bits - 1);
tformat->idx_mask = (1 << tformat->idx_bits) - 1;
tformat->color_mask = (1 << PPOD_IDX_SHIFT) - 1;
tformat->idx_clr_mask = (1 << (bits - 1)) - 1;
tformat->rsvd_mask = (1 << bits) - 1;
pr_info("ippm: tagmask 0x%x, rsvd %u=%u+%u+1, mask 0x%x,0x%x, "
"pg %u,%u,%u,%u.\n",
tagmask, tformat->rsvd_bits, tformat->idx_bits,
tformat->color_bits, tformat->no_ddp_mask, tformat->rsvd_mask,
tformat->pgsz_order[0], tformat->pgsz_order[1],
tformat->pgsz_order[2], tformat->pgsz_order[3]);
}
int cxgbi_ppm_find_page_index(struct cxgbi_ppm *ppm, unsigned long pgsz);
void cxgbi_ppm_make_ppod_hdr(struct cxgbi_ppm *ppm, u32 tag,
unsigned int tid, unsigned int offset,
unsigned int length,
struct cxgbi_pagepod_hdr *hdr);
void cxgbi_ppm_ppod_release(struct cxgbi_ppm *, u32 idx);
int cxgbi_ppm_ppods_reserve(struct cxgbi_ppm *, unsigned short nr_pages,
u32 per_tag_pg_idx, u32 *ppod_idx, u32 *ddp_tag,
unsigned long caller_data);
int cxgbi_ppm_init(void **ppm_pp, struct net_device *, struct pci_dev *,
void *lldev, struct cxgbi_tag_format *,
unsigned int ppmax, unsigned int llimit,
unsigned int start,
unsigned int reserve_factor);
int cxgbi_ppm_release(struct cxgbi_ppm *ppm);
void cxgbi_tagmask_check(unsigned int tagmask, struct cxgbi_tag_format *);
unsigned int cxgbi_tagmask_set(unsigned int ppmax);
#endif /*__CXGB4PPM_H__*/
|
