1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
|
/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */
/*
* Copyright(c) 2020 - Cornelis Networks, Inc.
* Copyright(c) 2015 - 2018 Intel Corporation.
*/
#ifndef _HFI1_USER_SDMA_H
#define _HFI1_USER_SDMA_H
#include <linux/device.h>
#include <linux/wait.h>
#include "common.h"
#include "iowait.h"
#include "user_exp_rcv.h"
#include "mmu_rb.h"
/* The maximum number of Data io vectors per message/request */
#define MAX_VECTORS_PER_REQ 8
/*
* Maximum number of packet to send from each message/request
* before moving to the next one.
*/
#define MAX_PKTS_PER_QUEUE 16
#define num_pages(x) (1 + ((((x) - 1) & PAGE_MASK) >> PAGE_SHIFT))
#define req_opcode(x) \
(((x) >> HFI1_SDMA_REQ_OPCODE_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
#define req_version(x) \
(((x) >> HFI1_SDMA_REQ_VERSION_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
#define req_iovcnt(x) \
(((x) >> HFI1_SDMA_REQ_IOVCNT_SHIFT) & HFI1_SDMA_REQ_IOVCNT_MASK)
/* Number of BTH.PSN bits used for sequence number in expected rcvs */
#define BTH_SEQ_MASK 0x7ffull
#define AHG_KDETH_INTR_SHIFT 12
#define AHG_KDETH_SH_SHIFT 13
#define AHG_KDETH_ARRAY_SIZE 9
#define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4)
#define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff)
/**
* Build an SDMA AHG header update descriptor and save it to an array.
* @arr - Array to save the descriptor to.
* @idx - Index of the array at which the descriptor will be saved.
* @array_size - Size of the array arr.
* @dw - Update index into the header in DWs.
* @bit - Start bit.
* @width - Field width.
* @value - 16 bits of immediate data to write into the field.
* Returns -ERANGE if idx is invalid. If successful, returns the next index
* (idx + 1) of the array to be used for the next descriptor.
*/
static inline int ahg_header_set(u32 *arr, int idx, size_t array_size,
u8 dw, u8 bit, u8 width, u16 value)
{
if ((size_t)idx >= array_size)
return -ERANGE;
arr[idx++] = sdma_build_ahg_descriptor(value, dw, bit, width);
return idx;
}
/* Tx request flag bits */
#define TXREQ_FLAGS_REQ_ACK BIT(0) /* Set the ACK bit in the header */
#define TXREQ_FLAGS_REQ_DISABLE_SH BIT(1) /* Disable header suppression */
enum pkt_q_sdma_state {
SDMA_PKT_Q_ACTIVE,
SDMA_PKT_Q_DEFERRED,
};
#define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */
#define SDMA_DBG(req, fmt, ...) \
hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \
(req)->pq->ctxt, (req)->pq->subctxt, (req)->info.comp_idx, \
##__VA_ARGS__)
struct hfi1_user_sdma_pkt_q {
u16 ctxt;
u16 subctxt;
u16 n_max_reqs;
atomic_t n_reqs;
u16 reqidx;
struct hfi1_devdata *dd;
struct kmem_cache *txreq_cache;
struct user_sdma_request *reqs;
unsigned long *req_in_use;
struct iowait busy;
enum pkt_q_sdma_state state;
wait_queue_head_t wait;
unsigned long unpinned;
struct mmu_rb_handler *handler;
atomic_t n_locked;
};
struct hfi1_user_sdma_comp_q {
u16 nentries;
struct hfi1_sdma_comp_entry *comps;
};
struct sdma_mmu_node {
struct mmu_rb_node rb;
struct hfi1_user_sdma_pkt_q *pq;
atomic_t refcount;
struct page **pages;
unsigned int npages;
};
struct user_sdma_iovec {
struct list_head list;
struct iovec iov;
/* number of pages in this vector */
unsigned int npages;
/* array of pinned pages for this vector */
struct page **pages;
/*
* offset into the virtual address space of the vector at
* which we last left off.
*/
u64 offset;
struct sdma_mmu_node *node;
};
/* evict operation argument */
struct evict_data {
u32 cleared; /* count evicted so far */
u32 target; /* target count to evict */
};
struct user_sdma_request {
/* This is the original header from user space */
struct hfi1_pkt_header hdr;
/* Read mostly fields */
struct hfi1_user_sdma_pkt_q *pq ____cacheline_aligned_in_smp;
struct hfi1_user_sdma_comp_q *cq;
/*
* Pointer to the SDMA engine for this request.
* Since different request could be on different VLs,
* each request will need it's own engine pointer.
*/
struct sdma_engine *sde;
struct sdma_req_info info;
/* TID array values copied from the tid_iov vector */
u32 *tids;
/* total length of the data in the request */
u32 data_len;
/* number of elements copied to the tids array */
u16 n_tids;
/*
* We copy the iovs for this request (based on
* info.iovcnt). These are only the data vectors
*/
u8 data_iovs;
s8 ahg_idx;
/* Writeable fields shared with interrupt */
u16 seqcomp ____cacheline_aligned_in_smp;
u16 seqsubmitted;
/* Send side fields */
struct list_head txps ____cacheline_aligned_in_smp;
u16 seqnum;
/*
* KDETH.OFFSET (TID) field
* The offset can cover multiple packets, depending on the
* size of the TID entry.
*/
u32 tidoffset;
/*
* KDETH.Offset (Eager) field
* We need to remember the initial value so the headers
* can be updated properly.
*/
u32 koffset;
u32 sent;
/* TID index copied from the tid_iov vector */
u16 tididx;
/* progress index moving along the iovs array */
u8 iov_idx;
u8 has_error;
struct user_sdma_iovec iovs[MAX_VECTORS_PER_REQ];
} ____cacheline_aligned_in_smp;
/*
* A single txreq could span up to 3 physical pages when the MTU
* is sufficiently large (> 4K). Each of the IOV pointers also
* needs it's own set of flags so the vector has been handled
* independently of each other.
*/
struct user_sdma_txreq {
/* Packet header for the txreq */
struct hfi1_pkt_header hdr;
struct sdma_txreq txreq;
struct list_head list;
struct user_sdma_request *req;
u16 flags;
u16 seqnum;
};
int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
struct hfi1_filedata *fd);
int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd,
struct hfi1_ctxtdata *uctxt);
int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
struct iovec *iovec, unsigned long dim,
unsigned long *count);
static inline struct mm_struct *mm_from_sdma_node(struct sdma_mmu_node *node)
{
return node->rb.handler->mn.mm;
}
#endif /* _HFI1_USER_SDMA_H */
|
