summaryrefslogtreecommitdiffstats
path: root/drivers/misc/mic/host/mic_smpt.c
blob: 50d1bebecd54286950c92a126106523c9726e64e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Intel MIC Platform Software Stack (MPSS)
 *
 * Copyright(c) 2013 Intel Corporation.
 *
 * Intel MIC Host driver.
 */
#include <linux/pci.h>

#include "../common/mic_dev.h"
#include "mic_device.h"
#include "mic_smpt.h"

static inline u64 mic_system_page_mask(struct mic_device *mdev)
{
	return (1ULL << mdev->smpt->info.page_shift) - 1ULL;
}

static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa)
{
	return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift;
}

static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index)
{
	return mdev->smpt->info.base + (index * mdev->smpt->info.page_size);
}

static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa)
{
	return pa & mic_system_page_mask(mdev);
}

static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa)
{
	return ALIGN(pa - mic_system_page_mask(mdev),
		mdev->smpt->info.page_size);
}

static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa)
{
	return ALIGN(pa, mdev->smpt->info.page_size);
}

/* Total Cumulative system memory accessible by MIC across all SMPT entries */
static inline u64 mic_max_system_memory(struct mic_device *mdev)
{
	return mdev->smpt->info.num_reg * mdev->smpt->info.page_size;
}

/* Maximum system memory address accessible by MIC */
static inline u64 mic_max_system_addr(struct mic_device *mdev)
{
	return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL;
}

/* Check if the DMA address is a MIC system memory address */
static inline bool
mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa)
{
	return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev);
}

/* Populate an SMPT entry and update the reference counts. */
static void mic_add_smpt_entry(int spt, s64 *ref, u64 addr,
			       int entries, struct mic_device *mdev)
{
	struct mic_smpt_info *smpt_info = mdev->smpt;
	int i;

	for (i = spt; i < spt + entries; i++,
		addr += smpt_info->info.page_size) {
		if (!smpt_info->entry[i].ref_count &&
		    (smpt_info->entry[i].dma_addr != addr)) {
			mdev->smpt_ops->set(mdev, addr, i);
			smpt_info->entry[i].dma_addr = addr;
		}
		smpt_info->entry[i].ref_count += ref[i - spt];
	}
}

/*
 * Find an available MIC address in MIC SMPT address space
 * for a given DMA address and size.
 */
static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr,
			      int entries, s64 *ref, size_t size)
{
	int spt;
	int ae = 0;
	int i;
	unsigned long flags;
	dma_addr_t mic_addr = 0;
	dma_addr_t addr = dma_addr;
	struct mic_smpt_info *smpt_info = mdev->smpt;

	spin_lock_irqsave(&smpt_info->smpt_lock, flags);

	/* find existing entries */
	for (i = 0; i < smpt_info->info.num_reg; i++) {
		if (smpt_info->entry[i].dma_addr == addr) {
			ae++;
			addr += smpt_info->info.page_size;
		} else if (ae) /* cannot find contiguous entries */
			goto not_found;

		if (ae == entries)
			goto found;
	}

	/* find free entry */
	for (ae = 0, i = 0; i < smpt_info->info.num_reg; i++) {
		ae = (smpt_info->entry[i].ref_count == 0) ? ae + 1 : 0;
		if (ae == entries)
			goto found;
	}

not_found:
	spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
	return mic_addr;

found:
	spt = i - entries + 1;
	mic_addr = mic_smpt_to_pa(mdev, spt);
	mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev);
	smpt_info->map_count++;
	smpt_info->ref_count += (s64)size;
	spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
	return mic_addr;
}

/*
 * Returns number of smpt entries needed for dma_addr to dma_addr + size
 * also returns the reference count array for each of those entries
 * and the starting smpt address
 */
static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr,
				  size_t size, s64 *ref,  u64 *smpt_start)
{
	u64 start =  dma_addr;
	u64 end = dma_addr + size;
	int i = 0;

	while (start < end) {
		ref[i++] = min(mic_smpt_align_high(mdev, start + 1),
			end) - start;
		start = mic_smpt_align_high(mdev, start + 1);
	}

	if (smpt_start)
		*smpt_start = mic_smpt_align_low(mdev, dma_addr);

	return i;
}

/*
 * mic_to_dma_addr - Converts a MIC address to a DMA address.
 *
 * @mdev: pointer to mic_device instance.
 * @mic_addr: MIC address.
 *
 * returns a DMA address.
 */
dma_addr_t mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr)
{
	struct mic_smpt_info *smpt_info = mdev->smpt;
	int spt;
	dma_addr_t dma_addr;

	if (!mic_is_system_addr(mdev, mic_addr)) {
		dev_err(&mdev->pdev->dev,
			"mic_addr is invalid. mic_addr = 0x%llx\n", mic_addr);
		return -EINVAL;
	}
	spt = mic_sys_addr_to_smpt(mdev, mic_addr);
	dma_addr = smpt_info->entry[spt].dma_addr +
		mic_smpt_offset(mdev, mic_addr);
	return dma_addr;
}

/**
 * mic_map - Maps a DMA address to a MIC physical address.
 *
 * @mdev: pointer to mic_device instance.
 * @dma_addr: DMA address.
 * @size: Size of the region to be mapped.
 *
 * This API converts the DMA address provided to a DMA address understood
 * by MIC. Caller should check for errors by calling mic_map_error(..).
 *
 * returns DMA address as required by MIC.
 */
dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size)
{
	dma_addr_t mic_addr = 0;
	int num_entries;
	s64 *ref;
	u64 smpt_start;

	if (!size || size > mic_max_system_memory(mdev))
		return mic_addr;

	ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);
	if (!ref)
		return mic_addr;

	num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size,
					     ref, &smpt_start);

	/* Set the smpt table appropriately and get 16G aligned mic address */
	mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size);

	kfree(ref);

	/*
	 * If mic_addr is zero then its an error case
	 * since mic_addr can never be zero.
	 * else generate mic_addr by adding the 16G offset in dma_addr
	 */
	if (!mic_addr && MIC_FAMILY_X100 == mdev->family) {
		dev_err(&mdev->pdev->dev,
			"mic_map failed dma_addr 0x%llx size 0x%lx\n",
			dma_addr, size);
		return mic_addr;
	} else {
		return mic_addr + mic_smpt_offset(mdev, dma_addr);
	}
}

/**
 * mic_unmap - Unmaps a MIC physical address.
 *
 * @mdev: pointer to mic_device instance.
 * @mic_addr: MIC physical address.
 * @size: Size of the region to be unmapped.
 *
 * This API unmaps the mappings created by mic_map(..).
 *
 * returns None.
 */
void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)
{
	struct mic_smpt_info *smpt_info = mdev->smpt;
	s64 *ref;
	int num_smpt;
	int spt;
	int i;
	unsigned long flags;

	if (!size)
		return;

	if (!mic_is_system_addr(mdev, mic_addr)) {
		dev_err(&mdev->pdev->dev,
			"invalid address: 0x%llx\n", mic_addr);
		return;
	}

	spt = mic_sys_addr_to_smpt(mdev, mic_addr);
	ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);
	if (!ref)
		return;

	/* Get number of smpt entries to be mapped, ref count array */
	num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL);

	spin_lock_irqsave(&smpt_info->smpt_lock, flags);
	smpt_info->unmap_count++;
	smpt_info->ref_count -= (s64)size;

	for (i = spt; i < spt + num_smpt; i++) {
		smpt_info->entry[i].ref_count -= ref[i - spt];
		if (smpt_info->entry[i].ref_count < 0)
			dev_warn(&mdev->pdev->dev,
				 "ref count for entry %d is negative\n", i);
	}
	spin_unlock_irqrestore(&smpt_info->smpt_lock, flags);
	kfree(ref);
}

/**
 * mic_map_single - Maps a virtual address to a MIC physical address.
 *
 * @mdev: pointer to mic_device instance.
 * @va: Kernel direct mapped virtual address.
 * @size: Size of the region to be mapped.
 *
 * This API calls pci_map_single(..) for the direct mapped virtual address
 * and then converts the DMA address provided to a DMA address understood
 * by MIC. Caller should check for errors by calling mic_map_error(..).
 *
 * returns DMA address as required by MIC.
 */
dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size)
{
	dma_addr_t mic_addr = 0;
	struct pci_dev *pdev = mdev->pdev;
	dma_addr_t dma_addr =
		pci_map_single(pdev, va, size, PCI_DMA_BIDIRECTIONAL);

	if (!pci_dma_mapping_error(pdev, dma_addr)) {
		mic_addr = mic_map(mdev, dma_addr, size);
		if (!mic_addr) {
			dev_err(&mdev->pdev->dev,
				"mic_map failed dma_addr 0x%llx size 0x%lx\n",
				dma_addr, size);
			pci_unmap_single(pdev, dma_addr,
					 size, PCI_DMA_BIDIRECTIONAL);
		}
	}
	return mic_addr;
}

/**
 * mic_unmap_single - Unmaps a MIC physical address.
 *
 * @mdev: pointer to mic_device instance.
 * @mic_addr: MIC physical address.
 * @size: Size of the region to be unmapped.
 *
 * This API unmaps the mappings created by mic_map_single(..).
 *
 * returns None.
 */
void
mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)
{
	struct pci_dev *pdev = mdev->pdev;
	dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr);
	mic_unmap(mdev, mic_addr, size);
	pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
}

/**
 * mic_smpt_init - Initialize MIC System Memory Page Tables.
 *
 * @mdev: pointer to mic_device instance.
 *
 * returns 0 for success and -errno for error.
 */
int mic_smpt_init(struct mic_device *mdev)
{
	int i, err = 0;
	dma_addr_t dma_addr;
	struct mic_smpt_info *smpt_info;

	mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL);
	if (!mdev->smpt)
		return -ENOMEM;

	smpt_info = mdev->smpt;
	mdev->smpt_ops->init(mdev);
	smpt_info->entry = kmalloc_array(smpt_info->info.num_reg,
					 sizeof(*smpt_info->entry), GFP_KERNEL);
	if (!smpt_info->entry) {
		err = -ENOMEM;
		goto free_smpt;
	}
	spin_lock_init(&smpt_info->smpt_lock);
	for (i = 0; i < smpt_info->info.num_reg; i++) {
		dma_addr = i * smpt_info->info.page_size;
		smpt_info->entry[i].dma_addr = dma_addr;
		smpt_info->entry[i].ref_count = 0;
		mdev->smpt_ops->set(mdev, dma_addr, i);
	}
	smpt_info->ref_count = 0;
	smpt_info->map_count = 0;
	smpt_info->unmap_count = 0;
	return 0;
free_smpt:
	kfree(smpt_info);
	return err;
}

/**
 * mic_smpt_uninit - UnInitialize MIC System Memory Page Tables.
 *
 * @mdev: pointer to mic_device instance.
 *
 * returns None.
 */
void mic_smpt_uninit(struct mic_device *mdev)
{
	struct mic_smpt_info *smpt_info = mdev->smpt;
	int i;

	dev_dbg(&mdev->pdev->dev,
		"nodeid %d SMPT ref count %lld map %lld unmap %lld\n",
		mdev->id, smpt_info->ref_count,
		smpt_info->map_count, smpt_info->unmap_count);

	for (i = 0; i < smpt_info->info.num_reg; i++) {
		dev_dbg(&mdev->pdev->dev,
			"SMPT entry[%d] dma_addr = 0x%llx ref_count = %lld\n",
			i, smpt_info->entry[i].dma_addr,
			smpt_info->entry[i].ref_count);
		if (smpt_info->entry[i].ref_count)
			dev_warn(&mdev->pdev->dev,
				 "ref count for entry %d is not zero\n", i);
	}
	kfree(smpt_info->entry);
	kfree(smpt_info);
}

/**
 * mic_smpt_restore - Restore MIC System Memory Page Tables.
 *
 * @mdev: pointer to mic_device instance.
 *
 * Restore the SMPT registers to values previously stored in the
 * SW data structures. Some MIC steppings lose register state
 * across resets and this API should be called for performing
 * a restore operation if required.
 *
 * returns None.
 */
void mic_smpt_restore(struct mic_device *mdev)
{
	int i;
	dma_addr_t dma_addr;

	for (i = 0; i < mdev->smpt->info.num_reg; i++) {
		dma_addr = mdev->smpt->entry[i].dma_addr;
		mdev->smpt_ops->set(mdev, dma_addr, i);
	}
}