summaryrefslogtreecommitdiffstats
path: root/drivers/crypto/ccree/cc_request_mgr.c
blob: 33fb27745d522fbfa407f5df270d040543325c9b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */

#include <linux/kernel.h>
#include <linux/nospec.h>
#include "cc_driver.h"
#include "cc_buffer_mgr.h"
#include "cc_request_mgr.h"
#include "cc_pm.h"

#define CC_MAX_POLL_ITER	10
/* The highest descriptor count in used */
#define CC_MAX_DESC_SEQ_LEN	23

struct cc_req_mgr_handle {
	/* Request manager resources */
	unsigned int hw_queue_size; /* HW capability */
	unsigned int min_free_hw_slots;
	unsigned int max_used_sw_slots;
	struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
	u32 req_queue_head;
	u32 req_queue_tail;
	u32 axi_completed;
	u32 q_free_slots;
	/* This lock protects access to HW register
	 * that must be single request at a time
	 */
	spinlock_t hw_lock;
	struct cc_hw_desc compl_desc;
	u8 *dummy_comp_buff;
	dma_addr_t dummy_comp_buff_dma;

	/* backlog queue */
	struct list_head backlog;
	unsigned int bl_len;
	spinlock_t bl_lock; /* protect backlog queue */

#ifdef COMP_IN_WQ
	struct workqueue_struct *workq;
	struct delayed_work compwork;
#else
	struct tasklet_struct comptask;
#endif
};

struct cc_bl_item {
	struct cc_crypto_req creq;
	struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];
	unsigned int len;
	struct list_head list;
	bool notif;
};

static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = {
	{ BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) },
	{ BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT),
	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) }
};

static void comp_handler(unsigned long devarg);
#ifdef COMP_IN_WQ
static void comp_work_handler(struct work_struct *work);
#endif

static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot)
{
	alg = array_index_nospec(alg, CC_CPP_NUM_ALGS);
	slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS);

	return cc_cpp_int_masks[alg][slot];
}

void cc_req_mgr_fini(struct cc_drvdata *drvdata)
{
	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
	struct device *dev = drvdata_to_dev(drvdata);

	if (!req_mgr_h)
		return; /* Not allocated */

	if (req_mgr_h->dummy_comp_buff_dma) {
		dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,
				  req_mgr_h->dummy_comp_buff_dma);
	}

	dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
						req_mgr_h->min_free_hw_slots));
	dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);

#ifdef COMP_IN_WQ
	flush_workqueue(req_mgr_h->workq);
	destroy_workqueue(req_mgr_h->workq);
#else
	/* Kill tasklet */
	tasklet_kill(&req_mgr_h->comptask);
#endif
	kfree_sensitive(req_mgr_h);
	drvdata->request_mgr_handle = NULL;
}

int cc_req_mgr_init(struct cc_drvdata *drvdata)
{
	struct cc_req_mgr_handle *req_mgr_h;
	struct device *dev = drvdata_to_dev(drvdata);
	int rc = 0;

	req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);
	if (!req_mgr_h) {
		rc = -ENOMEM;
		goto req_mgr_init_err;
	}

	drvdata->request_mgr_handle = req_mgr_h;

	spin_lock_init(&req_mgr_h->hw_lock);
	spin_lock_init(&req_mgr_h->bl_lock);
	INIT_LIST_HEAD(&req_mgr_h->backlog);

#ifdef COMP_IN_WQ
	dev_dbg(dev, "Initializing completion workqueue\n");
	req_mgr_h->workq = create_singlethread_workqueue("ccree");
	if (!req_mgr_h->workq) {
		dev_err(dev, "Failed creating work queue\n");
		rc = -ENOMEM;
		goto req_mgr_init_err;
	}
	INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
#else
	dev_dbg(dev, "Initializing completion tasklet\n");
	tasklet_init(&req_mgr_h->comptask, comp_handler,
		     (unsigned long)drvdata);
#endif
	req_mgr_h->hw_queue_size = cc_ioread(drvdata,
					     CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));
	dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
	if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
		dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",
			req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
		rc = -ENOMEM;
		goto req_mgr_init_err;
	}
	req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
	req_mgr_h->max_used_sw_slots = 0;

	/* Allocate DMA word for "dummy" completion descriptor use */
	req_mgr_h->dummy_comp_buff =
		dma_alloc_coherent(dev, sizeof(u32),
				   &req_mgr_h->dummy_comp_buff_dma,
				   GFP_KERNEL);
	if (!req_mgr_h->dummy_comp_buff) {
		dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",
			sizeof(u32));
		rc = -ENOMEM;
		goto req_mgr_init_err;
	}

	/* Init. "dummy" completion descriptor */
	hw_desc_init(&req_mgr_h->compl_desc);
	set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));
	set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,
		      sizeof(u32), NS_BIT, 1);
	set_flow_mode(&req_mgr_h->compl_desc, BYPASS);
	set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);

	return 0;

req_mgr_init_err:
	cc_req_mgr_fini(drvdata);
	return rc;
}

static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],
			unsigned int seq_len)
{
	int i, w;
	void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);
	struct device *dev = drvdata_to_dev(drvdata);

	/*
	 * We do indeed write all 6 command words to the same
	 * register. The HW supports this.
	 */

	for (i = 0; i < seq_len; i++) {
		for (w = 0; w <= 5; w++)
			writel_relaxed(seq[i].word[w], reg);

		if (cc_dump_desc)
			dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
				i, seq[i].word[0], seq[i].word[1],
				seq[i].word[2], seq[i].word[3],
				seq[i].word[4], seq[i].word[5]);
	}
}

/**
 * request_mgr_complete() - Completion will take place if and only if user
 * requested completion by cc_send_sync_request().
 *
 * @dev: Device pointer
 * @dx_compl_h: The completion event to signal
 * @dummy: unused error code
 */
static void request_mgr_complete(struct device *dev, void *dx_compl_h,
				 int dummy)
{
	struct completion *this_compl = dx_compl_h;

	complete(this_compl);
}

static int cc_queues_status(struct cc_drvdata *drvdata,
			    struct cc_req_mgr_handle *req_mgr_h,
			    unsigned int total_seq_len)
{
	unsigned long poll_queue;
	struct device *dev = drvdata_to_dev(drvdata);

	/* SW queue is checked only once as it will not
	 * be changed during the poll because the spinlock_bh
	 * is held by the thread
	 */
	if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==
	    req_mgr_h->req_queue_tail) {
		dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
			req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
		return -ENOSPC;
	}

	if (req_mgr_h->q_free_slots >= total_seq_len)
		return 0;

	/* Wait for space in HW queue. Poll constant num of iterations. */
	for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {
		req_mgr_h->q_free_slots =
			cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
		if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)
			req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;

		if (req_mgr_h->q_free_slots >= total_seq_len) {
			/* If there is enough place return */
			return 0;
		}

		dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
			req_mgr_h->q_free_slots, total_seq_len);
	}
	/* No room in the HW queue try again later */
	dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
		req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,
		req_mgr_h->q_free_slots, total_seq_len);
	return -ENOSPC;
}

/**
 * cc_do_send_request() - Enqueue caller request to crypto hardware.
 * Need to be called with HW lock held and PM running
 *
 * @drvdata: Associated device driver context
 * @cc_req: The request to enqueue
 * @desc: The crypto sequence
 * @len: The crypto sequence length
 * @add_comp: If "true": add an artificial dout DMA to mark completion
 *
 */
static void cc_do_send_request(struct cc_drvdata *drvdata,
			       struct cc_crypto_req *cc_req,
			       struct cc_hw_desc *desc, unsigned int len,
			       bool add_comp)
{
	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
	unsigned int used_sw_slots;
	unsigned int total_seq_len = len; /*initial sequence length*/
	struct device *dev = drvdata_to_dev(drvdata);

	used_sw_slots = ((req_mgr_h->req_queue_head -
			  req_mgr_h->req_queue_tail) &
			 (MAX_REQUEST_QUEUE_SIZE - 1));
	if (used_sw_slots > req_mgr_h->max_used_sw_slots)
		req_mgr_h->max_used_sw_slots = used_sw_slots;

	/* Enqueue request - must be locked with HW lock*/
	req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;
	req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &
				    (MAX_REQUEST_QUEUE_SIZE - 1);

	dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);

	/*
	 * We are about to push command to the HW via the command registers
	 * that may reference host memory. We need to issue a memory barrier
	 * to make sure there are no outstanding memory writes
	 */
	wmb();

	/* STAT_PHASE_4: Push sequence */

	enqueue_seq(drvdata, desc, len);

	if (add_comp) {
		enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);
		total_seq_len++;
	}

	if (req_mgr_h->q_free_slots < total_seq_len) {
		/* This situation should never occur. Maybe indicating problem
		 * with resuming power. Set the free slot count to 0 and hope
		 * for the best.
		 */
		dev_err(dev, "HW free slot count mismatch.");
		req_mgr_h->q_free_slots = 0;
	} else {
		/* Update the free slots in HW queue */
		req_mgr_h->q_free_slots -= total_seq_len;
	}
}

static void cc_enqueue_backlog(struct cc_drvdata *drvdata,
			       struct cc_bl_item *bli)
{
	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
	struct device *dev = drvdata_to_dev(drvdata);

	spin_lock_bh(&mgr->bl_lock);
	list_add_tail(&bli->list, &mgr->backlog);
	++mgr->bl_len;
	dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len);
	spin_unlock_bh(&mgr->bl_lock);
	tasklet_schedule(&mgr->comptask);
}

static void cc_proc_backlog(struct cc_drvdata *drvdata)
{
	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
	struct cc_bl_item *bli;
	struct cc_crypto_req *creq;
	void *req;
	struct device *dev = drvdata_to_dev(drvdata);
	int rc;

	spin_lock(&mgr->bl_lock);

	while (mgr->bl_len) {
		bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);
		dev_dbg(dev, "---bl len: %d\n", mgr->bl_len);

		spin_unlock(&mgr->bl_lock);


		creq = &bli->creq;
		req = creq->user_arg;

		/*
		 * Notify the request we're moving out of the backlog
		 * but only if we haven't done so already.
		 */
		if (!bli->notif) {
			creq->user_cb(dev, req, -EINPROGRESS);
			bli->notif = true;
		}

		spin_lock(&mgr->hw_lock);

		rc = cc_queues_status(drvdata, mgr, bli->len);
		if (rc) {
			/*
			 * There is still no room in the FIFO for
			 * this request. Bail out. We'll return here
			 * on the next completion irq.
			 */
			spin_unlock(&mgr->hw_lock);
			return;
		}

		cc_do_send_request(drvdata, &bli->creq, bli->desc, bli->len,
				   false);
		spin_unlock(&mgr->hw_lock);

		/* Remove ourselves from the backlog list */
		spin_lock(&mgr->bl_lock);
		list_del(&bli->list);
		--mgr->bl_len;
		kfree(bli);
	}

	spin_unlock(&mgr->bl_lock);
}

int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
		    struct cc_hw_desc *desc, unsigned int len,
		    struct crypto_async_request *req)
{
	int rc;
	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
	struct device *dev = drvdata_to_dev(drvdata);
	bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
	gfp_t flags = cc_gfp_flags(req);
	struct cc_bl_item *bli;

	rc = cc_pm_get(dev);
	if (rc) {
		dev_err(dev, "cc_pm_get returned %x\n", rc);
		return rc;
	}

	spin_lock_bh(&mgr->hw_lock);
	rc = cc_queues_status(drvdata, mgr, len);

#ifdef CC_DEBUG_FORCE_BACKLOG
	if (backlog_ok)
		rc = -ENOSPC;
#endif /* CC_DEBUG_FORCE_BACKLOG */

	if (rc == -ENOSPC && backlog_ok) {
		spin_unlock_bh(&mgr->hw_lock);

		bli = kmalloc(sizeof(*bli), flags);
		if (!bli) {
			cc_pm_put_suspend(dev);
			return -ENOMEM;
		}

		memcpy(&bli->creq, cc_req, sizeof(*cc_req));
		memcpy(&bli->desc, desc, len * sizeof(*desc));
		bli->len = len;
		bli->notif = false;
		cc_enqueue_backlog(drvdata, bli);
		return -EBUSY;
	}

	if (!rc) {
		cc_do_send_request(drvdata, cc_req, desc, len, false);
		rc = -EINPROGRESS;
	}

	spin_unlock_bh(&mgr->hw_lock);
	return rc;
}

int cc_send_sync_request(struct cc_drvdata *drvdata,
			 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,
			 unsigned int len)
{
	int rc;
	struct device *dev = drvdata_to_dev(drvdata);
	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;

	init_completion(&cc_req->seq_compl);
	cc_req->user_cb = request_mgr_complete;
	cc_req->user_arg = &cc_req->seq_compl;

	rc = cc_pm_get(dev);
	if (rc) {
		dev_err(dev, "cc_pm_get returned %x\n", rc);
		return rc;
	}

	while (true) {
		spin_lock_bh(&mgr->hw_lock);
		rc = cc_queues_status(drvdata, mgr, len + 1);

		if (!rc)
			break;

		spin_unlock_bh(&mgr->hw_lock);
		wait_for_completion_interruptible(&drvdata->hw_queue_avail);
		reinit_completion(&drvdata->hw_queue_avail);
	}

	cc_do_send_request(drvdata, cc_req, desc, len, true);
	spin_unlock_bh(&mgr->hw_lock);
	wait_for_completion(&cc_req->seq_compl);
	return 0;
}

/**
 * send_request_init() - Enqueue caller request to crypto hardware during init
 * process.
 * Assume this function is not called in the middle of a flow,
 * since we set QUEUE_LAST_IND flag in the last descriptor.
 *
 * @drvdata: Associated device driver context
 * @desc: The crypto sequence
 * @len: The crypto sequence length
 *
 * Return:
 * Returns "0" upon success
 */
int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,
		      unsigned int len)
{
	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
	unsigned int total_seq_len = len; /*initial sequence length*/
	int rc = 0;

	/* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT.
	 */
	rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);
	if (rc)
		return rc;

	set_queue_last_ind(drvdata, &desc[(len - 1)]);

	/*
	 * We are about to push command to the HW via the command registers
	 * that may reference host memory. We need to issue a memory barrier
	 * to make sure there are no outstanding memory writes
	 */
	wmb();
	enqueue_seq(drvdata, desc, len);

	/* Update the free slots in HW queue */
	req_mgr_h->q_free_slots =
		cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));

	return 0;
}

void complete_request(struct cc_drvdata *drvdata)
{
	struct cc_req_mgr_handle *request_mgr_handle =
						drvdata->request_mgr_handle;

	complete(&drvdata->hw_queue_avail);
#ifdef COMP_IN_WQ
	queue_delayed_work(request_mgr_handle->workq,
			   &request_mgr_handle->compwork, 0);
#else
	tasklet_schedule(&request_mgr_handle->comptask);
#endif
}

#ifdef COMP_IN_WQ
static void comp_work_handler(struct work_struct *work)
{
	struct cc_drvdata *drvdata =
		container_of(work, struct cc_drvdata, compwork.work);

	comp_handler((unsigned long)drvdata);
}
#endif

static void proc_completions(struct cc_drvdata *drvdata)
{
	struct cc_crypto_req *cc_req;
	struct device *dev = drvdata_to_dev(drvdata);
	struct cc_req_mgr_handle *request_mgr_handle =
						drvdata->request_mgr_handle;
	unsigned int *tail = &request_mgr_handle->req_queue_tail;
	unsigned int *head = &request_mgr_handle->req_queue_head;
	int rc;
	u32 mask;

	while (request_mgr_handle->axi_completed) {
		request_mgr_handle->axi_completed--;

		/* Dequeue request */
		if (*head == *tail) {
			/* We are supposed to handle a completion but our
			 * queue is empty. This is not normal. Return and
			 * hope for the best.
			 */
			dev_err(dev, "Request queue is empty head == tail %u\n",
				*head);
			break;
		}

		cc_req = &request_mgr_handle->req_queue[*tail];

		if (cc_req->cpp.is_cpp) {

			dev_dbg(dev, "CPP request completion slot: %d alg:%d\n",
				cc_req->cpp.slot, cc_req->cpp.alg);
			mask = cc_cpp_int_mask(cc_req->cpp.alg,
					       cc_req->cpp.slot);
			rc = (drvdata->irq & mask ? -EPERM : 0);
			dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask,
				drvdata->irq, rc);
		} else {
			dev_dbg(dev, "None CPP request completion\n");
			rc = 0;
		}

		if (cc_req->user_cb)
			cc_req->user_cb(dev, cc_req->user_arg, rc);
		*tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
		dev_dbg(dev, "Dequeue request tail=%u\n", *tail);
		dev_dbg(dev, "Request completed. axi_completed=%d\n",
			request_mgr_handle->axi_completed);
		cc_pm_put_suspend(dev);
	}
}

static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)
{
	return FIELD_GET(AXIM_MON_COMP_VALUE,
			 cc_ioread(drvdata, drvdata->axim_mon_offset));
}

/* Deferred service handler, run as interrupt-fired tasklet */
static void comp_handler(unsigned long devarg)
{
	struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;
	struct cc_req_mgr_handle *request_mgr_handle =
						drvdata->request_mgr_handle;
	struct device *dev = drvdata_to_dev(drvdata);
	u32 irq;

	dev_dbg(dev, "Completion handler called!\n");
	irq = (drvdata->irq & drvdata->comp_mask);

	/* To avoid the interrupt from firing as we unmask it,
	 * we clear it now
	 */
	cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);

	/* Avoid race with above clear: Test completion counter once more */

	request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);

	dev_dbg(dev, "AXI completion after updated: %d\n",
		request_mgr_handle->axi_completed);

	while (request_mgr_handle->axi_completed) {
		do {
			drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR));
			irq = (drvdata->irq & drvdata->comp_mask);
			proc_completions(drvdata);

			/* At this point (after proc_completions()),
			 * request_mgr_handle->axi_completed is 0.
			 */
			request_mgr_handle->axi_completed +=
						cc_axi_comp_count(drvdata);
		} while (request_mgr_handle->axi_completed > 0);

		cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);

		request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
	}

	/* after verifying that there is nothing to do,
	 * unmask AXI completion interrupt
	 */
	cc_iowrite(drvdata, CC_REG(HOST_IMR),
		   cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask);

	cc_proc_backlog(drvdata);
	dev_dbg(dev, "Comp. handler done.\n");
}