summaryrefslogtreecommitdiffstats
path: root/drivers/hid/intel-ish-hid/ishtp-fw-loader.c
blob: aa2dbed30fc36aef68c81a7d16b355034f0a677d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
// SPDX-License-Identifier: GPL-2.0
/*
 * ISH-TP client driver for ISH firmware loading
 *
 * Copyright (c) 2019, Intel Corporation.
 */

#include <linux/firmware.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/intel-ish-client-if.h>
#include <linux/property.h>
#include <asm/cacheflush.h>

/* Number of times we attempt to load the firmware before giving up */
#define MAX_LOAD_ATTEMPTS			3

/* ISH TX/RX ring buffer pool size */
#define LOADER_CL_RX_RING_SIZE			1
#define LOADER_CL_TX_RING_SIZE			1

/*
 * ISH Shim firmware loader reserves 4 Kb buffer in SRAM. The buffer is
 * used to temporarily hold the data transferred from host to Shim
 * firmware loader. Reason for the odd size of 3968 bytes? Each IPC
 * transfer is 128 bytes (= 4 bytes header + 124 bytes payload). So the
 * 4 Kb buffer can hold maximum of 32 IPC transfers, which means we can
 * have a max payload of 3968 bytes (= 32 x 124 payload).
 */
#define LOADER_SHIM_IPC_BUF_SIZE		3968

/**
 * enum ish_loader_commands -	ISH loader host commands.
 * LOADER_CMD_XFER_QUERY	Query the Shim firmware loader for
 *				capabilities
 * LOADER_CMD_XFER_FRAGMENT	Transfer one firmware image fragment at a
 *				time. The command may be executed
 *				multiple times until the entire firmware
 *				image is downloaded to SRAM.
 * LOADER_CMD_START		Start executing the main firmware.
 */
enum ish_loader_commands {
	LOADER_CMD_XFER_QUERY = 0,
	LOADER_CMD_XFER_FRAGMENT,
	LOADER_CMD_START,
};

/* Command bit mask */
#define	CMD_MASK				GENMASK(6, 0)
#define	IS_RESPONSE				BIT(7)

/*
 * ISH firmware max delay for one transmit failure is 1 Hz,
 * and firmware will retry 2 times, so 3 Hz is used for timeout.
 */
#define ISHTP_SEND_TIMEOUT			(3 * HZ)

/*
 * Loader transfer modes:
 *
 * LOADER_XFER_MODE_ISHTP mode uses the existing ISH-TP mechanism to
 * transfer data. This may use IPC or DMA if supported in firmware.
 * The buffer size is limited to 4 Kb by the IPC/ISH-TP protocol for
 * both IPC & DMA (legacy).
 *
 * LOADER_XFER_MODE_DIRECT_DMA - firmware loading is a bit different
 * from the sensor data streaming. Here we download a large (300+ Kb)
 * image directly to ISH SRAM memory. There is limited benefit of
 * DMA'ing 300 Kb image in 4 Kb chucks limit. Hence, we introduce
 * this "direct dma" mode, where we do not use ISH-TP for DMA, but
 * instead manage the DMA directly in kernel driver and Shim firmware
 * loader (allocate buffer, break in chucks and transfer). This allows
 * to overcome 4 Kb limit, and optimize the data flow path in firmware.
 */
#define LOADER_XFER_MODE_DIRECT_DMA		BIT(0)
#define LOADER_XFER_MODE_ISHTP			BIT(1)

/* ISH Transport Loader client unique GUID */
static const guid_t loader_ishtp_guid =
	GUID_INIT(0xc804d06a, 0x55bd, 0x4ea7,
		  0xad, 0xed, 0x1e, 0x31, 0x22, 0x8c, 0x76, 0xdc);

#define FILENAME_SIZE				256

/*
 * The firmware loading latency will be minimum if we can DMA the
 * entire ISH firmware image in one go. This requires that we allocate
 * a large DMA buffer in kernel, which could be problematic on some
 * platforms. So here we limit the DMA buffer size via a module_param.
 * We default to 4 pages, but a customer can set it to higher limit if
 * deemed appropriate for his platform.
 */
static int dma_buf_size_limit = 4 * PAGE_SIZE;

/**
 * struct loader_msg_hdr - Header for ISH Loader commands.
 * @command:		LOADER_CMD* commands. Bit 7 is the response.
 * @status:		Command response status. Non 0, is error
 *			condition.
 *
 * This structure is used as header for every command/data sent/received
 * between Host driver and ISH Shim firmware loader.
 */
struct loader_msg_hdr {
	u8 command;
	u8 reserved[2];
	u8 status;
} __packed;

struct loader_xfer_query {
	struct loader_msg_hdr hdr;
	u32 image_size;
} __packed;

struct ish_fw_version {
	u16 major;
	u16 minor;
	u16 hotfix;
	u16 build;
} __packed;

union loader_version {
	u32 value;
	struct {
		u8 major;
		u8 minor;
		u8 hotfix;
		u8 build;
	};
} __packed;

struct loader_capability {
	u32 max_fw_image_size;
	u32 xfer_mode;
	u32 max_dma_buf_size; /* only for dma mode, multiples of cacheline */
} __packed;

struct shim_fw_info {
	struct ish_fw_version ish_fw_version;
	u32 protocol_version;
	union loader_version ldr_version;
	struct loader_capability ldr_capability;
} __packed;

struct loader_xfer_query_response {
	struct loader_msg_hdr hdr;
	struct shim_fw_info fw_info;
} __packed;

struct loader_xfer_fragment {
	struct loader_msg_hdr hdr;
	u32 xfer_mode;
	u32 offset;
	u32 size;
	u32 is_last;
} __packed;

struct loader_xfer_ipc_fragment {
	struct loader_xfer_fragment fragment;
	u8 data[] ____cacheline_aligned; /* variable length payload here */
} __packed;

struct loader_xfer_dma_fragment {
	struct loader_xfer_fragment fragment;
	u64 ddr_phys_addr;
} __packed;

struct loader_start {
	struct loader_msg_hdr hdr;
} __packed;

/**
 * struct response_info - Encapsulate firmware response related
 *			information for passing between function
 *			loader_cl_send() and process_recv() callback.
 * @data		Copy the data received from firmware here.
 * @max_size		Max size allocated for the @data buffer. If the
 *			received data exceeds this value, we log an
 *			error.
 * @size		Actual size of data received from firmware.
 * @error		Returns 0 for success, negative error code for a
 *			failure in function process_recv().
 * @received		Set to true on receiving a valid firmware
 *			response to host command
 * @wait_queue		Wait queue for Host firmware loading where the
 *			client sends message to ISH firmware and waits
 *			for response
 */
struct response_info {
	void *data;
	size_t max_size;
	size_t size;
	int error;
	bool received;
	wait_queue_head_t wait_queue;
};

/**
 * struct ishtp_cl_data - Encapsulate per ISH-TP Client Data.
 * @work_ishtp_reset:	Work queue for reset handling.
 * @work_fw_load:	Work queue for host firmware loading.
 * @flag_retry		Flag for indicating host firmware loading should
 *			be retried.
 * @retry_count		Count the number of retries.
 *
 * This structure is used to store data per client.
 */
struct ishtp_cl_data {
	struct ishtp_cl *loader_ishtp_cl;
	struct ishtp_cl_device *cl_device;

	/*
	 * Used for passing firmware response information between
	 * loader_cl_send() and process_recv() callback.
	 */
	struct response_info response;

	struct work_struct work_ishtp_reset;
	struct work_struct work_fw_load;

	/*
	 * In certain failure scenrios, it makes sense to reset the ISH
	 * subsystem and retry Host firmware loading (e.g. bad message
	 * packet, ENOMEM, etc.). On the other hand, failures due to
	 * protocol mismatch, etc., are not recoverable. We do not
	 * retry them.
	 *
	 * If set, the flag indicates that we should re-try the
	 * particular failure.
	 */
	bool flag_retry;
	int retry_count;
};

#define IPC_FRAGMENT_DATA_PREAMBLE				\
	offsetof(struct loader_xfer_ipc_fragment, data)

#define cl_data_to_dev(client_data) ishtp_device((client_data)->cl_device)

/**
 * get_firmware_variant() - Gets the filename of firmware image to be
 *			loaded based on platform variant.
 * @client_data		Client data instance.
 * @filename		Returns firmware filename.
 *
 * Queries the firmware-name device property string.
 *
 * Return: 0 for success, negative error code for failure.
 */
static int get_firmware_variant(struct ishtp_cl_data *client_data,
				char *filename)
{
	int rv;
	const char *val;
	struct device *devc = ishtp_get_pci_device(client_data->cl_device);

	rv = device_property_read_string(devc, "firmware-name", &val);
	if (rv < 0) {
		dev_err(devc,
			"Error: ISH firmware-name device property required\n");
		return rv;
	}
	return snprintf(filename, FILENAME_SIZE, "intel/%s", val);
}

/**
 * loader_cl_send()	Send message from host to firmware
 * @client_data:	Client data instance
 * @out_msg		Message buffer to be sent to firmware
 * @out_size		Size of out going message
 * @in_msg		Message buffer where the incoming data copied.
 *			This buffer is allocated by calling
 * @in_size		Max size of incoming message
 *
 * Return: Number of bytes copied in the in_msg on success, negative
 * error code on failure.
 */
static int loader_cl_send(struct ishtp_cl_data *client_data,
			  u8 *out_msg, size_t out_size,
			  u8 *in_msg, size_t in_size)
{
	int rv;
	struct loader_msg_hdr *out_hdr = (struct loader_msg_hdr *)out_msg;
	struct ishtp_cl *loader_ishtp_cl = client_data->loader_ishtp_cl;

	dev_dbg(cl_data_to_dev(client_data),
		"%s: command=%02lx is_response=%u status=%02x\n",
		__func__,
		out_hdr->command & CMD_MASK,
		out_hdr->command & IS_RESPONSE ? 1 : 0,
		out_hdr->status);

	/* Setup in coming buffer & size */
	client_data->response.data = in_msg;
	client_data->response.max_size = in_size;
	client_data->response.error = 0;
	client_data->response.received = false;

	rv = ishtp_cl_send(loader_ishtp_cl, out_msg, out_size);
	if (rv < 0) {
		dev_err(cl_data_to_dev(client_data),
			"ishtp_cl_send error %d\n", rv);
		return rv;
	}

	wait_event_interruptible_timeout(client_data->response.wait_queue,
					 client_data->response.received,
					 ISHTP_SEND_TIMEOUT);
	if (!client_data->response.received) {
		dev_err(cl_data_to_dev(client_data),
			"Timed out for response to command=%02lx",
			out_hdr->command & CMD_MASK);
		return -ETIMEDOUT;
	}

	if (client_data->response.error < 0)
		return client_data->response.error;

	return client_data->response.size;
}

/**
 * process_recv() -	Receive and parse incoming packet
 * @loader_ishtp_cl:	Client instance to get stats
 * @rb_in_proc:		ISH received message buffer
 *
 * Parse the incoming packet. If it is a response packet then it will
 * update received and wake up the caller waiting to for the response.
 */
static void process_recv(struct ishtp_cl *loader_ishtp_cl,
			 struct ishtp_cl_rb *rb_in_proc)
{
	struct loader_msg_hdr *hdr;
	size_t data_len = rb_in_proc->buf_idx;
	struct ishtp_cl_data *client_data =
		ishtp_get_client_data(loader_ishtp_cl);

	/* Sanity check */
	if (!client_data->response.data) {
		dev_err(cl_data_to_dev(client_data),
			"Receiving buffer is null. Should be allocated by calling function\n");
		client_data->response.error = -EINVAL;
		goto end;
	}

	if (client_data->response.received) {
		dev_err(cl_data_to_dev(client_data),
			"Previous firmware message not yet processed\n");
		client_data->response.error = -EINVAL;
		goto end;
	}
	/*
	 * All firmware messages have a header. Check buffer size
	 * before accessing elements inside.
	 */
	if (!rb_in_proc->buffer.data) {
		dev_warn(cl_data_to_dev(client_data),
			 "rb_in_proc->buffer.data returned null");
		client_data->response.error = -EBADMSG;
		goto end;
	}

	if (data_len < sizeof(struct loader_msg_hdr)) {
		dev_err(cl_data_to_dev(client_data),
			"data size %zu is less than header %zu\n",
			data_len, sizeof(struct loader_msg_hdr));
		client_data->response.error = -EMSGSIZE;
		goto end;
	}

	hdr = (struct loader_msg_hdr *)rb_in_proc->buffer.data;

	dev_dbg(cl_data_to_dev(client_data),
		"%s: command=%02lx is_response=%u status=%02x\n",
		__func__,
		hdr->command & CMD_MASK,
		hdr->command & IS_RESPONSE ? 1 : 0,
		hdr->status);

	if (((hdr->command & CMD_MASK) != LOADER_CMD_XFER_QUERY) &&
	    ((hdr->command & CMD_MASK) != LOADER_CMD_XFER_FRAGMENT) &&
	    ((hdr->command & CMD_MASK) != LOADER_CMD_START)) {
		dev_err(cl_data_to_dev(client_data),
			"Invalid command=%02lx\n",
			hdr->command & CMD_MASK);
		client_data->response.error = -EPROTO;
		goto end;
	}

	if (data_len > client_data->response.max_size) {
		dev_err(cl_data_to_dev(client_data),
			"Received buffer size %zu is larger than allocated buffer %zu\n",
			data_len, client_data->response.max_size);
		client_data->response.error = -EMSGSIZE;
		goto end;
	}

	/* We expect only "response" messages from firmware */
	if (!(hdr->command & IS_RESPONSE)) {
		dev_err(cl_data_to_dev(client_data),
			"Invalid response to command\n");
		client_data->response.error = -EIO;
		goto end;
	}

	if (hdr->status) {
		dev_err(cl_data_to_dev(client_data),
			"Loader returned status %d\n",
			hdr->status);
		client_data->response.error = -EIO;
		goto end;
	}

	/* Update the actual received buffer size */
	client_data->response.size = data_len;

	/*
	 * Copy the buffer received in firmware response for the
	 * calling thread.
	 */
	memcpy(client_data->response.data,
	       rb_in_proc->buffer.data, data_len);

	/* Set flag before waking up the caller */
	client_data->response.received = true;

end:
	/* Free the buffer */
	ishtp_cl_io_rb_recycle(rb_in_proc);
	rb_in_proc = NULL;

	/* Wake the calling thread */
	wake_up_interruptible(&client_data->response.wait_queue);
}

/**
 * loader_cl_event_cb() - bus driver callback for incoming message
 * @device:		Pointer to the ishtp client device for which this
 *			message is targeted
 *
 * Remove the packet from the list and process the message by calling
 * process_recv
 */
static void loader_cl_event_cb(struct ishtp_cl_device *cl_device)
{
	struct ishtp_cl_rb *rb_in_proc;
	struct ishtp_cl	*loader_ishtp_cl = ishtp_get_drvdata(cl_device);

	while ((rb_in_proc = ishtp_cl_rx_get_rb(loader_ishtp_cl)) != NULL) {
		/* Process the data packet from firmware */
		process_recv(loader_ishtp_cl, rb_in_proc);
	}
}

/**
 * ish_query_loader_prop() -  Query ISH Shim firmware loader
 * @client_data:	Client data instance
 * @fw:			Poiner to firmware data struct in host memory
 * @fw_info:		Loader firmware properties
 *
 * This function queries the ISH Shim firmware loader for capabilities.
 *
 * Return: 0 for success, negative error code for failure.
 */
static int ish_query_loader_prop(struct ishtp_cl_data *client_data,
				 const struct firmware *fw,
				 struct shim_fw_info *fw_info)
{
	int rv;
	struct loader_xfer_query ldr_xfer_query;
	struct loader_xfer_query_response ldr_xfer_query_resp;

	memset(&ldr_xfer_query, 0, sizeof(ldr_xfer_query));
	ldr_xfer_query.hdr.command = LOADER_CMD_XFER_QUERY;
	ldr_xfer_query.image_size = fw->size;
	rv = loader_cl_send(client_data,
			    (u8 *)&ldr_xfer_query,
			    sizeof(ldr_xfer_query),
			    (u8 *)&ldr_xfer_query_resp,
			    sizeof(ldr_xfer_query_resp));
	if (rv < 0) {
		client_data->flag_retry = true;
		return rv;
	}

	/* On success, the return value is the received buffer size */
	if (rv != sizeof(struct loader_xfer_query_response)) {
		dev_err(cl_data_to_dev(client_data),
			"data size %d is not equal to size of loader_xfer_query_response %zu\n",
			rv, sizeof(struct loader_xfer_query_response));
		client_data->flag_retry = true;
		return -EMSGSIZE;
	}

	/* Save fw_info for use outside this function */
	*fw_info = ldr_xfer_query_resp.fw_info;

	/* Loader firmware properties */
	dev_dbg(cl_data_to_dev(client_data),
		"ish_fw_version: major=%d minor=%d hotfix=%d build=%d protocol_version=0x%x loader_version=%d\n",
		fw_info->ish_fw_version.major,
		fw_info->ish_fw_version.minor,
		fw_info->ish_fw_version.hotfix,
		fw_info->ish_fw_version.build,
		fw_info->protocol_version,
		fw_info->ldr_version.value);

	dev_dbg(cl_data_to_dev(client_data),
		"loader_capability: max_fw_image_size=0x%x xfer_mode=%d max_dma_buf_size=0x%x dma_buf_size_limit=0x%x\n",
		fw_info->ldr_capability.max_fw_image_size,
		fw_info->ldr_capability.xfer_mode,
		fw_info->ldr_capability.max_dma_buf_size,
		dma_buf_size_limit);

	/* Sanity checks */
	if (fw_info->ldr_capability.max_fw_image_size < fw->size) {
		dev_err(cl_data_to_dev(client_data),
			"ISH firmware size %zu is greater than Shim firmware loader max supported %d\n",
			fw->size,
			fw_info->ldr_capability.max_fw_image_size);
		return -ENOSPC;
	}

	/* For DMA the buffer size should be multiple of cacheline size */
	if ((fw_info->ldr_capability.xfer_mode & LOADER_XFER_MODE_DIRECT_DMA) &&
	    (fw_info->ldr_capability.max_dma_buf_size % L1_CACHE_BYTES)) {
		dev_err(cl_data_to_dev(client_data),
			"Shim firmware loader buffer size %d should be multiple of cacheline\n",
			fw_info->ldr_capability.max_dma_buf_size);
		return -EINVAL;
	}

	return 0;
}

/**
 * ish_fw_xfer_ishtp()	Loads ISH firmware using ishtp interface
 * @client_data:	Client data instance
 * @fw:			Pointer to firmware data struct in host memory
 *
 * This function uses ISH-TP to transfer ISH firmware from host to
 * ISH SRAM. Lower layers may use IPC or DMA depending on firmware
 * support.
 *
 * Return: 0 for success, negative error code for failure.
 */
static int ish_fw_xfer_ishtp(struct ishtp_cl_data *client_data,
			     const struct firmware *fw)
{
	int rv;
	u32 fragment_offset, fragment_size, payload_max_size;
	struct loader_xfer_ipc_fragment *ldr_xfer_ipc_frag;
	struct loader_msg_hdr ldr_xfer_ipc_ack;

	payload_max_size =
		LOADER_SHIM_IPC_BUF_SIZE - IPC_FRAGMENT_DATA_PREAMBLE;

	ldr_xfer_ipc_frag = kzalloc(LOADER_SHIM_IPC_BUF_SIZE, GFP_KERNEL);
	if (!ldr_xfer_ipc_frag) {
		client_data->flag_retry = true;
		return -ENOMEM;
	}

	ldr_xfer_ipc_frag->fragment.hdr.command = LOADER_CMD_XFER_FRAGMENT;
	ldr_xfer_ipc_frag->fragment.xfer_mode = LOADER_XFER_MODE_ISHTP;

	/* Break the firmware image into fragments and send as ISH-TP payload */
	fragment_offset = 0;
	while (fragment_offset < fw->size) {
		if (fragment_offset + payload_max_size < fw->size) {
			fragment_size = payload_max_size;
			ldr_xfer_ipc_frag->fragment.is_last = 0;
		} else {
			fragment_size = fw->size - fragment_offset;
			ldr_xfer_ipc_frag->fragment.is_last = 1;
		}

		ldr_xfer_ipc_frag->fragment.offset = fragment_offset;
		ldr_xfer_ipc_frag->fragment.size = fragment_size;
		memcpy(ldr_xfer_ipc_frag->data,
		       &fw->data[fragment_offset],
		       fragment_size);

		dev_dbg(cl_data_to_dev(client_data),
			"xfer_mode=ipc offset=0x%08x size=0x%08x is_last=%d\n",
			ldr_xfer_ipc_frag->fragment.offset,
			ldr_xfer_ipc_frag->fragment.size,
			ldr_xfer_ipc_frag->fragment.is_last);

		rv = loader_cl_send(client_data,
				    (u8 *)ldr_xfer_ipc_frag,
				    IPC_FRAGMENT_DATA_PREAMBLE + fragment_size,
				    (u8 *)&ldr_xfer_ipc_ack,
				    sizeof(ldr_xfer_ipc_ack));
		if (rv < 0) {
			client_data->flag_retry = true;
			goto end_err_resp_buf_release;
		}

		fragment_offset += fragment_size;
	}

	kfree(ldr_xfer_ipc_frag);
	return 0;

end_err_resp_buf_release:
	/* Free ISH buffer if not done already, in error case */
	kfree(ldr_xfer_ipc_frag);
	return rv;
}

/**
 * ish_fw_xfer_direct_dma() - Loads ISH firmware using direct dma
 * @client_data:	Client data instance
 * @fw:			Pointer to firmware data struct in host memory
 * @fw_info:		Loader firmware properties
 *
 * Host firmware load is a unique case where we need to download
 * a large firmware image (200+ Kb). This function implements
 * direct DMA transfer in kernel and ISH firmware. This allows
 * us to overcome the ISH-TP 4 Kb limit, and allows us to DMA
 * directly to ISH UMA at location of choice.
 * Function depends on corresponding support in ISH firmware.
 *
 * Return: 0 for success, negative error code for failure.
 */
static int ish_fw_xfer_direct_dma(struct ishtp_cl_data *client_data,
				  const struct firmware *fw,
				  const struct shim_fw_info fw_info)
{
	int rv;
	void *dma_buf;
	dma_addr_t dma_buf_phy;
	u32 fragment_offset, fragment_size, payload_max_size;
	struct loader_msg_hdr ldr_xfer_dma_frag_ack;
	struct loader_xfer_dma_fragment ldr_xfer_dma_frag;
	struct device *devc = ishtp_get_pci_device(client_data->cl_device);
	u32 shim_fw_buf_size =
		fw_info.ldr_capability.max_dma_buf_size;

	/*
	 * payload_max_size should be set to minimum of
	 *  (1) Size of firmware to be loaded,
	 *  (2) Max DMA buffer size supported by Shim firmware,
	 *  (3) DMA buffer size limit set by boot_param dma_buf_size_limit.
	 */
	payload_max_size = min3(fw->size,
				(size_t)shim_fw_buf_size,
				(size_t)dma_buf_size_limit);

	/*
	 * Buffer size should be multiple of cacheline size
	 * if it's not, select the previous cacheline boundary.
	 */
	payload_max_size &= ~(L1_CACHE_BYTES - 1);

	dma_buf = kmalloc(payload_max_size, GFP_KERNEL | GFP_DMA32);
	if (!dma_buf) {
		client_data->flag_retry = true;
		return -ENOMEM;
	}

	dma_buf_phy = dma_map_single(devc, dma_buf, payload_max_size,
				     DMA_TO_DEVICE);
	if (dma_mapping_error(devc, dma_buf_phy)) {
		dev_err(cl_data_to_dev(client_data), "DMA map failed\n");
		client_data->flag_retry = true;
		rv = -ENOMEM;
		goto end_err_dma_buf_release;
	}

	ldr_xfer_dma_frag.fragment.hdr.command = LOADER_CMD_XFER_FRAGMENT;
	ldr_xfer_dma_frag.fragment.xfer_mode = LOADER_XFER_MODE_DIRECT_DMA;
	ldr_xfer_dma_frag.ddr_phys_addr = (u64)dma_buf_phy;

	/* Send the firmware image in chucks of payload_max_size */
	fragment_offset = 0;
	while (fragment_offset < fw->size) {
		if (fragment_offset + payload_max_size < fw->size) {
			fragment_size = payload_max_size;
			ldr_xfer_dma_frag.fragment.is_last = 0;
		} else {
			fragment_size = fw->size - fragment_offset;
			ldr_xfer_dma_frag.fragment.is_last = 1;
		}

		ldr_xfer_dma_frag.fragment.offset = fragment_offset;
		ldr_xfer_dma_frag.fragment.size = fragment_size;
		memcpy(dma_buf, &fw->data[fragment_offset], fragment_size);

		dma_sync_single_for_device(devc, dma_buf_phy,
					   payload_max_size,
					   DMA_TO_DEVICE);

		/*
		 * Flush cache here because the dma_sync_single_for_device()
		 * does not do for x86.
		 */
		clflush_cache_range(dma_buf, payload_max_size);

		dev_dbg(cl_data_to_dev(client_data),
			"xfer_mode=dma offset=0x%08x size=0x%x is_last=%d ddr_phys_addr=0x%016llx\n",
			ldr_xfer_dma_frag.fragment.offset,
			ldr_xfer_dma_frag.fragment.size,
			ldr_xfer_dma_frag.fragment.is_last,
			ldr_xfer_dma_frag.ddr_phys_addr);

		rv = loader_cl_send(client_data,
				    (u8 *)&ldr_xfer_dma_frag,
				    sizeof(ldr_xfer_dma_frag),
				    (u8 *)&ldr_xfer_dma_frag_ack,
				    sizeof(ldr_xfer_dma_frag_ack));
		if (rv < 0) {
			client_data->flag_retry = true;
			goto end_err_resp_buf_release;
		}

		fragment_offset += fragment_size;
	}

	dma_unmap_single(devc, dma_buf_phy, payload_max_size, DMA_TO_DEVICE);
	kfree(dma_buf);
	return 0;

end_err_resp_buf_release:
	/* Free ISH buffer if not done already, in error case */
	dma_unmap_single(devc, dma_buf_phy, payload_max_size, DMA_TO_DEVICE);
end_err_dma_buf_release:
	kfree(dma_buf);
	return rv;
}

/**
 * ish_fw_start()	Start executing ISH main firmware
 * @client_data:	client data instance
 *
 * This function sends message to Shim firmware loader to start
 * the execution of ISH main firmware.
 *
 * Return: 0 for success, negative error code for failure.
 */
static int ish_fw_start(struct ishtp_cl_data *client_data)
{
	struct loader_start ldr_start;
	struct loader_msg_hdr ldr_start_ack;

	memset(&ldr_start, 0, sizeof(ldr_start));
	ldr_start.hdr.command = LOADER_CMD_START;
	return loader_cl_send(client_data,
			    (u8 *)&ldr_start,
			    sizeof(ldr_start),
			    (u8 *)&ldr_start_ack,
			    sizeof(ldr_start_ack));
}

/**
 * load_fw_from_host()	Loads ISH firmware from host
 * @client_data:	Client data instance
 *
 * This function loads the ISH firmware to ISH SRAM and starts execution
 *
 * Return: 0 for success, negative error code for failure.
 */
static int load_fw_from_host(struct ishtp_cl_data *client_data)
{
	int rv;
	u32 xfer_mode;
	char *filename;
	const struct firmware *fw;
	struct shim_fw_info fw_info;
	struct ishtp_cl *loader_ishtp_cl = client_data->loader_ishtp_cl;

	client_data->flag_retry = false;

	filename = kzalloc(FILENAME_SIZE, GFP_KERNEL);
	if (!filename) {
		client_data->flag_retry = true;
		rv = -ENOMEM;
		goto end_error;
	}

	/* Get filename of the ISH firmware to be loaded */
	rv = get_firmware_variant(client_data, filename);
	if (rv < 0)
		goto end_err_filename_buf_release;

	rv = request_firmware(&fw, filename, cl_data_to_dev(client_data));
	if (rv < 0)
		goto end_err_filename_buf_release;

	/* Step 1: Query Shim firmware loader properties */

	rv = ish_query_loader_prop(client_data, fw, &fw_info);
	if (rv < 0)
		goto end_err_fw_release;

	/* Step 2: Send the main firmware image to be loaded, to ISH SRAM */

	xfer_mode = fw_info.ldr_capability.xfer_mode;
	if (xfer_mode & LOADER_XFER_MODE_DIRECT_DMA) {
		rv = ish_fw_xfer_direct_dma(client_data, fw, fw_info);
	} else if (xfer_mode & LOADER_XFER_MODE_ISHTP) {
		rv = ish_fw_xfer_ishtp(client_data, fw);
	} else {
		dev_err(cl_data_to_dev(client_data),
			"No transfer mode selected in firmware\n");
		rv = -EINVAL;
	}
	if (rv < 0)
		goto end_err_fw_release;

	/* Step 3: Start ISH main firmware exeuction */

	rv = ish_fw_start(client_data);
	if (rv < 0)
		goto end_err_fw_release;

	release_firmware(fw);
	dev_info(cl_data_to_dev(client_data), "ISH firmware %s loaded\n",
		 filename);
	kfree(filename);
	return 0;

end_err_fw_release:
	release_firmware(fw);
end_err_filename_buf_release:
	kfree(filename);
end_error:
	/* Keep a count of retries, and give up after 3 attempts */
	if (client_data->flag_retry &&
	    client_data->retry_count++ < MAX_LOAD_ATTEMPTS) {
		dev_warn(cl_data_to_dev(client_data),
			 "ISH host firmware load failed %d. Resetting ISH, and trying again..\n",
			 rv);
		ish_hw_reset(ishtp_get_ishtp_device(loader_ishtp_cl));
	} else {
		dev_err(cl_data_to_dev(client_data),
			"ISH host firmware load failed %d\n", rv);
	}
	return rv;
}

static void load_fw_from_host_handler(struct work_struct *work)
{
	struct ishtp_cl_data *client_data;

	client_data = container_of(work, struct ishtp_cl_data,
				   work_fw_load);
	load_fw_from_host(client_data);
}

/**
 * loader_init() -	Init function for ISH-TP client
 * @loader_ishtp_cl:	ISH-TP client instance
 * @reset:		true if called for init after reset
 *
 * Return: 0 for success, negative error code for failure
 */
static int loader_init(struct ishtp_cl *loader_ishtp_cl, int reset)
{
	int rv;
	struct ishtp_fw_client *fw_client;
	struct ishtp_cl_data *client_data =
		ishtp_get_client_data(loader_ishtp_cl);

	dev_dbg(cl_data_to_dev(client_data), "reset flag: %d\n", reset);

	rv = ishtp_cl_link(loader_ishtp_cl);
	if (rv < 0) {
		dev_err(cl_data_to_dev(client_data), "ishtp_cl_link failed\n");
		return rv;
	}

	/* Connect to firmware client */
	ishtp_set_tx_ring_size(loader_ishtp_cl, LOADER_CL_TX_RING_SIZE);
	ishtp_set_rx_ring_size(loader_ishtp_cl, LOADER_CL_RX_RING_SIZE);

	fw_client =
		ishtp_fw_cl_get_client(ishtp_get_ishtp_device(loader_ishtp_cl),
				       &loader_ishtp_guid);
	if (!fw_client) {
		dev_err(cl_data_to_dev(client_data),
			"ISH client uuid not found\n");
		rv = -ENOENT;
		goto err_cl_unlink;
	}

	ishtp_cl_set_fw_client_id(loader_ishtp_cl,
				  ishtp_get_fw_client_id(fw_client));
	ishtp_set_connection_state(loader_ishtp_cl, ISHTP_CL_CONNECTING);

	rv = ishtp_cl_connect(loader_ishtp_cl);
	if (rv < 0) {
		dev_err(cl_data_to_dev(client_data), "Client connect fail\n");
		goto err_cl_unlink;
	}

	dev_dbg(cl_data_to_dev(client_data), "Client connected\n");

	ishtp_register_event_cb(client_data->cl_device, loader_cl_event_cb);

	return 0;

err_cl_unlink:
	ishtp_cl_unlink(loader_ishtp_cl);
	return rv;
}

static void loader_deinit(struct ishtp_cl *loader_ishtp_cl)
{
	ishtp_set_connection_state(loader_ishtp_cl, ISHTP_CL_DISCONNECTING);
	ishtp_cl_disconnect(loader_ishtp_cl);
	ishtp_cl_unlink(loader_ishtp_cl);
	ishtp_cl_flush_queues(loader_ishtp_cl);

	/* Disband and free all Tx and Rx client-level rings */
	ishtp_cl_free(loader_ishtp_cl);
}

static void reset_handler(struct work_struct *work)
{
	int rv;
	struct ishtp_cl_data *client_data;
	struct ishtp_cl *loader_ishtp_cl;
	struct ishtp_cl_device *cl_device;

	client_data = container_of(work, struct ishtp_cl_data,
				   work_ishtp_reset);

	loader_ishtp_cl = client_data->loader_ishtp_cl;
	cl_device = client_data->cl_device;

	/* Unlink, flush queues & start again */
	ishtp_cl_unlink(loader_ishtp_cl);
	ishtp_cl_flush_queues(loader_ishtp_cl);
	ishtp_cl_free(loader_ishtp_cl);

	loader_ishtp_cl = ishtp_cl_allocate(cl_device);
	if (!loader_ishtp_cl)
		return;

	ishtp_set_drvdata(cl_device, loader_ishtp_cl);
	ishtp_set_client_data(loader_ishtp_cl, client_data);
	client_data->loader_ishtp_cl = loader_ishtp_cl;
	client_data->cl_device = cl_device;

	rv = loader_init(loader_ishtp_cl, 1);
	if (rv < 0) {
		dev_err(ishtp_device(cl_device), "Reset Failed\n");
		return;
	}

	/* ISH firmware loading from host */
	load_fw_from_host(client_data);
}

/**
 * loader_ishtp_cl_probe() - ISH-TP client driver probe
 * @cl_device:		ISH-TP client device instance
 *
 * This function gets called on device create on ISH-TP bus
 *
 * Return: 0 for success, negative error code for failure
 */
static int loader_ishtp_cl_probe(struct ishtp_cl_device *cl_device)
{
	struct ishtp_cl *loader_ishtp_cl;
	struct ishtp_cl_data *client_data;
	int rv;

	client_data = devm_kzalloc(ishtp_device(cl_device),
				   sizeof(*client_data),
				   GFP_KERNEL);
	if (!client_data)
		return -ENOMEM;

	loader_ishtp_cl = ishtp_cl_allocate(cl_device);
	if (!loader_ishtp_cl)
		return -ENOMEM;

	ishtp_set_drvdata(cl_device, loader_ishtp_cl);
	ishtp_set_client_data(loader_ishtp_cl, client_data);
	client_data->loader_ishtp_cl = loader_ishtp_cl;
	client_data->cl_device = cl_device;

	init_waitqueue_head(&client_data->response.wait_queue);

	INIT_WORK(&client_data->work_ishtp_reset,
		  reset_handler);
	INIT_WORK(&client_data->work_fw_load,
		  load_fw_from_host_handler);

	rv = loader_init(loader_ishtp_cl, 0);
	if (rv < 0) {
		ishtp_cl_free(loader_ishtp_cl);
		return rv;
	}
	ishtp_get_device(cl_device);

	client_data->retry_count = 0;

	/* ISH firmware loading from host */
	schedule_work(&client_data->work_fw_load);

	return 0;
}

/**
 * loader_ishtp_cl_remove() - ISH-TP client driver remove
 * @cl_device:		ISH-TP client device instance
 *
 * This function gets called on device remove on ISH-TP bus
 *
 * Return: 0
 */
static int loader_ishtp_cl_remove(struct ishtp_cl_device *cl_device)
{
	struct ishtp_cl_data *client_data;
	struct ishtp_cl	*loader_ishtp_cl = ishtp_get_drvdata(cl_device);

	client_data = ishtp_get_client_data(loader_ishtp_cl);

	/*
	 * The sequence of the following two cancel_work_sync() is
	 * important. The work_fw_load can in turn schedue
	 * work_ishtp_reset, so first cancel work_fw_load then
	 * cancel work_ishtp_reset.
	 */
	cancel_work_sync(&client_data->work_fw_load);
	cancel_work_sync(&client_data->work_ishtp_reset);
	loader_deinit(loader_ishtp_cl);
	ishtp_put_device(cl_device);

	return 0;
}

/**
 * loader_ishtp_cl_reset() - ISH-TP client driver reset
 * @cl_device:		ISH-TP client device instance
 *
 * This function gets called on device reset on ISH-TP bus
 *
 * Return: 0
 */
static int loader_ishtp_cl_reset(struct ishtp_cl_device *cl_device)
{
	struct ishtp_cl_data *client_data;
	struct ishtp_cl	*loader_ishtp_cl = ishtp_get_drvdata(cl_device);

	client_data = ishtp_get_client_data(loader_ishtp_cl);

	schedule_work(&client_data->work_ishtp_reset);

	return 0;
}

static struct ishtp_cl_driver	loader_ishtp_cl_driver = {
	.name = "ish-loader",
	.guid = &loader_ishtp_guid,
	.probe = loader_ishtp_cl_probe,
	.remove = loader_ishtp_cl_remove,
	.reset = loader_ishtp_cl_reset,
};

static int __init ish_loader_init(void)
{
	return ishtp_cl_driver_register(&loader_ishtp_cl_driver, THIS_MODULE);
}

static void __exit ish_loader_exit(void)
{
	ishtp_cl_driver_unregister(&loader_ishtp_cl_driver);
}

late_initcall(ish_loader_init);
module_exit(ish_loader_exit);

module_param(dma_buf_size_limit, int, 0644);
MODULE_PARM_DESC(dma_buf_size_limit, "Limit the DMA buf size to this value in bytes");

MODULE_DESCRIPTION("ISH ISH-TP Host firmware Loader Client Driver");
MODULE_AUTHOR("Rushikesh S Kadam <rushikesh.s.kadam@intel.com>");

MODULE_LICENSE("GPL v2");
MODULE_ALIAS("ishtp:*");