1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2021, MediaTek Inc.
4 * Copyright (c) 2021-2022, Intel Corporation.
5 *
6 * Authors:
7 * Haijun Liu <[email protected]>
8 * Ricardo Martinez <[email protected]>
9 * Sreehari Kancharla <[email protected]>
10 *
11 * Contributors:
12 * Amir Hanania <[email protected]>
13 * Andy Shevchenko <[email protected]>
14 * Chiranjeevi Rapolu <[email protected]>
15 * Eliot Lee <[email protected]>
16 * Moises Veleta <[email protected]>
17 */
18
19 #include <linux/atomic.h>
20 #include <linux/bits.h>
21 #include <linux/completion.h>
22 #include <linux/device.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/gfp.h>
25 #include <linux/interrupt.h>
26 #include <linux/io.h>
27 #include <linux/iopoll.h>
28 #include <linux/jiffies.h>
29 #include <linux/list.h>
30 #include <linux/module.h>
31 #include <linux/mutex.h>
32 #include <linux/pci.h>
33 #include <linux/pm.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/pm_wakeup.h>
36 #include <linux/spinlock.h>
37
38 #include "t7xx_mhccif.h"
39 #include "t7xx_modem_ops.h"
40 #include "t7xx_pci.h"
41 #include "t7xx_pcie_mac.h"
42 #include "t7xx_reg.h"
43 #include "t7xx_state_monitor.h"
44 #include "t7xx_port_proxy.h"
45
46 #define DRIVER_NAME "mtk_t7xx"
47
48 #define T7XX_PCI_IREG_BASE 0
49 #define T7XX_PCI_EREG_BASE 2
50
51 #define T7XX_INIT_TIMEOUT 20
52 #define PM_SLEEP_DIS_TIMEOUT_MS 20
53 #define PM_ACK_TIMEOUT_MS 1500
54 #define PM_AUTOSUSPEND_MS 5000
55 #define PM_RESOURCE_POLL_TIMEOUT_US 10000
56 #define PM_RESOURCE_POLL_STEP_US 100
57
58 static const char * const t7xx_mode_names[] = {
59 [T7XX_UNKNOWN] = "unknown",
60 [T7XX_READY] = "ready",
61 [T7XX_RESET] = "reset",
62 [T7XX_FASTBOOT_SWITCHING] = "fastboot_switching",
63 [T7XX_FASTBOOT_DOWNLOAD] = "fastboot_download",
64 [T7XX_FASTBOOT_DUMP] = "fastboot_dump",
65 };
66
67 static_assert(ARRAY_SIZE(t7xx_mode_names) == T7XX_MODE_LAST);
68
t7xx_mode_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)69 static ssize_t t7xx_mode_store(struct device *dev,
70 struct device_attribute *attr,
71 const char *buf, size_t count)
72 {
73 struct t7xx_pci_dev *t7xx_dev;
74 struct pci_dev *pdev;
75 enum t7xx_mode mode;
76 int index = 0;
77
78 pdev = to_pci_dev(dev);
79 t7xx_dev = pci_get_drvdata(pdev);
80 if (!t7xx_dev)
81 return -ENODEV;
82
83 mode = READ_ONCE(t7xx_dev->mode);
84
85 index = sysfs_match_string(t7xx_mode_names, buf);
86 if (index == mode)
87 return -EBUSY;
88
89 if (index == T7XX_FASTBOOT_SWITCHING) {
90 if (mode == T7XX_FASTBOOT_DOWNLOAD)
91 return count;
92
93 WRITE_ONCE(t7xx_dev->mode, T7XX_FASTBOOT_SWITCHING);
94 pm_runtime_resume(dev);
95 t7xx_reset_device(t7xx_dev, FASTBOOT);
96 } else if (index == T7XX_RESET) {
97 pm_runtime_resume(dev);
98 t7xx_reset_device(t7xx_dev, PLDR);
99 }
100
101 return count;
102 };
103
t7xx_mode_show(struct device * dev,struct device_attribute * attr,char * buf)104 static ssize_t t7xx_mode_show(struct device *dev,
105 struct device_attribute *attr,
106 char *buf)
107 {
108 enum t7xx_mode mode = T7XX_UNKNOWN;
109 struct t7xx_pci_dev *t7xx_dev;
110 struct pci_dev *pdev;
111
112 pdev = to_pci_dev(dev);
113 t7xx_dev = pci_get_drvdata(pdev);
114 if (!t7xx_dev)
115 return -ENODEV;
116
117 mode = READ_ONCE(t7xx_dev->mode);
118 if (mode < T7XX_MODE_LAST)
119 return sysfs_emit(buf, "%s\n", t7xx_mode_names[mode]);
120
121 return sysfs_emit(buf, "%s\n", t7xx_mode_names[T7XX_UNKNOWN]);
122 }
123
124 static DEVICE_ATTR_RW(t7xx_mode);
125
t7xx_debug_ports_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)126 static ssize_t t7xx_debug_ports_store(struct device *dev,
127 struct device_attribute *attr,
128 const char *buf, size_t count)
129 {
130 struct t7xx_pci_dev *t7xx_dev;
131 struct pci_dev *pdev;
132 bool show;
133 int ret;
134
135 pdev = to_pci_dev(dev);
136 t7xx_dev = pci_get_drvdata(pdev);
137 if (!t7xx_dev)
138 return -ENODEV;
139
140 ret = kstrtobool(buf, &show);
141 if (ret < 0)
142 return ret;
143
144 t7xx_proxy_debug_ports_show(t7xx_dev, show);
145 WRITE_ONCE(t7xx_dev->debug_ports_show, show);
146
147 return count;
148 };
149
t7xx_debug_ports_show(struct device * dev,struct device_attribute * attr,char * buf)150 static ssize_t t7xx_debug_ports_show(struct device *dev,
151 struct device_attribute *attr,
152 char *buf)
153 {
154 struct t7xx_pci_dev *t7xx_dev;
155 struct pci_dev *pdev;
156 bool show;
157
158 pdev = to_pci_dev(dev);
159 t7xx_dev = pci_get_drvdata(pdev);
160 if (!t7xx_dev)
161 return -ENODEV;
162
163 show = READ_ONCE(t7xx_dev->debug_ports_show);
164
165 return sysfs_emit(buf, "%d\n", show);
166 }
167
168 static DEVICE_ATTR_RW(t7xx_debug_ports);
169
170 static struct attribute *t7xx_attr[] = {
171 &dev_attr_t7xx_mode.attr,
172 &dev_attr_t7xx_debug_ports.attr,
173 NULL
174 };
175
176 static const struct attribute_group t7xx_attribute_group = {
177 .attrs = t7xx_attr,
178 };
179
t7xx_mode_update(struct t7xx_pci_dev * t7xx_dev,enum t7xx_mode mode)180 void t7xx_mode_update(struct t7xx_pci_dev *t7xx_dev, enum t7xx_mode mode)
181 {
182 if (!t7xx_dev)
183 return;
184
185 WRITE_ONCE(t7xx_dev->mode, mode);
186 sysfs_notify(&t7xx_dev->pdev->dev.kobj, NULL, "t7xx_mode");
187 }
188
189 enum t7xx_pm_state {
190 MTK_PM_EXCEPTION,
191 MTK_PM_INIT, /* Device initialized, but handshake not completed */
192 MTK_PM_SUSPENDED,
193 MTK_PM_RESUMED,
194 };
195
t7xx_dev_set_sleep_capability(struct t7xx_pci_dev * t7xx_dev,bool enable)196 static void t7xx_dev_set_sleep_capability(struct t7xx_pci_dev *t7xx_dev, bool enable)
197 {
198 void __iomem *ctrl_reg = IREG_BASE(t7xx_dev) + T7XX_PCIE_MISC_CTRL;
199 u32 value;
200
201 value = ioread32(ctrl_reg);
202
203 if (enable)
204 value &= ~T7XX_PCIE_MISC_MAC_SLEEP_DIS;
205 else
206 value |= T7XX_PCIE_MISC_MAC_SLEEP_DIS;
207
208 iowrite32(value, ctrl_reg);
209 }
210
t7xx_wait_pm_config(struct t7xx_pci_dev * t7xx_dev)211 static int t7xx_wait_pm_config(struct t7xx_pci_dev *t7xx_dev)
212 {
213 int ret, val;
214
215 ret = read_poll_timeout(ioread32, val,
216 (val & T7XX_PCIE_RESOURCE_STS_MSK) == T7XX_PCIE_RESOURCE_STS_MSK,
217 PM_RESOURCE_POLL_STEP_US, PM_RESOURCE_POLL_TIMEOUT_US, true,
218 IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS);
219 if (ret == -ETIMEDOUT)
220 dev_err(&t7xx_dev->pdev->dev, "PM configuration timed out\n");
221
222 return ret;
223 }
224
t7xx_pci_pm_init(struct t7xx_pci_dev * t7xx_dev)225 static int t7xx_pci_pm_init(struct t7xx_pci_dev *t7xx_dev)
226 {
227 struct pci_dev *pdev = t7xx_dev->pdev;
228
229 INIT_LIST_HEAD(&t7xx_dev->md_pm_entities);
230 mutex_init(&t7xx_dev->md_pm_entity_mtx);
231 spin_lock_init(&t7xx_dev->md_pm_lock);
232 init_completion(&t7xx_dev->sleep_lock_acquire);
233 init_completion(&t7xx_dev->pm_sr_ack);
234 init_completion(&t7xx_dev->init_done);
235 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT);
236
237 device_init_wakeup(&pdev->dev, true);
238 dev_pm_set_driver_flags(&pdev->dev, pdev->dev.power.driver_flags |
239 DPM_FLAG_NO_DIRECT_COMPLETE);
240
241 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
242 pm_runtime_set_autosuspend_delay(&pdev->dev, PM_AUTOSUSPEND_MS);
243 pm_runtime_use_autosuspend(&pdev->dev);
244
245 return 0;
246 }
247
t7xx_pci_pm_init_late(struct t7xx_pci_dev * t7xx_dev)248 void t7xx_pci_pm_init_late(struct t7xx_pci_dev *t7xx_dev)
249 {
250 /* Enable the PCIe resource lock only after MD deep sleep is done */
251 t7xx_mhccif_mask_clr(t7xx_dev,
252 D2H_INT_DS_LOCK_ACK |
253 D2H_INT_SUSPEND_ACK |
254 D2H_INT_RESUME_ACK |
255 D2H_INT_SUSPEND_ACK_AP |
256 D2H_INT_RESUME_ACK_AP);
257 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
258 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
259
260 pm_runtime_mark_last_busy(&t7xx_dev->pdev->dev);
261 pm_runtime_allow(&t7xx_dev->pdev->dev);
262 pm_runtime_put_noidle(&t7xx_dev->pdev->dev);
263 complete_all(&t7xx_dev->init_done);
264 }
265
t7xx_pci_pm_reinit(struct t7xx_pci_dev * t7xx_dev)266 static int t7xx_pci_pm_reinit(struct t7xx_pci_dev *t7xx_dev)
267 {
268 /* The device is kept in FSM re-init flow
269 * so just roll back PM setting to the init setting.
270 */
271 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT);
272
273 pm_runtime_get_noresume(&t7xx_dev->pdev->dev);
274
275 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
276 return t7xx_wait_pm_config(t7xx_dev);
277 }
278
t7xx_pci_pm_exp_detected(struct t7xx_pci_dev * t7xx_dev)279 void t7xx_pci_pm_exp_detected(struct t7xx_pci_dev *t7xx_dev)
280 {
281 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
282 t7xx_wait_pm_config(t7xx_dev);
283 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_EXCEPTION);
284 }
285
t7xx_pci_pm_entity_register(struct t7xx_pci_dev * t7xx_dev,struct md_pm_entity * pm_entity)286 int t7xx_pci_pm_entity_register(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity)
287 {
288 struct md_pm_entity *entity;
289
290 mutex_lock(&t7xx_dev->md_pm_entity_mtx);
291 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
292 if (entity->id == pm_entity->id) {
293 mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
294 return -EEXIST;
295 }
296 }
297
298 list_add_tail(&pm_entity->entity, &t7xx_dev->md_pm_entities);
299 mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
300 return 0;
301 }
302
t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev * t7xx_dev,struct md_pm_entity * pm_entity)303 int t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity)
304 {
305 struct md_pm_entity *entity, *tmp_entity;
306
307 mutex_lock(&t7xx_dev->md_pm_entity_mtx);
308 list_for_each_entry_safe(entity, tmp_entity, &t7xx_dev->md_pm_entities, entity) {
309 if (entity->id == pm_entity->id) {
310 list_del(&pm_entity->entity);
311 mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
312 return 0;
313 }
314 }
315
316 mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
317
318 return -ENXIO;
319 }
320
t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev * t7xx_dev)321 int t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev *t7xx_dev)
322 {
323 struct device *dev = &t7xx_dev->pdev->dev;
324 int ret;
325
326 ret = wait_for_completion_timeout(&t7xx_dev->sleep_lock_acquire,
327 msecs_to_jiffies(PM_SLEEP_DIS_TIMEOUT_MS));
328 if (!ret)
329 dev_err_ratelimited(dev, "Resource wait complete timed out\n");
330
331 return ret;
332 }
333
334 /**
335 * t7xx_pci_disable_sleep() - Disable deep sleep capability.
336 * @t7xx_dev: MTK device.
337 *
338 * Lock the deep sleep capability, note that the device can still go into deep sleep
339 * state while device is in D0 state, from the host's point-of-view.
340 *
341 * If device is in deep sleep state, wake up the device and disable deep sleep capability.
342 */
t7xx_pci_disable_sleep(struct t7xx_pci_dev * t7xx_dev)343 void t7xx_pci_disable_sleep(struct t7xx_pci_dev *t7xx_dev)
344 {
345 unsigned long flags;
346
347 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags);
348 t7xx_dev->sleep_disable_count++;
349 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED)
350 goto unlock_and_complete;
351
352 if (t7xx_dev->sleep_disable_count == 1) {
353 u32 status;
354
355 reinit_completion(&t7xx_dev->sleep_lock_acquire);
356 t7xx_dev_set_sleep_capability(t7xx_dev, false);
357
358 status = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS);
359 if (status & T7XX_PCIE_RESOURCE_STS_MSK)
360 goto unlock_and_complete;
361
362 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_DS_LOCK);
363 }
364 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
365 return;
366
367 unlock_and_complete:
368 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
369 complete_all(&t7xx_dev->sleep_lock_acquire);
370 }
371
372 /**
373 * t7xx_pci_enable_sleep() - Enable deep sleep capability.
374 * @t7xx_dev: MTK device.
375 *
376 * After enabling deep sleep, device can enter into deep sleep state.
377 */
t7xx_pci_enable_sleep(struct t7xx_pci_dev * t7xx_dev)378 void t7xx_pci_enable_sleep(struct t7xx_pci_dev *t7xx_dev)
379 {
380 unsigned long flags;
381
382 spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags);
383 t7xx_dev->sleep_disable_count--;
384 if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED)
385 goto unlock;
386
387 if (t7xx_dev->sleep_disable_count == 0)
388 t7xx_dev_set_sleep_capability(t7xx_dev, true);
389
390 unlock:
391 spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
392 }
393
t7xx_send_pm_request(struct t7xx_pci_dev * t7xx_dev,u32 request)394 static int t7xx_send_pm_request(struct t7xx_pci_dev *t7xx_dev, u32 request)
395 {
396 unsigned long wait_ret;
397
398 reinit_completion(&t7xx_dev->pm_sr_ack);
399 t7xx_mhccif_h2d_swint_trigger(t7xx_dev, request);
400 wait_ret = wait_for_completion_timeout(&t7xx_dev->pm_sr_ack,
401 msecs_to_jiffies(PM_ACK_TIMEOUT_MS));
402 if (!wait_ret)
403 return -ETIMEDOUT;
404
405 return 0;
406 }
407
__t7xx_pci_pm_suspend(struct pci_dev * pdev)408 static int __t7xx_pci_pm_suspend(struct pci_dev *pdev)
409 {
410 enum t7xx_pm_id entity_id = PM_ENTITY_ID_INVALID;
411 struct t7xx_pci_dev *t7xx_dev;
412 struct md_pm_entity *entity;
413 int ret;
414
415 t7xx_dev = pci_get_drvdata(pdev);
416 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT ||
417 READ_ONCE(t7xx_dev->mode) != T7XX_READY) {
418 dev_err(&pdev->dev, "[PM] Exiting suspend, modem in invalid state\n");
419 return -EFAULT;
420 }
421
422 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
423 ret = t7xx_wait_pm_config(t7xx_dev);
424 if (ret) {
425 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
426 return ret;
427 }
428
429 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
430 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT);
431 t7xx_dev->rgu_pci_irq_en = false;
432
433 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
434 if (!entity->suspend)
435 continue;
436
437 ret = entity->suspend(t7xx_dev, entity->entity_param);
438 if (ret) {
439 entity_id = entity->id;
440 dev_err(&pdev->dev, "[PM] Suspend error: %d, id: %d\n", ret, entity_id);
441 goto abort_suspend;
442 }
443 }
444
445 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ);
446 if (ret) {
447 dev_err(&pdev->dev, "[PM] MD suspend error: %d\n", ret);
448 goto abort_suspend;
449 }
450
451 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP);
452 if (ret) {
453 t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ);
454 dev_err(&pdev->dev, "[PM] SAP suspend error: %d\n", ret);
455 goto abort_suspend;
456 }
457
458 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
459 if (entity->suspend_late)
460 entity->suspend_late(t7xx_dev, entity->entity_param);
461 }
462
463 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
464 return 0;
465
466 abort_suspend:
467 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
468 if (entity_id == entity->id)
469 break;
470
471 if (entity->resume)
472 entity->resume(t7xx_dev, entity->entity_param);
473 }
474
475 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
476 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
477 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
478 return ret;
479 }
480
t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev * t7xx_dev)481 static void t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev *t7xx_dev)
482 {
483 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM);
484
485 /* Disable interrupt first and let the IPs enable them */
486 iowrite32(MSIX_MSK_SET_ALL, IREG_BASE(t7xx_dev) + IMASK_HOST_MSIX_CLR_GRP0_0);
487
488 /* Device disables PCIe interrupts during resume and
489 * following function will re-enable PCIe interrupts.
490 */
491 t7xx_pcie_mac_interrupts_en(t7xx_dev);
492 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT);
493 }
494
t7xx_pcie_reinit(struct t7xx_pci_dev * t7xx_dev,bool is_d3)495 static int t7xx_pcie_reinit(struct t7xx_pci_dev *t7xx_dev, bool is_d3)
496 {
497 int ret;
498
499 ret = pcim_enable_device(t7xx_dev->pdev);
500 if (ret)
501 return ret;
502
503 t7xx_pcie_mac_atr_init(t7xx_dev);
504 t7xx_pcie_interrupt_reinit(t7xx_dev);
505
506 if (is_d3) {
507 t7xx_mhccif_init(t7xx_dev);
508 t7xx_pci_pm_reinit(t7xx_dev);
509 }
510
511 return 0;
512 }
513
t7xx_send_fsm_command(struct t7xx_pci_dev * t7xx_dev,u32 event)514 static int t7xx_send_fsm_command(struct t7xx_pci_dev *t7xx_dev, u32 event)
515 {
516 struct t7xx_fsm_ctl *fsm_ctl = t7xx_dev->md->fsm_ctl;
517 struct device *dev = &t7xx_dev->pdev->dev;
518 int ret = -EINVAL;
519
520 switch (event) {
521 case FSM_CMD_STOP:
522 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION);
523 break;
524
525 case FSM_CMD_START:
526 t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT);
527 t7xx_pcie_mac_clear_int_status(t7xx_dev, SAP_RGU_INT);
528 t7xx_dev->rgu_pci_irq_en = true;
529 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
530 ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_START, 0);
531 break;
532
533 default:
534 break;
535 }
536
537 if (ret)
538 dev_err(dev, "Failure handling FSM command %u, %d\n", event, ret);
539
540 return ret;
541 }
542
t7xx_pci_reprobe_early(struct t7xx_pci_dev * t7xx_dev)543 int t7xx_pci_reprobe_early(struct t7xx_pci_dev *t7xx_dev)
544 {
545 enum t7xx_mode mode = READ_ONCE(t7xx_dev->mode);
546 int ret;
547
548 if (mode == T7XX_FASTBOOT_DOWNLOAD)
549 pm_runtime_put_noidle(&t7xx_dev->pdev->dev);
550
551 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP);
552 if (ret)
553 return ret;
554
555 return 0;
556 }
557
t7xx_pci_reprobe(struct t7xx_pci_dev * t7xx_dev,bool boot)558 int t7xx_pci_reprobe(struct t7xx_pci_dev *t7xx_dev, bool boot)
559 {
560 int ret;
561
562 ret = t7xx_pcie_reinit(t7xx_dev, boot);
563 if (ret)
564 return ret;
565
566 t7xx_clear_rgu_irq(t7xx_dev);
567 return t7xx_send_fsm_command(t7xx_dev, FSM_CMD_START);
568 }
569
__t7xx_pci_pm_resume(struct pci_dev * pdev,bool state_check)570 static int __t7xx_pci_pm_resume(struct pci_dev *pdev, bool state_check)
571 {
572 struct t7xx_pci_dev *t7xx_dev;
573 struct md_pm_entity *entity;
574 u32 prev_state;
575 int ret = 0;
576
577 t7xx_dev = pci_get_drvdata(pdev);
578 if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT) {
579 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
580 return 0;
581 }
582
583 t7xx_pcie_mac_interrupts_en(t7xx_dev);
584 prev_state = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_PM_RESUME_STATE);
585
586 if (state_check) {
587 /* For D3/L3 resume, the device could boot so quickly that the
588 * initial value of the dummy register might be overwritten.
589 * Identify new boots if the ATR source address register is not initialized.
590 */
591 u32 atr_reg_val = ioread32(IREG_BASE(t7xx_dev) +
592 ATR_PCIE_WIN0_T0_ATR_PARAM_SRC_ADDR);
593 if (prev_state == PM_RESUME_REG_STATE_L3 ||
594 (prev_state == PM_RESUME_REG_STATE_INIT &&
595 atr_reg_val == ATR_SRC_ADDR_INVALID)) {
596 ret = t7xx_pci_reprobe_early(t7xx_dev);
597 if (ret)
598 return ret;
599
600 return t7xx_pci_reprobe(t7xx_dev, true);
601 }
602
603 if (prev_state == PM_RESUME_REG_STATE_EXP ||
604 prev_state == PM_RESUME_REG_STATE_L2_EXP) {
605 if (prev_state == PM_RESUME_REG_STATE_L2_EXP) {
606 ret = t7xx_pcie_reinit(t7xx_dev, false);
607 if (ret)
608 return ret;
609 }
610
611 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
612 t7xx_dev->rgu_pci_irq_en = true;
613 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
614
615 t7xx_mhccif_mask_clr(t7xx_dev,
616 D2H_INT_EXCEPTION_INIT |
617 D2H_INT_EXCEPTION_INIT_DONE |
618 D2H_INT_EXCEPTION_CLEARQ_DONE |
619 D2H_INT_EXCEPTION_ALLQ_RESET |
620 D2H_INT_PORT_ENUM);
621
622 return ret;
623 }
624
625 if (prev_state == PM_RESUME_REG_STATE_L2) {
626 ret = t7xx_pcie_reinit(t7xx_dev, false);
627 if (ret)
628 return ret;
629
630 } else if (prev_state != PM_RESUME_REG_STATE_L1 &&
631 prev_state != PM_RESUME_REG_STATE_INIT) {
632 ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP);
633 if (ret)
634 return ret;
635
636 t7xx_clear_rgu_irq(t7xx_dev);
637 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
638 return 0;
639 }
640 }
641
642 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
643 t7xx_wait_pm_config(t7xx_dev);
644
645 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
646 if (entity->resume_early)
647 entity->resume_early(t7xx_dev, entity->entity_param);
648 }
649
650 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ);
651 if (ret)
652 dev_err(&pdev->dev, "[PM] MD resume error: %d\n", ret);
653
654 ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ_AP);
655 if (ret)
656 dev_err(&pdev->dev, "[PM] SAP resume error: %d\n", ret);
657
658 list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
659 if (entity->resume) {
660 ret = entity->resume(t7xx_dev, entity->entity_param);
661 if (ret)
662 dev_err(&pdev->dev, "[PM] Resume entry ID: %d error: %d\n",
663 entity->id, ret);
664 }
665 }
666
667 t7xx_dev->rgu_pci_irq_en = true;
668 t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
669 iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
670 pm_runtime_mark_last_busy(&pdev->dev);
671 atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
672
673 return ret;
674 }
675
t7xx_pci_pm_resume_noirq(struct device * dev)676 static int t7xx_pci_pm_resume_noirq(struct device *dev)
677 {
678 struct pci_dev *pdev = to_pci_dev(dev);
679 struct t7xx_pci_dev *t7xx_dev;
680
681 t7xx_dev = pci_get_drvdata(pdev);
682 t7xx_pcie_mac_interrupts_dis(t7xx_dev);
683
684 return 0;
685 }
686
t7xx_pci_shutdown(struct pci_dev * pdev)687 static void t7xx_pci_shutdown(struct pci_dev *pdev)
688 {
689 __t7xx_pci_pm_suspend(pdev);
690 }
691
t7xx_pci_pm_prepare(struct device * dev)692 static int t7xx_pci_pm_prepare(struct device *dev)
693 {
694 struct pci_dev *pdev = to_pci_dev(dev);
695 struct t7xx_pci_dev *t7xx_dev;
696
697 t7xx_dev = pci_get_drvdata(pdev);
698 if (!wait_for_completion_timeout(&t7xx_dev->init_done, T7XX_INIT_TIMEOUT * HZ)) {
699 dev_warn(dev, "Not ready for system sleep.\n");
700 return -ETIMEDOUT;
701 }
702
703 return 0;
704 }
705
t7xx_pci_pm_suspend(struct device * dev)706 static int t7xx_pci_pm_suspend(struct device *dev)
707 {
708 return __t7xx_pci_pm_suspend(to_pci_dev(dev));
709 }
710
t7xx_pci_pm_resume(struct device * dev)711 static int t7xx_pci_pm_resume(struct device *dev)
712 {
713 return __t7xx_pci_pm_resume(to_pci_dev(dev), true);
714 }
715
t7xx_pci_pm_thaw(struct device * dev)716 static int t7xx_pci_pm_thaw(struct device *dev)
717 {
718 return __t7xx_pci_pm_resume(to_pci_dev(dev), false);
719 }
720
t7xx_pci_pm_runtime_suspend(struct device * dev)721 static int t7xx_pci_pm_runtime_suspend(struct device *dev)
722 {
723 return __t7xx_pci_pm_suspend(to_pci_dev(dev));
724 }
725
t7xx_pci_pm_runtime_resume(struct device * dev)726 static int t7xx_pci_pm_runtime_resume(struct device *dev)
727 {
728 return __t7xx_pci_pm_resume(to_pci_dev(dev), true);
729 }
730
731 static const struct dev_pm_ops t7xx_pci_pm_ops = {
732 .prepare = t7xx_pci_pm_prepare,
733 .suspend = t7xx_pci_pm_suspend,
734 .resume = t7xx_pci_pm_resume,
735 .resume_noirq = t7xx_pci_pm_resume_noirq,
736 .freeze = t7xx_pci_pm_suspend,
737 .thaw = t7xx_pci_pm_thaw,
738 .poweroff = t7xx_pci_pm_suspend,
739 .restore = t7xx_pci_pm_resume,
740 .restore_noirq = t7xx_pci_pm_resume_noirq,
741 .runtime_suspend = t7xx_pci_pm_runtime_suspend,
742 .runtime_resume = t7xx_pci_pm_runtime_resume
743 };
744
t7xx_request_irq(struct pci_dev * pdev)745 static int t7xx_request_irq(struct pci_dev *pdev)
746 {
747 struct t7xx_pci_dev *t7xx_dev;
748 int ret = 0, i;
749
750 t7xx_dev = pci_get_drvdata(pdev);
751
752 for (i = 0; i < EXT_INT_NUM; i++) {
753 const char *irq_descr;
754 int irq_vec;
755
756 if (!t7xx_dev->intr_handler[i])
757 continue;
758
759 irq_descr = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
760 dev_driver_string(&pdev->dev), i);
761 if (!irq_descr) {
762 ret = -ENOMEM;
763 break;
764 }
765
766 irq_vec = pci_irq_vector(pdev, i);
767 ret = request_threaded_irq(irq_vec, t7xx_dev->intr_handler[i],
768 t7xx_dev->intr_thread[i], 0, irq_descr,
769 t7xx_dev->callback_param[i]);
770 if (ret) {
771 dev_err(&pdev->dev, "Failed to request IRQ: %d\n", ret);
772 break;
773 }
774 }
775
776 if (ret) {
777 while (i--) {
778 if (!t7xx_dev->intr_handler[i])
779 continue;
780
781 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]);
782 }
783 }
784
785 return ret;
786 }
787
t7xx_setup_msix(struct t7xx_pci_dev * t7xx_dev)788 static int t7xx_setup_msix(struct t7xx_pci_dev *t7xx_dev)
789 {
790 struct pci_dev *pdev = t7xx_dev->pdev;
791 int ret;
792
793 /* Only using 6 interrupts, but HW-design requires power-of-2 IRQs allocation */
794 ret = pci_alloc_irq_vectors(pdev, EXT_INT_NUM, EXT_INT_NUM, PCI_IRQ_MSIX);
795 if (ret < 0) {
796 dev_err(&pdev->dev, "Failed to allocate MSI-X entry: %d\n", ret);
797 return ret;
798 }
799
800 ret = t7xx_request_irq(pdev);
801 if (ret) {
802 pci_free_irq_vectors(pdev);
803 return ret;
804 }
805
806 t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM);
807 return 0;
808 }
809
t7xx_interrupt_init(struct t7xx_pci_dev * t7xx_dev)810 static int t7xx_interrupt_init(struct t7xx_pci_dev *t7xx_dev)
811 {
812 int ret, i;
813
814 if (!t7xx_dev->pdev->msix_cap)
815 return -EINVAL;
816
817 ret = t7xx_setup_msix(t7xx_dev);
818 if (ret)
819 return ret;
820
821 /* IPs enable interrupts when ready */
822 for (i = 0; i < EXT_INT_NUM; i++)
823 t7xx_pcie_mac_set_int(t7xx_dev, i);
824
825 return 0;
826 }
827
t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev * t7xx_dev)828 static void t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev *t7xx_dev)
829 {
830 t7xx_dev->base_addr.infracfg_ao_base = t7xx_dev->base_addr.pcie_ext_reg_base +
831 INFRACFG_AO_DEV_CHIP -
832 t7xx_dev->base_addr.pcie_dev_reg_trsl_addr;
833 }
834
t7xx_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)835 static int t7xx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
836 {
837 struct t7xx_pci_dev *t7xx_dev;
838 void __iomem *iomem;
839 int ret;
840
841 t7xx_dev = devm_kzalloc(&pdev->dev, sizeof(*t7xx_dev), GFP_KERNEL);
842 if (!t7xx_dev)
843 return -ENOMEM;
844
845 pci_set_drvdata(pdev, t7xx_dev);
846 t7xx_dev->pdev = pdev;
847
848 ret = pcim_enable_device(pdev);
849 if (ret)
850 return ret;
851
852 pci_set_master(pdev);
853
854 iomem = pcim_iomap_region(pdev, T7XX_PCI_IREG_BASE, DRIVER_NAME);
855 ret = PTR_ERR_OR_ZERO(iomem);
856 if (ret) {
857 dev_err(&pdev->dev, "Could not request IREG BAR: %d\n", ret);
858 return -ENOMEM;
859 }
860 IREG_BASE(t7xx_dev) = iomem;
861
862 iomem = pcim_iomap_region(pdev, T7XX_PCI_EREG_BASE, DRIVER_NAME);
863 ret = PTR_ERR_OR_ZERO(iomem);
864 if (ret) {
865 dev_err(&pdev->dev, "Could not request EREG BAR: %d\n", ret);
866 return -ENOMEM;
867 }
868 t7xx_dev->base_addr.pcie_ext_reg_base = iomem;
869
870 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
871 if (ret) {
872 dev_err(&pdev->dev, "Could not set PCI DMA mask: %d\n", ret);
873 return ret;
874 }
875
876 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
877 if (ret) {
878 dev_err(&pdev->dev, "Could not set consistent PCI DMA mask: %d\n", ret);
879 return ret;
880 }
881
882 ret = t7xx_pci_pm_init(t7xx_dev);
883 if (ret)
884 return ret;
885
886 t7xx_pcie_mac_atr_init(t7xx_dev);
887 t7xx_pci_infracfg_ao_calc(t7xx_dev);
888 t7xx_mhccif_init(t7xx_dev);
889
890 ret = t7xx_md_init(t7xx_dev);
891 if (ret)
892 return ret;
893
894 t7xx_pcie_mac_interrupts_dis(t7xx_dev);
895
896 ret = sysfs_create_group(&t7xx_dev->pdev->dev.kobj,
897 &t7xx_attribute_group);
898 if (ret)
899 goto err_md_exit;
900
901 ret = t7xx_interrupt_init(t7xx_dev);
902 if (ret)
903 goto err_remove_group;
904
905
906 t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT);
907 t7xx_pcie_mac_interrupts_en(t7xx_dev);
908
909 return 0;
910
911 err_remove_group:
912 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj,
913 &t7xx_attribute_group);
914
915 err_md_exit:
916 t7xx_md_exit(t7xx_dev);
917 return ret;
918 }
919
t7xx_pci_remove(struct pci_dev * pdev)920 static void t7xx_pci_remove(struct pci_dev *pdev)
921 {
922 struct t7xx_pci_dev *t7xx_dev;
923 int i;
924
925 t7xx_dev = pci_get_drvdata(pdev);
926
927 sysfs_remove_group(&t7xx_dev->pdev->dev.kobj,
928 &t7xx_attribute_group);
929 t7xx_md_exit(t7xx_dev);
930
931 for (i = 0; i < EXT_INT_NUM; i++) {
932 if (!t7xx_dev->intr_handler[i])
933 continue;
934
935 free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]);
936 }
937
938 pci_free_irq_vectors(t7xx_dev->pdev);
939 }
940
941 static const struct pci_device_id t7xx_pci_table[] = {
942 { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x4d75) },
943 { PCI_DEVICE(0x14c0, 0x4d75) }, // Dell DW5933e
944 { }
945 };
946 MODULE_DEVICE_TABLE(pci, t7xx_pci_table);
947
948 static struct pci_driver t7xx_pci_driver = {
949 .name = DRIVER_NAME,
950 .id_table = t7xx_pci_table,
951 .probe = t7xx_pci_probe,
952 .remove = t7xx_pci_remove,
953 .driver.pm = &t7xx_pci_pm_ops,
954 .shutdown = t7xx_pci_shutdown,
955 };
956
957 module_pci_driver(t7xx_pci_driver);
958
959 MODULE_AUTHOR("MediaTek Inc");
960 MODULE_DESCRIPTION("MediaTek PCIe 5G WWAN modem T7xx driver");
961 MODULE_LICENSE("GPL");
962