53 		nvkm_wr32(device, 0x405804, gr->zbc_color[zbc].ds[0]);  in gf100_gr_zbc_clear_color()
54 		nvkm_wr32(device, 0x405808, gr->zbc_color[zbc].ds[1]);  in gf100_gr_zbc_clear_color()
55 		nvkm_wr32(device, 0x40580c, gr->zbc_color[zbc].ds[2]);  in gf100_gr_zbc_clear_color()
56 		nvkm_wr32(device, 0x405810, gr->zbc_color[zbc].ds[3]);  in gf100_gr_zbc_clear_color()
58 	nvkm_wr32(device, 0x405814, gr->zbc_color[zbc].format);  in gf100_gr_zbc_clear_color()
59 	nvkm_wr32(device, 0x405820, zbc);  in gf100_gr_zbc_clear_color()
60 	nvkm_wr32(device, 0x405824, 0x00000004); /* TRIGGER | WRITE | COLOR */  in gf100_gr_zbc_clear_color()
84 			zbc = (zbc < 0) ? i : zbc;  in gf100_gr_zbc_color_get()
88 	if (zbc < 0)  in gf100_gr_zbc_color_get()
104 		nvkm_wr32(device, 0x405818, gr->zbc_depth[zbc].ds);  in gf100_gr_zbc_clear_depth()
105 	nvkm_wr32(device, 0x40581c, gr->zbc_depth[zbc].format);  in gf100_gr_zbc_clear_depth()
106 	nvkm_wr32(device, 0x405820, zbc);  in gf100_gr_zbc_clear_depth()
107 	nvkm_wr32(device, 0x405824, 0x00000005); /* TRIGGER | WRITE | DEPTH */  in gf100_gr_zbc_clear_depth()
129 			zbc = (zbc < 0) ? i : zbc;  in gf100_gr_zbc_depth_get()
133 	if (zbc < 0)  in gf100_gr_zbc_depth_get()
169 	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {  in gf100_fermi_mthd_zbc_color()
193 			if (ret >= 0) {  in gf100_fermi_mthd_zbc_color()
195 				return 0;  in gf100_fermi_mthd_zbc_color()
215 	if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {  in gf100_fermi_mthd_zbc_depth()
221 			return (ret >= 0) ? 0 : -ENOSPC;  in gf100_fermi_mthd_zbc_depth()
253 	nvkm_wr32(device, 0x419e44, data ? 0xffffffff : 0x00000000);  in gf100_gr_mthd_set_shader_exceptions()
254 	nvkm_wr32(device, 0x419e4c, data ? 0xffffffff : 0x00000000);  in gf100_gr_mthd_set_shader_exceptions()
260 	switch (class & 0x00ff) {  in gf100_gr_mthd_sw()
261 	case 0x97:  in gf100_gr_mthd_sw()
262 	case 0xc0:  in gf100_gr_mthd_sw()
264 		case 0x1528:  in gf100_gr_mthd_sw()
295 	return 0;  in gf100_gr_object_new()
302 	int c = 0;  in gf100_gr_object_get()
333 	for (i = 0; i < gr->size; i += 4)  in gf100_gr_chan_bind()
337 		nvkm_wo32(*pgpuobj, 0x00, chan->mmio_nr / 2);  in gf100_gr_chan_bind()
338 		nvkm_wo32(*pgpuobj, 0x04, chan->mmio_vma->addr >> 8);  in gf100_gr_chan_bind()
340 		nvkm_wo32(*pgpuobj, 0xf4, 0);  in gf100_gr_chan_bind()
341 		nvkm_wo32(*pgpuobj, 0xf8, 0);  in gf100_gr_chan_bind()
342 		nvkm_wo32(*pgpuobj, 0x10, chan->mmio_nr / 2);  in gf100_gr_chan_bind()
343 		nvkm_wo32(*pgpuobj, 0x14, lower_32_bits(chan->mmio_vma->addr));  in gf100_gr_chan_bind()
344 		nvkm_wo32(*pgpuobj, 0x18, upper_32_bits(chan->mmio_vma->addr));  in gf100_gr_chan_bind()
345 		nvkm_wo32(*pgpuobj, 0x1c, 1);  in gf100_gr_chan_bind()
346 		nvkm_wo32(*pgpuobj, 0x20, 0);  in gf100_gr_chan_bind()
347 		nvkm_wo32(*pgpuobj, 0x28, 0);  in gf100_gr_chan_bind()
348 		nvkm_wo32(*pgpuobj, 0x2c, 0);  in gf100_gr_chan_bind()
351 	return 0;  in gf100_gr_chan_bind()
399 	ret = nvkm_memory_map(gr->pagepool, 0, chan->vmm, chan->pagepool, &args, sizeof(args));  in gf100_gr_chan_new()
408 	ret = nvkm_memory_map(gr->bundle_cb, 0, chan->vmm, chan->bundle_cb, &args, sizeof(args));  in gf100_gr_chan_new()
418 		ret = nvkm_memory_map(gr->attrib_cb, 0, chan->vmm, chan->attrib_cb, NULL, 0);  in gf100_gr_chan_new()
422 		ret = nvkm_memory_map(gr->attrib_cb, 0, chan->vmm, chan->attrib_cb,  in gf100_gr_chan_new()
434 		ret = nvkm_memory_map(gr->unknown, 0, chan->vmm, chan->unknown,  in gf100_gr_chan_new()
456 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x100,  in gf100_gr_chan_new()
461 	ret = nvkm_vmm_get(fifoch->vmm, 12, 0x1000, &chan->mmio_vma);  in gf100_gr_chan_new()
465 	ret = nvkm_memory_map(chan->mmio, 0, fifoch->vmm,  in gf100_gr_chan_new()
481 	return 0;  in gf100_gr_chan_new()
490 	{ 0x400080,   1, 0x04, 0x003083c2 },
491 	{ 0x400088,   1, 0x04, 0x00006fe7 },
492 	{ 0x40008c,   1, 0x04, 0x00000000 },
493 	{ 0x400090,   1, 0x04, 0x00000030 },
494 	{ 0x40013c,   1, 0x04, 0x013901f7 },
495 	{ 0x400140,   1, 0x04, 0x00000100 },
496 	{ 0x400144,   1, 0x04, 0x00000000 },
497 	{ 0x400148,   1, 0x04, 0x00000110 },
498 	{ 0x400138,   1, 0x04, 0x00000000 },
499 	{ 0x400130,   2, 0x04, 0x00000000 },
500 	{ 0x400124,   1, 0x04, 0x00000002 },
506 	{ 0x40415c,   1, 0x04, 0x00000000 },
507 	{ 0x404170,   1, 0x04, 0x00000000 },
513 	{ 0x404488,   2, 0x04, 0x00000000 },
519 	{ 0x407808,   1, 0x04, 0x00000000 },
525 	{ 0x406024,   1, 0x04, 0x00000000 },
531 	{ 0x405844,   1, 0x04, 0x00ffffff },
532 	{ 0x405850,   1, 0x04, 0x00000000 },
533 	{ 0x405908,   1, 0x04, 0x00000000 },
539 	{ 0x40803c,   1, 0x04, 0x00000000 },
545 	{ 0x4184a0,   1, 0x04, 0x00000000 },
551 	{ 0x418604,   1, 0x04, 0x00000000 },
552 	{ 0x418680,   1, 0x04, 0x00000000 },
553 	{ 0x418714,   1, 0x04, 0x80000000 },
554 	{ 0x418384,   1, 0x04, 0x00000000 },
560 	{ 0x418814,   3, 0x04, 0x00000000 },
566 	{ 0x418b04,   1, 0x04, 0x00000000 },
572 	{ 0x4188c8,   1, 0x04, 0x80000000 },
573 	{ 0x4188cc,   1, 0x04, 0x00000000 },
574 	{ 0x4188d0,   1, 0x04, 0x00010000 },
575 	{ 0x4188d4,   1, 0x04, 0x00000001 },
581 	{ 0x418910,   1, 0x04, 0x00010001 },
582 	{ 0x418914,   1, 0x04, 0x00000301 },
583 	{ 0x418918,   1, 0x04, 0x00800000 },
584 	{ 0x418980,   1, 0x04, 0x77777770 },
585 	{ 0x418984,   3, 0x04, 0x77777777 },
591 	{ 0x418c04,   1, 0x04, 0x00000000 },
592 	{ 0x418c88,   1, 0x04, 0x00000000 },
598 	{ 0x418d00,   1, 0x04, 0x00000000 },
599 	{ 0x418f08,   1, 0x04, 0x00000000 },
600 	{ 0x418e00,   1, 0x04, 0x00000050 },
601 	{ 0x418e08,   1, 0x04, 0x00000000 },
607 	{ 0x41900c,   1, 0x04, 0x00000000 },
608 	{ 0x419018,   1, 0x04, 0x00000000 },
614 	{ 0x419d08,   2, 0x04, 0x00000000 },
615 	{ 0x419d10,   1, 0x04, 0x00000014 },
621 	{ 0x419ab0,   1, 0x04, 0x00000000 },
622 	{ 0x419ab8,   1, 0x04, 0x000000e7 },
623 	{ 0x419abc,   2, 0x04, 0x00000000 },
629 	{ 0x41980c,   3, 0x04, 0x00000000 },
630 	{ 0x419844,   1, 0x04, 0x00000000 },
631 	{ 0x41984c,   1, 0x04, 0x00005bc5 },
632 	{ 0x419850,   4, 0x04, 0x00000000 },
638 	{ 0x419c98,   1, 0x04, 0x00000000 },
639 	{ 0x419ca8,   1, 0x04, 0x80000000 },
640 	{ 0x419cb4,   1, 0x04, 0x00000000 },
641 	{ 0x419cb8,   1, 0x04, 0x00008bf4 },
642 	{ 0x419cbc,   1, 0x04, 0x28137606 },
643 	{ 0x419cc0,   2, 0x04, 0x00000000 },
649 	{ 0x419bd4,   1, 0x04, 0x00800000 },
650 	{ 0x419bdc,   1, 0x04, 0x00000000 },
656 	{ 0x419d2c,   1, 0x04, 0x00000000 },
662 	{ 0x419c0c,   1, 0x04, 0x00000000 },
668 	{ 0x419e00,   1, 0x04, 0x00000000 },
669 	{ 0x419ea0,   1, 0x04, 0x00000000 },
670 	{ 0x419ea4,   1, 0x04, 0x00000100 },
671 	{ 0x419ea8,   1, 0x04, 0x00001100 },
672 	{ 0x419eac,   1, 0x04, 0x11100702 },
673 	{ 0x419eb0,   1, 0x04, 0x00000003 },
674 	{ 0x419eb4,   4, 0x04, 0x00000000 },
675 	{ 0x419ec8,   1, 0x04, 0x06060618 },
676 	{ 0x419ed0,   1, 0x04, 0x0eff0e38 },
677 	{ 0x419ed4,   1, 0x04, 0x011104f1 },
678 	{ 0x419edc,   1, 0x04, 0x00000000 },
679 	{ 0x419f00,   1, 0x04, 0x00000000 },
680 	{ 0x419f2c,   1, 0x04, 0x00000000 },
686 	{ 0x40880c,   1, 0x04, 0x00000000 },
687 	{ 0x408910,   9, 0x04, 0x00000000 },
688 	{ 0x408950,   1, 0x04, 0x00000000 },
689 	{ 0x408954,   1, 0x04, 0x0000ffff },
690 	{ 0x408984,   1, 0x04, 0x00000000 },
691 	{ 0x408988,   1, 0x04, 0x08040201 },
692 	{ 0x40898c,   1, 0x04, 0x80402010 },
698 	{ 0x4040f0,   1, 0x04, 0x00000000 },
704 	{ 0x419880,   1, 0x04, 0x00000002 },
747 	return nvkm_rd32(gr->engine.subdev.device, 0x409b00);  in gf100_gr_ctxsw_inst()
755 	nvkm_wr32(device, 0x409804, 0xffffffff);  in gf100_gr_fecs_ctrl_ctxsw()
756 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_fecs_ctrl_ctxsw()
757 	nvkm_wr32(device, 0x409500, 0xffffffff);  in gf100_gr_fecs_ctrl_ctxsw()
758 	nvkm_wr32(device, 0x409504, mthd);  in gf100_gr_fecs_ctrl_ctxsw()
760 		u32 stat = nvkm_rd32(device, 0x409804);  in gf100_gr_fecs_ctrl_ctxsw()
761 		if (stat == 0x00000002)  in gf100_gr_fecs_ctrl_ctxsw()
763 		if (stat == 0x00000001)  in gf100_gr_fecs_ctrl_ctxsw()
764 			return 0;  in gf100_gr_fecs_ctrl_ctxsw()
774 	int ret = 0;  in gf100_gr_fecs_start_ctxsw()
778 		if (WARN_ON(ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x39)))  in gf100_gr_fecs_start_ctxsw()
789 	int ret = 0;  in gf100_gr_fecs_stop_ctxsw()
793 		if (WARN_ON(ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x38)))  in gf100_gr_fecs_stop_ctxsw()
803 	int ret = 0;  in gf100_gr_fecs_halt_pipeline()
807 		ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x04);  in gf100_gr_fecs_halt_pipeline()
819 	nvkm_mask(device, 0x409800, 0x00000003, 0x00000000);  in gf100_gr_fecs_wfi_golden_save()
820 	nvkm_wr32(device, 0x409500, inst);  in gf100_gr_fecs_wfi_golden_save()
821 	nvkm_wr32(device, 0x409504, 0x00000009);  in gf100_gr_fecs_wfi_golden_save()
823 		u32 stat = nvkm_rd32(device, 0x409800);  in gf100_gr_fecs_wfi_golden_save()
824 		if (stat & 0x00000002)  in gf100_gr_fecs_wfi_golden_save()
826 		if (stat & 0x00000001)  in gf100_gr_fecs_wfi_golden_save()
827 			return 0;  in gf100_gr_fecs_wfi_golden_save()
838 	nvkm_mask(device, 0x409800, 0x00000030, 0x00000000);  in gf100_gr_fecs_bind_pointer()
839 	nvkm_wr32(device, 0x409500, inst);  in gf100_gr_fecs_bind_pointer()
840 	nvkm_wr32(device, 0x409504, 0x00000003);  in gf100_gr_fecs_bind_pointer()
842 		u32 stat = nvkm_rd32(device, 0x409800);  in gf100_gr_fecs_bind_pointer()
843 		if (stat & 0x00000020)  in gf100_gr_fecs_bind_pointer()
845 		if (stat & 0x00000010)  in gf100_gr_fecs_bind_pointer()
846 			return 0;  in gf100_gr_fecs_bind_pointer()
857 	nvkm_wr32(device, 0x409810, addr >> 8);  in gf100_gr_fecs_set_reglist_virtual_address()
858 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_fecs_set_reglist_virtual_address()
859 	nvkm_wr32(device, 0x409500, 0x00000001);  in gf100_gr_fecs_set_reglist_virtual_address()
860 	nvkm_wr32(device, 0x409504, 0x00000032);  in gf100_gr_fecs_set_reglist_virtual_address()
862 		if (nvkm_rd32(device, 0x409800) == 0x00000001)  in gf100_gr_fecs_set_reglist_virtual_address()
863 			return 0;  in gf100_gr_fecs_set_reglist_virtual_address()
874 	nvkm_wr32(device, 0x409810, inst);  in gf100_gr_fecs_set_reglist_bind_instance()
875 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_fecs_set_reglist_bind_instance()
876 	nvkm_wr32(device, 0x409500, 0x00000001);  in gf100_gr_fecs_set_reglist_bind_instance()
877 	nvkm_wr32(device, 0x409504, 0x00000031);  in gf100_gr_fecs_set_reglist_bind_instance()
879 		if (nvkm_rd32(device, 0x409800) == 0x00000001)  in gf100_gr_fecs_set_reglist_bind_instance()
880 			return 0;  in gf100_gr_fecs_set_reglist_bind_instance()
891 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_fecs_discover_reglist_image_size()
892 	nvkm_wr32(device, 0x409500, 0x00000001);  in gf100_gr_fecs_discover_reglist_image_size()
893 	nvkm_wr32(device, 0x409504, 0x00000030);  in gf100_gr_fecs_discover_reglist_image_size()
895 		if ((*psize = nvkm_rd32(device, 0x409800)))  in gf100_gr_fecs_discover_reglist_image_size()
896 			return 0;  in gf100_gr_fecs_discover_reglist_image_size()
917 	ret = gf100_gr_fecs_set_reglist_bind_instance(gr, 0);  in gf100_gr_fecs_elpg_bind()
921 	return gf100_gr_fecs_set_reglist_virtual_address(gr, 0);  in gf100_gr_fecs_elpg_bind()
929 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_fecs_discover_pm_image_size()
930 	nvkm_wr32(device, 0x409500, 0x00000000);  in gf100_gr_fecs_discover_pm_image_size()
931 	nvkm_wr32(device, 0x409504, 0x00000025);  in gf100_gr_fecs_discover_pm_image_size()
933 		if ((*psize = nvkm_rd32(device, 0x409800)))  in gf100_gr_fecs_discover_pm_image_size()
934 			return 0;  in gf100_gr_fecs_discover_pm_image_size()
945 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_fecs_discover_zcull_image_size()
946 	nvkm_wr32(device, 0x409500, 0x00000000);  in gf100_gr_fecs_discover_zcull_image_size()
947 	nvkm_wr32(device, 0x409504, 0x00000016);  in gf100_gr_fecs_discover_zcull_image_size()
949 		if ((*psize = nvkm_rd32(device, 0x409800)))  in gf100_gr_fecs_discover_zcull_image_size()
950 			return 0;  in gf100_gr_fecs_discover_zcull_image_size()
961 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_fecs_discover_image_size()
962 	nvkm_wr32(device, 0x409500, 0x00000000);  in gf100_gr_fecs_discover_image_size()
963 	nvkm_wr32(device, 0x409504, 0x00000010);  in gf100_gr_fecs_discover_image_size()
965 		if ((*psize = nvkm_rd32(device, 0x409800)))  in gf100_gr_fecs_discover_image_size()
966 			return 0;  in gf100_gr_fecs_discover_image_size()
977 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_fecs_set_watchdog_timeout()
978 	nvkm_wr32(device, 0x409500, timeout);  in gf100_gr_fecs_set_watchdog_timeout()
979 	nvkm_wr32(device, 0x409504, 0x00000021);  in gf100_gr_fecs_set_watchdog_timeout()
987 		u32 trace = nvkm_rd32(gr->base.engine.subdev.device, 0x40981c);  in gf100_gr_chsw_load()
988 		if (trace & 0x00000040)  in gf100_gr_chsw_load()
991 		u32 mthd = nvkm_rd32(gr->base.engine.subdev.device, 0x409808);  in gf100_gr_chsw_load()
992 		if (mthd & 0x00080000)  in gf100_gr_chsw_load()
1002 	return (nvkm_rd32(device, 0x409604) & 0x001f0000) >> 16;  in gf100_gr_rops()
1008 	const u32  zero[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000,  in gf100_gr_zbc_init()
1009 			      0x00000000, 0x00000000, 0x00000000, 0x00000000 };  in gf100_gr_zbc_init()
1010 	const u32   one[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,  in gf100_gr_zbc_init()
1011 			      0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff };  in gf100_gr_zbc_init()
1012 	const u32 f32_0[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000,  in gf100_gr_zbc_init()
1013 			      0x00000000, 0x00000000, 0x00000000, 0x00000000 };  in gf100_gr_zbc_init()
1014 	const u32 f32_1[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,  in gf100_gr_zbc_init()
1015 			      0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 };  in gf100_gr_zbc_init()
1019 	if (!gr->zbc_color[0].format) {  in gf100_gr_zbc_init()
1020 		gf100_gr_zbc_color_get(gr, 1,  & zero[0],   &zero[4]); c++;  in gf100_gr_zbc_init()
1021 		gf100_gr_zbc_color_get(gr, 2,  &  one[0],    &one[4]); c++;  in gf100_gr_zbc_init()
1022 		gf100_gr_zbc_color_get(gr, 4,  &f32_0[0],  &f32_0[4]); c++;  in gf100_gr_zbc_init()
1023 		gf100_gr_zbc_color_get(gr, 4,  &f32_1[0],  &f32_1[4]); c++;  in gf100_gr_zbc_init()
1024 		gf100_gr_zbc_depth_get(gr, 1, 0x00000000, 0x00000000); d++;  in gf100_gr_zbc_init()
1025 		gf100_gr_zbc_depth_get(gr, 1, 0x3f800000, 0x3f800000); d++;  in gf100_gr_zbc_init()
1027 			gr->func->zbc->stencil_get(gr, 1, 0x00, 0x00); s++;  in gf100_gr_zbc_init()
1028 			gr->func->zbc->stencil_get(gr, 1, 0x01, 0x01); s++;  in gf100_gr_zbc_init()
1029 			gr->func->zbc->stencil_get(gr, 1, 0xff, 0xff); s++;  in gf100_gr_zbc_init()
1046  * MC (0x200) register, or GR is not busy and a context switch is not in
1059 		 * required to make sure FIFO_ENGINE_STATUS (0x2640) is  in gf100_gr_wait_idle()
1062 		nvkm_rd32(device, 0x400700);  in gf100_gr_wait_idle()
1064 		gr_enabled = nvkm_rd32(device, 0x200) & 0x1000;  in gf100_gr_wait_idle()
1066 		gr_busy = nvkm_rd32(device, 0x40060c) & 0x1;  in gf100_gr_wait_idle()
1069 			return 0;  in gf100_gr_wait_idle()
1101 	u64 data = 0;  in gf100_gr_icmd()
1103 	nvkm_wr32(device, 0x400208, 0x80000000);  in gf100_gr_icmd()
1110 			nvkm_wr32(device, 0x400204, init->data);  in gf100_gr_icmd()
1112 				nvkm_wr32(device, 0x40020c, upper_32_bits(init->data));  in gf100_gr_icmd()
1117 			nvkm_wr32(device, 0x400200, addr);  in gf100_gr_icmd()
1122 			if ((addr & 0xffff) == 0xe100)  in gf100_gr_icmd()
1125 				if (!(nvkm_rd32(device, 0x400700) & 0x00000004))  in gf100_gr_icmd()
1132 	nvkm_wr32(device, 0x400208, 0x00000000);  in gf100_gr_icmd()
1141 	u32 data = 0;  in gf100_gr_mthd()
1144 		u32 ctrl = 0x80000000 | pack->type;  in gf100_gr_mthd()
1149 			nvkm_wr32(device, 0x40448c, init->data);  in gf100_gr_mthd()
1154 			nvkm_wr32(device, 0x404488, ctrl | (addr << 14));  in gf100_gr_mthd()
1174 	{ 0x00000001, "INJECTED_BUNDLE_ERROR" },
1175 	{ 0x00000002, "CLASS_SUBCH_MISMATCH" },
1176 	{ 0x00000004, "SUBCHSW_DURING_NOTIFY" },
1181 	{ 0x00000001, "PUSH_TOO_MUCH_DATA" },
1182 	{ 0x00000002, "PUSH_NOT_ENOUGH_DATA" },
1187 	{ 0x00000001, "TEMP_TOO_SMALL" },
1192 	{ 0x00000001, "INTR" },
1193 	{ 0x00000002, "LDCONST_OOB" },
1198 	{ 0x00000001, "TOO_FEW_PARAMS" },
1199 	{ 0x00000002, "TOO_MANY_PARAMS" },
1200 	{ 0x00000004, "ILLEGAL_OPCODE" },
1201 	{ 0x00000008, "DOUBLE_BRANCH" },
1202 	{ 0x00000010, "WATCHDOG" },
1207 	{ 0x00000040, "CTA_RESUME" },
1208 	{ 0x00000080, "CONSTANT_BUFFER_SIZE" },
1209 	{ 0x00000200, "LOCAL_MEMORY_SIZE_POS" },
1210 	{ 0x00000400, "LOCAL_MEMORY_SIZE_NEG" },
1211 	{ 0x00000800, "WARP_CSTACK_SIZE" },
1212 	{ 0x00001000, "TOTAL_TEMP_SIZE" },
1213 	{ 0x00002000, "REGISTER_COUNT" },
1214 	{ 0x00040000, "TOTAL_THREADS" },
1215 	{ 0x00100000, "PROGRAM_OFFSET" },
1216 	{ 0x00200000, "SHARED_MEMORY_SIZE" },
1217 	{ 0x00800000, "CTA_THREAD_DIMENSION_ZERO" },
1218 	{ 0x01000000, "MEMORY_WINDOW_OVERLAP" },
1219 	{ 0x02000000, "SHARED_CONFIG_TOO_SMALL" },
1220 	{ 0x04000000, "TOTAL_REGISTER_COUNT" },
1225 	{ 0x00000002, "RT_PITCH_OVERRUN" },
1226 	{ 0x00000010, "RT_WIDTH_OVERRUN" },
1227 	{ 0x00000020, "RT_HEIGHT_OVERRUN" },
1228 	{ 0x00000080, "ZETA_STORAGE_TYPE_MISMATCH" },
1229 	{ 0x00000100, "RT_STORAGE_TYPE_MISMATCH" },
1230 	{ 0x00000400, "RT_LINEAR_MISMATCH" },
1242 	trap[0] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0420)) & 0x3fffffff;  in gf100_gr_trap_gpc_rop()
1243 	trap[1] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0434));  in gf100_gr_trap_gpc_rop()
1244 	trap[2] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0438));  in gf100_gr_trap_gpc_rop()
1245 	trap[3] = nvkm_rd32(device, GPC_UNIT(gpc, 0x043c));  in gf100_gr_trap_gpc_rop()
1247 	nvkm_snprintbf(error, sizeof(error), gf100_gpc_rop_error, trap[0]);  in gf100_gr_trap_gpc_rop()
1251 		   gpc, trap[0], error, trap[1] & 0xffff, trap[1] >> 16,  in gf100_gr_trap_gpc_rop()
1252 		   (trap[2] >> 8) & 0x3f, trap[3] & 0xff);  in gf100_gr_trap_gpc_rop()
1253 	nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);  in gf100_gr_trap_gpc_rop()
1257 	{ 0x01, "STACK_ERROR" },
1258 	{ 0x02, "API_STACK_ERROR" },
1259 	{ 0x03, "RET_EMPTY_STACK_ERROR" },
1260 	{ 0x04, "PC_WRAP" },
1261 	{ 0x05, "MISALIGNED_PC" },
1262 	{ 0x06, "PC_OVERFLOW" },
1263 	{ 0x07, "MISALIGNED_IMMC_ADDR" },
1264 	{ 0x08, "MISALIGNED_REG" },
1265 	{ 0x09, "ILLEGAL_INSTR_ENCODING" },
1266 	{ 0x0a, "ILLEGAL_SPH_INSTR_COMBO" },
1267 	{ 0x0b, "ILLEGAL_INSTR_PARAM" },
1268 	{ 0x0c, "INVALID_CONST_ADDR" },
1269 	{ 0x0d, "OOR_REG" },
1270 	{ 0x0e, "OOR_ADDR" },
1271 	{ 0x0f, "MISALIGNED_ADDR" },
1272 	{ 0x10, "INVALID_ADDR_SPACE" },
1273 	{ 0x11, "ILLEGAL_INSTR_PARAM2" },
1274 	{ 0x12, "INVALID_CONST_ADDR_LDC" },
1275 	{ 0x13, "GEOMETRY_SM_ERROR" },
1276 	{ 0x14, "DIVERGENT" },
1277 	{ 0x15, "WARP_EXIT" },
1282 	{ 0x00000001, "SM_TO_SM_FAULT" },
1283 	{ 0x00000002, "L1_ERROR" },
1284 	{ 0x00000004, "MULTIPLE_WARP_ERRORS" },
1285 	{ 0x00000008, "PHYSICAL_STACK_OVERFLOW" },
1286 	{ 0x00000010, "BPT_INT" },
1287 	{ 0x00000020, "BPT_PAUSE" },
1288 	{ 0x00000040, "SINGLE_STEP_COMPLETE" },
1289 	{ 0x20000000, "ECC_SEC_ERROR" },
1290 	{ 0x40000000, "ECC_DED_ERROR" },
1291 	{ 0x80000000, "TIMEOUT" },
1300 	u32 werr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x648));  in gf100_gr_trap_mp()
1301 	u32 gerr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x650));  in gf100_gr_trap_mp()
1306 	warp = nvkm_enum_find(gf100_mp_warp_error, werr & 0xffff);  in gf100_gr_trap_mp()
1312 	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x648), 0x00000000);  in gf100_gr_trap_mp()
1313 	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x650), gerr);  in gf100_gr_trap_mp()
1321 	u32 stat = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0508));  in gf100_gr_trap_tpc()
1323 	if (stat & 0x00000001) {  in gf100_gr_trap_tpc()
1324 		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0224));  in gf100_gr_trap_tpc()
1326 		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);  in gf100_gr_trap_tpc()
1327 		stat &= ~0x00000001;  in gf100_gr_trap_tpc()
1330 	if (stat & 0x00000002) {  in gf100_gr_trap_tpc()
1332 		stat &= ~0x00000002;  in gf100_gr_trap_tpc()
1335 	if (stat & 0x00000004) {  in gf100_gr_trap_tpc()
1336 		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0084));  in gf100_gr_trap_tpc()
1338 		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);  in gf100_gr_trap_tpc()
1339 		stat &= ~0x00000004;  in gf100_gr_trap_tpc()
1342 	if (stat & 0x00000008) {  in gf100_gr_trap_tpc()
1343 		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x048c));  in gf100_gr_trap_tpc()
1345 		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);  in gf100_gr_trap_tpc()
1346 		stat &= ~0x00000008;  in gf100_gr_trap_tpc()
1349 	if (stat & 0x00000010) {  in gf100_gr_trap_tpc()
1350 		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0430));  in gf100_gr_trap_tpc()
1352 		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0430), 0xc0000000);  in gf100_gr_trap_tpc()
1353 		stat &= ~0x00000010;  in gf100_gr_trap_tpc()
1366 	u32 stat = nvkm_rd32(device, GPC_UNIT(gpc, 0x2c90));  in gf100_gr_trap_gpc()
1369 	if (stat & 0x00000001) {  in gf100_gr_trap_gpc()
1371 		stat &= ~0x00000001;  in gf100_gr_trap_gpc()
1374 	if (stat & 0x00000002) {  in gf100_gr_trap_gpc()
1375 		u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x0900));  in gf100_gr_trap_gpc()
1377 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);  in gf100_gr_trap_gpc()
1378 		stat &= ~0x00000002;  in gf100_gr_trap_gpc()
1381 	if (stat & 0x00000004) {  in gf100_gr_trap_gpc()
1382 		u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x1028));  in gf100_gr_trap_gpc()
1384 		nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);  in gf100_gr_trap_gpc()
1385 		stat &= ~0x00000004;  in gf100_gr_trap_gpc()
1388 	if (stat & 0x00000008) {  in gf100_gr_trap_gpc()
1389 		u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x0824));  in gf100_gr_trap_gpc()
1391 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);  in gf100_gr_trap_gpc()
1392 		stat &= ~0x00000009;  in gf100_gr_trap_gpc()
1395 	for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {  in gf100_gr_trap_gpc()
1396 		u32 mask = 0x00010000 << tpc;  in gf100_gr_trap_gpc()
1399 			nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), mask);  in gf100_gr_trap_gpc()
1415 	u32 trap = nvkm_rd32(device, 0x400108);  in gf100_gr_trap_intr()
1418 	if (trap & 0x00000001) {  in gf100_gr_trap_intr()
1419 		u32 stat = nvkm_rd32(device, 0x404000);  in gf100_gr_trap_intr()
1422 			       stat & 0x3fffffff);  in gf100_gr_trap_intr()
1424 		nvkm_wr32(device, 0x404000, 0xc0000000);  in gf100_gr_trap_intr()
1425 		nvkm_wr32(device, 0x400108, 0x00000001);  in gf100_gr_trap_intr()
1426 		trap &= ~0x00000001;  in gf100_gr_trap_intr()
1429 	if (trap & 0x00000002) {  in gf100_gr_trap_intr()
1430 		u32 stat = nvkm_rd32(device, 0x404600);  in gf100_gr_trap_intr()
1433 			       stat & 0x3fffffff);  in gf100_gr_trap_intr()
1436 		nvkm_wr32(device, 0x404600, 0xc0000000);  in gf100_gr_trap_intr()
1437 		nvkm_wr32(device, 0x400108, 0x00000002);  in gf100_gr_trap_intr()
1438 		trap &= ~0x00000002;  in gf100_gr_trap_intr()
1441 	if (trap & 0x00000008) {  in gf100_gr_trap_intr()
1442 		u32 stat = nvkm_rd32(device, 0x408030);  in gf100_gr_trap_intr()
1445 			       stat & 0x3fffffff);  in gf100_gr_trap_intr()
1447 		nvkm_wr32(device, 0x408030, 0xc0000000);  in gf100_gr_trap_intr()
1448 		nvkm_wr32(device, 0x400108, 0x00000008);  in gf100_gr_trap_intr()
1449 		trap &= ~0x00000008;  in gf100_gr_trap_intr()
1452 	if (trap & 0x00000010) {  in gf100_gr_trap_intr()
1453 		u32 stat = nvkm_rd32(device, 0x405840);  in gf100_gr_trap_intr()
1454 		nvkm_error(subdev, "SHADER %08x, sph: 0x%06x, stage: 0x%02x\n",  in gf100_gr_trap_intr()
1455 			   stat, stat & 0xffffff, (stat >> 24) & 0x3f);  in gf100_gr_trap_intr()
1456 		nvkm_wr32(device, 0x405840, 0xc0000000);  in gf100_gr_trap_intr()
1457 		nvkm_wr32(device, 0x400108, 0x00000010);  in gf100_gr_trap_intr()
1458 		trap &= ~0x00000010;  in gf100_gr_trap_intr()
1461 	if (trap & 0x00000040) {  in gf100_gr_trap_intr()
1462 		u32 stat = nvkm_rd32(device, 0x40601c);  in gf100_gr_trap_intr()
1465 			       stat & 0x3fffffff);  in gf100_gr_trap_intr()
1468 		nvkm_wr32(device, 0x40601c, 0xc0000000);  in gf100_gr_trap_intr()
1469 		nvkm_wr32(device, 0x400108, 0x00000040);  in gf100_gr_trap_intr()
1470 		trap &= ~0x00000040;  in gf100_gr_trap_intr()
1473 	if (trap & 0x00000080) {  in gf100_gr_trap_intr()
1474 		u32 stat = nvkm_rd32(device, 0x404490);  in gf100_gr_trap_intr()
1475 		u32 pc = nvkm_rd32(device, 0x404494);  in gf100_gr_trap_intr()
1476 		u32 op = nvkm_rd32(device, 0x40449c);  in gf100_gr_trap_intr()
1479 			       stat & 0x1fffffff);  in gf100_gr_trap_intr()
1480 		nvkm_error(subdev, "MACRO %08x [%s], pc: 0x%03x%s, op: 0x%08x\n",  in gf100_gr_trap_intr()
1481 			   stat, error, pc & 0x7ff,  in gf100_gr_trap_intr()
1482 			   (pc & 0x10000000) ? "" : " (invalid)",  in gf100_gr_trap_intr()
1485 		nvkm_wr32(device, 0x404490, 0xc0000000);  in gf100_gr_trap_intr()
1486 		nvkm_wr32(device, 0x400108, 0x00000080);  in gf100_gr_trap_intr()
1487 		trap &= ~0x00000080;  in gf100_gr_trap_intr()
1490 	if (trap & 0x00000100) {  in gf100_gr_trap_intr()
1491 		u32 stat = nvkm_rd32(device, 0x407020) & 0x3fffffff;  in gf100_gr_trap_intr()
1497 			nvkm_wr32(device, 0x407020, 0x40000000);  in gf100_gr_trap_intr()
1498 		nvkm_wr32(device, 0x400108, 0x00000100);  in gf100_gr_trap_intr()
1499 		trap &= ~0x00000100;  in gf100_gr_trap_intr()
1502 	if (trap & 0x01000000) {  in gf100_gr_trap_intr()
1503 		u32 stat = nvkm_rd32(device, 0x400118);  in gf100_gr_trap_intr()
1504 		for (gpc = 0; stat && gpc < gr->gpc_nr; gpc++) {  in gf100_gr_trap_intr()
1505 			u32 mask = 0x00000001 << gpc;  in gf100_gr_trap_intr()
1508 				nvkm_wr32(device, 0x400118, mask);  in gf100_gr_trap_intr()
1512 		nvkm_wr32(device, 0x400108, 0x01000000);  in gf100_gr_trap_intr()
1513 		trap &= ~0x01000000;  in gf100_gr_trap_intr()
1516 	if (trap & 0x02000000) {  in gf100_gr_trap_intr()
1517 		for (rop = 0; rop < gr->rop_nr; rop++) {  in gf100_gr_trap_intr()
1518 			u32 statz = nvkm_rd32(device, ROP_UNIT(rop, 0x070));  in gf100_gr_trap_intr()
1519 			u32 statc = nvkm_rd32(device, ROP_UNIT(rop, 0x144));  in gf100_gr_trap_intr()
1522 			nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0xc0000000);  in gf100_gr_trap_intr()
1523 			nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0xc0000000);  in gf100_gr_trap_intr()
1525 		nvkm_wr32(device, 0x400108, 0x02000000);  in gf100_gr_trap_intr()
1526 		trap &= ~0x02000000;  in gf100_gr_trap_intr()
1531 		nvkm_wr32(device, 0x400108, trap);  in gf100_gr_trap_intr()
1541 		   nvkm_rd32(device, base + 0x400));  in gf100_gr_ctxctl_debug_unit()
1543 		   nvkm_rd32(device, base + 0x800),  in gf100_gr_ctxctl_debug_unit()
1544 		   nvkm_rd32(device, base + 0x804),  in gf100_gr_ctxctl_debug_unit()
1545 		   nvkm_rd32(device, base + 0x808),  in gf100_gr_ctxctl_debug_unit()
1546 		   nvkm_rd32(device, base + 0x80c));  in gf100_gr_ctxctl_debug_unit()
1548 		   nvkm_rd32(device, base + 0x810),  in gf100_gr_ctxctl_debug_unit()
1549 		   nvkm_rd32(device, base + 0x814),  in gf100_gr_ctxctl_debug_unit()
1550 		   nvkm_rd32(device, base + 0x818),  in gf100_gr_ctxctl_debug_unit()
1551 		   nvkm_rd32(device, base + 0x81c));  in gf100_gr_ctxctl_debug_unit()
1558 	u32 gpcnr = nvkm_rd32(device, 0x409604) & 0xffff;  in gf100_gr_ctxctl_debug()
1561 	gf100_gr_ctxctl_debug_unit(gr, 0x409000);  in gf100_gr_ctxctl_debug()
1562 	for (gpc = 0; gpc < gpcnr; gpc++)  in gf100_gr_ctxctl_debug()
1563 		gf100_gr_ctxctl_debug_unit(gr, 0x502000 + (gpc * 0x8000));  in gf100_gr_ctxctl_debug()
1571 	u32 stat = nvkm_rd32(device, 0x409c18);  in gf100_gr_ctxctl_isr()
1573 	if (!gr->firmware && (stat & 0x00000001)) {  in gf100_gr_ctxctl_isr()
1574 		u32 code = nvkm_rd32(device, 0x409814);  in gf100_gr_ctxctl_isr()
1576 			u32 class = nvkm_rd32(device, 0x409808);  in gf100_gr_ctxctl_isr()
1577 			u32  addr = nvkm_rd32(device, 0x40980c);  in gf100_gr_ctxctl_isr()
1578 			u32  subc = (addr & 0x00070000) >> 16;  in gf100_gr_ctxctl_isr()
1579 			u32  mthd = (addr & 0x00003ffc);  in gf100_gr_ctxctl_isr()
1580 			u32  data = nvkm_rd32(device, 0x409810);  in gf100_gr_ctxctl_isr()
1588 		nvkm_wr32(device, 0x409c20, 0x00000001);  in gf100_gr_ctxctl_isr()
1589 		stat &= ~0x00000001;  in gf100_gr_ctxctl_isr()
1592 	if (!gr->firmware && (stat & 0x00080000)) {  in gf100_gr_ctxctl_isr()
1595 		nvkm_wr32(device, 0x409c20, 0x00080000);  in gf100_gr_ctxctl_isr()
1596 		stat &= ~0x00080000;  in gf100_gr_ctxctl_isr()
1602 		nvkm_wr32(device, 0x409c20, stat);  in gf100_gr_ctxctl_isr()
1614 	u64 inst = nvkm_rd32(device, 0x409b00) & 0x0fffffff;  in gf100_gr_intr()
1615 	u32 stat = nvkm_rd32(device, 0x400100);  in gf100_gr_intr()
1616 	u32 addr = nvkm_rd32(device, 0x400704);  in gf100_gr_intr()
1617 	u32 mthd = (addr & 0x00003ffc);  in gf100_gr_intr()
1618 	u32 subc = (addr & 0x00070000) >> 16;  in gf100_gr_intr()
1619 	u32 data = nvkm_rd32(device, 0x400708);  in gf100_gr_intr()
1620 	u32 code = nvkm_rd32(device, 0x400110);  in gf100_gr_intr()
1632 		class = nvkm_rd32(device, 0x404200 + (subc * 4));  in gf100_gr_intr()
1634 		class = 0x0000;  in gf100_gr_intr()
1636 	if (stat & 0x00000001) {  in gf100_gr_intr()
1641 		nvkm_wr32(device, 0x400100, 0x00000001);  in gf100_gr_intr()
1642 		stat &= ~0x00000001;  in gf100_gr_intr()
1645 	if (stat & 0x00000010) {  in gf100_gr_intr()
1652 		nvkm_wr32(device, 0x400100, 0x00000010);  in gf100_gr_intr()
1653 		stat &= ~0x00000010;  in gf100_gr_intr()
1656 	if (stat & 0x00000020) {  in gf100_gr_intr()
1660 		nvkm_wr32(device, 0x400100, 0x00000020);  in gf100_gr_intr()
1661 		stat &= ~0x00000020;  in gf100_gr_intr()
1664 	if (stat & 0x00100000) {  in gf100_gr_intr()
1671 		nvkm_wr32(device, 0x400100, 0x00100000);  in gf100_gr_intr()
1672 		stat &= ~0x00100000;  in gf100_gr_intr()
1675 	if (stat & 0x00200000) {  in gf100_gr_intr()
1679 		nvkm_wr32(device, 0x400100, 0x00200000);  in gf100_gr_intr()
1680 		stat &= ~0x00200000;  in gf100_gr_intr()
1683 	if (stat & 0x00080000) {  in gf100_gr_intr()
1685 		nvkm_wr32(device, 0x400100, 0x00080000);  in gf100_gr_intr()
1686 		stat &= ~0x00080000;  in gf100_gr_intr()
1691 		nvkm_wr32(device, 0x400100, stat);  in gf100_gr_intr()
1694 	nvkm_wr32(device, 0x400500, 0x00010001);  in gf100_gr_intr()
1703 	nvkm_falcon_load_dmem(falcon, data->data, 0x0, data->size, 0);  in gf100_gr_init_fw()
1704 	nvkm_falcon_load_imem(falcon, code->data, 0x0, code->size, 0, 0, false);  in gf100_gr_init_fw()
1715 	u32 addr = ~0, prev = ~0, xfer = 0;  in gf100_gr_init_csdata()
1718 	nvkm_wr32(device, falcon + 0x01c0, 0x02000000 + starstar);  in gf100_gr_init_csdata()
1719 	star = nvkm_rd32(device, falcon + 0x01c4);  in gf100_gr_init_csdata()
1720 	temp = nvkm_rd32(device, falcon + 0x01c4);  in gf100_gr_init_csdata()
1723 	nvkm_wr32(device, falcon + 0x01c0, 0x01000000 + star);  in gf100_gr_init_csdata()
1732 					nvkm_wr32(device, falcon + 0x01c4, data);  in gf100_gr_init_csdata()
1736 				xfer = 0;  in gf100_gr_init_csdata()
1744 	nvkm_wr32(device, falcon + 0x01c4, (--xfer << 26) | addr);  in gf100_gr_init_csdata()
1745 	nvkm_wr32(device, falcon + 0x01c0, 0x01000004 + starstar);  in gf100_gr_init_csdata()
1746 	nvkm_wr32(device, falcon + 0x01c4, star + 4);  in gf100_gr_init_csdata()
1755 	u32 lsf_mask = 0;  in gf100_gr_init_ctxctl_ext()
1759 	nvkm_mc_unk260(device, 0);  in gf100_gr_init_ctxctl_ext()
1786 	nvkm_wr32(device, 0x409800, 0x00000000);  in gf100_gr_init_ctxctl_ext()
1787 	nvkm_wr32(device, 0x41a10c, 0x00000000);  in gf100_gr_init_ctxctl_ext()
1788 	nvkm_wr32(device, 0x40910c, 0x00000000);  in gf100_gr_init_ctxctl_ext()
1794 		if (nvkm_rd32(device, 0x409800) & 0x00000001)  in gf100_gr_init_ctxctl_ext()
1796 	) < 0)  in gf100_gr_init_ctxctl_ext()
1799 	gf100_gr_fecs_set_watchdog_timeout(gr, 0x7fffffff);  in gf100_gr_init_ctxctl_ext()
1822 	if (0) {  in gf100_gr_init_ctxctl_ext()
1828 	return 0;  in gf100_gr_init_ctxctl_ext()
1843 	nvkm_mc_unk260(device, 0);  in gf100_gr_init_ctxctl_int()
1845 			      gr->func->fecs.ucode->data.data, 0x0,  in gf100_gr_init_ctxctl_int()
1846 			      gr->func->fecs.ucode->data.size, 0);  in gf100_gr_init_ctxctl_int()
1848 			      gr->func->fecs.ucode->code.data, 0x0,  in gf100_gr_init_ctxctl_int()
1849 			      gr->func->fecs.ucode->code.size, 0, 0, false);  in gf100_gr_init_ctxctl_int()
1853 			      gr->func->gpccs.ucode->data.data, 0x0,  in gf100_gr_init_ctxctl_int()
1854 			      gr->func->gpccs.ucode->data.size, 0);  in gf100_gr_init_ctxctl_int()
1856 			      gr->func->gpccs.ucode->code.data, 0x0,  in gf100_gr_init_ctxctl_int()
1857 			      gr->func->gpccs.ucode->code.size, 0, 0, false);  in gf100_gr_init_ctxctl_int()
1861 	gf100_gr_init_csdata(gr, grctx->hub, 0x409000, 0x000, 0x000000);  in gf100_gr_init_ctxctl_int()
1862 	gf100_gr_init_csdata(gr, grctx->gpc_0, 0x41a000, 0x000, 0x418000);  in gf100_gr_init_ctxctl_int()
1863 	gf100_gr_init_csdata(gr, grctx->gpc_1, 0x41a000, 0x000, 0x418000);  in gf100_gr_init_ctxctl_int()
1864 	gf100_gr_init_csdata(gr, grctx->tpc, 0x41a000, 0x004, 0x419800);  in gf100_gr_init_ctxctl_int()
1865 	gf100_gr_init_csdata(gr, grctx->ppc, 0x41a000, 0x008, 0x41be00);  in gf100_gr_init_ctxctl_int()
1868 	nvkm_wr32(device, 0x40910c, 0x00000000);  in gf100_gr_init_ctxctl_int()
1869 	nvkm_wr32(device, 0x409100, 0x00000002);  in gf100_gr_init_ctxctl_int()
1871 		if (nvkm_rd32(device, 0x409800) & 0x80000000)  in gf100_gr_init_ctxctl_int()
1873 	) < 0) {  in gf100_gr_init_ctxctl_int()
1878 	gr->size = nvkm_rd32(device, 0x409804);  in gf100_gr_init_ctxctl_int()
1879 	return 0;  in gf100_gr_init_ctxctl_int()
1900 	for (tpc = 0; tpc < gr->tpc_max; tpc++) {  in gf100_gr_oneinit_sm_id()
1901 		for (gpc = 0; gpc < gr->gpc_nr; gpc++) {  in gf100_gr_oneinit_sm_id()
1910 	return 0;  in gf100_gr_oneinit_sm_id()
1925 	case 15: gr->screen_tile_row_offset = 0x06; break;  in gf100_gr_oneinit_tiles()
1926 	case 14: gr->screen_tile_row_offset = 0x05; break;  in gf100_gr_oneinit_tiles()
1927 	case 13: gr->screen_tile_row_offset = 0x02; break;  in gf100_gr_oneinit_tiles()
1928 	case 11: gr->screen_tile_row_offset = 0x07; break;  in gf100_gr_oneinit_tiles()
1929 	case 10: gr->screen_tile_row_offset = 0x06; break;  in gf100_gr_oneinit_tiles()
1931 	case  5: gr->screen_tile_row_offset = 0x01; break;  in gf100_gr_oneinit_tiles()
1932 	case  3: gr->screen_tile_row_offset = 0x02; break;  in gf100_gr_oneinit_tiles()
1934 	case  1: gr->screen_tile_row_offset = 0x01; break;  in gf100_gr_oneinit_tiles()
1935 	default: gr->screen_tile_row_offset = 0x03;  in gf100_gr_oneinit_tiles()
1936 		for (i = 0; i < ARRAY_SIZE(primes); i++) {  in gf100_gr_oneinit_tiles()
1946 	for (i = 0; i < gr->gpc_nr; i++)  in gf100_gr_oneinit_tiles()
1951 		for (sorted = true, i = 0; i < gr->gpc_nr - 1; i++) {  in gf100_gr_oneinit_tiles()
1953 			    gr->tpc_nr[gpc_map[i + 0]]) {  in gf100_gr_oneinit_tiles()
1955 				gpc_map[i + 0] = gpc_map[i + 1];  in gf100_gr_oneinit_tiles()
1971 	for (i = 0; i < gr->gpc_nr; i++) {  in gf100_gr_oneinit_tiles()
1977 	for (i = 0; i < gr->tpc_total;) {  in gf100_gr_oneinit_tiles()
1978 		for (j = 0; j < gr->gpc_nr; j++) {  in gf100_gr_oneinit_tiles()
2010 	gr->gpc_nr = nvkm_rd32(device, 0x409604) & 0x0000001f;  in gf100_gr_oneinit()
2011 	for (i = 0; i < gr->gpc_nr; i++) {  in gf100_gr_oneinit()
2012 		gr->tpc_nr[i]  = nvkm_rd32(device, GPC_UNIT(i, 0x2608));  in gf100_gr_oneinit()
2015 		for (j = 0; j < gr->func->ppc_nr; j++) {  in gf100_gr_oneinit()
2017 				nvkm_rd32(device, GPC_UNIT(i, 0x0c30 + (j * 4)));  in gf100_gr_oneinit()
2018 			if (gr->ppc_tpc_mask[i][j] == 0)  in gf100_gr_oneinit()
2025 			if (gr->ppc_tpc_min == 0 ||  in gf100_gr_oneinit()
2037 			      gr->func->grctx->pagepool_size, 0x100, false, &gr->pagepool);  in gf100_gr_oneinit()
2042 			      0x100, false, &gr->bundle_cb);  in gf100_gr_oneinit()
2047 			      gr->func->grctx->attrib_cb_size(gr), 0x1000, false, &gr->attrib_cb);  in gf100_gr_oneinit()
2053 				      0x100, false, &gr->unknown);  in gf100_gr_oneinit()
2058 	memset(gr->tile, 0xff, sizeof(gr->tile));  in gf100_gr_oneinit()
2070 	bool reset = device->chipset == 0x137 || device->chipset == 0x138;  in gf100_gr_init_()
2090 		nvkm_mask(device, 0x000200, 0x00001000, 0x00000000);  in gf100_gr_init_()
2091 		nvkm_rd32(device, 0x000200);  in gf100_gr_init_()
2093 		nvkm_mask(device, 0x000200, 0x00001000, 0x00001000);  in gf100_gr_init_()
2094 		nvkm_rd32(device, 0x000200);  in gf100_gr_init_()
2112 	return 0;  in gf100_gr_init_()
2125 	return 0;  in gf100_gr_fini()
2176 	for (gpc = 0, i = 0; i < 4; i++) {  in gf100_gr_init_num_tpc_per_gpc()
2177 		for (data = 0, j = 0; j < 8 && gpc < gr->gpc_nr; j++, gpc++)  in gf100_gr_init_num_tpc_per_gpc()
2180 			nvkm_wr32(device, 0x406028 + (i * 4), data);  in gf100_gr_init_num_tpc_per_gpc()
2182 			nvkm_wr32(device, 0x405870 + (i * 4), data);  in gf100_gr_init_num_tpc_per_gpc()
2189 	nvkm_wr32(gr->base.engine.subdev.device, 0x400054, 0x34ce3464);  in gf100_gr_init_400054()
2197 	nvkm_wr32(device, 0x40011c, 0xffffffff);  in gf100_gr_init_exception2()
2198 	nvkm_wr32(device, 0x400134, 0xffffffff);  in gf100_gr_init_exception2()
2207 	for (rop = 0; rop < gr->rop_nr; rop++) {  in gf100_gr_init_rop_exceptions()
2208 		nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000);  in gf100_gr_init_rop_exceptions()
2209 		nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000);  in gf100_gr_init_rop_exceptions()
2210 		nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);  in gf100_gr_init_rop_exceptions()
2211 		nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);  in gf100_gr_init_rop_exceptions()
2219 	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);  in gf100_gr_init_shader_exceptions()
2220 	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);  in gf100_gr_init_shader_exceptions()
2227 	nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);  in gf100_gr_init_tex_hww_esr()
2234 	nvkm_mask(device, 0x419eb4, 0x00001000, 0x00001000);  in gf100_gr_init_419eb4()
2243 	nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);  in gf100_gr_init_419cc0()
2245 	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {  in gf100_gr_init_419cc0()
2246 		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++)  in gf100_gr_init_419cc0()
2247 			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);  in gf100_gr_init_419cc0()
2254 	nvkm_wr32(gr->base.engine.subdev.device, 0x40601c, 0xc0000000);  in gf100_gr_init_40601c()
2260 	const u32 data = gr->firmware ? 0x000e0000 : 0x000e0001;  in gf100_gr_init_fecs_exceptions()
2261 	nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, data);  in gf100_gr_init_fecs_exceptions()
2270 	nvkm_wr32(device, 0x418880, nvkm_rd32(device, 0x100c80) & 0x00000001);  in gf100_gr_init_gpc_mmu()
2271 	nvkm_wr32(device, 0x4188a4, 0x03000000);  in gf100_gr_init_gpc_mmu()
2272 	nvkm_wr32(device, 0x418888, 0x00000000);  in gf100_gr_init_gpc_mmu()
2273 	nvkm_wr32(device, 0x41888c, 0x00000000);  in gf100_gr_init_gpc_mmu()
2274 	nvkm_wr32(device, 0x418890, 0x00000000);  in gf100_gr_init_gpc_mmu()
2275 	nvkm_wr32(device, 0x418894, 0x00000000);  in gf100_gr_init_gpc_mmu()
2276 	nvkm_wr32(device, 0x4188b4, nvkm_memory_addr(fb->mmu_wr) >> 8);  in gf100_gr_init_gpc_mmu()
2277 	nvkm_wr32(device, 0x4188b8, nvkm_memory_addr(fb->mmu_rd) >> 8);  in gf100_gr_init_gpc_mmu()
2284 	nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));  in gf100_gr_init_num_active_ltcs()
2291 	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);  in gf100_gr_init_zcull()
2296 	for (i = 0; i < tile_nr; i += 8) {  in gf100_gr_init_zcull()
2297 		for (data = 0, j = 0; j < 8 && i + j < gr->tpc_total; j++) {  in gf100_gr_init_zcull()
2301 		nvkm_wr32(device, GPC_BCAST(0x0980 + ((i / 8) * 4)), data);  in gf100_gr_init_zcull()
2304 	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {  in gf100_gr_init_zcull()
2305 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),  in gf100_gr_init_zcull()
2307 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |  in gf100_gr_init_zcull()
2309 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);  in gf100_gr_init_zcull()
2312 	nvkm_wr32(device, GPC_BCAST(0x1bd4), magicgpc918);  in gf100_gr_init_zcull()
2319 	nvkm_mask(device, TPC_UNIT(0, 0, 0x05c), 0x00000001, 0x00000001);  in gf100_gr_init_vsc_stream_master()
2329 	nvkm_mask(device, 0x400500, 0x00000001, 0x00000000);  in gf100_gr_reset()
2348 	nvkm_mask(device, 0x400500, 0x00010001, 0x00000000);  in gf100_gr_init()
2387 	nvkm_wr32(device, 0x400500, 0x00010001);  in gf100_gr_init()
2389 	nvkm_wr32(device, 0x400100, 0xffffffff);  in gf100_gr_init()
2390 	nvkm_wr32(device, 0x40013c, 0xffffffff);  in gf100_gr_init()
2391 	nvkm_wr32(device, 0x400124, 0x00000002);  in gf100_gr_init()
2401 	nvkm_wr32(device, 0x404000, 0xc0000000);  in gf100_gr_init()
2402 	nvkm_wr32(device, 0x404600, 0xc0000000);  in gf100_gr_init()
2403 	nvkm_wr32(device, 0x408030, 0xc0000000);  in gf100_gr_init()
2408 	nvkm_wr32(device, 0x406018, 0xc0000000);  in gf100_gr_init()
2409 	nvkm_wr32(device, 0x404490, 0xc0000000);  in gf100_gr_init()
2414 	nvkm_wr32(device, 0x405840, 0xc0000000);  in gf100_gr_init()
2415 	nvkm_wr32(device, 0x405844, 0x00ffffff);  in gf100_gr_init()
2427 	for (gpc = 0; gpc < gr->gpc_nr; gpc++) {  in gf100_gr_init()
2428 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);  in gf100_gr_init()
2429 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);  in gf100_gr_init()
2430 		nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);  in gf100_gr_init()
2431 		nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);  in gf100_gr_init()
2432 		for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {  in gf100_gr_init()
2433 			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);  in gf100_gr_init()
2434 			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);  in gf100_gr_init()
2437 			nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);  in gf100_gr_init()
2442 		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);  in gf100_gr_init()
2443 		nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);  in gf100_gr_init()
2448 	nvkm_wr32(device, 0x400108, 0xffffffff);  in gf100_gr_init()
2449 	nvkm_wr32(device, 0x400138, 0xffffffff);  in gf100_gr_init()
2450 	nvkm_wr32(device, 0x400118, 0xffffffff);  in gf100_gr_init()
2451 	nvkm_wr32(device, 0x400130, 0xffffffff);  in gf100_gr_init()
2471 	nvkm_wr32(device, 0x409614, 0x00000070);  in gf100_gr_fecs_reset()
2473 	nvkm_mask(device, 0x409614, 0x00000700, 0x00000700);  in gf100_gr_fecs_reset()
2475 	nvkm_rd32(device, 0x409614);  in gf100_gr_fecs_reset()
2565 	return 0;  in gf100_gr_nofw()
2592 	return (blob->data != NULL) ? 0 : -ENOMEM;  in gf100_gr_load_fw()
2610 	return 0;  in gf100_gr_load()
2642 			       "fecs", 0x409000, &gr->fecs.falcon);  in gf100_gr_new_()
2649 			       "gpccs", 0x41a000, &gr->gpccs.falcon);  in gf100_gr_new_()
2653 	return 0;  in gf100_gr_new_()