On Fri, Feb 7, 2025 at 6:57 PM Jay Cornwall <jay.cornwall@xxxxxxx> wrote:
>
> VALU instructions with SGPR source need wait states to avoid hazard
> with SALU using different SGPR.
>
> v2: Eliminate some hazards to reduce code explosion
>
> Signed-off-by: Jay Cornwall <jay.cornwall@xxxxxxx>
> Cc: Lancelot Six <lancelot.six@xxxxxxx>
Acked-by: Alex Deucher <alexander.deucher@xxxxxxx>
> ---
> .../gpu/drm/amd/amdkfd/cwsr_trap_handler.h | 677 ++++++++++--------
> .../amd/amdkfd/cwsr_trap_handler_gfx12.asm | 82 ++-
> 2 files changed, 404 insertions(+), 355 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
> index 651660958e5b..0320163b6e74 100644
> --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
> +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
> @@ -3644,7 +3644,7 @@ static const uint32_t cwsr_trap_gfx9_4_3_hex[] = {
> };
>
> static const uint32_t cwsr_trap_gfx12_hex[] = {
> - 0xbfa00001, 0xbfa0024b,
> + 0xbfa00001, 0xbfa002a2,
> 0xb0804009, 0xb8f8f804,
> 0x9178ff78, 0x00008c00,
> 0xb8fbf811, 0x8b6eff78,
> @@ -3718,7 +3718,15 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
> 0x00011677, 0xd7610000,
> 0x00011a79, 0xd7610000,
> 0x00011c7e, 0xd7610000,
> - 0x00011e7f, 0xbefe00ff,
> + 0x00011e7f, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xbefe00ff,
> 0x00003fff, 0xbeff0080,
> 0xee0a407a, 0x000c0000,
> 0x00004000, 0xd760007a,
> @@ -3755,38 +3763,46 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
> 0x00000200, 0xbef600ff,
> 0x01000000, 0x7e000280,
> 0x7e020280, 0x7e040280,
> - 0xbefd0080, 0xbe804ec2,
> - 0xbf94fffe, 0xb8faf804,
> - 0x8b7a847a, 0x91788478,
> - 0x8c787a78, 0xd7610002,
> - 0x0000fa71, 0x807d817d,
> - 0xd7610002, 0x0000fa6c,
> - 0x807d817d, 0x917aff6d,
> - 0x80000000, 0xd7610002,
> - 0x0000fa7a, 0x807d817d,
> - 0xd7610002, 0x0000fa6e,
> - 0x807d817d, 0xd7610002,
> - 0x0000fa6f, 0x807d817d,
> - 0xd7610002, 0x0000fa78,
> - 0x807d817d, 0xb8faf811,
> - 0xd7610002, 0x0000fa7a,
> - 0x807d817d, 0xd7610002,
> - 0x0000fa7b, 0x807d817d,
> - 0xb8f1f801, 0xd7610002,
> - 0x0000fa71, 0x807d817d,
> - 0xb8f1f814, 0xd7610002,
> - 0x0000fa71, 0x807d817d,
> - 0xb8f1f815, 0xd7610002,
> - 0x0000fa71, 0x807d817d,
> - 0xb8f1f812, 0xd7610002,
> - 0x0000fa71, 0x807d817d,
> - 0xb8f1f813, 0xd7610002,
> - 0x0000fa71, 0x807d817d,
> + 0xbe804ec2, 0xbf94fffe,
> + 0xb8faf804, 0x8b7a847a,
> + 0x91788478, 0x8c787a78,
> + 0x917aff6d, 0x80000000,
> + 0xd7610002, 0x00010071,
> + 0xd7610002, 0x0001026c,
> + 0xd7610002, 0x0001047a,
> + 0xd7610002, 0x0001066e,
> + 0xd7610002, 0x0001086f,
> + 0xd7610002, 0x00010a78,
> + 0xd7610002, 0x00010e7b,
> + 0xd8500000, 0x00000000,
> + 0xd8500000, 0x00000000,
> + 0xd8500000, 0x00000000,
> + 0xd8500000, 0x00000000,
> + 0xd8500000, 0x00000000,
> + 0xd8500000, 0x00000000,
> + 0xd8500000, 0x00000000,
> + 0xd8500000, 0x00000000,
> + 0xb8faf811, 0xd7610002,
> + 0x00010c7a, 0xb8faf801,
> + 0xd7610002, 0x0001107a,
> + 0xb8faf814, 0xd7610002,
> + 0x0001127a, 0xb8faf815,
> + 0xd7610002, 0x0001147a,
> + 0xb8faf812, 0xd7610002,
> + 0x0001167a, 0xb8faf813,
> + 0xd7610002, 0x0001187a,
> 0xb8faf802, 0xd7610002,
> - 0x0000fa7a, 0x807d817d,
> - 0xbefa50c1, 0xbfc70000,
> - 0xd7610002, 0x0000fa7a,
> - 0x807d817d, 0xbefe00ff,
> + 0x00011a7a, 0xbefa50c1,
> + 0xbfc70000, 0xd7610002,
> + 0x00011c7a, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xbefe00ff,
> 0x0000ffff, 0xbeff0080,
> 0xc4068070, 0x008ce802,
> 0x00000000, 0xbefe00c1,
> @@ -3801,331 +3817,358 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
> 0xbe824102, 0xbe844104,
> 0xbe864106, 0xbe884108,
> 0xbe8a410a, 0xbe8c410c,
> - 0xbe8e410e, 0xd7610002,
> - 0x0000f200, 0x80798179,
> - 0xd7610002, 0x0000f201,
> - 0x80798179, 0xd7610002,
> - 0x0000f202, 0x80798179,
> - 0xd7610002, 0x0000f203,
> - 0x80798179, 0xd7610002,
> - 0x0000f204, 0x80798179,
> - 0xd7610002, 0x0000f205,
> - 0x80798179, 0xd7610002,
> - 0x0000f206, 0x80798179,
> - 0xd7610002, 0x0000f207,
> - 0x80798179, 0xd7610002,
> - 0x0000f208, 0x80798179,
> - 0xd7610002, 0x0000f209,
> - 0x80798179, 0xd7610002,
> - 0x0000f20a, 0x80798179,
> - 0xd7610002, 0x0000f20b,
> - 0x80798179, 0xd7610002,
> - 0x0000f20c, 0x80798179,
> - 0xd7610002, 0x0000f20d,
> - 0x80798179, 0xd7610002,
> - 0x0000f20e, 0x80798179,
> - 0xd7610002, 0x0000f20f,
> - 0x80798179, 0xbf06a079,
> - 0xbfa10007, 0xc4068070,
> + 0xbe8e410e, 0xbf068079,
> + 0xbfa10032, 0xd7610002,
> + 0x00010000, 0xd7610002,
> + 0x00010201, 0xd7610002,
> + 0x00010402, 0xd7610002,
> + 0x00010603, 0xd7610002,
> + 0x00010804, 0xd7610002,
> + 0x00010a05, 0xd7610002,
> + 0x00010c06, 0xd7610002,
> + 0x00010e07, 0xd7610002,
> + 0x00011008, 0xd7610002,
> + 0x00011209, 0xd7610002,
> + 0x0001140a, 0xd7610002,
> + 0x0001160b, 0xd7610002,
> + 0x0001180c, 0xd7610002,
> + 0x00011a0d, 0xd7610002,
> + 0x00011c0e, 0xd7610002,
> + 0x00011e0f, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0x80799079,
> + 0xbfa00038, 0xd7610002,
> + 0x00012000, 0xd7610002,
> + 0x00012201, 0xd7610002,
> + 0x00012402, 0xd7610002,
> + 0x00012603, 0xd7610002,
> + 0x00012804, 0xd7610002,
> + 0x00012a05, 0xd7610002,
> + 0x00012c06, 0xd7610002,
> + 0x00012e07, 0xd7610002,
> + 0x00013008, 0xd7610002,
> + 0x00013209, 0xd7610002,
> + 0x0001340a, 0xd7610002,
> + 0x0001360b, 0xd7610002,
> + 0x0001380c, 0xd7610002,
> + 0x00013a0d, 0xd7610002,
> + 0x00013c0e, 0xd7610002,
> + 0x00013e0f, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0x80799079,
> + 0xc4068070, 0x008ce802,
> + 0x00000000, 0x8070ff70,
> + 0x00000080, 0xbef90080,
> + 0x7e040280, 0x807d907d,
> + 0xbf0aff7d, 0x00000060,
> + 0xbfa2ff88, 0xbe804100,
> + 0xbe824102, 0xbe844104,
> + 0xbe864106, 0xbe884108,
> + 0xbe8a410a, 0xd7610002,
> + 0x00010000, 0xd7610002,
> + 0x00010201, 0xd7610002,
> + 0x00010402, 0xd7610002,
> + 0x00010603, 0xd7610002,
> + 0x00010804, 0xd7610002,
> + 0x00010a05, 0xd7610002,
> + 0x00010c06, 0xd7610002,
> + 0x00010e07, 0xd7610002,
> + 0x00011008, 0xd7610002,
> + 0x00011209, 0xd7610002,
> + 0x0001140a, 0xd7610002,
> + 0x0001160b, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xd8500000,
> + 0x00000000, 0xc4068070,
> 0x008ce802, 0x00000000,
> + 0xbefe00c1, 0x857d9973,
> + 0x8b7d817d, 0xbf06817d,
> + 0xbfa20002, 0xbeff0080,
> + 0xbfa00001, 0xbeff00c1,
> + 0xb8fb4306, 0x8b7bc17b,
> + 0xbfa10044, 0x8b7aff6d,
> + 0x80000000, 0xbfa10041,
> + 0x847b897b, 0xbef6007b,
> + 0xb8f03b05, 0x80708170,
> + 0xbf0d9973, 0xbfa20002,
> + 0x84708970, 0xbfa00001,
> + 0x84708a70, 0xb8fa1e06,
> + 0x847a8a7a, 0x80707a70,
> + 0x8070ff70, 0x00000200,
> 0x8070ff70, 0x00000080,
> - 0xbef90080, 0x7e040280,
> - 0x807d907d, 0xbf0aff7d,
> - 0x00000060, 0xbfa2ffbb,
> - 0xbe804100, 0xbe824102,
> - 0xbe844104, 0xbe864106,
> - 0xbe884108, 0xbe8a410a,
> - 0xd7610002, 0x0000f200,
> - 0x80798179, 0xd7610002,
> - 0x0000f201, 0x80798179,
> - 0xd7610002, 0x0000f202,
> - 0x80798179, 0xd7610002,
> - 0x0000f203, 0x80798179,
> - 0xd7610002, 0x0000f204,
> - 0x80798179, 0xd7610002,
> - 0x0000f205, 0x80798179,
> - 0xd7610002, 0x0000f206,
> - 0x80798179, 0xd7610002,
> - 0x0000f207, 0x80798179,
> - 0xd7610002, 0x0000f208,
> - 0x80798179, 0xd7610002,
> - 0x0000f209, 0x80798179,
> - 0xd7610002, 0x0000f20a,
> - 0x80798179, 0xd7610002,
> - 0x0000f20b, 0x80798179,
> - 0xc4068070, 0x008ce802,
> - 0x00000000, 0xbefe00c1,
> - 0x857d9973, 0x8b7d817d,
> - 0xbf06817d, 0xbfa20002,
> - 0xbeff0080, 0xbfa00001,
> - 0xbeff00c1, 0xb8fb4306,
> - 0x8b7bc17b, 0xbfa10044,
> - 0x8b7aff6d, 0x80000000,
> - 0xbfa10041, 0x847b897b,
> - 0xbef6007b, 0xb8f03b05,
> - 0x80708170, 0xbf0d9973,
> - 0xbfa20002, 0x84708970,
> - 0xbfa00001, 0x84708a70,
> - 0xb8fa1e06, 0x847a8a7a,
> - 0x80707a70, 0x8070ff70,
> - 0x00000200, 0x8070ff70,
> - 0x00000080, 0xbef600ff,
> - 0x01000000, 0xd71f0000,
> - 0x000100c1, 0xd7200000,
> - 0x000200c1, 0x16000084,
> - 0x857d9973, 0x8b7d817d,
> - 0xbf06817d, 0xbefd0080,
> - 0xbfa20013, 0xbe8300ff,
> - 0x00000080, 0xbf800000,
> - 0xbf800000, 0xbf800000,
> - 0xd8d80000, 0x01000000,
> - 0xbf8a0000, 0xc4068070,
> - 0x008ce801, 0x00000000,
> - 0x807d037d, 0x80700370,
> - 0xd5250000, 0x0001ff00,
> - 0x00000080, 0xbf0a7b7d,
> - 0xbfa2fff3, 0xbfa00012,
> - 0xbe8300ff, 0x00000100,
> + 0xbef600ff, 0x01000000,
> + 0xd71f0000, 0x000100c1,
> + 0xd7200000, 0x000200c1,
> + 0x16000084, 0x857d9973,
> + 0x8b7d817d, 0xbf06817d,
> + 0xbefd0080, 0xbfa20013,
> + 0xbe8300ff, 0x00000080,
> 0xbf800000, 0xbf800000,
> 0xbf800000, 0xd8d80000,
> 0x01000000, 0xbf8a0000,
> 0xc4068070, 0x008ce801,
> 0x00000000, 0x807d037d,
> 0x80700370, 0xd5250000,
> - 0x0001ff00, 0x00000100,
> + 0x0001ff00, 0x00000080,
> 0xbf0a7b7d, 0xbfa2fff3,
> - 0xbefe00c1, 0x857d9973,
> - 0x8b7d817d, 0xbf06817d,
> - 0xbfa20004, 0xbef000ff,
> - 0x00000200, 0xbeff0080,
> - 0xbfa00003, 0xbef000ff,
> - 0x00000400, 0xbeff00c1,
> - 0xb8fb3b05, 0x807b817b,
> - 0x847b827b, 0x857d9973,
> - 0x8b7d817d, 0xbf06817d,
> - 0xbfa2001b, 0xbef600ff,
> - 0x01000000, 0xbefd0084,
> - 0xbf0a7b7d, 0xbfa10040,
> - 0x7e008700, 0x7e028701,
> - 0x7e048702, 0x7e068703,
> - 0xc4068070, 0x008ce800,
> - 0x00000000, 0xc4068070,
> - 0x008ce801, 0x00008000,
> - 0xc4068070, 0x008ce802,
> - 0x00010000, 0xc4068070,
> - 0x008ce803, 0x00018000,
> - 0x807d847d, 0x8070ff70,
> - 0x00000200, 0xbf0a7b7d,
> - 0xbfa2ffeb, 0xbfa0002a,
> + 0xbfa00012, 0xbe8300ff,
> + 0x00000100, 0xbf800000,
> + 0xbf800000, 0xbf800000,
> + 0xd8d80000, 0x01000000,
> + 0xbf8a0000, 0xc4068070,
> + 0x008ce801, 0x00000000,
> + 0x807d037d, 0x80700370,
> + 0xd5250000, 0x0001ff00,
> + 0x00000100, 0xbf0a7b7d,
> + 0xbfa2fff3, 0xbefe00c1,
> + 0x857d9973, 0x8b7d817d,
> + 0xbf06817d, 0xbfa20004,
> + 0xbef000ff, 0x00000200,
> + 0xbeff0080, 0xbfa00003,
> + 0xbef000ff, 0x00000400,
> + 0xbeff00c1, 0xb8fb3b05,
> + 0x807b817b, 0x847b827b,
> + 0x857d9973, 0x8b7d817d,
> + 0xbf06817d, 0xbfa2001b,
> 0xbef600ff, 0x01000000,
> 0xbefd0084, 0xbf0a7b7d,
> - 0xbfa10015, 0x7e008700,
> + 0xbfa10040, 0x7e008700,
> 0x7e028701, 0x7e048702,
> 0x7e068703, 0xc4068070,
> 0x008ce800, 0x00000000,
> 0xc4068070, 0x008ce801,
> - 0x00010000, 0xc4068070,
> - 0x008ce802, 0x00020000,
> + 0x00008000, 0xc4068070,
> + 0x008ce802, 0x00010000,
> 0xc4068070, 0x008ce803,
> - 0x00030000, 0x807d847d,
> - 0x8070ff70, 0x00000400,
> + 0x00018000, 0x807d847d,
> + 0x8070ff70, 0x00000200,
> 0xbf0a7b7d, 0xbfa2ffeb,
> - 0xb8fb1e06, 0x8b7bc17b,
> - 0xbfa1000d, 0x847b837b,
> - 0x807b7d7b, 0xbefe00c1,
> - 0xbeff0080, 0x7e008700,
> + 0xbfa0002a, 0xbef600ff,
> + 0x01000000, 0xbefd0084,
> + 0xbf0a7b7d, 0xbfa10015,
> + 0x7e008700, 0x7e028701,
> + 0x7e048702, 0x7e068703,
> 0xc4068070, 0x008ce800,
> - 0x00000000, 0x807d817d,
> - 0x8070ff70, 0x00000080,
> - 0xbf0a7b7d, 0xbfa2fff7,
> - 0xbfa0016e, 0xbef4007e,
> - 0x8b75ff7f, 0x0000ffff,
> - 0x8c75ff75, 0x00040000,
> - 0xbef60080, 0xbef700ff,
> - 0x10807fac, 0xbef1007f,
> - 0xb8f20742, 0x84729972,
> - 0x8b6eff7f, 0x04000000,
> - 0xbfa1003b, 0xbefe00c1,
> - 0x857d9972, 0x8b7d817d,
> - 0xbf06817d, 0xbfa20002,
> - 0xbeff0080, 0xbfa00001,
> - 0xbeff00c1, 0xb8ef4306,
> - 0x8b6fc16f, 0xbfa10030,
> - 0x846f896f, 0xbef6006f,
> + 0x00000000, 0xc4068070,
> + 0x008ce801, 0x00010000,
> + 0xc4068070, 0x008ce802,
> + 0x00020000, 0xc4068070,
> + 0x008ce803, 0x00030000,
> + 0x807d847d, 0x8070ff70,
> + 0x00000400, 0xbf0a7b7d,
> + 0xbfa2ffeb, 0xb8fb1e06,
> + 0x8b7bc17b, 0xbfa1000d,
> + 0x847b837b, 0x807b7d7b,
> + 0xbefe00c1, 0xbeff0080,
> + 0x7e008700, 0xc4068070,
> + 0x008ce800, 0x00000000,
> + 0x807d817d, 0x8070ff70,
> + 0x00000080, 0xbf0a7b7d,
> + 0xbfa2fff7, 0xbfa0016e,
> + 0xbef4007e, 0x8b75ff7f,
> + 0x0000ffff, 0x8c75ff75,
> + 0x00040000, 0xbef60080,
> + 0xbef700ff, 0x10807fac,
> + 0xbef1007f, 0xb8f20742,
> + 0x84729972, 0x8b6eff7f,
> + 0x04000000, 0xbfa1003b,
> + 0xbefe00c1, 0x857d9972,
> + 0x8b7d817d, 0xbf06817d,
> + 0xbfa20002, 0xbeff0080,
> + 0xbfa00001, 0xbeff00c1,
> + 0xb8ef4306, 0x8b6fc16f,
> + 0xbfa10030, 0x846f896f,
> + 0xbef6006f, 0xb8f83b05,
> + 0x80788178, 0xbf0d9972,
> + 0xbfa20002, 0x84788978,
> + 0xbfa00001, 0x84788a78,
> + 0xb8ee1e06, 0x846e8a6e,
> + 0x80786e78, 0x8078ff78,
> + 0x00000200, 0x8078ff78,
> + 0x00000080, 0xbef600ff,
> + 0x01000000, 0x857d9972,
> + 0x8b7d817d, 0xbf06817d,
> + 0xbefd0080, 0xbfa2000d,
> + 0xc4050078, 0x0080e800,
> + 0x00000000, 0xbf8a0000,
> + 0xdac00000, 0x00000000,
> + 0x807dff7d, 0x00000080,
> + 0x8078ff78, 0x00000080,
> + 0xbf0a6f7d, 0xbfa2fff4,
> + 0xbfa0000c, 0xc4050078,
> + 0x0080e800, 0x00000000,
> + 0xbf8a0000, 0xdac00000,
> + 0x00000000, 0x807dff7d,
> + 0x00000100, 0x8078ff78,
> + 0x00000100, 0xbf0a6f7d,
> + 0xbfa2fff4, 0xbef80080,
> + 0xbefe00c1, 0x857d9972,
> + 0x8b7d817d, 0xbf06817d,
> + 0xbfa20002, 0xbeff0080,
> + 0xbfa00001, 0xbeff00c1,
> + 0xb8ef3b05, 0x806f816f,
> + 0x846f826f, 0x857d9972,
> + 0x8b7d817d, 0xbf06817d,
> + 0xbfa2002c, 0xbef600ff,
> + 0x01000000, 0xbeee0078,
> + 0x8078ff78, 0x00000200,
> + 0xbefd0084, 0xbf0a6f7d,
> + 0xbfa10061, 0xc4050078,
> + 0x008ce800, 0x00000000,
> + 0xc4050078, 0x008ce801,
> + 0x00008000, 0xc4050078,
> + 0x008ce802, 0x00010000,
> + 0xc4050078, 0x008ce803,
> + 0x00018000, 0xbf8a0000,
> + 0x7e008500, 0x7e028501,
> + 0x7e048502, 0x7e068503,
> + 0x807d847d, 0x8078ff78,
> + 0x00000200, 0xbf0a6f7d,
> + 0xbfa2ffea, 0xc405006e,
> + 0x008ce800, 0x00000000,
> + 0xc405006e, 0x008ce801,
> + 0x00008000, 0xc405006e,
> + 0x008ce802, 0x00010000,
> + 0xc405006e, 0x008ce803,
> + 0x00018000, 0xbf8a0000,
> + 0xbfa0003d, 0xbef600ff,
> + 0x01000000, 0xbeee0078,
> + 0x8078ff78, 0x00000400,
> + 0xbefd0084, 0xbf0a6f7d,
> + 0xbfa10016, 0xc4050078,
> + 0x008ce800, 0x00000000,
> + 0xc4050078, 0x008ce801,
> + 0x00010000, 0xc4050078,
> + 0x008ce802, 0x00020000,
> + 0xc4050078, 0x008ce803,
> + 0x00030000, 0xbf8a0000,
> + 0x7e008500, 0x7e028501,
> + 0x7e048502, 0x7e068503,
> + 0x807d847d, 0x8078ff78,
> + 0x00000400, 0xbf0a6f7d,
> + 0xbfa2ffea, 0xb8ef1e06,
> + 0x8b6fc16f, 0xbfa1000f,
> + 0x846f836f, 0x806f7d6f,
> + 0xbefe00c1, 0xbeff0080,
> + 0xc4050078, 0x008ce800,
> + 0x00000000, 0xbf8a0000,
> + 0x7e008500, 0x807d817d,
> + 0x8078ff78, 0x00000080,
> + 0xbf0a6f7d, 0xbfa2fff6,
> + 0xbeff00c1, 0xc405006e,
> + 0x008ce800, 0x00000000,
> + 0xc405006e, 0x008ce801,
> + 0x00010000, 0xc405006e,
> + 0x008ce802, 0x00020000,
> + 0xc405006e, 0x008ce803,
> + 0x00030000, 0xbf8a0000,
> 0xb8f83b05, 0x80788178,
> 0xbf0d9972, 0xbfa20002,
> 0x84788978, 0xbfa00001,
> 0x84788a78, 0xb8ee1e06,
> 0x846e8a6e, 0x80786e78,
> 0x8078ff78, 0x00000200,
> - 0x8078ff78, 0x00000080,
> - 0xbef600ff, 0x01000000,
> - 0x857d9972, 0x8b7d817d,
> - 0xbf06817d, 0xbefd0080,
> - 0xbfa2000d, 0xc4050078,
> - 0x0080e800, 0x00000000,
> - 0xbf8a0000, 0xdac00000,
> - 0x00000000, 0x807dff7d,
> - 0x00000080, 0x8078ff78,
> - 0x00000080, 0xbf0a6f7d,
> - 0xbfa2fff4, 0xbfa0000c,
> - 0xc4050078, 0x0080e800,
> - 0x00000000, 0xbf8a0000,
> - 0xdac00000, 0x00000000,
> - 0x807dff7d, 0x00000100,
> - 0x8078ff78, 0x00000100,
> - 0xbf0a6f7d, 0xbfa2fff4,
> - 0xbef80080, 0xbefe00c1,
> - 0x857d9972, 0x8b7d817d,
> - 0xbf06817d, 0xbfa20002,
> - 0xbeff0080, 0xbfa00001,
> - 0xbeff00c1, 0xb8ef3b05,
> - 0x806f816f, 0x846f826f,
> - 0x857d9972, 0x8b7d817d,
> - 0xbf06817d, 0xbfa2002c,
> + 0x80f8ff78, 0x00000050,
> 0xbef600ff, 0x01000000,
> - 0xbeee0078, 0x8078ff78,
> - 0x00000200, 0xbefd0084,
> - 0xbf0a6f7d, 0xbfa10061,
> - 0xc4050078, 0x008ce800,
> - 0x00000000, 0xc4050078,
> - 0x008ce801, 0x00008000,
> - 0xc4050078, 0x008ce802,
> - 0x00010000, 0xc4050078,
> - 0x008ce803, 0x00018000,
> - 0xbf8a0000, 0x7e008500,
> - 0x7e028501, 0x7e048502,
> - 0x7e068503, 0x807d847d,
> + 0xbefd00ff, 0x0000006c,
> + 0x80f89078, 0xf462403a,
> + 0xf0000000, 0xbf8a0000,
> + 0x80fd847d, 0xbf800000,
> + 0xbe804300, 0xbe824302,
> + 0x80f8a078, 0xf462603a,
> + 0xf0000000, 0xbf8a0000,
> + 0x80fd887d, 0xbf800000,
> + 0xbe804300, 0xbe824302,
> + 0xbe844304, 0xbe864306,
> + 0x80f8c078, 0xf462803a,
> + 0xf0000000, 0xbf8a0000,
> + 0x80fd907d, 0xbf800000,
> + 0xbe804300, 0xbe824302,
> + 0xbe844304, 0xbe864306,
> + 0xbe884308, 0xbe8a430a,
> + 0xbe8c430c, 0xbe8e430e,
> + 0xbf06807d, 0xbfa1fff0,
> + 0xb980f801, 0x00000000,
> + 0xb8f83b05, 0x80788178,
> + 0xbf0d9972, 0xbfa20002,
> + 0x84788978, 0xbfa00001,
> + 0x84788a78, 0xb8ee1e06,
> + 0x846e8a6e, 0x80786e78,
> 0x8078ff78, 0x00000200,
> - 0xbf0a6f7d, 0xbfa2ffea,
> - 0xc405006e, 0x008ce800,
> - 0x00000000, 0xc405006e,
> - 0x008ce801, 0x00008000,
> - 0xc405006e, 0x008ce802,
> - 0x00010000, 0xc405006e,
> - 0x008ce803, 0x00018000,
> - 0xbf8a0000, 0xbfa0003d,
> 0xbef600ff, 0x01000000,
> - 0xbeee0078, 0x8078ff78,
> - 0x00000400, 0xbefd0084,
> - 0xbf0a6f7d, 0xbfa10016,
> - 0xc4050078, 0x008ce800,
> - 0x00000000, 0xc4050078,
> - 0x008ce801, 0x00010000,
> - 0xc4050078, 0x008ce802,
> - 0x00020000, 0xc4050078,
> - 0x008ce803, 0x00030000,
> - 0xbf8a0000, 0x7e008500,
> - 0x7e028501, 0x7e048502,
> - 0x7e068503, 0x807d847d,
> - 0x8078ff78, 0x00000400,
> - 0xbf0a6f7d, 0xbfa2ffea,
> - 0xb8ef1e06, 0x8b6fc16f,
> - 0xbfa1000f, 0x846f836f,
> - 0x806f7d6f, 0xbefe00c1,
> - 0xbeff0080, 0xc4050078,
> - 0x008ce800, 0x00000000,
> - 0xbf8a0000, 0x7e008500,
> - 0x807d817d, 0x8078ff78,
> - 0x00000080, 0xbf0a6f7d,
> - 0xbfa2fff6, 0xbeff00c1,
> - 0xc405006e, 0x008ce800,
> - 0x00000000, 0xc405006e,
> - 0x008ce801, 0x00010000,
> - 0xc405006e, 0x008ce802,
> - 0x00020000, 0xc405006e,
> - 0x008ce803, 0x00030000,
> - 0xbf8a0000, 0xb8f83b05,
> - 0x80788178, 0xbf0d9972,
> - 0xbfa20002, 0x84788978,
> - 0xbfa00001, 0x84788a78,
> - 0xb8ee1e06, 0x846e8a6e,
> - 0x80786e78, 0x8078ff78,
> - 0x00000200, 0x80f8ff78,
> - 0x00000050, 0xbef600ff,
> - 0x01000000, 0xbefd00ff,
> - 0x0000006c, 0x80f89078,
> - 0xf462403a, 0xf0000000,
> - 0xbf8a0000, 0x80fd847d,
> - 0xbf800000, 0xbe804300,
> - 0xbe824302, 0x80f8a078,
> - 0xf462603a, 0xf0000000,
> - 0xbf8a0000, 0x80fd887d,
> - 0xbf800000, 0xbe804300,
> - 0xbe824302, 0xbe844304,
> - 0xbe864306, 0x80f8c078,
> - 0xf462803a, 0xf0000000,
> - 0xbf8a0000, 0x80fd907d,
> - 0xbf800000, 0xbe804300,
> - 0xbe824302, 0xbe844304,
> - 0xbe864306, 0xbe884308,
> - 0xbe8a430a, 0xbe8c430c,
> - 0xbe8e430e, 0xbf06807d,
> - 0xbfa1fff0, 0xb980f801,
> - 0x00000000, 0xb8f83b05,
> - 0x80788178, 0xbf0d9972,
> - 0xbfa20002, 0x84788978,
> - 0xbfa00001, 0x84788a78,
> - 0xb8ee1e06, 0x846e8a6e,
> - 0x80786e78, 0x8078ff78,
> - 0x00000200, 0xbef600ff,
> - 0x01000000, 0xbeff0071,
> - 0xf4621bfa, 0xf0000000,
> - 0x80788478, 0xf4621b3a,
> + 0xbeff0071, 0xf4621bfa,
> 0xf0000000, 0x80788478,
> - 0xf4621b7a, 0xf0000000,
> - 0x80788478, 0xf4621c3a,
> + 0xf4621b3a, 0xf0000000,
> + 0x80788478, 0xf4621b7a,
> 0xf0000000, 0x80788478,
> - 0xf4621c7a, 0xf0000000,
> - 0x80788478, 0xf4621eba,
> + 0xf4621c3a, 0xf0000000,
> + 0x80788478, 0xf4621c7a,
> 0xf0000000, 0x80788478,
> - 0xf4621efa, 0xf0000000,
> - 0x80788478, 0xf4621e7a,
> + 0xf4621eba, 0xf0000000,
> + 0x80788478, 0xf4621efa,
> 0xf0000000, 0x80788478,
> - 0xf4621cfa, 0xf0000000,
> - 0x80788478, 0xf4621bba,
> + 0xf4621e7a, 0xf0000000,
> + 0x80788478, 0xf4621cfa,
> 0xf0000000, 0x80788478,
> - 0xbf8a0000, 0xb96ef814,
> 0xf4621bba, 0xf0000000,
> 0x80788478, 0xbf8a0000,
> - 0xb96ef815, 0xf4621bba,
> + 0xb96ef814, 0xf4621bba,
> 0xf0000000, 0x80788478,
> - 0xbf8a0000, 0xb96ef812,
> + 0xbf8a0000, 0xb96ef815,
> 0xf4621bba, 0xf0000000,
> 0x80788478, 0xbf8a0000,
> - 0xb96ef813, 0x8b6eff7f,
> - 0x04000000, 0xbfa1000d,
> - 0x80788478, 0xf4621bba,
> + 0xb96ef812, 0xf4621bba,
> 0xf0000000, 0x80788478,
> - 0xbf8a0000, 0xbf0d806e,
> - 0xbfa10006, 0x856e906e,
> - 0x8b6e6e6e, 0xbfa10003,
> - 0xbe804ec1, 0x816ec16e,
> - 0xbfa0fffb, 0xbefd006f,
> - 0xbefe0070, 0xbeff0071,
> - 0xb97b2011, 0x857b867b,
> - 0xb97b0191, 0x857b827b,
> - 0xb97bba11, 0xb973f801,
> - 0xb8ee3b05, 0x806e816e,
> - 0xbf0d9972, 0xbfa20002,
> - 0x846e896e, 0xbfa00001,
> - 0x846e8a6e, 0xb8ef1e06,
> - 0x846f8a6f, 0x806e6f6e,
> - 0x806eff6e, 0x00000200,
> - 0x806e746e, 0x826f8075,
> - 0x8b6fff6f, 0x0000ffff,
> - 0xf4605c37, 0xf8000050,
> - 0xf4605d37, 0xf8000060,
> - 0xf4601e77, 0xf8000074,
> - 0xbf8a0000, 0x8b6dff6d,
> - 0x0000ffff, 0x8bfe7e7e,
> - 0x8bea6a6a, 0xb97af804,
> + 0xbf8a0000, 0xb96ef813,
> + 0x8b6eff7f, 0x04000000,
> + 0xbfa1000d, 0x80788478,
> + 0xf4621bba, 0xf0000000,
> + 0x80788478, 0xbf8a0000,
> + 0xbf0d806e, 0xbfa10006,
> + 0x856e906e, 0x8b6e6e6e,
> + 0xbfa10003, 0xbe804ec1,
> + 0x816ec16e, 0xbfa0fffb,
> + 0xbefd006f, 0xbefe0070,
> + 0xbeff0071, 0xb97b2011,
> + 0x857b867b, 0xb97b0191,
> + 0x857b827b, 0xb97bba11,
> + 0xb973f801, 0xb8ee3b05,
> + 0x806e816e, 0xbf0d9972,
> + 0xbfa20002, 0x846e896e,
> + 0xbfa00001, 0x846e8a6e,
> + 0xb8ef1e06, 0x846f8a6f,
> + 0x806e6f6e, 0x806eff6e,
> + 0x00000200, 0x806e746e,
> + 0x826f8075, 0x8b6fff6f,
> + 0x0000ffff, 0xf4605c37,
> + 0xf8000050, 0xf4605d37,
> + 0xf8000060, 0xf4601e77,
> + 0xf8000074, 0xbf8a0000,
> + 0x8b6dff6d, 0x0000ffff,
> + 0x8bfe7e7e, 0x8bea6a6a,
> + 0xb97af804, 0xbe804ec2,
> + 0xbf94fffe, 0xbe804a6c,
> 0xbe804ec2, 0xbf94fffe,
> - 0xbe804a6c, 0xbe804ec2,
> - 0xbf94fffe, 0xbfb10000,
> + 0xbfb10000, 0xbf9f0000,
> 0xbf9f0000, 0xbf9f0000,
> 0xbf9f0000, 0xbf9f0000,
> - 0xbf9f0000, 0x00000000,
> };
>
> static const uint32_t cwsr_trap_gfx9_5_0_hex[] = {
> diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
> index 7b9d36e5fa43..5a1a1b1f897f 100644
> --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
> +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
> @@ -30,6 +30,7 @@
> #define CHIP_GFX12 37
>
> #define SINGLE_STEP_MISSED_WORKAROUND 1 //workaround for lost TRAP_AFTER_INST exception when SAVECTX raised
> +#define HAVE_VALU_SGPR_HAZARD (ASIC_FAMILY == CHIP_GFX12)
>
> var SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_MASK = 0x4
> var SQ_WAVE_STATE_PRIV_SCC_SHIFT = 9
> @@ -351,6 +352,7 @@ L_HAVE_VGPRS:
> v_writelane_b32 v0, ttmp13, 0xD
> v_writelane_b32 v0, exec_lo, 0xE
> v_writelane_b32 v0, exec_hi, 0xF
> + valu_sgpr_hazard()
>
> s_mov_b32 exec_lo, 0x3FFF
> s_mov_b32 exec_hi, 0x0
> @@ -417,7 +419,6 @@ L_SAVE_HWREG:
> v_mov_b32 v0, 0x0 //Offset[31:0] from buffer resource
> v_mov_b32 v1, 0x0 //Offset[63:32] from buffer resource
> v_mov_b32 v2, 0x0 //Set of SGPRs for TCP store
> - s_mov_b32 m0, 0x0 //Next lane of v2 to write to
>
> // Ensure no further changes to barrier or LDS state.
> // STATE_PRIV.BARRIER_COMPLETE may change up to this point.
> @@ -430,40 +431,41 @@ L_SAVE_HWREG:
> s_andn2_b32 s_save_state_priv, s_save_state_priv, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_MASK
> s_or_b32 s_save_state_priv, s_save_state_priv, s_save_tmp
>
> - write_hwreg_to_v2(s_save_m0)
> - write_hwreg_to_v2(s_save_pc_lo)
> s_andn2_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
> - write_hwreg_to_v2(s_save_tmp)
> - write_hwreg_to_v2(s_save_exec_lo)
> - write_hwreg_to_v2(s_save_exec_hi)
> - write_hwreg_to_v2(s_save_state_priv)
> + v_writelane_b32 v2, s_save_m0, 0x0
> + v_writelane_b32 v2, s_save_pc_lo, 0x1
> + v_writelane_b32 v2, s_save_tmp, 0x2
> + v_writelane_b32 v2, s_save_exec_lo, 0x3
> + v_writelane_b32 v2, s_save_exec_hi, 0x4
> + v_writelane_b32 v2, s_save_state_priv, 0x5
> + v_writelane_b32 v2, s_save_xnack_mask, 0x7
> + valu_sgpr_hazard()
>
> s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV)
> - write_hwreg_to_v2(s_save_tmp)
> + v_writelane_b32 v2, s_save_tmp, 0x6
>
> - write_hwreg_to_v2(s_save_xnack_mask)
> + s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_MODE)
> + v_writelane_b32 v2, s_save_tmp, 0x8
>
> - s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_MODE)
> - write_hwreg_to_v2(s_save_m0)
> + s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_SCRATCH_BASE_LO)
> + v_writelane_b32 v2, s_save_tmp, 0x9
>
> - s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_SCRATCH_BASE_LO)
> - write_hwreg_to_v2(s_save_m0)
> + s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_SCRATCH_BASE_HI)
> + v_writelane_b32 v2, s_save_tmp, 0xA
>
> - s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_SCRATCH_BASE_HI)
> - write_hwreg_to_v2(s_save_m0)
> + s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_EXCP_FLAG_USER)
> + v_writelane_b32 v2, s_save_tmp, 0xB
>
> - s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_EXCP_FLAG_USER)
> - write_hwreg_to_v2(s_save_m0)
> -
> - s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_TRAP_CTRL)
> - write_hwreg_to_v2(s_save_m0)
> + s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_TRAP_CTRL)
> + v_writelane_b32 v2, s_save_tmp, 0xC
>
> s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_STATUS)
> - write_hwreg_to_v2(s_save_tmp)
> + v_writelane_b32 v2, s_save_tmp, 0xD
>
> s_get_barrier_state s_save_tmp, -1
> s_wait_kmcnt (0)
> - write_hwreg_to_v2(s_save_tmp)
> + v_writelane_b32 v2, s_save_tmp, 0xE
> + valu_sgpr_hazard()
>
> // Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this.
> s_mov_b32 exec_lo, 0xFFFF
> @@ -497,10 +499,12 @@ L_SAVE_SGPR_LOOP:
> s_movrels_b64 s12, s12 //s12 = s[12+m0], s13 = s[13+m0]
> s_movrels_b64 s14, s14 //s14 = s[14+m0], s15 = s[15+m0]
>
> - write_16sgpr_to_v2(s0)
> -
> - s_cmp_eq_u32 ttmp13, 0x20 //have 32 VGPR lanes filled?
> - s_cbranch_scc0 L_SAVE_SGPR_SKIP_TCP_STORE
> + s_cmp_eq_u32 ttmp13, 0x0
> + s_cbranch_scc0 L_WRITE_V2_SECOND_HALF
> + write_16sgpr_to_v2(s0, 0x0)
> + s_branch L_SAVE_SGPR_SKIP_TCP_STORE
> +L_WRITE_V2_SECOND_HALF:
> + write_16sgpr_to_v2(s0, 0x10)
>
> buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS
> s_add_u32 s_save_mem_offset, s_save_mem_offset, 0x80
> @@ -1056,27 +1060,21 @@ L_END_PGM:
> s_endpgm_saved
> end
>
> -function write_hwreg_to_v2(s)
> - // Copy into VGPR for later TCP store.
> - v_writelane_b32 v2, s, m0
> - s_add_u32 m0, m0, 0x1
> -end
> -
> -
> -function write_16sgpr_to_v2(s)
> +function write_16sgpr_to_v2(s, lane_offset)
> // Copy into VGPR for later TCP store.
> for var sgpr_idx = 0; sgpr_idx < 16; sgpr_idx ++
> - v_writelane_b32 v2, s[sgpr_idx], ttmp13
> - s_add_u32 ttmp13, ttmp13, 0x1
> + v_writelane_b32 v2, s[sgpr_idx], sgpr_idx + lane_offset
> end
> + valu_sgpr_hazard()
> + s_add_u32 ttmp13, ttmp13, 0x10
> end
>
> function write_12sgpr_to_v2(s)
> // Copy into VGPR for later TCP store.
> for var sgpr_idx = 0; sgpr_idx < 12; sgpr_idx ++
> - v_writelane_b32 v2, s[sgpr_idx], ttmp13
> - s_add_u32 ttmp13, ttmp13, 0x1
> + v_writelane_b32 v2, s[sgpr_idx], sgpr_idx
> end
> + valu_sgpr_hazard()
> end
>
> function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
> @@ -1128,3 +1126,11 @@ function get_wave_size2(s_reg)
> s_getreg_b32 s_reg, hwreg(HW_REG_WAVE_STATUS,SQ_WAVE_STATUS_WAVE64_SHIFT,SQ_WAVE_STATUS_WAVE64_SIZE)
> s_lshl_b32 s_reg, s_reg, S_WAVE_SIZE
> end
> +
> +function valu_sgpr_hazard
> +#if HAVE_VALU_SGPR_HAZARD
> + for var rep = 0; rep < 8; rep ++
> + ds_nop
> + end
> +#endif
> +end
> --
> 2.34.1
>
