diff --git a/src/nouveau/compiler/nak_encode_sm70.rs b/src/nouveau/compiler/nak_encode_sm70.rs index f11ade5622a..4e1ce63fd9f 100644 --- a/src/nouveau/compiler/nak_encode_sm70.rs +++ b/src/nouveau/compiler/nak_encode_sm70.rs @@ -1263,7 +1263,7 @@ impl SM70Instr { fn set_mem_access(&mut self, access: &MemAccess) { self.set_field( 72..73, - match access.addr_type { + match access.space.addr_type() { MemAddrType::A32 => 0_u8, MemAddrType::A64 => 1_u8, }, @@ -1291,7 +1291,6 @@ impl SM70Instr { self.set_reg_src(24..32, op.addr); self.set_field(40..64, op.offset); - assert!(op.access.addr_type == MemAddrType::A32); self.set_mem_type(73..76, op.access.mem_type); assert!(op.access.order == MemOrder::Strong(MemScope::CTA)); assert!(op.access.eviction_priority == MemEvictionPriority::Normal); @@ -1304,7 +1303,6 @@ impl SM70Instr { self.set_reg_src(24..32, op.addr); self.set_field(40..64, op.offset); - assert!(op.access.addr_type == MemAddrType::A32); self.set_mem_type(73..76, op.access.mem_type); assert!(op.access.order == MemOrder::Strong(MemScope::CTA)); assert!(op.access.eviction_priority == MemEvictionPriority::Normal); @@ -1314,7 +1312,7 @@ impl SM70Instr { fn encode_ld(&mut self, op: &OpLd) { match op.access.space { - MemSpace::Global => self.encode_ldg(op), + MemSpace::Global(_) => self.encode_ldg(op), MemSpace::Local => self.encode_ldl(op), MemSpace::Shared => self.encode_lds(op), } @@ -1351,7 +1349,6 @@ impl SM70Instr { self.set_reg_src(32..40, op.data); self.set_field(40..64, op.offset); - assert!(op.access.addr_type == MemAddrType::A32); self.set_mem_type(73..76, op.access.mem_type); assert!(op.access.order == MemOrder::Strong(MemScope::CTA)); assert!(op.access.eviction_priority == MemEvictionPriority::Normal); @@ -1364,7 +1361,6 @@ impl SM70Instr { self.set_reg_src(32..40, op.data); self.set_field(40..64, op.offset); - assert!(op.access.addr_type == MemAddrType::A32); self.set_mem_type(73..76, op.access.mem_type); assert!(op.access.order == MemOrder::Strong(MemScope::CTA)); assert!(op.access.eviction_priority == MemEvictionPriority::Normal); @@ -1372,7 +1368,7 @@ impl SM70Instr { fn encode_st(&mut self, op: &OpSt) { match op.access.space { - MemSpace::Global => self.encode_stg(op), + MemSpace::Global(_) => self.encode_stg(op), MemSpace::Local => self.encode_stl(op), MemSpace::Shared => self.encode_sts(op), } @@ -1434,7 +1430,7 @@ impl SM70Instr { self.set_field( 72..73, - match op.addr_type { + match op.mem_space.addr_type() { MemAddrType::A32 => 0_u8, MemAddrType::A64 => 1_u8, }, @@ -1463,7 +1459,6 @@ impl SM70Instr { self.set_reg_src(24..32, op.addr); self.set_field(40..64, op.addr_offset); - assert!(op.addr_type == MemAddrType::A32); assert!(op.mem_order == MemOrder::Strong(MemScope::CTA)); assert!(op.mem_eviction_priority == MemEvictionPriority::Normal); @@ -1472,7 +1467,7 @@ impl SM70Instr { fn encode_atom(&mut self, op: &OpAtom) { match op.mem_space { - MemSpace::Global => self.encode_atomg(op), + MemSpace::Global(_) => self.encode_atomg(op), MemSpace::Local => panic!("Atomics do not support local"), MemSpace::Shared => self.encode_atoms(op), } @@ -1564,7 +1559,7 @@ impl SM70Instr { } fn encode_cctl(&mut self, op: &OpCCtl) { - assert!(op.mem_space == MemSpace::Global); + assert!(matches!(op.mem_space, MemSpace::Global(_))); self.set_opcode(0x98f); self.set_reg_src(24..32, op.addr); diff --git a/src/nouveau/compiler/nak_from_nir.rs b/src/nouveau/compiler/nak_from_nir.rs index 40550924b37..f48860e6a00 100644 --- a/src/nouveau/compiler/nak_from_nir.rs +++ b/src/nouveau/compiler/nak_from_nir.rs @@ -1527,9 +1527,8 @@ impl<'a> ShaderFromNir<'a> { data: data, atom_op: atom_op, atom_type: atom_type, - addr_type: MemAddrType::A64, addr_offset: offset, - mem_space: MemSpace::Global, + mem_space: MemSpace::Global(MemAddrType::A64), mem_order: MemOrder::Strong(MemScope::System), mem_eviction_priority: MemEvictionPriority::Normal, // Note: no intrinic access }); @@ -1553,9 +1552,8 @@ impl<'a> ShaderFromNir<'a> { data: data, atom_op: AtomOp::CmpExch, atom_type: atom_type, - addr_type: MemAddrType::A64, addr_offset: offset, - mem_space: MemSpace::Global, + mem_space: MemSpace::Global(MemAddrType::A64), mem_order: MemOrder::Strong(MemScope::System), mem_eviction_priority: MemEvictionPriority::Normal, // Note: no intrinic access }); @@ -1634,9 +1632,8 @@ impl<'a> ShaderFromNir<'a> { MemOrder::Strong(MemScope::System) }; let access = MemAccess { - addr_type: MemAddrType::A64, mem_type: MemType::from_size(size_B, false), - space: MemSpace::Global, + space: MemSpace::Global(MemAddrType::A64), order: order, eviction_priority: self .get_eviction_priority(intrin.access()), @@ -1748,7 +1745,6 @@ impl<'a> ShaderFromNir<'a> { (intrin.def.bit_size() / 8) * intrin.def.num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::from_size(size_B, false), space: MemSpace::Local, order: MemOrder::Strong(MemScope::CTA), @@ -1770,7 +1766,6 @@ impl<'a> ShaderFromNir<'a> { (intrin.def.bit_size() / 8) * intrin.def.num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::from_size(size_B, false), space: MemSpace::Shared, order: MemOrder::Strong(MemScope::CTA), @@ -1845,7 +1840,7 @@ impl<'a> ShaderFromNir<'a> { { b.push_op(OpCCtl { op: CCtlOp::WBAll, - mem_space: MemSpace::Global, + mem_space: MemSpace::Global(MemAddrType::A64), addr: 0.into(), addr_offset: 0, }); @@ -1875,7 +1870,7 @@ impl<'a> ShaderFromNir<'a> { { b.push_op(OpCCtl { op: CCtlOp::IVAll, - mem_space: MemSpace::Global, + mem_space: MemSpace::Global(MemAddrType::A64), addr: 0.into(), addr_offset: 0, }); @@ -1957,7 +1952,6 @@ impl<'a> ShaderFromNir<'a> { data: data, atom_op: atom_op, atom_type: atom_type, - addr_type: MemAddrType::A32, addr_offset: offset, mem_space: MemSpace::Shared, mem_order: MemOrder::Strong(MemScope::CTA), @@ -1983,7 +1977,6 @@ impl<'a> ShaderFromNir<'a> { data: data, atom_op: AtomOp::CmpExch, atom_type: atom_type, - addr_type: MemAddrType::A32, addr_offset: offset, mem_space: MemSpace::Shared, mem_order: MemOrder::Strong(MemScope::CTA), @@ -1997,9 +1990,8 @@ impl<'a> ShaderFromNir<'a> { (srcs[0].bit_size() / 8) * srcs[0].num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A64, mem_type: MemType::from_size(size_B, false), - space: MemSpace::Global, + space: MemSpace::Global(MemAddrType::A64), order: MemOrder::Strong(MemScope::System), eviction_priority: self .get_eviction_priority(intrin.access()), @@ -2035,7 +2027,6 @@ impl<'a> ShaderFromNir<'a> { (srcs[0].bit_size() / 8) * srcs[0].num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::from_size(size_B, false), space: MemSpace::Local, order: MemOrder::Strong(MemScope::CTA), @@ -2056,7 +2047,6 @@ impl<'a> ShaderFromNir<'a> { (srcs[0].bit_size() / 8) * srcs[0].num_components(); assert!(u32::from(size_B) <= intrin.align()); let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::from_size(size_B, false), space: MemSpace::Shared, order: MemOrder::Strong(MemScope::CTA), diff --git a/src/nouveau/compiler/nak_ir.rs b/src/nouveau/compiler/nak_ir.rs index 4aee9455799..4d2f3dff7bc 100644 --- a/src/nouveau/compiler/nak_ir.rs +++ b/src/nouveau/compiler/nak_ir.rs @@ -1851,15 +1851,25 @@ impl fmt::Display for MemScope { #[derive(Clone, Copy, Eq, Hash, PartialEq)] pub enum MemSpace { - Global, + Global(MemAddrType), Local, Shared, } +impl MemSpace { + pub fn addr_type(&self) -> MemAddrType { + match self { + MemSpace::Global(t) => *t, + MemSpace::Local => MemAddrType::A32, + MemSpace::Shared => MemAddrType::A32, + } + } +} + impl fmt::Display for MemSpace { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - MemSpace::Global => write!(f, ".global"), + MemSpace::Global(t) => write!(f, ".global{t}"), MemSpace::Local => write!(f, ".local"), MemSpace::Shared => write!(f, ".shared"), } @@ -1888,7 +1898,6 @@ impl fmt::Display for MemEvictionPriority { #[derive(Clone)] pub struct MemAccess { - pub addr_type: MemAddrType, pub mem_type: MemType, pub space: MemSpace, pub order: MemOrder, @@ -1899,8 +1908,7 @@ impl fmt::Display for MemAccess { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "{}{}{}{}{}", - self.addr_type, + "{}{}{}{}", self.mem_type, self.space, self.order, @@ -3290,7 +3298,6 @@ pub struct OpAtom { pub atom_op: AtomOp, pub atom_type: AtomType, - pub addr_type: MemAddrType, pub addr_offset: i32, pub mem_space: MemSpace, @@ -3302,9 +3309,10 @@ impl DisplayOp for OpAtom { fn fmt_op(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "atom{}{}{}{}", + "atom{}{}{}{}{}", self.atom_op, self.atom_type, + self.mem_space, self.mem_order, self.mem_eviction_priority, )?; @@ -4655,8 +4663,8 @@ impl Instr { pub fn writes_global_mem(&self) -> bool { match &self.op { - Op::Atom(op) => op.mem_space == MemSpace::Global, - Op::St(op) => op.access.space == MemSpace::Global, + Op::Atom(op) => matches!(op.mem_space, MemSpace::Global(_)), + Op::St(op) => matches!(op.access.space, MemSpace::Global(_)), Op::SuAtom(_) | Op::SuSt(_) => true, _ => false, } diff --git a/src/nouveau/compiler/nak_lower_copy_swap.rs b/src/nouveau/compiler/nak_lower_copy_swap.rs index 292dacfc2ae..476310cc457 100644 --- a/src/nouveau/compiler/nak_lower_copy_swap.rs +++ b/src/nouveau/compiler/nak_lower_copy_swap.rs @@ -45,7 +45,6 @@ impl LowerCopySwap { } RegFile::Mem => { let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::B32, space: MemSpace::Local, order: MemOrder::Strong(MemScope::CTA), @@ -101,7 +100,6 @@ impl LowerCopySwap { SrcRef::Reg(src_reg) => match src_reg.file() { RegFile::GPR => { let access = MemAccess { - addr_type: MemAddrType::A32, mem_type: MemType::B32, space: MemSpace::Local, order: MemOrder::Strong(MemScope::CTA),