No public description PiperOrigin-RevId: 968645551 Change-Id: I04e3c08a4c43e969d37f7ce81883b609600dac1d
diff --git a/riscv/riscv_vector_permute_instructions.cc b/riscv/riscv_vector_permute_instructions.cc index f69903c..9dda8e9 100644 --- a/riscv/riscv_vector_permute_instructions.cc +++ b/riscv/riscv_vector_permute_instructions.cc
@@ -35,8 +35,10 @@ void VrgatherHelper(RiscVVectorState* rv_vector, Instruction* inst) { if (rv_vector->vector_exception()) return; int num_elements = rv_vector->vector_length(); - int elements_per_vector = - rv_vector->vector_register_byte_length() / sizeof(Vd); + int vlenb = rv_vector->vector_register_byte_length(); + int elements_per_vector = vlenb / sizeof(Vd); + int vlmax = rv_vector->max_vector_length(); + // Verify that the lmul is compatible with index size. int index_emul = rv_vector->vector_length_multiplier() * sizeof(Vs1) / sizeof(Vd); @@ -44,10 +46,12 @@ rv_vector->set_vector_exception(); return; } + int max_regs = std::max( 1, (num_elements + elements_per_vector - 1) / elements_per_vector); auto* dest_op = static_cast<RV32VectorDestinationOperand*>(inst->Destination(0)); + // Verify that there are enough registers in the destination operand. if (dest_op->size() < max_regs) { rv_vector->set_vector_exception(); @@ -56,46 +60,66 @@ dest_op->size(), ") than required by the operation (", max_regs, ")"); return; } + // Get the vector mask. auto* mask_op = static_cast<RV32VectorSourceOperand*>(inst->Source(2)); auto mask_span = mask_op->GetRegister(0)->data_buffer()->Get<uint8_t>(); - // Get the vector start element index and compute the where to start - // the operation. + + auto* src0_op = static_cast<RV32VectorSourceOperand*>(inst->Source(0)); + + // Determine the type of the index source (Source 1). + // .vi: Source(1) is an immediate operand. + // .vx: Source(1) is a scalar GPR operand. + // .vv: Source(1) is a vector operand. + bool is_scalar_or_imm = (inst->Source(1)->shape()[0] == 1); + auto* src1_op = is_scalar_or_imm + ? nullptr + : static_cast<RV32VectorSourceOperand*>(inst->Source(1)); + int vector_index = rv_vector->vstart(); int start_reg = vector_index / elements_per_vector; int item_index = vector_index % elements_per_vector; - // Determine if it's vector-vector or vector-scalar. - bool vector_scalar = inst->Source(1)->shape()[0] == 1; - auto src0_op = static_cast<RV32VectorSourceOperand*>(inst->Source(0)); - int max_index = src0_op->size() * elements_per_vector; - // Iterate over the number of registers to write. + + int vs1_els_per_reg = vlenb / sizeof(Vs1); + int vs2_els_per_reg = vlenb / sizeof(Vs2); + for (int reg = start_reg; (reg < max_regs) && (vector_index < num_elements); reg++) { // Allocate data buffer for the new register data. auto* dest_db = dest_op->CopyDataBuffer(reg); auto dest_span = dest_db->Get<Vd>(); - // Write data into register subject to masking. + int element_count = std::min(elements_per_vector, num_elements); for (int i = item_index; (i < element_count) && (vector_index < num_elements); i++) { - // Get the mask value. - int mask_index = i >> 3; - int mask_offset = i & 0b111; + int mask_index = vector_index >> 3; + int mask_offset = vector_index & 0b111; bool mask_value = ((mask_span[mask_index] >> mask_offset) & 0b1) != 0; + if (mask_value) { - // Compute result. - RV32Register::ValueType vs1; - if (vector_scalar) { - vs1 = generic::GetInstructionSource<RV32Register::ValueType>(inst, 1, - 0); + uint64_t index_val = 0; + if (is_scalar_or_imm) { + // For .vx and .vi, index is the same for all elements. + index_val = generic::GetInstructionSource<uint32_t>(inst, 1, 0); } else { - vs1 = generic::GetInstructionSource<Vs1>(inst, 1, vector_index); + // For .vv and .ei16.vv, index is read from the vector register vs1. + int vs1_reg_off = vector_index / vs1_els_per_reg; + int vs1_el_off = vector_index % vs1_els_per_reg; + index_val = src1_op->GetRegister(vs1_reg_off) + ->data_buffer() + ->Get<Vs1>()[vs1_el_off]; } - Vs2 vs2 = 0; - if (vs1 < max_index) { - vs2 = generic::GetInstructionSource<Vs2>(inst, 0, vs1); + + Vs2 res_vs2 = 0; + // Spec: vd[i] = (index >= VLMAX) ? 0 : vs2[index] + if (index_val < static_cast<uint64_t>(vlmax)) { + int vs2_reg_off = index_val / vs2_els_per_reg; + int vs2_el_off = index_val % vs2_els_per_reg; + res_vs2 = src0_op->GetRegister(vs2_reg_off) + ->data_buffer() + ->Get<Vs2>()[vs2_el_off]; } - dest_span[i] = vs2; + dest_span[i] = static_cast<Vd>(res_vs2); } vector_index++; }
diff --git a/riscv/test/riscv_vector_permute_instructions_test.cc b/riscv/test/riscv_vector_permute_instructions_test.cc index f71a970..9a39786 100644 --- a/riscv/test/riscv_vector_permute_instructions_test.cc +++ b/riscv/test/riscv_vector_permute_instructions_test.cc
@@ -15,6 +15,7 @@ #include "riscv/riscv_vector_permute_instructions.h" #include <cstdint> +#include <cstring> #include "absl/random/random.h" #include "googlemock/include/gmock/gmock.h" @@ -624,4 +625,232 @@ CompressHelper<uint64_t>(this, instruction_); } +// Test vrgather.vv where indices are >= VLMAX (with LMUL < 8). +// Per spec: vd[i] = (vs1[i] >= VLMAX) ? 0 : vs2[vs1[i]]. +TEST_F(RiscVVectorPermuteInstructionsTest, + VrgatherVVIndexOutOfBoundsZeroesElement) { + SetSemanticFunction(&Vrgather); + AppendVectorRegisterOperands({kVs2, kVs1, kVmask}, {kVd}); + + // LMUL = 1, SEW = 32 bits (4 bytes). VLMAX = 64 / 4 = 16 elements. + uint32_t vtype = (kSewSettingsByByteSize[sizeof(uint32_t)] << 3) | + kLmulSettingByLogSize[4]; + ConfigureVectorUnit(vtype, 16); + + int num_values_per_reg = kVectorLengthInBytes / sizeof(uint32_t); // 16 + // Set vs2 (v24) elements to distinct non-zero values. + auto vs2_span = vreg()[kVs2]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_span[i] = 100 + i; + } + // Also populate v25 with non-zero values so that reading beyond VLMAX would + // yield non-zero. + auto vs2_plus1_span = vreg()[kVs2 + 1]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_plus1_span[i] = 500 + i; + } + + // Set vs1 (v16) with some indices in-bounds (< 16) and some out-of-bounds (>= + // 16). + auto vs1_span = vreg()[kVs1]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs1_span[i] = (i % 2 == 0) ? (i / 2) : (16 + i); + } + + // Unmasked (mask all 1s). + uint8_t all_ones_mask[kVectorLengthInBytes]; + memset(all_ones_mask, 0xff, sizeof(all_ones_mask)); + SetVectorRegisterValues<uint8_t>({{kVmaskName, all_ones_mask}}); + + instruction_->Execute(); + + auto vd_span = vreg()[kVd]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + if (i % 2 == 0) { + EXPECT_EQ(vd_span[i], vs2_span[i / 2]) << "at index " << i; + } else { + EXPECT_EQ(vd_span[i], 0) + << "at index " << i << " (index value " << (16 + i) << " >= VLMAX)"; + } + } +} + +// Test vrgather.vx where scalar index rs1 >= VLMAX (with LMUL < 8). +// Per spec: vd[i] = (rs1 >= VLMAX) ? 0 : vs2[rs1]. +TEST_F(RiscVVectorPermuteInstructionsTest, + VrgatherVXIndexOutOfBoundsZeroesElement) { + SetSemanticFunction(&Vrgather); + AppendVectorRegisterOperands({kVs2}, {}); + AppendRegisterOperands<RV32Register>({kRs1Name}, {}); + AppendVectorRegisterOperands({kVmask}, {kVd}); + + // LMUL = 1, SEW = 32 bits (4 bytes). VLMAX = 16 elements. + uint32_t vtype = (kSewSettingsByByteSize[sizeof(uint32_t)] << 3) | + kLmulSettingByLogSize[4]; + ConfigureVectorUnit(vtype, 16); + + int num_values_per_reg = kVectorLengthInBytes / sizeof(uint32_t); // 16 + auto vs2_span = vreg()[kVs2]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_span[i] = 100 + i; + } + auto vs2_plus1_span = vreg()[kVs2 + 1]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_plus1_span[i] = 500 + i; + } + + uint8_t all_ones_mask[kVectorLengthInBytes]; + memset(all_ones_mask, 0xff, sizeof(all_ones_mask)); + SetVectorRegisterValues<uint8_t>({{kVmaskName, all_ones_mask}}); + + // rs1 = 16 (>= VLMAX) + SetRegisterValues<RV32Register::ValueType>({{kRs1Name, 16}}); + + instruction_->Execute(); + + auto vd_span = vreg()[kVd]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + EXPECT_EQ(vd_span[i], 0) << "at index " << i; + } +} + +// Test vrgather.vi (immediate index) in-bounds and out-of-bounds. +TEST_F(RiscVVectorPermuteInstructionsTest, VrgatherVIInBoundsAndOutOfBounds) { + // LMUL = 1, SEW = 64 bits (8 bytes). VLMAX = 8 elements. + uint32_t vtype = (kSewSettingsByByteSize[sizeof(uint64_t)] << 3) | + kLmulSettingByLogSize[4]; + ConfigureVectorUnit(vtype, 8); + + int num_values_per_reg = kVectorLengthInBytes / sizeof(uint64_t); // 8 + auto vs2_span = vreg()[kVs2]->data_buffer()->Get<uint64_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_span[i] = 1000 + i; + } + auto vs2_plus1_span = vreg()[kVs2 + 1]->data_buffer()->Get<uint64_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_plus1_span[i] = 5000 + i; + } + + uint8_t all_ones_mask[kVectorLengthInBytes]; + memset(all_ones_mask, 0xff, sizeof(all_ones_mask)); + SetVectorRegisterValues<uint8_t>({{kVmaskName, all_ones_mask}}); + + // Test in-bounds immediate: uimm = 3 (< VLMAX = 8). + ResetInstruction(); + SetSemanticFunction(&Vrgather); + AppendVectorRegisterOperands({kVs2}, {}); + AppendImmediateOperands<uint32_t>({3}); + AppendVectorRegisterOperands({kVmask}, {kVd}); + + instruction_->Execute(); + + auto vd_span = vreg()[kVd]->data_buffer()->Get<uint64_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + EXPECT_EQ(vd_span[i], 1003) << "at index " << i; + } + + // Test out-of-bounds immediate: uimm = 10 (>= VLMAX = 8). + ResetInstruction(); + SetSemanticFunction(&Vrgather); + AppendVectorRegisterOperands({kVs2}, {}); + AppendImmediateOperands<uint32_t>({10}); + AppendVectorRegisterOperands({kVmask}, {kVd}); + + instruction_->Execute(); + + vd_span = vreg()[kVd]->data_buffer()->Get<uint64_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + EXPECT_EQ(vd_span[i], 0) << "at index " << i; + } +} + +// Test vrgatherei16.vv where indices are >= VLMAX (with LMUL < 8). +TEST_F(RiscVVectorPermuteInstructionsTest, + Vrgatherei16VVIndexOutOfBoundsZeroesElement) { + SetSemanticFunction(&Vrgatherei16); + AppendVectorRegisterOperands({kVs2, kVs1, kVmask}, {kVd}); + + // LMUL = 1, SEW = 32 bits (4 bytes), index SEW = 16 bits (2 bytes). VLMAX = + // 16 elements. + uint32_t vtype = (kSewSettingsByByteSize[sizeof(uint32_t)] << 3) | + kLmulSettingByLogSize[4]; + ConfigureVectorUnit(vtype, 16); + + int num_values_per_reg = kVectorLengthInBytes / sizeof(uint32_t); // 16 + auto vs2_span = vreg()[kVs2]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_span[i] = 200 + i; + } + auto vs2_plus1_span = vreg()[kVs2 + 1]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_plus1_span[i] = 600 + i; + } + + // Indices in vs1 (uint16_t). + auto vs1_span = vreg()[kVs1]->data_buffer()->Get<uint16_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs1_span[i] = (i % 2 == 0) ? (i / 2) : (16 + i); + } + + uint8_t all_ones_mask[kVectorLengthInBytes]; + memset(all_ones_mask, 0xff, sizeof(all_ones_mask)); + SetVectorRegisterValues<uint8_t>({{kVmaskName, all_ones_mask}}); + + instruction_->Execute(); + + auto vd_span = vreg()[kVd]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + if (i % 2 == 0) { + EXPECT_EQ(vd_span[i], vs2_span[i / 2]) << "at index " << i; + } else { + EXPECT_EQ(vd_span[i], 0) + << "at index " << i << " (index value " << (16 + i) << " >= VLMAX)"; + } + } +} + +// Test multi-register masking across register boundaries. +TEST_F(RiscVVectorPermuteInstructionsTest, VrgatherMultiRegisterMask) { + SetSemanticFunction(&Vrgather); + AppendVectorRegisterOperands({kVs2, kVs1, kVmask}, {kVd}); + + // LMUL = 2, SEW = 32 bits (4 bytes). 2 registers = 32 elements. + uint32_t vtype = (kSewSettingsByByteSize[sizeof(uint32_t)] << 3) | + kLmulSettingByLogSize[5]; + ConfigureVectorUnit(vtype, 32); + + int num_values_per_reg = kVectorLengthInBytes / sizeof(uint32_t); // 16 + for (int reg = 0; reg < 2; reg++) { + auto vs2_span = vreg()[kVs2 + reg]->data_buffer()->Get<uint32_t>(); + auto vs1_span = vreg()[kVs1 + reg]->data_buffer()->Get<uint32_t>(); + auto vd_span = vreg()[kVd + reg]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + vs2_span[i] = 100 * (reg + 1) + i; + vs1_span[i] = reg * num_values_per_reg + i; + vd_span[i] = 0x55555555; + } + } + + // Set mask: reg 0 (elements 0..15, bytes 0..1) = 0xFF (active). + // reg 1 (elements 16..31, bytes 2..3) = 0x00 (inactive). + uint8_t mask[kVectorLengthInBytes]; + memset(mask, 0, sizeof(mask)); + mask[0] = 0xff; + mask[1] = 0xff; + SetVectorRegisterValues<uint8_t>({{kVmaskName, mask}}); + + instruction_->Execute(); + + // Elements 0..15 in vd (v8) should be updated. + auto vd0_span = vreg()[kVd]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + EXPECT_EQ(vd0_span[i], 100 + i) << "at reg 0 index " << i; + } + // Elements 16..31 in vd (v9) should be UNMODIFIED (0x55555555). + auto vd1_span = vreg()[kVd + 1]->data_buffer()->Get<uint32_t>(); + for (int i = 0; i < num_values_per_reg; i++) { + EXPECT_EQ(vd1_span[i], 0x55555555) << "at reg 1 index " << i; + } +} + } // namespace