/* * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under the BSD-style license found in the * LICENSE file in the root directory of this source tree. */ #include #include #include #include using exec_aten::ScalarType; using exec_aten::Tensor; using executorch::runtime::KernelRuntimeContext; using executorch::runtime::kTensorDimensionLimit; using executorch::runtime::resize_tensor; using executorch::runtime::tensor_is_default_dim_order; using executorch::runtime::tensors_have_same_dim_order; using torch::executor::check_mm_args; using torch::executor::Error; using torch::executor::get_mm_out_target_size; namespace cadence { namespace impl { namespace HiFi { namespace native { Tensor& mm_out( KernelRuntimeContext& ctx, const Tensor& in, const Tensor& mat2, Tensor& out) { ET_KERNEL_CHECK(ctx, check_mm_args(in, mat2, out), InvalidArgument, out); size_t output_ndim = 0; exec_aten::SizesType output_sizes[kTensorDimensionLimit]; get_mm_out_target_size(in, mat2, output_sizes, &output_ndim); ET_KERNEL_CHECK( ctx, resize_tensor(out, {output_sizes, output_ndim}) == Error::Ok, InvalidArgument, out); ET_KERNEL_CHECK( ctx, tensors_have_same_dim_order(in, mat2, out), InvalidArgument, out); ET_KERNEL_CHECK(ctx, tensor_is_default_dim_order(in), InvalidArgument, out); ScalarType out_type = out.scalar_type(); constexpr auto name = "mm.out"; bool optimized = true; if (out_type != ScalarType::Float) optimized = false; if (optimized) { const float* in_data = in.const_data_ptr(); const float* mat2_data = mat2.const_data_ptr(); float* out_data = out.mutable_data_ptr(); int64_t m = in.size(0); int64_t n = in.size(1); int64_t p = mat2.size(1); WORD32 rows = m; WORD32 cols1 = n; WORD32 row_stride1 = n; WORD32 vec_count = p; WORD32 vec_offset = n; WORD32 out_offset = 1; WORD32 out_stride = p; WORD32* __restrict__ p_o = (WORD32* __restrict__)kernels::allocate_temp_memory( ctx, (n * p) * sizeof(WORD32)); WORD32 p_inp_shape[2]; p_inp_shape[0] = n; p_inp_shape[1] = p; WORD32 p_out_shape[2]; p_out_shape[0] = p; p_out_shape[1] = n; WORD32 p_permute_vec[2] = {1, 0}; WORD32 num_out_dims = 2; WORD32 num_inp_dims = 2; const FLOAT32* __restrict__ p_mat1 = in_data; const FLOAT32* __restrict__ p_vec1 = mat2_data; FLOAT32* __restrict__ p_out = out_data; WORD32* p_inp = (WORD32*)p_vec1; WORD32 t = xa_nn_transpose_32_32( p_o, p_out_shape, p_inp, p_inp_shape, p_permute_vec, num_out_dims, num_inp_dims); const FLOAT32* __restrict__ p_vec = (const FLOAT32* __restrict__)p_o; WORD32 val = xa_nn_matmul_f32xf32_f32( p_out, p_mat1, p_vec, NULL, rows, cols1, row_stride1, vec_count, vec_offset, out_offset, out_stride); return out; } ET_SWITCH_REAL_TYPES_AND2( Half, BFloat16, in.scalar_type(), ctx, name, CTYPE, [&]() { size_t m = in.size(0); size_t n = in.size(1); size_t p = mat2.size(1); torch::executor::vec_matmul( out.mutable_data_ptr(), in.const_data_ptr(), mat2.const_data_ptr(), m, n, p); }); return out; } } // namespace native } // namespace HiFi } // namespace impl } // namespace cadence