RTT works
This commit is contained in:
@@ -0,0 +1,235 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_1x1_HWC_q7_fast_nonsquare.c
|
||||
* Description: Fast Q7 version of 1x1 convolution (non-square shape)
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Fast Q7 version of 1x1 convolution (non-sqaure shape)
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in_x input tensor dimention x
|
||||
* @param[in] dim_im_in_y input tensor dimention y
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel_x filter kernel size x
|
||||
* @param[in] dim_kernel_y filter kernel size y
|
||||
* @param[in] padding_x padding size x
|
||||
* @param[in] padding_y padding size y
|
||||
* @param[in] stride_x convolution stride x
|
||||
* @param[in] stride_y convolution stride y
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out_x output tensor dimension x
|
||||
* @param[in] dim_im_out_y output tensor dimension y
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns either
|
||||
* <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
|
||||
*
|
||||
* This function is optimized for convolution with 1x1 kernel size (i.e., dim_kernel_x=1
|
||||
* and dim_kernel_y=1). It can be used for the second half of MobileNets [1] after depthwise
|
||||
* separable convolution.
|
||||
*
|
||||
* This function is the version with full list of optimization tricks, but with
|
||||
* some contraints:
|
||||
* ch_im_in is multiple of 4
|
||||
* ch_im_out is multiple of 2
|
||||
*
|
||||
* [1] MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications
|
||||
* https://arxiv.org/abs/1704.04861
|
||||
*/
|
||||
|
||||
arm_status arm_convolve_1x1_HWC_q7_fast_nonsquare(const q7_t * Im_in,
|
||||
const uint16_t dim_im_in_x,
|
||||
const uint16_t dim_im_in_y,
|
||||
const uint16_t ch_im_in,
|
||||
const q7_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel_x,
|
||||
const uint16_t dim_kernel_y,
|
||||
const uint16_t padding_x,
|
||||
const uint16_t padding_y,
|
||||
const uint16_t stride_x,
|
||||
const uint16_t stride_y,
|
||||
const q7_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q7_t * Im_out,
|
||||
const uint16_t dim_im_out_x,
|
||||
const uint16_t dim_im_out_y,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
int16_t i_out_y, i_out_x;
|
||||
int16_t i_ch_out;
|
||||
|
||||
/* -----------------------
|
||||
* Here we use bufferA as q15_t internally as computation are done with q15_t level
|
||||
* im2col are done to output in q15_t format from q7_t input
|
||||
*/
|
||||
|
||||
q15_t *pBuffer = bufferA;
|
||||
q7_t *pOut = Im_out;
|
||||
|
||||
if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1
|
||||
|| padding_x != 0 || padding_y != 0 || stride_x != 1 || stride_y != 1)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_out_y * dim_im_in_x + i_out_x) * ch_im_in, pBuffer,
|
||||
ch_im_in);
|
||||
pBuffer += ch_im_in;
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in, bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* check if there is left-over for compute */
|
||||
if (pBuffer != bufferA)
|
||||
{
|
||||
const q7_t *pA = wt;
|
||||
for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
|
||||
{
|
||||
q31_t sum = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
|
||||
q15_t *pB = bufferA;
|
||||
/* basically each time it process 4 entries */
|
||||
uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2;
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
|
||||
q31_t inA1, inA2;
|
||||
q31_t inB1, inB2;
|
||||
|
||||
pA = (const q7_t *)read_and_pad_reordered((void *)pA, &inA1, &inA2);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
inB2 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA2, inB2, sum);
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
sum += inA1 * inB1;
|
||||
colCnt--;
|
||||
}
|
||||
*pOut = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
pOut++;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
|
||||
int i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
int in_row, in_col;
|
||||
|
||||
if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1
|
||||
|| padding_x != 0 || padding_y != 0 || stride_x != 1 || stride_y != 1)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out_y; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out_x; k++)
|
||||
{
|
||||
conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel_y; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel_x; n++)
|
||||
{
|
||||
// if-for implementation
|
||||
in_row = stride_y * j + m - padding_y;
|
||||
in_col = stride_x * k + n - padding_x;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
|
||||
wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_y + n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_HWC_q15_basic.c
|
||||
* Description: Q15 version of convolution
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Basic Q15 convolution function
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in input tensor dimention
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel filter kernel size
|
||||
* @param[in] padding padding sizes
|
||||
* @param[in] stride convolution stride
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out output tensor dimension
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns <code>ARM_MATH_SUCCESS</code>
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* <b>Buffer size:</b>
|
||||
*
|
||||
* bufferA size: ch_im_in*dim_kernel*dim_kernel
|
||||
*
|
||||
* bufferB size: 0
|
||||
*
|
||||
* This basic version is designed to work for any input tensor and weight
|
||||
* dimension.
|
||||
*/
|
||||
|
||||
arm_status
|
||||
arm_convolve_HWC_q15_basic(const q15_t * Im_in,
|
||||
const uint16_t dim_im_in,
|
||||
const uint16_t ch_im_in,
|
||||
const q15_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel,
|
||||
const uint16_t padding,
|
||||
const uint16_t stride,
|
||||
const q15_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q15_t * Im_out,
|
||||
const uint16_t dim_im_out,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
|
||||
|
||||
uint16_t im2col_out_pixel_index = 0;
|
||||
q15_t *pBuffer = bufferA;
|
||||
q15_t *pOut = Im_out;
|
||||
q15_t *im_buffer = bufferA;
|
||||
const q15_t *pA;
|
||||
int i;
|
||||
|
||||
/* This part implements the im2col function */
|
||||
for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* Filling 0 for out-of-bound paddings */
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
/* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
|
||||
memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
pA = wt;
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
q15_t *pB = im_buffer;
|
||||
uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1 = *__SIMD32(pA)++;
|
||||
q31_t inB1 = *__SIMD32(pB)++;
|
||||
q31_t inA2 = *__SIMD32(pA)++;
|
||||
q31_t inB2 = *__SIMD32(pB)++;
|
||||
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
sum = __SMLAD(inA2, inB2, sum);
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q15_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
sum += inA1 * inB1;
|
||||
colCnt--;
|
||||
}
|
||||
*pOut = (q15_t) __SSAT((sum >> out_shift), 16);
|
||||
pOut++;
|
||||
}
|
||||
|
||||
/* counter reset */
|
||||
pBuffer = im_buffer;
|
||||
im2col_out_pixel_index++;
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
uint16_t i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
signed char in_row, in_col;
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out; k++)
|
||||
{
|
||||
conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel; n++)
|
||||
{
|
||||
in_row = stride * j + m - padding;
|
||||
in_col = stride * k + n - padding;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out +=
|
||||
Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
|
||||
l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
|
||||
n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,255 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_HWC_q15_fast.c
|
||||
* Description: Fast Q15 version of convolution
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Fast Q15 convolution function
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in input tensor dimention
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel filter kernel size
|
||||
* @param[in] padding padding sizes
|
||||
* @param[in] stride convolution stride
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out output tensor dimension
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns either
|
||||
* <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* <b>Buffer size:</b>
|
||||
*
|
||||
* bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
|
||||
*
|
||||
* bufferB size: 0
|
||||
*
|
||||
* <b>Input dimension constraints:</b>
|
||||
*
|
||||
* ch_im_in is multiple of 2
|
||||
*
|
||||
* ch_im_out is multipe of 2
|
||||
*
|
||||
*/
|
||||
|
||||
arm_status
|
||||
arm_convolve_HWC_q15_fast(const q15_t * Im_in,
|
||||
const uint16_t dim_im_in,
|
||||
const uint16_t ch_im_in,
|
||||
const q15_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel,
|
||||
const uint16_t padding,
|
||||
const uint16_t stride,
|
||||
const q15_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q15_t * Im_out,
|
||||
const uint16_t dim_im_out,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
|
||||
|
||||
q15_t *pBuffer = bufferA;
|
||||
q15_t *im_buffer = bufferA;
|
||||
q15_t *pOut = Im_out;
|
||||
|
||||
if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
/* This part implements the im2col function */
|
||||
for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
/* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
|
||||
memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (i_out_x & 0x1)
|
||||
{
|
||||
int i;
|
||||
/* initialize the matrix pointers for A */
|
||||
const q15_t *pA = wt;
|
||||
|
||||
/* set up the second output pointers */
|
||||
q15_t *pOut2 = pOut + ch_im_out;
|
||||
|
||||
/* this loop over rows in A */
|
||||
for (i = 0; i < ch_im_out; i += 2)
|
||||
{
|
||||
/* setup pointers for B */
|
||||
q15_t *pB = im_buffer;
|
||||
const q15_t *pB2 = pB + ch_im_in * dim_kernel * dim_kernel;
|
||||
|
||||
/* aling the second pointer for A */
|
||||
const q15_t *pA2 = pA + ch_im_in * dim_kernel * dim_kernel;
|
||||
|
||||
/* init the sum with bias */
|
||||
q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 1;
|
||||
/* accumulate over the vector */
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1 = *__SIMD32(pA)++;
|
||||
q31_t inB1 = *__SIMD32(pB)++;
|
||||
q31_t inA2 = *__SIMD32(pA2)++;
|
||||
q31_t inB2 = *__SIMD32(pB2)++;
|
||||
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
sum2 = __SMLAD(inA1, inB2, sum2);
|
||||
sum3 = __SMLAD(inA2, inB1, sum3);
|
||||
sum4 = __SMLAD(inA2, inB2, sum4);
|
||||
|
||||
colCnt--;
|
||||
} /* while over colCnt */
|
||||
colCnt = ch_im_in * dim_kernel * dim_kernel & 0x1;
|
||||
while (colCnt)
|
||||
{
|
||||
q15_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
q15_t inA2 = *pA2++;
|
||||
q15_t inB2 = *pB2++;
|
||||
|
||||
sum += inA1 * inB1;
|
||||
sum2 += inA1 * inB2;
|
||||
sum3 += inA2 * inB1;
|
||||
sum4 += inA2 * inB2;
|
||||
colCnt--;
|
||||
} /* while over colCnt */
|
||||
*pOut++ = (q15_t) __SSAT(sum >> out_shift, 16);
|
||||
*pOut++ = (q15_t) __SSAT(sum3 >> out_shift, 16);
|
||||
*pOut2++ = (q15_t) __SSAT(sum2 >> out_shift, 16);
|
||||
*pOut2++ = (q15_t) __SSAT(sum4 >> out_shift, 16);
|
||||
|
||||
/* skip the row computed with A2 */
|
||||
pA += ch_im_in * dim_kernel * dim_kernel;
|
||||
} /* for over ch_im_out */
|
||||
|
||||
pOut += ch_im_out;
|
||||
/* counter reset */
|
||||
pBuffer = im_buffer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
uint16_t i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
signed char in_row, in_col;
|
||||
|
||||
if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out; k++)
|
||||
{
|
||||
conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel; n++)
|
||||
{
|
||||
in_row = stride * j + m - padding;
|
||||
in_col = stride * k + n - padding;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out +=
|
||||
Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
|
||||
l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
|
||||
n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,265 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_HWC_q15_fast.c
|
||||
* Description: Fast Q15 version of convolution
|
||||
*
|
||||
* $Date: 24. May 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Fast Q15 convolution function (non-sqaure shape)
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in_x input tensor dimention x
|
||||
* @param[in] dim_im_in_y input tensor dimention y
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel_x filter kernel size x
|
||||
* @param[in] dim_kernel_y filter kernel size y
|
||||
* @param[in] padding_x padding size x
|
||||
* @param[in] padding_y padding size y
|
||||
* @param[in] stride_x convolution stride x
|
||||
* @param[in] stride_y convolution stride y
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out_x output tensor dimension x
|
||||
* @param[in] dim_im_out_y output tensor dimension y
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns either
|
||||
* <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* <b>Buffer size:</b>
|
||||
*
|
||||
* bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
|
||||
*
|
||||
* bufferB size: 0
|
||||
*
|
||||
* <b>Input dimension constraints:</b>
|
||||
*
|
||||
* ch_im_in is multiple of 2
|
||||
*
|
||||
* ch_im_out is multipe of 2
|
||||
*
|
||||
*/
|
||||
|
||||
arm_status
|
||||
arm_convolve_HWC_q15_fast_nonsquare(const q15_t * Im_in,
|
||||
const uint16_t dim_im_in_x,
|
||||
const uint16_t dim_im_in_y,
|
||||
const uint16_t ch_im_in,
|
||||
const q15_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel_x,
|
||||
const uint16_t dim_kernel_y,
|
||||
const uint16_t padding_x,
|
||||
const uint16_t padding_y,
|
||||
const uint16_t stride_x,
|
||||
const uint16_t stride_y,
|
||||
const q15_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q15_t * Im_out,
|
||||
const uint16_t dim_im_out_x,
|
||||
const uint16_t dim_im_out_y,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
|
||||
|
||||
q15_t *pBuffer = bufferA;
|
||||
q15_t *im_buffer = bufferA;
|
||||
q15_t *pOut = Im_out;
|
||||
|
||||
if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
/* This part implements the im2col function */
|
||||
for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
|
||||
{
|
||||
for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
/* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
|
||||
memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (i_out_x & 0x1)
|
||||
{
|
||||
int i;
|
||||
/* initialize the matrix pointers for A */
|
||||
const q15_t *pA = wt;
|
||||
|
||||
/* set up the second output pointers */
|
||||
q15_t *pOut2 = pOut + ch_im_out;
|
||||
|
||||
/* this loop over rows in A */
|
||||
for (i = 0; i < ch_im_out; i += 2)
|
||||
{
|
||||
/* setup pointers for B */
|
||||
q15_t *pB = im_buffer;
|
||||
const q15_t *pB2 = pB + ch_im_in * dim_kernel_y * dim_kernel_x;
|
||||
|
||||
/* aling the second pointer for A */
|
||||
const q15_t *pA2 = pA + ch_im_in * dim_kernel_y * dim_kernel_x;
|
||||
|
||||
/* init the sum with bias */
|
||||
q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 1;
|
||||
/* accumulate over the vector */
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1 = *__SIMD32(pA)++;
|
||||
q31_t inB1 = *__SIMD32(pB)++;
|
||||
q31_t inA2 = *__SIMD32(pA2)++;
|
||||
q31_t inB2 = *__SIMD32(pB2)++;
|
||||
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
sum2 = __SMLAD(inA1, inB2, sum2);
|
||||
sum3 = __SMLAD(inA2, inB1, sum3);
|
||||
sum4 = __SMLAD(inA2, inB2, sum4);
|
||||
|
||||
colCnt--;
|
||||
} /* while over colCnt */
|
||||
colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x1;
|
||||
while (colCnt)
|
||||
{
|
||||
q15_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
q15_t inA2 = *pA2++;
|
||||
q15_t inB2 = *pB2++;
|
||||
|
||||
sum += inA1 * inB1;
|
||||
sum2 += inA1 * inB2;
|
||||
sum3 += inA2 * inB1;
|
||||
sum4 += inA2 * inB2;
|
||||
colCnt--;
|
||||
} /* while over colCnt */
|
||||
*pOut++ = (q15_t) __SSAT(sum >> out_shift, 16);
|
||||
*pOut++ = (q15_t) __SSAT(sum3 >> out_shift, 16);
|
||||
*pOut2++ = (q15_t) __SSAT(sum2 >> out_shift, 16);
|
||||
*pOut2++ = (q15_t) __SSAT(sum4 >> out_shift, 16);
|
||||
|
||||
/* skip the row computed with A2 */
|
||||
pA += ch_im_in * dim_kernel_y * dim_kernel_x;
|
||||
} /* for over ch_im_out */
|
||||
|
||||
pOut += ch_im_out;
|
||||
/* counter reset */
|
||||
pBuffer = im_buffer;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
uint16_t i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
signed char in_row, in_col;
|
||||
|
||||
if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out_y; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out_x; k++)
|
||||
{
|
||||
conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel_y; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel_x; n++)
|
||||
{
|
||||
in_row = stride_y * j + m - padding_y;
|
||||
in_col = stride_x * k + n - padding_x;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out +=
|
||||
Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in +
|
||||
l] * wt[i * ch_im_in * dim_kernel_x * dim_kernel_y + (m * dim_kernel_x +
|
||||
n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,279 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_HWC_q7_RGB.c
|
||||
* Description: Q7 version of convolution for RGB image
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Q7 convolution function for RGB image
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in input tensor dimention
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel filter kernel size
|
||||
* @param[in] padding padding sizes
|
||||
* @param[in] stride convolution stride
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out output tensor dimension
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns either
|
||||
* <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* <b>Buffer size:</b>
|
||||
*
|
||||
* bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
|
||||
*
|
||||
* bufferB size: 0
|
||||
*
|
||||
* <b>Input dimension constraints:</b>
|
||||
*
|
||||
* ch_im_in equals 3
|
||||
*
|
||||
* This kernel is written exclusively for convolution with ch_im_in
|
||||
* equals 3. This applies on the first layer of CNNs which has input
|
||||
* image with RGB format.
|
||||
*/
|
||||
|
||||
arm_status
|
||||
arm_convolve_HWC_q7_RGB(const q7_t * Im_in,
|
||||
const uint16_t dim_im_in,
|
||||
const uint16_t ch_im_in,
|
||||
const q7_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel,
|
||||
const uint16_t padding,
|
||||
const uint16_t stride,
|
||||
const q7_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q7_t * Im_out, const uint16_t dim_im_out, q15_t * bufferA, q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
|
||||
|
||||
/*
|
||||
* Here we use bufferA as q15_t internally as computation are done with q15_t level
|
||||
* im2col are done to output in q15_t format from q7_t input
|
||||
*/
|
||||
q15_t *pBuffer = bufferA;
|
||||
q7_t *pOut = Im_out;
|
||||
|
||||
// check if number of input channels is 3
|
||||
if (ch_im_in != 3)
|
||||
{
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
// This part implements the im2col function
|
||||
for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* Equivalent to arm_fill_q15(0, pBuffer, ch_im_in) with assumption: ch_im_in = 3 */
|
||||
*__SIMD32(pBuffer) = 0x0;
|
||||
*(pBuffer + 2) = 0;
|
||||
pBuffer += 3;
|
||||
} else
|
||||
{
|
||||
/*
|
||||
* Equivalent to:
|
||||
* arm_q7_to_q15_no_shift( (q7_t*)Im_in+(i_ker_y*dim_im_in+i_ker_x)*3, pBuffer, 3);
|
||||
*/
|
||||
|
||||
const q7_t *pPixel = Im_in + (i_ker_y * dim_im_in + i_ker_x) * 3;
|
||||
q31_t buf = *__SIMD32(pPixel);
|
||||
|
||||
union arm_nnword top;
|
||||
union arm_nnword bottom;
|
||||
|
||||
top.word = __SXTB16(buf);
|
||||
bottom.word = __SXTB16(__ROR(buf, 8));
|
||||
|
||||
#ifndef ARM_MATH_BIG_ENDIAN
|
||||
/*
|
||||
* little-endian, | omit | 3rd | 2nd | 1st |
|
||||
* MSB LSB
|
||||
* top | 3rd | 1st |; bottom | omit | 2nd |
|
||||
*
|
||||
* version 1, need to swap 2nd and 3rd weight
|
||||
* *__SIMD32(pBuffer) = top.word;
|
||||
* *(pBuffer+2) = bottom.half_words[0];
|
||||
*
|
||||
* version 2, no weight shuffling required
|
||||
*/
|
||||
*pBuffer++ = top.half_words[0];
|
||||
*__SIMD32(pBuffer) = __PKHBT(bottom.word, top.word, 0);
|
||||
#else
|
||||
/*
|
||||
* big-endian, | 1st | 2nd | 3rd | omit |
|
||||
* MSB LSB
|
||||
* top | 2nd | omit |; bottom | 1st | 3rd |
|
||||
*
|
||||
* version 1, need to swap 2nd and 3rd weight
|
||||
* *__SIMD32(pBuffer) = bottom.word;
|
||||
* *(pBuffer+2) = top.half_words[1];
|
||||
*
|
||||
* version 2, no weight shuffling required
|
||||
*/
|
||||
*pBuffer++ = bottom.half_words[0];
|
||||
*__SIMD32(pBuffer) = __PKHTB(top.word, bottom.word, 0);
|
||||
#endif
|
||||
pBuffer += 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * 3 * dim_kernel * dim_kernel)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
|
||||
ch_im_out,
|
||||
3 * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
|
||||
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* left-over because odd number of output pixels */
|
||||
if (pBuffer != bufferA)
|
||||
{
|
||||
const q7_t *pA = wt;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
q15_t *pB = bufferA;
|
||||
/* basically each time it process 4 entries */
|
||||
uint16_t colCnt = 3 * dim_kernel * dim_kernel >> 2;
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
|
||||
q31_t inA1, inA2;
|
||||
q31_t inB1, inB2;
|
||||
|
||||
pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
inB2 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA2, inB2, sum);
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
colCnt = 3 * dim_kernel * dim_kernel & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
sum += inA1 * inB1;
|
||||
colCnt--;
|
||||
}
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
|
||||
uint16_t i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
signed char in_row, in_col;
|
||||
|
||||
// check if number of input channels is 3
|
||||
if (ch_im_in != 3)
|
||||
{
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out; k++)
|
||||
{
|
||||
conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel; n++)
|
||||
{
|
||||
/* if-for implementation */
|
||||
in_row = stride * j + m - padding;
|
||||
in_col = stride * k + n - padding;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out +=
|
||||
Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
|
||||
l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
|
||||
n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return (ARM_MATH_SUCCESS);
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,230 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_HWC_q7_basic.c
|
||||
* Description: Q7 version of convolution
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Basic Q7 convolution function
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in input tensor dimention
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel filter kernel size
|
||||
* @param[in] padding padding sizes
|
||||
* @param[in] stride convolution stride
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out output tensor dimension
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns <code>ARM_MATH_SUCCESS</code>
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* <b>Buffer size:</b>
|
||||
*
|
||||
* bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
|
||||
*
|
||||
* bufferB size: 0
|
||||
*
|
||||
* This basic version is designed to work for any input tensor and weight
|
||||
* dimension.
|
||||
*/
|
||||
|
||||
arm_status
|
||||
arm_convolve_HWC_q7_basic(const q7_t * Im_in,
|
||||
const uint16_t dim_im_in,
|
||||
const uint16_t ch_im_in,
|
||||
const q7_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel,
|
||||
const uint16_t padding,
|
||||
const uint16_t stride,
|
||||
const q7_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q7_t * Im_out,
|
||||
const uint16_t dim_im_out,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
|
||||
|
||||
/*
|
||||
* Here we use bufferA as q15_t internally as computation are done with q15_t level
|
||||
* im2col are done to output in q15_t format from q7_t input
|
||||
*/
|
||||
q15_t *pBuffer = bufferA;
|
||||
q7_t *pOut = Im_out;
|
||||
|
||||
/* This part implements the im2col function */
|
||||
for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* Filling 0 for out-of-bound paddings */
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
/* Copying the pixel data to column */
|
||||
arm_q7_to_q15_no_shift((q7_t *)
|
||||
Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
/* Computation is filed for every 2 columns */
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
|
||||
ch_im_out,
|
||||
ch_im_in *
|
||||
dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
|
||||
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* left-over because odd number of output pixels */
|
||||
if (pBuffer != bufferA)
|
||||
{
|
||||
const q7_t *pA = wt;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
/* Load the accumulator with bias first */
|
||||
q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
/* Point to the beging of the im2col buffer */
|
||||
q15_t *pB = bufferA;
|
||||
|
||||
/* Each time it process 4 entries */
|
||||
uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1, inA2;
|
||||
q31_t inB1, inB2;
|
||||
|
||||
pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
inB2 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA2, inB2, sum);
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
sum += inA1 * inB1;
|
||||
colCnt--;
|
||||
}
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
|
||||
uint16_t i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
signed char in_row, in_col;
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out; k++)
|
||||
{
|
||||
conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel; n++)
|
||||
{
|
||||
// if-for implementation
|
||||
in_row = stride * j + m - padding;
|
||||
in_col = stride * k + n - padding;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out +=
|
||||
Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
|
||||
l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
|
||||
n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,228 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_HWC_q7_basic.c
|
||||
* Description: Q7 version of convolution
|
||||
*
|
||||
* $Date: 13. July 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Basic Q7 convolution function (non-sqaure shape)
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in_x input tensor dimention x
|
||||
* @param[in] dim_im_in_y input tensor dimention y
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel_x filter kernel size x
|
||||
* @param[in] dim_kernel_y filter kernel size y
|
||||
* @param[in] padding_x padding size x
|
||||
* @param[in] padding_y padding size y
|
||||
* @param[in] stride_x convolution stride x
|
||||
* @param[in] stride_y convolution stride y
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out_x output tensor dimension x
|
||||
* @param[in] dim_im_out_y output tensor dimension y
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns <code>ARM_MATH_SUCCESS</code>
|
||||
*/
|
||||
|
||||
arm_status arm_convolve_HWC_q7_basic_nonsquare(const q7_t * Im_in,
|
||||
const uint16_t dim_im_in_x,
|
||||
const uint16_t dim_im_in_y,
|
||||
const uint16_t ch_im_in,
|
||||
const q7_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel_x,
|
||||
const uint16_t dim_kernel_y,
|
||||
const uint16_t padding_x,
|
||||
const uint16_t padding_y,
|
||||
const uint16_t stride_x,
|
||||
const uint16_t stride_y,
|
||||
const q7_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q7_t * Im_out,
|
||||
const uint16_t dim_im_out_x,
|
||||
const uint16_t dim_im_out_y,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
|
||||
|
||||
/*
|
||||
* Here we use bufferA as q15_t internally as computation are done with q15_t level
|
||||
* im2col are done to output in q15_t format from q7_t input
|
||||
*/
|
||||
q15_t *pBuffer = bufferA;
|
||||
q7_t *pOut = Im_out;
|
||||
|
||||
/* This part implements the im2col function */
|
||||
for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
|
||||
{
|
||||
for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
|
||||
{
|
||||
/* Filling 0 for out-of-bound paddings */
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
/* Copying the pixel data to column */
|
||||
arm_q7_to_q15_no_shift((q7_t *)
|
||||
Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
/* Computation is filed for every 2 columns */
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_y * dim_kernel_x)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
|
||||
ch_im_out,
|
||||
ch_im_in *
|
||||
dim_kernel_y * dim_kernel_x, bias_shift, out_shift, bias, pOut);
|
||||
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* left-over because odd number of output pixels */
|
||||
if (pBuffer != bufferA)
|
||||
{
|
||||
const q7_t *pA = wt;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
/* Load the accumulator with bias first */
|
||||
q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
/* Point to the beging of the im2col buffer */
|
||||
q15_t *pB = bufferA;
|
||||
|
||||
/* Each time it process 4 entries */
|
||||
uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 2;
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1, inA2;
|
||||
q31_t inB1, inB2;
|
||||
|
||||
pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
inB2 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA2, inB2, sum);
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
sum += inA1 * inB1;
|
||||
colCnt--;
|
||||
}
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
|
||||
uint16_t i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
signed char in_row, in_col;
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out_y; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out_x; k++)
|
||||
{
|
||||
conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel_y; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel_x; n++)
|
||||
{
|
||||
// if-for implementation
|
||||
in_row = stride_y * j + m - padding_y;
|
||||
in_col = stride_x * k + n - padding_x;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out +=
|
||||
Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
|
||||
wt[i * ch_im_in * dim_kernel_y * dim_kernel_x +
|
||||
(m * dim_kernel_x + n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,408 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_HWC_q7_fast.c
|
||||
* Description: Fast Q7 version of convolution
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Fast Q7 convolution function
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in input tensor dimention
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel filter kernel size
|
||||
* @param[in] padding padding sizes
|
||||
* @param[in] stride convolution stride
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out output tensor dimension
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns either
|
||||
* <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* <b>Buffer size:</b>
|
||||
*
|
||||
* bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
|
||||
*
|
||||
* bufferB size: 0
|
||||
*
|
||||
* <b>Input dimension constraints:</b>
|
||||
*
|
||||
* ch_im_in is multiple of 4 ( because of the SIMD32 read and swap )
|
||||
*
|
||||
* ch_im_out is multipe of 2 ( bacause 2x2 mat_mult kernel )
|
||||
*
|
||||
* The im2col converts the Q7 tensor input into Q15 column, which is stored in
|
||||
* bufferA. There is reordering happenning during this im2col process with
|
||||
* arm_q7_to_q15_reordered_no_shift. For every four elements, the second and
|
||||
* third elements are swapped.
|
||||
*
|
||||
* The computation kernel arm_nn_mat_mult_kernel_q7_q15_reordered does the
|
||||
* GEMM computation with the reordered columns.
|
||||
*
|
||||
* To speed-up the determination of the padding condition, we split the
|
||||
* computation into 3x3 parts, i.e., {top, mid, bottom} X {left, mid, right}.
|
||||
* This reduces the total number of boundary condition checks and improves
|
||||
* the data copying performance.
|
||||
*/
|
||||
|
||||
arm_status
|
||||
arm_convolve_HWC_q7_fast(const q7_t * Im_in,
|
||||
const uint16_t dim_im_in,
|
||||
const uint16_t ch_im_in,
|
||||
const q7_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel,
|
||||
const uint16_t padding,
|
||||
const uint16_t stride,
|
||||
const q7_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q7_t * Im_out,
|
||||
const uint16_t dim_im_out,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
|
||||
|
||||
/*
|
||||
* Here we use bufferA as q15_t internally as computation are done with q15_t level
|
||||
* im2col are done to output in q15_t format from q7_t input
|
||||
*/
|
||||
|
||||
q15_t *pBuffer = bufferA;
|
||||
q7_t *pOut = Im_out;
|
||||
|
||||
if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
/*
|
||||
* Here we split the entire matrix into three regions depending on the padding situation
|
||||
* Top: i_out_y from 0 to padding - 1
|
||||
* Middle: i_out_y from padding to dim_im_out-padding-1
|
||||
* Bottom: i_out_y from dim_im_out-padding to dim_im_out-1
|
||||
*/
|
||||
|
||||
/* top part */
|
||||
for (i_out_y = 0; i_out_y < padding; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift
|
||||
((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
|
||||
bufferA,
|
||||
ch_im_out,
|
||||
ch_im_in
|
||||
*
|
||||
dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* middle part, here we also divide the x into left, mid and right */
|
||||
for (; i_out_y < dim_im_out - padding; i_out_y++)
|
||||
{
|
||||
|
||||
/* left part */
|
||||
for (i_out_x = 0; i_out_x < padding; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift
|
||||
((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
|
||||
bufferA,
|
||||
ch_im_out,
|
||||
ch_im_in
|
||||
*
|
||||
dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
|
||||
/* mid part */
|
||||
for (; i_out_x < dim_im_out - padding; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in
|
||||
+
|
||||
(i_ker_y *
|
||||
dim_im_in +
|
||||
i_out_x *
|
||||
stride - padding) * ch_im_in, pBuffer, ch_im_in * dim_kernel);
|
||||
pBuffer += ch_im_in * dim_kernel;
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
|
||||
bufferA,
|
||||
ch_im_out,
|
||||
ch_im_in
|
||||
*
|
||||
dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
|
||||
/* right part */
|
||||
for (; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift
|
||||
((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
|
||||
bufferA,
|
||||
ch_im_out,
|
||||
ch_im_in
|
||||
*
|
||||
dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (; i_out_y < dim_im_out; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift
|
||||
((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
|
||||
bufferA,
|
||||
ch_im_out,
|
||||
ch_im_in
|
||||
*
|
||||
dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* check if there is left-over for compute */
|
||||
if (pBuffer != bufferA)
|
||||
{
|
||||
const q7_t *pA = wt;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
q15_t *pB = bufferA;
|
||||
/* each time it process 4 entries */
|
||||
uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
|
||||
q31_t inA1, inA2;
|
||||
q31_t inB1, inB2;
|
||||
|
||||
pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA1, &inA2);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
inB2 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA2, inB2, sum);
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
sum += inA1 * inB1;
|
||||
colCnt--;
|
||||
}
|
||||
*pOut = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
pOut++;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
|
||||
uint16_t i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
signed char in_row, in_col;
|
||||
|
||||
if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out; k++)
|
||||
{
|
||||
conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel; n++)
|
||||
{
|
||||
// if-for implementation
|
||||
in_row = stride * j + m - padding;
|
||||
in_col = stride * k + n - padding;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out +=
|
||||
Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
|
||||
l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
|
||||
n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,379 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_convolve_HWC_q7_fast_nonsquare.c
|
||||
* Description: Fast Q7 version of convolution (non-sqaure shape)
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Fast Q7 convolution function (non-sqaure shape)
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in_x input tensor dimention x
|
||||
* @param[in] dim_im_in_y input tensor dimention y
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel_x filter kernel size x
|
||||
* @param[in] dim_kernel_y filter kernel size y
|
||||
* @param[in] padding_x padding size x
|
||||
* @param[in] padding_y padding size y
|
||||
* @param[in] stride_x convolution stride x
|
||||
* @param[in] stride_y convolution stride y
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out_x output tensor dimension x
|
||||
* @param[in] dim_im_out_y output tensor dimension y
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns either
|
||||
* <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
|
||||
*
|
||||
* This function is the version with full list of optimization tricks, but with
|
||||
* some contraints:
|
||||
* ch_im_in is multiple of 4
|
||||
* ch_im_out is multiple of 2
|
||||
*/
|
||||
|
||||
arm_status arm_convolve_HWC_q7_fast_nonsquare(const q7_t * Im_in,
|
||||
const uint16_t dim_im_in_x,
|
||||
const uint16_t dim_im_in_y,
|
||||
const uint16_t ch_im_in,
|
||||
const q7_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel_x,
|
||||
const uint16_t dim_kernel_y,
|
||||
const uint16_t padding_x,
|
||||
const uint16_t padding_y,
|
||||
const uint16_t stride_x,
|
||||
const uint16_t stride_y,
|
||||
const q7_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q7_t * Im_out,
|
||||
const uint16_t dim_im_out_x,
|
||||
const uint16_t dim_im_out_y,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
|
||||
|
||||
/* -----------------------
|
||||
* Here we use bufferA as q15_t internally as computation are done with q15_t level
|
||||
* im2col are done to output in q15_t format from q7_t input
|
||||
*/
|
||||
|
||||
q15_t *pBuffer = bufferA;
|
||||
q7_t *pOut = Im_out;
|
||||
|
||||
if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
/*
|
||||
* Here we split the entire matrix into three regions depending on the padding situation
|
||||
* Top: i_out_y from 0 to padding - 1
|
||||
* Middle: i_out_y from padding to dim_im_out-padding-1
|
||||
* Bottom: i_out_y from dim_im_out-padding to dim_im_out-1
|
||||
*/
|
||||
|
||||
/* top part */
|
||||
for (i_out_y = 0; i_out_y < padding_y; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
|
||||
i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
|
||||
i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
|
||||
pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
|
||||
bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* middle part, here we also divide the x into left, mid and right */
|
||||
for (; i_out_y < dim_im_out_y - padding_y; i_out_y++)
|
||||
{
|
||||
|
||||
/* left part */
|
||||
for (i_out_x = 0; i_out_x < padding_x; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
|
||||
i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
|
||||
i_ker_x++)
|
||||
{
|
||||
if (i_ker_x < 0 || i_ker_x >= dim_im_in_x)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
|
||||
pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
|
||||
bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
|
||||
/* mid part */
|
||||
for (; i_out_x < dim_im_out_x - padding_x; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
|
||||
i_ker_y++)
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in +
|
||||
(i_ker_y * dim_im_in_x + i_out_x * stride_x - padding_x) * ch_im_in,
|
||||
pBuffer, ch_im_in * dim_kernel_x);
|
||||
pBuffer += ch_im_in * dim_kernel_x;
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
|
||||
bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
|
||||
/* right part */
|
||||
for (; i_out_x < dim_im_out_x; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
|
||||
i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
|
||||
i_ker_x++)
|
||||
{
|
||||
if (i_ker_x < 0 || i_ker_x >= dim_im_in_x)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
|
||||
pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
|
||||
bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (; i_out_y < dim_im_out_y; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
|
||||
{
|
||||
/* This part implements the im2col function */
|
||||
for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
|
||||
i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
|
||||
i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
|
||||
{
|
||||
/* arm_fill_q15(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
|
||||
} else
|
||||
{
|
||||
arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
|
||||
pBuffer, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
|
||||
{
|
||||
pOut =
|
||||
arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
|
||||
bias_shift, out_shift, bias, pOut);
|
||||
/* counter reset */
|
||||
pBuffer = bufferA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* check if there is left-over for compute */
|
||||
if (pBuffer != bufferA)
|
||||
{
|
||||
const q7_t *pA = wt;
|
||||
int i;
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
|
||||
q15_t *pB = bufferA;
|
||||
/* basically each time it process 4 entries */
|
||||
uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2;
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
|
||||
q31_t inA1, inA2;
|
||||
q31_t inB1, inB2;
|
||||
|
||||
pA = (const q7_t *)read_and_pad_reordered((void *)pA, &inA1, &inA2);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA1, inB1, sum);
|
||||
inB2 = *__SIMD32(pB)++;
|
||||
sum = __SMLAD(inA2, inB2, sum);
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
colCnt = (ch_im_in * dim_kernel_y * dim_kernel_x) & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
sum += inA1 * inB1;
|
||||
colCnt--;
|
||||
}
|
||||
*pOut = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
pOut++;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
int i, j, k, l, m, n;
|
||||
int conv_out;
|
||||
int in_row, in_col;
|
||||
|
||||
if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
|
||||
{
|
||||
/* check if the input dimension meets the constraints */
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i = 0; i < ch_im_out; i++)
|
||||
{
|
||||
for (j = 0; j < dim_im_out_y; j++)
|
||||
{
|
||||
for (k = 0; k < dim_im_out_x; k++)
|
||||
{
|
||||
conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
|
||||
for (m = 0; m < dim_kernel_y; m++)
|
||||
{
|
||||
for (n = 0; n < dim_kernel_x; n++)
|
||||
{
|
||||
/* if-for implementation */
|
||||
in_row = stride_y * j + m - padding_y;
|
||||
in_col = stride_x * k + n - padding_x;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
|
||||
{
|
||||
for (l = 0; l < ch_im_in; l++)
|
||||
{
|
||||
conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
|
||||
wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_x + n) * ch_im_in + l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,418 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_depthwise_separable_conv_HWC_q7.c
|
||||
* Description: Q7 depthwise separable convolution function
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Q7 depthwise separable convolution function
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in input tensor dimention
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel filter kernel size
|
||||
* @param[in] padding padding sizes
|
||||
* @param[in] stride convolution stride
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out output tensor dimension
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns either
|
||||
* <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* <b>Buffer size:</b>
|
||||
*
|
||||
* bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
|
||||
*
|
||||
* bufferB size: 0
|
||||
*
|
||||
* <b>Input dimension constraints:</b>
|
||||
*
|
||||
* ch_im_in equals ch_im_out
|
||||
*
|
||||
* Implementation:
|
||||
* There are 3 nested loop here:
|
||||
* Inner loop: calculate each output value with MAC instruction over an accumulator
|
||||
* Mid loop: loop over different output channel
|
||||
* Outer loop: loop over different output (x, y)
|
||||
*/
|
||||
|
||||
arm_status arm_depthwise_separable_conv_HWC_q7(const q7_t * Im_in,
|
||||
const uint16_t dim_im_in,
|
||||
const uint16_t ch_im_in,
|
||||
const q7_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel,
|
||||
const uint16_t padding,
|
||||
const uint16_t stride,
|
||||
const q7_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q7_t * Im_out,
|
||||
const uint16_t dim_im_out,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
int16_t i_out_y, i_out_x;
|
||||
int16_t i_ker_y, i_ker_x;
|
||||
q7_t *colBuffer = (q7_t *) bufferA;
|
||||
q7_t *pBuffer = colBuffer;
|
||||
const q7_t *pBias = bias;
|
||||
q7_t *pOut = Im_out;
|
||||
uint16_t rowCnt;
|
||||
uint16_t row_shift;
|
||||
|
||||
/* do some checking here, basically ch_im_in == ch_im_out */
|
||||
if (ch_im_in != ch_im_out)
|
||||
{
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
/* we first do im2col here */
|
||||
for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
|
||||
{
|
||||
/* arm_fill_q7(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, ch_im_in);
|
||||
} else
|
||||
{
|
||||
/* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
|
||||
memcpy(pBuffer, (q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
/* we will do the computation here for each channel */
|
||||
rowCnt = ch_im_out >> 2;
|
||||
row_shift = 0;
|
||||
pBias = bias;
|
||||
|
||||
while (rowCnt)
|
||||
{
|
||||
q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
uint16_t colCnt = (dim_kernel * dim_kernel) >> 1;
|
||||
q7_t *pB = colBuffer + row_shift;
|
||||
const q7_t *pA = wt + row_shift;
|
||||
row_shift += 4;
|
||||
|
||||
#ifdef USE_INTRINSIC
|
||||
|
||||
#ifndef ARM_MATH_BIG_ENDIAN
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1, inA2, inB1, inB2, opA, opB;
|
||||
|
||||
inB1 = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
opB = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
inB2 = __PKHTB(opB, inB1, 16);
|
||||
inB1 = __PKHBT(inB1, opB, 16);
|
||||
inA1 = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
opB = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
inA2 = __PKHTB(opB, inA1, 16);
|
||||
inA1 = __PKHBT(inA1, opB, 16);
|
||||
opA = __SXTB16(inA1);
|
||||
opB = __SXTB16(inB1);
|
||||
sum = __SMLAD(opA, opB, sum);
|
||||
opA = __SXTB16(__ROR(inA1, 8));
|
||||
opB = __SXTB16(__ROR(inB1, 8));
|
||||
sum2 = __SMLAD(opA, opB, sum2);
|
||||
opA = __SXTB16(inA2);
|
||||
opB = __SXTB16(inB2);
|
||||
sum3 = __SMLAD(opA, opB, sum3);
|
||||
opA = __SXTB16(__ROR(inA2, 8));
|
||||
opB = __SXTB16(__ROR(inB2, 8));
|
||||
sum4 = __SMLAD(opA, opB, sum4);
|
||||
colCnt--;
|
||||
}
|
||||
#else
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1, inA2, inB1, inB2, opA, opB;
|
||||
|
||||
inB1 = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
opB = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
inB2 = __PKHBT(opB, inB1, 16);
|
||||
inB1 = __PKHTB(inB1, opB, 16);
|
||||
inA1 = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
opB = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
inA2 = __PKHBT(opB, inA1, 16);
|
||||
inA1 = __PKHTB(inA1, opB, 16);
|
||||
opA = __SXTB16(inA1);
|
||||
opB = __SXTB16(inB1);
|
||||
sum2 = __SMLAD(opA, opB, sum2);
|
||||
opA = __SXTB16(__ROR(inA1, 8));
|
||||
opB = __SXTB16(__ROR(inB1, 8));
|
||||
sum = __SMLAD(opA, opB, sum);
|
||||
opA = __SXTB16(inA2);
|
||||
opB = __SXTB16(inB2);
|
||||
sum4 = __SMLAD(opA, opB, sum4);
|
||||
opA = __SXTB16(__ROR(inA2, 8));
|
||||
opB = __SXTB16(__ROR(inB2, 8));
|
||||
sum3 = __SMLAD(opA, opB, sum3);
|
||||
colCnt--;
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_BIG_ENDIAN */
|
||||
|
||||
#else
|
||||
|
||||
#ifndef ARM_MATH_BIG_ENDIAN
|
||||
/*
|
||||
* r0 r1 r2 r3 r4 r5
|
||||
* inA1, inA2, inB1, inB2, opA, opB
|
||||
*/
|
||||
|
||||
asm volatile ("COL_LOOP_%=:\n"
|
||||
"ldr.w r2, [%[pB], #0]\n"
|
||||
"add.w %[pB], %[pB], %[ch_im_in]\n"
|
||||
"ldr.w r5, [%[pB], #0]\n"
|
||||
"add.w %[pB], %[pB], %[ch_im_in]\n"
|
||||
"pkhtb r3, r5, r2, ASR #16\n"
|
||||
"pkhbt r2, r2, r5, LSL #16\n"
|
||||
"ldr.w r0, [%[pA], #0]\n"
|
||||
"add.w %[pA], %[pA], %[ch_im_in]\n"
|
||||
"ldr.w r5, [%[pA], #0]\n"
|
||||
"add.w %[pA], %[pA], %[ch_im_in]\n"
|
||||
"pkhtb r1, r5, r0, ASR #16\n"
|
||||
"pkhbt r0, r0, r5, LSL #16\n"
|
||||
"sxtb16 r4, r0\n"
|
||||
"sxtb16 r5, r2\n"
|
||||
"smlad %[sum], r4, r5, %[sum]\n"
|
||||
"mov.w r4, r0, ror #8\n"
|
||||
"mov.w r5, r2, ror #8\n"
|
||||
"sxtb16 r4, r4\n"
|
||||
"sxtb16 r5, r5\n"
|
||||
"smlad %[sum2], r4, r5, %[sum2]\n"
|
||||
"sxtb16 r4, r1\n"
|
||||
"sxtb16 r5, r3\n"
|
||||
"smlad %[sum3], r4, r5, %[sum3]\n"
|
||||
"mov.w r4, r1, ror #8\n"
|
||||
"mov.w r5, r3, ror #8\n"
|
||||
"sxtb16 r4, r4\n"
|
||||
"sxtb16 r5, r5\n"
|
||||
"smlad %[sum4], r4, r5, %[sum4]\n"
|
||||
"subs %[colCnt], #1\n"
|
||||
"bne COL_LOOP_%=\n":[sum]
|
||||
"+r"(sum),[sum2] "+r"(sum2),
|
||||
[sum3] "+r"(sum3),
|
||||
[sum4] "+r"(sum4),[pB] "+r"(pB),
|
||||
[pA] "+r"(pA):[colCnt]
|
||||
"r"(colCnt),[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
|
||||
#else
|
||||
/*
|
||||
* r0 r1 r2 r3 r4 r5
|
||||
* inA1, inA2, inB1, inB2, opA, opB
|
||||
*/
|
||||
asm volatile ("COL_LOOP_%=:\n"
|
||||
"ldr.w r2, [%[pB], #0]\n"
|
||||
"add.w %[pB], %[pB], %[ch_im_in]\n"
|
||||
"ldr.w r5, [%[pB], #0]\n"
|
||||
"add.w %[pB], %[pB], %[ch_im_in]\n"
|
||||
"pkhbt r3, r5, r2, LSL #16\n"
|
||||
"pkhtb r2, r2, r5, ASR #16\n"
|
||||
"ldr.w r0, [%[pA], #0]\n"
|
||||
"add.w %[pA], %[pA], %[ch_im_in]\n"
|
||||
"ldr.w r5, [%[pA], #0]\n"
|
||||
"add.w %[pA], %[pA], %[ch_im_in]\n"
|
||||
"pkhbt r1, r5, r0, LSL #16\n"
|
||||
"pkhtb r0, r0, r5, ASR #16\n"
|
||||
"sxtb16 r4, r0\n"
|
||||
"sxtb16 r5, r2\n"
|
||||
"smlad %[sum2], r4, r5, %[sum2]\n"
|
||||
"mov.w r4, r0, ror #8\n"
|
||||
"mov.w r5, r2, ror #8\n"
|
||||
"sxtb16 r4, r4\n"
|
||||
"sxtb16 r5, r5\n"
|
||||
"smlad %[sum], r4, r5, %[sum]\n"
|
||||
"sxtb16 r4, r1\n"
|
||||
"sxtb16 r5, r3\n"
|
||||
"smlad %[sum4], r4, r5, %[sum4]\n"
|
||||
"mov.w r4, r1, ror #8\n"
|
||||
"mov.w r5, r3, ror #8\n"
|
||||
"sxtb16 r4, r4\n"
|
||||
"sxtb16 r5, r5\n"
|
||||
"smlad %[sum3], r4, r5, %[sum3]\n"
|
||||
"subs %[colCnt], #1\n"
|
||||
"bne COL_LOOP_%=\n":[sum]
|
||||
"+r"(sum),[sum2] "+r"(sum2),
|
||||
[sum3] "+r"(sum3),
|
||||
[sum4] "+r"(sum4),[pB] "+r"(pB),
|
||||
[pA] "+r"(pA):[colCnt]
|
||||
"r"(colCnt),[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
|
||||
|
||||
#endif /* ARM_MATH_BIG_ENDIAN */
|
||||
|
||||
#endif /* USE_INTRINSIC */
|
||||
|
||||
colCnt = (dim_kernel * dim_kernel) & 0x1;
|
||||
while (colCnt)
|
||||
{
|
||||
union arm_nnword inA, inB;
|
||||
inA.word = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
inB.word = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
sum += inA.bytes[0] * inB.bytes[0];
|
||||
sum2 += inA.bytes[1] * inB.bytes[1];
|
||||
sum3 += inA.bytes[2] * inB.bytes[2];
|
||||
sum4 += inA.bytes[3] * inB.bytes[3];
|
||||
colCnt--;
|
||||
}
|
||||
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
*pOut++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
|
||||
*pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
|
||||
*pOut++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
|
||||
|
||||
rowCnt--;
|
||||
}
|
||||
|
||||
rowCnt = ch_im_out & 0x3;
|
||||
while (rowCnt)
|
||||
{
|
||||
q7_t *pB = colBuffer + row_shift;
|
||||
const q7_t *pA = wt + row_shift;
|
||||
q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
uint16_t colCnt = (dim_kernel * dim_kernel);
|
||||
|
||||
row_shift += 1;
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t A1 = *pA;
|
||||
q7_t B1 = *pB;
|
||||
pA += ch_im_in;
|
||||
pB += ch_im_in;
|
||||
sum += A1 * B1;
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
rowCnt--;
|
||||
}
|
||||
|
||||
/* clear counter and pointers */
|
||||
pBuffer = colBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
int i_out_y, i_out_x, i_ch_out, i_ker_x, i_ker_y;
|
||||
int conv_out;
|
||||
|
||||
/* do some checking here, basically ch_im_in == ch_im_out */
|
||||
if (ch_im_in != ch_im_out)
|
||||
{
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
|
||||
{
|
||||
for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
|
||||
{
|
||||
// for each output
|
||||
conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
|
||||
for (i_ker_y = 0; i_ker_y < dim_kernel; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = 0; i_ker_x < dim_kernel; i_ker_x++)
|
||||
{
|
||||
int in_row = stride * i_out_y + i_ker_y - padding;
|
||||
int in_col = stride * i_out_x + i_ker_x - padding;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
|
||||
{
|
||||
conv_out +=
|
||||
Im_in[(in_row *
|
||||
dim_im_in +
|
||||
in_col) *
|
||||
ch_im_in +
|
||||
i_ch_out] * wt[(i_ker_y * dim_kernel + i_ker_x) * ch_im_out + i_ch_out];
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[(i_out_y * dim_im_out +
|
||||
i_out_x) * ch_im_out + i_ch_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_depthwise_separable_conv_HWC_q7_nonsquare.c
|
||||
* Description: Q7 depthwise separable convolution function (non-square shape)
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
*
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @ingroup groupNN
|
||||
*/
|
||||
|
||||
/**
|
||||
* @addtogroup NNConv
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Q7 depthwise separable convolution function (non-square shape)
|
||||
* @param[in] Im_in pointer to input tensor
|
||||
* @param[in] dim_im_in_x input tensor dimention x
|
||||
* @param[in] dim_im_in_y input tensor dimention y
|
||||
* @param[in] ch_im_in number of input tensor channels
|
||||
* @param[in] wt pointer to kernel weights
|
||||
* @param[in] ch_im_out number of filters, i.e., output tensor channels
|
||||
* @param[in] dim_kernel_x filter kernel size x
|
||||
* @param[in] dim_kernel_y filter kernel size y
|
||||
* @param[in] padding_x padding sizes x
|
||||
* @param[in] padding_y padding sizes y
|
||||
* @param[in] stride_x convolution stride x
|
||||
* @param[in] stride_y convolution stride y
|
||||
* @param[in] bias pointer to bias
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in,out] Im_out pointer to output tensor
|
||||
* @param[in] dim_im_out_x output tensor dimension x
|
||||
* @param[in] dim_im_out_y output tensor dimension y
|
||||
* @param[in,out] bufferA pointer to buffer space for input
|
||||
* @param[in,out] bufferB pointer to buffer space for output
|
||||
* @return The function returns either
|
||||
* <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
|
||||
*
|
||||
* This function is the version with full list of optimization tricks, but with
|
||||
* some contraints:
|
||||
* ch_im_in is multiple of 2
|
||||
* ch_im_out is multiple of 2
|
||||
*/
|
||||
|
||||
arm_status arm_depthwise_separable_conv_HWC_q7_nonsquare(const q7_t * Im_in,
|
||||
const uint16_t dim_im_in_x,
|
||||
const uint16_t dim_im_in_y,
|
||||
const uint16_t ch_im_in,
|
||||
const q7_t * wt,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t dim_kernel_x,
|
||||
const uint16_t dim_kernel_y,
|
||||
const uint16_t padding_x,
|
||||
const uint16_t padding_y,
|
||||
const uint16_t stride_x,
|
||||
const uint16_t stride_y,
|
||||
const q7_t * bias,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
q7_t * Im_out,
|
||||
const uint16_t dim_im_out_x,
|
||||
const uint16_t dim_im_out_y,
|
||||
q15_t * bufferA,
|
||||
q7_t * bufferB)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* Run the following code for Cortex-M4 and Cortex-M7 */
|
||||
|
||||
/*
|
||||
* Implementation:
|
||||
* There are 3 nested loop here:
|
||||
* Inner loop: calculate each output value with MAC instruction over an accumulator
|
||||
* Mid loop: loop over different output channel
|
||||
* Outer loop: loop over different output (x, y)
|
||||
*
|
||||
*/
|
||||
|
||||
int16_t i_out_y, i_out_x;
|
||||
int16_t i_ker_y, i_ker_x;
|
||||
q7_t *colBuffer = (q7_t *) bufferA;
|
||||
q7_t *pBuffer = colBuffer;
|
||||
const q7_t *pBias = bias;
|
||||
q7_t *pOut = Im_out;
|
||||
uint16_t rowCnt;
|
||||
uint16_t row_shift;
|
||||
|
||||
/* do some checking here, basically ch_im_in == ch_im_out */
|
||||
if (ch_im_in != ch_im_out)
|
||||
{
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
|
||||
{
|
||||
/* we first do im2col here */
|
||||
for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
|
||||
i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
|
||||
i_ker_x++)
|
||||
{
|
||||
if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
|
||||
{
|
||||
/* arm_fill_q7(0, pBuffer, ch_im_in); */
|
||||
memset(pBuffer, 0, ch_im_in);
|
||||
} else
|
||||
{
|
||||
/* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
|
||||
memcpy(pBuffer, (q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, ch_im_in);
|
||||
}
|
||||
pBuffer += ch_im_in;
|
||||
}
|
||||
}
|
||||
|
||||
/* we will do the computation here for each channel */
|
||||
rowCnt = ch_im_out >> 2;
|
||||
row_shift = 0;
|
||||
pBias = bias;
|
||||
|
||||
while (rowCnt)
|
||||
{
|
||||
q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
uint16_t colCnt = (dim_kernel_x * dim_kernel_y) >> 1;
|
||||
q7_t *pB = colBuffer + row_shift;
|
||||
const q7_t *pA = wt + row_shift;
|
||||
row_shift += 4;
|
||||
|
||||
#ifdef USE_INTRINSIC
|
||||
|
||||
#ifndef ARM_MATH_BIG_ENDIAN
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1, inA2, inB1, inB2, opA, opB;
|
||||
|
||||
inB1 = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
opB = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
inB2 = __PKHTB(opB, inB1, 16);
|
||||
inB1 = __PKHBT(inB1, opB, 16);
|
||||
inA1 = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
opB = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
inA2 = __PKHTB(opB, inA1, 16);
|
||||
inA1 = __PKHBT(inA1, opB, 16);
|
||||
opA = __SXTB16(inA1);
|
||||
opB = __SXTB16(inB1);
|
||||
sum = __SMLAD(opA, opB, sum);
|
||||
opA = __SXTB16(__ROR(inA1, 8));
|
||||
opB = __SXTB16(__ROR(inB1, 8));
|
||||
sum2 = __SMLAD(opA, opB, sum2);
|
||||
opA = __SXTB16(inA2);
|
||||
opB = __SXTB16(inB2);
|
||||
sum3 = __SMLAD(opA, opB, sum3);
|
||||
opA = __SXTB16(__ROR(inA2, 8));
|
||||
opB = __SXTB16(__ROR(inB2, 8));
|
||||
sum4 = __SMLAD(opA, opB, sum4);
|
||||
colCnt--;
|
||||
}
|
||||
#else
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA1, inA2, inB1, inB2, opA, opB;
|
||||
|
||||
inB1 = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
opB = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
inB2 = __PKHBT(opB, inB1, 16);
|
||||
inB1 = __PKHTB(inB1, opB, 16);
|
||||
inA1 = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
opB = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
inA2 = __PKHBT(opB, inA1, 16);
|
||||
inA1 = __PKHTB(inA1, opB, 16);
|
||||
opA = __SXTB16(inA1);
|
||||
opB = __SXTB16(inB1);
|
||||
sum2 = __SMLAD(opA, opB, sum2);
|
||||
opA = __SXTB16(__ROR(inA1, 8));
|
||||
opB = __SXTB16(__ROR(inB1, 8));
|
||||
sum = __SMLAD(opA, opB, sum);
|
||||
opA = __SXTB16(inA2);
|
||||
opB = __SXTB16(inB2);
|
||||
sum4 = __SMLAD(opA, opB, sum4);
|
||||
opA = __SXTB16(__ROR(inA2, 8));
|
||||
opB = __SXTB16(__ROR(inB2, 8));
|
||||
sum3 = __SMLAD(opA, opB, sum3);
|
||||
colCnt--;
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_BIG_ENDIAN */
|
||||
|
||||
#else
|
||||
|
||||
#ifndef ARM_MATH_BIG_ENDIAN
|
||||
// r0 r1 r2 r3 r4 r5
|
||||
// inA1, inA2, inB1, inB2, opA, opB
|
||||
asm volatile ("COL_LOOP:\n"
|
||||
"ldr.w r2, [%[pB], #0]\n"
|
||||
"add.w %[pB], %[pB], %[ch_im_in]\n"
|
||||
"ldr.w r5, [%[pB], #0]\n"
|
||||
"add.w %[pB], %[pB], %[ch_im_in]\n"
|
||||
"pkhtb r3, r5, r2, ASR #16\n"
|
||||
"pkhbt r2, r2, r5, LSL #16\n"
|
||||
"ldr.w r0, [%[pA], #0]\n"
|
||||
"add.w %[pA], %[pA], %[ch_im_in]\n"
|
||||
"ldr.w r5, [%[pA], #0]\n"
|
||||
"add.w %[pA], %[pA], %[ch_im_in]\n"
|
||||
"pkhtb r1, r5, r0, ASR #16\n"
|
||||
"pkhbt r0, r0, r5, LSL #16\n"
|
||||
"sxtb16 r4, r0\n"
|
||||
"sxtb16 r5, r2\n"
|
||||
"smlad %[sum], r4, r5, %[sum]\n"
|
||||
"mov.w r4, r0, ror #8\n"
|
||||
"mov.w r5, r2, ror #8\n"
|
||||
"sxtb16 r4, r4\n"
|
||||
"sxtb16 r5, r5\n"
|
||||
"smlad %[sum2], r4, r5, %[sum2]\n"
|
||||
"sxtb16 r4, r1\n"
|
||||
"sxtb16 r5, r3\n"
|
||||
"smlad %[sum3], r4, r5, %[sum3]\n"
|
||||
"mov.w r4, r1, ror #8\n"
|
||||
"mov.w r5, r3, ror #8\n"
|
||||
"sxtb16 r4, r4\n"
|
||||
"sxtb16 r5, r5\n"
|
||||
"smlad %[sum4], r4, r5, %[sum4]\n"
|
||||
"subs %[colCnt], #1\n"
|
||||
"bne COL_LOOP\n":[sum] "+r"(sum),[sum2] "+r"(sum2),[sum3] "+r"(sum3),
|
||||
[sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt),
|
||||
[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
|
||||
#else
|
||||
// r0 r1 r2 r3 r4 r5
|
||||
// inA1, inA2, inB1, inB2, opA, opB
|
||||
asm volatile ("COL_LOOP:\n"
|
||||
"ldr.w r2, [%[pB], #0]\n"
|
||||
"add.w %[pB], %[pB], %[ch_im_in]\n"
|
||||
"ldr.w r5, [%[pB], #0]\n"
|
||||
"add.w %[pB], %[pB], %[ch_im_in]\n"
|
||||
"pkhbt r3, r5, r2, LSL #16\n"
|
||||
"pkhtb r2, r2, r5, ASR #16\n"
|
||||
"ldr.w r0, [%[pA], #0]\n"
|
||||
"add.w %[pA], %[pA], %[ch_im_in]\n"
|
||||
"ldr.w r5, [%[pA], #0]\n"
|
||||
"add.w %[pA], %[pA], %[ch_im_in]\n"
|
||||
"pkhbt r1, r5, r0, LSL #16\n"
|
||||
"pkhtb r0, r0, r5, ASR #16\n"
|
||||
"sxtb16 r4, r0\n"
|
||||
"sxtb16 r5, r2\n"
|
||||
"smlad %[sum2], r4, r5, %[sum2]\n"
|
||||
"mov.w r4, r0, ror #8\n"
|
||||
"mov.w r5, r2, ror #8\n"
|
||||
"sxtb16 r4, r4\n"
|
||||
"sxtb16 r5, r5\n"
|
||||
"smlad %[sum], r4, r5, %[sum]\n"
|
||||
"sxtb16 r4, r1\n"
|
||||
"sxtb16 r5, r3\n"
|
||||
"smlad %[sum4], r4, r5, %[sum4]\n"
|
||||
"mov.w r4, r1, ror #8\n"
|
||||
"mov.w r5, r3, ror #8\n"
|
||||
"sxtb16 r4, r4\n"
|
||||
"sxtb16 r5, r5\n"
|
||||
"smlad %[sum3], r4, r5, %[sum3]\n"
|
||||
"subs %[colCnt], #1\n"
|
||||
"bne COL_LOOP\n":[sum] "+r"(sum),[sum2] "+r"(sum2),[sum3] "+r"(sum3),
|
||||
[sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt),
|
||||
[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
|
||||
#endif /*ARM_MATH_BIG_ENDIAN */
|
||||
|
||||
#endif /* USE_INTRINSIC */
|
||||
|
||||
colCnt = (dim_kernel_x * dim_kernel_y) & 0x1;
|
||||
while (colCnt)
|
||||
{
|
||||
union arm_nnword inA, inB;
|
||||
inA.word = *__SIMD32(pA);
|
||||
pA += ch_im_in;
|
||||
inB.word = *__SIMD32(pB);
|
||||
pB += ch_im_in;
|
||||
sum += inA.bytes[0] * inB.bytes[0];
|
||||
sum2 += inA.bytes[1] * inB.bytes[1];
|
||||
sum3 += inA.bytes[2] * inB.bytes[2];
|
||||
sum4 += inA.bytes[3] * inB.bytes[3];
|
||||
colCnt--;
|
||||
}
|
||||
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
*pOut++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
|
||||
*pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
|
||||
*pOut++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
|
||||
|
||||
rowCnt--;
|
||||
}
|
||||
|
||||
rowCnt = ch_im_out & 0x3;
|
||||
while (rowCnt)
|
||||
{
|
||||
q7_t *pB = colBuffer + row_shift;
|
||||
const q7_t *pA = wt + row_shift;
|
||||
q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
uint16_t colCnt = (dim_kernel_x * dim_kernel_y);
|
||||
|
||||
row_shift += 1;
|
||||
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t A1 = *pA;
|
||||
q7_t B1 = *pB;
|
||||
pA += ch_im_in;
|
||||
pB += ch_im_in;
|
||||
sum += A1 * B1;
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
rowCnt--;
|
||||
}
|
||||
|
||||
// clear counter and pointers
|
||||
pBuffer = colBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
/* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
|
||||
int i_out_y, i_out_x, i_ch_out;
|
||||
int i_ker_y, i_ker_x;
|
||||
|
||||
/* do some checking here, basically ch_im_in == ch_im_out */
|
||||
if (ch_im_in != ch_im_out)
|
||||
{
|
||||
return ARM_MATH_SIZE_MISMATCH;
|
||||
}
|
||||
|
||||
for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
|
||||
{
|
||||
for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
|
||||
{
|
||||
for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
|
||||
{
|
||||
// for each output
|
||||
int conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
|
||||
for (i_ker_y = 0; i_ker_y < dim_kernel_y; i_ker_y++)
|
||||
{
|
||||
for (i_ker_x = 0; i_ker_x < dim_kernel_x; i_ker_x++)
|
||||
{
|
||||
int in_row = stride_y * i_out_y + i_ker_y - padding_y;
|
||||
int in_col = stride_x * i_out_x + i_ker_x - padding_x;
|
||||
if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
|
||||
{
|
||||
conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + i_ch_out] *
|
||||
wt[(i_ker_y * dim_kernel_x + i_ker_x) * ch_im_out + i_ch_out];
|
||||
}
|
||||
}
|
||||
}
|
||||
Im_out[(i_out_y * dim_im_out_x + i_out_x) * ch_im_out + i_ch_out] =
|
||||
(q7_t) __SSAT((conv_out >> out_shift), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
|
||||
/* Return to application */
|
||||
return ARM_MATH_SUCCESS;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @} end of NNConv group
|
||||
*/
|
||||
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_nn_mat_mult_kernel_q7_q15.c
|
||||
* Description: Matrix-multiplication function for convolution
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_math.h"
|
||||
#include "arm_nnfunctions.h"
|
||||
|
||||
/**
|
||||
* @brief Matrix-multiplication function for convolution
|
||||
* @param[in] pA pointer to operand A
|
||||
* @param[in] pInBuffer pointer to operand B, always conssists of 2 vectors
|
||||
* @param[in] ch_im_out numRow of A
|
||||
* @param[in] numCol_A numCol of A
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in] bias the bias
|
||||
* @param[in,out] pOut pointer to output
|
||||
* @return The function returns the incremented output pointer
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* This function does the matrix multiplication with weight matrix
|
||||
* and 2 columns from im2col.
|
||||
*/
|
||||
|
||||
q7_t *arm_nn_mat_mult_kernel_q7_q15(const q7_t * pA,
|
||||
const q15_t * pInBuffer,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t numCol_A,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
const q7_t * bias,
|
||||
q7_t * pOut)
|
||||
{
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* set up the second output pointers */
|
||||
q7_t *pOut2 = pOut + ch_im_out;
|
||||
const q7_t *pBias = bias;
|
||||
|
||||
uint16_t rowCnt = ch_im_out >> 1;
|
||||
/* this loop over rows in A */
|
||||
while (rowCnt)
|
||||
{
|
||||
/* setup pointers for B */
|
||||
const q15_t *pB = pInBuffer;
|
||||
const q15_t *pB2 = pB + numCol_A;
|
||||
|
||||
/* align the second pointer for A */
|
||||
const q7_t *pA2 = pA + numCol_A;
|
||||
|
||||
/* init the sum with bias */
|
||||
q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum3 = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
uint16_t colCnt = numCol_A >> 2;
|
||||
/* accumulate over the vector */
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA11, inA12, inA21, inA22;
|
||||
q31_t inB1 = *__SIMD32(pB)++;
|
||||
q31_t inB2 = *__SIMD32(pB2)++;
|
||||
|
||||
pA = (q7_t *) read_and_pad((void *)pA, &inA11, &inA12);
|
||||
pA2 = (q7_t *) read_and_pad((void *)pA2, &inA21, &inA22);
|
||||
|
||||
sum = __SMLAD(inA11, inB1, sum);
|
||||
sum2 = __SMLAD(inA11, inB2, sum2);
|
||||
sum3 = __SMLAD(inA21, inB1, sum3);
|
||||
sum4 = __SMLAD(inA21, inB2, sum4);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
inB2 = *__SIMD32(pB2)++;
|
||||
|
||||
sum = __SMLAD(inA12, inB1, sum);
|
||||
sum2 = __SMLAD(inA12, inB2, sum2);
|
||||
sum3 = __SMLAD(inA22, inB1, sum3);
|
||||
sum4 = __SMLAD(inA22, inB2, sum4);
|
||||
|
||||
colCnt--;
|
||||
} /* while over colCnt */
|
||||
colCnt = numCol_A & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
q7_t inA2 = *pA2++;
|
||||
q15_t inB2 = *pB2++;
|
||||
|
||||
sum += inA1 * inB1;
|
||||
sum2 += inA1 * inB2;
|
||||
sum3 += inA2 * inB1;
|
||||
sum4 += inA2 * inB2;
|
||||
colCnt--;
|
||||
} /* while over colCnt */
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
*pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
|
||||
*pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
|
||||
*pOut2++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
|
||||
|
||||
/* skip the row computed with A2 */
|
||||
pA += numCol_A;
|
||||
rowCnt--;
|
||||
} /* for over ch_im_out */
|
||||
|
||||
/* compute left-over row if any */
|
||||
if (ch_im_out & 0x1)
|
||||
{
|
||||
/* setup pointers for B */
|
||||
const q15_t *pB = pInBuffer;
|
||||
const q15_t *pB2 = pB + numCol_A;
|
||||
|
||||
/* load the bias */
|
||||
q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
uint16_t colCnt = numCol_A >> 2;
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA11, inA12;
|
||||
q31_t inB1 = *__SIMD32(pB)++;
|
||||
q31_t inB2 = *__SIMD32(pB2)++;
|
||||
|
||||
pA = (q7_t *) read_and_pad((void *)pA, &inA11, &inA12);
|
||||
|
||||
sum = __SMLAD(inA11, inB1, sum);
|
||||
sum2 = __SMLAD(inA11, inB2, sum2);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
inB2 = *__SIMD32(pB2)++;
|
||||
sum = __SMLAD(inA12, inB1, sum);
|
||||
sum2 = __SMLAD(inA12, inB2, sum2);
|
||||
|
||||
colCnt--;
|
||||
}
|
||||
colCnt = numCol_A & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
q15_t inB2 = *pB2++;
|
||||
|
||||
sum += inA1 * inB1;
|
||||
sum2 += inA1 * inB2;
|
||||
colCnt--;
|
||||
}
|
||||
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
*pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
|
||||
}
|
||||
|
||||
pOut += ch_im_out;
|
||||
|
||||
/* return the new output pointer with offset */
|
||||
return pOut;
|
||||
#else
|
||||
/* To be completed */
|
||||
return NULL;
|
||||
#endif /* ARM_MATH_DSP */
|
||||
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/*
|
||||
* Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the License); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an AS IS BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* ----------------------------------------------------------------------
|
||||
* Project: CMSIS NN Library
|
||||
* Title: arm_nn_mat_mult_kernel_q7_q15_reordered.c
|
||||
* Description: Matrix-multiplication function for convolution with reordered columns
|
||||
*
|
||||
* $Date: 17. January 2018
|
||||
* $Revision: V.1.0.0
|
||||
*
|
||||
* Target Processor: Cortex-M cores
|
||||
* -------------------------------------------------------------------- */
|
||||
|
||||
#include "arm_nnfunctions.h"
|
||||
#include "arm_math.h"
|
||||
|
||||
/**
|
||||
* @brief Matrix-multiplication function for convolution with reordered columns
|
||||
* @param[in] pA pointer to operand A
|
||||
* @param[in] pInBuffer pointer to operand B, always conssists of 2 vectors
|
||||
* @param[in] ch_im_out numRow of A
|
||||
* @param[in] numCol_A numCol of A
|
||||
* @param[in] bias_shift amount of left-shift for bias
|
||||
* @param[in] out_shift amount of right-shift for output
|
||||
* @param[in] bias the bias
|
||||
* @param[in,out] pOut pointer to output
|
||||
* @return The function returns the incremented output pointer
|
||||
*
|
||||
* @details
|
||||
*
|
||||
* This function assumes that data in pInBuffer are reordered
|
||||
*/
|
||||
|
||||
q7_t *arm_nn_mat_mult_kernel_q7_q15_reordered(const q7_t * pA,
|
||||
const q15_t * pInBuffer,
|
||||
const uint16_t ch_im_out,
|
||||
const uint16_t numCol_A,
|
||||
const uint16_t bias_shift,
|
||||
const uint16_t out_shift,
|
||||
const q7_t * bias,
|
||||
q7_t * pOut)
|
||||
{
|
||||
|
||||
#if defined (ARM_MATH_DSP)
|
||||
/* set up the second output pointers */
|
||||
q7_t *pOut2 = pOut + ch_im_out;
|
||||
int i;
|
||||
|
||||
/* this loop over rows in A */
|
||||
for (i = 0; i < ch_im_out; i += 2)
|
||||
{
|
||||
/* setup pointers for B */
|
||||
const q15_t *pB = pInBuffer;
|
||||
const q15_t *pB2 = pB + numCol_A;
|
||||
|
||||
/* align the second pointer for A */
|
||||
const q7_t *pA2 = pA + numCol_A;
|
||||
|
||||
/* init the sum with bias */
|
||||
q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum2 = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum3 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift);
|
||||
q31_t sum4 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift);
|
||||
|
||||
uint16_t colCnt = numCol_A >> 2;
|
||||
/* accumulate over the vector */
|
||||
while (colCnt)
|
||||
{
|
||||
q31_t inA11, inA12, inA21, inA22;
|
||||
q31_t inB1 = *__SIMD32(pB)++;
|
||||
q31_t inB2 = *__SIMD32(pB2)++;
|
||||
|
||||
pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA11, &inA12);
|
||||
pA2 = (q7_t *) read_and_pad_reordered((void *)pA2, &inA21, &inA22);
|
||||
|
||||
sum = __SMLAD(inA11, inB1, sum);
|
||||
sum2 = __SMLAD(inA11, inB2, sum2);
|
||||
sum3 = __SMLAD(inA21, inB1, sum3);
|
||||
sum4 = __SMLAD(inA21, inB2, sum4);
|
||||
|
||||
inB1 = *__SIMD32(pB)++;
|
||||
inB2 = *__SIMD32(pB2)++;
|
||||
|
||||
sum = __SMLAD(inA12, inB1, sum);
|
||||
sum2 = __SMLAD(inA12, inB2, sum2);
|
||||
sum3 = __SMLAD(inA22, inB1, sum3);
|
||||
sum4 = __SMLAD(inA22, inB2, sum4);
|
||||
|
||||
colCnt--;
|
||||
} /* while over colCnt */
|
||||
colCnt = numCol_A & 0x3;
|
||||
while (colCnt)
|
||||
{
|
||||
q7_t inA1 = *pA++;
|
||||
q15_t inB1 = *pB++;
|
||||
q7_t inA2 = *pA2++;
|
||||
q15_t inB2 = *pB2++;
|
||||
|
||||
sum += inA1 * inB1;
|
||||
sum2 += inA1 * inB2;
|
||||
sum3 += inA2 * inB1;
|
||||
sum4 += inA2 * inB2;
|
||||
colCnt--;
|
||||
} /* while over colCnt */
|
||||
*pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
|
||||
*pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
|
||||
*pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
|
||||
*pOut2++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
|
||||
|
||||
/* skip the row computed with A2 */
|
||||
pA += numCol_A;
|
||||
} /* for over ch_im_out */
|
||||
|
||||
pOut += ch_im_out;
|
||||
|
||||
/* return the new output pointer with offset */
|
||||
return pOut;
|
||||
#else
|
||||
/* To be completed */
|
||||
return NULL;
|
||||
#endif /* ARM_MATH_DSP */
|
||||
}
|
||||
Reference in New Issue
Block a user