bli_gemm_cntl.h

/*

   BLIS
   An object-based framework for developing high-performance BLAS-like
   libraries.

   Copyright (C) 2014, The University of Texas at Austin
   Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are
   met:
    - Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    - Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    - Neither the name(s) of the copyright holder(s) nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
   HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/


// -----------------------------------------------------------------------------

struct gemm_var_cntl_s
{
	cntl_t      cntl; //this field must be present and come first
	num_t       dt_comp;
	num_t       dt_out;
	gemm_ukr_ft ukr;
	gemm_ukr_ft real_ukr;
	const void* params;
	const void* real_params;
	dim_t       mr;
	dim_t       nr;
	dim_t       mr_scale;
	dim_t       nr_scale;
	bool        row_pref;
};
typedef struct gemm_var_cntl_s gemm_var_cntl_t;

// -----------------------------------------------------------------------------

BLIS_INLINE gemm_ukr_ft bli_gemm_var_cntl_ukr( const cntl_t* cntl )
{
	return ( ( const gemm_var_cntl_t* ) cntl )->ukr;
}

BLIS_INLINE gemm_ukr_ft bli_gemm_var_cntl_real_ukr( const cntl_t* cntl )
{
	return ( ( const gemm_var_cntl_t* ) cntl )->real_ukr;
}

BLIS_INLINE bool bli_gemm_var_cntl_row_pref( const cntl_t* cntl )
{
	return ( ( const gemm_var_cntl_t* ) cntl )->row_pref;
}

BLIS_INLINE const void* bli_gemm_var_cntl_params( const cntl_t* cntl )
{
	return ( ( const gemm_var_cntl_t* ) cntl )->params;
}

BLIS_INLINE const void* bli_gemm_var_cntl_real_params( const cntl_t* cntl )
{
	return ( ( const gemm_var_cntl_t* ) cntl )->real_params;
}

BLIS_INLINE dim_t bli_gemm_var_cntl_mr( const cntl_t* cntl )
{
	return ( ( const gemm_var_cntl_t* ) cntl )->mr;
}

BLIS_INLINE dim_t bli_gemm_var_cntl_nr( const cntl_t* cntl )
{
	return ( ( const gemm_var_cntl_t* ) cntl )->nr;
}

BLIS_INLINE num_t bli_gemm_var_cntl_comp_dt( const cntl_t* cntl )
{
	return ( ( const gemm_var_cntl_t* ) cntl )->dt_comp;
}

// -----------------------------------------------------------------------------

BLIS_INLINE void bli_gemm_var_cntl_set_ukr( const func2_t* ukr, cntl_t* cntl_ )
{
	gemm_var_cntl_t* cntl = ( gemm_var_cntl_t* )cntl_;
	num_t dt_comp = cntl->dt_comp;
	num_t dt_out = cntl->dt_out;
	cntl->ukr = ( gemm_ukr_ft )bli_func2_get_dt( dt_comp, dt_out, ukr );
}

BLIS_INLINE void bli_gemm_var_cntl_set_real_ukr( const func2_t* ukr, cntl_t* cntl_ )
{
	gemm_var_cntl_t* cntl = ( gemm_var_cntl_t* )cntl_;
	num_t dt_comp = cntl->dt_comp;
	num_t dt_out = cntl->dt_out;
	cntl->real_ukr = ( gemm_ukr_ft )bli_func2_get_dt( dt_comp, dt_out, ukr );
}

BLIS_INLINE err_t bli_gemm_var_cntl_set_ukr_simple( const func_t* ukr, cntl_t* cntl_ )
{
	gemm_var_cntl_t* cntl = ( gemm_var_cntl_t* )cntl_;
	num_t dt_comp = cntl->dt_comp;
	num_t dt_out = cntl->dt_out;
	if ( dt_comp != dt_out )
		return BLIS_INCONSISTENT_DATATYPES;
	cntl->ukr = ( gemm_ukr_ft )bli_func_get_dt( dt_comp, ukr );
	return BLIS_SUCCESS;
}

BLIS_INLINE err_t bli_gemm_var_cntl_set_real_ukr_simple( const func_t* ukr, cntl_t* cntl_ )
{
	gemm_var_cntl_t* cntl = ( gemm_var_cntl_t* )cntl_;
	num_t dt_comp = cntl->dt_comp;
	num_t dt_out = cntl->dt_out;
	if ( dt_comp != dt_out )
		return BLIS_INCONSISTENT_DATATYPES;
	cntl->real_ukr = ( gemm_ukr_ft )bli_func_get_dt( dt_comp, ukr );
	return BLIS_SUCCESS;
}

BLIS_INLINE void bli_gemm_var_cntl_set_row_pref( const mbool_t* row_pref, cntl_t* cntl_ )
{
	gemm_var_cntl_t* cntl = ( gemm_var_cntl_t* )cntl_;
	num_t dt_comp = cntl->dt_comp;
	cntl->row_pref = bli_mbool_get_dt( dt_comp, row_pref );
}

BLIS_INLINE void bli_gemm_var_cntl_set_params( const void* params, cntl_t* cntl )
{
	( ( gemm_var_cntl_t* ) cntl )->params = params;
}

BLIS_INLINE void bli_gemm_var_cntl_set_real_params( const void* params, cntl_t* cntl )
{
	( ( gemm_var_cntl_t* ) cntl )->real_params = params;
}

BLIS_INLINE void bli_gemm_var_cntl_set_mr( dim_t mr, cntl_t* cntl )
{
	( ( gemm_var_cntl_t* ) cntl )->mr = mr / ( ( gemm_var_cntl_t* ) cntl )->mr_scale;
}

BLIS_INLINE void bli_gemm_var_cntl_set_nr( dim_t nr, cntl_t* cntl )
{
	( ( gemm_var_cntl_t* ) cntl )->nr = nr / ( ( gemm_var_cntl_t* ) cntl )->nr_scale;
}

BLIS_INLINE void bli_gemm_var_cntl_set_comp_dt( num_t dt, const cntl_t* cntl )
{
	( ( gemm_var_cntl_t* ) cntl )->dt_comp = dt;
}

// -----------------------------------------------------------------------------

void bli_gemm_var_cntl_init_node
     (
       void_fp          var_func,
       num_t            dt_comp,
       num_t            dt_out,
       gemm_ukr_ft      ukr,
       gemm_ukr_ft      real_ukr,
       bool             row_pref,
       dim_t            mr,
       dim_t            nr,
       dim_t            mr_scale,
       dim_t            nr_scale,
       gemm_var_cntl_t* cntl
     );

// -----------------------------------------------------------------------------

struct gemm_cntl_s
{
         part_cntl_t part_jc;
         part_cntl_t part_pc;
    packm_def_cntl_t pack_b;
         part_cntl_t part_ic;
    packm_def_cntl_t pack_a;
     gemm_var_cntl_t ker;
              cntl_t ir_loop;
};
typedef struct gemm_cntl_s gemm_cntl_t;

// -----------------------------------------------------------------------------

BLIS_EXPORT_BLIS bool bli_gemm_cntl_init
     (
             ind_t        im,
             opid_t       family,
       const obj_t*       alpha,
             obj_t*       a,
             obj_t*       b,
       const obj_t*       beta,
             obj_t*       c,
       const cntx_t*      cntx,
             gemm_cntl_t* cntl
     );

BLIS_EXPORT_BLIS void bli_gemm_cntl_finalize
     (
             opid_t       family,
       const obj_t*       a,
       const obj_t*       b,
       const obj_t*       c,
             gemm_cntl_t* cntl
     );

// -----------------------------------------------------------------------------

BLIS_INLINE gemm_ukr_ft bli_gemm_cntl_ukr( gemm_cntl_t* cntl )
{
	gemm_ukr_ft real_ukr = bli_gemm_var_cntl_real_ukr( ( cntl_t* )&cntl->ker );
	return real_ukr ? real_ukr : bli_gemm_var_cntl_ukr( ( cntl_t* )&cntl->ker );
}

BLIS_INLINE bool bli_gemm_cntl_row_pref( gemm_cntl_t* cntl )
{
	return bli_gemm_var_cntl_row_pref( ( cntl_t* )&cntl->ker );
}

BLIS_INLINE const void* bli_gemm_cntl_params( gemm_cntl_t* cntl )
{
	gemm_ukr_ft real_ukr = bli_gemm_var_cntl_real_ukr( ( cntl_t* )&cntl->ker );
	return real_ukr ? bli_gemm_var_cntl_real_params( ( cntl_t* )&cntl->ker )
	                : bli_gemm_var_cntl_params( ( cntl_t* )&cntl->ker );
}

BLIS_INLINE l3_var_oft bli_gemm_cntl_var( gemm_cntl_t* cntl )
{
	return ( l3_var_oft )bli_cntl_var_func( ( cntl_t* )&cntl->ker );
}

BLIS_INLINE packm_ker_ft bli_gemm_cntl_packa_ukr( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_ukr( ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE pack_t bli_gemm_cntl_packa_schema( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_pack_schema( ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE const void* bli_gemm_cntl_packa_params( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_ukr_params( ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE packm_var_oft bli_gemm_cntl_packa_var( gemm_cntl_t* cntl )
{
	return bli_packm_cntl_variant( ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE packm_ker_ft bli_gemm_cntl_packb_ukr( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_ukr( ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE pack_t bli_gemm_cntl_packb_schema( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_pack_schema( ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE const void* bli_gemm_cntl_packb_params( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_ukr_params( ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE packm_var_oft bli_gemm_cntl_packb_var( gemm_cntl_t* cntl )
{
	return bli_packm_cntl_variant( ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE dim_t bli_gemm_cntl_mr_def( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_bmult_m_def( ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE dim_t bli_gemm_cntl_mr_pack( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_bmult_m_pack( ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE dim_t bli_gemm_cntl_nr_def( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_bmult_m_def( ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE dim_t bli_gemm_cntl_nr_pack( gemm_cntl_t* cntl )
{
	return bli_packm_def_cntl_bmult_m_pack( ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE dim_t bli_gemm_cntl_mc_def( gemm_cntl_t* cntl )
{
	return bli_part_cntl_blksz_alg( ( cntl_t* )&cntl->part_ic );
}

BLIS_INLINE dim_t bli_gemm_cntl_mc_max( gemm_cntl_t* cntl )
{
	return bli_part_cntl_blksz_max( ( cntl_t* )&cntl->part_ic );
}

BLIS_INLINE dim_t bli_gemm_cntl_nc_def( gemm_cntl_t* cntl )
{
	return bli_part_cntl_blksz_alg( ( cntl_t* )&cntl->part_jc );
}

BLIS_INLINE dim_t bli_gemm_cntl_nc_max( gemm_cntl_t* cntl )
{
	return bli_part_cntl_blksz_max( ( cntl_t* )&cntl->part_jc );
}

BLIS_INLINE dim_t bli_gemm_cntl_kc_def( gemm_cntl_t* cntl )
{
	return bli_part_cntl_blksz_alg( ( cntl_t* )&cntl->part_pc );
}

BLIS_INLINE dim_t bli_gemm_cntl_kc_max( gemm_cntl_t* cntl )
{
	return bli_part_cntl_blksz_max( ( cntl_t* )&cntl->part_pc );
}

// -----------------------------------------------------------------------------

BLIS_INLINE void bli_gemm_cntl_set_ukr( const func2_t* ukr, gemm_cntl_t* cntl )
{
	if ( bli_gemm_var_cntl_real_ukr( ( cntl_t* )&cntl->ker ) )
	{
		bli_gemm_var_cntl_set_real_ukr( ukr, ( cntl_t* )&cntl->ker );
	}
	else
	{
		bli_gemm_var_cntl_set_ukr( ukr, ( cntl_t* )&cntl->ker );
	}
}

BLIS_INLINE void bli_gemm_cntl_set_ukr_simple( const func_t* ukr, gemm_cntl_t* cntl )
{
	if ( bli_gemm_var_cntl_real_ukr( ( cntl_t* )&cntl->ker ) )
	{
		bli_gemm_var_cntl_set_real_ukr_simple( ukr, ( cntl_t* )&cntl->ker );
	}
	else
	{
		bli_gemm_var_cntl_set_ukr_simple( ukr, ( cntl_t* )&cntl->ker );
	}
}

BLIS_INLINE void bli_gemm_cntl_set_row_pref( const mbool_t* row_pref, gemm_cntl_t* cntl )
{
	bli_gemm_var_cntl_set_row_pref( row_pref, ( cntl_t* )&cntl->ker );
}

BLIS_INLINE void bli_gemm_cntl_set_params( const void* params, gemm_cntl_t* cntl )
{
	if ( bli_gemm_var_cntl_real_ukr( ( cntl_t* )&cntl->ker ) )
	{
		bli_gemm_var_cntl_set_real_params( params, ( cntl_t* )&cntl->ker );
	}
	else
	{
		bli_gemm_var_cntl_set_params( params, ( cntl_t* )&cntl->ker );
	}
}

BLIS_INLINE void bli_gemm_cntl_set_var( l3_var_oft var, gemm_cntl_t* cntl )
{
	bli_cntl_set_var_func( ( void_fp )var, ( cntl_t* )&cntl->ker );
}

BLIS_INLINE void bli_gemm_cntl_set_packa_ukr( const func2_t* ukr, gemm_cntl_t* cntl )
{
	bli_packm_def_cntl_set_ukr( ukr, ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE void bli_gemm_cntl_set_packa_ukr_simple( const func_t* ukr, gemm_cntl_t* cntl )
{
	bli_packm_def_cntl_set_ukr_simple( ukr, ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE void bli_gemm_cntl_set_packa_schema( pack_t schema, gemm_cntl_t* cntl )
{
	bli_packm_def_cntl_set_pack_schema( schema, ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE void bli_gemm_cntl_set_packa_params( const void* params, gemm_cntl_t* cntl )
{
	bli_packm_def_cntl_set_ukr_params( params, ( cntl_t* )&cntl->pack_a );
	bli_packm_cntl_set_variant_params( params, ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE void bli_gemm_cntl_set_packa_var( packm_var_oft var, gemm_cntl_t* cntl )
{
	bli_packm_cntl_set_variant( var, ( cntl_t* )&cntl->pack_a );
}

BLIS_INLINE void bli_gemm_cntl_set_packb_ukr( const func2_t* ukr, gemm_cntl_t* cntl )
{
	bli_packm_def_cntl_set_ukr( ukr, ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE void bli_gemm_cntl_set_packb_ukr_simple( const func_t* ukr, gemm_cntl_t* cntl )
{
	bli_packm_def_cntl_set_ukr_simple( ukr, ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE void bli_gemm_cntl_set_packb_schema( pack_t schema, gemm_cntl_t* cntl )
{
	bli_packm_def_cntl_set_pack_schema( schema, ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE void bli_gemm_cntl_set_packb_params( const void* params, gemm_cntl_t* cntl )
{
	bli_packm_def_cntl_set_ukr_params( params, ( cntl_t* )&cntl->pack_b );
	bli_packm_cntl_set_variant_params( params, ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE void bli_gemm_cntl_set_packb_var( packm_var_oft var, gemm_cntl_t* cntl )
{
	bli_packm_cntl_set_variant( var, ( cntl_t* )&cntl->pack_b );
}

BLIS_INLINE void bli_gemm_cntl_set_mr( const blksz_t* mr, gemm_cntl_t* cntl )
{
	num_t dt = cntl->ker.dt_comp;
	dim_t mr_dt = bli_blksz_get_def( dt, mr );
	bli_packm_def_cntl_set_bmult_m( mr, ( cntl_t* )&cntl->pack_a );
	bli_part_cntl_set_blksz_mult( mr, ( cntl_t* )&cntl->part_ic );
	bli_gemm_var_cntl_set_mr( mr_dt, ( cntl_t* )&cntl->ker );
}

BLIS_INLINE void bli_gemm_cntl_set_nr( const blksz_t* nr, gemm_cntl_t* cntl )
{
	num_t dt = cntl->ker.dt_comp;
	dim_t nr_dt = bli_blksz_get_def( dt, nr );
	bli_packm_def_cntl_set_bmult_m( nr, ( cntl_t* )&cntl->pack_b );
	bli_part_cntl_set_blksz_mult( nr, ( cntl_t* )&cntl->part_jc );
	bli_gemm_var_cntl_set_nr( nr_dt, ( cntl_t* )&cntl->ker );
}

BLIS_INLINE void bli_gemm_cntl_set_mc( const blksz_t* mc, gemm_cntl_t* cntl )
{
	bli_part_cntl_set_blksz( mc, ( cntl_t* )&cntl->part_ic );
}

BLIS_INLINE void bli_gemm_cntl_set_nc( const blksz_t* nc, gemm_cntl_t* cntl )
{
	bli_part_cntl_set_blksz( nc, ( cntl_t* )&cntl->part_jc );
}

BLIS_INLINE void bli_gemm_cntl_set_kc( const blksz_t* kc, gemm_cntl_t* cntl )
{
	bli_part_cntl_set_blksz( kc, ( cntl_t* )&cntl->part_pc );
}